birdclaw 0.9.3 → 0.9.4

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # CHANGELOG
 
+## 0.9.4 - 2026-07-05
+
+### Fixed
+
+- `search tweets` now returns in well under a second on large archives (previously ~60s, minutes for common terms): the FTS5 match set is materialized once instead of SQLite re-running the MATCH scan per timeline edge row, the join order adapts to term frequency, the LIMIT is resolved on an id-only inner query, and snippets are computed only for returned rows.
+
 ## 0.9.3 - 2026-07-04
 
 ### Changed

package/dist/cli/birdclaw.js

@@ -11166,7 +11166,8 @@ function getTimelineQualityReason(row, lowQualityThreshold) {
   return "keep:long-text";
 }
 var RECENT_TIMELINE_EDGE_CANDIDATES = 5e3;
-function listTimelineItems({
+var FTS_DRIVE_FROM_MATCHES_MAX = 1e4;
+function buildTimelineItemsQuery({
   resource,
   account,
   listAccountId,
@@ -11179,13 +11180,12 @@ function listTimelineItems({
   includeReplies = true,
   qualityFilter = "all",
   lowQualityThreshold,
-  includeQualityReason = false,
   likedOnly = false,
   bookmarkedOnly = false,
   limit = 18
-}) {
-  const db = getReadDb();
+}, ftsMatchCountHint = 0) {
   const kind = resource === "mentions" ? "mention" : resource;
+  const cteParams = [];
   const params = [];
   const normalizedLowQualityThreshold = normalizeLowQualityThreshold(lowQualityThreshold);
   const shouldDedupeAcrossAccounts = !account || account === "all";
@@ -11198,9 +11198,8 @@ function listTimelineItems({
 	    `;
   const unwindowedTimelineEdgesCte = timelineEdgesCte;
   let usedRecentEdgeWindow = false;
-  let join3 = "";
   let where = "where e.kind = ?";
-  let searchSnippetSelect = "";
+  const ftsSearch = search?.trim() ? toFtsSearchQuery(search) : "";
   const canUseRecentEdgeWindow = !likedOnly && !bookmarkedOnly && !listId && !account && !search?.trim() && replyFilter === "all" && !since?.trim() && !until?.trim() && includeReplies && qualityFilter === "all";
   if (likedOnly || bookmarkedOnly) {
     if (likedOnly && bookmarkedOnly) {
@@ -11224,7 +11223,7 @@ function listTimelineItems({
 	          where kind = ?
 	        )
 				`;
-      params.push(collectionKind2);
+      cteParams.push(collectionKind2);
     }
     where = "where 1 = 1";
   } else if (canUseRecentEdgeWindow) {
@@ -11246,11 +11245,11 @@ function listTimelineItems({
       RECENT_TIMELINE_EDGE_CANDIDATES,
       limit * 50
     );
-    params.push(kind, candidateLimit);
+    cteParams.push(kind, candidateLimit);
     where = "where e.kind = ?";
     params.push(kind);
   } else {
-    params.push(kind);
+    cteParams.push(kind);
     where = "where e.kind = ?";
     params.push(kind);
   }
@@ -11313,16 +11312,32 @@ function listTimelineItems({
     `;
     params.push(listAccountId, listId);
   }
-  const ftsSearch = search?.trim() ? toFtsSearchQuery(search) : "";
+  const ftsMatchesCte = ftsSearch ? `, fts_matches as materialized (
+        select tweet_id
+        from tweets_fts
+        where tweets_fts.text match ?
+      )` : "";
   if (ftsSearch) {
-    join3 += " join tweets_fts on tweets_fts.tweet_id = t.id ";
-    where += " and tweets_fts.text match ?";
-    searchSnippetSelect = ", snippet(tweets_fts, 1, '<mark>', '</mark>', '...', 16) as search_snippet";
-    params.push(ftsSearch);
+    cteParams.push(ftsSearch);
   }
+  const searchDrivenFrom = ftsMatchCountHint > FTS_DRIVE_FROM_MATCHES_MAX ? `tweets t
+        cross join fts_matches on fts_matches.tweet_id = t.id
+        cross join timeline_edges e on e.tweet_id = t.id` : `fts_matches
+        cross join tweets t on t.id = fts_matches.tweet_id
+        cross join timeline_edges e on e.tweet_id = t.id`;
   params.push(limit);
+  if (ftsSearch) {
+    params.push(limit);
+  }
+  const searchSelectionCte = ftsSearch ? `, search_selection as materialized (
+        select t.id as tweet_id, e.account_id, e.kind, e.raw_json
+        from ${searchDrivenFrom}
+        ${where}
+        order by t.created_at desc, t.id desc
+        limit ?
+      )` : "";
   const buildTimelineSelectSql = (timelineEdgesSql) => `
-      ${timelineEdgesSql}
+      ${timelineEdgesSql}${ftsMatchesCte}${searchSelectionCte}
       select
         t.id,
         e.account_id,
@@ -11399,8 +11414,7 @@ function listTimelineItems({
         qp.avatar_hue as quoted_avatar_hue,
         qp.avatar_url as quoted_avatar_url,
         qp.created_at as quoted_profile_created_at
-        ${searchSnippetSelect}
-      from timeline_edges e
+      from ${ftsSearch ? "search_selection e" : "timeline_edges e"}
       join tweets t on t.id = e.tweet_id
       join accounts a on a.id = e.account_id
       join profiles p on p.id = t.author_profile_id
@@ -11408,14 +11422,52 @@ function listTimelineItems({
       left join profiles rp on rp.id = rt.author_profile_id
       left join tweets qt on qt.id = t.quoted_tweet_id
       left join profiles qp on qp.id = qt.author_profile_id
-      ${join3}
-      ${where}
+      ${ftsSearch ? "" : where}
       order by t.created_at desc, t.id desc
       limit ?
       `;
-  let rows = db.prepare(buildTimelineSelectSql(timelineEdgesCte)).all(...params);
-  if (usedRecentEdgeWindow && rows.length < limit) {
-    rows = db.prepare(buildTimelineSelectSql(unwindowedTimelineEdgesCte)).all(kind, kind, limit);
+  return {
+    sql: buildTimelineSelectSql(timelineEdgesCte),
+    params: [...cteParams, ...params],
+    fallbackSql: buildTimelineSelectSql(unwindowedTimelineEdgesCte),
+    fallbackParams: [kind, kind, limit],
+    usedRecentEdgeWindow,
+    ftsSearch
+  };
+}
+var SEARCH_SNIPPET_SQL = "snippet(tweets_fts, 1, '<mark>', '</mark>', '...', 16)";
+function listTimelineItems(query) {
+  const db = getReadDb();
+  const {
+    includeQualityReason = false,
+    lowQualityThreshold,
+    limit = 18
+  } = query;
+  const normalizedLowQualityThreshold = normalizeLowQualityThreshold(lowQualityThreshold);
+  const ftsSearch = query.search?.trim() ? toFtsSearchQuery(query.search) : "";
+  const ftsMatchCount = ftsSearch ? Number(
+    db.prepare(
+      "select count(*) as match_count from tweets_fts where tweets_fts.text match ?"
+    ).get(ftsSearch).match_count
+  ) : 0;
+  const plan = buildTimelineItemsQuery(query, ftsMatchCount);
+  let rows = db.prepare(plan.sql).all(...plan.params);
+  if (plan.usedRecentEdgeWindow && rows.length < limit) {
+    rows = db.prepare(plan.fallbackSql).all(...plan.fallbackParams);
+  }
+  if (plan.ftsSearch && rows.length > 0) {
+    const snippetRows = db.prepare(
+      `select tweet_id, ${SEARCH_SNIPPET_SQL} as search_snippet
+         from tweets_fts
+         where tweets_fts.text match ?
+           and tweet_id in (${rows.map(() => "?").join(",")})`
+    ).all(plan.ftsSearch, ...rows.map((row) => String(row.id)));
+    const snippetByTweetId = new Map(
+      snippetRows.map((row) => [String(row.tweet_id), row.search_snippet])
+    );
+    for (const row of rows) {
+      row.search_snippet = snippetByTweetId.get(String(row.id));
+    }
   }
   const urlExpansionCache = /* @__PURE__ */ new Map();
   const profileByHandleCache = /* @__PURE__ */ new Map();

package/dist/server/assets/{router-B874XqWN.js → router-DVQ3M870.js}

@@ -8902,9 +8902,10 @@ function getTimelineQualityReason(row, lowQualityThreshold) {
 	return "keep:long-text";
 }
 var RECENT_TIMELINE_EDGE_CANDIDATES = 5e3;
-function listTimelineItems({ resource, account, listAccountId, listId, search, replyFilter = "all", since, until, untilId, includeReplies = true, qualityFilter = "all", lowQualityThreshold, includeQualityReason = false, likedOnly = false, bookmarkedOnly = false, limit = 18 }) {
-	const db = getReadDb();
+var FTS_DRIVE_FROM_MATCHES_MAX = 1e4;
+function buildTimelineItemsQuery({ resource, account, listAccountId, listId, search, replyFilter = "all", since, until, untilId, includeReplies = true, qualityFilter = "all", lowQualityThreshold, likedOnly = false, bookmarkedOnly = false, limit = 18 }, ftsMatchCountHint = 0) {
 	const kind = resource === "mentions" ? "mention" : resource;
+	const cteParams = [];
 	const params = [];
 	const normalizedLowQualityThreshold = normalizeLowQualityThreshold(lowQualityThreshold);
 	const shouldDedupeAcrossAccounts = !account || account === "all";
@@ -8917,9 +8918,8 @@ function listTimelineItems({ resource, account, listAccountId, listId, search, r
 	    `;
 	const unwindowedTimelineEdgesCte = timelineEdgesCte;
 	let usedRecentEdgeWindow = false;
-	let join = "";
 	let where = "where e.kind = ?";
-	let searchSnippetSelect = "";
+	const ftsSearch = search?.trim() ? toFtsSearchQuery(search) : "";
 	const canUseRecentEdgeWindow = !likedOnly && !bookmarkedOnly && !listId && !account && !search?.trim() && replyFilter === "all" && !since?.trim() && !until?.trim() && includeReplies && qualityFilter === "all";
 	if (likedOnly || bookmarkedOnly) {
 		if (likedOnly && bookmarkedOnly) timelineEdgesCte = `
@@ -8942,7 +8942,7 @@ function listTimelineItems({ resource, account, listAccountId, listId, search, r
 	          where kind = ?
 	        )
 				`;
-			params.push(collectionKind);
+			cteParams.push(collectionKind);
 		}
 		where = "where 1 = 1";
 	} else if (canUseRecentEdgeWindow) {
@@ -8961,11 +8961,11 @@ function listTimelineItems({ resource, account, listAccountId, listId, search, r
 	      )
 	    `;
 		const candidateLimit = Math.max(RECENT_TIMELINE_EDGE_CANDIDATES, limit * 50);
-		params.push(kind, candidateLimit);
+		cteParams.push(kind, candidateLimit);
 		where = "where e.kind = ?";
 		params.push(kind);
 	} else {
-		params.push(kind);
+		cteParams.push(kind);
 		where = "where e.kind = ?";
 		params.push(kind);
 	}
@@ -9014,16 +9014,28 @@ function listTimelineItems({ resource, account, listAccountId, listId, search, r
     `;
 		params.push(listAccountId, listId);
 	}
-	const ftsSearch = search?.trim() ? toFtsSearchQuery(search) : "";
-	if (ftsSearch) {
-		join += " join tweets_fts on tweets_fts.tweet_id = t.id ";
-		where += " and tweets_fts.text match ?";
-		searchSnippetSelect = ", snippet(tweets_fts, 1, '<mark>', '</mark>', '...', 16) as search_snippet";
-		params.push(ftsSearch);
-	}
+	const ftsMatchesCte = ftsSearch ? `, fts_matches as materialized (
+        select tweet_id
+        from tweets_fts
+        where tweets_fts.text match ?
+      )` : "";
+	if (ftsSearch) cteParams.push(ftsSearch);
+	const searchDrivenFrom = ftsMatchCountHint > FTS_DRIVE_FROM_MATCHES_MAX ? `tweets t
+        cross join fts_matches on fts_matches.tweet_id = t.id
+        cross join timeline_edges e on e.tweet_id = t.id` : `fts_matches
+        cross join tweets t on t.id = fts_matches.tweet_id
+        cross join timeline_edges e on e.tweet_id = t.id`;
 	params.push(limit);
+	if (ftsSearch) params.push(limit);
+	const searchSelectionCte = ftsSearch ? `, search_selection as materialized (
+        select t.id as tweet_id, e.account_id, e.kind, e.raw_json
+        from ${searchDrivenFrom}
+        ${where}
+        order by t.created_at desc, t.id desc
+        limit ?
+      )` : "";
 	const buildTimelineSelectSql = (timelineEdgesSql) => `
-      ${timelineEdgesSql}
+      ${timelineEdgesSql}${ftsMatchesCte}${searchSelectionCte}
       select
         t.id,
         e.account_id,
@@ -9100,8 +9112,7 @@ function listTimelineItems({ resource, account, listAccountId, listId, search, r
         qp.avatar_hue as quoted_avatar_hue,
         qp.avatar_url as quoted_avatar_url,
         qp.created_at as quoted_profile_created_at
-        ${searchSnippetSelect}
-      from timeline_edges e
+      from ${ftsSearch ? "search_selection e" : "timeline_edges e"}
       join tweets t on t.id = e.tweet_id
       join accounts a on a.id = e.account_id
       join profiles p on p.id = t.author_profile_id
@@ -9109,13 +9120,40 @@ function listTimelineItems({ resource, account, listAccountId, listId, search, r
       left join profiles rp on rp.id = rt.author_profile_id
       left join tweets qt on qt.id = t.quoted_tweet_id
       left join profiles qp on qp.id = qt.author_profile_id
-      ${join}
-      ${where}
+      ${ftsSearch ? "" : where}
       order by t.created_at desc, t.id desc
       limit ?
       `;
-	let rows = db.prepare(buildTimelineSelectSql(timelineEdgesCte)).all(...params);
-	if (usedRecentEdgeWindow && rows.length < limit) rows = db.prepare(buildTimelineSelectSql(unwindowedTimelineEdgesCte)).all(kind, kind, limit);
+	return {
+		sql: buildTimelineSelectSql(timelineEdgesCte),
+		params: [...cteParams, ...params],
+		fallbackSql: buildTimelineSelectSql(unwindowedTimelineEdgesCte),
+		fallbackParams: [
+			kind,
+			kind,
+			limit
+		],
+		usedRecentEdgeWindow,
+		ftsSearch
+	};
+}
+var SEARCH_SNIPPET_SQL = "snippet(tweets_fts, 1, '<mark>', '</mark>', '...', 16)";
+function listTimelineItems(query) {
+	const db = getReadDb();
+	const { includeQualityReason = false, lowQualityThreshold, limit = 18 } = query;
+	const normalizedLowQualityThreshold = normalizeLowQualityThreshold(lowQualityThreshold);
+	const ftsSearch = query.search?.trim() ? toFtsSearchQuery(query.search) : "";
+	const plan = buildTimelineItemsQuery(query, ftsSearch ? Number(db.prepare("select count(*) as match_count from tweets_fts where tweets_fts.text match ?").get(ftsSearch).match_count) : 0);
+	let rows = db.prepare(plan.sql).all(...plan.params);
+	if (plan.usedRecentEdgeWindow && rows.length < limit) rows = db.prepare(plan.fallbackSql).all(...plan.fallbackParams);
+	if (plan.ftsSearch && rows.length > 0) {
+		const snippetRows = db.prepare(`select tweet_id, ${SEARCH_SNIPPET_SQL} as search_snippet
+         from tweets_fts
+         where tweets_fts.text match ?
+           and tweet_id in (${rows.map(() => "?").join(",")})`).all(plan.ftsSearch, ...rows.map((row) => String(row.id)));
+		const snippetByTweetId = new Map(snippetRows.map((row) => [String(row.tweet_id), row.search_snippet]));
+		for (const row of rows) row.search_snippet = snippetByTweetId.get(String(row.id));
+	}
 	const urlExpansionCache = /* @__PURE__ */ new Map();
 	const profileByHandleCache = /* @__PURE__ */ new Map();
 	return rows.map((row) => {

package/dist/server/server.js

@@ -1767,7 +1767,7 @@ var getBaseManifest = getProdBaseManifest;
 var createEarlyHintsForRequest = createEarlyHintsCollector;
 async function loadEntries() {
 	const [routerEntry, startEntry, pluginAdapters] = await Promise.all([
-		import("./assets/router-B874XqWN.js"),
+		import("./assets/router-DVQ3M870.js"),
 		import("./assets/start-5Z2QO8AU.js"),
 		import("./assets/empty-plugin-adapters-D9UWiqvJ.js")
 	]);

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "birdclaw",
-	"version": "0.9.3",
+	"version": "0.9.4",
 	"description": "Local Twitter memory in SQLite for archives, DMs, likes, bookmarks, and moderation",
 	"homepage": "https://github.com/steipete/birdclaw#readme",
 	"license": "MIT",