birdclaw 0.3.0 → 0.4.1

package/src/lib/backup.ts

@@ -4,7 +4,7 @@ import { existsSync } from "node:fs";
 import fs from "node:fs/promises";
 import path from "node:path";
 import { promisify } from "node:util";
-import type Database from "better-sqlite3";
+import type { Database } from "./sqlite";
 import { getBirdclawConfig } from "./config";
 import { getNativeDb } from "./db";
 
@@ -160,17 +160,13 @@ function yearFromTimestamp(value: unknown) {
 	return match[1];
 }
 
-function rowsForQuery(
-	db: Database.Database,
-	sql: string,
-	params: unknown[] = [],
-) {
+function rowsForQuery(db: Database, sql: string, params: unknown[] = []) {
 	return (db.prepare(sql).all(...params) as Record<string, unknown>[]).map(
 		toJsonRecord,
 	);
 }
 
-function getExportRowSets(db: Database.Database) {
+function getExportRowSets(db: Database) {
 	const rowSets: Array<{ logicalName: string; rows: JsonRecord[] }> = [
 		{
 			logicalName: "accounts",
@@ -188,9 +184,49 @@ function getExportRowSets(db: Database.Database) {
 			rows: rowsForQuery(
 				db,
 				`
-        select id, handle, display_name, bio, followers_count, following_count, avatar_hue, avatar_url, created_at
+        select id, handle, display_name, bio, followers_count, following_count,
+          avatar_hue, avatar_url, location, url, verified_type, entities_json,
+          raw_json, created_at
         from profiles
         order by id
+        `,
+			),
+		},
+		{
+			logicalName: "profile_affiliations",
+			rows: rowsForQuery(
+				db,
+				`
+        select subject_profile_id, organization_profile_id, organization_name,
+          organization_handle, badge_url, url, label, source, is_active,
+          first_seen_at, last_seen_at, raw_json, updated_at
+        from profile_affiliations
+        order by subject_profile_id, organization_profile_id
+        `,
+			),
+		},
+		{
+			logicalName: "profile_snapshots",
+			rows: rowsForQuery(
+				db,
+				`
+        select profile_id, snapshot_hash, observed_at, last_seen_at, source,
+          handle, display_name, bio, location, url, verified_type,
+          followers_count, following_count, affiliations_json, raw_json
+        from profile_snapshots
+        order by profile_id, last_seen_at, snapshot_hash
+        `,
+			),
+		},
+		{
+			logicalName: "profile_bio_entities",
+			rows: rowsForQuery(
+				db,
+				`
+        select profile_id, kind, value, source, is_active, first_seen_at,
+          last_seen_at, raw_json
+        from profile_bio_entities
+        order by profile_id, kind, value
         `,
 			),
 		},
@@ -215,6 +251,18 @@ function getExportRowSets(db: Database.Database) {
         select account_id, tweet_id, kind, collected_at, source, raw_json, updated_at
         from tweet_collections
         order by kind, account_id, coalesce(collected_at, ''), tweet_id
+        `,
+			),
+		},
+		{
+			logicalName: "tweet_account_edges",
+			rows: rowsForQuery(
+				db,
+				`
+        select account_id, tweet_id, kind, first_seen_at, last_seen_at,
+          seen_count, source, raw_json, updated_at
+        from tweet_account_edges
+        order by kind, account_id, last_seen_at, tweet_id
         `,
 			),
 		},
@@ -303,7 +351,7 @@ function addRows(
 	shards.set(relativePath, existing);
 }
 
-function buildShards(db: Database.Database) {
+function buildShards(db: Database) {
 	const shards = new Map<string, JsonRecord[]>();
 	const rowSets = getExportRowSets(db);
 
@@ -315,6 +363,15 @@ function buildShards(db: Database.Database) {
 			case "profiles":
 				addRows(shards, "data/profiles.jsonl", rowSet.rows);
 				break;
+			case "profile_affiliations":
+				addRows(shards, "data/profile_affiliations.jsonl", rowSet.rows);
+				break;
+			case "profile_snapshots":
+				addRows(shards, "data/profile_snapshots.jsonl", rowSet.rows);
+				break;
+			case "profile_bio_entities":
+				addRows(shards, "data/profile_bio_entities.jsonl", rowSet.rows);
+				break;
 			case "tweets":
 				for (const row of rowSet.rows) {
 					addRows(
@@ -333,6 +390,15 @@ function buildShards(db: Database.Database) {
 					addRows(shards, `data/collections/${kind}.jsonl`, [row]);
 				}
 				break;
+			case "tweet_account_edges":
+				for (const row of rowSet.rows) {
+					const kind =
+						row.kind === "home" || row.kind === "mention"
+							? row.kind
+							: "unknown";
+					addRows(shards, `data/timeline_edges/${kind}.jsonl`, [row]);
+				}
+				break;
 			case "dm_conversations":
 				addRows(shards, "data/dms/conversations.jsonl", rowSet.rows);
 				break;
@@ -442,15 +508,17 @@ function computeCounts(files: BackupFileManifest[]) {
 				? "tweets"
 				: second === "collections"
 					? `collections_${third?.replace(/\.jsonl$/, "") ?? "unknown"}`
-					: second === "dms" && third === "conversations.jsonl"
-						? "dm_conversations"
-						: second === "dms"
-							? "dm_messages"
-							: second === "moderation"
-								? third?.replace(/\.jsonl$/, "") || "moderation"
-								: second === "actions"
-									? third?.replace(/\.jsonl$/, "") || "actions"
-									: second?.replace(/\.jsonl$/, "") || "unknown";
+					: second === "timeline_edges"
+						? `timeline_edges_${third?.replace(/\.jsonl$/, "") ?? "unknown"}`
+						: second === "dms" && third === "conversations.jsonl"
+							? "dm_conversations"
+							: second === "dms"
+								? "dm_messages"
+								: second === "moderation"
+									? third?.replace(/\.jsonl$/, "") || "moderation"
+									: second === "actions"
+										? third?.replace(/\.jsonl$/, "") || "actions"
+										: second?.replace(/\.jsonl$/, "") || "unknown";
 		counts[key] = (counts[key] ?? 0) + file.rows;
 	}
 	return counts;
@@ -473,8 +541,13 @@ Private text backup for Birdclaw data. The committed files are canonical JSONL s
 manifest.json
 data/accounts.jsonl
 data/profiles.jsonl
+data/profile_affiliations.jsonl
+data/profile_snapshots.jsonl
+data/profile_bio_entities.jsonl
 data/tweets/YYYY.jsonl
 data/tweets/unknown.jsonl
+data/timeline_edges/home.jsonl
+data/timeline_edges/mention.jsonl
 data/collections/likes.jsonl
 data/collections/bookmarks.jsonl
 data/dms/conversations.jsonl
@@ -483,7 +556,7 @@ data/moderation/blocks.jsonl
 data/moderation/mutes.jsonl
 \`\`\`
 
-Tweets are sharded by creation year. Collection-only tweets whose creation date is unknown live in \`data/tweets/unknown.jsonl\`. DMs are sharded by year and keep \`conversation_id\` in each row.
+Tweets are sharded by creation year. Collection-only tweets whose creation date is unknown live in \`data/tweets/unknown.jsonl\`. Timeline edges keep account-scoped home/mention membership separate from canonical tweet content. DMs are sharded by year and keep \`conversation_id\` in each row.
 
 Never commit live tokens, browser cookies, raw SQLite WAL/SHM sidecars, or temporary cache files here.
 `,
@@ -735,7 +808,7 @@ export async function exportBackup({
 	validate = true,
 }: {
 	repoPath: string;
-	db?: Database.Database;
+	db?: Database;
 	commit?: boolean;
 	push?: boolean;
 	message?: string;
@@ -847,7 +920,7 @@ function rowsForManifestPath(
 }
 
 function insertRows(
-	db: Database.Database,
+	db: Database,
 	sql: string,
 	rows: JsonRecord[],
 	keys: string[],
@@ -859,7 +932,7 @@ function insertRows(
 }
 
 function readFtsIds(
-	db: Database.Database,
+	db: Database,
 	tableName: "tweets_fts" | "dm_fts",
 	idColumn: "tweet_id" | "message_id",
 ) {
@@ -870,7 +943,7 @@ function readFtsIds(
 }
 
 function insertFtsRows(
-	db: Database.Database,
+	db: Database,
 	tableName: "tweets_fts" | "dm_fts",
 	idColumn: "tweet_id" | "message_id",
 	rows: JsonRecord[],
@@ -892,11 +965,14 @@ function insertFtsRows(
 	}
 }
 
-function clearBackupImportData(db: Database.Database) {
+function clearBackupImportData(db: Database) {
 	db.exec(`
     delete from ai_scores;
     delete from tweet_actions;
+    delete from tweet_account_edges;
     delete from tweet_collections;
+    delete from link_occurrences;
+    delete from url_expansions;
     delete from blocks;
     delete from mutes;
     delete from dm_fts;
@@ -904,6 +980,9 @@ function clearBackupImportData(db: Database.Database) {
     delete from dm_messages;
     delete from dm_conversations;
     delete from tweets;
+    delete from profile_bio_entities;
+    delete from profile_snapshots;
+    delete from profile_affiliations;
     delete from profiles;
     delete from accounts;
     delete from sync_cache;
@@ -917,7 +996,7 @@ export async function importBackup({
 	mode = "merge",
 }: {
 	repoPath: string;
-	db?: Database.Database;
+	db?: Database;
 	validate?: boolean;
 	mode?: BackupImportMode;
 }): Promise<BackupImportResult> {
@@ -938,8 +1017,12 @@ export async function importBackup({
 	const [
 		accounts,
 		profiles,
+		profileAffiliations,
+		profileSnapshots,
+		profileBioEntities,
 		tweets,
 		collections,
+		timelineEdges,
 		conversations,
 		messages,
 		blocks,
@@ -949,8 +1032,12 @@ export async function importBackup({
 	] = await Promise.all([
 		readRows((file) => file === "data/accounts.jsonl"),
 		readRows((file) => file === "data/profiles.jsonl"),
+		readRows((file) => file === "data/profile_affiliations.jsonl"),
+		readRows((file) => file === "data/profile_snapshots.jsonl"),
+		readRows((file) => file === "data/profile_bio_entities.jsonl"),
 		readRows((file) => file.startsWith("data/tweets/")),
 		readRows((file) => file.startsWith("data/collections/")),
+		readRows((file) => file.startsWith("data/timeline_edges/")),
 		readRows((file) => file === "data/dms/conversations.jsonl"),
 		readRows(
 			(file) =>
@@ -1001,9 +1088,73 @@ export async function importBackup({
 		insertRows(
 			db,
 			`
+      insert into profile_snapshots (
+        profile_id, snapshot_hash, observed_at, last_seen_at, source, handle,
+        display_name, bio, location, url, verified_type, followers_count,
+        following_count, affiliations_json, raw_json
+      ) values (?, ?, ?, ?, coalesce(?, 'backup'), ?, ?, ?, ?, ?, ?, coalesce(?, 0), coalesce(?, 0), coalesce(?, '[]'), coalesce(?, '{}'))
+      on conflict(profile_id, snapshot_hash) do update set
+        last_seen_at = max(profile_snapshots.last_seen_at, excluded.last_seen_at),
+        source = excluded.source,
+        raw_json = case
+          when excluded.raw_json not in ('', '{}', 'null') then excluded.raw_json
+          else profile_snapshots.raw_json
+        end
+      `,
+			profileSnapshots,
+			[
+				"profile_id",
+				"snapshot_hash",
+				"observed_at",
+				"last_seen_at",
+				"source",
+				"handle",
+				"display_name",
+				"bio",
+				"location",
+				"url",
+				"verified_type",
+				"followers_count",
+				"following_count",
+				"affiliations_json",
+				"raw_json",
+			],
+		);
+		insertRows(
+			db,
+			`
+      insert into profile_bio_entities (
+        profile_id, kind, value, source, is_active, first_seen_at, last_seen_at, raw_json
+      ) values (?, ?, ?, coalesce(?, 'backup'), coalesce(?, 1), ?, ?, coalesce(?, '{}'))
+      on conflict(profile_id, kind, value) do update set
+        source = excluded.source,
+        is_active = excluded.is_active,
+        last_seen_at = max(profile_bio_entities.last_seen_at, excluded.last_seen_at),
+        raw_json = case
+          when excluded.raw_json not in ('', '{}', 'null') then excluded.raw_json
+          else profile_bio_entities.raw_json
+        end
+      `,
+			profileBioEntities,
+			[
+				"profile_id",
+				"kind",
+				"value",
+				"source",
+				"is_active",
+				"first_seen_at",
+				"last_seen_at",
+				"raw_json",
+			],
+		);
+		insertRows(
+			db,
+			`
       insert into profiles (
-        id, handle, display_name, bio, followers_count, following_count, avatar_hue, avatar_url, created_at
-      ) values (?, ?, ?, ?, ?, coalesce(?, 0), ?, ?, ?)
+        id, handle, display_name, bio, followers_count, following_count,
+        avatar_hue, avatar_url, location, url, verified_type, entities_json,
+        raw_json, created_at
+      ) values (?, ?, ?, ?, ?, coalesce(?, 0), ?, ?, ?, ?, ?, coalesce(?, '{}'), coalesce(?, '{}'), ?)
       on conflict(id) do update set
         handle = coalesce(nullif(excluded.handle, ''), profiles.handle),
         display_name = coalesce(nullif(excluded.display_name, ''), profiles.display_name),
@@ -1012,6 +1163,17 @@ export async function importBackup({
         following_count = max(profiles.following_count, excluded.following_count),
         avatar_hue = case when profiles.avatar_hue = 0 then excluded.avatar_hue else profiles.avatar_hue end,
         avatar_url = coalesce(excluded.avatar_url, profiles.avatar_url),
+        location = coalesce(excluded.location, profiles.location),
+        url = coalesce(excluded.url, profiles.url),
+        verified_type = coalesce(excluded.verified_type, profiles.verified_type),
+        entities_json = case
+          when excluded.entities_json not in ('', '{}', 'null') then excluded.entities_json
+          else profiles.entities_json
+        end,
+        raw_json = case
+          when excluded.raw_json not in ('', '{}', 'null') then excluded.raw_json
+          else profiles.raw_json
+        end,
         created_at = min(profiles.created_at, excluded.created_at)
       `,
 			profiles,
@@ -1024,12 +1186,57 @@ export async function importBackup({
 				"following_count",
 				"avatar_hue",
 				"avatar_url",
+				"location",
+				"url",
+				"verified_type",
+				"entities_json",
+				"raw_json",
 				"created_at",
 			],
 		);
 		insertRows(
 			db,
 			`
+      insert into profile_affiliations (
+        subject_profile_id, organization_profile_id, organization_name,
+        organization_handle, badge_url, url, label, source, is_active,
+        first_seen_at, last_seen_at, raw_json, updated_at
+      ) values (?, ?, ?, ?, ?, ?, ?, coalesce(?, 'backup'), coalesce(?, 1), ?, ?, coalesce(?, '{}'), ?)
+      on conflict(subject_profile_id, organization_profile_id) do update set
+        organization_name = coalesce(excluded.organization_name, profile_affiliations.organization_name),
+        organization_handle = coalesce(excluded.organization_handle, profile_affiliations.organization_handle),
+        badge_url = coalesce(excluded.badge_url, profile_affiliations.badge_url),
+        url = coalesce(excluded.url, profile_affiliations.url),
+        label = coalesce(excluded.label, profile_affiliations.label),
+        source = excluded.source,
+        is_active = excluded.is_active,
+        last_seen_at = max(profile_affiliations.last_seen_at, excluded.last_seen_at),
+        raw_json = case
+          when excluded.raw_json not in ('', '{}', 'null') then excluded.raw_json
+          else profile_affiliations.raw_json
+        end,
+        updated_at = excluded.updated_at
+      `,
+			profileAffiliations,
+			[
+				"subject_profile_id",
+				"organization_profile_id",
+				"organization_name",
+				"organization_handle",
+				"badge_url",
+				"url",
+				"label",
+				"source",
+				"is_active",
+				"first_seen_at",
+				"last_seen_at",
+				"raw_json",
+				"updated_at",
+			],
+		);
+		insertRows(
+			db,
+			`
       insert into tweets (
         id, account_id, author_profile_id, kind, text, created_at, is_replied,
         reply_to_id, like_count, media_count, bookmarked, liked, entities_json,
@@ -1118,6 +1325,37 @@ export async function importBackup({
 		insertRows(
 			db,
 			`
+      insert into tweet_account_edges (
+        account_id, tweet_id, kind, first_seen_at, last_seen_at, seen_count,
+        source, raw_json, updated_at
+      ) values (?, ?, ?, ?, ?, coalesce(?, 1), coalesce(?, 'backup'), coalesce(?, '{}'), ?)
+      on conflict(account_id, tweet_id, kind) do update set
+        first_seen_at = min(tweet_account_edges.first_seen_at, excluded.first_seen_at),
+        last_seen_at = max(tweet_account_edges.last_seen_at, excluded.last_seen_at),
+        seen_count = max(tweet_account_edges.seen_count, excluded.seen_count),
+        source = coalesce(nullif(excluded.source, ''), tweet_account_edges.source),
+        raw_json = case
+          when excluded.raw_json not in ('', '{}', 'null') then excluded.raw_json
+          else tweet_account_edges.raw_json
+        end,
+        updated_at = max(tweet_account_edges.updated_at, excluded.updated_at)
+      `,
+			timelineEdges,
+			[
+				"account_id",
+				"tweet_id",
+				"kind",
+				"first_seen_at",
+				"last_seen_at",
+				"seen_count",
+				"source",
+				"raw_json",
+				"updated_at",
+			],
+		);
+		insertRows(
+			db,
+			`
       insert into dm_conversations (
         id, account_id, participant_profile_id, title, last_message_at, unread_count, needs_reply
       ) values (?, ?, ?, ?, ?, ?, ?)
@@ -1251,7 +1489,7 @@ export async function syncBackup({
 }: {
 	repoPath: string;
 	remote?: string;
-	db?: Database.Database;
+	db?: Database;
 	message?: string;
 }): Promise<BackupSyncResult> {
 	const resolvedRepoPath = path.resolve(repoPath);
@@ -1291,7 +1529,7 @@ export async function updateBackupFromGit({
 }: {
 	repoPath: string;
 	remote?: string;
-	db?: Database.Database;
+	db?: Database;
 }): Promise<{
 	ok: true;
 	repoPath: string;
@@ -1322,7 +1560,7 @@ export async function updateBackupFromGit({
 	};
 }
 
-function readAutoSyncState(db: Database.Database) {
+function readAutoSyncState(db: Database) {
 	const row = db
 		.prepare("select value_json from sync_cache where cache_key = ?")
 		.get(AUTO_SYNC_CACHE_KEY) as { value_json: string } | undefined;
@@ -1341,7 +1579,7 @@ function readAutoSyncState(db: Database.Database) {
 }
 
 function writeAutoSyncState(
-	db: Database.Database,
+	db: Database,
 	value: { checkedAt: string; ok: boolean; error?: string },
 ) {
 	db.prepare(
@@ -1382,7 +1620,7 @@ function resolveAutoSyncConfig() {
 }
 
 export async function maybeAutoUpdateBackup(
-	db?: Database.Database,
+	db?: Database,
 ): Promise<BackupAutoUpdateResult> {
 	if (process.env.BIRDCLAW_BACKUP_AUTO_SYNC === "0") {
 		return {
@@ -1449,7 +1687,7 @@ export async function maybeAutoUpdateBackup(
 }
 
 export async function maybeAutoSyncBackup(
-	db?: Database.Database,
+	db?: Database,
 ): Promise<BackupAutoUpdateResult> {
 	if (process.env.BIRDCLAW_BACKUP_AUTO_SYNC === "0") {
 		return {

package/src/lib/bird-actions.ts

@@ -90,10 +90,27 @@ function toBirdLookupUser(payload: Record<string, unknown>): XurlMentionUser {
 		username,
 		description:
 			typeof user.description === "string" ? user.description : undefined,
+		location: typeof user.location === "string" ? user.location : undefined,
+		url: typeof user.url === "string" ? user.url : undefined,
+		verified: typeof user.verified === "boolean" ? user.verified : undefined,
+		verified_type:
+			typeof user.verifiedType === "string"
+				? user.verifiedType
+				: typeof user.verified_type === "string"
+					? user.verified_type
+					: undefined,
 		profile_image_url:
 			typeof user.profileImageUrl === "string"
 				? user.profileImageUrl
 				: undefined,
+		entities:
+			user.entities && typeof user.entities === "object"
+				? (user.entities as Record<string, unknown>)
+				: undefined,
+		affiliation:
+			user.affiliation && typeof user.affiliation === "object"
+				? (user.affiliation as Record<string, unknown>)
+				: undefined,
 		public_metrics: {
 			followers_count: Number.isFinite(followersCount) ? followersCount : 0,
 			...(Number.isFinite(followingCount)