birdclaw 0.8.3 → 0.8.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (284) hide show
  1. package/CHANGELOG.md +17 -0
  2. package/README.md +10 -8
  3. package/bin/birdclaw.mjs +4 -66
  4. package/dist/cli/birdclaw.js +28165 -0
  5. package/dist/client/assets/AvatarChip-DLoqM5xD.js +1 -0
  6. package/dist/client/assets/ProfilePreview-BejAuYsJ.js +1 -0
  7. package/dist/client/assets/SavedTimelineView-BIS7HKpp.js +1 -0
  8. package/dist/client/assets/SmartTimestamp-BR0-9se2.js +1 -0
  9. package/dist/client/assets/TimelineRouteFrame-CyGPj0eM.js +1 -0
  10. package/dist/client/assets/blocks-7UDKQp8H.js +1 -0
  11. package/dist/client/assets/bookmarks-CbBkMvXk.js +1 -0
  12. package/dist/client/assets/chunk-C0phanGV.js +1 -0
  13. package/dist/client/assets/circle-check-C_Vq3ElD.js +1 -0
  14. package/dist/client/assets/client-stream-contracts-ys3XhWG4.js +5 -0
  15. package/dist/client/assets/createLucideIcon-JV8isFFR.js +1 -0
  16. package/dist/client/assets/data-sources-Cm2bpNvg.js +1 -0
  17. package/dist/client/assets/discuss-DTjaOugw.js +1 -0
  18. package/dist/client/assets/dms-CGd6Eogq.js +1 -0
  19. package/dist/client/assets/inbox-BHlGuOy2.js +1 -0
  20. package/dist/client/assets/index-CEIG7S6s.js +31 -0
  21. package/dist/client/assets/index-bpFrQ5yG.css +1 -0
  22. package/dist/client/assets/likes-cLrcAQem.js +1 -0
  23. package/dist/client/assets/links-CM9QO1LC.js +1 -0
  24. package/dist/client/assets/mapbox-BxTlqXt9.js +2 -0
  25. package/dist/client/assets/mapbox-gl-dgf_hX3a.js +3215 -0
  26. package/dist/client/assets/mentions-CxKCgzIf.js +1 -0
  27. package/dist/client/assets/network-map-B5a3whqj.js +2 -0
  28. package/dist/client/assets/profile-analyze-CRtNEwlb.js +1 -0
  29. package/dist/client/assets/profile-hydration-client-YEtJsNTv.js +1 -0
  30. package/dist/client/assets/profiles._handle-C82_HoKX.js +1 -0
  31. package/dist/client/assets/query-client-fYu2rzRC.js +17 -0
  32. package/dist/client/assets/rate-limits-ScaEwdVK.js +1 -0
  33. package/dist/client/assets/react-dom-DHl6Ug7F.js +1 -0
  34. package/dist/client/assets/react-nBylq1U5.js +1 -0
  35. package/dist/client/assets/repeat-2-xCwHM9hD.js +1 -0
  36. package/dist/client/assets/routes-q7ZaW4En.js +1 -0
  37. package/dist/client/assets/schemas-I9r6F6ya.js +64 -0
  38. package/dist/client/assets/search-Dq5kYsxA.js +1 -0
  39. package/dist/client/assets/sparkles-CQVlvHsZ.js +1 -0
  40. package/dist/client/assets/styles-B8s3tt3r.css +2 -0
  41. package/dist/client/assets/today-CeG1O7Py.js +1 -0
  42. package/dist/client/assets/triangle-alert-CBL7s2pf.js +1 -0
  43. package/dist/client/assets/useTimelineRouteData-1HsqgvUi.js +1 -0
  44. package/dist/client/assets/users-CHbKlqUr.js +1 -0
  45. package/dist/server/assets/AvatarChip-zUDAEmjL.js +105 -0
  46. package/dist/server/assets/FeedState-ntW0Q2GB.js +214 -0
  47. package/dist/server/assets/ProfileAnalysisStream-BFvhVdhn.js +198 -0
  48. package/dist/server/assets/ProfilePreview-C6gAor_j.js +752 -0
  49. package/dist/server/assets/SavedTimelineView-BG1jnBp1.js +64 -0
  50. package/dist/server/assets/SmartTimestamp-DfwE1NGf.js +45 -0
  51. package/dist/server/assets/SyncNowButton-BOn_IBXS.js +76 -0
  52. package/dist/server/assets/TimelineRouteFrame-CU7ilM7U.js +85 -0
  53. package/dist/server/assets/TweetRichText-DhCVv1-6.js +74 -0
  54. package/dist/server/assets/_tanstack-start-manifest_v-C-ZRnG_n.js +148 -0
  55. package/dist/server/assets/account-selection-CTdFGWRk.js +44 -0
  56. package/dist/server/assets/blocks-C1mgGTB3.js +299 -0
  57. package/dist/server/assets/blocks-DSgQghqP.js +17 -0
  58. package/dist/server/assets/bookmarks-BJN8tykY.js +14 -0
  59. package/dist/server/assets/client-stream-contracts-C2xbWVjm.js +769 -0
  60. package/dist/server/assets/data-sources-BnA1Q94p.js +180 -0
  61. package/dist/server/assets/discuss-CJ1dau1j.js +15 -0
  62. package/dist/server/assets/discuss-QTo08NCN.js +338 -0
  63. package/dist/server/assets/dms-BuLj4bp8.js +368 -0
  64. package/dist/server/assets/dms-DAyQrVM2.js +240 -0
  65. package/dist/server/assets/empty-plugin-adapters-DzrEkdBF.js +5 -0
  66. package/dist/server/assets/inbox-CkGOaW1q.js +214 -0
  67. package/dist/server/assets/inbox-DpWd1EzC.js +146 -0
  68. package/dist/server/assets/likes-DBOAtsJU.js +14 -0
  69. package/dist/server/assets/links-DAhSjynu.js +240 -0
  70. package/dist/server/assets/links-DmH5lxHO.js +468 -0
  71. package/dist/server/assets/mentions-BH2MzKtB.js +26 -0
  72. package/dist/server/assets/network-map-D3CzRcC4.js +224 -0
  73. package/dist/server/assets/network-map-D5DoLOwR.js +582 -0
  74. package/dist/server/assets/profile-analyze-BP65PED0.js +90 -0
  75. package/dist/server/assets/profile-analyze-C1NoM0tV.js +9 -0
  76. package/dist/server/assets/profile-hydration-client-GJ4kwJS8.js +133 -0
  77. package/dist/server/assets/profiles._handle-DjwlN_ac.js +184 -0
  78. package/dist/server/assets/profiles._handle-DsLNj2Ca.js +12 -0
  79. package/dist/server/assets/query-client-BdauHBgd.js +923 -0
  80. package/dist/server/assets/rate-limits-VAr2qzYD.js +262 -0
  81. package/dist/server/assets/route-search-BUlQ5lGM.js +117 -0
  82. package/dist/server/assets/router-DqkwSPiM.js +17026 -0
  83. package/dist/server/assets/routes-A9c9ZhwN.js +26 -0
  84. package/dist/server/assets/start-FaDOTmto.js +4 -0
  85. package/dist/server/assets/today-B5hVM57Z.js +17 -0
  86. package/dist/server/assets/today-fvGiZ7ud.js +303 -0
  87. package/dist/server/assets/ui-qbmcMSSX.js +137 -0
  88. package/{src/components/useDebouncedValue.ts → dist/server/assets/useDebouncedValue-DOnA6Xcy.js} +4 -4
  89. package/dist/server/assets/useTimelineRouteData-27cxynxa.js +1101 -0
  90. package/dist/server/server.js +5520 -0
  91. package/package.json +26 -34
  92. package/playwright.config.ts +0 -35
  93. package/scripts/browser-perf.mjs +0 -427
  94. package/scripts/build-docs-site.mjs +0 -967
  95. package/scripts/docs-site-assets.mjs +0 -311
  96. package/scripts/run-vitest.mjs +0 -21
  97. package/scripts/sanitize-node-options.mjs +0 -23
  98. package/scripts/start-test-server.mjs +0 -42
  99. package/src/cli/command-context.ts +0 -17
  100. package/src/cli/register-analysis.ts +0 -500
  101. package/src/cli/register-compose.ts +0 -40
  102. package/src/cli/register-graph.ts +0 -132
  103. package/src/cli/register-inbox.ts +0 -41
  104. package/src/cli/register-storage.ts +0 -106
  105. package/src/cli-moderation.ts +0 -210
  106. package/src/cli.ts +0 -1616
  107. package/src/components/AccountSwitcher.tsx +0 -178
  108. package/src/components/AppNav.tsx +0 -118
  109. package/src/components/AvatarChip.tsx +0 -58
  110. package/src/components/AvatarPreload.ts +0 -149
  111. package/src/components/BrandMark.tsx +0 -67
  112. package/src/components/ConversationThread.tsx +0 -131
  113. package/src/components/DmWorkspace.tsx +0 -286
  114. package/src/components/EmbeddedTweetCard.tsx +0 -55
  115. package/src/components/FeedState.tsx +0 -147
  116. package/src/components/FloatingPreview.tsx +0 -382
  117. package/src/components/InboxCard.tsx +0 -152
  118. package/src/components/LinkPreviewCard.tsx +0 -292
  119. package/src/components/MarkdownCitations.tsx +0 -680
  120. package/src/components/MarkdownViewer.tsx +0 -111
  121. package/src/components/ProfileAnalysisClient.ts +0 -191
  122. package/src/components/ProfileAnalysisStream.tsx +0 -259
  123. package/src/components/ProfilePreview.tsx +0 -131
  124. package/src/components/SavedTimelineView.tsx +0 -155
  125. package/src/components/SmartTimestamp.tsx +0 -72
  126. package/src/components/SyncNowButton.tsx +0 -140
  127. package/src/components/ThemeSlider.tsx +0 -80
  128. package/src/components/TimelineCard.tsx +0 -533
  129. package/src/components/TimelineRouteFrame.tsx +0 -186
  130. package/src/components/TweetArticleCard.tsx +0 -66
  131. package/src/components/TweetMediaGrid.tsx +0 -185
  132. package/src/components/TweetRichText.tsx +0 -159
  133. package/src/components/account-selection.ts +0 -64
  134. package/src/components/links-controller.ts +0 -162
  135. package/src/components/links-model.ts +0 -198
  136. package/src/components/network-map-controller.ts +0 -84
  137. package/src/components/network-map-model.ts +0 -255
  138. package/src/components/useTimelineRouteData.ts +0 -193
  139. package/src/lib/account-sync-job.ts +0 -666
  140. package/src/lib/actions-transport.ts +0 -300
  141. package/src/lib/analysis-runtime.ts +0 -238
  142. package/src/lib/api-client.ts +0 -220
  143. package/src/lib/api-contracts.ts +0 -328
  144. package/src/lib/archive-finder.ts +0 -185
  145. package/src/lib/archive-import-plan.ts +0 -102
  146. package/src/lib/archive-import.ts +0 -2695
  147. package/src/lib/authored-live.ts +0 -1027
  148. package/src/lib/avatar-cache.ts +0 -349
  149. package/src/lib/backup.ts +0 -2652
  150. package/src/lib/bird-actions.ts +0 -294
  151. package/src/lib/bird-command.ts +0 -130
  152. package/src/lib/bird.ts +0 -1140
  153. package/src/lib/blocklist.ts +0 -136
  154. package/src/lib/blocks-write.ts +0 -171
  155. package/src/lib/blocks.ts +0 -387
  156. package/src/lib/bookmark-sync-job.ts +0 -536
  157. package/src/lib/config.ts +0 -224
  158. package/src/lib/conversation-surface.ts +0 -205
  159. package/src/lib/data-sources.ts +0 -219
  160. package/src/lib/database-metrics.ts +0 -88
  161. package/src/lib/database-migrations.ts +0 -34
  162. package/src/lib/database-schema.ts +0 -312
  163. package/src/lib/database-writer.ts +0 -69
  164. package/src/lib/db.ts +0 -889
  165. package/src/lib/dm-read-model.ts +0 -533
  166. package/src/lib/dms-live.ts +0 -870
  167. package/src/lib/effect-runtime.ts +0 -45
  168. package/src/lib/follow-graph.ts +0 -1087
  169. package/src/lib/geocoding.ts +0 -296
  170. package/src/lib/http-effect.ts +0 -222
  171. package/src/lib/identity-search-index.ts +0 -369
  172. package/src/lib/import-repository.ts +0 -138
  173. package/src/lib/inbox.ts +0 -242
  174. package/src/lib/link-index.ts +0 -641
  175. package/src/lib/link-insights.ts +0 -860
  176. package/src/lib/link-preview-metadata.ts +0 -754
  177. package/src/lib/live-sync-engine.ts +0 -209
  178. package/src/lib/live-transport-gateway.ts +0 -128
  179. package/src/lib/location.ts +0 -137
  180. package/src/lib/media-fetch.ts +0 -860
  181. package/src/lib/media-includes.ts +0 -165
  182. package/src/lib/mention-threads-live.ts +0 -830
  183. package/src/lib/mentions-export.ts +0 -147
  184. package/src/lib/mentions-live.ts +0 -1114
  185. package/src/lib/moderation-target.ts +0 -227
  186. package/src/lib/moderation-write.ts +0 -131
  187. package/src/lib/mutes-write.ts +0 -171
  188. package/src/lib/mutes.ts +0 -84
  189. package/src/lib/ndjson-stream.ts +0 -106
  190. package/src/lib/network-map.ts +0 -382
  191. package/src/lib/openai-response-runtime.ts +0 -251
  192. package/src/lib/openai.ts +0 -117
  193. package/src/lib/paginated-sync.ts +0 -93
  194. package/src/lib/period-digest.ts +0 -1364
  195. package/src/lib/present.ts +0 -118
  196. package/src/lib/profile-affiliation-hydration.ts +0 -228
  197. package/src/lib/profile-affiliations.ts +0 -262
  198. package/src/lib/profile-analysis.ts +0 -1198
  199. package/src/lib/profile-bio-entities.ts +0 -318
  200. package/src/lib/profile-history.ts +0 -164
  201. package/src/lib/profile-hydration.ts +0 -180
  202. package/src/lib/profile-replies.ts +0 -77
  203. package/src/lib/profile-resolver.ts +0 -650
  204. package/src/lib/queries.ts +0 -6
  205. package/src/lib/query-actions.ts +0 -437
  206. package/src/lib/query-client.tsx +0 -47
  207. package/src/lib/query-read-model-shared.ts +0 -52
  208. package/src/lib/query-read-models.ts +0 -5
  209. package/src/lib/query-resource.ts +0 -41
  210. package/src/lib/query-status.ts +0 -164
  211. package/src/lib/research.ts +0 -652
  212. package/src/lib/runtime-services.ts +0 -20
  213. package/src/lib/search-discussion.ts +0 -812
  214. package/src/lib/seed.ts +0 -613
  215. package/src/lib/server-runtime-services.ts +0 -30
  216. package/src/lib/sqlite.ts +0 -183
  217. package/src/lib/streaming-ingestion.ts +0 -240
  218. package/src/lib/sync-cache.ts +0 -70
  219. package/src/lib/sync-plan.ts +0 -175
  220. package/src/lib/theme-transition.ts +0 -113
  221. package/src/lib/theme.tsx +0 -165
  222. package/src/lib/timeline-collections-live.ts +0 -407
  223. package/src/lib/timeline-live.ts +0 -302
  224. package/src/lib/timeline-read-model.ts +0 -1191
  225. package/src/lib/tweet-account-edges.ts +0 -48
  226. package/src/lib/tweet-lookup.ts +0 -46
  227. package/src/lib/tweet-render.ts +0 -430
  228. package/src/lib/tweet-repository.ts +0 -156
  229. package/src/lib/tweet-search-live.ts +0 -462
  230. package/src/lib/types.ts +0 -723
  231. package/src/lib/ui.ts +0 -529
  232. package/src/lib/url-expansion-store.ts +0 -110
  233. package/src/lib/url-expansion.ts +0 -343
  234. package/src/lib/url-safety.ts +0 -220
  235. package/src/lib/web-sync.ts +0 -533
  236. package/src/lib/whois.ts +0 -1080
  237. package/src/lib/x-profile.ts +0 -365
  238. package/src/lib/x-web.ts +0 -199
  239. package/src/lib/xurl-rate-limits.ts +0 -267
  240. package/src/lib/xurl.ts +0 -1677
  241. package/src/routeTree.gen.ts +0 -723
  242. package/src/router.tsx +0 -20
  243. package/src/routes/__root.tsx +0 -82
  244. package/src/routes/api/action.tsx +0 -147
  245. package/src/routes/api/avatar.tsx +0 -50
  246. package/src/routes/api/blocks.tsx +0 -35
  247. package/src/routes/api/conversation.tsx +0 -45
  248. package/src/routes/api/data-sources.tsx +0 -24
  249. package/src/routes/api/inbox.tsx +0 -43
  250. package/src/routes/api/link-insights.tsx +0 -85
  251. package/src/routes/api/link-preview.tsx +0 -47
  252. package/src/routes/api/network-map.tsx +0 -55
  253. package/src/routes/api/period-digest.tsx +0 -101
  254. package/src/routes/api/profile-analysis.tsx +0 -97
  255. package/src/routes/api/profile-hydrate.tsx +0 -54
  256. package/src/routes/api/query.tsx +0 -121
  257. package/src/routes/api/search-discussion.tsx +0 -113
  258. package/src/routes/api/status.tsx +0 -30
  259. package/src/routes/api/sync.tsx +0 -108
  260. package/src/routes/api/xurl-rate-limits.tsx +0 -24
  261. package/src/routes/blocks.tsx +0 -343
  262. package/src/routes/bookmarks.tsx +0 -18
  263. package/src/routes/data-sources.tsx +0 -249
  264. package/src/routes/discuss.tsx +0 -419
  265. package/src/routes/dms.tsx +0 -415
  266. package/src/routes/inbox.tsx +0 -232
  267. package/src/routes/index.tsx +0 -32
  268. package/src/routes/likes.tsx +0 -18
  269. package/src/routes/links.tsx +0 -631
  270. package/src/routes/mentions.tsx +0 -32
  271. package/src/routes/network-map.tsx +0 -732
  272. package/src/routes/profile-analyze.tsx +0 -112
  273. package/src/routes/profiles.$handle.tsx +0 -228
  274. package/src/routes/rate-limits.tsx +0 -305
  275. package/src/routes/today.tsx +0 -511
  276. package/src/styles.css +0 -257
  277. package/tsconfig.json +0 -29
  278. package/vite.config.ts +0 -29
  279. /package/{public → dist/client}/birdclaw-mark.png +0 -0
  280. /package/{public → dist/client}/favicon.ico +0 -0
  281. /package/{public → dist/client}/logo192.png +0 -0
  282. /package/{public → dist/client}/logo512.png +0 -0
  283. /package/{public → dist/client}/manifest.json +0 -0
  284. /package/{public → dist/client}/robots.txt +0 -0
package/src/lib/db.ts DELETED
@@ -1,889 +0,0 @@
1
- import NativeSqliteDatabase, {
2
- type Database,
3
- SQLITE_BUSY_TIMEOUT_MS,
4
- } from "./sqlite";
5
- import { Kysely, SqliteDialect } from "kysely";
6
- import { ensureBirdclawDirs, getBirdclawPaths } from "./config";
7
- import {
8
- type DatabaseMigration,
9
- runDatabaseMigrations,
10
- } from "./database-migrations";
11
- import { seedDemoData } from "./seed";
12
- import {
13
- type DatabaseConnectionRole,
14
- recordDatabaseStatement,
15
- } from "./database-metrics";
16
-
17
- import type { BirdclawDatabase } from "./database-schema";
18
- export * from "./database-schema";
19
-
20
- let nativeDb: Database | undefined;
21
- let readDbs: Database[] = [];
22
- let readDbIndex = 0;
23
- let kyselyDb: Kysely<BirdclawDatabase> | undefined;
24
- let demoSeedAttempted = false;
25
-
26
- export interface InitDatabaseOptions {
27
- seedDemoData?: boolean;
28
- }
29
-
30
- const BASE_SCHEMA_SQL = `
31
- pragma journal_mode = wal;
32
- pragma busy_timeout = ${SQLITE_BUSY_TIMEOUT_MS};
33
- pragma foreign_keys = on;
34
-
35
- create table if not exists accounts (
36
- id text primary key,
37
- name text not null,
38
- handle text not null unique,
39
- external_user_id text,
40
- transport text not null,
41
- is_default integer not null default 0,
42
- created_at text not null
43
- );
44
-
45
- create table if not exists profiles (
46
- id text primary key,
47
- handle text not null unique,
48
- display_name text not null,
49
- bio text not null,
50
- followers_count integer not null default 0,
51
- following_count integer not null default 0,
52
- public_metrics_json text not null default '{}',
53
- avatar_hue integer not null default 0,
54
- avatar_url text,
55
- location text,
56
- url text,
57
- verified_type text,
58
- entities_json text not null default '{}',
59
- raw_json text not null default '{}',
60
- created_at text not null
61
- );
62
-
63
- create table if not exists profile_affiliations (
64
- subject_profile_id text not null,
65
- organization_profile_id text not null,
66
- organization_name text,
67
- organization_handle text,
68
- badge_url text,
69
- url text,
70
- label text,
71
- source text not null,
72
- is_active integer not null default 1,
73
- first_seen_at text not null,
74
- last_seen_at text not null,
75
- raw_json text not null default '{}',
76
- updated_at text not null,
77
- primary key (subject_profile_id, organization_profile_id)
78
- );
79
-
80
- create table if not exists profile_snapshots (
81
- profile_id text not null,
82
- snapshot_hash text not null,
83
- observed_at text not null,
84
- last_seen_at text not null,
85
- source text not null,
86
- handle text not null,
87
- display_name text not null,
88
- bio text not null,
89
- location text,
90
- url text,
91
- verified_type text,
92
- followers_count integer not null default 0,
93
- following_count integer not null default 0,
94
- affiliations_json text not null default '[]',
95
- raw_json text not null default '{}',
96
- primary key (profile_id, snapshot_hash)
97
- );
98
-
99
- create table if not exists profile_bio_entities (
100
- profile_id text not null,
101
- kind text not null,
102
- value text not null,
103
- source text not null,
104
- is_active integer not null default 1,
105
- first_seen_at text not null,
106
- last_seen_at text not null,
107
- raw_json text not null default '{}',
108
- primary key (profile_id, kind, value)
109
- );
110
-
111
- create table if not exists identity_search_index (
112
- profile_id text not null,
113
- kind text not null,
114
- value text not null,
115
- normalized_value text not null,
116
- source text not null,
117
- weight integer not null,
118
- updated_at text not null,
119
- primary key (profile_id, kind, value, source)
120
- );
121
-
122
- create table if not exists tweets (
123
- id text primary key,
124
- account_id text not null,
125
- author_profile_id text not null,
126
- kind text not null,
127
- text text not null,
128
- created_at text not null,
129
- is_replied integer not null default 0,
130
- reply_to_id text,
131
- like_count integer not null default 0,
132
- media_count integer not null default 0,
133
- bookmarked integer not null default 0,
134
- liked integer not null default 0,
135
- entities_json text not null default '{}',
136
- media_json text not null default '[]',
137
- quoted_tweet_id text
138
- );
139
-
140
- create table if not exists tweet_collections (
141
- account_id text not null,
142
- tweet_id text not null,
143
- kind text not null,
144
- collected_at text,
145
- source text not null,
146
- raw_json text not null default '{}',
147
- updated_at text not null,
148
- primary key (account_id, tweet_id, kind)
149
- );
150
-
151
- create table if not exists tweet_account_edges (
152
- account_id text not null,
153
- tweet_id text not null,
154
- kind text not null,
155
- first_seen_at text not null,
156
- last_seen_at text not null,
157
- seen_count integer not null default 1,
158
- source text not null,
159
- raw_json text not null default '{}',
160
- updated_at text not null,
161
- primary key (account_id, tweet_id, kind)
162
- );
163
-
164
- create table if not exists dm_conversations (
165
- id text primary key,
166
- account_id text not null,
167
- participant_profile_id text not null,
168
- title text not null,
169
- inbox_kind text not null default 'accepted',
170
- last_message_at text not null,
171
- unread_count integer not null default 0,
172
- needs_reply integer not null default 0
173
- );
174
-
175
- create table if not exists dm_messages (
176
- id text primary key,
177
- conversation_id text not null,
178
- sender_profile_id text not null,
179
- text text not null,
180
- created_at text not null,
181
- direction text not null,
182
- is_replied integer not null default 0,
183
- media_count integer not null default 0
184
- );
185
-
186
- create table if not exists tweet_actions (
187
- id text primary key,
188
- account_id text not null,
189
- tweet_id text,
190
- kind text not null,
191
- body text not null,
192
- created_at text not null
193
- );
194
-
195
- create table if not exists blocks (
196
- account_id text not null,
197
- profile_id text not null,
198
- source text not null,
199
- created_at text not null,
200
- primary key (account_id, profile_id)
201
- );
202
-
203
- create table if not exists mutes (
204
- account_id text not null,
205
- profile_id text not null,
206
- source text not null,
207
- created_at text not null,
208
- primary key (account_id, profile_id)
209
- );
210
-
211
- create table if not exists ai_scores (
212
- entity_kind text not null,
213
- entity_id text not null,
214
- model text not null,
215
- score integer not null,
216
- summary text not null,
217
- reasoning text not null,
218
- updated_at text not null,
219
- primary key (entity_kind, entity_id)
220
- );
221
-
222
- create table if not exists sync_cache (
223
- cache_key text primary key,
224
- value_json text not null,
225
- updated_at text not null
226
- );
227
-
228
- create table if not exists url_expansions (
229
- short_url text primary key,
230
- expanded_url text not null,
231
- final_url text not null,
232
- status text not null,
233
- expanded_tweet_id text,
234
- expanded_handle text,
235
- title text,
236
- description text,
237
- image_url text,
238
- site_name text,
239
- error text,
240
- source text not null,
241
- updated_at text not null
242
- );
243
-
244
- create table if not exists link_occurrences (
245
- source_kind text not null,
246
- source_id text not null,
247
- source_position integer not null,
248
- short_url text not null,
249
- account_id text,
250
- conversation_id text,
251
- direction text,
252
- created_at text not null,
253
- primary key (source_kind, source_id, source_position, short_url)
254
- );
255
-
256
- create table if not exists follow_snapshots (
257
- id text primary key,
258
- account_id text not null,
259
- direction text not null,
260
- source text not null,
261
- status text not null,
262
- page_count integer not null default 0,
263
- result_count integer not null default 0,
264
- started_at text not null,
265
- completed_at text not null,
266
- raw_meta_json text not null default '{}'
267
- );
268
-
269
- create table if not exists follow_snapshot_members (
270
- snapshot_id text not null,
271
- profile_id text not null,
272
- external_user_id text not null,
273
- position integer not null,
274
- primary key (snapshot_id, profile_id)
275
- );
276
-
277
- create table if not exists follow_edges (
278
- account_id text not null,
279
- direction text not null,
280
- profile_id text not null,
281
- external_user_id text not null,
282
- source text not null,
283
- current integer not null default 1,
284
- first_seen_at text not null,
285
- last_seen_at text not null,
286
- ended_at text,
287
- updated_at text not null,
288
- primary key (account_id, direction, profile_id)
289
- );
290
-
291
- create table if not exists follow_events (
292
- id text primary key,
293
- account_id text not null,
294
- direction text not null,
295
- profile_id text not null,
296
- external_user_id text not null,
297
- kind text not null,
298
- event_at text not null,
299
- snapshot_id text not null
300
- );
301
-
302
- create table if not exists geocoded_locations (
303
- normalized_key text primary key,
304
- original text not null,
305
- lat real not null,
306
- lng real not null,
307
- formatted text,
308
- country_code text,
309
- confidence integer,
310
- provider text not null,
311
- approx_radius_m real,
312
- bounds_json text not null default '{}',
313
- components_json text not null default '{}',
314
- hits integer not null default 1,
315
- created_at text not null,
316
- last_used_at text not null
317
- );
318
-
319
- create table if not exists geocoded_locations_unresolved (
320
- normalized_key text primary key,
321
- original text not null,
322
- reason text not null,
323
- last_attempted_at text not null,
324
- ttl_until text
325
- );
326
-
327
- create virtual table if not exists tweets_fts using fts5(
328
- tweet_id unindexed,
329
- text
330
- );
331
-
332
- create virtual table if not exists dm_fts using fts5(
333
- message_id unindexed,
334
- text
335
- );
336
- `;
337
-
338
- const INDEX_SQL = `
339
- create index if not exists idx_tweets_kind_created on tweets(kind, created_at desc);
340
- create index if not exists idx_tweets_created on tweets(created_at desc);
341
- create index if not exists idx_tweets_account_created on tweets(account_id, created_at desc);
342
- create index if not exists idx_tweets_quoted on tweets(quoted_tweet_id);
343
- create index if not exists idx_tweet_collections_kind_account on tweet_collections(kind, account_id, collected_at desc, tweet_id);
344
- create index if not exists idx_tweet_collections_tweet on tweet_collections(tweet_id);
345
- create index if not exists idx_tweet_account_edges_kind_account on tweet_account_edges(kind, account_id, last_seen_at desc, tweet_id);
346
- create index if not exists idx_tweet_account_edges_kind_tweet on tweet_account_edges(kind, tweet_id, account_id);
347
- create index if not exists idx_tweet_account_edges_tweet on tweet_account_edges(tweet_id);
348
- create index if not exists idx_dm_conversations_account on dm_conversations(account_id, last_message_at desc);
349
- create index if not exists idx_dm_messages_conversation on dm_messages(conversation_id, created_at asc);
350
- create index if not exists idx_profiles_followers on profiles(followers_count desc);
351
- create index if not exists idx_profiles_following on profiles(following_count desc);
352
- create index if not exists idx_profiles_handle on profiles(handle);
353
- create index if not exists idx_profile_affiliations_subject on profile_affiliations(subject_profile_id, is_active, last_seen_at desc);
354
- create index if not exists idx_profile_affiliations_org on profile_affiliations(organization_profile_id, is_active, last_seen_at desc);
355
- create index if not exists idx_profile_snapshots_profile on profile_snapshots(profile_id, last_seen_at desc);
356
- create index if not exists idx_profile_bio_entities_profile on profile_bio_entities(profile_id, is_active, last_seen_at desc);
357
- create index if not exists idx_profile_bio_entities_value on profile_bio_entities(kind, value, is_active);
358
- create index if not exists idx_identity_search_index_profile on identity_search_index(profile_id);
359
- create index if not exists idx_identity_search_index_value on identity_search_index(normalized_value, kind, weight desc);
360
- create index if not exists idx_blocks_account_created on blocks(account_id, created_at desc);
361
- create index if not exists idx_mutes_account_created on mutes(account_id, created_at desc);
362
- create index if not exists idx_ai_scores_updated on ai_scores(updated_at desc);
363
- create index if not exists idx_sync_cache_updated on sync_cache(updated_at desc);
364
- create index if not exists idx_url_expansions_expanded on url_expansions(expanded_url);
365
- create index if not exists idx_url_expansions_tweet on url_expansions(expanded_tweet_id);
366
- create index if not exists idx_url_expansions_handle on url_expansions(expanded_handle);
367
- create index if not exists idx_link_occurrences_url on link_occurrences(short_url);
368
- create index if not exists idx_link_occurrences_created on link_occurrences(created_at desc);
369
- create index if not exists idx_link_occurrences_account on link_occurrences(account_id, created_at desc);
370
- create index if not exists idx_link_occurrences_direction on link_occurrences(direction, created_at desc);
371
- create index if not exists idx_follow_edges_current on follow_edges(account_id, direction, current, last_seen_at desc);
372
- create index if not exists idx_follow_edges_profile on follow_edges(profile_id, current);
373
- create index if not exists idx_follow_snapshots_account on follow_snapshots(account_id, direction, completed_at desc);
374
- create index if not exists idx_follow_events_account on follow_events(account_id, direction, kind, event_at desc);
375
- create index if not exists idx_geocoded_locations_country on geocoded_locations(country_code);
376
- create index if not exists idx_geocoded_locations_last_used on geocoded_locations(last_used_at desc);
377
- create index if not exists idx_geocoded_unresolved_ttl on geocoded_locations_unresolved(ttl_until desc);
378
- `;
379
-
380
- function getColumnNames(db: Database, tableName: string): Set<string> {
381
- const rows = db.prepare(`pragma table_info(${tableName})`).all() as Array<{
382
- name: string;
383
- }>;
384
- return new Set(rows.map((row) => row.name));
385
- }
386
-
387
- function ensureTweetMetadataColumns(db: Database) {
388
- const columnNames = getColumnNames(db, "tweets");
389
- if (!columnNames.has("entities_json")) {
390
- db.exec(
391
- "alter table tweets add column entities_json text not null default '{}'",
392
- );
393
- }
394
- if (!columnNames.has("media_json")) {
395
- db.exec(
396
- "alter table tweets add column media_json text not null default '[]'",
397
- );
398
- }
399
- if (!columnNames.has("quoted_tweet_id")) {
400
- db.exec("alter table tweets add column quoted_tweet_id text");
401
- }
402
- }
403
-
404
- function ensureProfileAvatarColumns(db: Database) {
405
- const columnNames = getColumnNames(db, "profiles");
406
- if (!columnNames.has("following_count")) {
407
- db.exec(
408
- "alter table profiles add column following_count integer not null default 0",
409
- );
410
- }
411
- if (!columnNames.has("avatar_url")) {
412
- db.exec("alter table profiles add column avatar_url text");
413
- }
414
- if (!columnNames.has("location")) {
415
- db.exec("alter table profiles add column location text");
416
- }
417
- if (!columnNames.has("url")) {
418
- db.exec("alter table profiles add column url text");
419
- }
420
- if (!columnNames.has("verified_type")) {
421
- db.exec("alter table profiles add column verified_type text");
422
- }
423
- if (!columnNames.has("entities_json")) {
424
- db.exec(
425
- "alter table profiles add column entities_json text not null default '{}'",
426
- );
427
- }
428
- if (!columnNames.has("raw_json")) {
429
- db.exec(
430
- "alter table profiles add column raw_json text not null default '{}'",
431
- );
432
- }
433
- if (!columnNames.has("public_metrics_json")) {
434
- db.exec(
435
- "alter table profiles add column public_metrics_json text not null default '{}'",
436
- );
437
- }
438
- }
439
-
440
- function ensureAccountExternalUserIdColumn(db: Database) {
441
- const columnNames = getColumnNames(db, "accounts");
442
- if (!columnNames.has("external_user_id")) {
443
- db.exec("alter table accounts add column external_user_id text");
444
- }
445
- }
446
-
447
- function ensureDmConversationInboxColumns(db: Database) {
448
- const columnNames = getColumnNames(db, "dm_conversations");
449
- if (!columnNames.has("inbox_kind")) {
450
- db.exec(
451
- "alter table dm_conversations add column inbox_kind text not null default 'accepted'",
452
- );
453
- }
454
- }
455
-
456
- function ensureTweetCollectionsTable(db: Database) {
457
- db.exec(`
458
- create table if not exists tweet_collections (
459
- account_id text not null,
460
- tweet_id text not null,
461
- kind text not null,
462
- collected_at text,
463
- source text not null,
464
- raw_json text not null default '{}',
465
- updated_at text not null,
466
- primary key (account_id, tweet_id, kind)
467
- );
468
- `);
469
- }
470
-
471
- function ensureTweetAccountEdgesTable(db: Database) {
472
- db.exec(`
473
- create table if not exists tweet_account_edges (
474
- account_id text not null,
475
- tweet_id text not null,
476
- kind text not null,
477
- first_seen_at text not null,
478
- last_seen_at text not null,
479
- seen_count integer not null default 1,
480
- source text not null,
481
- raw_json text not null default '{}',
482
- updated_at text not null,
483
- primary key (account_id, tweet_id, kind)
484
- );
485
- `);
486
- }
487
-
488
- function ensureProfileAffiliationsTable(db: Database) {
489
- db.exec(`
490
- create table if not exists profile_affiliations (
491
- subject_profile_id text not null,
492
- organization_profile_id text not null,
493
- organization_name text,
494
- organization_handle text,
495
- badge_url text,
496
- url text,
497
- label text,
498
- source text not null,
499
- is_active integer not null default 1,
500
- first_seen_at text not null,
501
- last_seen_at text not null,
502
- raw_json text not null default '{}',
503
- updated_at text not null,
504
- primary key (subject_profile_id, organization_profile_id)
505
- );
506
- `);
507
- }
508
-
509
- function ensureProfileSnapshotsTable(db: Database) {
510
- db.exec(`
511
- create table if not exists profile_snapshots (
512
- profile_id text not null,
513
- snapshot_hash text not null,
514
- observed_at text not null,
515
- last_seen_at text not null,
516
- source text not null,
517
- handle text not null,
518
- display_name text not null,
519
- bio text not null,
520
- location text,
521
- url text,
522
- verified_type text,
523
- followers_count integer not null default 0,
524
- following_count integer not null default 0,
525
- affiliations_json text not null default '[]',
526
- raw_json text not null default '{}',
527
- primary key (profile_id, snapshot_hash)
528
- );
529
- `);
530
- }
531
-
532
- function ensureProfileBioEntitiesTable(db: Database) {
533
- db.exec(`
534
- create table if not exists profile_bio_entities (
535
- profile_id text not null,
536
- kind text not null,
537
- value text not null,
538
- source text not null,
539
- is_active integer not null default 1,
540
- first_seen_at text not null,
541
- last_seen_at text not null,
542
- raw_json text not null default '{}',
543
- primary key (profile_id, kind, value)
544
- );
545
- `);
546
- }
547
-
548
- function ensureIdentitySearchIndexTable(db: Database) {
549
- db.exec(`
550
- create table if not exists identity_search_index (
551
- profile_id text not null,
552
- kind text not null,
553
- value text not null,
554
- normalized_value text not null,
555
- source text not null,
556
- weight integer not null,
557
- updated_at text not null,
558
- primary key (profile_id, kind, value, source)
559
- );
560
- `);
561
- }
562
-
563
- function ensureLinkIndexTables(db: Database) {
564
- db.exec(`
565
- create table if not exists url_expansions (
566
- short_url text primary key,
567
- expanded_url text not null,
568
- final_url text not null,
569
- status text not null,
570
- expanded_tweet_id text,
571
- expanded_handle text,
572
- title text,
573
- description text,
574
- image_url text,
575
- site_name text,
576
- error text,
577
- source text not null,
578
- updated_at text not null
579
- );
580
-
581
- create table if not exists link_occurrences (
582
- source_kind text not null,
583
- source_id text not null,
584
- source_position integer not null,
585
- short_url text not null,
586
- account_id text,
587
- conversation_id text,
588
- direction text,
589
- created_at text not null,
590
- primary key (source_kind, source_id, source_position, short_url)
591
- );
592
- `);
593
-
594
- const urlExpansionColumns = getColumnNames(db, "url_expansions");
595
- if (!urlExpansionColumns.has("image_url")) {
596
- db.exec("alter table url_expansions add column image_url text");
597
- }
598
- if (!urlExpansionColumns.has("site_name")) {
599
- db.exec("alter table url_expansions add column site_name text");
600
- }
601
- }
602
-
603
- function ensureFollowGraphTables(db: Database) {
604
- db.exec(`
605
- create table if not exists follow_snapshots (
606
- id text primary key,
607
- account_id text not null,
608
- direction text not null,
609
- source text not null,
610
- status text not null,
611
- page_count integer not null default 0,
612
- result_count integer not null default 0,
613
- started_at text not null,
614
- completed_at text not null,
615
- raw_meta_json text not null default '{}'
616
- );
617
-
618
- create table if not exists follow_snapshot_members (
619
- snapshot_id text not null,
620
- profile_id text not null,
621
- external_user_id text not null,
622
- position integer not null,
623
- primary key (snapshot_id, profile_id)
624
- );
625
-
626
- create table if not exists follow_edges (
627
- account_id text not null,
628
- direction text not null,
629
- profile_id text not null,
630
- external_user_id text not null,
631
- source text not null,
632
- current integer not null default 1,
633
- first_seen_at text not null,
634
- last_seen_at text not null,
635
- ended_at text,
636
- updated_at text not null,
637
- primary key (account_id, direction, profile_id)
638
- );
639
-
640
- create table if not exists follow_events (
641
- id text primary key,
642
- account_id text not null,
643
- direction text not null,
644
- profile_id text not null,
645
- external_user_id text not null,
646
- kind text not null,
647
- event_at text not null,
648
- snapshot_id text not null
649
- );
650
- `);
651
- }
652
-
653
- function backfillTweetCollections(db: Database) {
654
- const missingKinds = (
655
- [
656
- ["likes", "liked"],
657
- ["bookmarks", "bookmarked"],
658
- ] as const
659
- ).filter(([, column]) =>
660
- db
661
- .prepare(
662
- `
663
- select 1
664
- from tweets as tweet
665
- where tweet.${column} = 1
666
- and not exists (
667
- select 1
668
- from tweet_collections as collection
669
- where collection.account_id = tweet.account_id
670
- and collection.tweet_id = tweet.id
671
- and collection.kind = ?
672
- )
673
- limit 1
674
- `,
675
- )
676
- .get(column === "liked" ? "likes" : "bookmarks"),
677
- );
678
- if (missingKinds.length === 0) {
679
- return;
680
- }
681
-
682
- const now = new Date().toISOString();
683
- const insert = db.prepare(`
684
- insert or ignore into tweet_collections (
685
- account_id, tweet_id, kind, collected_at, source, raw_json, updated_at
686
- )
687
- select account_id, id, ?, null, 'legacy', '{}', ?
688
- from tweets
689
- where
690
- case
691
- when ? = 'likes' then liked
692
- else bookmarked
693
- end = 1
694
- `);
695
-
696
- db.transaction(() => {
697
- for (const [kind] of missingKinds) {
698
- insert.run(kind, now, kind);
699
- }
700
- })();
701
- }
702
-
703
- function backfillTweetAccountEdges(db: Database) {
704
- const missing = db
705
- .prepare(
706
- `
707
- select 1
708
- from tweets as tweet
709
- where tweet.kind in ('home', 'mention')
710
- and not exists (
711
- select 1
712
- from tweet_account_edges as edge
713
- where edge.account_id = tweet.account_id
714
- and edge.tweet_id = tweet.id
715
- and edge.kind = tweet.kind
716
- )
717
- limit 1
718
- `,
719
- )
720
- .get();
721
- if (!missing) {
722
- return;
723
- }
724
-
725
- const now = new Date().toISOString();
726
- db.prepare(`
727
- insert or ignore into tweet_account_edges (
728
- account_id, tweet_id, kind, first_seen_at, last_seen_at, seen_count,
729
- source, raw_json, updated_at
730
- )
731
- select
732
- account_id,
733
- id,
734
- kind,
735
- created_at,
736
- created_at,
737
- 1,
738
- 'legacy',
739
- '{}',
740
- ?
741
- from tweets
742
- where kind in ('home', 'mention')
743
- `).run(now);
744
- }
745
-
746
- function ensureSchemaIndexes(db: Database) {
747
- db.exec(INDEX_SQL);
748
- }
749
-
750
- const DATABASE_MIGRATIONS: readonly DatabaseMigration[] = [
751
- {
752
- version: 1,
753
- name: "canonical schema baseline",
754
- up: (db) => {
755
- db.exec(BASE_SCHEMA_SQL);
756
- ensureAccountExternalUserIdColumn(db);
757
- ensureDmConversationInboxColumns(db);
758
- ensureTweetMetadataColumns(db);
759
- ensureProfileAvatarColumns(db);
760
- ensureTweetCollectionsTable(db);
761
- ensureTweetAccountEdgesTable(db);
762
- ensureProfileAffiliationsTable(db);
763
- ensureProfileSnapshotsTable(db);
764
- ensureProfileBioEntitiesTable(db);
765
- ensureIdentitySearchIndexTable(db);
766
- ensureLinkIndexTables(db);
767
- ensureFollowGraphTables(db);
768
- ensureSchemaIndexes(db);
769
- backfillTweetCollections(db);
770
- backfillTweetAccountEdges(db);
771
- },
772
- },
773
- ];
774
-
775
- function ensureDemoData(db: Database) {
776
- if (demoSeedAttempted) {
777
- return;
778
- }
779
-
780
- seedDemoData(db);
781
- backfillTweetCollections(db);
782
- backfillTweetAccountEdges(db);
783
- demoSeedAttempted = true;
784
- }
785
-
786
- function initDatabase(options: InitDatabaseOptions = {}) {
787
- ensureBirdclawDirs();
788
-
789
- if (!nativeDb) {
790
- const { dbPath } = getBirdclawPaths();
791
- nativeDb = createDatabaseConnection(dbPath, "writer");
792
- nativeDb.exec(`
793
- pragma busy_timeout = ${SQLITE_BUSY_TIMEOUT_MS};
794
- pragma foreign_keys = on;
795
- `);
796
- runDatabaseMigrations(nativeDb, DATABASE_MIGRATIONS);
797
- if (options.seedDemoData !== false) {
798
- ensureDemoData(nativeDb);
799
- }
800
- } else if (options.seedDemoData !== false) {
801
- ensureDemoData(nativeDb);
802
- }
803
-
804
- if (!kyselyDb) {
805
- kyselyDb = new Kysely<BirdclawDatabase>({
806
- dialect: new SqliteDialect({
807
- database: nativeDb,
808
- }),
809
- });
810
- }
811
- }
812
-
813
- function createDatabaseConnection(
814
- dbPath: string,
815
- role: DatabaseConnectionRole,
816
- options: { readonly?: boolean } = {},
817
- ) {
818
- return new NativeSqliteDatabase(dbPath, {
819
- ...options,
820
- onStatement: (sql, durationMs) =>
821
- recordDatabaseStatement(role, sql, durationMs),
822
- });
823
- }
824
-
825
- export function getNativeDb(options: InitDatabaseOptions = {}) {
826
- initDatabase(options);
827
- return nativeDb as Database;
828
- }
829
-
830
- export function getReadDb(options: InitDatabaseOptions = {}) {
831
- initDatabase(options);
832
- if (readDbs.length === 0) {
833
- const { dbPath } = getBirdclawPaths();
834
- readDbs = Array.from({ length: 2 }, () => {
835
- const db = createDatabaseConnection(dbPath, "reader", {
836
- readonly: true,
837
- });
838
- db.exec(`
839
- pragma busy_timeout = ${SQLITE_BUSY_TIMEOUT_MS};
840
- pragma foreign_keys = on;
841
- pragma query_only = on;
842
- `);
843
- return db;
844
- });
845
- }
846
- const db = readDbs[readDbIndex % readDbs.length] as Database;
847
- readDbIndex = (readDbIndex + 1) % readDbs.length;
848
- return db;
849
- }
850
-
851
- export function getDb() {
852
- initDatabase();
853
- return kyselyDb as Kysely<BirdclawDatabase>;
854
- }
855
-
856
- export async function closeDatabase() {
857
- const db = kyselyDb;
858
- const native = nativeDb;
859
- const readers = readDbs;
860
- kyselyDb = undefined;
861
- nativeDb = undefined;
862
- readDbs = [];
863
- readDbIndex = 0;
864
- demoSeedAttempted = false;
865
-
866
- for (const reader of readers) reader.close();
867
- if (db) {
868
- await db.destroy();
869
- } else {
870
- native?.close();
871
- }
872
- }
873
-
874
- export function resetDatabaseForTests() {
875
- const db = kyselyDb;
876
- const native = nativeDb;
877
- const readers = readDbs;
878
- kyselyDb = undefined;
879
- nativeDb = undefined;
880
- readDbs = [];
881
- readDbIndex = 0;
882
- demoSeedAttempted = false;
883
- for (const reader of readers) reader.close();
884
- if (db) {
885
- void db.destroy();
886
- } else {
887
- native?.close();
888
- }
889
- }