birdclaw 0.8.3 → 0.8.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +10 -8
- package/bin/birdclaw.mjs +4 -66
- package/dist/cli/birdclaw.js +28165 -0
- package/dist/client/assets/AvatarChip-DLoqM5xD.js +1 -0
- package/dist/client/assets/ProfilePreview-BejAuYsJ.js +1 -0
- package/dist/client/assets/SavedTimelineView-BIS7HKpp.js +1 -0
- package/dist/client/assets/SmartTimestamp-BR0-9se2.js +1 -0
- package/dist/client/assets/TimelineRouteFrame-CyGPj0eM.js +1 -0
- package/dist/client/assets/blocks-7UDKQp8H.js +1 -0
- package/dist/client/assets/bookmarks-CbBkMvXk.js +1 -0
- package/dist/client/assets/chunk-C0phanGV.js +1 -0
- package/dist/client/assets/circle-check-C_Vq3ElD.js +1 -0
- package/dist/client/assets/client-stream-contracts-ys3XhWG4.js +5 -0
- package/dist/client/assets/createLucideIcon-JV8isFFR.js +1 -0
- package/dist/client/assets/data-sources-Cm2bpNvg.js +1 -0
- package/dist/client/assets/discuss-DTjaOugw.js +1 -0
- package/dist/client/assets/dms-CGd6Eogq.js +1 -0
- package/dist/client/assets/inbox-BHlGuOy2.js +1 -0
- package/dist/client/assets/index-CEIG7S6s.js +31 -0
- package/dist/client/assets/index-bpFrQ5yG.css +1 -0
- package/dist/client/assets/likes-cLrcAQem.js +1 -0
- package/dist/client/assets/links-CM9QO1LC.js +1 -0
- package/dist/client/assets/mapbox-BxTlqXt9.js +2 -0
- package/dist/client/assets/mapbox-gl-dgf_hX3a.js +3215 -0
- package/dist/client/assets/mentions-CxKCgzIf.js +1 -0
- package/dist/client/assets/network-map-B5a3whqj.js +2 -0
- package/dist/client/assets/profile-analyze-CRtNEwlb.js +1 -0
- package/dist/client/assets/profile-hydration-client-YEtJsNTv.js +1 -0
- package/dist/client/assets/profiles._handle-C82_HoKX.js +1 -0
- package/dist/client/assets/query-client-fYu2rzRC.js +17 -0
- package/dist/client/assets/rate-limits-ScaEwdVK.js +1 -0
- package/dist/client/assets/react-dom-DHl6Ug7F.js +1 -0
- package/dist/client/assets/react-nBylq1U5.js +1 -0
- package/dist/client/assets/repeat-2-xCwHM9hD.js +1 -0
- package/dist/client/assets/routes-q7ZaW4En.js +1 -0
- package/dist/client/assets/schemas-I9r6F6ya.js +64 -0
- package/dist/client/assets/search-Dq5kYsxA.js +1 -0
- package/dist/client/assets/sparkles-CQVlvHsZ.js +1 -0
- package/dist/client/assets/styles-B8s3tt3r.css +2 -0
- package/dist/client/assets/today-CeG1O7Py.js +1 -0
- package/dist/client/assets/triangle-alert-CBL7s2pf.js +1 -0
- package/dist/client/assets/useTimelineRouteData-1HsqgvUi.js +1 -0
- package/dist/client/assets/users-CHbKlqUr.js +1 -0
- package/dist/server/assets/AvatarChip-zUDAEmjL.js +105 -0
- package/dist/server/assets/FeedState-ntW0Q2GB.js +214 -0
- package/dist/server/assets/ProfileAnalysisStream-BFvhVdhn.js +198 -0
- package/dist/server/assets/ProfilePreview-C6gAor_j.js +752 -0
- package/dist/server/assets/SavedTimelineView-BG1jnBp1.js +64 -0
- package/dist/server/assets/SmartTimestamp-DfwE1NGf.js +45 -0
- package/dist/server/assets/SyncNowButton-BOn_IBXS.js +76 -0
- package/dist/server/assets/TimelineRouteFrame-CU7ilM7U.js +85 -0
- package/dist/server/assets/TweetRichText-DhCVv1-6.js +74 -0
- package/dist/server/assets/_tanstack-start-manifest_v-C-ZRnG_n.js +148 -0
- package/dist/server/assets/account-selection-CTdFGWRk.js +44 -0
- package/dist/server/assets/blocks-C1mgGTB3.js +299 -0
- package/dist/server/assets/blocks-DSgQghqP.js +17 -0
- package/dist/server/assets/bookmarks-BJN8tykY.js +14 -0
- package/dist/server/assets/client-stream-contracts-C2xbWVjm.js +769 -0
- package/dist/server/assets/data-sources-BnA1Q94p.js +180 -0
- package/dist/server/assets/discuss-CJ1dau1j.js +15 -0
- package/dist/server/assets/discuss-QTo08NCN.js +338 -0
- package/dist/server/assets/dms-BuLj4bp8.js +368 -0
- package/dist/server/assets/dms-DAyQrVM2.js +240 -0
- package/dist/server/assets/empty-plugin-adapters-DzrEkdBF.js +5 -0
- package/dist/server/assets/inbox-CkGOaW1q.js +214 -0
- package/dist/server/assets/inbox-DpWd1EzC.js +146 -0
- package/dist/server/assets/likes-DBOAtsJU.js +14 -0
- package/dist/server/assets/links-DAhSjynu.js +240 -0
- package/dist/server/assets/links-DmH5lxHO.js +468 -0
- package/dist/server/assets/mentions-BH2MzKtB.js +26 -0
- package/dist/server/assets/network-map-D3CzRcC4.js +224 -0
- package/dist/server/assets/network-map-D5DoLOwR.js +582 -0
- package/dist/server/assets/profile-analyze-BP65PED0.js +90 -0
- package/dist/server/assets/profile-analyze-C1NoM0tV.js +9 -0
- package/dist/server/assets/profile-hydration-client-GJ4kwJS8.js +133 -0
- package/dist/server/assets/profiles._handle-DjwlN_ac.js +184 -0
- package/dist/server/assets/profiles._handle-DsLNj2Ca.js +12 -0
- package/dist/server/assets/query-client-BdauHBgd.js +923 -0
- package/dist/server/assets/rate-limits-VAr2qzYD.js +262 -0
- package/dist/server/assets/route-search-BUlQ5lGM.js +117 -0
- package/dist/server/assets/router-DqkwSPiM.js +17026 -0
- package/dist/server/assets/routes-A9c9ZhwN.js +26 -0
- package/dist/server/assets/start-FaDOTmto.js +4 -0
- package/dist/server/assets/today-B5hVM57Z.js +17 -0
- package/dist/server/assets/today-fvGiZ7ud.js +303 -0
- package/dist/server/assets/ui-qbmcMSSX.js +137 -0
- package/{src/components/useDebouncedValue.ts → dist/server/assets/useDebouncedValue-DOnA6Xcy.js} +4 -4
- package/dist/server/assets/useTimelineRouteData-27cxynxa.js +1101 -0
- package/dist/server/server.js +5520 -0
- package/package.json +26 -34
- package/playwright.config.ts +0 -35
- package/scripts/browser-perf.mjs +0 -427
- package/scripts/build-docs-site.mjs +0 -967
- package/scripts/docs-site-assets.mjs +0 -311
- package/scripts/run-vitest.mjs +0 -21
- package/scripts/sanitize-node-options.mjs +0 -23
- package/scripts/start-test-server.mjs +0 -42
- package/src/cli/command-context.ts +0 -17
- package/src/cli/register-analysis.ts +0 -500
- package/src/cli/register-compose.ts +0 -40
- package/src/cli/register-graph.ts +0 -132
- package/src/cli/register-inbox.ts +0 -41
- package/src/cli/register-storage.ts +0 -106
- package/src/cli-moderation.ts +0 -210
- package/src/cli.ts +0 -1616
- package/src/components/AccountSwitcher.tsx +0 -178
- package/src/components/AppNav.tsx +0 -118
- package/src/components/AvatarChip.tsx +0 -58
- package/src/components/AvatarPreload.ts +0 -149
- package/src/components/BrandMark.tsx +0 -67
- package/src/components/ConversationThread.tsx +0 -131
- package/src/components/DmWorkspace.tsx +0 -286
- package/src/components/EmbeddedTweetCard.tsx +0 -55
- package/src/components/FeedState.tsx +0 -147
- package/src/components/FloatingPreview.tsx +0 -382
- package/src/components/InboxCard.tsx +0 -152
- package/src/components/LinkPreviewCard.tsx +0 -292
- package/src/components/MarkdownCitations.tsx +0 -680
- package/src/components/MarkdownViewer.tsx +0 -111
- package/src/components/ProfileAnalysisClient.ts +0 -191
- package/src/components/ProfileAnalysisStream.tsx +0 -259
- package/src/components/ProfilePreview.tsx +0 -131
- package/src/components/SavedTimelineView.tsx +0 -155
- package/src/components/SmartTimestamp.tsx +0 -72
- package/src/components/SyncNowButton.tsx +0 -140
- package/src/components/ThemeSlider.tsx +0 -80
- package/src/components/TimelineCard.tsx +0 -533
- package/src/components/TimelineRouteFrame.tsx +0 -186
- package/src/components/TweetArticleCard.tsx +0 -66
- package/src/components/TweetMediaGrid.tsx +0 -185
- package/src/components/TweetRichText.tsx +0 -159
- package/src/components/account-selection.ts +0 -64
- package/src/components/links-controller.ts +0 -162
- package/src/components/links-model.ts +0 -198
- package/src/components/network-map-controller.ts +0 -84
- package/src/components/network-map-model.ts +0 -255
- package/src/components/useTimelineRouteData.ts +0 -193
- package/src/lib/account-sync-job.ts +0 -666
- package/src/lib/actions-transport.ts +0 -300
- package/src/lib/analysis-runtime.ts +0 -238
- package/src/lib/api-client.ts +0 -220
- package/src/lib/api-contracts.ts +0 -328
- package/src/lib/archive-finder.ts +0 -185
- package/src/lib/archive-import-plan.ts +0 -102
- package/src/lib/archive-import.ts +0 -2695
- package/src/lib/authored-live.ts +0 -1027
- package/src/lib/avatar-cache.ts +0 -349
- package/src/lib/backup.ts +0 -2652
- package/src/lib/bird-actions.ts +0 -294
- package/src/lib/bird-command.ts +0 -130
- package/src/lib/bird.ts +0 -1140
- package/src/lib/blocklist.ts +0 -136
- package/src/lib/blocks-write.ts +0 -171
- package/src/lib/blocks.ts +0 -387
- package/src/lib/bookmark-sync-job.ts +0 -536
- package/src/lib/config.ts +0 -224
- package/src/lib/conversation-surface.ts +0 -205
- package/src/lib/data-sources.ts +0 -219
- package/src/lib/database-metrics.ts +0 -88
- package/src/lib/database-migrations.ts +0 -34
- package/src/lib/database-schema.ts +0 -312
- package/src/lib/database-writer.ts +0 -69
- package/src/lib/db.ts +0 -889
- package/src/lib/dm-read-model.ts +0 -533
- package/src/lib/dms-live.ts +0 -870
- package/src/lib/effect-runtime.ts +0 -45
- package/src/lib/follow-graph.ts +0 -1087
- package/src/lib/geocoding.ts +0 -296
- package/src/lib/http-effect.ts +0 -222
- package/src/lib/identity-search-index.ts +0 -369
- package/src/lib/import-repository.ts +0 -138
- package/src/lib/inbox.ts +0 -242
- package/src/lib/link-index.ts +0 -641
- package/src/lib/link-insights.ts +0 -860
- package/src/lib/link-preview-metadata.ts +0 -754
- package/src/lib/live-sync-engine.ts +0 -209
- package/src/lib/live-transport-gateway.ts +0 -128
- package/src/lib/location.ts +0 -137
- package/src/lib/media-fetch.ts +0 -860
- package/src/lib/media-includes.ts +0 -165
- package/src/lib/mention-threads-live.ts +0 -830
- package/src/lib/mentions-export.ts +0 -147
- package/src/lib/mentions-live.ts +0 -1114
- package/src/lib/moderation-target.ts +0 -227
- package/src/lib/moderation-write.ts +0 -131
- package/src/lib/mutes-write.ts +0 -171
- package/src/lib/mutes.ts +0 -84
- package/src/lib/ndjson-stream.ts +0 -106
- package/src/lib/network-map.ts +0 -382
- package/src/lib/openai-response-runtime.ts +0 -251
- package/src/lib/openai.ts +0 -117
- package/src/lib/paginated-sync.ts +0 -93
- package/src/lib/period-digest.ts +0 -1364
- package/src/lib/present.ts +0 -118
- package/src/lib/profile-affiliation-hydration.ts +0 -228
- package/src/lib/profile-affiliations.ts +0 -262
- package/src/lib/profile-analysis.ts +0 -1198
- package/src/lib/profile-bio-entities.ts +0 -318
- package/src/lib/profile-history.ts +0 -164
- package/src/lib/profile-hydration.ts +0 -180
- package/src/lib/profile-replies.ts +0 -77
- package/src/lib/profile-resolver.ts +0 -650
- package/src/lib/queries.ts +0 -6
- package/src/lib/query-actions.ts +0 -437
- package/src/lib/query-client.tsx +0 -47
- package/src/lib/query-read-model-shared.ts +0 -52
- package/src/lib/query-read-models.ts +0 -5
- package/src/lib/query-resource.ts +0 -41
- package/src/lib/query-status.ts +0 -164
- package/src/lib/research.ts +0 -652
- package/src/lib/runtime-services.ts +0 -20
- package/src/lib/search-discussion.ts +0 -812
- package/src/lib/seed.ts +0 -613
- package/src/lib/server-runtime-services.ts +0 -30
- package/src/lib/sqlite.ts +0 -183
- package/src/lib/streaming-ingestion.ts +0 -240
- package/src/lib/sync-cache.ts +0 -70
- package/src/lib/sync-plan.ts +0 -175
- package/src/lib/theme-transition.ts +0 -113
- package/src/lib/theme.tsx +0 -165
- package/src/lib/timeline-collections-live.ts +0 -407
- package/src/lib/timeline-live.ts +0 -302
- package/src/lib/timeline-read-model.ts +0 -1191
- package/src/lib/tweet-account-edges.ts +0 -48
- package/src/lib/tweet-lookup.ts +0 -46
- package/src/lib/tweet-render.ts +0 -430
- package/src/lib/tweet-repository.ts +0 -156
- package/src/lib/tweet-search-live.ts +0 -462
- package/src/lib/types.ts +0 -723
- package/src/lib/ui.ts +0 -529
- package/src/lib/url-expansion-store.ts +0 -110
- package/src/lib/url-expansion.ts +0 -343
- package/src/lib/url-safety.ts +0 -220
- package/src/lib/web-sync.ts +0 -533
- package/src/lib/whois.ts +0 -1080
- package/src/lib/x-profile.ts +0 -365
- package/src/lib/x-web.ts +0 -199
- package/src/lib/xurl-rate-limits.ts +0 -267
- package/src/lib/xurl.ts +0 -1677
- package/src/routeTree.gen.ts +0 -723
- package/src/router.tsx +0 -20
- package/src/routes/__root.tsx +0 -82
- package/src/routes/api/action.tsx +0 -147
- package/src/routes/api/avatar.tsx +0 -50
- package/src/routes/api/blocks.tsx +0 -35
- package/src/routes/api/conversation.tsx +0 -45
- package/src/routes/api/data-sources.tsx +0 -24
- package/src/routes/api/inbox.tsx +0 -43
- package/src/routes/api/link-insights.tsx +0 -85
- package/src/routes/api/link-preview.tsx +0 -47
- package/src/routes/api/network-map.tsx +0 -55
- package/src/routes/api/period-digest.tsx +0 -101
- package/src/routes/api/profile-analysis.tsx +0 -97
- package/src/routes/api/profile-hydrate.tsx +0 -54
- package/src/routes/api/query.tsx +0 -121
- package/src/routes/api/search-discussion.tsx +0 -113
- package/src/routes/api/status.tsx +0 -30
- package/src/routes/api/sync.tsx +0 -108
- package/src/routes/api/xurl-rate-limits.tsx +0 -24
- package/src/routes/blocks.tsx +0 -343
- package/src/routes/bookmarks.tsx +0 -18
- package/src/routes/data-sources.tsx +0 -249
- package/src/routes/discuss.tsx +0 -419
- package/src/routes/dms.tsx +0 -415
- package/src/routes/inbox.tsx +0 -232
- package/src/routes/index.tsx +0 -32
- package/src/routes/likes.tsx +0 -18
- package/src/routes/links.tsx +0 -631
- package/src/routes/mentions.tsx +0 -32
- package/src/routes/network-map.tsx +0 -732
- package/src/routes/profile-analyze.tsx +0 -112
- package/src/routes/profiles.$handle.tsx +0 -228
- package/src/routes/rate-limits.tsx +0 -305
- package/src/routes/today.tsx +0 -511
- package/src/styles.css +0 -257
- package/tsconfig.json +0 -29
- package/vite.config.ts +0 -29
- /package/{public → dist/client}/birdclaw-mark.png +0 -0
- /package/{public → dist/client}/favicon.ico +0 -0
- /package/{public → dist/client}/logo192.png +0 -0
- /package/{public → dist/client}/logo512.png +0 -0
- /package/{public → dist/client}/manifest.json +0 -0
- /package/{public → dist/client}/robots.txt +0 -0
package/src/lib/db.ts
DELETED
|
@@ -1,889 +0,0 @@
|
|
|
1
|
-
import NativeSqliteDatabase, {
|
|
2
|
-
type Database,
|
|
3
|
-
SQLITE_BUSY_TIMEOUT_MS,
|
|
4
|
-
} from "./sqlite";
|
|
5
|
-
import { Kysely, SqliteDialect } from "kysely";
|
|
6
|
-
import { ensureBirdclawDirs, getBirdclawPaths } from "./config";
|
|
7
|
-
import {
|
|
8
|
-
type DatabaseMigration,
|
|
9
|
-
runDatabaseMigrations,
|
|
10
|
-
} from "./database-migrations";
|
|
11
|
-
import { seedDemoData } from "./seed";
|
|
12
|
-
import {
|
|
13
|
-
type DatabaseConnectionRole,
|
|
14
|
-
recordDatabaseStatement,
|
|
15
|
-
} from "./database-metrics";
|
|
16
|
-
|
|
17
|
-
import type { BirdclawDatabase } from "./database-schema";
|
|
18
|
-
export * from "./database-schema";
|
|
19
|
-
|
|
20
|
-
let nativeDb: Database | undefined;
|
|
21
|
-
let readDbs: Database[] = [];
|
|
22
|
-
let readDbIndex = 0;
|
|
23
|
-
let kyselyDb: Kysely<BirdclawDatabase> | undefined;
|
|
24
|
-
let demoSeedAttempted = false;
|
|
25
|
-
|
|
26
|
-
export interface InitDatabaseOptions {
|
|
27
|
-
seedDemoData?: boolean;
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
const BASE_SCHEMA_SQL = `
|
|
31
|
-
pragma journal_mode = wal;
|
|
32
|
-
pragma busy_timeout = ${SQLITE_BUSY_TIMEOUT_MS};
|
|
33
|
-
pragma foreign_keys = on;
|
|
34
|
-
|
|
35
|
-
create table if not exists accounts (
|
|
36
|
-
id text primary key,
|
|
37
|
-
name text not null,
|
|
38
|
-
handle text not null unique,
|
|
39
|
-
external_user_id text,
|
|
40
|
-
transport text not null,
|
|
41
|
-
is_default integer not null default 0,
|
|
42
|
-
created_at text not null
|
|
43
|
-
);
|
|
44
|
-
|
|
45
|
-
create table if not exists profiles (
|
|
46
|
-
id text primary key,
|
|
47
|
-
handle text not null unique,
|
|
48
|
-
display_name text not null,
|
|
49
|
-
bio text not null,
|
|
50
|
-
followers_count integer not null default 0,
|
|
51
|
-
following_count integer not null default 0,
|
|
52
|
-
public_metrics_json text not null default '{}',
|
|
53
|
-
avatar_hue integer not null default 0,
|
|
54
|
-
avatar_url text,
|
|
55
|
-
location text,
|
|
56
|
-
url text,
|
|
57
|
-
verified_type text,
|
|
58
|
-
entities_json text not null default '{}',
|
|
59
|
-
raw_json text not null default '{}',
|
|
60
|
-
created_at text not null
|
|
61
|
-
);
|
|
62
|
-
|
|
63
|
-
create table if not exists profile_affiliations (
|
|
64
|
-
subject_profile_id text not null,
|
|
65
|
-
organization_profile_id text not null,
|
|
66
|
-
organization_name text,
|
|
67
|
-
organization_handle text,
|
|
68
|
-
badge_url text,
|
|
69
|
-
url text,
|
|
70
|
-
label text,
|
|
71
|
-
source text not null,
|
|
72
|
-
is_active integer not null default 1,
|
|
73
|
-
first_seen_at text not null,
|
|
74
|
-
last_seen_at text not null,
|
|
75
|
-
raw_json text not null default '{}',
|
|
76
|
-
updated_at text not null,
|
|
77
|
-
primary key (subject_profile_id, organization_profile_id)
|
|
78
|
-
);
|
|
79
|
-
|
|
80
|
-
create table if not exists profile_snapshots (
|
|
81
|
-
profile_id text not null,
|
|
82
|
-
snapshot_hash text not null,
|
|
83
|
-
observed_at text not null,
|
|
84
|
-
last_seen_at text not null,
|
|
85
|
-
source text not null,
|
|
86
|
-
handle text not null,
|
|
87
|
-
display_name text not null,
|
|
88
|
-
bio text not null,
|
|
89
|
-
location text,
|
|
90
|
-
url text,
|
|
91
|
-
verified_type text,
|
|
92
|
-
followers_count integer not null default 0,
|
|
93
|
-
following_count integer not null default 0,
|
|
94
|
-
affiliations_json text not null default '[]',
|
|
95
|
-
raw_json text not null default '{}',
|
|
96
|
-
primary key (profile_id, snapshot_hash)
|
|
97
|
-
);
|
|
98
|
-
|
|
99
|
-
create table if not exists profile_bio_entities (
|
|
100
|
-
profile_id text not null,
|
|
101
|
-
kind text not null,
|
|
102
|
-
value text not null,
|
|
103
|
-
source text not null,
|
|
104
|
-
is_active integer not null default 1,
|
|
105
|
-
first_seen_at text not null,
|
|
106
|
-
last_seen_at text not null,
|
|
107
|
-
raw_json text not null default '{}',
|
|
108
|
-
primary key (profile_id, kind, value)
|
|
109
|
-
);
|
|
110
|
-
|
|
111
|
-
create table if not exists identity_search_index (
|
|
112
|
-
profile_id text not null,
|
|
113
|
-
kind text not null,
|
|
114
|
-
value text not null,
|
|
115
|
-
normalized_value text not null,
|
|
116
|
-
source text not null,
|
|
117
|
-
weight integer not null,
|
|
118
|
-
updated_at text not null,
|
|
119
|
-
primary key (profile_id, kind, value, source)
|
|
120
|
-
);
|
|
121
|
-
|
|
122
|
-
create table if not exists tweets (
|
|
123
|
-
id text primary key,
|
|
124
|
-
account_id text not null,
|
|
125
|
-
author_profile_id text not null,
|
|
126
|
-
kind text not null,
|
|
127
|
-
text text not null,
|
|
128
|
-
created_at text not null,
|
|
129
|
-
is_replied integer not null default 0,
|
|
130
|
-
reply_to_id text,
|
|
131
|
-
like_count integer not null default 0,
|
|
132
|
-
media_count integer not null default 0,
|
|
133
|
-
bookmarked integer not null default 0,
|
|
134
|
-
liked integer not null default 0,
|
|
135
|
-
entities_json text not null default '{}',
|
|
136
|
-
media_json text not null default '[]',
|
|
137
|
-
quoted_tweet_id text
|
|
138
|
-
);
|
|
139
|
-
|
|
140
|
-
create table if not exists tweet_collections (
|
|
141
|
-
account_id text not null,
|
|
142
|
-
tweet_id text not null,
|
|
143
|
-
kind text not null,
|
|
144
|
-
collected_at text,
|
|
145
|
-
source text not null,
|
|
146
|
-
raw_json text not null default '{}',
|
|
147
|
-
updated_at text not null,
|
|
148
|
-
primary key (account_id, tweet_id, kind)
|
|
149
|
-
);
|
|
150
|
-
|
|
151
|
-
create table if not exists tweet_account_edges (
|
|
152
|
-
account_id text not null,
|
|
153
|
-
tweet_id text not null,
|
|
154
|
-
kind text not null,
|
|
155
|
-
first_seen_at text not null,
|
|
156
|
-
last_seen_at text not null,
|
|
157
|
-
seen_count integer not null default 1,
|
|
158
|
-
source text not null,
|
|
159
|
-
raw_json text not null default '{}',
|
|
160
|
-
updated_at text not null,
|
|
161
|
-
primary key (account_id, tweet_id, kind)
|
|
162
|
-
);
|
|
163
|
-
|
|
164
|
-
create table if not exists dm_conversations (
|
|
165
|
-
id text primary key,
|
|
166
|
-
account_id text not null,
|
|
167
|
-
participant_profile_id text not null,
|
|
168
|
-
title text not null,
|
|
169
|
-
inbox_kind text not null default 'accepted',
|
|
170
|
-
last_message_at text not null,
|
|
171
|
-
unread_count integer not null default 0,
|
|
172
|
-
needs_reply integer not null default 0
|
|
173
|
-
);
|
|
174
|
-
|
|
175
|
-
create table if not exists dm_messages (
|
|
176
|
-
id text primary key,
|
|
177
|
-
conversation_id text not null,
|
|
178
|
-
sender_profile_id text not null,
|
|
179
|
-
text text not null,
|
|
180
|
-
created_at text not null,
|
|
181
|
-
direction text not null,
|
|
182
|
-
is_replied integer not null default 0,
|
|
183
|
-
media_count integer not null default 0
|
|
184
|
-
);
|
|
185
|
-
|
|
186
|
-
create table if not exists tweet_actions (
|
|
187
|
-
id text primary key,
|
|
188
|
-
account_id text not null,
|
|
189
|
-
tweet_id text,
|
|
190
|
-
kind text not null,
|
|
191
|
-
body text not null,
|
|
192
|
-
created_at text not null
|
|
193
|
-
);
|
|
194
|
-
|
|
195
|
-
create table if not exists blocks (
|
|
196
|
-
account_id text not null,
|
|
197
|
-
profile_id text not null,
|
|
198
|
-
source text not null,
|
|
199
|
-
created_at text not null,
|
|
200
|
-
primary key (account_id, profile_id)
|
|
201
|
-
);
|
|
202
|
-
|
|
203
|
-
create table if not exists mutes (
|
|
204
|
-
account_id text not null,
|
|
205
|
-
profile_id text not null,
|
|
206
|
-
source text not null,
|
|
207
|
-
created_at text not null,
|
|
208
|
-
primary key (account_id, profile_id)
|
|
209
|
-
);
|
|
210
|
-
|
|
211
|
-
create table if not exists ai_scores (
|
|
212
|
-
entity_kind text not null,
|
|
213
|
-
entity_id text not null,
|
|
214
|
-
model text not null,
|
|
215
|
-
score integer not null,
|
|
216
|
-
summary text not null,
|
|
217
|
-
reasoning text not null,
|
|
218
|
-
updated_at text not null,
|
|
219
|
-
primary key (entity_kind, entity_id)
|
|
220
|
-
);
|
|
221
|
-
|
|
222
|
-
create table if not exists sync_cache (
|
|
223
|
-
cache_key text primary key,
|
|
224
|
-
value_json text not null,
|
|
225
|
-
updated_at text not null
|
|
226
|
-
);
|
|
227
|
-
|
|
228
|
-
create table if not exists url_expansions (
|
|
229
|
-
short_url text primary key,
|
|
230
|
-
expanded_url text not null,
|
|
231
|
-
final_url text not null,
|
|
232
|
-
status text not null,
|
|
233
|
-
expanded_tweet_id text,
|
|
234
|
-
expanded_handle text,
|
|
235
|
-
title text,
|
|
236
|
-
description text,
|
|
237
|
-
image_url text,
|
|
238
|
-
site_name text,
|
|
239
|
-
error text,
|
|
240
|
-
source text not null,
|
|
241
|
-
updated_at text not null
|
|
242
|
-
);
|
|
243
|
-
|
|
244
|
-
create table if not exists link_occurrences (
|
|
245
|
-
source_kind text not null,
|
|
246
|
-
source_id text not null,
|
|
247
|
-
source_position integer not null,
|
|
248
|
-
short_url text not null,
|
|
249
|
-
account_id text,
|
|
250
|
-
conversation_id text,
|
|
251
|
-
direction text,
|
|
252
|
-
created_at text not null,
|
|
253
|
-
primary key (source_kind, source_id, source_position, short_url)
|
|
254
|
-
);
|
|
255
|
-
|
|
256
|
-
create table if not exists follow_snapshots (
|
|
257
|
-
id text primary key,
|
|
258
|
-
account_id text not null,
|
|
259
|
-
direction text not null,
|
|
260
|
-
source text not null,
|
|
261
|
-
status text not null,
|
|
262
|
-
page_count integer not null default 0,
|
|
263
|
-
result_count integer not null default 0,
|
|
264
|
-
started_at text not null,
|
|
265
|
-
completed_at text not null,
|
|
266
|
-
raw_meta_json text not null default '{}'
|
|
267
|
-
);
|
|
268
|
-
|
|
269
|
-
create table if not exists follow_snapshot_members (
|
|
270
|
-
snapshot_id text not null,
|
|
271
|
-
profile_id text not null,
|
|
272
|
-
external_user_id text not null,
|
|
273
|
-
position integer not null,
|
|
274
|
-
primary key (snapshot_id, profile_id)
|
|
275
|
-
);
|
|
276
|
-
|
|
277
|
-
create table if not exists follow_edges (
|
|
278
|
-
account_id text not null,
|
|
279
|
-
direction text not null,
|
|
280
|
-
profile_id text not null,
|
|
281
|
-
external_user_id text not null,
|
|
282
|
-
source text not null,
|
|
283
|
-
current integer not null default 1,
|
|
284
|
-
first_seen_at text not null,
|
|
285
|
-
last_seen_at text not null,
|
|
286
|
-
ended_at text,
|
|
287
|
-
updated_at text not null,
|
|
288
|
-
primary key (account_id, direction, profile_id)
|
|
289
|
-
);
|
|
290
|
-
|
|
291
|
-
create table if not exists follow_events (
|
|
292
|
-
id text primary key,
|
|
293
|
-
account_id text not null,
|
|
294
|
-
direction text not null,
|
|
295
|
-
profile_id text not null,
|
|
296
|
-
external_user_id text not null,
|
|
297
|
-
kind text not null,
|
|
298
|
-
event_at text not null,
|
|
299
|
-
snapshot_id text not null
|
|
300
|
-
);
|
|
301
|
-
|
|
302
|
-
create table if not exists geocoded_locations (
|
|
303
|
-
normalized_key text primary key,
|
|
304
|
-
original text not null,
|
|
305
|
-
lat real not null,
|
|
306
|
-
lng real not null,
|
|
307
|
-
formatted text,
|
|
308
|
-
country_code text,
|
|
309
|
-
confidence integer,
|
|
310
|
-
provider text not null,
|
|
311
|
-
approx_radius_m real,
|
|
312
|
-
bounds_json text not null default '{}',
|
|
313
|
-
components_json text not null default '{}',
|
|
314
|
-
hits integer not null default 1,
|
|
315
|
-
created_at text not null,
|
|
316
|
-
last_used_at text not null
|
|
317
|
-
);
|
|
318
|
-
|
|
319
|
-
create table if not exists geocoded_locations_unresolved (
|
|
320
|
-
normalized_key text primary key,
|
|
321
|
-
original text not null,
|
|
322
|
-
reason text not null,
|
|
323
|
-
last_attempted_at text not null,
|
|
324
|
-
ttl_until text
|
|
325
|
-
);
|
|
326
|
-
|
|
327
|
-
create virtual table if not exists tweets_fts using fts5(
|
|
328
|
-
tweet_id unindexed,
|
|
329
|
-
text
|
|
330
|
-
);
|
|
331
|
-
|
|
332
|
-
create virtual table if not exists dm_fts using fts5(
|
|
333
|
-
message_id unindexed,
|
|
334
|
-
text
|
|
335
|
-
);
|
|
336
|
-
`;
|
|
337
|
-
|
|
338
|
-
const INDEX_SQL = `
|
|
339
|
-
create index if not exists idx_tweets_kind_created on tweets(kind, created_at desc);
|
|
340
|
-
create index if not exists idx_tweets_created on tweets(created_at desc);
|
|
341
|
-
create index if not exists idx_tweets_account_created on tweets(account_id, created_at desc);
|
|
342
|
-
create index if not exists idx_tweets_quoted on tweets(quoted_tweet_id);
|
|
343
|
-
create index if not exists idx_tweet_collections_kind_account on tweet_collections(kind, account_id, collected_at desc, tweet_id);
|
|
344
|
-
create index if not exists idx_tweet_collections_tweet on tweet_collections(tweet_id);
|
|
345
|
-
create index if not exists idx_tweet_account_edges_kind_account on tweet_account_edges(kind, account_id, last_seen_at desc, tweet_id);
|
|
346
|
-
create index if not exists idx_tweet_account_edges_kind_tweet on tweet_account_edges(kind, tweet_id, account_id);
|
|
347
|
-
create index if not exists idx_tweet_account_edges_tweet on tweet_account_edges(tweet_id);
|
|
348
|
-
create index if not exists idx_dm_conversations_account on dm_conversations(account_id, last_message_at desc);
|
|
349
|
-
create index if not exists idx_dm_messages_conversation on dm_messages(conversation_id, created_at asc);
|
|
350
|
-
create index if not exists idx_profiles_followers on profiles(followers_count desc);
|
|
351
|
-
create index if not exists idx_profiles_following on profiles(following_count desc);
|
|
352
|
-
create index if not exists idx_profiles_handle on profiles(handle);
|
|
353
|
-
create index if not exists idx_profile_affiliations_subject on profile_affiliations(subject_profile_id, is_active, last_seen_at desc);
|
|
354
|
-
create index if not exists idx_profile_affiliations_org on profile_affiliations(organization_profile_id, is_active, last_seen_at desc);
|
|
355
|
-
create index if not exists idx_profile_snapshots_profile on profile_snapshots(profile_id, last_seen_at desc);
|
|
356
|
-
create index if not exists idx_profile_bio_entities_profile on profile_bio_entities(profile_id, is_active, last_seen_at desc);
|
|
357
|
-
create index if not exists idx_profile_bio_entities_value on profile_bio_entities(kind, value, is_active);
|
|
358
|
-
create index if not exists idx_identity_search_index_profile on identity_search_index(profile_id);
|
|
359
|
-
create index if not exists idx_identity_search_index_value on identity_search_index(normalized_value, kind, weight desc);
|
|
360
|
-
create index if not exists idx_blocks_account_created on blocks(account_id, created_at desc);
|
|
361
|
-
create index if not exists idx_mutes_account_created on mutes(account_id, created_at desc);
|
|
362
|
-
create index if not exists idx_ai_scores_updated on ai_scores(updated_at desc);
|
|
363
|
-
create index if not exists idx_sync_cache_updated on sync_cache(updated_at desc);
|
|
364
|
-
create index if not exists idx_url_expansions_expanded on url_expansions(expanded_url);
|
|
365
|
-
create index if not exists idx_url_expansions_tweet on url_expansions(expanded_tweet_id);
|
|
366
|
-
create index if not exists idx_url_expansions_handle on url_expansions(expanded_handle);
|
|
367
|
-
create index if not exists idx_link_occurrences_url on link_occurrences(short_url);
|
|
368
|
-
create index if not exists idx_link_occurrences_created on link_occurrences(created_at desc);
|
|
369
|
-
create index if not exists idx_link_occurrences_account on link_occurrences(account_id, created_at desc);
|
|
370
|
-
create index if not exists idx_link_occurrences_direction on link_occurrences(direction, created_at desc);
|
|
371
|
-
create index if not exists idx_follow_edges_current on follow_edges(account_id, direction, current, last_seen_at desc);
|
|
372
|
-
create index if not exists idx_follow_edges_profile on follow_edges(profile_id, current);
|
|
373
|
-
create index if not exists idx_follow_snapshots_account on follow_snapshots(account_id, direction, completed_at desc);
|
|
374
|
-
create index if not exists idx_follow_events_account on follow_events(account_id, direction, kind, event_at desc);
|
|
375
|
-
create index if not exists idx_geocoded_locations_country on geocoded_locations(country_code);
|
|
376
|
-
create index if not exists idx_geocoded_locations_last_used on geocoded_locations(last_used_at desc);
|
|
377
|
-
create index if not exists idx_geocoded_unresolved_ttl on geocoded_locations_unresolved(ttl_until desc);
|
|
378
|
-
`;
|
|
379
|
-
|
|
380
|
-
function getColumnNames(db: Database, tableName: string): Set<string> {
|
|
381
|
-
const rows = db.prepare(`pragma table_info(${tableName})`).all() as Array<{
|
|
382
|
-
name: string;
|
|
383
|
-
}>;
|
|
384
|
-
return new Set(rows.map((row) => row.name));
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
function ensureTweetMetadataColumns(db: Database) {
|
|
388
|
-
const columnNames = getColumnNames(db, "tweets");
|
|
389
|
-
if (!columnNames.has("entities_json")) {
|
|
390
|
-
db.exec(
|
|
391
|
-
"alter table tweets add column entities_json text not null default '{}'",
|
|
392
|
-
);
|
|
393
|
-
}
|
|
394
|
-
if (!columnNames.has("media_json")) {
|
|
395
|
-
db.exec(
|
|
396
|
-
"alter table tweets add column media_json text not null default '[]'",
|
|
397
|
-
);
|
|
398
|
-
}
|
|
399
|
-
if (!columnNames.has("quoted_tweet_id")) {
|
|
400
|
-
db.exec("alter table tweets add column quoted_tweet_id text");
|
|
401
|
-
}
|
|
402
|
-
}
|
|
403
|
-
|
|
404
|
-
function ensureProfileAvatarColumns(db: Database) {
|
|
405
|
-
const columnNames = getColumnNames(db, "profiles");
|
|
406
|
-
if (!columnNames.has("following_count")) {
|
|
407
|
-
db.exec(
|
|
408
|
-
"alter table profiles add column following_count integer not null default 0",
|
|
409
|
-
);
|
|
410
|
-
}
|
|
411
|
-
if (!columnNames.has("avatar_url")) {
|
|
412
|
-
db.exec("alter table profiles add column avatar_url text");
|
|
413
|
-
}
|
|
414
|
-
if (!columnNames.has("location")) {
|
|
415
|
-
db.exec("alter table profiles add column location text");
|
|
416
|
-
}
|
|
417
|
-
if (!columnNames.has("url")) {
|
|
418
|
-
db.exec("alter table profiles add column url text");
|
|
419
|
-
}
|
|
420
|
-
if (!columnNames.has("verified_type")) {
|
|
421
|
-
db.exec("alter table profiles add column verified_type text");
|
|
422
|
-
}
|
|
423
|
-
if (!columnNames.has("entities_json")) {
|
|
424
|
-
db.exec(
|
|
425
|
-
"alter table profiles add column entities_json text not null default '{}'",
|
|
426
|
-
);
|
|
427
|
-
}
|
|
428
|
-
if (!columnNames.has("raw_json")) {
|
|
429
|
-
db.exec(
|
|
430
|
-
"alter table profiles add column raw_json text not null default '{}'",
|
|
431
|
-
);
|
|
432
|
-
}
|
|
433
|
-
if (!columnNames.has("public_metrics_json")) {
|
|
434
|
-
db.exec(
|
|
435
|
-
"alter table profiles add column public_metrics_json text not null default '{}'",
|
|
436
|
-
);
|
|
437
|
-
}
|
|
438
|
-
}
|
|
439
|
-
|
|
440
|
-
function ensureAccountExternalUserIdColumn(db: Database) {
|
|
441
|
-
const columnNames = getColumnNames(db, "accounts");
|
|
442
|
-
if (!columnNames.has("external_user_id")) {
|
|
443
|
-
db.exec("alter table accounts add column external_user_id text");
|
|
444
|
-
}
|
|
445
|
-
}
|
|
446
|
-
|
|
447
|
-
function ensureDmConversationInboxColumns(db: Database) {
|
|
448
|
-
const columnNames = getColumnNames(db, "dm_conversations");
|
|
449
|
-
if (!columnNames.has("inbox_kind")) {
|
|
450
|
-
db.exec(
|
|
451
|
-
"alter table dm_conversations add column inbox_kind text not null default 'accepted'",
|
|
452
|
-
);
|
|
453
|
-
}
|
|
454
|
-
}
|
|
455
|
-
|
|
456
|
-
function ensureTweetCollectionsTable(db: Database) {
|
|
457
|
-
db.exec(`
|
|
458
|
-
create table if not exists tweet_collections (
|
|
459
|
-
account_id text not null,
|
|
460
|
-
tweet_id text not null,
|
|
461
|
-
kind text not null,
|
|
462
|
-
collected_at text,
|
|
463
|
-
source text not null,
|
|
464
|
-
raw_json text not null default '{}',
|
|
465
|
-
updated_at text not null,
|
|
466
|
-
primary key (account_id, tweet_id, kind)
|
|
467
|
-
);
|
|
468
|
-
`);
|
|
469
|
-
}
|
|
470
|
-
|
|
471
|
-
function ensureTweetAccountEdgesTable(db: Database) {
|
|
472
|
-
db.exec(`
|
|
473
|
-
create table if not exists tweet_account_edges (
|
|
474
|
-
account_id text not null,
|
|
475
|
-
tweet_id text not null,
|
|
476
|
-
kind text not null,
|
|
477
|
-
first_seen_at text not null,
|
|
478
|
-
last_seen_at text not null,
|
|
479
|
-
seen_count integer not null default 1,
|
|
480
|
-
source text not null,
|
|
481
|
-
raw_json text not null default '{}',
|
|
482
|
-
updated_at text not null,
|
|
483
|
-
primary key (account_id, tweet_id, kind)
|
|
484
|
-
);
|
|
485
|
-
`);
|
|
486
|
-
}
|
|
487
|
-
|
|
488
|
-
function ensureProfileAffiliationsTable(db: Database) {
|
|
489
|
-
db.exec(`
|
|
490
|
-
create table if not exists profile_affiliations (
|
|
491
|
-
subject_profile_id text not null,
|
|
492
|
-
organization_profile_id text not null,
|
|
493
|
-
organization_name text,
|
|
494
|
-
organization_handle text,
|
|
495
|
-
badge_url text,
|
|
496
|
-
url text,
|
|
497
|
-
label text,
|
|
498
|
-
source text not null,
|
|
499
|
-
is_active integer not null default 1,
|
|
500
|
-
first_seen_at text not null,
|
|
501
|
-
last_seen_at text not null,
|
|
502
|
-
raw_json text not null default '{}',
|
|
503
|
-
updated_at text not null,
|
|
504
|
-
primary key (subject_profile_id, organization_profile_id)
|
|
505
|
-
);
|
|
506
|
-
`);
|
|
507
|
-
}
|
|
508
|
-
|
|
509
|
-
function ensureProfileSnapshotsTable(db: Database) {
|
|
510
|
-
db.exec(`
|
|
511
|
-
create table if not exists profile_snapshots (
|
|
512
|
-
profile_id text not null,
|
|
513
|
-
snapshot_hash text not null,
|
|
514
|
-
observed_at text not null,
|
|
515
|
-
last_seen_at text not null,
|
|
516
|
-
source text not null,
|
|
517
|
-
handle text not null,
|
|
518
|
-
display_name text not null,
|
|
519
|
-
bio text not null,
|
|
520
|
-
location text,
|
|
521
|
-
url text,
|
|
522
|
-
verified_type text,
|
|
523
|
-
followers_count integer not null default 0,
|
|
524
|
-
following_count integer not null default 0,
|
|
525
|
-
affiliations_json text not null default '[]',
|
|
526
|
-
raw_json text not null default '{}',
|
|
527
|
-
primary key (profile_id, snapshot_hash)
|
|
528
|
-
);
|
|
529
|
-
`);
|
|
530
|
-
}
|
|
531
|
-
|
|
532
|
-
function ensureProfileBioEntitiesTable(db: Database) {
|
|
533
|
-
db.exec(`
|
|
534
|
-
create table if not exists profile_bio_entities (
|
|
535
|
-
profile_id text not null,
|
|
536
|
-
kind text not null,
|
|
537
|
-
value text not null,
|
|
538
|
-
source text not null,
|
|
539
|
-
is_active integer not null default 1,
|
|
540
|
-
first_seen_at text not null,
|
|
541
|
-
last_seen_at text not null,
|
|
542
|
-
raw_json text not null default '{}',
|
|
543
|
-
primary key (profile_id, kind, value)
|
|
544
|
-
);
|
|
545
|
-
`);
|
|
546
|
-
}
|
|
547
|
-
|
|
548
|
-
function ensureIdentitySearchIndexTable(db: Database) {
|
|
549
|
-
db.exec(`
|
|
550
|
-
create table if not exists identity_search_index (
|
|
551
|
-
profile_id text not null,
|
|
552
|
-
kind text not null,
|
|
553
|
-
value text not null,
|
|
554
|
-
normalized_value text not null,
|
|
555
|
-
source text not null,
|
|
556
|
-
weight integer not null,
|
|
557
|
-
updated_at text not null,
|
|
558
|
-
primary key (profile_id, kind, value, source)
|
|
559
|
-
);
|
|
560
|
-
`);
|
|
561
|
-
}
|
|
562
|
-
|
|
563
|
-
function ensureLinkIndexTables(db: Database) {
|
|
564
|
-
db.exec(`
|
|
565
|
-
create table if not exists url_expansions (
|
|
566
|
-
short_url text primary key,
|
|
567
|
-
expanded_url text not null,
|
|
568
|
-
final_url text not null,
|
|
569
|
-
status text not null,
|
|
570
|
-
expanded_tweet_id text,
|
|
571
|
-
expanded_handle text,
|
|
572
|
-
title text,
|
|
573
|
-
description text,
|
|
574
|
-
image_url text,
|
|
575
|
-
site_name text,
|
|
576
|
-
error text,
|
|
577
|
-
source text not null,
|
|
578
|
-
updated_at text not null
|
|
579
|
-
);
|
|
580
|
-
|
|
581
|
-
create table if not exists link_occurrences (
|
|
582
|
-
source_kind text not null,
|
|
583
|
-
source_id text not null,
|
|
584
|
-
source_position integer not null,
|
|
585
|
-
short_url text not null,
|
|
586
|
-
account_id text,
|
|
587
|
-
conversation_id text,
|
|
588
|
-
direction text,
|
|
589
|
-
created_at text not null,
|
|
590
|
-
primary key (source_kind, source_id, source_position, short_url)
|
|
591
|
-
);
|
|
592
|
-
`);
|
|
593
|
-
|
|
594
|
-
const urlExpansionColumns = getColumnNames(db, "url_expansions");
|
|
595
|
-
if (!urlExpansionColumns.has("image_url")) {
|
|
596
|
-
db.exec("alter table url_expansions add column image_url text");
|
|
597
|
-
}
|
|
598
|
-
if (!urlExpansionColumns.has("site_name")) {
|
|
599
|
-
db.exec("alter table url_expansions add column site_name text");
|
|
600
|
-
}
|
|
601
|
-
}
|
|
602
|
-
|
|
603
|
-
function ensureFollowGraphTables(db: Database) {
|
|
604
|
-
db.exec(`
|
|
605
|
-
create table if not exists follow_snapshots (
|
|
606
|
-
id text primary key,
|
|
607
|
-
account_id text not null,
|
|
608
|
-
direction text not null,
|
|
609
|
-
source text not null,
|
|
610
|
-
status text not null,
|
|
611
|
-
page_count integer not null default 0,
|
|
612
|
-
result_count integer not null default 0,
|
|
613
|
-
started_at text not null,
|
|
614
|
-
completed_at text not null,
|
|
615
|
-
raw_meta_json text not null default '{}'
|
|
616
|
-
);
|
|
617
|
-
|
|
618
|
-
create table if not exists follow_snapshot_members (
|
|
619
|
-
snapshot_id text not null,
|
|
620
|
-
profile_id text not null,
|
|
621
|
-
external_user_id text not null,
|
|
622
|
-
position integer not null,
|
|
623
|
-
primary key (snapshot_id, profile_id)
|
|
624
|
-
);
|
|
625
|
-
|
|
626
|
-
create table if not exists follow_edges (
|
|
627
|
-
account_id text not null,
|
|
628
|
-
direction text not null,
|
|
629
|
-
profile_id text not null,
|
|
630
|
-
external_user_id text not null,
|
|
631
|
-
source text not null,
|
|
632
|
-
current integer not null default 1,
|
|
633
|
-
first_seen_at text not null,
|
|
634
|
-
last_seen_at text not null,
|
|
635
|
-
ended_at text,
|
|
636
|
-
updated_at text not null,
|
|
637
|
-
primary key (account_id, direction, profile_id)
|
|
638
|
-
);
|
|
639
|
-
|
|
640
|
-
create table if not exists follow_events (
|
|
641
|
-
id text primary key,
|
|
642
|
-
account_id text not null,
|
|
643
|
-
direction text not null,
|
|
644
|
-
profile_id text not null,
|
|
645
|
-
external_user_id text not null,
|
|
646
|
-
kind text not null,
|
|
647
|
-
event_at text not null,
|
|
648
|
-
snapshot_id text not null
|
|
649
|
-
);
|
|
650
|
-
`);
|
|
651
|
-
}
|
|
652
|
-
|
|
653
|
-
function backfillTweetCollections(db: Database) {
|
|
654
|
-
const missingKinds = (
|
|
655
|
-
[
|
|
656
|
-
["likes", "liked"],
|
|
657
|
-
["bookmarks", "bookmarked"],
|
|
658
|
-
] as const
|
|
659
|
-
).filter(([, column]) =>
|
|
660
|
-
db
|
|
661
|
-
.prepare(
|
|
662
|
-
`
|
|
663
|
-
select 1
|
|
664
|
-
from tweets as tweet
|
|
665
|
-
where tweet.${column} = 1
|
|
666
|
-
and not exists (
|
|
667
|
-
select 1
|
|
668
|
-
from tweet_collections as collection
|
|
669
|
-
where collection.account_id = tweet.account_id
|
|
670
|
-
and collection.tweet_id = tweet.id
|
|
671
|
-
and collection.kind = ?
|
|
672
|
-
)
|
|
673
|
-
limit 1
|
|
674
|
-
`,
|
|
675
|
-
)
|
|
676
|
-
.get(column === "liked" ? "likes" : "bookmarks"),
|
|
677
|
-
);
|
|
678
|
-
if (missingKinds.length === 0) {
|
|
679
|
-
return;
|
|
680
|
-
}
|
|
681
|
-
|
|
682
|
-
const now = new Date().toISOString();
|
|
683
|
-
const insert = db.prepare(`
|
|
684
|
-
insert or ignore into tweet_collections (
|
|
685
|
-
account_id, tweet_id, kind, collected_at, source, raw_json, updated_at
|
|
686
|
-
)
|
|
687
|
-
select account_id, id, ?, null, 'legacy', '{}', ?
|
|
688
|
-
from tweets
|
|
689
|
-
where
|
|
690
|
-
case
|
|
691
|
-
when ? = 'likes' then liked
|
|
692
|
-
else bookmarked
|
|
693
|
-
end = 1
|
|
694
|
-
`);
|
|
695
|
-
|
|
696
|
-
db.transaction(() => {
|
|
697
|
-
for (const [kind] of missingKinds) {
|
|
698
|
-
insert.run(kind, now, kind);
|
|
699
|
-
}
|
|
700
|
-
})();
|
|
701
|
-
}
|
|
702
|
-
|
|
703
|
-
function backfillTweetAccountEdges(db: Database) {
|
|
704
|
-
const missing = db
|
|
705
|
-
.prepare(
|
|
706
|
-
`
|
|
707
|
-
select 1
|
|
708
|
-
from tweets as tweet
|
|
709
|
-
where tweet.kind in ('home', 'mention')
|
|
710
|
-
and not exists (
|
|
711
|
-
select 1
|
|
712
|
-
from tweet_account_edges as edge
|
|
713
|
-
where edge.account_id = tweet.account_id
|
|
714
|
-
and edge.tweet_id = tweet.id
|
|
715
|
-
and edge.kind = tweet.kind
|
|
716
|
-
)
|
|
717
|
-
limit 1
|
|
718
|
-
`,
|
|
719
|
-
)
|
|
720
|
-
.get();
|
|
721
|
-
if (!missing) {
|
|
722
|
-
return;
|
|
723
|
-
}
|
|
724
|
-
|
|
725
|
-
const now = new Date().toISOString();
|
|
726
|
-
db.prepare(`
|
|
727
|
-
insert or ignore into tweet_account_edges (
|
|
728
|
-
account_id, tweet_id, kind, first_seen_at, last_seen_at, seen_count,
|
|
729
|
-
source, raw_json, updated_at
|
|
730
|
-
)
|
|
731
|
-
select
|
|
732
|
-
account_id,
|
|
733
|
-
id,
|
|
734
|
-
kind,
|
|
735
|
-
created_at,
|
|
736
|
-
created_at,
|
|
737
|
-
1,
|
|
738
|
-
'legacy',
|
|
739
|
-
'{}',
|
|
740
|
-
?
|
|
741
|
-
from tweets
|
|
742
|
-
where kind in ('home', 'mention')
|
|
743
|
-
`).run(now);
|
|
744
|
-
}
|
|
745
|
-
|
|
746
|
-
function ensureSchemaIndexes(db: Database) {
|
|
747
|
-
db.exec(INDEX_SQL);
|
|
748
|
-
}
|
|
749
|
-
|
|
750
|
-
const DATABASE_MIGRATIONS: readonly DatabaseMigration[] = [
|
|
751
|
-
{
|
|
752
|
-
version: 1,
|
|
753
|
-
name: "canonical schema baseline",
|
|
754
|
-
up: (db) => {
|
|
755
|
-
db.exec(BASE_SCHEMA_SQL);
|
|
756
|
-
ensureAccountExternalUserIdColumn(db);
|
|
757
|
-
ensureDmConversationInboxColumns(db);
|
|
758
|
-
ensureTweetMetadataColumns(db);
|
|
759
|
-
ensureProfileAvatarColumns(db);
|
|
760
|
-
ensureTweetCollectionsTable(db);
|
|
761
|
-
ensureTweetAccountEdgesTable(db);
|
|
762
|
-
ensureProfileAffiliationsTable(db);
|
|
763
|
-
ensureProfileSnapshotsTable(db);
|
|
764
|
-
ensureProfileBioEntitiesTable(db);
|
|
765
|
-
ensureIdentitySearchIndexTable(db);
|
|
766
|
-
ensureLinkIndexTables(db);
|
|
767
|
-
ensureFollowGraphTables(db);
|
|
768
|
-
ensureSchemaIndexes(db);
|
|
769
|
-
backfillTweetCollections(db);
|
|
770
|
-
backfillTweetAccountEdges(db);
|
|
771
|
-
},
|
|
772
|
-
},
|
|
773
|
-
];
|
|
774
|
-
|
|
775
|
-
function ensureDemoData(db: Database) {
|
|
776
|
-
if (demoSeedAttempted) {
|
|
777
|
-
return;
|
|
778
|
-
}
|
|
779
|
-
|
|
780
|
-
seedDemoData(db);
|
|
781
|
-
backfillTweetCollections(db);
|
|
782
|
-
backfillTweetAccountEdges(db);
|
|
783
|
-
demoSeedAttempted = true;
|
|
784
|
-
}
|
|
785
|
-
|
|
786
|
-
function initDatabase(options: InitDatabaseOptions = {}) {
|
|
787
|
-
ensureBirdclawDirs();
|
|
788
|
-
|
|
789
|
-
if (!nativeDb) {
|
|
790
|
-
const { dbPath } = getBirdclawPaths();
|
|
791
|
-
nativeDb = createDatabaseConnection(dbPath, "writer");
|
|
792
|
-
nativeDb.exec(`
|
|
793
|
-
pragma busy_timeout = ${SQLITE_BUSY_TIMEOUT_MS};
|
|
794
|
-
pragma foreign_keys = on;
|
|
795
|
-
`);
|
|
796
|
-
runDatabaseMigrations(nativeDb, DATABASE_MIGRATIONS);
|
|
797
|
-
if (options.seedDemoData !== false) {
|
|
798
|
-
ensureDemoData(nativeDb);
|
|
799
|
-
}
|
|
800
|
-
} else if (options.seedDemoData !== false) {
|
|
801
|
-
ensureDemoData(nativeDb);
|
|
802
|
-
}
|
|
803
|
-
|
|
804
|
-
if (!kyselyDb) {
|
|
805
|
-
kyselyDb = new Kysely<BirdclawDatabase>({
|
|
806
|
-
dialect: new SqliteDialect({
|
|
807
|
-
database: nativeDb,
|
|
808
|
-
}),
|
|
809
|
-
});
|
|
810
|
-
}
|
|
811
|
-
}
|
|
812
|
-
|
|
813
|
-
function createDatabaseConnection(
|
|
814
|
-
dbPath: string,
|
|
815
|
-
role: DatabaseConnectionRole,
|
|
816
|
-
options: { readonly?: boolean } = {},
|
|
817
|
-
) {
|
|
818
|
-
return new NativeSqliteDatabase(dbPath, {
|
|
819
|
-
...options,
|
|
820
|
-
onStatement: (sql, durationMs) =>
|
|
821
|
-
recordDatabaseStatement(role, sql, durationMs),
|
|
822
|
-
});
|
|
823
|
-
}
|
|
824
|
-
|
|
825
|
-
export function getNativeDb(options: InitDatabaseOptions = {}) {
|
|
826
|
-
initDatabase(options);
|
|
827
|
-
return nativeDb as Database;
|
|
828
|
-
}
|
|
829
|
-
|
|
830
|
-
export function getReadDb(options: InitDatabaseOptions = {}) {
|
|
831
|
-
initDatabase(options);
|
|
832
|
-
if (readDbs.length === 0) {
|
|
833
|
-
const { dbPath } = getBirdclawPaths();
|
|
834
|
-
readDbs = Array.from({ length: 2 }, () => {
|
|
835
|
-
const db = createDatabaseConnection(dbPath, "reader", {
|
|
836
|
-
readonly: true,
|
|
837
|
-
});
|
|
838
|
-
db.exec(`
|
|
839
|
-
pragma busy_timeout = ${SQLITE_BUSY_TIMEOUT_MS};
|
|
840
|
-
pragma foreign_keys = on;
|
|
841
|
-
pragma query_only = on;
|
|
842
|
-
`);
|
|
843
|
-
return db;
|
|
844
|
-
});
|
|
845
|
-
}
|
|
846
|
-
const db = readDbs[readDbIndex % readDbs.length] as Database;
|
|
847
|
-
readDbIndex = (readDbIndex + 1) % readDbs.length;
|
|
848
|
-
return db;
|
|
849
|
-
}
|
|
850
|
-
|
|
851
|
-
export function getDb() {
|
|
852
|
-
initDatabase();
|
|
853
|
-
return kyselyDb as Kysely<BirdclawDatabase>;
|
|
854
|
-
}
|
|
855
|
-
|
|
856
|
-
export async function closeDatabase() {
|
|
857
|
-
const db = kyselyDb;
|
|
858
|
-
const native = nativeDb;
|
|
859
|
-
const readers = readDbs;
|
|
860
|
-
kyselyDb = undefined;
|
|
861
|
-
nativeDb = undefined;
|
|
862
|
-
readDbs = [];
|
|
863
|
-
readDbIndex = 0;
|
|
864
|
-
demoSeedAttempted = false;
|
|
865
|
-
|
|
866
|
-
for (const reader of readers) reader.close();
|
|
867
|
-
if (db) {
|
|
868
|
-
await db.destroy();
|
|
869
|
-
} else {
|
|
870
|
-
native?.close();
|
|
871
|
-
}
|
|
872
|
-
}
|
|
873
|
-
|
|
874
|
-
export function resetDatabaseForTests() {
|
|
875
|
-
const db = kyselyDb;
|
|
876
|
-
const native = nativeDb;
|
|
877
|
-
const readers = readDbs;
|
|
878
|
-
kyselyDb = undefined;
|
|
879
|
-
nativeDb = undefined;
|
|
880
|
-
readDbs = [];
|
|
881
|
-
readDbIndex = 0;
|
|
882
|
-
demoSeedAttempted = false;
|
|
883
|
-
for (const reader of readers) reader.close();
|
|
884
|
-
if (db) {
|
|
885
|
-
void db.destroy();
|
|
886
|
-
} else {
|
|
887
|
-
native?.close();
|
|
888
|
-
}
|
|
889
|
-
}
|