@mepuka/skygent 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/README.md +59 -0
  2. package/index.ts +146 -0
  3. package/package.json +56 -0
  4. package/src/cli/app.ts +75 -0
  5. package/src/cli/config-command.ts +140 -0
  6. package/src/cli/config.ts +91 -0
  7. package/src/cli/derive.ts +205 -0
  8. package/src/cli/doc/annotation.ts +36 -0
  9. package/src/cli/doc/filter.ts +69 -0
  10. package/src/cli/doc/index.ts +9 -0
  11. package/src/cli/doc/post.ts +155 -0
  12. package/src/cli/doc/primitives.ts +25 -0
  13. package/src/cli/doc/render.ts +18 -0
  14. package/src/cli/doc/table.ts +114 -0
  15. package/src/cli/doc/thread.ts +46 -0
  16. package/src/cli/doc/tree.ts +126 -0
  17. package/src/cli/errors.ts +59 -0
  18. package/src/cli/exit-codes.ts +52 -0
  19. package/src/cli/feed.ts +177 -0
  20. package/src/cli/filter-dsl.ts +1411 -0
  21. package/src/cli/filter-errors.ts +208 -0
  22. package/src/cli/filter-help.ts +70 -0
  23. package/src/cli/filter-input.ts +54 -0
  24. package/src/cli/filter.ts +435 -0
  25. package/src/cli/graph.ts +472 -0
  26. package/src/cli/help.ts +14 -0
  27. package/src/cli/interval.ts +35 -0
  28. package/src/cli/jetstream.ts +173 -0
  29. package/src/cli/layers.ts +180 -0
  30. package/src/cli/logging.ts +136 -0
  31. package/src/cli/output-format.ts +26 -0
  32. package/src/cli/output.ts +82 -0
  33. package/src/cli/parse.ts +80 -0
  34. package/src/cli/post.ts +193 -0
  35. package/src/cli/preferences.ts +11 -0
  36. package/src/cli/query-fields.ts +247 -0
  37. package/src/cli/query.ts +415 -0
  38. package/src/cli/range.ts +44 -0
  39. package/src/cli/search.ts +465 -0
  40. package/src/cli/shared-options.ts +169 -0
  41. package/src/cli/shared.ts +20 -0
  42. package/src/cli/store-errors.ts +80 -0
  43. package/src/cli/store-tree.ts +392 -0
  44. package/src/cli/store.ts +395 -0
  45. package/src/cli/sync-factory.ts +107 -0
  46. package/src/cli/sync.ts +366 -0
  47. package/src/cli/view-thread.ts +196 -0
  48. package/src/cli/view.ts +47 -0
  49. package/src/cli/watch.ts +344 -0
  50. package/src/db/migrations/store-catalog/001_init.ts +14 -0
  51. package/src/db/migrations/store-index/001_init.ts +34 -0
  52. package/src/db/migrations/store-index/002_event_log.ts +24 -0
  53. package/src/db/migrations/store-index/003_fts_and_derived.ts +52 -0
  54. package/src/db/migrations/store-index/004_query_indexes.ts +9 -0
  55. package/src/db/migrations/store-index/005_post_lang.ts +15 -0
  56. package/src/db/migrations/store-index/006_has_embed.ts +10 -0
  57. package/src/db/migrations/store-index/007_event_seq_and_checkpoints.ts +68 -0
  58. package/src/domain/bsky.ts +467 -0
  59. package/src/domain/config.ts +11 -0
  60. package/src/domain/credentials.ts +6 -0
  61. package/src/domain/defaults.ts +8 -0
  62. package/src/domain/derivation.ts +55 -0
  63. package/src/domain/errors.ts +71 -0
  64. package/src/domain/events.ts +55 -0
  65. package/src/domain/extract.ts +64 -0
  66. package/src/domain/filter-describe.ts +551 -0
  67. package/src/domain/filter-explain.ts +9 -0
  68. package/src/domain/filter.ts +797 -0
  69. package/src/domain/format.ts +91 -0
  70. package/src/domain/index.ts +13 -0
  71. package/src/domain/indexes.ts +17 -0
  72. package/src/domain/policies.ts +16 -0
  73. package/src/domain/post.ts +88 -0
  74. package/src/domain/primitives.ts +50 -0
  75. package/src/domain/raw.ts +140 -0
  76. package/src/domain/store.ts +103 -0
  77. package/src/domain/sync.ts +211 -0
  78. package/src/domain/text-width.ts +56 -0
  79. package/src/services/app-config.ts +278 -0
  80. package/src/services/bsky-client.ts +2113 -0
  81. package/src/services/credential-store.ts +408 -0
  82. package/src/services/derivation-engine.ts +502 -0
  83. package/src/services/derivation-settings.ts +61 -0
  84. package/src/services/derivation-validator.ts +68 -0
  85. package/src/services/filter-compiler.ts +269 -0
  86. package/src/services/filter-library.ts +371 -0
  87. package/src/services/filter-runtime.ts +821 -0
  88. package/src/services/filter-settings.ts +30 -0
  89. package/src/services/identity-resolver.ts +563 -0
  90. package/src/services/jetstream-sync.ts +636 -0
  91. package/src/services/lineage-store.ts +89 -0
  92. package/src/services/link-validator.ts +244 -0
  93. package/src/services/output-manager.ts +274 -0
  94. package/src/services/post-parser.ts +62 -0
  95. package/src/services/profile-resolver.ts +223 -0
  96. package/src/services/resource-monitor.ts +106 -0
  97. package/src/services/shared.ts +69 -0
  98. package/src/services/store-cleaner.ts +43 -0
  99. package/src/services/store-commit.ts +168 -0
  100. package/src/services/store-db.ts +248 -0
  101. package/src/services/store-event-log.ts +285 -0
  102. package/src/services/store-index-sql.ts +289 -0
  103. package/src/services/store-index.ts +1152 -0
  104. package/src/services/store-keys.ts +4 -0
  105. package/src/services/store-manager.ts +358 -0
  106. package/src/services/store-stats.ts +522 -0
  107. package/src/services/store-writer.ts +200 -0
  108. package/src/services/sync-checkpoint-store.ts +169 -0
  109. package/src/services/sync-engine.ts +547 -0
  110. package/src/services/sync-reporter.ts +16 -0
  111. package/src/services/sync-settings.ts +72 -0
  112. package/src/services/trending-topics.ts +226 -0
  113. package/src/services/view-checkpoint-store.ts +238 -0
  114. package/src/typeclass/chunk.ts +84 -0
@@ -0,0 +1,344 @@
1
+ import { Command, Options } from "@effect/cli";
2
+ import { Effect, Layer, Option, Stream } from "effect";
3
+ import { Jetstream } from "effect-jetstream";
4
+ import { filterExprSignature } from "../domain/filter.js";
5
+ import { DataSource } from "../domain/sync.js";
6
+ import { SyncReporter } from "../services/sync-reporter.js";
7
+ import { JetstreamSyncEngine } from "../services/jetstream-sync.js";
8
+ import { parseFilterExpr } from "./filter-input.js";
9
+ import { CliOutput, writeJsonStream } from "./output.js";
10
+ import { storeOptions } from "./store.js";
11
+ import { logInfo, makeSyncReporter } from "./logging.js";
12
+ import { ResourceMonitor } from "../services/resource-monitor.js";
13
+ import { withExamples } from "./help.js";
14
+ import { buildJetstreamSelection, jetstreamOptions } from "./jetstream.js";
15
+ import { makeWatchCommandBody } from "./sync-factory.js";
16
+ import {
17
+ feedUriArg,
18
+ listUriArg,
19
+ postUriArg,
20
+ actorArg,
21
+ storeNameOption,
22
+ filterOption,
23
+ filterJsonOption,
24
+ postFilterOption,
25
+ postFilterJsonOption,
26
+ authorFilterOption,
27
+ includePinsOption,
28
+ decodeActor,
29
+ quietOption,
30
+ refreshOption,
31
+ strictOption,
32
+ maxErrorsOption,
33
+ parseMaxErrors,
34
+ parseBoundedIntOption
35
+ } from "./shared-options.js";
36
+
37
+ const intervalOption = Options.text("interval").pipe(
38
+ Options.withDescription(
39
+ "Polling interval (e.g. \"30 seconds\", \"500 millis\") (default: 30 seconds)"
40
+ ),
41
+ Options.optional
42
+ );
43
+ const depthOption = Options.integer("depth").pipe(
44
+ Options.withDescription("Thread reply depth to include (0-1000, default 6)"),
45
+ Options.optional
46
+ );
47
+ const parentHeightOption = Options.integer("parent-height").pipe(
48
+ Options.withDescription("Thread parent height to include (0-1000, default 80)"),
49
+ Options.optional
50
+ );
51
+
52
+ const timelineCommand = Command.make(
53
+ "timeline",
54
+ {
55
+ store: storeNameOption,
56
+ filter: filterOption,
57
+ filterJson: filterJsonOption,
58
+ interval: intervalOption,
59
+ quiet: quietOption,
60
+ refresh: refreshOption
61
+ },
62
+ makeWatchCommandBody("timeline", () => DataSource.timeline())
63
+ ).pipe(
64
+ Command.withDescription(
65
+ withExamples(
66
+ "Watch timeline updates and emit sync results",
67
+ [
68
+ "skygent watch timeline --store my-store",
69
+ "skygent watch timeline --store my-store --interval \"5 minutes\" --quiet"
70
+ ],
71
+ ["Tip: add --quiet to suppress progress logs."]
72
+ )
73
+ )
74
+ );
75
+
76
+ const feedCommand = Command.make(
77
+ "feed",
78
+ {
79
+ uri: feedUriArg,
80
+ store: storeNameOption,
81
+ filter: filterOption,
82
+ filterJson: filterJsonOption,
83
+ interval: intervalOption,
84
+ quiet: quietOption,
85
+ refresh: refreshOption
86
+ },
87
+ ({ uri, ...rest }) => makeWatchCommandBody("feed", () => DataSource.feed(uri), { uri })(rest)
88
+ ).pipe(
89
+ Command.withDescription(
90
+ withExamples(
91
+ "Watch a feed URI and emit sync results",
92
+ [
93
+ "skygent watch feed at://did:plc:example/app.bsky.feed.generator/xyz --store my-store --interval \"2 minutes\""
94
+ ],
95
+ ["Tip: add --quiet to suppress progress logs."]
96
+ )
97
+ )
98
+ );
99
+
100
+ const listCommand = Command.make(
101
+ "list",
102
+ {
103
+ uri: listUriArg,
104
+ store: storeNameOption,
105
+ filter: filterOption,
106
+ filterJson: filterJsonOption,
107
+ interval: intervalOption,
108
+ quiet: quietOption,
109
+ refresh: refreshOption
110
+ },
111
+ ({ uri, ...rest }) => makeWatchCommandBody("list", () => DataSource.list(uri), { uri })(rest)
112
+ ).pipe(
113
+ Command.withDescription(
114
+ withExamples(
115
+ "Watch a list feed URI and emit sync results",
116
+ [
117
+ "skygent watch list at://did:plc:example/app.bsky.graph.list/xyz --store my-store --interval \"2 minutes\""
118
+ ],
119
+ ["Tip: add --quiet to suppress progress logs."]
120
+ )
121
+ )
122
+ );
123
+
124
+ const notificationsCommand = Command.make(
125
+ "notifications",
126
+ {
127
+ store: storeNameOption,
128
+ filter: filterOption,
129
+ filterJson: filterJsonOption,
130
+ interval: intervalOption,
131
+ quiet: quietOption,
132
+ refresh: refreshOption
133
+ },
134
+ makeWatchCommandBody("notifications", () => DataSource.notifications())
135
+ ).pipe(
136
+ Command.withDescription(
137
+ withExamples(
138
+ "Watch notifications and emit sync results",
139
+ ["skygent watch notifications --store my-store --interval \"1 minute\" --quiet"],
140
+ ["Tip: add --quiet to suppress progress logs."]
141
+ )
142
+ )
143
+ );
144
+
145
+ const authorCommand = Command.make(
146
+ "author",
147
+ {
148
+ actor: actorArg,
149
+ store: storeNameOption,
150
+ filter: authorFilterOption,
151
+ includePins: includePinsOption,
152
+ postFilter: postFilterOption,
153
+ postFilterJson: postFilterJsonOption,
154
+ interval: intervalOption,
155
+ quiet: quietOption,
156
+ refresh: refreshOption
157
+ },
158
+ ({ actor, filter, includePins, postFilter, postFilterJson, interval, store, quiet, refresh }) =>
159
+ Effect.gen(function* () {
160
+ const resolvedActor = yield* decodeActor(actor);
161
+ const apiFilter = Option.getOrUndefined(filter);
162
+ const source = DataSource.author(resolvedActor, {
163
+ ...(apiFilter !== undefined ? { filter: apiFilter } : {}),
164
+ ...(includePins ? { includePins: true } : {})
165
+ });
166
+ const run = makeWatchCommandBody("author", () => source, {
167
+ actor: resolvedActor,
168
+ ...(apiFilter !== undefined ? { filter: apiFilter } : {}),
169
+ ...(includePins ? { includePins: true } : {})
170
+ });
171
+ return yield* run({
172
+ store,
173
+ filter: postFilter,
174
+ filterJson: postFilterJson,
175
+ interval,
176
+ quiet,
177
+ refresh
178
+ });
179
+ })
180
+ ).pipe(
181
+ Command.withDescription(
182
+ withExamples(
183
+ "Watch an author's feed and emit sync results",
184
+ [
185
+ "skygent watch author alice.bsky.social --store my-store",
186
+ "skygent watch author did:plc:example --store my-store --filter posts_no_replies --include-pins"
187
+ ],
188
+ ["Tip: use --post-filter to apply the DSL filter to synced posts."]
189
+ )
190
+ )
191
+ );
192
+
193
+ const threadCommand = Command.make(
194
+ "thread",
195
+ {
196
+ uri: postUriArg,
197
+ store: storeNameOption,
198
+ depth: depthOption,
199
+ parentHeight: parentHeightOption,
200
+ filter: filterOption,
201
+ filterJson: filterJsonOption,
202
+ interval: intervalOption,
203
+ quiet: quietOption,
204
+ refresh: refreshOption
205
+ },
206
+ ({ uri, depth, parentHeight, filter, filterJson, interval, store, quiet, refresh }) =>
207
+ Effect.gen(function* () {
208
+ const parsedDepth = yield* parseBoundedIntOption(depth, "depth", 0, 1000);
209
+ const parsedParentHeight = yield* parseBoundedIntOption(
210
+ parentHeight,
211
+ "parent-height",
212
+ 0,
213
+ 1000
214
+ );
215
+ const depthValue = Option.getOrUndefined(parsedDepth);
216
+ const parentHeightValue = Option.getOrUndefined(parsedParentHeight);
217
+ const source = DataSource.thread(uri, {
218
+ ...(depthValue !== undefined ? { depth: depthValue } : {}),
219
+ ...(parentHeightValue !== undefined ? { parentHeight: parentHeightValue } : {})
220
+ });
221
+ const run = makeWatchCommandBody("thread", () => source, {
222
+ uri,
223
+ ...(depthValue !== undefined ? { depth: depthValue } : {}),
224
+ ...(parentHeightValue !== undefined ? { parentHeight: parentHeightValue } : {})
225
+ });
226
+ return yield* run({ store, filter, filterJson, interval, quiet, refresh });
227
+ })
228
+ ).pipe(
229
+ Command.withDescription(
230
+ withExamples(
231
+ "Watch a thread and emit sync results",
232
+ [
233
+ "skygent watch thread at://did:plc:example/app.bsky.feed.post/xyz --store my-store",
234
+ "skygent watch thread at://did:plc:example/app.bsky.feed.post/xyz --store my-store --depth 10 --parent-height 5"
235
+ ],
236
+ ["Tip: use --filter to apply the DSL filter to thread posts."]
237
+ )
238
+ )
239
+ );
240
+
241
+ const jetstreamCommand = Command.make(
242
+ "jetstream",
243
+ {
244
+ store: storeNameOption,
245
+ filter: filterOption,
246
+ filterJson: filterJsonOption,
247
+ quiet: quietOption,
248
+ endpoint: jetstreamOptions.endpoint,
249
+ collections: jetstreamOptions.collections,
250
+ dids: jetstreamOptions.dids,
251
+ cursor: jetstreamOptions.cursor,
252
+ compress: jetstreamOptions.compress,
253
+ maxMessageSize: jetstreamOptions.maxMessageSize,
254
+ strict: strictOption,
255
+ maxErrors: maxErrorsOption
256
+ },
257
+ ({
258
+ store,
259
+ filter,
260
+ filterJson,
261
+ quiet,
262
+ endpoint,
263
+ collections,
264
+ dids,
265
+ cursor,
266
+ compress,
267
+ maxMessageSize,
268
+ strict,
269
+ maxErrors
270
+ }) =>
271
+ Effect.gen(function* () {
272
+ const monitor = yield* ResourceMonitor;
273
+ const output = yield* CliOutput;
274
+ const storeRef = yield* storeOptions.loadStoreRef(store);
275
+ const expr = yield* parseFilterExpr(filter, filterJson);
276
+ const filterHash = filterExprSignature(expr);
277
+ const selection = yield* buildJetstreamSelection(
278
+ {
279
+ endpoint,
280
+ collections,
281
+ dids,
282
+ cursor,
283
+ compress,
284
+ maxMessageSize
285
+ },
286
+ storeRef,
287
+ filterHash
288
+ );
289
+ const parsedMaxErrors = yield* parseMaxErrors(maxErrors);
290
+ const engineLayer = JetstreamSyncEngine.layer.pipe(
291
+ Layer.provideMerge(Jetstream.live(selection.config))
292
+ );
293
+ yield* logInfo("Starting watch", { source: "jetstream", store: storeRef.name });
294
+ yield* Effect.gen(function* () {
295
+ const engine = yield* JetstreamSyncEngine;
296
+ const maxErrorsValue = Option.getOrUndefined(parsedMaxErrors);
297
+ const stream = engine.watch({
298
+ source: selection.source,
299
+ store: storeRef,
300
+ filter: expr,
301
+ command: "watch jetstream",
302
+ ...(selection.cursor !== undefined ? { cursor: selection.cursor } : {}),
303
+ ...(strict ? { strict } : {}),
304
+ ...(maxErrorsValue !== undefined ? { maxErrors: maxErrorsValue } : {})
305
+ });
306
+ const outputStream = stream.pipe(
307
+ Stream.map((event) => event.result),
308
+ Stream.provideService(
309
+ SyncReporter,
310
+ makeSyncReporter(quiet, monitor, output)
311
+ )
312
+ );
313
+ return yield* writeJsonStream(outputStream);
314
+ }).pipe(Effect.provide(engineLayer));
315
+ })
316
+ ).pipe(
317
+ Command.withDescription(
318
+ withExamples(
319
+ "Watch Jetstream updates and emit sync results (posts only)",
320
+ [
321
+ "skygent watch jetstream --store my-store",
322
+ "skygent watch jetstream --store my-store --quiet"
323
+ ],
324
+ ["Tip: use --collections to override subscribed collections."]
325
+ )
326
+ )
327
+ );
328
+
329
+ export const watchCommand = Command.make("watch", {}).pipe(
330
+ Command.withSubcommands([
331
+ timelineCommand,
332
+ feedCommand,
333
+ listCommand,
334
+ notificationsCommand,
335
+ authorCommand,
336
+ threadCommand,
337
+ jetstreamCommand
338
+ ]),
339
+ Command.withDescription(
340
+ withExamples("Continuously sync and emit results", [
341
+ "skygent watch timeline --store my-store --interval \"2 minutes\""
342
+ ])
343
+ )
344
+ );
@@ -0,0 +1,14 @@
1
+ import * as SqlClient from "@effect/sql/SqlClient";
2
+ import { Effect } from "effect";
3
+
4
+ export default Effect.gen(function* () {
5
+ const sql = yield* SqlClient.SqlClient;
6
+
7
+ yield* sql`CREATE TABLE IF NOT EXISTS stores (
8
+ name TEXT PRIMARY KEY,
9
+ root TEXT NOT NULL,
10
+ created_at TEXT NOT NULL,
11
+ updated_at TEXT NOT NULL,
12
+ config_json TEXT NOT NULL
13
+ )`;
14
+ });
@@ -0,0 +1,34 @@
1
+ import * as SqlClient from "@effect/sql/SqlClient";
2
+ import { Effect } from "effect";
3
+
4
+ export default Effect.gen(function* () {
5
+ const sql = yield* SqlClient.SqlClient;
6
+
7
+ yield* sql`CREATE TABLE IF NOT EXISTS posts (
8
+ uri TEXT PRIMARY KEY,
9
+ created_at TEXT NOT NULL,
10
+ created_date TEXT NOT NULL,
11
+ author TEXT,
12
+ post_json TEXT NOT NULL
13
+ )`;
14
+
15
+ yield* sql`CREATE TABLE IF NOT EXISTS post_hashtag (
16
+ uri TEXT NOT NULL,
17
+ tag TEXT NOT NULL,
18
+ PRIMARY KEY (uri, tag),
19
+ FOREIGN KEY (uri) REFERENCES posts(uri) ON DELETE CASCADE
20
+ )`;
21
+
22
+ yield* sql`CREATE TABLE IF NOT EXISTS index_checkpoints (
23
+ index_name TEXT PRIMARY KEY,
24
+ version INTEGER NOT NULL,
25
+ last_event_id TEXT NOT NULL,
26
+ event_count INTEGER NOT NULL,
27
+ updated_at TEXT NOT NULL
28
+ )`;
29
+
30
+ yield* sql`CREATE INDEX IF NOT EXISTS posts_created_date_idx ON posts(created_date)`;
31
+ yield* sql`CREATE INDEX IF NOT EXISTS posts_created_at_idx ON posts(created_at)`;
32
+ yield* sql`CREATE INDEX IF NOT EXISTS posts_author_idx ON posts(author)`;
33
+ yield* sql`CREATE INDEX IF NOT EXISTS post_hashtag_tag_idx ON post_hashtag(tag)`;
34
+ });
@@ -0,0 +1,24 @@
1
+ import * as SqlClient from "@effect/sql/SqlClient";
2
+ import { Effect } from "effect";
3
+
4
+ export default Effect.gen(function* () {
5
+ const sql = yield* SqlClient.SqlClient;
6
+
7
+ yield* sql`CREATE TABLE IF NOT EXISTS event_log (
8
+ event_id TEXT PRIMARY KEY,
9
+ event_type TEXT NOT NULL,
10
+ post_uri TEXT NOT NULL,
11
+ payload_json TEXT NOT NULL,
12
+ created_at TEXT NOT NULL,
13
+ source TEXT NOT NULL
14
+ )`;
15
+
16
+ yield* sql`CREATE TABLE IF NOT EXISTS event_log_meta (
17
+ key TEXT PRIMARY KEY,
18
+ value TEXT NOT NULL
19
+ )`;
20
+
21
+ yield* sql`CREATE INDEX IF NOT EXISTS event_log_created_at_idx ON event_log(created_at)`;
22
+ yield* sql`CREATE INDEX IF NOT EXISTS event_log_post_uri_idx ON event_log(post_uri)`;
23
+ yield* sql`CREATE INDEX IF NOT EXISTS event_log_source_idx ON event_log(source)`;
24
+ });
@@ -0,0 +1,52 @@
1
+ import * as SqlClient from "@effect/sql/SqlClient";
2
+ import { Effect } from "effect";
3
+
4
+ export default Effect.gen(function* () {
5
+ const sql = yield* SqlClient.SqlClient;
6
+
7
+ yield* sql`ALTER TABLE posts ADD COLUMN text TEXT NOT NULL DEFAULT ''`;
8
+ yield* sql`ALTER TABLE posts ADD COLUMN lang TEXT`;
9
+ yield* sql`ALTER TABLE posts ADD COLUMN is_reply INTEGER NOT NULL DEFAULT 0`;
10
+ yield* sql`ALTER TABLE posts ADD COLUMN is_quote INTEGER NOT NULL DEFAULT 0`;
11
+ yield* sql`ALTER TABLE posts ADD COLUMN is_repost INTEGER NOT NULL DEFAULT 0`;
12
+ yield* sql`ALTER TABLE posts ADD COLUMN is_original INTEGER NOT NULL DEFAULT 0`;
13
+ yield* sql`ALTER TABLE posts ADD COLUMN has_links INTEGER NOT NULL DEFAULT 0`;
14
+ yield* sql`ALTER TABLE posts ADD COLUMN has_media INTEGER NOT NULL DEFAULT 0`;
15
+ yield* sql`ALTER TABLE posts ADD COLUMN has_images INTEGER NOT NULL DEFAULT 0`;
16
+ yield* sql`ALTER TABLE posts ADD COLUMN has_video INTEGER NOT NULL DEFAULT 0`;
17
+ yield* sql`ALTER TABLE posts ADD COLUMN like_count INTEGER NOT NULL DEFAULT 0`;
18
+ yield* sql`ALTER TABLE posts ADD COLUMN repost_count INTEGER NOT NULL DEFAULT 0`;
19
+ yield* sql`ALTER TABLE posts ADD COLUMN reply_count INTEGER NOT NULL DEFAULT 0`;
20
+
21
+ yield* sql`CREATE VIRTUAL TABLE IF NOT EXISTS posts_fts USING fts5(
22
+ text,
23
+ content='posts',
24
+ content_rowid='rowid'
25
+ )`;
26
+
27
+ yield* sql`CREATE TRIGGER IF NOT EXISTS posts_ai AFTER INSERT ON posts BEGIN
28
+ INSERT INTO posts_fts(rowid, text) VALUES (new.rowid, new.text);
29
+ END`;
30
+
31
+ yield* sql`CREATE TRIGGER IF NOT EXISTS posts_ad AFTER DELETE ON posts BEGIN
32
+ INSERT INTO posts_fts(posts_fts, rowid, text) VALUES ('delete', old.rowid, old.text);
33
+ END`;
34
+
35
+ yield* sql`CREATE TRIGGER IF NOT EXISTS posts_au AFTER UPDATE ON posts BEGIN
36
+ INSERT INTO posts_fts(posts_fts, rowid, text) VALUES ('delete', old.rowid, old.text);
37
+ INSERT INTO posts_fts(rowid, text) VALUES (new.rowid, new.text);
38
+ END`;
39
+
40
+ yield* sql`CREATE INDEX IF NOT EXISTS posts_lang_idx ON posts(lang)`;
41
+ yield* sql`CREATE INDEX IF NOT EXISTS posts_is_reply_idx ON posts(is_reply)`;
42
+ yield* sql`CREATE INDEX IF NOT EXISTS posts_is_quote_idx ON posts(is_quote)`;
43
+ yield* sql`CREATE INDEX IF NOT EXISTS posts_is_repost_idx ON posts(is_repost)`;
44
+ yield* sql`CREATE INDEX IF NOT EXISTS posts_is_original_idx ON posts(is_original)`;
45
+ yield* sql`CREATE INDEX IF NOT EXISTS posts_has_links_idx ON posts(has_links)`;
46
+ yield* sql`CREATE INDEX IF NOT EXISTS posts_has_media_idx ON posts(has_media)`;
47
+ yield* sql`CREATE INDEX IF NOT EXISTS posts_has_images_idx ON posts(has_images)`;
48
+ yield* sql`CREATE INDEX IF NOT EXISTS posts_has_video_idx ON posts(has_video)`;
49
+ yield* sql`CREATE INDEX IF NOT EXISTS posts_like_count_idx ON posts(like_count)`;
50
+ yield* sql`CREATE INDEX IF NOT EXISTS posts_repost_count_idx ON posts(repost_count)`;
51
+ yield* sql`CREATE INDEX IF NOT EXISTS posts_reply_count_idx ON posts(reply_count)`;
52
+ });
@@ -0,0 +1,9 @@
1
+ import * as SqlClient from "@effect/sql/SqlClient";
2
+ import { Effect } from "effect";
3
+
4
+ export default Effect.gen(function* () {
5
+ const sql = yield* SqlClient.SqlClient;
6
+
7
+ yield* sql`CREATE INDEX IF NOT EXISTS posts_author_created_at_idx ON posts(author, created_at)`;
8
+ yield* sql`CREATE INDEX IF NOT EXISTS posts_created_at_uri_idx ON posts(created_at, uri)`;
9
+ });
@@ -0,0 +1,15 @@
1
+ import * as SqlClient from "@effect/sql/SqlClient";
2
+ import { Effect } from "effect";
3
+
4
+ export default Effect.gen(function* () {
5
+ const sql = yield* SqlClient.SqlClient;
6
+
7
+ yield* sql`CREATE TABLE IF NOT EXISTS post_lang (
8
+ uri TEXT NOT NULL,
9
+ lang TEXT NOT NULL,
10
+ PRIMARY KEY (uri, lang),
11
+ FOREIGN KEY (uri) REFERENCES posts(uri) ON DELETE CASCADE
12
+ )`;
13
+
14
+ yield* sql`CREATE INDEX IF NOT EXISTS post_lang_lang_uri_idx ON post_lang(lang, uri)`;
15
+ });
@@ -0,0 +1,10 @@
1
+ import * as SqlClient from "@effect/sql/SqlClient";
2
+ import { Effect } from "effect";
3
+
4
+ export default Effect.gen(function* () {
5
+ const sql = yield* SqlClient.SqlClient;
6
+
7
+ yield* sql`ALTER TABLE posts ADD COLUMN has_embed INTEGER NOT NULL DEFAULT 0`;
8
+ yield* sql`UPDATE posts SET has_embed = CASE WHEN has_media = 1 OR is_quote = 1 THEN 1 ELSE 0 END`;
9
+ yield* sql`CREATE INDEX IF NOT EXISTS posts_has_embed_idx ON posts(has_embed)`;
10
+ });
@@ -0,0 +1,68 @@
1
+ import * as SqlClient from "@effect/sql/SqlClient";
2
+ import { Effect } from "effect";
3
+
4
+ export default Effect.gen(function* () {
5
+ const sql = yield* SqlClient.SqlClient;
6
+
7
+ yield* sql`DROP TABLE IF EXISTS event_log_new`;
8
+ yield* sql`CREATE TABLE IF NOT EXISTS event_log_new (
9
+ event_seq INTEGER PRIMARY KEY AUTOINCREMENT,
10
+ event_id TEXT NOT NULL UNIQUE,
11
+ event_type TEXT NOT NULL,
12
+ post_uri TEXT NOT NULL,
13
+ payload_json TEXT NOT NULL,
14
+ created_at TEXT NOT NULL,
15
+ source TEXT NOT NULL
16
+ )`;
17
+ yield* sql`INSERT INTO event_log_new (event_seq, event_id, event_type, post_uri, payload_json, created_at, source)
18
+ SELECT rowid, event_id, event_type, post_uri, payload_json, created_at, source
19
+ FROM event_log
20
+ ORDER BY rowid`;
21
+ yield* sql`DROP TABLE event_log`;
22
+ yield* sql`ALTER TABLE event_log_new RENAME TO event_log`;
23
+ yield* sql`CREATE INDEX IF NOT EXISTS event_log_created_at_idx ON event_log(created_at)`;
24
+ yield* sql`CREATE INDEX IF NOT EXISTS event_log_post_uri_idx ON event_log(post_uri)`;
25
+ yield* sql`CREATE INDEX IF NOT EXISTS event_log_source_idx ON event_log(source)`;
26
+
27
+ yield* sql`ALTER TABLE index_checkpoints RENAME TO index_checkpoints_old`;
28
+ yield* sql`CREATE TABLE IF NOT EXISTS index_checkpoints (
29
+ index_name TEXT PRIMARY KEY,
30
+ version INTEGER NOT NULL,
31
+ last_event_seq INTEGER NOT NULL,
32
+ event_count INTEGER NOT NULL,
33
+ updated_at TEXT NOT NULL
34
+ )`;
35
+ yield* sql`INSERT INTO index_checkpoints (index_name, version, last_event_seq, event_count, updated_at)
36
+ SELECT index_name,
37
+ version,
38
+ COALESCE((SELECT event_seq FROM event_log WHERE event_id = index_checkpoints_old.last_event_id), 0),
39
+ event_count,
40
+ updated_at
41
+ FROM index_checkpoints_old`;
42
+ yield* sql`DROP TABLE index_checkpoints_old`;
43
+
44
+ yield* sql`CREATE TABLE IF NOT EXISTS sync_checkpoints (
45
+ source_key TEXT PRIMARY KEY,
46
+ source_json TEXT NOT NULL,
47
+ cursor TEXT,
48
+ last_event_seq INTEGER,
49
+ filter_hash TEXT,
50
+ updated_at TEXT NOT NULL
51
+ )`;
52
+ yield* sql`CREATE INDEX IF NOT EXISTS sync_checkpoints_updated_at_idx ON sync_checkpoints(updated_at)`;
53
+
54
+ yield* sql`CREATE TABLE IF NOT EXISTS derivation_checkpoints (
55
+ view_name TEXT NOT NULL,
56
+ source_store TEXT NOT NULL,
57
+ target_store TEXT NOT NULL,
58
+ filter_hash TEXT NOT NULL,
59
+ evaluation_mode TEXT NOT NULL,
60
+ last_source_event_seq INTEGER,
61
+ events_processed INTEGER NOT NULL,
62
+ events_matched INTEGER NOT NULL,
63
+ deletes_propagated INTEGER NOT NULL,
64
+ updated_at TEXT NOT NULL,
65
+ PRIMARY KEY (view_name, source_store)
66
+ )`;
67
+ yield* sql`CREATE INDEX IF NOT EXISTS derivation_checkpoints_updated_at_idx ON derivation_checkpoints(updated_at)`;
68
+ });