@mepuka/skygent 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -0
- package/index.ts +146 -0
- package/package.json +56 -0
- package/src/cli/app.ts +75 -0
- package/src/cli/config-command.ts +140 -0
- package/src/cli/config.ts +91 -0
- package/src/cli/derive.ts +205 -0
- package/src/cli/doc/annotation.ts +36 -0
- package/src/cli/doc/filter.ts +69 -0
- package/src/cli/doc/index.ts +9 -0
- package/src/cli/doc/post.ts +155 -0
- package/src/cli/doc/primitives.ts +25 -0
- package/src/cli/doc/render.ts +18 -0
- package/src/cli/doc/table.ts +114 -0
- package/src/cli/doc/thread.ts +46 -0
- package/src/cli/doc/tree.ts +126 -0
- package/src/cli/errors.ts +59 -0
- package/src/cli/exit-codes.ts +52 -0
- package/src/cli/feed.ts +177 -0
- package/src/cli/filter-dsl.ts +1411 -0
- package/src/cli/filter-errors.ts +208 -0
- package/src/cli/filter-help.ts +70 -0
- package/src/cli/filter-input.ts +54 -0
- package/src/cli/filter.ts +435 -0
- package/src/cli/graph.ts +472 -0
- package/src/cli/help.ts +14 -0
- package/src/cli/interval.ts +35 -0
- package/src/cli/jetstream.ts +173 -0
- package/src/cli/layers.ts +180 -0
- package/src/cli/logging.ts +136 -0
- package/src/cli/output-format.ts +26 -0
- package/src/cli/output.ts +82 -0
- package/src/cli/parse.ts +80 -0
- package/src/cli/post.ts +193 -0
- package/src/cli/preferences.ts +11 -0
- package/src/cli/query-fields.ts +247 -0
- package/src/cli/query.ts +415 -0
- package/src/cli/range.ts +44 -0
- package/src/cli/search.ts +465 -0
- package/src/cli/shared-options.ts +169 -0
- package/src/cli/shared.ts +20 -0
- package/src/cli/store-errors.ts +80 -0
- package/src/cli/store-tree.ts +392 -0
- package/src/cli/store.ts +395 -0
- package/src/cli/sync-factory.ts +107 -0
- package/src/cli/sync.ts +366 -0
- package/src/cli/view-thread.ts +196 -0
- package/src/cli/view.ts +47 -0
- package/src/cli/watch.ts +344 -0
- package/src/db/migrations/store-catalog/001_init.ts +14 -0
- package/src/db/migrations/store-index/001_init.ts +34 -0
- package/src/db/migrations/store-index/002_event_log.ts +24 -0
- package/src/db/migrations/store-index/003_fts_and_derived.ts +52 -0
- package/src/db/migrations/store-index/004_query_indexes.ts +9 -0
- package/src/db/migrations/store-index/005_post_lang.ts +15 -0
- package/src/db/migrations/store-index/006_has_embed.ts +10 -0
- package/src/db/migrations/store-index/007_event_seq_and_checkpoints.ts +68 -0
- package/src/domain/bsky.ts +467 -0
- package/src/domain/config.ts +11 -0
- package/src/domain/credentials.ts +6 -0
- package/src/domain/defaults.ts +8 -0
- package/src/domain/derivation.ts +55 -0
- package/src/domain/errors.ts +71 -0
- package/src/domain/events.ts +55 -0
- package/src/domain/extract.ts +64 -0
- package/src/domain/filter-describe.ts +551 -0
- package/src/domain/filter-explain.ts +9 -0
- package/src/domain/filter.ts +797 -0
- package/src/domain/format.ts +91 -0
- package/src/domain/index.ts +13 -0
- package/src/domain/indexes.ts +17 -0
- package/src/domain/policies.ts +16 -0
- package/src/domain/post.ts +88 -0
- package/src/domain/primitives.ts +50 -0
- package/src/domain/raw.ts +140 -0
- package/src/domain/store.ts +103 -0
- package/src/domain/sync.ts +211 -0
- package/src/domain/text-width.ts +56 -0
- package/src/services/app-config.ts +278 -0
- package/src/services/bsky-client.ts +2113 -0
- package/src/services/credential-store.ts +408 -0
- package/src/services/derivation-engine.ts +502 -0
- package/src/services/derivation-settings.ts +61 -0
- package/src/services/derivation-validator.ts +68 -0
- package/src/services/filter-compiler.ts +269 -0
- package/src/services/filter-library.ts +371 -0
- package/src/services/filter-runtime.ts +821 -0
- package/src/services/filter-settings.ts +30 -0
- package/src/services/identity-resolver.ts +563 -0
- package/src/services/jetstream-sync.ts +636 -0
- package/src/services/lineage-store.ts +89 -0
- package/src/services/link-validator.ts +244 -0
- package/src/services/output-manager.ts +274 -0
- package/src/services/post-parser.ts +62 -0
- package/src/services/profile-resolver.ts +223 -0
- package/src/services/resource-monitor.ts +106 -0
- package/src/services/shared.ts +69 -0
- package/src/services/store-cleaner.ts +43 -0
- package/src/services/store-commit.ts +168 -0
- package/src/services/store-db.ts +248 -0
- package/src/services/store-event-log.ts +285 -0
- package/src/services/store-index-sql.ts +289 -0
- package/src/services/store-index.ts +1152 -0
- package/src/services/store-keys.ts +4 -0
- package/src/services/store-manager.ts +358 -0
- package/src/services/store-stats.ts +522 -0
- package/src/services/store-writer.ts +200 -0
- package/src/services/sync-checkpoint-store.ts +169 -0
- package/src/services/sync-engine.ts +547 -0
- package/src/services/sync-reporter.ts +16 -0
- package/src/services/sync-settings.ts +72 -0
- package/src/services/trending-topics.ts +226 -0
- package/src/services/view-checkpoint-store.ts +238 -0
- package/src/typeclass/chunk.ts +84 -0
package/src/cli/watch.ts
ADDED
|
@@ -0,0 +1,344 @@
|
|
|
1
|
+
import { Command, Options } from "@effect/cli";
|
|
2
|
+
import { Effect, Layer, Option, Stream } from "effect";
|
|
3
|
+
import { Jetstream } from "effect-jetstream";
|
|
4
|
+
import { filterExprSignature } from "../domain/filter.js";
|
|
5
|
+
import { DataSource } from "../domain/sync.js";
|
|
6
|
+
import { SyncReporter } from "../services/sync-reporter.js";
|
|
7
|
+
import { JetstreamSyncEngine } from "../services/jetstream-sync.js";
|
|
8
|
+
import { parseFilterExpr } from "./filter-input.js";
|
|
9
|
+
import { CliOutput, writeJsonStream } from "./output.js";
|
|
10
|
+
import { storeOptions } from "./store.js";
|
|
11
|
+
import { logInfo, makeSyncReporter } from "./logging.js";
|
|
12
|
+
import { ResourceMonitor } from "../services/resource-monitor.js";
|
|
13
|
+
import { withExamples } from "./help.js";
|
|
14
|
+
import { buildJetstreamSelection, jetstreamOptions } from "./jetstream.js";
|
|
15
|
+
import { makeWatchCommandBody } from "./sync-factory.js";
|
|
16
|
+
import {
|
|
17
|
+
feedUriArg,
|
|
18
|
+
listUriArg,
|
|
19
|
+
postUriArg,
|
|
20
|
+
actorArg,
|
|
21
|
+
storeNameOption,
|
|
22
|
+
filterOption,
|
|
23
|
+
filterJsonOption,
|
|
24
|
+
postFilterOption,
|
|
25
|
+
postFilterJsonOption,
|
|
26
|
+
authorFilterOption,
|
|
27
|
+
includePinsOption,
|
|
28
|
+
decodeActor,
|
|
29
|
+
quietOption,
|
|
30
|
+
refreshOption,
|
|
31
|
+
strictOption,
|
|
32
|
+
maxErrorsOption,
|
|
33
|
+
parseMaxErrors,
|
|
34
|
+
parseBoundedIntOption
|
|
35
|
+
} from "./shared-options.js";
|
|
36
|
+
|
|
37
|
+
const intervalOption = Options.text("interval").pipe(
|
|
38
|
+
Options.withDescription(
|
|
39
|
+
"Polling interval (e.g. \"30 seconds\", \"500 millis\") (default: 30 seconds)"
|
|
40
|
+
),
|
|
41
|
+
Options.optional
|
|
42
|
+
);
|
|
43
|
+
const depthOption = Options.integer("depth").pipe(
|
|
44
|
+
Options.withDescription("Thread reply depth to include (0-1000, default 6)"),
|
|
45
|
+
Options.optional
|
|
46
|
+
);
|
|
47
|
+
const parentHeightOption = Options.integer("parent-height").pipe(
|
|
48
|
+
Options.withDescription("Thread parent height to include (0-1000, default 80)"),
|
|
49
|
+
Options.optional
|
|
50
|
+
);
|
|
51
|
+
|
|
52
|
+
const timelineCommand = Command.make(
|
|
53
|
+
"timeline",
|
|
54
|
+
{
|
|
55
|
+
store: storeNameOption,
|
|
56
|
+
filter: filterOption,
|
|
57
|
+
filterJson: filterJsonOption,
|
|
58
|
+
interval: intervalOption,
|
|
59
|
+
quiet: quietOption,
|
|
60
|
+
refresh: refreshOption
|
|
61
|
+
},
|
|
62
|
+
makeWatchCommandBody("timeline", () => DataSource.timeline())
|
|
63
|
+
).pipe(
|
|
64
|
+
Command.withDescription(
|
|
65
|
+
withExamples(
|
|
66
|
+
"Watch timeline updates and emit sync results",
|
|
67
|
+
[
|
|
68
|
+
"skygent watch timeline --store my-store",
|
|
69
|
+
"skygent watch timeline --store my-store --interval \"5 minutes\" --quiet"
|
|
70
|
+
],
|
|
71
|
+
["Tip: add --quiet to suppress progress logs."]
|
|
72
|
+
)
|
|
73
|
+
)
|
|
74
|
+
);
|
|
75
|
+
|
|
76
|
+
const feedCommand = Command.make(
|
|
77
|
+
"feed",
|
|
78
|
+
{
|
|
79
|
+
uri: feedUriArg,
|
|
80
|
+
store: storeNameOption,
|
|
81
|
+
filter: filterOption,
|
|
82
|
+
filterJson: filterJsonOption,
|
|
83
|
+
interval: intervalOption,
|
|
84
|
+
quiet: quietOption,
|
|
85
|
+
refresh: refreshOption
|
|
86
|
+
},
|
|
87
|
+
({ uri, ...rest }) => makeWatchCommandBody("feed", () => DataSource.feed(uri), { uri })(rest)
|
|
88
|
+
).pipe(
|
|
89
|
+
Command.withDescription(
|
|
90
|
+
withExamples(
|
|
91
|
+
"Watch a feed URI and emit sync results",
|
|
92
|
+
[
|
|
93
|
+
"skygent watch feed at://did:plc:example/app.bsky.feed.generator/xyz --store my-store --interval \"2 minutes\""
|
|
94
|
+
],
|
|
95
|
+
["Tip: add --quiet to suppress progress logs."]
|
|
96
|
+
)
|
|
97
|
+
)
|
|
98
|
+
);
|
|
99
|
+
|
|
100
|
+
const listCommand = Command.make(
|
|
101
|
+
"list",
|
|
102
|
+
{
|
|
103
|
+
uri: listUriArg,
|
|
104
|
+
store: storeNameOption,
|
|
105
|
+
filter: filterOption,
|
|
106
|
+
filterJson: filterJsonOption,
|
|
107
|
+
interval: intervalOption,
|
|
108
|
+
quiet: quietOption,
|
|
109
|
+
refresh: refreshOption
|
|
110
|
+
},
|
|
111
|
+
({ uri, ...rest }) => makeWatchCommandBody("list", () => DataSource.list(uri), { uri })(rest)
|
|
112
|
+
).pipe(
|
|
113
|
+
Command.withDescription(
|
|
114
|
+
withExamples(
|
|
115
|
+
"Watch a list feed URI and emit sync results",
|
|
116
|
+
[
|
|
117
|
+
"skygent watch list at://did:plc:example/app.bsky.graph.list/xyz --store my-store --interval \"2 minutes\""
|
|
118
|
+
],
|
|
119
|
+
["Tip: add --quiet to suppress progress logs."]
|
|
120
|
+
)
|
|
121
|
+
)
|
|
122
|
+
);
|
|
123
|
+
|
|
124
|
+
const notificationsCommand = Command.make(
|
|
125
|
+
"notifications",
|
|
126
|
+
{
|
|
127
|
+
store: storeNameOption,
|
|
128
|
+
filter: filterOption,
|
|
129
|
+
filterJson: filterJsonOption,
|
|
130
|
+
interval: intervalOption,
|
|
131
|
+
quiet: quietOption,
|
|
132
|
+
refresh: refreshOption
|
|
133
|
+
},
|
|
134
|
+
makeWatchCommandBody("notifications", () => DataSource.notifications())
|
|
135
|
+
).pipe(
|
|
136
|
+
Command.withDescription(
|
|
137
|
+
withExamples(
|
|
138
|
+
"Watch notifications and emit sync results",
|
|
139
|
+
["skygent watch notifications --store my-store --interval \"1 minute\" --quiet"],
|
|
140
|
+
["Tip: add --quiet to suppress progress logs."]
|
|
141
|
+
)
|
|
142
|
+
)
|
|
143
|
+
);
|
|
144
|
+
|
|
145
|
+
const authorCommand = Command.make(
|
|
146
|
+
"author",
|
|
147
|
+
{
|
|
148
|
+
actor: actorArg,
|
|
149
|
+
store: storeNameOption,
|
|
150
|
+
filter: authorFilterOption,
|
|
151
|
+
includePins: includePinsOption,
|
|
152
|
+
postFilter: postFilterOption,
|
|
153
|
+
postFilterJson: postFilterJsonOption,
|
|
154
|
+
interval: intervalOption,
|
|
155
|
+
quiet: quietOption,
|
|
156
|
+
refresh: refreshOption
|
|
157
|
+
},
|
|
158
|
+
({ actor, filter, includePins, postFilter, postFilterJson, interval, store, quiet, refresh }) =>
|
|
159
|
+
Effect.gen(function* () {
|
|
160
|
+
const resolvedActor = yield* decodeActor(actor);
|
|
161
|
+
const apiFilter = Option.getOrUndefined(filter);
|
|
162
|
+
const source = DataSource.author(resolvedActor, {
|
|
163
|
+
...(apiFilter !== undefined ? { filter: apiFilter } : {}),
|
|
164
|
+
...(includePins ? { includePins: true } : {})
|
|
165
|
+
});
|
|
166
|
+
const run = makeWatchCommandBody("author", () => source, {
|
|
167
|
+
actor: resolvedActor,
|
|
168
|
+
...(apiFilter !== undefined ? { filter: apiFilter } : {}),
|
|
169
|
+
...(includePins ? { includePins: true } : {})
|
|
170
|
+
});
|
|
171
|
+
return yield* run({
|
|
172
|
+
store,
|
|
173
|
+
filter: postFilter,
|
|
174
|
+
filterJson: postFilterJson,
|
|
175
|
+
interval,
|
|
176
|
+
quiet,
|
|
177
|
+
refresh
|
|
178
|
+
});
|
|
179
|
+
})
|
|
180
|
+
).pipe(
|
|
181
|
+
Command.withDescription(
|
|
182
|
+
withExamples(
|
|
183
|
+
"Watch an author's feed and emit sync results",
|
|
184
|
+
[
|
|
185
|
+
"skygent watch author alice.bsky.social --store my-store",
|
|
186
|
+
"skygent watch author did:plc:example --store my-store --filter posts_no_replies --include-pins"
|
|
187
|
+
],
|
|
188
|
+
["Tip: use --post-filter to apply the DSL filter to synced posts."]
|
|
189
|
+
)
|
|
190
|
+
)
|
|
191
|
+
);
|
|
192
|
+
|
|
193
|
+
const threadCommand = Command.make(
|
|
194
|
+
"thread",
|
|
195
|
+
{
|
|
196
|
+
uri: postUriArg,
|
|
197
|
+
store: storeNameOption,
|
|
198
|
+
depth: depthOption,
|
|
199
|
+
parentHeight: parentHeightOption,
|
|
200
|
+
filter: filterOption,
|
|
201
|
+
filterJson: filterJsonOption,
|
|
202
|
+
interval: intervalOption,
|
|
203
|
+
quiet: quietOption,
|
|
204
|
+
refresh: refreshOption
|
|
205
|
+
},
|
|
206
|
+
({ uri, depth, parentHeight, filter, filterJson, interval, store, quiet, refresh }) =>
|
|
207
|
+
Effect.gen(function* () {
|
|
208
|
+
const parsedDepth = yield* parseBoundedIntOption(depth, "depth", 0, 1000);
|
|
209
|
+
const parsedParentHeight = yield* parseBoundedIntOption(
|
|
210
|
+
parentHeight,
|
|
211
|
+
"parent-height",
|
|
212
|
+
0,
|
|
213
|
+
1000
|
|
214
|
+
);
|
|
215
|
+
const depthValue = Option.getOrUndefined(parsedDepth);
|
|
216
|
+
const parentHeightValue = Option.getOrUndefined(parsedParentHeight);
|
|
217
|
+
const source = DataSource.thread(uri, {
|
|
218
|
+
...(depthValue !== undefined ? { depth: depthValue } : {}),
|
|
219
|
+
...(parentHeightValue !== undefined ? { parentHeight: parentHeightValue } : {})
|
|
220
|
+
});
|
|
221
|
+
const run = makeWatchCommandBody("thread", () => source, {
|
|
222
|
+
uri,
|
|
223
|
+
...(depthValue !== undefined ? { depth: depthValue } : {}),
|
|
224
|
+
...(parentHeightValue !== undefined ? { parentHeight: parentHeightValue } : {})
|
|
225
|
+
});
|
|
226
|
+
return yield* run({ store, filter, filterJson, interval, quiet, refresh });
|
|
227
|
+
})
|
|
228
|
+
).pipe(
|
|
229
|
+
Command.withDescription(
|
|
230
|
+
withExamples(
|
|
231
|
+
"Watch a thread and emit sync results",
|
|
232
|
+
[
|
|
233
|
+
"skygent watch thread at://did:plc:example/app.bsky.feed.post/xyz --store my-store",
|
|
234
|
+
"skygent watch thread at://did:plc:example/app.bsky.feed.post/xyz --store my-store --depth 10 --parent-height 5"
|
|
235
|
+
],
|
|
236
|
+
["Tip: use --filter to apply the DSL filter to thread posts."]
|
|
237
|
+
)
|
|
238
|
+
)
|
|
239
|
+
);
|
|
240
|
+
|
|
241
|
+
const jetstreamCommand = Command.make(
|
|
242
|
+
"jetstream",
|
|
243
|
+
{
|
|
244
|
+
store: storeNameOption,
|
|
245
|
+
filter: filterOption,
|
|
246
|
+
filterJson: filterJsonOption,
|
|
247
|
+
quiet: quietOption,
|
|
248
|
+
endpoint: jetstreamOptions.endpoint,
|
|
249
|
+
collections: jetstreamOptions.collections,
|
|
250
|
+
dids: jetstreamOptions.dids,
|
|
251
|
+
cursor: jetstreamOptions.cursor,
|
|
252
|
+
compress: jetstreamOptions.compress,
|
|
253
|
+
maxMessageSize: jetstreamOptions.maxMessageSize,
|
|
254
|
+
strict: strictOption,
|
|
255
|
+
maxErrors: maxErrorsOption
|
|
256
|
+
},
|
|
257
|
+
({
|
|
258
|
+
store,
|
|
259
|
+
filter,
|
|
260
|
+
filterJson,
|
|
261
|
+
quiet,
|
|
262
|
+
endpoint,
|
|
263
|
+
collections,
|
|
264
|
+
dids,
|
|
265
|
+
cursor,
|
|
266
|
+
compress,
|
|
267
|
+
maxMessageSize,
|
|
268
|
+
strict,
|
|
269
|
+
maxErrors
|
|
270
|
+
}) =>
|
|
271
|
+
Effect.gen(function* () {
|
|
272
|
+
const monitor = yield* ResourceMonitor;
|
|
273
|
+
const output = yield* CliOutput;
|
|
274
|
+
const storeRef = yield* storeOptions.loadStoreRef(store);
|
|
275
|
+
const expr = yield* parseFilterExpr(filter, filterJson);
|
|
276
|
+
const filterHash = filterExprSignature(expr);
|
|
277
|
+
const selection = yield* buildJetstreamSelection(
|
|
278
|
+
{
|
|
279
|
+
endpoint,
|
|
280
|
+
collections,
|
|
281
|
+
dids,
|
|
282
|
+
cursor,
|
|
283
|
+
compress,
|
|
284
|
+
maxMessageSize
|
|
285
|
+
},
|
|
286
|
+
storeRef,
|
|
287
|
+
filterHash
|
|
288
|
+
);
|
|
289
|
+
const parsedMaxErrors = yield* parseMaxErrors(maxErrors);
|
|
290
|
+
const engineLayer = JetstreamSyncEngine.layer.pipe(
|
|
291
|
+
Layer.provideMerge(Jetstream.live(selection.config))
|
|
292
|
+
);
|
|
293
|
+
yield* logInfo("Starting watch", { source: "jetstream", store: storeRef.name });
|
|
294
|
+
yield* Effect.gen(function* () {
|
|
295
|
+
const engine = yield* JetstreamSyncEngine;
|
|
296
|
+
const maxErrorsValue = Option.getOrUndefined(parsedMaxErrors);
|
|
297
|
+
const stream = engine.watch({
|
|
298
|
+
source: selection.source,
|
|
299
|
+
store: storeRef,
|
|
300
|
+
filter: expr,
|
|
301
|
+
command: "watch jetstream",
|
|
302
|
+
...(selection.cursor !== undefined ? { cursor: selection.cursor } : {}),
|
|
303
|
+
...(strict ? { strict } : {}),
|
|
304
|
+
...(maxErrorsValue !== undefined ? { maxErrors: maxErrorsValue } : {})
|
|
305
|
+
});
|
|
306
|
+
const outputStream = stream.pipe(
|
|
307
|
+
Stream.map((event) => event.result),
|
|
308
|
+
Stream.provideService(
|
|
309
|
+
SyncReporter,
|
|
310
|
+
makeSyncReporter(quiet, monitor, output)
|
|
311
|
+
)
|
|
312
|
+
);
|
|
313
|
+
return yield* writeJsonStream(outputStream);
|
|
314
|
+
}).pipe(Effect.provide(engineLayer));
|
|
315
|
+
})
|
|
316
|
+
).pipe(
|
|
317
|
+
Command.withDescription(
|
|
318
|
+
withExamples(
|
|
319
|
+
"Watch Jetstream updates and emit sync results (posts only)",
|
|
320
|
+
[
|
|
321
|
+
"skygent watch jetstream --store my-store",
|
|
322
|
+
"skygent watch jetstream --store my-store --quiet"
|
|
323
|
+
],
|
|
324
|
+
["Tip: use --collections to override subscribed collections."]
|
|
325
|
+
)
|
|
326
|
+
)
|
|
327
|
+
);
|
|
328
|
+
|
|
329
|
+
export const watchCommand = Command.make("watch", {}).pipe(
|
|
330
|
+
Command.withSubcommands([
|
|
331
|
+
timelineCommand,
|
|
332
|
+
feedCommand,
|
|
333
|
+
listCommand,
|
|
334
|
+
notificationsCommand,
|
|
335
|
+
authorCommand,
|
|
336
|
+
threadCommand,
|
|
337
|
+
jetstreamCommand
|
|
338
|
+
]),
|
|
339
|
+
Command.withDescription(
|
|
340
|
+
withExamples("Continuously sync and emit results", [
|
|
341
|
+
"skygent watch timeline --store my-store --interval \"2 minutes\""
|
|
342
|
+
])
|
|
343
|
+
)
|
|
344
|
+
);
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import * as SqlClient from "@effect/sql/SqlClient";
|
|
2
|
+
import { Effect } from "effect";
|
|
3
|
+
|
|
4
|
+
export default Effect.gen(function* () {
|
|
5
|
+
const sql = yield* SqlClient.SqlClient;
|
|
6
|
+
|
|
7
|
+
yield* sql`CREATE TABLE IF NOT EXISTS stores (
|
|
8
|
+
name TEXT PRIMARY KEY,
|
|
9
|
+
root TEXT NOT NULL,
|
|
10
|
+
created_at TEXT NOT NULL,
|
|
11
|
+
updated_at TEXT NOT NULL,
|
|
12
|
+
config_json TEXT NOT NULL
|
|
13
|
+
)`;
|
|
14
|
+
});
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import * as SqlClient from "@effect/sql/SqlClient";
|
|
2
|
+
import { Effect } from "effect";
|
|
3
|
+
|
|
4
|
+
export default Effect.gen(function* () {
|
|
5
|
+
const sql = yield* SqlClient.SqlClient;
|
|
6
|
+
|
|
7
|
+
yield* sql`CREATE TABLE IF NOT EXISTS posts (
|
|
8
|
+
uri TEXT PRIMARY KEY,
|
|
9
|
+
created_at TEXT NOT NULL,
|
|
10
|
+
created_date TEXT NOT NULL,
|
|
11
|
+
author TEXT,
|
|
12
|
+
post_json TEXT NOT NULL
|
|
13
|
+
)`;
|
|
14
|
+
|
|
15
|
+
yield* sql`CREATE TABLE IF NOT EXISTS post_hashtag (
|
|
16
|
+
uri TEXT NOT NULL,
|
|
17
|
+
tag TEXT NOT NULL,
|
|
18
|
+
PRIMARY KEY (uri, tag),
|
|
19
|
+
FOREIGN KEY (uri) REFERENCES posts(uri) ON DELETE CASCADE
|
|
20
|
+
)`;
|
|
21
|
+
|
|
22
|
+
yield* sql`CREATE TABLE IF NOT EXISTS index_checkpoints (
|
|
23
|
+
index_name TEXT PRIMARY KEY,
|
|
24
|
+
version INTEGER NOT NULL,
|
|
25
|
+
last_event_id TEXT NOT NULL,
|
|
26
|
+
event_count INTEGER NOT NULL,
|
|
27
|
+
updated_at TEXT NOT NULL
|
|
28
|
+
)`;
|
|
29
|
+
|
|
30
|
+
yield* sql`CREATE INDEX IF NOT EXISTS posts_created_date_idx ON posts(created_date)`;
|
|
31
|
+
yield* sql`CREATE INDEX IF NOT EXISTS posts_created_at_idx ON posts(created_at)`;
|
|
32
|
+
yield* sql`CREATE INDEX IF NOT EXISTS posts_author_idx ON posts(author)`;
|
|
33
|
+
yield* sql`CREATE INDEX IF NOT EXISTS post_hashtag_tag_idx ON post_hashtag(tag)`;
|
|
34
|
+
});
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import * as SqlClient from "@effect/sql/SqlClient";
|
|
2
|
+
import { Effect } from "effect";
|
|
3
|
+
|
|
4
|
+
export default Effect.gen(function* () {
|
|
5
|
+
const sql = yield* SqlClient.SqlClient;
|
|
6
|
+
|
|
7
|
+
yield* sql`CREATE TABLE IF NOT EXISTS event_log (
|
|
8
|
+
event_id TEXT PRIMARY KEY,
|
|
9
|
+
event_type TEXT NOT NULL,
|
|
10
|
+
post_uri TEXT NOT NULL,
|
|
11
|
+
payload_json TEXT NOT NULL,
|
|
12
|
+
created_at TEXT NOT NULL,
|
|
13
|
+
source TEXT NOT NULL
|
|
14
|
+
)`;
|
|
15
|
+
|
|
16
|
+
yield* sql`CREATE TABLE IF NOT EXISTS event_log_meta (
|
|
17
|
+
key TEXT PRIMARY KEY,
|
|
18
|
+
value TEXT NOT NULL
|
|
19
|
+
)`;
|
|
20
|
+
|
|
21
|
+
yield* sql`CREATE INDEX IF NOT EXISTS event_log_created_at_idx ON event_log(created_at)`;
|
|
22
|
+
yield* sql`CREATE INDEX IF NOT EXISTS event_log_post_uri_idx ON event_log(post_uri)`;
|
|
23
|
+
yield* sql`CREATE INDEX IF NOT EXISTS event_log_source_idx ON event_log(source)`;
|
|
24
|
+
});
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import * as SqlClient from "@effect/sql/SqlClient";
|
|
2
|
+
import { Effect } from "effect";
|
|
3
|
+
|
|
4
|
+
export default Effect.gen(function* () {
|
|
5
|
+
const sql = yield* SqlClient.SqlClient;
|
|
6
|
+
|
|
7
|
+
yield* sql`ALTER TABLE posts ADD COLUMN text TEXT NOT NULL DEFAULT ''`;
|
|
8
|
+
yield* sql`ALTER TABLE posts ADD COLUMN lang TEXT`;
|
|
9
|
+
yield* sql`ALTER TABLE posts ADD COLUMN is_reply INTEGER NOT NULL DEFAULT 0`;
|
|
10
|
+
yield* sql`ALTER TABLE posts ADD COLUMN is_quote INTEGER NOT NULL DEFAULT 0`;
|
|
11
|
+
yield* sql`ALTER TABLE posts ADD COLUMN is_repost INTEGER NOT NULL DEFAULT 0`;
|
|
12
|
+
yield* sql`ALTER TABLE posts ADD COLUMN is_original INTEGER NOT NULL DEFAULT 0`;
|
|
13
|
+
yield* sql`ALTER TABLE posts ADD COLUMN has_links INTEGER NOT NULL DEFAULT 0`;
|
|
14
|
+
yield* sql`ALTER TABLE posts ADD COLUMN has_media INTEGER NOT NULL DEFAULT 0`;
|
|
15
|
+
yield* sql`ALTER TABLE posts ADD COLUMN has_images INTEGER NOT NULL DEFAULT 0`;
|
|
16
|
+
yield* sql`ALTER TABLE posts ADD COLUMN has_video INTEGER NOT NULL DEFAULT 0`;
|
|
17
|
+
yield* sql`ALTER TABLE posts ADD COLUMN like_count INTEGER NOT NULL DEFAULT 0`;
|
|
18
|
+
yield* sql`ALTER TABLE posts ADD COLUMN repost_count INTEGER NOT NULL DEFAULT 0`;
|
|
19
|
+
yield* sql`ALTER TABLE posts ADD COLUMN reply_count INTEGER NOT NULL DEFAULT 0`;
|
|
20
|
+
|
|
21
|
+
yield* sql`CREATE VIRTUAL TABLE IF NOT EXISTS posts_fts USING fts5(
|
|
22
|
+
text,
|
|
23
|
+
content='posts',
|
|
24
|
+
content_rowid='rowid'
|
|
25
|
+
)`;
|
|
26
|
+
|
|
27
|
+
yield* sql`CREATE TRIGGER IF NOT EXISTS posts_ai AFTER INSERT ON posts BEGIN
|
|
28
|
+
INSERT INTO posts_fts(rowid, text) VALUES (new.rowid, new.text);
|
|
29
|
+
END`;
|
|
30
|
+
|
|
31
|
+
yield* sql`CREATE TRIGGER IF NOT EXISTS posts_ad AFTER DELETE ON posts BEGIN
|
|
32
|
+
INSERT INTO posts_fts(posts_fts, rowid, text) VALUES ('delete', old.rowid, old.text);
|
|
33
|
+
END`;
|
|
34
|
+
|
|
35
|
+
yield* sql`CREATE TRIGGER IF NOT EXISTS posts_au AFTER UPDATE ON posts BEGIN
|
|
36
|
+
INSERT INTO posts_fts(posts_fts, rowid, text) VALUES ('delete', old.rowid, old.text);
|
|
37
|
+
INSERT INTO posts_fts(rowid, text) VALUES (new.rowid, new.text);
|
|
38
|
+
END`;
|
|
39
|
+
|
|
40
|
+
yield* sql`CREATE INDEX IF NOT EXISTS posts_lang_idx ON posts(lang)`;
|
|
41
|
+
yield* sql`CREATE INDEX IF NOT EXISTS posts_is_reply_idx ON posts(is_reply)`;
|
|
42
|
+
yield* sql`CREATE INDEX IF NOT EXISTS posts_is_quote_idx ON posts(is_quote)`;
|
|
43
|
+
yield* sql`CREATE INDEX IF NOT EXISTS posts_is_repost_idx ON posts(is_repost)`;
|
|
44
|
+
yield* sql`CREATE INDEX IF NOT EXISTS posts_is_original_idx ON posts(is_original)`;
|
|
45
|
+
yield* sql`CREATE INDEX IF NOT EXISTS posts_has_links_idx ON posts(has_links)`;
|
|
46
|
+
yield* sql`CREATE INDEX IF NOT EXISTS posts_has_media_idx ON posts(has_media)`;
|
|
47
|
+
yield* sql`CREATE INDEX IF NOT EXISTS posts_has_images_idx ON posts(has_images)`;
|
|
48
|
+
yield* sql`CREATE INDEX IF NOT EXISTS posts_has_video_idx ON posts(has_video)`;
|
|
49
|
+
yield* sql`CREATE INDEX IF NOT EXISTS posts_like_count_idx ON posts(like_count)`;
|
|
50
|
+
yield* sql`CREATE INDEX IF NOT EXISTS posts_repost_count_idx ON posts(repost_count)`;
|
|
51
|
+
yield* sql`CREATE INDEX IF NOT EXISTS posts_reply_count_idx ON posts(reply_count)`;
|
|
52
|
+
});
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import * as SqlClient from "@effect/sql/SqlClient";
|
|
2
|
+
import { Effect } from "effect";
|
|
3
|
+
|
|
4
|
+
export default Effect.gen(function* () {
|
|
5
|
+
const sql = yield* SqlClient.SqlClient;
|
|
6
|
+
|
|
7
|
+
yield* sql`CREATE INDEX IF NOT EXISTS posts_author_created_at_idx ON posts(author, created_at)`;
|
|
8
|
+
yield* sql`CREATE INDEX IF NOT EXISTS posts_created_at_uri_idx ON posts(created_at, uri)`;
|
|
9
|
+
});
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import * as SqlClient from "@effect/sql/SqlClient";
|
|
2
|
+
import { Effect } from "effect";
|
|
3
|
+
|
|
4
|
+
export default Effect.gen(function* () {
|
|
5
|
+
const sql = yield* SqlClient.SqlClient;
|
|
6
|
+
|
|
7
|
+
yield* sql`CREATE TABLE IF NOT EXISTS post_lang (
|
|
8
|
+
uri TEXT NOT NULL,
|
|
9
|
+
lang TEXT NOT NULL,
|
|
10
|
+
PRIMARY KEY (uri, lang),
|
|
11
|
+
FOREIGN KEY (uri) REFERENCES posts(uri) ON DELETE CASCADE
|
|
12
|
+
)`;
|
|
13
|
+
|
|
14
|
+
yield* sql`CREATE INDEX IF NOT EXISTS post_lang_lang_uri_idx ON post_lang(lang, uri)`;
|
|
15
|
+
});
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import * as SqlClient from "@effect/sql/SqlClient";
|
|
2
|
+
import { Effect } from "effect";
|
|
3
|
+
|
|
4
|
+
export default Effect.gen(function* () {
|
|
5
|
+
const sql = yield* SqlClient.SqlClient;
|
|
6
|
+
|
|
7
|
+
yield* sql`ALTER TABLE posts ADD COLUMN has_embed INTEGER NOT NULL DEFAULT 0`;
|
|
8
|
+
yield* sql`UPDATE posts SET has_embed = CASE WHEN has_media = 1 OR is_quote = 1 THEN 1 ELSE 0 END`;
|
|
9
|
+
yield* sql`CREATE INDEX IF NOT EXISTS posts_has_embed_idx ON posts(has_embed)`;
|
|
10
|
+
});
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import * as SqlClient from "@effect/sql/SqlClient";
|
|
2
|
+
import { Effect } from "effect";
|
|
3
|
+
|
|
4
|
+
export default Effect.gen(function* () {
|
|
5
|
+
const sql = yield* SqlClient.SqlClient;
|
|
6
|
+
|
|
7
|
+
yield* sql`DROP TABLE IF EXISTS event_log_new`;
|
|
8
|
+
yield* sql`CREATE TABLE IF NOT EXISTS event_log_new (
|
|
9
|
+
event_seq INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
10
|
+
event_id TEXT NOT NULL UNIQUE,
|
|
11
|
+
event_type TEXT NOT NULL,
|
|
12
|
+
post_uri TEXT NOT NULL,
|
|
13
|
+
payload_json TEXT NOT NULL,
|
|
14
|
+
created_at TEXT NOT NULL,
|
|
15
|
+
source TEXT NOT NULL
|
|
16
|
+
)`;
|
|
17
|
+
yield* sql`INSERT INTO event_log_new (event_seq, event_id, event_type, post_uri, payload_json, created_at, source)
|
|
18
|
+
SELECT rowid, event_id, event_type, post_uri, payload_json, created_at, source
|
|
19
|
+
FROM event_log
|
|
20
|
+
ORDER BY rowid`;
|
|
21
|
+
yield* sql`DROP TABLE event_log`;
|
|
22
|
+
yield* sql`ALTER TABLE event_log_new RENAME TO event_log`;
|
|
23
|
+
yield* sql`CREATE INDEX IF NOT EXISTS event_log_created_at_idx ON event_log(created_at)`;
|
|
24
|
+
yield* sql`CREATE INDEX IF NOT EXISTS event_log_post_uri_idx ON event_log(post_uri)`;
|
|
25
|
+
yield* sql`CREATE INDEX IF NOT EXISTS event_log_source_idx ON event_log(source)`;
|
|
26
|
+
|
|
27
|
+
yield* sql`ALTER TABLE index_checkpoints RENAME TO index_checkpoints_old`;
|
|
28
|
+
yield* sql`CREATE TABLE IF NOT EXISTS index_checkpoints (
|
|
29
|
+
index_name TEXT PRIMARY KEY,
|
|
30
|
+
version INTEGER NOT NULL,
|
|
31
|
+
last_event_seq INTEGER NOT NULL,
|
|
32
|
+
event_count INTEGER NOT NULL,
|
|
33
|
+
updated_at TEXT NOT NULL
|
|
34
|
+
)`;
|
|
35
|
+
yield* sql`INSERT INTO index_checkpoints (index_name, version, last_event_seq, event_count, updated_at)
|
|
36
|
+
SELECT index_name,
|
|
37
|
+
version,
|
|
38
|
+
COALESCE((SELECT event_seq FROM event_log WHERE event_id = index_checkpoints_old.last_event_id), 0),
|
|
39
|
+
event_count,
|
|
40
|
+
updated_at
|
|
41
|
+
FROM index_checkpoints_old`;
|
|
42
|
+
yield* sql`DROP TABLE index_checkpoints_old`;
|
|
43
|
+
|
|
44
|
+
yield* sql`CREATE TABLE IF NOT EXISTS sync_checkpoints (
|
|
45
|
+
source_key TEXT PRIMARY KEY,
|
|
46
|
+
source_json TEXT NOT NULL,
|
|
47
|
+
cursor TEXT,
|
|
48
|
+
last_event_seq INTEGER,
|
|
49
|
+
filter_hash TEXT,
|
|
50
|
+
updated_at TEXT NOT NULL
|
|
51
|
+
)`;
|
|
52
|
+
yield* sql`CREATE INDEX IF NOT EXISTS sync_checkpoints_updated_at_idx ON sync_checkpoints(updated_at)`;
|
|
53
|
+
|
|
54
|
+
yield* sql`CREATE TABLE IF NOT EXISTS derivation_checkpoints (
|
|
55
|
+
view_name TEXT NOT NULL,
|
|
56
|
+
source_store TEXT NOT NULL,
|
|
57
|
+
target_store TEXT NOT NULL,
|
|
58
|
+
filter_hash TEXT NOT NULL,
|
|
59
|
+
evaluation_mode TEXT NOT NULL,
|
|
60
|
+
last_source_event_seq INTEGER,
|
|
61
|
+
events_processed INTEGER NOT NULL,
|
|
62
|
+
events_matched INTEGER NOT NULL,
|
|
63
|
+
deletes_propagated INTEGER NOT NULL,
|
|
64
|
+
updated_at TEXT NOT NULL,
|
|
65
|
+
PRIMARY KEY (view_name, source_store)
|
|
66
|
+
)`;
|
|
67
|
+
yield* sql`CREATE INDEX IF NOT EXISTS derivation_checkpoints_updated_at_idx ON derivation_checkpoints(updated_at)`;
|
|
68
|
+
});
|