@mepuka/skygent 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/README.md +59 -0
  2. package/index.ts +146 -0
  3. package/package.json +56 -0
  4. package/src/cli/app.ts +75 -0
  5. package/src/cli/config-command.ts +140 -0
  6. package/src/cli/config.ts +91 -0
  7. package/src/cli/derive.ts +205 -0
  8. package/src/cli/doc/annotation.ts +36 -0
  9. package/src/cli/doc/filter.ts +69 -0
  10. package/src/cli/doc/index.ts +9 -0
  11. package/src/cli/doc/post.ts +155 -0
  12. package/src/cli/doc/primitives.ts +25 -0
  13. package/src/cli/doc/render.ts +18 -0
  14. package/src/cli/doc/table.ts +114 -0
  15. package/src/cli/doc/thread.ts +46 -0
  16. package/src/cli/doc/tree.ts +126 -0
  17. package/src/cli/errors.ts +59 -0
  18. package/src/cli/exit-codes.ts +52 -0
  19. package/src/cli/feed.ts +177 -0
  20. package/src/cli/filter-dsl.ts +1411 -0
  21. package/src/cli/filter-errors.ts +208 -0
  22. package/src/cli/filter-help.ts +70 -0
  23. package/src/cli/filter-input.ts +54 -0
  24. package/src/cli/filter.ts +435 -0
  25. package/src/cli/graph.ts +472 -0
  26. package/src/cli/help.ts +14 -0
  27. package/src/cli/interval.ts +35 -0
  28. package/src/cli/jetstream.ts +173 -0
  29. package/src/cli/layers.ts +180 -0
  30. package/src/cli/logging.ts +136 -0
  31. package/src/cli/output-format.ts +26 -0
  32. package/src/cli/output.ts +82 -0
  33. package/src/cli/parse.ts +80 -0
  34. package/src/cli/post.ts +193 -0
  35. package/src/cli/preferences.ts +11 -0
  36. package/src/cli/query-fields.ts +247 -0
  37. package/src/cli/query.ts +415 -0
  38. package/src/cli/range.ts +44 -0
  39. package/src/cli/search.ts +465 -0
  40. package/src/cli/shared-options.ts +169 -0
  41. package/src/cli/shared.ts +20 -0
  42. package/src/cli/store-errors.ts +80 -0
  43. package/src/cli/store-tree.ts +392 -0
  44. package/src/cli/store.ts +395 -0
  45. package/src/cli/sync-factory.ts +107 -0
  46. package/src/cli/sync.ts +366 -0
  47. package/src/cli/view-thread.ts +196 -0
  48. package/src/cli/view.ts +47 -0
  49. package/src/cli/watch.ts +344 -0
  50. package/src/db/migrations/store-catalog/001_init.ts +14 -0
  51. package/src/db/migrations/store-index/001_init.ts +34 -0
  52. package/src/db/migrations/store-index/002_event_log.ts +24 -0
  53. package/src/db/migrations/store-index/003_fts_and_derived.ts +52 -0
  54. package/src/db/migrations/store-index/004_query_indexes.ts +9 -0
  55. package/src/db/migrations/store-index/005_post_lang.ts +15 -0
  56. package/src/db/migrations/store-index/006_has_embed.ts +10 -0
  57. package/src/db/migrations/store-index/007_event_seq_and_checkpoints.ts +68 -0
  58. package/src/domain/bsky.ts +467 -0
  59. package/src/domain/config.ts +11 -0
  60. package/src/domain/credentials.ts +6 -0
  61. package/src/domain/defaults.ts +8 -0
  62. package/src/domain/derivation.ts +55 -0
  63. package/src/domain/errors.ts +71 -0
  64. package/src/domain/events.ts +55 -0
  65. package/src/domain/extract.ts +64 -0
  66. package/src/domain/filter-describe.ts +551 -0
  67. package/src/domain/filter-explain.ts +9 -0
  68. package/src/domain/filter.ts +797 -0
  69. package/src/domain/format.ts +91 -0
  70. package/src/domain/index.ts +13 -0
  71. package/src/domain/indexes.ts +17 -0
  72. package/src/domain/policies.ts +16 -0
  73. package/src/domain/post.ts +88 -0
  74. package/src/domain/primitives.ts +50 -0
  75. package/src/domain/raw.ts +140 -0
  76. package/src/domain/store.ts +103 -0
  77. package/src/domain/sync.ts +211 -0
  78. package/src/domain/text-width.ts +56 -0
  79. package/src/services/app-config.ts +278 -0
  80. package/src/services/bsky-client.ts +2113 -0
  81. package/src/services/credential-store.ts +408 -0
  82. package/src/services/derivation-engine.ts +502 -0
  83. package/src/services/derivation-settings.ts +61 -0
  84. package/src/services/derivation-validator.ts +68 -0
  85. package/src/services/filter-compiler.ts +269 -0
  86. package/src/services/filter-library.ts +371 -0
  87. package/src/services/filter-runtime.ts +821 -0
  88. package/src/services/filter-settings.ts +30 -0
  89. package/src/services/identity-resolver.ts +563 -0
  90. package/src/services/jetstream-sync.ts +636 -0
  91. package/src/services/lineage-store.ts +89 -0
  92. package/src/services/link-validator.ts +244 -0
  93. package/src/services/output-manager.ts +274 -0
  94. package/src/services/post-parser.ts +62 -0
  95. package/src/services/profile-resolver.ts +223 -0
  96. package/src/services/resource-monitor.ts +106 -0
  97. package/src/services/shared.ts +69 -0
  98. package/src/services/store-cleaner.ts +43 -0
  99. package/src/services/store-commit.ts +168 -0
  100. package/src/services/store-db.ts +248 -0
  101. package/src/services/store-event-log.ts +285 -0
  102. package/src/services/store-index-sql.ts +289 -0
  103. package/src/services/store-index.ts +1152 -0
  104. package/src/services/store-keys.ts +4 -0
  105. package/src/services/store-manager.ts +358 -0
  106. package/src/services/store-stats.ts +522 -0
  107. package/src/services/store-writer.ts +200 -0
  108. package/src/services/sync-checkpoint-store.ts +169 -0
  109. package/src/services/sync-engine.ts +547 -0
  110. package/src/services/sync-reporter.ts +16 -0
  111. package/src/services/sync-settings.ts +72 -0
  112. package/src/services/trending-topics.ts +226 -0
  113. package/src/services/view-checkpoint-store.ts +238 -0
  114. package/src/typeclass/chunk.ts +84 -0
@@ -0,0 +1,408 @@
1
+ /**
2
+ * Credential Store Service
3
+ *
4
+ * Manages Bluesky authentication credentials with secure storage.
5
+ * Credentials are encrypted using AES-GCM with PBKDF2 key derivation
6
+ * and stored in a JSON file within the store root directory.
7
+ *
8
+ * The service supports multiple credential sources with the following priority:
9
+ * 1. Runtime overrides (via CredentialsOverrides context)
10
+ * 2. Environment variables (SKYGENT_IDENTIFIER, SKYGENT_PASSWORD)
11
+ * 3. Encrypted credentials file (requires SKYGENT_CREDENTIALS_KEY)
12
+ * 4. Config file defaults
13
+ *
14
+ * **Environment Variables:**
15
+ * - `SKYGENT_IDENTIFIER` - Bluesky handle or DID
16
+ * - `SKYGENT_PASSWORD` - App-specific password (redacted)
17
+ * - `SKYGENT_CREDENTIALS_KEY` - Master key for encrypting/decrypting stored credentials
18
+ *
19
+ * **Security:**
20
+ * - Passwords are stored as Redacted values to prevent accidental logging
21
+ * - Encryption uses 100,000 PBKDF2 iterations with SHA-256
22
+ * - Random 16-byte salt and 12-byte IV per encryption
23
+ *
24
+ * @module services/credential-store
25
+ *
26
+ * @example
27
+ * ```typescript
28
+ * import { CredentialStore } from "./services/credential-store.js";
29
+ * import { Effect } from "effect";
30
+ *
31
+ * const program = Effect.gen(function* () {
32
+ * const store = yield* CredentialStore;
33
+ *
34
+ * // Save credentials (requires SKYGENT_CREDENTIALS_KEY)
35
+ * yield* store.save({
36
+ * identifier: "handle.bsky.social",
37
+ * password: Redacted.make("app-password")
38
+ * });
39
+ *
40
+ * // Retrieve credentials
41
+ * const creds = yield* store.require();
42
+ * console.log(creds.identifier);
43
+ * });
44
+ * ```
45
+ */
46
+
47
+ import { FileSystem, Path } from "@effect/platform";
48
+ import { formatSchemaError } from "./shared.js";
49
+ import {
50
+ Config,
51
+ Context,
52
+ Effect,
53
+ Layer,
54
+ Option,
55
+ Redacted,
56
+ Schema
57
+ } from "effect";
58
+ import { AppConfigService } from "./app-config.js";
59
+ import { BskyCredentials } from "../domain/credentials.js";
60
+ import { CredentialError } from "../domain/errors.js";
61
+
62
+ /**
63
+ * Value type for runtime credential overrides.
64
+ * Used to inject credentials for testing or one-off operations.
65
+ */
66
+ export type CredentialsOverridesValue = {
67
+ /** Bluesky handle or DID */
68
+ readonly identifier?: string;
69
+ /** App-specific password (should be Redacted) */
70
+ readonly password?: Redacted.Redacted<string>;
71
+ };
72
+
73
+ /**
74
+ * Context tag for runtime credential overrides.
75
+ * Provides a way to inject credentials outside of the normal resolution chain.
76
+ *
77
+ * @example
78
+ * ```typescript
79
+ * const withOverrides = program.pipe(
80
+ * Effect.provideService(CredentialsOverrides, {
81
+ * identifier: "test.handle",
82
+ * password: Redacted.make("test-pass")
83
+ * })
84
+ * );
85
+ * ```
86
+ */
87
+ export class CredentialsOverrides extends Context.Tag("@skygent/CredentialsOverrides")<
88
+ CredentialsOverrides,
89
+ CredentialsOverridesValue
90
+ >() {
91
+ /** Empty layer for when no overrides are needed */
92
+ static readonly layer = Layer.succeed(CredentialsOverrides, {});
93
+ }
94
+
95
+ const credentialsFileName = "credentials.json";
96
+
97
+ class CredentialsPayload extends Schema.Class<CredentialsPayload>("CredentialsPayload")({
98
+ identifier: Schema.String,
99
+ password: Schema.String
100
+ }) {}
101
+
102
+ class CredentialsFile extends Schema.Class<CredentialsFile>("CredentialsFile")({
103
+ version: Schema.Literal(1),
104
+ salt: Schema.String,
105
+ iv: Schema.String,
106
+ ciphertext: Schema.String
107
+ }) {}
108
+
109
+
110
+ const toCredentialError = (message: string) => (cause: unknown) =>
111
+ CredentialError.make({ message, cause });
112
+
113
+ const encodeBase64 = (bytes: Uint8Array) => Buffer.from(bytes).toString("base64");
114
+ const decodeBase64 = (value: string) => new Uint8Array(Buffer.from(value, "base64"));
115
+
116
+ const deriveKey = (secret: string, salt: Uint8Array) =>
117
+ Effect.tryPromise({
118
+ try: async () => {
119
+ const saltBuffer: ArrayBuffer =
120
+ salt.buffer instanceof ArrayBuffer ? salt.buffer : new Uint8Array(salt).buffer;
121
+ const keyMaterial = await crypto.subtle.importKey(
122
+ "raw",
123
+ new TextEncoder().encode(secret),
124
+ "PBKDF2",
125
+ false,
126
+ ["deriveKey"]
127
+ );
128
+ return crypto.subtle.deriveKey(
129
+ {
130
+ name: "PBKDF2",
131
+ salt: saltBuffer,
132
+ iterations: 100_000,
133
+ hash: "SHA-256"
134
+ },
135
+ keyMaterial,
136
+ { name: "AES-GCM", length: 256 },
137
+ false,
138
+ ["encrypt", "decrypt"]
139
+ );
140
+ },
141
+ catch: toCredentialError("Failed to derive credentials key")
142
+ });
143
+
144
+ const encryptPayload = (secret: string, payload: CredentialsPayload) =>
145
+ Effect.gen(function* () {
146
+ const salt = crypto.getRandomValues(new Uint8Array(16));
147
+ const iv = crypto.getRandomValues(new Uint8Array(12));
148
+ const key = yield* deriveKey(secret, salt);
149
+ const payloadJson = yield* Schema.encodeUnknown(
150
+ Schema.parseJson(CredentialsPayload)
151
+ )(payload).pipe(
152
+ Effect.mapError((error) =>
153
+ CredentialError.make({
154
+ message: `Invalid credentials payload: ${formatSchemaError(error)}`,
155
+ cause: error
156
+ })
157
+ )
158
+ );
159
+ const encoded = new TextEncoder().encode(payloadJson);
160
+ const ciphertext = yield* Effect.tryPromise({
161
+ try: async () => {
162
+ const result = await crypto.subtle.encrypt({ name: "AES-GCM", iv }, key, encoded);
163
+ return new Uint8Array(result);
164
+ },
165
+ catch: toCredentialError("Failed to encrypt credentials")
166
+ });
167
+ return CredentialsFile.make({
168
+ version: 1,
169
+ salt: encodeBase64(salt),
170
+ iv: encodeBase64(iv),
171
+ ciphertext: encodeBase64(ciphertext)
172
+ });
173
+ });
174
+
175
+ const decryptPayload = (secret: string, file: CredentialsFile) =>
176
+ Effect.gen(function* () {
177
+ const salt = decodeBase64(file.salt);
178
+ const iv = decodeBase64(file.iv);
179
+ const key = yield* deriveKey(secret, salt);
180
+ const plaintext = yield* Effect.tryPromise({
181
+ try: async () => {
182
+ const result = await crypto.subtle.decrypt(
183
+ { name: "AES-GCM", iv },
184
+ key,
185
+ decodeBase64(file.ciphertext)
186
+ );
187
+ return new TextDecoder().decode(result);
188
+ },
189
+ catch: toCredentialError("Failed to decrypt credentials")
190
+ });
191
+ return yield* Schema.decodeUnknown(
192
+ Schema.parseJson(CredentialsPayload)
193
+ )(plaintext).pipe(
194
+ Effect.mapError((error) =>
195
+ CredentialError.make({
196
+ message: `Invalid credentials payload: ${formatSchemaError(error)}`,
197
+ cause: error
198
+ })
199
+ )
200
+ );
201
+ });
202
+
203
+ const decodeCredentialsFile = (raw: string) =>
204
+ Schema.decodeUnknown(Schema.parseJson(CredentialsFile))(raw).pipe(
205
+ Effect.mapError((error) =>
206
+ CredentialError.make({
207
+ message: `Invalid credentials file: ${formatSchemaError(error)}`,
208
+ cause: error
209
+ })
210
+ )
211
+ );
212
+
213
+ /**
214
+ * Interface for the credential store service.
215
+ * Defines operations for retrieving and storing Bluesky credentials.
216
+ */
217
+ export interface CredentialStoreService {
218
+ /**
219
+ * Retrieves credentials from any available source.
220
+ * Returns Option.none if no credentials are configured.
221
+ *
222
+ * @returns Effect resolving to Some(credentials) or None
223
+ */
224
+ readonly get: () => Effect.Effect<Option.Option<BskyCredentials>, CredentialError>;
225
+
226
+ /**
227
+ * Retrieves credentials, failing if none are available.
228
+ *
229
+ * @returns Effect resolving to credentials
230
+ * @throws {CredentialError} When no credentials are configured
231
+ */
232
+ readonly require: () => Effect.Effect<BskyCredentials, CredentialError>;
233
+
234
+ /**
235
+ * Saves credentials to the encrypted file.
236
+ * Requires SKYGENT_CREDENTIALS_KEY environment variable.
237
+ *
238
+ * @param credentials - The credentials to encrypt and store
239
+ * @returns Effect resolving to void on success
240
+ * @throws {CredentialError} When encryption fails or env key is missing
241
+ */
242
+ readonly save: (credentials: BskyCredentials) => Effect.Effect<void, CredentialError>;
243
+ }
244
+
245
+ /**
246
+ * Context tag and Layer implementation for the credential store service.
247
+ * Manages the lifecycle of credential storage and retrieval.
248
+ *
249
+ * **Resolution Priority:**
250
+ * 1. CredentialsOverrides context (for testing/runtime injection)
251
+ * 2. SKYGENT_IDENTIFIER / SKYGENT_PASSWORD environment variables
252
+ * 3. Encrypted credentials file (decrypted with SKYGENT_CREDENTIALS_KEY)
253
+ * 4. Config file identifier (password not supported in config)
254
+ *
255
+ * @example
256
+ * ```typescript
257
+ * // Basic usage with layer
258
+ * const program = Effect.gen(function* () {
259
+ * const store = yield* CredentialStore;
260
+ * const creds = yield* store.require();
261
+ * }).pipe(Effect.provide(CredentialStore.layer));
262
+ *
263
+ * // Testing with empty store
264
+ * const testProgram = program.pipe(
265
+ * Effect.provide(CredentialStore.testLayer)
266
+ * );
267
+ * ```
268
+ */
269
+ export class CredentialStore extends Context.Tag("@skygent/CredentialStore")<
270
+ CredentialStore,
271
+ CredentialStoreService
272
+ >() {
273
+ /**
274
+ * Production layer that creates the credential store service.
275
+ * Requires AppConfigService and FileSystem services.
276
+ */
277
+ static readonly layer = Layer.effect(
278
+ CredentialStore,
279
+ Effect.gen(function* () {
280
+ const overrides = yield* CredentialsOverrides;
281
+ const config = yield* AppConfigService;
282
+ const fs = yield* FileSystem.FileSystem;
283
+ const path = yield* Path.Path;
284
+
285
+ const credentialsPath = path.join(config.storeRoot, credentialsFileName);
286
+
287
+ const envIdentifier = yield* Config.string("SKYGENT_IDENTIFIER").pipe(Config.option);
288
+ const envPassword = yield* Config.redacted("SKYGENT_PASSWORD").pipe(Config.option);
289
+ const envKey = yield* Config.redacted("SKYGENT_CREDENTIALS_KEY").pipe(
290
+ Config.option
291
+ );
292
+
293
+ const loadFromFile = Effect.gen(function* () {
294
+ const exists = yield* fs.exists(credentialsPath).pipe(
295
+ Effect.mapError(toCredentialError("Failed to check credentials file"))
296
+ );
297
+ if (!exists) {
298
+ return Option.none<CredentialsPayload>();
299
+ }
300
+ if (Option.isNone(envKey)) {
301
+ return yield* CredentialError.make({
302
+ message:
303
+ "Credentials file exists but SKYGENT_CREDENTIALS_KEY is not set."
304
+ });
305
+ }
306
+ const raw = yield* fs.readFileString(credentialsPath).pipe(
307
+ Effect.mapError(toCredentialError("Failed to read credentials file"))
308
+ );
309
+ const file = yield* decodeCredentialsFile(raw);
310
+ const payload = yield* decryptPayload(Redacted.value(envKey.value), file);
311
+ return Option.some(payload);
312
+ });
313
+
314
+ const resolveIdentifier = (
315
+ filePayload: Option.Option<CredentialsPayload>
316
+ ): string | undefined =>
317
+ overrides.identifier ??
318
+ Option.getOrUndefined(envIdentifier) ??
319
+ Option.getOrUndefined(Option.map(filePayload, (payload) => payload.identifier)) ??
320
+ config.identifier;
321
+
322
+ const resolvePassword = (
323
+ filePayload: Option.Option<CredentialsPayload>
324
+ ): Redacted.Redacted<string> | undefined =>
325
+ overrides.password ??
326
+ Option.getOrUndefined(envPassword) ??
327
+ Option.getOrUndefined(
328
+ Option.map(filePayload, (payload) => Redacted.make(payload.password))
329
+ );
330
+
331
+ const get = Effect.fn("CredentialStore.get")(() =>
332
+ Effect.gen(function* () {
333
+ const filePayload = yield* loadFromFile;
334
+ const identifier = resolveIdentifier(filePayload);
335
+ const password = resolvePassword(filePayload);
336
+ if (!identifier || !password) {
337
+ return Option.none<BskyCredentials>();
338
+ }
339
+ const creds = BskyCredentials.make({ identifier, password });
340
+ return Option.some(creds);
341
+ })
342
+ );
343
+
344
+ const require = Effect.fn("CredentialStore.require")(() =>
345
+ get().pipe(
346
+ Effect.flatMap(
347
+ Option.match({
348
+ onNone: () =>
349
+ CredentialError.make({
350
+ message: "Missing Bluesky credentials."
351
+ }),
352
+ onSome: Effect.succeed
353
+ })
354
+ )
355
+ )
356
+ );
357
+
358
+ const save = Effect.fn("CredentialStore.save")((credentials: BskyCredentials) =>
359
+ Effect.gen(function* () {
360
+ if (Option.isNone(envKey)) {
361
+ return yield* CredentialError.make({
362
+ message:
363
+ "SKYGENT_CREDENTIALS_KEY is required to save encrypted credentials."
364
+ });
365
+ }
366
+ const payload = CredentialsPayload.make({
367
+ identifier: credentials.identifier,
368
+ password: Redacted.value(credentials.password)
369
+ });
370
+ const file = yield* encryptPayload(Redacted.value(envKey.value), payload);
371
+ yield* fs
372
+ .makeDirectory(config.storeRoot, { recursive: true })
373
+ .pipe(Effect.mapError(toCredentialError("Failed to create credentials directory")));
374
+ const encoded = yield* Schema.encodeUnknown(
375
+ Schema.parseJson(CredentialsFile)
376
+ )(file).pipe(
377
+ Effect.mapError((error) =>
378
+ CredentialError.make({
379
+ message: `Invalid credentials file: ${formatSchemaError(error)}`,
380
+ cause: error
381
+ })
382
+ )
383
+ );
384
+ yield* fs
385
+ .writeFileString(credentialsPath, encoded)
386
+ .pipe(Effect.mapError(toCredentialError("Failed to write credentials file")));
387
+ })
388
+ );
389
+
390
+ return CredentialStore.of({ get, require, save });
391
+ })
392
+ );
393
+
394
+ /**
395
+ * Test layer that provides an empty credential store.
396
+ * Useful for tests where authentication is not needed.
397
+ * get() returns None, require() fails with CredentialError.
398
+ */
399
+ static readonly testLayer = Layer.succeed(
400
+ CredentialStore,
401
+ CredentialStore.of({
402
+ get: () => Effect.succeed(Option.none()),
403
+ require: () =>
404
+ Effect.fail(CredentialError.make({ message: "Missing Bluesky credentials." })),
405
+ save: () => Effect.void
406
+ })
407
+ );
408
+ }