@mepuka/skygent 0.2.0 → 0.3.1

package/README.md

@@ -1,59 +1,297 @@
-# skygent-bsky
+# @mepuka/skygent
 
-To install dependencies:
+Composable Bluesky data filtering and monitoring CLI built with [Effect](https://effect.website).
+
+Sync posts from timelines, feeds, lists, authors, and the real-time Jetstream firehose into local SQLite stores. Query, filter, derive, and export data with a powerful filter DSL and multiple output formats.
+
+## Install
+
+### npm / bun
 
 ```bash
+bun add -g @mepuka/skygent
+```
+
+### From source
+
+```bash
+git clone https://github.com/mepuka/skygent-bsky.git
+cd skygent-bsky
 bun install
+bun run index.ts --help
 ```
 
-Create a local `.env` from `.env.example` and set credentials:
+### Standalone binary
+
+Download a prebuilt binary from [GitHub Releases](https://github.com/mepuka/skygent-bsky/releases), or build locally:
 
 ```bash
-cp .env.example .env
+bun run build:binary
+./skygent --help
 ```
 
-To run:
+Cross-platform targets: `linux-x64`, `linux-arm64`, `darwin-x64`, `darwin-arm64`.
+
+## Authentication
+
+Skygent needs a Bluesky handle and [app password](https://bsky.app/settings/app-passwords). Credentials are resolved in this order:
+
+1. CLI flags: `--identifier` and `--password`
+2. Environment variables: `SKYGENT_IDENTIFIER` and `SKYGENT_PASSWORD`
+3. Encrypted credential file (`~/.skygent/credentials.json`, requires `SKYGENT_CREDENTIALS_KEY`)
+
+The simplest setup:
 
 ```bash
-bun run index.ts
+cp .env.example .env
+# Edit .env with your handle and app password
 ```
 
+Bun loads `.env` automatically.
+
 ## Quickstart
 
 ```bash
-# List stores (compact JSON for agents)
-bun run index.ts store list --compact
+# Create a store
+skygent store create my-store
+
+# Sync your timeline
+skygent sync timeline --store my-store
 
-# Sync timeline into a store
-bun run index.ts sync timeline --store my-store --quiet
+# Query recent posts
+skygent query my-store --limit 10 --format table
 
-# Query recent posts as a table
-bun run index.ts query my-store --limit 10 --format table
+# Stream live posts from Jetstream
+skygent watch jetstream --store my-store
 
-# Stream Jetstream posts
-bun run index.ts watch jetstream --store my-store --quiet
+# Derive a filtered store
+skygent derive my-store ai-posts --filter 'hashtag:#ai OR hashtag:#ml'
+
+# Search posts locally
+skygent search posts "effect typescript" --store my-store
 ```
 
-Tips:
-- Add `--compact` to reduce JSON output size.
-- Add `--quiet` to suppress progress logs during sync/watch commands.
+## Commands
+
+### `store` -- Manage local stores
+
+| Subcommand | Description |
+|---|---|
+| `store create <name>` | Create a new store |
+| `store list` | List all stores |
+| `store show <name>` | Show store config and metadata |
+| `store rename <from> <to>` | Rename a store |
+| `store delete <name> --force` | Delete a store |
+| `store stats <name>` | Show store statistics |
+| `store summary` | Summarize all stores |
+| `store tree` | Visualize store lineage |
+| `store materialize <name>` | Materialize filter outputs to disk |
+
+### `sync` -- One-time data sync
+
+| Subcommand | Description |
+|---|---|
+| `sync timeline` | Sync your timeline |
+| `sync feed <uri>` | Sync a feed generator |
+| `sync list <uri>` | Sync a list feed |
+| `sync author <actor>` | Sync posts from an author |
+| `sync thread <uri>` | Sync a thread (parents + replies) |
+| `sync notifications` | Sync notifications |
+| `sync jetstream` | Sync from Jetstream firehose |
+
+All sync commands accept `--store`, `--filter`, `--quiet`, and `--refresh`.
+
+### `watch` -- Continuous sync
+
+Same subcommands as `sync`, with continuous polling. Supports `--interval` (default: 30s).
+
+```bash
+skygent watch timeline --store my-store --interval "5 minutes"
+```
+
+### `query` -- Query stored posts
+
+```bash
+skygent query my-store --limit 25 --format table
+skygent query my-store --filter 'hashtag:#ai' --sort desc --format json
+skygent query my-store --range 2024-01-01T00:00:00Z..2024-01-31T00:00:00Z
+skygent query my-store --fields @minimal --newest-first
+skygent query store-a,store-b --format ndjson
+```
+
+**Formats:** `json`, `ndjson`, `table`, `markdown`, `compact`, `card`, `thread`
+
+**Field presets:** `@minimal`, `@social`, `@full`, or comma-separated field names with dot notation.
+
+Multi-store queries accept comma-separated store lists or repeated store arguments and include store names in output by default.
+
+### `derive` -- Create derived stores
+
+Apply a filter to a source store to produce a new filtered store:
+
+```bash
+skygent derive source-store target-store --filter 'hashtag:#ai'
+```
+
+**Modes:**
+- `event-time` (default) -- Pure filters only, replayable
+- `derive-time` -- Allows effectful filters (Trending, HasValidLinks)
+
+### `filter` -- Filter management and testing
+
+Tip: run `skygent filter help` for a compact list of predicates and aliases.
+
+| Subcommand | Description |
+|---|---|
+| `filter create <name>` | Save a named filter |
+| `filter list` | List saved filters |
+| `filter show <name>` | Show a saved filter |
+| `filter delete <name>` | Delete a saved filter |
+| `filter help` | Show filter DSL and JSON help |
+| `filter validate` | Validate a filter expression |
+| `filter test` | Test a filter against a post |
+| `filter explain` | Explain why a post matches |
+| `filter benchmark` | Benchmark filter performance |
+| `filter describe` | Describe a filter in plain text |
+
+### `search` -- Search content
+
+| Subcommand | Description |
+|---|---|
+| `search posts <query>` | Search posts locally or `--network` |
+| `search handles <query>` | Search Bluesky profiles |
+| `search feeds <query>` | Search feed generators |
+
+### `graph` -- Social graph
+
+| Subcommand | Description |
+|---|---|
+| `graph followers <actor>` | List followers |
+| `graph follows <actor>` | List follows |
+| `graph known-followers <actor>` | Mutual followers |
+| `graph relationships <actor>` | Relationship status |
+| `graph lists <actor>` | Lists created by actor |
+| `graph list <uri>` | View a list's members |
+| `graph blocks` | Your blocked accounts |
+| `graph mutes` | Your muted accounts |
+
+### `feed` -- Feed generators
+
+| Subcommand | Description |
+|---|---|
+| `feed show <uri>` | Show feed details |
+| `feed batch <uri>...` | Fetch multiple feeds |
+| `feed by <actor>` | List feeds by an actor |
+
+### `post` -- Post engagement
+
+| Subcommand | Description |
+|---|---|
+| `post likes <uri>` | Who liked a post |
+| `post reposted-by <uri>` | Who reposted |
+| `post quotes <uri>` | Quote posts |
+
+### `view` -- Inspect threads and derivations
+
+| Subcommand | Description |
+|---|---|
+| `view thread <uri>` | Display a thread |
+| `view status <view> <source>` | Check if a derived view is stale |
+
+### `config` -- Configuration
+
+```bash
+skygent config check          # Run health checks
+```
+
+## Filter DSL
+
+Filters are passed via `--filter` (DSL string) or `--filter-json` (JSON AST).
+
+### Primitives
+
+| Filter | Example |
+|---|---|
+| `hashtag:#tag` | Match posts with hashtag |
+| `author:handle.bsky.social` | Match posts by author |
+| `contains:"text"` | Text search (case-insensitive by default) |
+| `regex:/pattern/i` | Regex match |
+| `language:en,es` | Match languages |
+| `date:<start>..<end>` | Date range (ISO 8601) |
+| `engagement:minLikes=100` | Engagement thresholds |
+| `is:reply` | Post type (`reply`, `quote`, `repost`, `original`) |
+| `has:images` | Media presence (`images`, `video`, `links`, `media`, `embed`) |
+| `@saved-name` | Reference a saved filter |
+
+### Aliases
+
+`from:` = `author:`, `tag:` = `hashtag:`, `text:` = `contains:`, `lang:` = `language:`
+
+### List filters
+
+`authorin:alice,bob,charlie` and `hashtagin:#ai,#ml,#dl`
+
+### Boolean operators
+
+```
+hashtag:#ai AND author:user.bsky.social
+hashtag:#ai OR hashtag:#ml
+NOT hashtag:#spam
+(hashtag:#ai OR hashtag:#ml) AND engagement:minLikes=10
+```
+
+Operators: `AND` / `&&`, `OR` / `||`, `NOT` / `!`, parentheses for grouping.
+
+## Configuration
+
+### Environment variables
+
+| Variable | Default | Description |
+|---|---|---|
+| `SKYGENT_IDENTIFIER` | -- | Bluesky handle or DID |
+| `SKYGENT_PASSWORD` | -- | App password |
+| `SKYGENT_CREDENTIALS_KEY` | -- | Master key for encrypted credential storage |
+| `SKYGENT_SERVICE` | `https://bsky.social` | Bluesky service URL |
+| `SKYGENT_STORE_ROOT` | `~/.skygent` | Root storage directory |
+| `SKYGENT_OUTPUT_FORMAT` | `ndjson` | Default output format |
+| `SKYGENT_BSKY_RATE_LIMIT` | `250 millis` | Min delay between API calls |
+| `SKYGENT_BSKY_RETRY_MAX` | `5` | Max retry attempts |
+| `SKYGENT_SYNC_CONCURRENCY` | `5` | Concurrent sync workers |
+
+### Global flags
+
+- `--full` -- Use verbose JSON output (compact is the default)
+- `--quiet` -- Suppress progress output
+- `--log-format json|human` -- Control log format
+
+## Architecture
+
+Skygent is built entirely on Effect with a layered service architecture:
+
+- **Domain** (`src/domain/`) -- Data models for posts, stores, filters, events, and derivations using Effect Schema
+- **Services** (`src/services/`) -- Business logic: Bluesky API client, SQLite store, sync engine, filter runtime, derivation engine
+- **CLI** (`src/cli/`) -- Command definitions, output formatting, error handling
+
+Stores are local SQLite databases with an append-only event log. Derivations track lineage between stores and support incremental processing with checkpoints.
+
+## Security
+
+- Passwords are handled as `Redacted` values and never logged
+- Encrypted credential storage uses AES-GCM with PBKDF2 (100,000 iterations)
+- Avoid putting passwords in config files; use environment variables or the credential store
 
 ## Documentation
 
-- docs/README.md
-- docs/getting-started.md
-- docs/cli.md
-- docs/filters.md
-- docs/configuration.md
-- docs/credentials.md
-- docs/stores.md
-- docs/outputs.md
+Detailed docs are in `docs/`:
 
-## Security notes
+- [Getting Started](docs/getting-started.md)
+- [CLI Reference](docs/cli.md)
+- [Filters](docs/filters.md)
+- [Configuration](docs/configuration.md)
+- [Credentials](docs/credentials.md)
+- [Stores](docs/stores.md)
+- [Output Formats](docs/outputs.md)
 
-- Credentials are loaded via Effect `Redacted` config and are not logged.
-- Encrypted credential storage uses `SKYGENT_CREDENTIALS_KEY` (AES-GCM).
-- Avoid putting passwords in `config.json`; use env or the credential store instead.
-- Resource warnings can be configured via `SKYGENT_RESOURCE_*` env vars.
+## License
 
-This project was created using `bun init` in bun v1.3.4. [Bun](https://bun.com) is a fast all-in-one JavaScript runtime.
+MIT

package/index.ts

@@ -1,7 +1,7 @@
 #!/usr/bin/env bun
 import { Command, HelpDoc, ValidationError } from "@effect/cli";
 import { BunContext, BunRuntime } from "@effect/platform-bun";
-import { Effect, Layer } from "effect";
+import { Clock, Effect, Layer } from "effect";
 import { app } from "./src/cli/app.js";
 import pkg from "./package.json" with { type: "json" };
 import {
@@ -13,7 +13,7 @@ import {
 import { logErrorEvent } from "./src/cli/logging.js";
 import { CliOutput } from "./src/cli/output.js";
 import { exitCodeFor, exitCodeFromExit } from "./src/cli/exit-codes.js";
-import { BskyError, StoreNotFound } from "./src/domain/errors.js";
+import { BskyError, StoreAlreadyExists, StoreNotFound } from "./src/domain/errors.js";
 
 const cli = Command.run(app, {
   name: "skygent",
@@ -51,6 +51,9 @@ const formatError = (error: unknown, agentPayload?: AgentErrorPayload) => {
   if (error instanceof StoreNotFound) {
     return `Store \"${error.name}\" does not exist.`;
   }
+  if (error instanceof StoreAlreadyExists) {
+    return `Store \"${error.name}\" already exists.`;
+  }
   if (error instanceof Error) {
     return error.message;
   }
@@ -88,6 +91,9 @@ const errorDetails = (
   if (error instanceof StoreNotFound) {
     return { error: { _tag: "StoreNotFound", name: error.name } };
   }
+  if (error instanceof StoreAlreadyExists) {
+    return { error: { _tag: "StoreAlreadyExists", name: error.name } };
+  }
   if (error instanceof BskyError) {
     return {
       error: {
@@ -114,6 +120,9 @@ const errorSuggestion = (
   if (error instanceof StoreNotFound) {
     return `Run: skygent store create ${error.name}`;
   }
+  if (error instanceof StoreAlreadyExists) {
+    return "Run: skygent store list";
+  }
   return undefined;
 };
 
@@ -130,7 +139,13 @@ const program = cli(process.argv).pipe(
       ...errorDetails(error, agentPayload)
     });
   }),
-  Effect.provide(Layer.mergeAll(BunContext.layer, CliOutput.layer))
+  Effect.provide(
+    Layer.mergeAll(
+      BunContext.layer,
+      CliOutput.layer,
+      Layer.succeed(Clock.Clock, Clock.make())
+    )
+  )
 );
 
 BunRuntime.runMain({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mepuka/skygent",
-  "version": "0.2.0",
+  "version": "0.3.1",
   "description": "Composable Bluesky data filtering and monitoring CLI built with Effect",
   "module": "index.ts",
   "type": "module",

package/src/cli/app.ts

@@ -17,6 +17,7 @@ import { searchCommand } from "./search.js";
 import { graphCommand } from "./graph.js";
 import { feedCommand } from "./feed.js";
 import { postCommand } from "./post.js";
+import { pipeCommand } from "./pipe.js";
 import { configCommand } from "./config-command.js";
 import {
   configOptions,
@@ -39,7 +40,8 @@ export const app = Command.make("skygent", configOptions).pipe(
     searchCommand,
     graphCommand,
     feedCommand,
-    postCommand
+    postCommand,
+    pipeCommand
   ]),
   Command.provide((config) =>
     Layer.mergeAll(
@@ -69,7 +71,7 @@ export const app = Command.make("skygent", configOptions).pipe(
         "skygent store list --compact",
         "skygent sync timeline --store my-store --quiet"
       ],
-      ["Tip: add --compact for shorter JSON output."]
+      ["Tip: compact output is the default; use --full for verbose JSON."]
     )
   )
 );

package/src/cli/compact-output.ts

@@ -0,0 +1,52 @@
+import type { FeedGeneratorView, ListItemView, ListView, PostLike, ProfileView } from "../domain/bsky.js";
+import type { Post } from "../domain/post.js";
+
+type CompactProfile = {
+  readonly did: ProfileView["did"];
+  readonly handle: ProfileView["handle"];
+  readonly displayName?: string;
+};
+
+const compactProfile = (profile: ProfileView): CompactProfile => ({
+  did: profile.did,
+  handle: profile.handle,
+  ...(profile.displayName ? { displayName: profile.displayName } : {})
+});
+
+export const compactProfileView = (profile: ProfileView) =>
+  compactProfile(profile);
+
+export const compactFeedGeneratorView = (feed: FeedGeneratorView) => ({
+  uri: feed.uri,
+  displayName: feed.displayName,
+  creator: compactProfile(feed.creator),
+  ...(feed.likeCount !== undefined ? { likeCount: feed.likeCount } : {})
+});
+
+export const compactListView = (list: ListView) => ({
+  uri: list.uri,
+  name: list.name,
+  purpose: list.purpose,
+  creator: compactProfile(list.creator),
+  ...(list.listItemCount !== undefined
+    ? { listItemCount: list.listItemCount }
+    : {})
+});
+
+export const compactListItemView = (item: ListItemView) => ({
+  uri: item.uri,
+  subject: compactProfile(item.subject)
+});
+
+export const compactPostLike = (like: PostLike) => ({
+  actor: compactProfile(like.actor),
+  createdAt: like.createdAt,
+  indexedAt: like.indexedAt
+});
+
+export const compactPost = (post: Post) => ({
+  uri: post.uri,
+  author: post.author,
+  text: post.text,
+  createdAt: post.createdAt
+});

package/src/cli/config.ts

@@ -1,4 +1,4 @@
-import { Options } from "@effect/cli";
+import { HelpDoc, Options } from "@effect/cli";
 import { Option, Redacted } from "effect";
 import { pickDefined } from "../services/shared.js";
 import { OutputFormat } from "../domain/config.js";
@@ -6,8 +6,35 @@ import { AppConfig } from "../domain/config.js";
 import type { LogFormat } from "./logging.js";
 import type { SyncSettingsValue } from "../services/sync-settings.js";
 import type { CredentialsOverridesValue } from "../services/credential-store.js";
+import { NonNegativeInt, PositiveInt } from "./option-schemas.js";
 
 
+const compactOption = Options.all({
+  full: Options.boolean("full").pipe(
+    Options.withDescription("Use full JSON output")
+  ),
+  compact: Options.boolean("compact").pipe(
+    Options.withDescription("Use compact JSON output (default)")
+  )
+}).pipe(
+  Options.mapTryCatch(
+    ({ full, compact }) => {
+      if (full && compact) {
+        throw new Error("Use either --full or --compact, not both.");
+      }
+      if (full) return false;
+      if (compact) return true;
+      return true;
+    },
+    (error) =>
+      HelpDoc.p(
+        typeof error === "object" && error !== null && "message" in error
+          ? String((error as { readonly message?: unknown }).message ?? error)
+          : String(error)
+      )
+  )
+);
+
 export const configOptions = {
   service: Options.text("service").pipe(
     Options.optional,
@@ -31,22 +58,33 @@ export const configOptions = {
     Options.optional,
     Options.withDescription("Override Bluesky password (redacted)")
   ),
-  compact: Options.boolean("compact").pipe(
-    Options.withDescription("Reduce JSON output verbosity for agent consumption")
-  ),
+  compact: compactOption,
   logFormat: Options.choice("log-format", ["json", "human"]).pipe(
     Options.optional,
     Options.withDescription("Override log format (json or human)")
   ),
   syncConcurrency: Options.integer("sync-concurrency").pipe(
+    Options.withSchema(PositiveInt),
     Options.optional,
     Options.withDescription("Concurrent sync preparation workers (default: 5)")
   ),
+  syncBatchSize: Options.integer("sync-batch-size").pipe(
+    Options.withSchema(PositiveInt),
+    Options.optional,
+    Options.withDescription("Batch size for sync store writes (default: 100)")
+  ),
+  syncPageLimit: Options.integer("sync-page-limit").pipe(
+    Options.withSchema(PositiveInt),
+    Options.optional,
+    Options.withDescription("Page size for sync fetches (default: 100)")
+  ),
   checkpointEvery: Options.integer("checkpoint-every").pipe(
+    Options.withSchema(PositiveInt),
     Options.optional,
     Options.withDescription("Checkpoint every N processed posts (default: 100)")
   ),
   checkpointIntervalMs: Options.integer("checkpoint-interval-ms").pipe(
+    Options.withSchema(NonNegativeInt),
     Options.optional,
     Options.withDescription("Checkpoint interval in milliseconds (default: 5000)")
   )
@@ -61,6 +99,8 @@ export type ConfigOptions = {
   readonly compact: boolean;
   readonly logFormat: Option.Option<LogFormat>;
   readonly syncConcurrency: Option.Option<number>;
+  readonly syncBatchSize: Option.Option<number>;
+  readonly syncPageLimit: Option.Option<number>;
   readonly checkpointEvery: Option.Option<number>;
   readonly checkpointIntervalMs: Option.Option<number>;
 };
@@ -86,6 +126,8 @@ export const toSyncSettingsOverrides = (
 ): Partial<SyncSettingsValue> =>
   pickDefined({
     concurrency: Option.getOrUndefined(options.syncConcurrency),
+    batchSize: Option.getOrUndefined(options.syncBatchSize),
+    pageLimit: Option.getOrUndefined(options.syncPageLimit),
     checkpointEvery: Option.getOrUndefined(options.checkpointEvery),
     checkpointIntervalMs: Option.getOrUndefined(options.checkpointIntervalMs)
   }) as Partial<SyncSettingsValue>;

package/src/cli/doc/table-renderers.ts

@@ -0,0 +1,29 @@
+import { renderTableLegacy } from "./table.js";
+import type { FeedGeneratorView, ProfileView } from "../../domain/bsky.js";
+
+export const renderProfileTable = (
+  actors: ReadonlyArray<ProfileView>,
+  cursor: string | undefined
+) => {
+  const rows = actors.map((actor) => [
+    actor.handle,
+    actor.displayName ?? "",
+    actor.did
+  ]);
+  const table = renderTableLegacy(["HANDLE", "DISPLAY NAME", "DID"], rows);
+  return cursor ? `${table}\n\nCursor: ${cursor}` : table;
+};
+
+export const renderFeedTable = (
+  feeds: ReadonlyArray<FeedGeneratorView>,
+  cursor: string | undefined
+) => {
+  const rows = feeds.map((feed) => [
+    feed.displayName,
+    feed.creator.handle,
+    feed.uri,
+    typeof feed.likeCount === "number" ? String(feed.likeCount) : ""
+  ]);
+  const table = renderTableLegacy(["NAME", "CREATOR", "URI", "LIKES"], rows);
+  return cursor ? `${table}\n\nCursor: ${cursor}` : table;
+};

package/src/cli/doc/thread.ts

@@ -3,6 +3,7 @@ import type { Annotation } from "./annotation.js";
 import { renderTree } from "./tree.js";
 import { renderPostCompact, renderPostCardLines } from "./post.js";
 import type { Post } from "../../domain/post.js";
+import { PostOrder } from "../../domain/order.js";
 
 export const renderThread = (
   posts: ReadonlyArray<Post>,
@@ -23,10 +24,7 @@ export const renderThread = (
     }
   }
 
-  const sortPosts = (arr: Post[]) =>
-    arr.sort((a, b) =>
-      a.createdAt.getTime() - b.createdAt.getTime() || a.uri.localeCompare(b.uri)
-    );
+  const sortPosts = (arr: Post[]) => arr.sort(PostOrder);
 
   sortPosts(roots);
   for (const children of childMap.values()) sortPosts(children);

package/src/cli/exit-codes.ts

@@ -8,6 +8,7 @@ import {
   FilterEvalError,
   FilterLibraryError,
   FilterNotFound,
+  StoreAlreadyExists,
   StoreIoError,
   StoreIndexError,
   StoreNotFound
@@ -21,6 +22,7 @@ export const exitCodeFor = (error: unknown): number => {
   if (error instanceof CliInputError) return 2;
   if (error instanceof ConfigError) return 2;
   if (error instanceof StoreNotFound) return 3;
+  if (error instanceof StoreAlreadyExists) return 2;
   if (error instanceof FilterNotFound) return 2;
   if (error instanceof FilterLibraryError) return 2;
   if (error instanceof StoreIoError || error instanceof StoreIndexError) return 7;