@mepuka/skygent 0.2.0

package/src/cli/sync.ts

@@ -0,0 +1,366 @@
+import { Command, Options } from "@effect/cli";
+import { Duration, Effect, Layer, Option } from "effect";
+import { Jetstream } from "effect-jetstream";
+import { filterExprSignature } from "../domain/filter.js";
+import { DataSource, SyncResult } from "../domain/sync.js";
+import { JetstreamSyncEngine } from "../services/jetstream-sync.js";
+import { storeOptions } from "./store.js";
+import { logInfo, makeSyncReporter } from "./logging.js";
+import { SyncReporter } from "../services/sync-reporter.js";
+import { ResourceMonitor } from "../services/resource-monitor.js";
+import { OutputManager } from "../services/output-manager.js";
+import { CliOutput, writeJson } from "./output.js";
+import { parseFilterExpr } from "./filter-input.js";
+import { withExamples } from "./help.js";
+import { buildJetstreamSelection, jetstreamOptions } from "./jetstream.js";
+import { CliInputError } from "./errors.js";
+import { makeSyncCommandBody } from "./sync-factory.js";
+import {
+  feedUriArg,
+  listUriArg,
+  postUriArg,
+  actorArg,
+  storeNameOption,
+  filterOption,
+  filterJsonOption,
+  postFilterOption,
+  postFilterJsonOption,
+  authorFilterOption,
+  includePinsOption,
+  decodeActor,
+  quietOption,
+  refreshOption,
+  strictOption,
+  maxErrorsOption,
+  parseMaxErrors,
+  parseLimit,
+  parseBoundedIntOption
+} from "./shared-options.js";
+
+const limitOption = Options.integer("limit").pipe(
+  Options.withDescription("Maximum number of Jetstream events to process"),
+  Options.optional
+);
+const durationOption = Options.text("duration").pipe(
+  Options.withDescription("Stop after a duration (e.g. \"2 minutes\")"),
+  Options.optional
+);
+const depthOption = Options.integer("depth").pipe(
+  Options.withDescription("Thread reply depth to include (0-1000, default 6)"),
+  Options.optional
+);
+const parentHeightOption = Options.integer("parent-height").pipe(
+  Options.withDescription("Thread parent height to include (0-1000, default 80)"),
+  Options.optional
+);
+
+const parseDuration = (value: Option.Option<string>) =>
+  Option.match(value, {
+    onNone: () => Effect.succeed(Option.none()),
+    onSome: (raw) =>
+      Effect.try({
+        try: () => Duration.decode(raw as Duration.DurationInput),
+        catch: (cause) =>
+          CliInputError.make({
+            message: `Invalid duration: ${raw}. Use formats like \"2 minutes\".`,
+            cause
+          })
+      }).pipe(
+        Effect.flatMap((duration) =>
+          Duration.toMillis(duration) < 0
+            ? Effect.fail(
+                CliInputError.make({
+                  message: "Duration must be non-negative.",
+                  cause: duration
+                })
+              )
+            : Effect.succeed(Option.some(duration))
+        )
+      )
+  });
+
+const timelineCommand = Command.make(
+  "timeline",
+  { store: storeNameOption, filter: filterOption, filterJson: filterJsonOption, quiet: quietOption, refresh: refreshOption },
+  makeSyncCommandBody("timeline", () => DataSource.timeline())
+).pipe(
+  Command.withDescription(
+    withExamples(
+      "Sync the authenticated timeline into a store",
+      [
+        "skygent sync timeline --store my-store",
+        "skygent sync timeline --store my-store --filter 'hashtag:#ai' --quiet"
+      ],
+      ["Tip: add --quiet to suppress progress logs."]
+    )
+  )
+);
+
+const feedCommand = Command.make(
+  "feed",
+  { uri: feedUriArg, store: storeNameOption, filter: filterOption, filterJson: filterJsonOption, quiet: quietOption, refresh: refreshOption },
+  ({ uri, ...rest }) => makeSyncCommandBody("feed", () => DataSource.feed(uri), { uri })(rest)
+).pipe(
+  Command.withDescription(
+    withExamples(
+      "Sync a feed URI into a store",
+      [
+        "skygent sync feed at://did:plc:example/app.bsky.feed.generator/xyz --store my-store"
+      ],
+      ["Tip: add --quiet to suppress progress logs."]
+    )
+  )
+);
+
+const listCommand = Command.make(
+  "list",
+  { uri: listUriArg, store: storeNameOption, filter: filterOption, filterJson: filterJsonOption, quiet: quietOption, refresh: refreshOption },
+  ({ uri, ...rest }) => makeSyncCommandBody("list", () => DataSource.list(uri), { uri })(rest)
+).pipe(
+  Command.withDescription(
+    withExamples(
+      "Sync a list feed URI into a store",
+      [
+        "skygent sync list at://did:plc:example/app.bsky.graph.list/xyz --store my-store"
+      ],
+      ["Tip: add --quiet to suppress progress logs."]
+    )
+  )
+);
+
+const notificationsCommand = Command.make(
+  "notifications",
+  { store: storeNameOption, filter: filterOption, filterJson: filterJsonOption, quiet: quietOption, refresh: refreshOption },
+  makeSyncCommandBody("notifications", () => DataSource.notifications())
+).pipe(
+  Command.withDescription(
+    withExamples(
+      "Sync notifications into a store",
+      ["skygent sync notifications --store my-store --quiet"],
+      ["Tip: add --quiet to suppress progress logs."]
+    )
+  )
+);
+
+const authorCommand = Command.make(
+  "author",
+  {
+    actor: actorArg,
+    store: storeNameOption,
+    filter: authorFilterOption,
+    includePins: includePinsOption,
+    postFilter: postFilterOption,
+    postFilterJson: postFilterJsonOption,
+    quiet: quietOption,
+    refresh: refreshOption
+  },
+  ({ actor, filter, includePins, postFilter, postFilterJson, store, quiet, refresh }) =>
+    Effect.gen(function* () {
+      const resolvedActor = yield* decodeActor(actor);
+      const apiFilter = Option.getOrUndefined(filter);
+      const source = DataSource.author(resolvedActor, {
+        ...(apiFilter !== undefined ? { filter: apiFilter } : {}),
+        ...(includePins ? { includePins: true } : {})
+      });
+      const run = makeSyncCommandBody("author", () => source, {
+        actor: resolvedActor,
+        ...(apiFilter !== undefined ? { filter: apiFilter } : {}),
+        ...(includePins ? { includePins: true } : {})
+      });
+      return yield* run({
+        store,
+        filter: postFilter,
+        filterJson: postFilterJson,
+        quiet,
+        refresh
+      });
+    })
+).pipe(
+  Command.withDescription(
+    withExamples(
+      "Sync posts from a specific author",
+      [
+        "skygent sync author alice.bsky.social --store my-store",
+        "skygent sync author did:plc:example --store my-store --filter posts_no_replies --include-pins"
+      ],
+      ["Tip: use --post-filter to apply the DSL filter to synced posts."]
+    )
+  )
+);
+
+const threadCommand = Command.make(
+  "thread",
+  {
+    uri: postUriArg,
+    store: storeNameOption,
+    depth: depthOption,
+    parentHeight: parentHeightOption,
+    filter: filterOption,
+    filterJson: filterJsonOption,
+    quiet: quietOption,
+    refresh: refreshOption
+  },
+  ({ uri, depth, parentHeight, filter, filterJson, store, quiet, refresh }) =>
+    Effect.gen(function* () {
+      const parsedDepth = yield* parseBoundedIntOption(depth, "depth", 0, 1000);
+      const parsedParentHeight = yield* parseBoundedIntOption(
+        parentHeight,
+        "parent-height",
+        0,
+        1000
+      );
+      const depthValue = Option.getOrUndefined(parsedDepth);
+      const parentHeightValue = Option.getOrUndefined(parsedParentHeight);
+      const source = DataSource.thread(uri, {
+        ...(depthValue !== undefined ? { depth: depthValue } : {}),
+        ...(parentHeightValue !== undefined ? { parentHeight: parentHeightValue } : {})
+      });
+      const run = makeSyncCommandBody("thread", () => source, {
+        uri,
+        ...(depthValue !== undefined ? { depth: depthValue } : {}),
+        ...(parentHeightValue !== undefined ? { parentHeight: parentHeightValue } : {})
+      });
+      return yield* run({ store, filter, filterJson, quiet, refresh });
+    })
+).pipe(
+  Command.withDescription(
+    withExamples(
+      "Sync a post thread (parents + replies) into a store",
+      [
+        "skygent sync thread at://did:plc:example/app.bsky.feed.post/xyz --store my-store",
+        "skygent sync thread at://did:plc:example/app.bsky.feed.post/xyz --store my-store --depth 10 --parent-height 5"
+      ],
+      ["Tip: use --filter to apply the DSL filter to thread posts."]
+    )
+  )
+);
+
+const jetstreamCommand = Command.make(
+  "jetstream",
+  {
+    store: storeNameOption,
+    filter: filterOption,
+    filterJson: filterJsonOption,
+    quiet: quietOption,
+    endpoint: jetstreamOptions.endpoint,
+    collections: jetstreamOptions.collections,
+    dids: jetstreamOptions.dids,
+    cursor: jetstreamOptions.cursor,
+    compress: jetstreamOptions.compress,
+    maxMessageSize: jetstreamOptions.maxMessageSize,
+    limit: limitOption,
+    duration: durationOption,
+    strict: strictOption,
+    maxErrors: maxErrorsOption
+  },
+  ({
+    store,
+    filter,
+    filterJson,
+    quiet,
+    endpoint,
+    collections,
+    dids,
+    cursor,
+    compress,
+    maxMessageSize,
+    limit,
+    duration,
+    strict,
+    maxErrors
+  }) =>
+    Effect.gen(function* () {
+      const monitor = yield* ResourceMonitor;
+      const output = yield* CliOutput;
+      const outputManager = yield* OutputManager;
+      const storeRef = yield* storeOptions.loadStoreRef(store);
+      const expr = yield* parseFilterExpr(filter, filterJson);
+      const filterHash = filterExprSignature(expr);
+      const selection = yield* buildJetstreamSelection(
+        {
+          endpoint,
+          collections,
+          dids,
+          cursor,
+          compress,
+          maxMessageSize
+        },
+        storeRef,
+        filterHash
+      );
+      const parsedLimit = yield* parseLimit(limit);
+      const parsedDuration = yield* parseDuration(duration);
+      const parsedMaxErrors = yield* parseMaxErrors(maxErrors);
+      if (Option.isNone(parsedLimit) && Option.isNone(parsedDuration)) {
+        return yield* CliInputError.make({
+          message:
+            "Jetstream sync requires --limit or --duration. Use watch jetstream for continuous streaming.",
+          cause: { limit, duration }
+        });
+      }
+      const engineLayer = JetstreamSyncEngine.layer.pipe(
+        Layer.provideMerge(Jetstream.live(selection.config))
+      );
+      yield* logInfo("Starting sync", {
+        source: "jetstream",
+        store: storeRef.name
+      });
+      const result = yield* Effect.gen(function* () {
+        const engine = yield* JetstreamSyncEngine;
+        const limitValue = Option.getOrUndefined(parsedLimit);
+        const durationValue = Option.getOrUndefined(parsedDuration);
+        const maxErrorsValue = Option.getOrUndefined(parsedMaxErrors);
+        return yield* engine.sync({
+          source: selection.source,
+          store: storeRef,
+          filter: expr,
+          command: "sync jetstream",
+          ...(limitValue !== undefined ? { limit: limitValue } : {}),
+          ...(durationValue !== undefined ? { duration: durationValue } : {}),
+          ...(selection.cursor !== undefined ? { cursor: selection.cursor } : {}),
+          ...(strict ? { strict } : {}),
+          ...(maxErrorsValue !== undefined ? { maxErrors: maxErrorsValue } : {})
+        });
+      }).pipe(
+        Effect.provide(engineLayer),
+        Effect.provideService(SyncReporter, makeSyncReporter(quiet, monitor, output))
+      );
+      const materialized = yield* outputManager.materializeStore(storeRef);
+      if (materialized.filters.length > 0) {
+        yield* logInfo("Materialized filter outputs", {
+          store: storeRef.name,
+          filters: materialized.filters.map((spec) => spec.name)
+        });
+      }
+      yield* logInfo("Sync complete", { source: "jetstream", store: storeRef.name });
+      yield* writeJson(result as SyncResult);
+    })
+).pipe(
+  Command.withDescription(
+    withExamples(
+      "Sync Jetstream events into a store (posts only)",
+      [
+        "skygent sync jetstream --store my-store --limit 500",
+        "skygent sync jetstream --store my-store --duration \"2 minutes\""
+      ],
+      ["Tip: use watch jetstream for continuous streaming."]
+    )
+  )
+);
+
+export const syncCommand = Command.make("sync", {}).pipe(
+  Command.withSubcommands([
+    timelineCommand,
+    feedCommand,
+    listCommand,
+    notificationsCommand,
+    authorCommand,
+    threadCommand,
+    jetstreamCommand
+  ]),
+  Command.withDescription(
+    withExamples("Sync content into stores", [
+      "skygent sync timeline --store my-store"
+    ])
+  )
+);

package/src/cli/view-thread.ts

@@ -0,0 +1,196 @@
+import { Args, Command, Options } from "@effect/cli";
+import { Chunk, Console, Effect, Option, Schema, Stream } from "effect";
+import { PostUri, StoreName } from "../domain/primitives.js";
+import type { Post } from "../domain/post.js";
+import { all } from "../domain/filter.js";
+import { StoreQuery } from "../domain/events.js";
+import { DataSource } from "../domain/sync.js";
+import { BskyClient } from "../services/bsky-client.js";
+import { PostParser } from "../services/post-parser.js";
+import { StoreIndex } from "../services/store-index.js";
+import { SyncEngine } from "../services/sync-engine.js";
+import { renderThread } from "./doc/thread.js";
+import { renderPlain, renderAnsi } from "./doc/render.js";
+import { writeJson, writeText } from "./output.js";
+import { storeOptions } from "./store.js";
+import { withExamples } from "./help.js";
+import { CliInputError } from "./errors.js";
+import { formatSchemaError } from "./shared.js";
+import { parseBoundedIntOption } from "./shared-options.js";
+import { textJsonFormats } from "./output-format.js";
+
+const uriArg = Args.text({ name: "uri" }).pipe(
+  Args.withDescription("AT-URI of any post in the thread")
+);
+
+const storeOption = Options.text("store").pipe(
+  Options.withSchema(StoreName),
+  Options.withDescription("Query from local store instead of API"),
+  Options.optional
+);
+
+const compactOption = Options.boolean("compact").pipe(
+  Options.withDescription("Single-line rendering (default: card)")
+);
+
+const ansiOption = Options.boolean("ansi").pipe(
+  Options.withDescription("Enable ANSI colors in output")
+);
+
+const widthOption = Options.integer("width").pipe(
+  Options.withDescription("Line width for terminal output"),
+  Options.optional
+);
+
+const formatOption = Options.choice("format", textJsonFormats).pipe(
+  Options.withDescription("Output format (default: text)"),
+  Options.optional
+);
+
+const depthOption = Options.integer("depth").pipe(
+  Options.withDescription("Reply depth (API only, default: 6)"),
+  Options.optional
+);
+
+const parentHeightOption = Options.integer("parent-height").pipe(
+  Options.withDescription("Parent height (API only, default: 80)"),
+  Options.optional
+);
+
+export const threadCommand = Command.make(
+  "thread",
+  {
+    uri: uriArg,
+    store: storeOption,
+    compact: compactOption,
+    ansi: ansiOption,
+    width: widthOption,
+    format: formatOption,
+    depth: depthOption,
+    parentHeight: parentHeightOption
+  },
+  ({ uri, store, compact, ansi, width, format, depth, parentHeight }) =>
+    Effect.gen(function* () {
+      const outputFormat = Option.getOrElse(format, () => "text" as const);
+      const w = Option.getOrUndefined(width);
+      const parsedDepth = yield* parseBoundedIntOption(depth, "depth", 0, 1000);
+      const parsedParentHeight = yield* parseBoundedIntOption(
+        parentHeight,
+        "parent-height",
+        0,
+        1000
+      );
+      const d = Option.getOrElse(parsedDepth, () => 6);
+      const ph = Option.getOrElse(parsedParentHeight, () => 80);
+
+      let posts: ReadonlyArray<Post>;
+
+      if (Option.isSome(store)) {
+        const index = yield* StoreIndex;
+        const storeRef = yield* storeOptions.loadStoreRef(store.value);
+        const targetUri = yield* Schema.decodeUnknown(PostUri)(uri).pipe(
+          Effect.mapError((error) =>
+            CliInputError.make({
+              message: `Invalid post URI: ${formatSchemaError(error)}`,
+              cause: error
+            })
+          )
+        );
+        const hasTarget = yield* index.hasUri(storeRef, targetUri);
+        if (!hasTarget) {
+          const engine = yield* SyncEngine;
+          const source = DataSource.thread(uri, { depth: d, parentHeight: ph });
+          yield* engine.sync(source, storeRef, all());
+        }
+        const query = StoreQuery.make({});
+        const stream = index.query(storeRef, query);
+        const collected = yield* Stream.runCollect(stream);
+        const allPosts = Chunk.toReadonlyArray(collected);
+        const threadPosts = selectThreadPosts(allPosts, String(targetUri));
+        if (threadPosts.length === 0) {
+          return yield* CliInputError.make({
+            message: `Thread not found for ${uri}.`,
+            cause: { uri, store: storeRef.name }
+          });
+        }
+        // B1: Hint when only root post exists in store
+        if (threadPosts.length === 1 && threadPosts[0]?.uri === targetUri) {
+          yield* Console.log("\nℹ️  Only root post found in store. Use --no-store to fetch full thread from API.\n");
+        }
+        posts = threadPosts;
+      } else {
+        const client = yield* BskyClient;
+        const parser = yield* PostParser;
+        const rawPosts = yield* client.getPostThread(uri, { depth: d, parentHeight: ph });
+        posts = yield* Effect.forEach(rawPosts, (raw) => parser.parsePost(raw));
+      }
+
+      if (outputFormat === "json") {
+        yield* writeJson(posts);
+        return;
+      }
+
+      const doc = renderThread(
+        posts,
+        w === undefined ? { compact } : { compact, lineWidth: w }
+      );
+      yield* writeText(ansi ? renderAnsi(doc, w) : renderPlain(doc, w));
+    })
+).pipe(
+  Command.withDescription(
+    withExamples(
+      "Display a thread from the API or a local store",
+      [
+        "skygent view thread at://did:plc:example/app.bsky.feed.post/xyz --ansi",
+        "skygent view thread at://did:plc:example/app.bsky.feed.post/xyz --compact --ansi",
+        "skygent view thread at://did:plc:example/app.bsky.feed.post/xyz --store my-store --ansi --width 100",
+        "skygent view thread at://did:plc:example/app.bsky.feed.post/xyz --format json"
+      ]
+    )
+  )
+);
+
+const selectThreadPosts = (posts: ReadonlyArray<Post>, targetUri: string) => {
+  const byUri = new Map(posts.map((post) => [String(post.uri), post]));
+  if (!byUri.has(targetUri)) {
+    return [] as ReadonlyArray<Post>;
+  }
+
+  const childMap = new Map<string, Post[]>();
+  for (const post of posts) {
+    const parentUri = post.reply?.parent?.uri ? String(post.reply.parent.uri) : undefined;
+    if (!parentUri || !byUri.has(parentUri)) {
+      continue;
+    }
+    const siblings = childMap.get(parentUri) ?? [];
+    siblings.push(post);
+    childMap.set(parentUri, siblings);
+  }
+
+  const threadUris = new Set<string>();
+  let current: Post | undefined = byUri.get(targetUri);
+  while (current) {
+    const currentUri = String(current.uri);
+    threadUris.add(currentUri);
+    const parentUri = current.reply?.parent?.uri
+      ? String(current.reply.parent.uri)
+      : undefined;
+    current = parentUri ? byUri.get(parentUri) : undefined;
+  }
+
+  const queue: Array<string> = [targetUri];
+  while (queue.length > 0) {
+    const next = queue.shift();
+    if (!next) break;
+    const children = childMap.get(next) ?? [];
+    for (const child of children) {
+      const childUri = String(child.uri);
+      if (!threadUris.has(childUri)) {
+        threadUris.add(childUri);
+        queue.push(childUri);
+      }
+    }
+  }
+
+  return posts.filter((post) => threadUris.has(String(post.uri)));
+};

package/src/cli/view.ts

@@ -0,0 +1,47 @@
+import { Args, Command } from "@effect/cli";
+import { Effect } from "effect";
+import { StoreName } from "../domain/primitives.js";
+import { DerivationValidator } from "../services/derivation-validator.js";
+import { writeJson } from "./output.js";
+import { withExamples } from "./help.js";
+import { threadCommand } from "./view-thread.js";
+
+const viewArg = Args.text({ name: "view" }).pipe(
+  Args.withSchema(StoreName),
+  Args.withDescription("Derived view store name")
+);
+const sourceArg = Args.text({ name: "source" }).pipe(
+  Args.withSchema(StoreName),
+  Args.withDescription("Source store name")
+);
+
+const statusCommand = Command.make(
+  "status",
+  { view: viewArg, source: sourceArg },
+  ({ view, source }) =>
+    Effect.gen(function* () {
+      const validator = yield* DerivationValidator;
+      const isStale = yield* validator.isStale(view, source);
+
+      yield* writeJson({
+        view,
+        source,
+        status: isStale ? "stale" : "ready"
+      });
+    })
+).pipe(
+  Command.withDescription(
+    withExamples("Check if a derived view is stale relative to its source", [
+      "skygent view status derived-store source-store"
+    ])
+  )
+);
+
+export const viewCommand = Command.make("view", {}).pipe(
+  Command.withSubcommands([statusCommand, threadCommand]),
+  Command.withDescription(
+    withExamples("View derivation status and metadata", [
+      "skygent view status derived-store source-store"
+    ])
+  )
+);