@mepuka/skygent 0.2.0

package/src/cli/preferences.ts

@@ -0,0 +1,11 @@
+import { Context } from "effect";
+
+export type CliPreferencesValue = {
+  readonly compact: boolean;
+  readonly logFormat?: "json" | "human";
+};
+
+export class CliPreferences extends Context.Tag("@skygent/CliPreferences")<
+  CliPreferences,
+  CliPreferencesValue
+>() {}

package/src/cli/query-fields.ts

@@ -0,0 +1,247 @@
+import { Effect, Option } from "effect";
+import { CliInputError } from "./errors.js";
+
+type FieldSelector = {
+  readonly path: ReadonlyArray<string>;
+  readonly wildcard: boolean;
+  readonly raw: string;
+};
+
+const fieldPresets: Record<string, ReadonlyArray<string>> = {
+  minimal: ["uri", "author", "text", "createdAt"],
+  social: ["uri", "author", "text", "metrics", "hashtags"],
+  full: []
+};
+
+const isObject = (value: unknown): value is Record<string, unknown> =>
+  typeof value === "object" && value !== null && !Array.isArray(value);
+
+const ensureNonEmpty = (value: string, message: string, cause: unknown) => {
+  if (value.length === 0) {
+    return Effect.fail(CliInputError.make({ message, cause }));
+  }
+  return Effect.void;
+};
+
+const scalarFieldHeads = new Set([
+  "uri",
+  "cid",
+  "author",
+  "authorDid",
+  "text",
+  "createdAt",
+  "hashtags",
+  "mentions",
+  "mentionDids",
+  "links",
+  "facets",
+  "langs",
+  "tags",
+  "selfLabels",
+  "labels",
+  "indexedAt"
+]);
+
+const parseFieldToken = (token: string) =>
+  Effect.gen(function* () {
+    const parts = token.split(".").map((segment) => segment.trim());
+    if (parts.length === 0) {
+      return yield* CliInputError.make({
+        message: "Field token cannot be empty.",
+        cause: token
+      });
+    }
+    for (const part of parts) {
+      yield* ensureNonEmpty(
+        part,
+        `Invalid field token "${token}".`,
+        token
+      );
+    }
+    if (parts.length > 1) {
+      const head = parts[0] ?? "";
+      if (scalarFieldHeads.has(head)) {
+        const suggestion =
+          head === "author"
+            ? ' Use "author" or "authorProfile.handle".'
+            : " Remove the dot path.";
+        return yield* CliInputError.make({
+          message: `Field "${token}" is not a valid path. "${head}" is a scalar field.${suggestion}`,
+          cause: token
+        });
+      }
+    }
+    const wildcardIndex = parts.indexOf("*");
+    if (wildcardIndex >= 0 && wildcardIndex !== parts.length - 1) {
+      return yield* CliInputError.make({
+        message: `Wildcard "*" must be the last segment in "${token}".`,
+        cause: token
+      });
+    }
+    const wildcardCount = parts.filter((part) => part === "*").length;
+    if (wildcardCount > 1) {
+      return yield* CliInputError.make({
+        message: `Wildcard "*" can only appear once in "${token}".`,
+        cause: token
+      });
+    }
+    const wildcard = wildcardIndex === parts.length - 1;
+    const path = wildcard ? parts.slice(0, -1) : parts;
+    return { path, wildcard, raw: token } satisfies FieldSelector;
+  });
+
+const normalizeTokens = (raw: string) =>
+  raw
+    .split(",")
+    .map((token) => token.trim())
+    .filter((token) => token.length > 0);
+
+const expandPresets = (tokens: ReadonlyArray<string>) =>
+  Effect.gen(function* () {
+    const expanded: string[] = [];
+    let fullRequested = false;
+
+    for (const token of tokens) {
+      if (token.startsWith("@")) {
+        const name = token.slice(1);
+        const preset = fieldPresets[name];
+        if (!preset) {
+          return yield* CliInputError.make({
+            message: `Unknown fields preset "${token}".`,
+            cause: token
+          });
+        }
+        if (name === "full") {
+          fullRequested = true;
+        } else {
+          expanded.push(...preset);
+        }
+      } else {
+        expanded.push(token);
+      }
+    }
+
+    if (fullRequested) {
+      if (expanded.length > 0) {
+        return yield* CliInputError.make({
+          message: "Preset @full cannot be combined with other fields.",
+          cause: tokens
+        });
+      }
+      return { kind: "full" as const, tokens: [] as ReadonlyArray<string> };
+    }
+
+    if (expanded.length === 0) {
+      return yield* CliInputError.make({
+        message: "Fields list cannot be empty.",
+        cause: tokens
+      });
+    }
+
+    return { kind: "partial" as const, tokens: expanded };
+  });
+
+export const parseFieldSelectors = (
+  raw: string
+): Effect.Effect<Option.Option<ReadonlyArray<FieldSelector>>, CliInputError> =>
+  Effect.gen(function* () {
+    const tokens = normalizeTokens(raw);
+    yield* ensureNonEmpty(raw.trim(), "Fields list cannot be empty.", raw);
+    const expanded = yield* expandPresets(tokens);
+    if (expanded.kind === "full") {
+      return Option.none();
+    }
+    const selectors = yield* Effect.forEach(
+      expanded.tokens,
+      (token) => parseFieldToken(token),
+      { discard: false }
+    );
+    return Option.some(selectors);
+  });
+
+export const resolveFieldSelectors = (
+  fields: Option.Option<string>,
+  compact: boolean
+): Effect.Effect<Option.Option<ReadonlyArray<FieldSelector>>, CliInputError> =>
+  Option.match(fields, {
+    onNone: () =>
+      compact ? parseFieldSelectors("@minimal") : Effect.succeed(Option.none()),
+    onSome: (raw) => parseFieldSelectors(raw)
+  });
+
+const getPathValue = (source: unknown, path: ReadonlyArray<string>): unknown => {
+  let current: unknown = source;
+  for (const segment of path) {
+    if (!isObject(current)) {
+      return undefined;
+    }
+    current = current[segment];
+  }
+  return current;
+};
+
+const ensureObjectPath = (target: Record<string, unknown>, path: string[]) => {
+  let current: Record<string, unknown> = target;
+  for (const segment of path) {
+    const existing = current[segment];
+    if (!isObject(existing)) {
+      current[segment] = {};
+    }
+    current = current[segment] as Record<string, unknown>;
+  }
+  return current;
+};
+
+const setPathValue = (
+  target: Record<string, unknown>,
+  path: ReadonlyArray<string>,
+  value: unknown
+) => {
+  if (path.length === 0) {
+    return;
+  }
+  const head = path.slice(0, -1);
+  const tail = path[path.length - 1]!;
+  const container = ensureObjectPath(target, head);
+  container[tail] = value;
+};
+
+const applySelector = (
+  target: Record<string, unknown>,
+  source: unknown,
+  selector: FieldSelector
+) => {
+  if (selector.wildcard) {
+    const value = getPathValue(source, selector.path);
+    if (value === undefined) return;
+    if (Array.isArray(value)) {
+      setPathValue(target, selector.path, value);
+      return;
+    }
+    if (!isObject(value)) {
+      return;
+    }
+    for (const [key, child] of Object.entries(value)) {
+      if (child !== undefined) {
+        setPathValue(target, [...selector.path, key], child);
+      }
+    }
+    return;
+  }
+
+  const value = getPathValue(source, selector.path);
+  if (value !== undefined) {
+    setPathValue(target, selector.path, value);
+  }
+};
+
+export const projectFields = (
+  source: unknown,
+  selectors: ReadonlyArray<FieldSelector>
+): Record<string, unknown> => {
+  const target: Record<string, unknown> = {};
+  for (const selector of selectors) {
+    applySelector(target, source, selector);
+  }
+  return target;
+};

package/src/cli/query.ts

@@ -0,0 +1,415 @@
+import { Args, Command, Options } from "@effect/cli";
+import { Chunk, Clock, Effect, Option, Ref, Stream } from "effect";
+import * as Doc from "@effect/printer/Doc";
+import { all } from "../domain/filter.js";
+import type { FilterExpr } from "../domain/filter.js";
+import { StoreQuery } from "../domain/events.js";
+import { StoreName } from "../domain/primitives.js";
+import type { Post } from "../domain/post.js";
+import { FilterRuntime } from "../services/filter-runtime.js";
+import { AppConfigService } from "../services/app-config.js";
+import { StoreIndex } from "../services/store-index.js";
+import { renderPostsMarkdown, renderPostsTable } from "../domain/format.js";
+import { renderPostCompact, renderPostCard } from "./doc/post.js";
+import { renderThread } from "./doc/thread.js";
+import { renderPlain, renderAnsi } from "./doc/render.js";
+import { parseOptionalFilterExpr } from "./filter-input.js";
+import { CliOutput, writeJson, writeJsonStream, writeText } from "./output.js";
+import { parseRange } from "./range.js";
+import { storeOptions } from "./store.js";
+import { CliPreferences } from "./preferences.js";
+import { projectFields, resolveFieldSelectors } from "./query-fields.js";
+import { CliInputError } from "./errors.js";
+import { withExamples } from "./help.js";
+import { filterOption, filterJsonOption } from "./shared-options.js";
+import { filterByFlags } from "../typeclass/chunk.js";
+
+const storeNameArg = Args.text({ name: "store" }).pipe(
+  Args.withSchema(StoreName),
+  Args.withDescription("Store name to query")
+);
+const rangeOption = Options.text("range").pipe(
+  Options.withDescription("ISO range as <start>..<end>"),
+  Options.optional
+);
+const limitOption = Options.integer("limit").pipe(
+  Options.withDescription("Maximum number of posts to return"),
+  Options.optional
+);
+const scanLimitOption = Options.integer("scan-limit").pipe(
+  Options.withDescription("Maximum rows to scan before filtering (advanced)"),
+  Options.optional
+);
+const sortOption = Options.choice("sort", ["asc", "desc"]).pipe(
+  Options.withDescription("Sort order for results (default: asc)"),
+  Options.optional
+);
+const newestFirstOption = Options.boolean("newest-first").pipe(
+  Options.withDescription("Sort newest posts first (alias for --sort desc)")
+);
+const formatOption = Options.choice("format", [
+  "json",
+  "ndjson",
+  "markdown",
+  "table",
+  "compact",
+  "card",
+  "thread"
+]).pipe(
+  Options.optional,
+  Options.withDescription("Output format (default: config output format)")
+);
+const ansiOption = Options.boolean("ansi").pipe(
+  Options.withDescription("Enable ANSI colors in output")
+);
+const widthOption = Options.integer("width").pipe(
+  Options.withDescription("Line width for terminal output"),
+  Options.optional
+);
+const fieldsOption = Options.text("fields").pipe(
+  Options.withDescription(
+    "Comma-separated fields to include (supports dot notation and presets: @minimal, @social, @full). Use author or authorProfile.handle for handles."
+  ),
+  Options.optional
+);
+const progressOption = Options.boolean("progress").pipe(
+  Options.withDescription("Show progress for filtered queries")
+);
+
+const DEFAULT_FILTER_SCAN_LIMIT = 5000;
+
+const isAscii = (value: string) => /^[\x00-\x7F]*$/.test(value);
+
+const hasUnicodeInsensitiveContains = (expr: FilterExpr): boolean => {
+  switch (expr._tag) {
+    case "Contains":
+      return !expr.caseSensitive && expr.text.length > 0 && !isAscii(expr.text);
+    case "And":
+      return (
+        hasUnicodeInsensitiveContains(expr.left) ||
+        hasUnicodeInsensitiveContains(expr.right)
+      );
+    case "Or":
+      return (
+        hasUnicodeInsensitiveContains(expr.left) ||
+        hasUnicodeInsensitiveContains(expr.right)
+      );
+    case "Not":
+      return hasUnicodeInsensitiveContains(expr.expr);
+    default:
+      return false;
+  }
+};
+
+const parseRangeOption = (range: Option.Option<string>) =>
+  Option.match(range, {
+    onNone: () => Effect.succeed(Option.none()),
+    onSome: (raw) => parseRange(raw).pipe(Effect.map(Option.some))
+  });
+
+
+export const queryCommand = Command.make(
+  "query",
+  {
+    store: storeNameArg,
+    range: rangeOption,
+    filter: filterOption,
+    filterJson: filterJsonOption,
+    limit: limitOption,
+    scanLimit: scanLimitOption,
+    sort: sortOption,
+    newestFirst: newestFirstOption,
+    format: formatOption,
+    ansi: ansiOption,
+    width: widthOption,
+    fields: fieldsOption,
+    progress: progressOption
+  },
+  ({ store, range, filter, filterJson, limit, scanLimit, sort, newestFirst, format, ansi, width, fields, progress }) =>
+    Effect.gen(function* () {
+      const appConfig = yield* AppConfigService;
+      const index = yield* StoreIndex;
+      const runtime = yield* FilterRuntime;
+      const output = yield* CliOutput;
+      const preferences = yield* CliPreferences;
+      const storeRef = yield* storeOptions.loadStoreRef(store);
+      const parsedRange = yield* parseRangeOption(range);
+      const parsedFilter = yield* parseOptionalFilterExpr(filter, filterJson);
+      const expr = Option.getOrElse(parsedFilter, () => all());
+      const outputFormat = Option.getOrElse(format, () => appConfig.outputFormat);
+      const compact = preferences.compact;
+      const selectorsOption = yield* resolveFieldSelectors(fields, compact);
+      const project = (post: Post) =>
+        Option.match(selectorsOption, {
+          onNone: () => post,
+          onSome: (selectors) => projectFields(post, selectors)
+        });
+      if (Option.isSome(selectorsOption) && outputFormat !== "json" && outputFormat !== "ndjson") {
+        return yield* CliInputError.make({
+          message: "--fields is only supported with json or ndjson output.",
+          cause: { format: outputFormat }
+        });
+      }
+
+      const w = Option.getOrUndefined(width);
+
+      if (Option.isSome(limit) && limit.value <= 0) {
+        return yield* CliInputError.make({
+          message: "--limit must be a positive integer.",
+          cause: { limit: limit.value }
+        });
+      }
+      if (Option.isSome(scanLimit) && scanLimit.value <= 0) {
+        return yield* CliInputError.make({
+          message: "--scan-limit must be a positive integer.",
+          cause: { scanLimit: scanLimit.value }
+        });
+      }
+      const sortValue = Option.getOrUndefined(sort);
+      const order =
+        newestFirst
+          ? "desc"
+          : sortValue;
+      if (newestFirst && sortValue === "asc") {
+        return yield* CliInputError.make({
+          message: "--newest-first conflicts with --sort asc.",
+          cause: { newestFirst, sort: sortValue }
+        });
+      }
+
+      const hasFilter = Option.isSome(parsedFilter);
+      if (hasFilter && hasUnicodeInsensitiveContains(expr)) {
+        yield* output
+          .writeStderr(
+            "Warning: Unicode case-insensitive contains filters cannot be pushed down; query may scan in-memory.\n"
+          )
+          .pipe(Effect.catchAll(() => Effect.void));
+      }
+      const userLimit = Option.getOrUndefined(limit);
+      const userScanLimit = Option.getOrUndefined(scanLimit);
+      const defaultScanLimit =
+        hasFilter && userScanLimit === undefined
+          ? Math.max(userLimit !== undefined ? userLimit * 50 : 0, DEFAULT_FILTER_SCAN_LIMIT)
+          : undefined;
+      const resolvedScanLimit =
+        hasFilter
+          ? userScanLimit ?? defaultScanLimit
+          : userScanLimit ?? userLimit;
+      if (defaultScanLimit !== undefined) {
+        yield* output
+          .writeStderr(
+            `ℹ️  Scanning up to ${defaultScanLimit} posts (filtered query). Use --scan-limit to scan more.`
+          )
+          .pipe(Effect.catchAll(() => Effect.void));
+      }
+
+      if (
+        hasFilter &&
+        Option.isNone(limit) &&
+        (outputFormat === "thread" || outputFormat === "table")
+      ) {
+        yield* output
+          .writeStderr(
+            "Warning: thread/table output collects all matched posts in memory. Consider adding --limit."
+          )
+          .pipe(Effect.catchAll(() => Effect.void));
+      }
+
+      const query = StoreQuery.make({
+        range: Option.getOrUndefined(parsedRange),
+        filter: Option.getOrUndefined(parsedFilter),
+        scanLimit: resolvedScanLimit,
+        order
+      });
+
+      const baseStream = index.query(storeRef, query);
+      const progressEnabled = hasFilter && progress;
+      const trackScanLimit = hasFilter && resolvedScanLimit !== undefined;
+      const scanRef = trackScanLimit
+        ? yield* Ref.make({ scanned: 0, matched: 0 })
+        : undefined;
+      let startTime = 0;
+      let progressRef: Ref.Ref<{ scanned: number; matched: number; lastReportAt: number }> | undefined;
+      if (progressEnabled) {
+        startTime = yield* Clock.currentTimeMillis;
+        progressRef = yield* Ref.make({ scanned: 0, matched: 0, lastReportAt: startTime });
+      }
+
+      const reportProgress =
+        progressEnabled && progressRef
+          ? (scanned: number, matched: number, now: number) =>
+              output
+                .writeStderr(
+                  `Query progress: scanned=${scanned} matched=${matched} elapsedMs=${now - startTime}`
+                )
+                .pipe(Effect.catchAll(() => Effect.void))
+          : undefined;
+
+      const onBatch = (scannedDelta: number, matchedDelta: number) =>
+        Effect.gen(function* () {
+          if (scanRef) {
+            yield* Ref.update(scanRef, (state) => ({
+              scanned: state.scanned + scannedDelta,
+              matched: state.matched + matchedDelta
+            }));
+          }
+          if (progressEnabled && progressRef && reportProgress) {
+            const now = yield* Clock.currentTimeMillis;
+            const state = yield* Ref.get(progressRef);
+            const scanned = state.scanned + scannedDelta;
+            const matched = state.matched + matchedDelta;
+            const shouldReport =
+              scanned % 1000 === 0 || now - state.lastReportAt >= 1000;
+            if (shouldReport) {
+              yield* reportProgress(scanned, matched, now);
+            }
+            yield* Ref.set(progressRef, {
+              scanned,
+              matched,
+              lastReportAt: shouldReport ? now : state.lastReportAt
+            });
+          }
+        });
+
+      const evaluateBatch = hasFilter ? yield* runtime.evaluateBatch(expr) : undefined;
+
+      const filtered = hasFilter && evaluateBatch
+        ? baseStream.pipe(
+            Stream.grouped(50),
+            Stream.mapEffect((batch) =>
+              evaluateBatch(batch).pipe(
+                Effect.map((flags) => {
+                  const matched = filterByFlags(batch, flags);
+                  return {
+                    matched,
+                    scanned: Chunk.size(batch),
+                    matchedCount: Chunk.size(matched)
+                  };
+                }),
+                Effect.tap(({ scanned, matchedCount }) => onBatch(scanned, matchedCount)),
+                Effect.map(({ matched }) => matched)
+              )
+            ),
+            Stream.mapConcat((chunk) => Chunk.toReadonlyArray(chunk))
+          )
+        : baseStream;
+
+      const stream = Option.match(limit, {
+        onNone: () => filtered,
+        onSome: (value) => filtered.pipe(Stream.take(value))
+      });
+
+      const warnIfScanLimitReached = scanRef && resolvedScanLimit !== undefined
+        ? () =>
+            Ref.get(scanRef).pipe(
+              Effect.flatMap((state) =>
+                state.scanned >= resolvedScanLimit
+                  ? output
+                      .writeStderr(
+                        `Warning: scan limit ${resolvedScanLimit} reached. Results may be truncated.\n`
+                      )
+                      .pipe(Effect.catchAll(() => Effect.void))
+                  : Effect.void
+              )
+            )
+        : () => Effect.void;
+
+      if (outputFormat === "ndjson") {
+        yield* writeJsonStream(stream.pipe(Stream.map(project)));
+        yield* warnIfScanLimitReached();
+        return;
+      }
+      if (outputFormat === "json") {
+        const writeChunk = (value: string) =>
+          Stream.fromIterable([value]).pipe(Stream.run(output.stdout));
+        let isFirst = true;
+        yield* writeChunk("[");
+        yield* Stream.runForEach(stream.pipe(Stream.map(project)), (post) => {
+          const json = JSON.stringify(post);
+          const prefix = isFirst ? "" : ",\n";
+          isFirst = false;
+          return writeChunk(`${prefix}${json}`);
+        });
+        const suffix = isFirst ? "]\n" : "\n]\n";
+        yield* writeChunk(suffix);
+        yield* warnIfScanLimitReached();
+        return;
+      }
+
+      switch (outputFormat) {
+        case "compact": {
+          const render = (post: Post) =>
+            ansi
+              ? renderAnsi(renderPostCompact(post), w)
+              : renderPlain(renderPostCompact(post), w);
+          yield* Stream.runForEach(stream, (post) => writeText(render(post)));
+          yield* warnIfScanLimitReached();
+          return;
+        }
+        case "card": {
+          const rendered = stream.pipe(
+            Stream.map((post) => {
+              const doc = Doc.vsep(renderPostCard(post));
+              return ansi ? renderAnsi(doc, w) : renderPlain(doc, w);
+            }),
+            Stream.mapAccum(true, (isFirst, text) => {
+              const output = isFirst ? text : `\\n${text}`;
+              return [false, output] as const;
+            })
+          );
+          yield* Stream.runForEach(rendered, (text) => writeText(text));
+          yield* warnIfScanLimitReached();
+          return;
+        }
+      }
+
+      const collected = yield* Stream.runCollect(stream);
+      const posts = Chunk.toReadonlyArray(collected);
+      const projectedPosts = Option.isSome(selectorsOption) ? posts.map(project) : posts;
+      yield* warnIfScanLimitReached();
+
+      switch (outputFormat) {
+        case "markdown":
+          yield* writeText(renderPostsMarkdown(posts));
+          return;
+        case "table":
+          yield* writeText(renderPostsTable(posts));
+          return;
+        case "thread": {
+          // B3: Warn if query doesn't have thread relationships
+          if (!hasFilter) {
+            yield* output
+              .writeStderr(
+                "ℹ️  Query results don't have thread relationships. Posts will display in chronological order.\n"
+              )
+              .pipe(Effect.catchAll(() => Effect.void));
+          }
+          const doc = renderThread(
+            posts,
+            w === undefined ? { compact: false } : { compact: false, lineWidth: w }
+          );
+          yield* writeText(ansi ? renderAnsi(doc, w) : renderPlain(doc, w));
+          return;
+        }
+        default:
+          yield* writeJson(projectedPosts);
+      }
+    })
+).pipe(
+  Command.withDescription(
+    withExamples(
+      "Query a store with optional range and filter",
+      [
+        "skygent query my-store --limit 25 --format table",
+        "skygent query my-store --range 2024-01-01T00:00:00Z..2024-01-31T00:00:00Z --filter 'hashtag:#ai'",
+        "skygent query my-store --format card --ansi",
+        "skygent query my-store --format thread --ansi --width 120",
+        "skygent query my-store --format compact --limit 50",
+        "skygent query my-store --sort desc --limit 25"
+      ],
+      [
+        "Tip: use --fields @minimal or --compact to reduce JSON output size."
+      ]
+    )
+  )
+);