@cursor/july 0.1.1

package/src/internal/cli-ax.ts

@@ -0,0 +1,474 @@
+/**
+ * Agent-experience CLI commands:
+ *   init / run / trajectory / eval
+ *
+ * Coding agents get JSON-first output, ephemeral local servers, and
+ * filesystem-first evals under evals/ (*.eval.ts). Live A/B metrics
+ * (`defineAB` under `agent/ab`) plug into serve — there is no `ab` CLI.
+ */
+
+import { mkdir, mkdtemp, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { serve } from "../index.js";
+import type { AgentServeHandle } from "../types.js";
+import { CLI_COMMAND_NAME } from "./distribution.js";
+import {
+  discoverEvals,
+  type EvalRunResult,
+  filterDiscoveredEvals,
+  runDiscoveredEvals,
+} from "./eval-runner.js";
+import { initProject, printInitGuide } from "./init-project.js";
+import { runSession } from "./run-client.js";
+import { createStreamProgress } from "./stream-progress.js";
+import {
+  formatTrajectoryText,
+  loadTrajectoryFromEventsFile,
+} from "./trajectory.js";
+
+export interface AxCliOptions {
+  cwd?: string;
+  dir: string;
+  /** Agent slug when multiple agents are mounted. */
+  slug?: string;
+  bearerToken?: string;
+  stateRoot?: string;
+  /** One or more user turns. */
+  messages?: string[];
+  sessionId?: string;
+  continuationToken?: string;
+  timeoutMs?: number;
+  /**
+   * Write raw NDJSON session events (the visualizable trace).
+   * - `true` / omitted: auto path under `<dir>/.agent-serve/traces/`
+   * - string: explicit path
+   * - `false`: skip writing
+   */
+  events?: string | true | false;
+  text?: boolean;
+  json?: boolean;
+  list?: boolean;
+  verbose?: boolean;
+  tags?: string[];
+  /** Eval ids / path prefixes. */
+  evalIds?: string[];
+  /** Scaffold target directory for `init`. */
+  initTarget?: string;
+  /** Tool name for `call`. */
+  toolName?: string;
+  /** JSON input for `call`. */
+  toolInput?: string;
+  /** Remote agent base URL (skip local host) for run/eval. */
+  url?: string;
+  /**
+   * Stream live turn progress to stderr (default: stderr is a TTY).
+   * Final JSON/text still goes to stdout.
+   */
+  stream?: boolean;
+}
+
+function shouldStreamProgress(options: AxCliOptions): boolean {
+  if (options.stream === false) {
+    return false;
+  }
+  if (options.stream === true) {
+    return true;
+  }
+  return process.stderr.isTTY === true;
+}
+
+function die(message: string): never {
+  process.stderr.write(`${message}\n`);
+  process.exit(1);
+}
+
+function pickAgentUrl(handle: AgentServeHandle, slug?: string): string {
+  if (handle.agents.length === 0) {
+    die("No agents discovered.");
+  }
+  if (slug !== undefined && slug !== "") {
+    const match = handle.agents.find((a) => a.slug === slug);
+    if (match === undefined) {
+      die(
+        `Unknown agent slug "${slug}". Available: ${handle.agents
+          .map((a) => a.slug || "(root)")
+          .join(", ")}`
+      );
+    }
+    return match.url;
+  }
+  const root = handle.agents.find((a) => a.slug === "");
+  return (root ?? handle.agents[0]!).url;
+}
+
+async function startEphemeral(
+  options: AxCliOptions
+): Promise<{ agentUrl: string; close: () => Promise<void> }> {
+  const dir = resolve(options.cwd ?? process.cwd(), options.dir);
+  const stateRoot =
+    options.stateRoot ?? (await mkdtemp(join(tmpdir(), "agent-serve-ax-")));
+
+  const handle = await serve(dir, {
+    host: "127.0.0.1",
+    port: 0,
+    stateRoot,
+    playground: false,
+    schedules: false,
+    // Keep host/tool timing off stdout so `run`/`eval` JSON stays clean.
+    logger: (line: string) => {
+      process.stderr.write(`${line}\n`);
+    },
+    ...(options.bearerToken === undefined
+      ? {}
+      : { authToken: options.bearerToken }),
+  });
+
+  return {
+    agentUrl: pickAgentUrl(handle, options.slug),
+    close: async () => {
+      // engine.close() can hang on open SDK streams; don't block the CLI.
+      await Promise.race([
+        handle.close(),
+        new Promise<void>((resolve) => setTimeout(resolve, 2_000)),
+      ]);
+    },
+  };
+}
+
+async function resolveTarget(
+  options: AxCliOptions
+): Promise<{ agentUrl: string; close?: () => Promise<void> }> {
+  if (options.url !== undefined) {
+    return { agentUrl: options.url.replace(/\/$/, "") };
+  }
+  return startEphemeral(options);
+}
+
+export async function cmdInit(options: AxCliOptions): Promise<number> {
+  const target = options.initTarget;
+  if (target === undefined || target === "") {
+    printInitGuide();
+    return 0;
+  }
+  const dir = resolve(options.cwd ?? process.cwd(), target);
+  const result = await initProject({ dir });
+  process.stdout.write(
+    `${JSON.stringify(
+      {
+        ok: true,
+        directory: result.rootDir,
+        created: result.files,
+        next: [
+          `cd ${dir === process.cwd() ? "." : dir}`,
+          `${CLI_COMMAND_NAME} validate`,
+          `${CLI_COMMAND_NAME} info --json`,
+          `${CLI_COMMAND_NAME} run --message "hello"`,
+          `${CLI_COMMAND_NAME} eval`,
+        ],
+      },
+      null,
+      2
+    )}\n`
+  );
+  return 0;
+}
+
+export async function cmdRun(options: AxCliOptions): Promise<number> {
+  const messages = options.messages ?? [];
+  if (messages.length === 0) {
+    die("Provide --message (repeatable) or --messages-file");
+  }
+
+  const stream = shouldStreamProgress(options);
+  const progress = stream ? createStreamProgress() : undefined;
+
+  const target = await resolveTarget(options);
+  try {
+    if (stream) {
+      process.stderr.write(`→ ${target.agentUrl}\n`);
+    }
+    const result = await runSession({
+      baseUrl: target.agentUrl,
+      messages,
+      ...(options.sessionId === undefined
+        ? {}
+        : { sessionId: options.sessionId }),
+      ...(options.continuationToken === undefined
+        ? {}
+        : { continuationToken: options.continuationToken }),
+      ...(options.bearerToken === undefined
+        ? {}
+        : { bearerToken: options.bearerToken }),
+      ...(options.timeoutMs === undefined
+        ? {}
+        : { timeoutMs: options.timeoutMs }),
+      ...(progress === undefined
+        ? {}
+        : { onEvent: (e) => progress.onEvent(e) }),
+    });
+
+    let eventsPath: string | undefined;
+    const writeEvents = options.events !== false;
+    if (writeEvents) {
+      if (typeof options.events === "string") {
+        eventsPath = resolve(options.events);
+      } else {
+        const root = resolve(options.cwd ?? process.cwd(), options.dir);
+        const tracesDir = join(root, ".agent-serve", "traces");
+        await mkdir(tracesDir, { recursive: true });
+        eventsPath = join(tracesDir, `${result.sessionId}.ndjson`);
+      }
+      await writeFile(
+        eventsPath,
+        `${result.events.map((e) => JSON.stringify(e)).join("\n")}\n`,
+        "utf8"
+      );
+    }
+
+    const playgroundUrl =
+      options.url !== undefined
+        ? `${target.agentUrl.replace(/\/$/, "")}/playground?sessionId=${encodeURIComponent(result.sessionId)}`
+        : null;
+
+    if (options.text) {
+      process.stdout.write(formatTrajectoryText(result.trajectory));
+      if (eventsPath !== undefined) {
+        process.stderr.write(`\ntrace: ${eventsPath}\n`);
+      }
+      if (playgroundUrl !== null) {
+        process.stderr.write(`playground: ${playgroundUrl}\n`);
+      }
+    } else {
+      console.log(
+        JSON.stringify(
+          {
+            ok: result.ok,
+            sessionId: result.sessionId,
+            continuationToken: result.continuationToken ?? null,
+            trace: eventsPath ?? null,
+            playgroundUrl,
+            visualize:
+              playgroundUrl !== null
+                ? `Open ${playgroundUrl}`
+                : eventsPath !== undefined
+                  ? `Start the playground (${CLI_COMMAND_NAME} serve --dir ${options.dir}), then Open trace → ${eventsPath}`
+                  : null,
+            trajectory: result.trajectory,
+          },
+          null,
+          2
+        )
+      );
+    }
+    return result.ok ? 0 : 1;
+  } finally {
+    progress?.close();
+    if (target.close !== undefined) {
+      await target.close();
+    }
+  }
+}
+
+/**
+ * `agent-serve call <tool>` — deterministic tool invocation over
+ * `POST /v1/tools/:toolName` (ephemeral local server unless --url).
+ */
+export async function cmdCall(options: AxCliOptions): Promise<number> {
+  const toolName = options.toolName;
+  if (toolName === undefined || toolName === "") {
+    die(
+      `Provide a tool name: ${CLI_COMMAND_NAME} call <tool> [--input '{...}']`
+    );
+  }
+  let input: unknown = {};
+  if (options.toolInput !== undefined) {
+    try {
+      input = JSON.parse(options.toolInput);
+    } catch (error) {
+      die(
+        `--input must be valid JSON: ${
+          error instanceof Error ? error.message : String(error)
+        }`
+      );
+    }
+  }
+
+  const target = await resolveTarget(options);
+  try {
+    const response = await fetch(
+      `${target.agentUrl}/v1/tools/${encodeURIComponent(toolName)}`,
+      {
+        method: "POST",
+        headers: {
+          "content-type": "application/json",
+          ...(options.bearerToken === undefined
+            ? {}
+            : { authorization: `Bearer ${options.bearerToken}` }),
+        },
+        body: JSON.stringify({
+          input,
+          ...(options.sessionId === undefined
+            ? {}
+            : { sessionId: options.sessionId }),
+        }),
+      }
+    );
+    const payload: unknown = await response
+      .json()
+      .catch(() => ({ ok: false, error: "invalid_response" }));
+    console.log(JSON.stringify(payload, null, 2));
+    const ok =
+      response.ok &&
+      typeof payload === "object" &&
+      payload !== null &&
+      (payload as { ok?: unknown }).ok === true;
+    return ok ? 0 : 1;
+  } finally {
+    if (target.close !== undefined) {
+      await target.close();
+    }
+  }
+}
+
+export async function cmdTrajectory(options: AxCliOptions): Promise<number> {
+  if (typeof options.events !== "string") {
+    die(
+      `trajectory requires --events <file.ndjson> (write one with: ${CLI_COMMAND_NAME} run --events …)`
+    );
+  }
+  const trajectory = await loadTrajectoryFromEventsFile(
+    resolve(options.events)
+  );
+  if (options.text) {
+    process.stdout.write(formatTrajectoryText(trajectory));
+  } else {
+    console.log(JSON.stringify(trajectory, null, 2));
+  }
+  return trajectory.ok ? 0 : 1;
+}
+
+export async function cmdEval(options: AxCliOptions): Promise<number> {
+  const projectRoot = resolve(options.cwd ?? process.cwd(), options.dir);
+
+  if (options.list) {
+    const { evals } = await discoverEvals(projectRoot);
+    const listed = filterDiscoveredEvals(evals, {
+      filterIds: options.evalIds,
+      tags: options.tags,
+    });
+    if (options.json) {
+      console.log(
+        JSON.stringify(
+          listed.map((e) => ({
+            id: e.id,
+            fileId: e.fileId,
+            path: e.path,
+            description: e.definition.description,
+            tags: e.definition.tags ?? [],
+          })),
+          null,
+          2
+        )
+      );
+    } else if (listed.length === 0) {
+      process.stdout.write(`No evals under ${projectRoot}/evals\n`);
+    } else {
+      for (const e of listed) {
+        const tags =
+          e.definition.tags && e.definition.tags.length > 0
+            ? `  [${e.definition.tags.join(", ")}]`
+            : "";
+        const desc =
+          e.definition.description !== undefined
+            ? ` — ${e.definition.description}`
+            : "";
+        process.stdout.write(`${e.id}${tags}${desc}\n`);
+      }
+    }
+    return 0;
+  }
+
+  const target = await resolveTarget(options);
+  const stream = shouldStreamProgress(options);
+  const progress = stream ? createStreamProgress() : undefined;
+  try {
+    if (options.verbose || stream) {
+      process.stderr.write(`eval target: ${target.agentUrl}\n`);
+    }
+    const results = await runDiscoveredEvals({
+      projectRoot,
+      baseUrl: target.agentUrl,
+      ...(options.evalIds === undefined || options.evalIds.length === 0
+        ? {}
+        : { filterIds: options.evalIds }),
+      ...(options.tags === undefined || options.tags.length === 0
+        ? {}
+        : { tags: options.tags }),
+      ...(options.timeoutMs === undefined
+        ? {}
+        : { timeoutMs: options.timeoutMs }),
+      ...(options.bearerToken === undefined
+        ? {}
+        : { bearerToken: options.bearerToken }),
+      verbose: Boolean(options.verbose) || stream,
+      ...(progress === undefined
+        ? {}
+        : { onEvent: (e) => progress.onEvent(e) }),
+    });
+
+    if (results.length === 0) {
+      process.stderr.write("No matching evals found.\n");
+      return 2;
+    }
+    return printEvalResults(results, options);
+  } finally {
+    progress?.close();
+    if (target.close !== undefined) {
+      await target.close();
+    }
+  }
+}
+
+function printEvalResults(
+  results: EvalRunResult[],
+  options: AxCliOptions
+): number {
+  const passed = results.filter((r) => r.ok).length;
+  const failed = results.length - passed;
+
+  if (options.json) {
+    console.log(
+      JSON.stringify({ ok: failed === 0, passed, failed, results }, null, 2)
+    );
+  } else {
+    for (const r of results) {
+      process.stdout.write(
+        `${r.ok ? "PASS" : "FAIL"}  ${r.id} (${(r.durationMs / 1000).toFixed(1)}s)`
+      );
+      if (r.description !== undefined) {
+        process.stdout.write(` — ${r.description}`);
+      }
+      process.stdout.write("\n");
+      for (const a of r.assertions) {
+        process.stdout.write(
+          `  ${a.passed ? "✓" : "✗"} ${a.name}${
+            a.detail !== undefined ? ` (${a.detail})` : ""
+          }\n`
+        );
+      }
+      if (r.error !== undefined) {
+        process.stdout.write(`  error: ${r.error}\n`);
+      }
+      if (options.verbose && r.finalText !== undefined) {
+        process.stdout.write(
+          `  reply: ${r.finalText.slice(0, 200).replace(/\s+/g, " ")}\n`
+        );
+      }
+    }
+    process.stdout.write(
+      `\n${passed} passed, ${failed} failed, ${results.length} total\n`
+    );
+  }
+  return failed === 0 ? 0 : 1;
+}