@cursor/july 0.1.5 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. package/dist/ab.d.ts +8 -95
  2. package/dist/ab.d.ts.map +1 -1
  3. package/dist/ab.js +9 -150
  4. package/dist/bin/agent-serve.js +41 -8
  5. package/dist/channels/slack/post-update-delivery.d.ts +85 -0
  6. package/dist/channels/slack/post-update-delivery.d.ts.map +1 -0
  7. package/dist/docs/404.html +2 -2
  8. package/dist/docs/ab.html +4 -4
  9. package/dist/docs/assets/{app.DabPG-io.js → app.COTN7wgo.js} +1 -1
  10. package/dist/docs/assets/chunks/@localSearchIndexroot.B7UcKvIn.js +1 -0
  11. package/dist/docs/assets/chunks/{VPLocalSearchBox.jmyr0bU0.js → VPLocalSearchBox.BW3TBdT0.js} +1 -1
  12. package/dist/docs/assets/chunks/{theme.DysN9-VN.js → theme.BEJW0vE7.js} +2 -2
  13. package/dist/docs/assets/deployment.md.BtfEsc9S.js +55 -0
  14. package/dist/docs/assets/deployment.md.BtfEsc9S.lean.js +1 -0
  15. package/dist/docs/assets/example-agents_approval-buddy.md.8R5phXb5.js +10 -0
  16. package/dist/docs/assets/example-agents_approval-buddy.md.8R5phXb5.lean.js +1 -0
  17. package/dist/docs/assets/example-agents_benny.md.B0gjhI-p.js +7 -0
  18. package/dist/docs/assets/example-agents_benny.md.B0gjhI-p.lean.js +1 -0
  19. package/dist/docs/assets/example-agents_bugbot.md.DelIdhxB.js +11 -0
  20. package/dist/docs/assets/example-agents_bugbot.md.DelIdhxB.lean.js +1 -0
  21. package/dist/docs/assets/example-agents_codebase-wiki.md.DC6sgwn0.js +8 -0
  22. package/dist/docs/assets/example-agents_codebase-wiki.md.DC6sgwn0.lean.js +1 -0
  23. package/dist/docs/assets/example-agents_codeowners-review.md.Ku_tG2RY.js +8 -0
  24. package/dist/docs/assets/example-agents_codeowners-review.md.Ku_tG2RY.lean.js +1 -0
  25. package/dist/docs/assets/example-agents_concierge.md.4rQTSMXt.js +23 -0
  26. package/dist/docs/assets/example-agents_concierge.md.4rQTSMXt.lean.js +1 -0
  27. package/dist/docs/assets/example-agents_fsd.md.CzgUrDfi.js +15 -0
  28. package/dist/docs/assets/example-agents_fsd.md.CzgUrDfi.lean.js +1 -0
  29. package/dist/docs/assets/example-agents_index.md.CRqJlnIf.js +2 -0
  30. package/dist/docs/assets/example-agents_index.md.CRqJlnIf.lean.js +1 -0
  31. package/dist/docs/assets/example-agents_knowledge-base.md.BPJiVueF.js +11 -0
  32. package/dist/docs/assets/example-agents_knowledge-base.md.BPJiVueF.lean.js +1 -0
  33. package/dist/docs/assets/example-agents_security-reviewer.md.D2rtwDTO.js +19 -0
  34. package/dist/docs/assets/example-agents_security-reviewer.md.D2rtwDTO.lean.js +1 -0
  35. package/dist/docs/assets/example-agents_slack-agent.md.buLbgvBf.js +5 -0
  36. package/dist/docs/assets/example-agents_slack-agent.md.buLbgvBf.lean.js +1 -0
  37. package/dist/docs/assets/example-agents_weather-agent.md.C9Qv-W0o.js +24 -0
  38. package/dist/docs/assets/example-agents_weather-agent.md.C9Qv-W0o.lean.js +1 -0
  39. package/dist/docs/assets/index.md.COiu-1jL.js +20 -0
  40. package/dist/docs/assets/{index.md.Cylk70gg.lean.js → index.md.COiu-1jL.lean.js} +1 -1
  41. package/dist/docs/assets/reference_cli.md.D189RBCH.js +60 -0
  42. package/dist/docs/assets/reference_cli.md.D189RBCH.lean.js +1 -0
  43. package/dist/docs/building-with-agents.html +4 -4
  44. package/dist/docs/concepts.html +4 -4
  45. package/dist/docs/deployment.html +58 -17
  46. package/dist/docs/evals.html +4 -4
  47. package/dist/docs/example-agents/approval-buddy.html +34 -0
  48. package/dist/docs/example-agents/benny.html +31 -0
  49. package/dist/docs/example-agents/bugbot.html +35 -0
  50. package/dist/docs/example-agents/codebase-wiki.html +32 -0
  51. package/dist/docs/example-agents/codeowners-review.html +32 -0
  52. package/dist/docs/example-agents/concierge.html +47 -0
  53. package/dist/docs/example-agents/fsd.html +39 -0
  54. package/dist/docs/example-agents/index.html +26 -0
  55. package/dist/docs/example-agents/knowledge-base.html +35 -0
  56. package/dist/docs/example-agents/security-reviewer.html +43 -0
  57. package/dist/docs/example-agents/slack-agent.html +29 -0
  58. package/dist/docs/example-agents/weather-agent.html +48 -0
  59. package/dist/docs/guides/agent-to-agent.html +4 -4
  60. package/dist/docs/guides/cloud-runtime.html +5 -5
  61. package/dist/docs/guides/github.html +4 -4
  62. package/dist/docs/guides/human-in-the-loop.html +4 -4
  63. package/dist/docs/guides/slack.html +4 -4
  64. package/dist/docs/guides/webhooks.html +4 -4
  65. package/dist/docs/hashmap.json +1 -1
  66. package/dist/docs/hillclimbing.html +4 -4
  67. package/dist/docs/index.html +7 -7
  68. package/dist/docs/quickstart.html +4 -4
  69. package/dist/docs/reference/agent-config.html +4 -4
  70. package/dist/docs/reference/channels.html +4 -4
  71. package/dist/docs/reference/cli.html +52 -30
  72. package/dist/docs/reference/connections.html +4 -4
  73. package/dist/docs/reference/hooks.html +4 -4
  74. package/dist/docs/reference/http-api.html +4 -4
  75. package/dist/docs/reference/instructions.html +4 -4
  76. package/dist/docs/reference/playground.html +4 -4
  77. package/dist/docs/reference/project-layout.html +4 -4
  78. package/dist/docs/reference/schedules.html +4 -4
  79. package/dist/docs/reference/sessions.html +4 -4
  80. package/dist/docs/reference/skills.html +4 -4
  81. package/dist/docs/reference/subagents.html +4 -4
  82. package/dist/docs/reference/tools.html +4 -4
  83. package/dist/docs/scaffolding-agents.html +4 -4
  84. package/dist/docs/storage.html +4 -4
  85. package/dist/docs/troubleshooting.html +4 -4
  86. package/dist/evals.d.ts +5 -62
  87. package/dist/evals.d.ts.map +1 -1
  88. package/dist/evals.js +3 -66
  89. package/dist/index.d.ts +1 -1
  90. package/dist/index.d.ts.map +1 -1
  91. package/dist/internal/ab-collector.d.ts +7 -5
  92. package/dist/internal/ab-collector.d.ts.map +1 -1
  93. package/dist/internal/ab-collector.js +3 -14
  94. package/dist/internal/ab-snapshot.d.ts +2 -4
  95. package/dist/internal/ab-snapshot.d.ts.map +1 -1
  96. package/dist/internal/cli-ax.d.ts +33 -5
  97. package/dist/internal/cli-ax.d.ts.map +1 -1
  98. package/dist/internal/cli-ax.js +428 -87
  99. package/dist/internal/cli-deploy.js +1 -1
  100. package/dist/internal/discovery.js +3 -3
  101. package/dist/internal/eval-run-store.d.ts +35 -30
  102. package/dist/internal/eval-run-store.d.ts.map +1 -1
  103. package/dist/internal/eval-run-store.js +88 -100
  104. package/dist/internal/evals-client.d.ts +96 -0
  105. package/dist/internal/evals-client.d.ts.map +1 -0
  106. package/dist/internal/evals-client.js +262 -0
  107. package/dist/internal/init-project.d.ts.map +1 -1
  108. package/dist/internal/init-project.js +1 -0
  109. package/dist/internal/persistence-coordinator.d.ts +127 -0
  110. package/dist/internal/persistence-coordinator.d.ts.map +1 -0
  111. package/dist/internal/playground-proxy.d.ts +5 -5
  112. package/dist/internal/playground-proxy.js +3 -3
  113. package/dist/internal/resolve-prod-target.d.ts +30 -0
  114. package/dist/internal/resolve-prod-target.d.ts.map +1 -1
  115. package/dist/internal/resolve-prod-target.js +74 -2
  116. package/dist/internal/server.d.ts.map +1 -1
  117. package/dist/internal/server.js +16 -5
  118. package/dist/internal/session-engine.d.ts +1 -2
  119. package/dist/internal/session-engine.d.ts.map +1 -1
  120. package/dist/internal/session-engine.js +14 -31
  121. package/dist/internal/storage-coordinator.d.ts +16 -15
  122. package/dist/internal/storage-coordinator.d.ts.map +1 -1
  123. package/dist/internal/storage-coordinator.js +73 -80
  124. package/dist/persistence.d.ts +184 -0
  125. package/dist/persistence.d.ts.map +1 -0
  126. package/dist/playground/assets/cursor-icons-16-CQ50JpfO.woff2 +0 -0
  127. package/dist/playground/assets/index-72vCOBWO.js +86 -0
  128. package/dist/playground/assets/index-BjnMwYoR.css +1 -0
  129. package/dist/playground/index.html +2 -2
  130. package/dist/storage.d.ts +51 -10
  131. package/dist/storage.d.ts.map +1 -1
  132. package/dist/storage.js +27 -10
  133. package/docs/README.md +34 -5
  134. package/docs/deployment.md +352 -149
  135. package/docs/example-agents/approval-buddy.md +270 -0
  136. package/docs/example-agents/benny.md +186 -0
  137. package/docs/example-agents/bugbot.md +231 -0
  138. package/docs/example-agents/codebase-wiki.md +174 -0
  139. package/docs/example-agents/codeowners-review.md +195 -0
  140. package/docs/example-agents/concierge.md +205 -0
  141. package/docs/example-agents/fsd.md +330 -0
  142. package/docs/example-agents/index.md +102 -0
  143. package/docs/example-agents/knowledge-base.md +171 -0
  144. package/docs/example-agents/security-reviewer.md +296 -0
  145. package/docs/example-agents/slack-agent.md +146 -0
  146. package/docs/example-agents/weather-agent.md +302 -0
  147. package/docs/reference/cli.md +546 -147
  148. package/package.json +1 -1
  149. package/src/ab.ts +9 -261
  150. package/src/bin/agent-serve.ts +46 -7
  151. package/src/evals.ts +5 -119
  152. package/src/index.ts +2 -0
  153. package/src/internal/ab-collector.ts +12 -22
  154. package/src/internal/ab-snapshot.ts +2 -4
  155. package/src/internal/cli-ax.ts +551 -104
  156. package/src/internal/cli-deploy.ts +1 -1
  157. package/src/internal/discovery.ts +2 -2
  158. package/src/internal/eval-run-store.ts +91 -100
  159. package/src/internal/evals-client.ts +431 -0
  160. package/src/internal/init-project.ts +1 -0
  161. package/src/internal/playground-proxy.ts +5 -5
  162. package/src/internal/resolve-prod-target.ts +101 -3
  163. package/src/internal/server.ts +17 -3
  164. package/src/internal/session-engine.ts +9 -29
  165. package/src/internal/storage-coordinator.ts +109 -101
  166. package/src/storage.ts +79 -14
  167. package/dist/docs/assets/chunks/@localSearchIndexroot.QwK5BtEH.js +0 -1
  168. package/dist/docs/assets/deployment.md.DTKwE15Z.js +0 -14
  169. package/dist/docs/assets/deployment.md.DTKwE15Z.lean.js +0 -1
  170. package/dist/docs/assets/index.md.Cylk70gg.js +0 -20
  171. package/dist/docs/assets/reference_cli.md.Bv6pOxcF.js +0 -38
  172. package/dist/docs/assets/reference_cli.md.Bv6pOxcF.lean.js +0 -1
  173. package/dist/internal/json-dir-store.js +0 -100
  174. package/dist/playground/assets/cursor-icons-outline-BxTT_FVJ.woff2 +0 -0
  175. package/dist/playground/assets/index-BEauYlII.css +0 -1
  176. package/dist/playground/assets/index-BtM0wEGg.js +0 -319
  177. package/src/internal/json-dir-store.ts +0 -109
@@ -25,12 +25,13 @@ import { resolveDeployAuth } from "./cli-deploy.js";
25
25
  import { resolveApiKey } from "./cursor/credentials.js";
26
26
  import { CLI_COMMAND_NAME } from "./distribution.js";
27
27
  import { discoverEvals, filterDiscoveredEvals, runDiscoveredEvals, } from "./eval-runner.js";
28
+ import { getRemoteEvalRun, listRemoteEvalRuns, listRemoteEvals, RemoteEvalError, runRemoteEvals, startRemoteEvalRun, } from "./evals-client.js";
28
29
  import { formatInitNextSteps, formatInitScaffoldSummary, initNextCommands, initProject, installScaffoldDependencies, printInitGuide, } from "./init-project.js";
29
30
  import { offerInstallCursorSkills } from "./install-cursor-skills.js";
30
31
  import { runLogsTail, sanitizeCustomerError } from "./logs-client.js";
31
32
  import { openBrowser } from "./open-browser.js";
32
33
  import { startPlaygroundProxy } from "./playground-proxy.js";
33
- import { resolveProdSlug, resolveProdTarget } from "./resolve-prod-target.js";
34
+ import { ProdTargetError, resolveProdAliasPlaygroundUrl, resolveProdSlug, resolveProdTarget, } from "./resolve-prod-target.js";
34
35
  import { runSession } from "./run-client.js";
35
36
  import { fetchSessionEvents, formatSessionsTable, listSessions, resolvePreviousSessionId, } from "./sessions-client.js";
36
37
  import { createStreamProgress } from "./stream-progress.js";
@@ -475,82 +476,451 @@ export function cmdTrajectory(options) {
475
476
  return trajectory.ok ? 0 : 1;
476
477
  });
477
478
  }
479
+ /**
480
+ * Which surface `eval` should target. Local (ephemeral) is the default;
481
+ * `--prod` uses the hosted deployment, `--url` a running server.
482
+ * Exported for tests.
483
+ */
484
+ export function resolveEvalTargetMode(options) {
485
+ const flags = [
486
+ options.prod === true ? "--prod" : undefined,
487
+ options.url !== undefined ? "--url" : undefined,
488
+ ].filter((flag) => flag !== undefined);
489
+ if (flags.length > 1) {
490
+ return { error: `${flags.join(" and ")} are mutually exclusive` };
491
+ }
492
+ if (options.url !== undefined) {
493
+ return "url";
494
+ }
495
+ if (options.prod === true) {
496
+ return "deployment";
497
+ }
498
+ return "local";
499
+ }
500
+ /** Resolve the eval target; prints a CLI-ready error and returns an exit code on failure. */
501
+ function resolveEvalTarget(options) {
502
+ return __awaiter(this, void 0, void 0, function* () {
503
+ const mode = resolveEvalTargetMode(options);
504
+ if (typeof mode === "object") {
505
+ process.stderr.write(`${mode.error}\n`);
506
+ return 2;
507
+ }
508
+ if (mode === "url") {
509
+ return { agentUrl: options.url.replace(/\/$/, "") };
510
+ }
511
+ if (mode === "local") {
512
+ return startEphemeral(options);
513
+ }
514
+ try {
515
+ const prod = yield resolveProdTarget({
516
+ dir: options.dir,
517
+ cwd: options.cwd,
518
+ slug: options.slug,
519
+ team: options.team,
520
+ apiKey: options.apiKey,
521
+ fetchImpl: options.fetchImpl,
522
+ });
523
+ return { agentUrl: prod.agentUrl, headers: prod.headers };
524
+ }
525
+ catch (error) {
526
+ if (error instanceof ProdTargetError && error.notFound) {
527
+ // Keep the API error text: a 404 can also mean a wrong API host or a
528
+ // disabled feature gate, and the message distinguishes those from a
529
+ // genuinely missing deployment.
530
+ process.stderr.write(`${sanitizeCustomerError(error.message)}\n` +
531
+ ` deploy it first: ${CLI_COMMAND_NAME} deploy --slug ${error.slug}\n` +
532
+ ` or omit --prod: ${CLI_COMMAND_NAME} eval --dir ${options.dir}\n`);
533
+ return 1;
534
+ }
535
+ process.stderr.write(`${sanitizeCustomerError(error instanceof Error ? error.message : String(error))}\n` +
536
+ `Pass --prod for the hosted deployment, or omit it to run on an ephemeral local server.\n`);
537
+ return 1;
538
+ }
539
+ });
540
+ }
478
541
  export function cmdEval(options) {
479
542
  return __awaiter(this, void 0, void 0, function* () {
480
543
  var _a;
481
544
  const projectRoot = resolve((_a = options.cwd) !== null && _a !== void 0 ? _a : process.cwd(), options.dir);
545
+ const mode = resolveEvalTargetMode(options);
546
+ if (typeof mode === "object") {
547
+ process.stderr.write(`${mode.error}\n`);
548
+ return 2;
549
+ }
550
+ if (options.noWait === true) {
551
+ if (options.list === true) {
552
+ process.stderr.write("--no-wait and --list are mutually exclusive\n");
553
+ return 2;
554
+ }
555
+ if (mode === "local") {
556
+ process.stderr.write("--no-wait requires --prod or --url (not the default local target)\n");
557
+ return 2;
558
+ }
559
+ }
560
+ const filterIds = options.evalIds === undefined || options.evalIds.length === 0
561
+ ? undefined
562
+ : options.evalIds;
563
+ const tags = options.tags === undefined || options.tags.length === 0
564
+ ? undefined
565
+ : options.tags;
482
566
  if (options.list) {
483
- const { evals } = yield discoverEvals(projectRoot);
484
- const listed = filterDiscoveredEvals(evals, {
485
- filterIds: options.evalIds,
486
- tags: options.tags,
487
- });
488
- if (options.json) {
489
- console.log(JSON.stringify(listed.map((e) => { var _a; return ({
567
+ // List the same surface a run will execute: local filesystem by default,
568
+ // GET /v1/dev/evals on --prod / --url.
569
+ if (mode === "local") {
570
+ return listLocalEvals(projectRoot, options, filterIds, tags);
571
+ }
572
+ const target = yield resolveEvalTarget(options);
573
+ if (typeof target === "number") {
574
+ return target;
575
+ }
576
+ try {
577
+ const listed = yield listRemoteEvals({
578
+ baseUrl: target.agentUrl,
579
+ auth: {
580
+ bearerToken: options.bearerToken,
581
+ headers: target.headers,
582
+ },
583
+ fetchImpl: options.fetchImpl,
584
+ filterIds,
585
+ tags,
586
+ });
587
+ return printListedEvals(listed.map((e) => { var _a; return ({
490
588
  id: e.id,
491
589
  fileId: e.fileId,
492
- path: e.path,
493
- description: e.definition.description,
494
- tags: (_a = e.definition.tags) !== null && _a !== void 0 ? _a : [],
495
- }); }), null, 2));
590
+ description: e.description,
591
+ tags: (_a = e.tags) !== null && _a !== void 0 ? _a : [],
592
+ }); }), options, `No evals on ${target.agentUrl}`);
496
593
  }
497
- else if (listed.length === 0) {
498
- process.stdout.write(`No evals under ${projectRoot}/evals\n`);
594
+ catch (error) {
595
+ process.stderr.write(`${sanitizeCustomerError(error instanceof Error ? error.message : String(error))}\n`);
596
+ return 1;
499
597
  }
500
- else {
501
- for (const e of listed) {
502
- const tags = e.definition.tags && e.definition.tags.length > 0
503
- ? ` [${e.definition.tags.join(", ")}]`
504
- : "";
505
- const desc = e.definition.description !== undefined
506
- ? ` — ${e.definition.description}`
507
- : "";
508
- process.stdout.write(`${e.id}${tags}${desc}\n`);
598
+ finally {
599
+ if (target.close !== undefined) {
600
+ yield target.close();
509
601
  }
510
602
  }
511
- return 0;
512
603
  }
513
- const target = yield resolveTarget(options);
604
+ const target = yield resolveEvalTarget(options);
605
+ if (typeof target === "number") {
606
+ return target;
607
+ }
514
608
  const stream = shouldStreamProgress(options);
515
- const progress = stream ? createStreamProgress() : undefined;
516
609
  try {
517
610
  if (options.verbose || stream) {
518
611
  process.stderr.write(`eval target: ${target.agentUrl}\n`);
519
612
  }
520
- const results = yield runDiscoveredEvals({
521
- projectRoot,
613
+ // Default local: ephemeral in-process harness against local eval sources.
614
+ // --prod / --url: run on the serve process via /v1/dev/evals/runs so the
615
+ // batch lands in playground history and defineStorage evals (when set).
616
+ if (mode === "local") {
617
+ const progress = stream ? createStreamProgress() : undefined;
618
+ try {
619
+ const results = yield runDiscoveredEvals({
620
+ projectRoot,
621
+ baseUrl: target.agentUrl,
622
+ filterIds,
623
+ tags,
624
+ timeoutMs: options.timeoutMs,
625
+ bearerToken: options.bearerToken,
626
+ headers: target.headers,
627
+ verbose: Boolean(options.verbose) || stream,
628
+ onEvent: progress === undefined ? undefined : (e) => progress.onEvent(e),
629
+ });
630
+ if (results.length === 0) {
631
+ process.stderr.write("No matching evals found.\n");
632
+ return 2;
633
+ }
634
+ return printEvalResults(results, options);
635
+ }
636
+ finally {
637
+ progress === null || progress === void 0 ? void 0 : progress.close();
638
+ }
639
+ }
640
+ try {
641
+ if (options.noWait === true) {
642
+ const started = yield startRemoteEvalRun({
643
+ baseUrl: target.agentUrl,
644
+ filterIds,
645
+ tags,
646
+ timeoutMs: options.timeoutMs,
647
+ verbose: Boolean(options.verbose),
648
+ auth: {
649
+ bearerToken: options.bearerToken,
650
+ headers: target.headers,
651
+ },
652
+ fetchImpl: options.fetchImpl,
653
+ });
654
+ return printEvalAccepted(started, options, target.agentUrl);
655
+ }
656
+ const progress = createRemoteEvalProgress(options.verbose === true || stream);
657
+ const { snapshot, results } = yield runRemoteEvals({
658
+ baseUrl: target.agentUrl,
659
+ filterIds,
660
+ tags,
661
+ timeoutMs: options.timeoutMs,
662
+ verbose: Boolean(options.verbose) || stream,
663
+ bearerToken: options.bearerToken,
664
+ headers: target.headers,
665
+ fetchImpl: options.fetchImpl,
666
+ onSnapshot: progress === undefined ? undefined : progress.onSnapshot,
667
+ });
668
+ if (results.length === 0 && snapshot.status !== "failed") {
669
+ process.stderr.write("No matching evals found.\n");
670
+ return 2;
671
+ }
672
+ return printEvalResults(results, options, {
673
+ runId: snapshot.runId,
674
+ status: snapshot.status,
675
+ error: snapshot.error,
676
+ });
677
+ }
678
+ catch (error) {
679
+ if (error instanceof RemoteEvalError &&
680
+ error.code === "no_matching_evals") {
681
+ process.stderr.write(`${error.message}\n`);
682
+ return 2;
683
+ }
684
+ process.stderr.write(`${sanitizeCustomerError(error instanceof Error ? error.message : String(error))}\n`);
685
+ return 1;
686
+ }
687
+ }
688
+ finally {
689
+ if (target.close !== undefined) {
690
+ yield target.close();
691
+ }
692
+ }
693
+ });
694
+ }
695
+ /**
696
+ * Flags to re-target the same surface from an `eval --no-wait` hint.
697
+ * Exported for tests.
698
+ */
699
+ export function formatEvalStatusFlags(options) {
700
+ const parts = [];
701
+ if (options.url !== undefined) {
702
+ parts.push(`--url ${options.url.replace(/\/$/, "")}`);
703
+ }
704
+ else {
705
+ parts.push("--prod");
706
+ if (options.slug !== undefined && options.slug !== "") {
707
+ parts.push(`--slug ${options.slug}`);
708
+ }
709
+ }
710
+ if (options.team !== undefined && options.team !== "") {
711
+ parts.push(`--team ${options.team}`);
712
+ }
713
+ return parts.length === 0 ? "" : ` ${parts.join(" ")}`;
714
+ }
715
+ /** `--no-wait`: print the accepted batch id (and how to poll) then exit. */
716
+ function printEvalAccepted(started, options, agentUrl) {
717
+ const pollPath = `${agentUrl.replace(/\/$/, "")}/v1/dev/evals/runs/${started.runId}`;
718
+ const statusFlags = formatEvalStatusFlags(options);
719
+ if (options.json) {
720
+ console.log(JSON.stringify({
721
+ ok: true,
722
+ accepted: true,
723
+ runId: started.runId,
724
+ status: started.status,
725
+ summary: started.summary,
726
+ pollUrl: pollPath,
727
+ durableRuns: started.config.durableRuns === true,
728
+ }, null, 2));
729
+ }
730
+ else {
731
+ const durable = started.config.durableRuns === true ? " (persisted)" : "";
732
+ process.stdout.write(`Accepted eval run ${started.runId}${durable} — ${started.summary.total} case(s)\n`);
733
+ process.stdout.write(`Status: ${CLI_COMMAND_NAME} eval status ${started.runId}${statusFlags}\n`);
734
+ process.stdout.write(`Poll: ${pollPath}\n`);
735
+ }
736
+ return 0;
737
+ }
738
+ /**
739
+ * `eval status [runId]` — fetch one batch (or list recent) on the deployment /
740
+ * `--url` target. Exit `0` when the batch is completed with all cases passing,
741
+ * `1` when failed / any case failed / unknown, `2` for usage errors, `3` when
742
+ * the batch is still running (for poll loops after `--no-wait`).
743
+ */
744
+ export function cmdEvalStatus(runId, options) {
745
+ return __awaiter(this, void 0, void 0, function* () {
746
+ const mode = resolveEvalTargetMode(options);
747
+ if (typeof mode === "object") {
748
+ process.stderr.write(`${mode.error}\n`);
749
+ return 2;
750
+ }
751
+ if (mode === "local") {
752
+ process.stderr.write("eval status requires --prod or --url (not the default local target)\n");
753
+ return 2;
754
+ }
755
+ if (options.list === true || options.noWait === true) {
756
+ process.stderr.write("eval status does not accept --list or --no-wait\n");
757
+ return 2;
758
+ }
759
+ const target = yield resolveEvalTarget(options);
760
+ if (typeof target === "number") {
761
+ return target;
762
+ }
763
+ const auth = {
764
+ bearerToken: options.bearerToken,
765
+ headers: target.headers,
766
+ };
767
+ try {
768
+ if (runId === undefined || runId === "") {
769
+ const { runs, activeRunId } = yield listRemoteEvalRuns({
770
+ baseUrl: target.agentUrl,
771
+ auth,
772
+ fetchImpl: options.fetchImpl,
773
+ });
774
+ if (options.json) {
775
+ console.log(JSON.stringify({ runs, activeRunId }, null, 2));
776
+ }
777
+ else if (runs.length === 0) {
778
+ process.stdout.write("No eval runs on this agent.\n");
779
+ }
780
+ else {
781
+ if (activeRunId !== undefined) {
782
+ process.stdout.write(`active: ${activeRunId}\n`);
783
+ }
784
+ for (const run of runs) {
785
+ process.stdout.write(`${formatEvalRunSummaryLine(run)}\n`);
786
+ }
787
+ }
788
+ return 0;
789
+ }
790
+ const snap = yield getRemoteEvalRun({
522
791
  baseUrl: target.agentUrl,
523
- filterIds: options.evalIds === undefined || options.evalIds.length === 0
524
- ? undefined
525
- : options.evalIds,
526
- tags: options.tags === undefined || options.tags.length === 0
527
- ? undefined
528
- : options.tags,
529
- timeoutMs: options.timeoutMs,
530
- bearerToken: options.bearerToken,
531
- headers: target.headers,
532
- verbose: Boolean(options.verbose) || stream,
533
- onEvent: progress === undefined ? undefined : (e) => progress.onEvent(e),
792
+ runId,
793
+ auth,
794
+ fetchImpl: options.fetchImpl,
534
795
  });
535
- if (results.length === 0) {
536
- process.stderr.write("No matching evals found.\n");
537
- return 2;
796
+ if (options.json) {
797
+ console.log(JSON.stringify(snap, null, 2));
798
+ }
799
+ else {
800
+ process.stdout.write(`${formatEvalRunSummaryLine(snap)}\n`);
801
+ if (snap.error !== undefined && snap.error !== "") {
802
+ process.stdout.write(`error: ${snap.error}\n`);
803
+ }
804
+ for (const c of snap.cases) {
805
+ const mark = c.status !== "done" ? c.status : c.ok === true ? "PASS" : "FAIL";
806
+ const dur = c.durationMs === undefined
807
+ ? ""
808
+ : ` (${(c.durationMs / 1000).toFixed(1)}s)`;
809
+ process.stdout.write(` ${mark} ${c.id}${dur}\n`);
810
+ if (c.error !== undefined && c.error !== "") {
811
+ process.stdout.write(` error: ${c.error}\n`);
812
+ }
813
+ }
814
+ }
815
+ if (snap.status === "running") {
816
+ return 3;
538
817
  }
539
- return printEvalResults(results, options);
818
+ const failed = snap.status === "failed" || snap.cases.some((c) => c.ok !== true);
819
+ return failed ? 1 : 0;
820
+ }
821
+ catch (error) {
822
+ process.stderr.write(`${sanitizeCustomerError(error instanceof Error ? error.message : String(error))}\n`);
823
+ return 1;
540
824
  }
541
825
  finally {
542
- progress === null || progress === void 0 ? void 0 : progress.close();
543
826
  if (target.close !== undefined) {
544
827
  yield target.close();
545
828
  }
546
829
  }
547
830
  });
548
831
  }
549
- function printEvalResults(results, options) {
832
+ function formatEvalRunSummaryLine(run) {
833
+ const { passed, failed, total, done } = run.summary;
834
+ const durable = run.config.durableRuns === true ? " persisted" : "";
835
+ return `${run.runId} ${run.status} ${done}/${total} done (${passed} passed, ${failed} failed)${durable}`;
836
+ }
837
+ function listLocalEvals(projectRoot, options, filterIds, tags) {
838
+ return __awaiter(this, void 0, void 0, function* () {
839
+ const { evals } = yield discoverEvals(projectRoot);
840
+ const listed = filterDiscoveredEvals(evals, { filterIds, tags });
841
+ return printListedEvals(listed.map((e) => { var _a; return ({
842
+ id: e.id,
843
+ fileId: e.fileId,
844
+ path: e.path,
845
+ description: e.definition.description,
846
+ tags: (_a = e.definition.tags) !== null && _a !== void 0 ? _a : [],
847
+ }); }), options, `No evals under ${projectRoot}/evals`);
848
+ });
849
+ }
850
+ function printListedEvals(listed, options, emptyMessage) {
851
+ if (options.json) {
852
+ console.log(JSON.stringify(listed, null, 2));
853
+ }
854
+ else if (listed.length === 0) {
855
+ process.stdout.write(`${emptyMessage}\n`);
856
+ }
857
+ else {
858
+ for (const e of listed) {
859
+ const tagSuffix = e.tags.length > 0 ? ` [${e.tags.join(", ")}]` : "";
860
+ const desc = e.description !== undefined ? ` — ${e.description}` : "";
861
+ process.stdout.write(`${e.id}${tagSuffix}${desc}\n`);
862
+ }
863
+ }
864
+ return 0;
865
+ }
866
+ /**
867
+ * Remote batches don't stream turn events to the CLI; print run/case progress
868
+ * (and case logs once they appear on the snapshot) instead.
869
+ */
870
+ function createRemoteEvalProgress(enabled) {
871
+ if (!enabled) {
872
+ return undefined;
873
+ }
874
+ let announced = false;
875
+ let lastSummaryKey = "";
876
+ const seenCaseKeys = new Set();
877
+ return {
878
+ onSnapshot(snap) {
879
+ var _a, _b;
880
+ if (!announced) {
881
+ announced = true;
882
+ const durable = snap.config.durableRuns === true ? " (persisted)" : "";
883
+ process.stderr.write(`eval run: ${snap.runId}${durable}\n`);
884
+ }
885
+ const summaryKey = `${snap.status}:${snap.summary.done}:${snap.summary.passed}:${snap.summary.failed}`;
886
+ if (summaryKey !== lastSummaryKey) {
887
+ lastSummaryKey = summaryKey;
888
+ const { done, total, passed, failed } = snap.summary;
889
+ process.stderr.write(`eval progress: ${done}/${total} done (${passed} passed, ${failed} failed) [${snap.status}]\n`);
890
+ }
891
+ for (const c of snap.cases) {
892
+ if (c.status !== "done") {
893
+ continue;
894
+ }
895
+ const caseKey = `${c.id}:${c.ok === true ? "pass" : "fail"}:${(_a = c.durationMs) !== null && _a !== void 0 ? _a : 0}`;
896
+ if (seenCaseKeys.has(caseKey)) {
897
+ continue;
898
+ }
899
+ seenCaseKeys.add(caseKey);
900
+ process.stderr.write(` eval ${c.id}: ${c.ok === true ? "PASS" : "FAIL"}${c.error !== undefined ? ` — ${c.error}` : ""}\n`);
901
+ for (const line of (_b = c.logs) !== null && _b !== void 0 ? _b : []) {
902
+ process.stderr.write(` ${line}\n`);
903
+ }
904
+ }
905
+ },
906
+ };
907
+ }
908
+ function printEvalResults(results, options, batch) {
550
909
  const passed = results.filter((r) => r.ok).length;
551
910
  const failed = results.length - passed;
911
+ const batchFailed = (batch === null || batch === void 0 ? void 0 : batch.status) === "failed";
912
+ const ok = failed === 0 && !batchFailed;
913
+ if ((batch === null || batch === void 0 ? void 0 : batch.error) !== undefined && batch.error !== "" && !options.json) {
914
+ // Always surface harness-level failures (not only --verbose).
915
+ process.stderr.write(`eval run error: ${batch.error}\n`);
916
+ }
552
917
  if (options.json) {
553
- console.log(JSON.stringify({ ok: failed === 0, passed, failed, results }, null, 2));
918
+ console.log(JSON.stringify(Object.assign(Object.assign(Object.assign({ ok,
919
+ passed,
920
+ failed,
921
+ results }, ((batch === null || batch === void 0 ? void 0 : batch.runId) === undefined ? {} : { runId: batch.runId })), ((batch === null || batch === void 0 ? void 0 : batch.status) === undefined ? {} : { status: batch.status })), ((batch === null || batch === void 0 ? void 0 : batch.error) === undefined || batch.error === ""
922
+ ? {}
923
+ : { error: batch.error })), null, 2));
554
924
  }
555
925
  else {
556
926
  for (const r of results) {
@@ -571,7 +941,7 @@ function printEvalResults(results, options) {
571
941
  }
572
942
  process.stdout.write(`\n${passed} passed, ${failed} failed, ${results.length} total\n`);
573
943
  }
574
- return failed === 0 ? 0 : 1;
944
+ return ok ? 0 : 1;
575
945
  }
576
946
  /**
577
947
  * `logs` — follow deploy progress (`--prod`) then runtime agent logs.
@@ -770,9 +1140,9 @@ export function cmdSession(sessionId, options) {
770
1140
  /**
771
1141
  * `playground` — open the web UI against local / `--url` / `--prod`.
772
1142
  *
773
- * Hosted engines need short-lived access headers a browser cannot set, so
774
- * `--prod` (and `--bearer-token`) run a loopback reverse proxy that injects
775
- * them. Ctrl-C stops the proxy.
1143
+ * `--prod` opens the stable hosted alias playground URL; the SPA collects
1144
+ * the alias token. `--bearer-token` (and other injected headers) still run a
1145
+ * loopback reverse proxy — Ctrl-C stops that proxy.
776
1146
  */
777
1147
  export function cmdPlayground(options) {
778
1148
  return __awaiter(this, void 0, void 0, function* () {
@@ -783,58 +1153,29 @@ export function cmdPlayground(options) {
783
1153
  const sessionId = options.sessionId !== undefined && options.sessionId.trim() !== ""
784
1154
  ? options.sessionId.trim()
785
1155
  : undefined;
786
- // Hosted: mint engineAccess and refresh it for the life of the proxy.
1156
+ // Hosted: open the alias playground; SPA owns alias-token sign-in.
787
1157
  if (options.prod === true) {
788
- let initial;
1158
+ let resolved;
789
1159
  try {
790
- initial = yield resolveProdTarget({
1160
+ resolved = yield resolveProdAliasPlaygroundUrl({
791
1161
  dir: options.dir,
792
1162
  cwd: options.cwd,
793
1163
  slug: options.slug,
794
1164
  team: options.team,
795
1165
  apiKey: options.apiKey,
796
1166
  fetchImpl: options.fetchImpl,
1167
+ sessionId,
797
1168
  });
798
1169
  }
799
1170
  catch (error) {
800
1171
  process.stderr.write(`${sanitizeCustomerError(error instanceof Error ? error.message : String(error))}\n`);
801
1172
  return 1;
802
1173
  }
803
- let proxy;
804
- try {
805
- proxy = yield startPlaygroundProxy({
806
- upstreamUrl: initial.agentUrl,
807
- headers: initial.headers,
808
- expiresAt: initial.expiresAt,
809
- refreshUpstream: () => __awaiter(this, void 0, void 0, function* () {
810
- var _a;
811
- const next = yield resolveProdTarget({
812
- dir: options.dir,
813
- cwd: options.cwd,
814
- slug: (_a = options.slug) !== null && _a !== void 0 ? _a : initial.slug,
815
- team: options.team,
816
- apiKey: options.apiKey,
817
- fetchImpl: options.fetchImpl,
818
- err: () => { },
819
- });
820
- return {
821
- upstreamUrl: next.agentUrl,
822
- headers: next.headers,
823
- expiresAt: next.expiresAt,
824
- };
825
- }),
826
- });
827
- }
828
- catch (error) {
829
- process.stderr.write(`${sanitizeCustomerError(error instanceof Error ? error.message : String(error))}\n`);
830
- return 1;
1174
+ process.stdout.write(`${resolved.playgroundUrl}\n`);
1175
+ if (options.print !== true) {
1176
+ openBrowser(resolved.playgroundUrl);
831
1177
  }
832
- return yield runPlaygroundProxySession({
833
- proxy,
834
- sessionId,
835
- print: options.print === true,
836
- signal: options.signal,
837
- });
1178
+ return 0;
838
1179
  }
839
1180
  const target = yield resolveSessionTarget(options);
840
1181
  if (typeof target === "number") {
@@ -464,7 +464,7 @@ function deployOne(ctx, options, target, extras) {
464
464
  ` kind: ${detail.deploymentKind}\n` +
465
465
  ` alias: ${aliasUrl}\n` +
466
466
  ` playground: ${aliasUrl}/playground (sign in with the alias token)\n` +
467
- ` or: ${CLI} playground --prod --slug ${slug} --team ${ctx.teamId}\n`);
467
+ ` open: ${CLI} playground --prod --slug ${slug} --team ${ctx.teamId}\n`);
468
468
  return 0;
469
469
  });
470
470
  }
@@ -1226,7 +1226,7 @@ function loadSeedFiles(ctx, agentDir) {
1226
1226
  // Info projection
1227
1227
  // ============================================================================
1228
1228
  export function projectInfo(project) {
1229
- var _a, _b, _c;
1229
+ var _a, _b, _c, _d;
1230
1230
  const agent = project.agent;
1231
1231
  const httpRoutes = [
1232
1232
  { method: "POST", path: "/v1/session", url: "/v1/session" },
@@ -1324,8 +1324,8 @@ export function projectInfo(project) {
1324
1324
  : {
1325
1325
  abConfig: {
1326
1326
  maxPlaygroundSessions: resolveABMaxPlaygroundSessions((_a = project.abConfig) === null || _a === void 0 ? void 0 : _a.maxPlaygroundSessions),
1327
- durableSamples: ((_b = project.abConfig) === null || _b === void 0 ? void 0 : _b.persistSamples) !== undefined,
1328
- durableSnapshots: ((_c = project.abConfig) === null || _c === void 0 ? void 0 : _c.persistSnapshots) !== undefined,
1327
+ durableSamples: ((_b = project.storage) === null || _b === void 0 ? void 0 : _b.abs) !== undefined,
1328
+ durableSnapshots: ((_d = (_c = project.storage) === null || _c === void 0 ? void 0 : _c.abs) === null || _d === void 0 ? void 0 : _d.putSnapshot) !== undefined,
1329
1329
  },
1330
1330
  })), { storage: project.storage === undefined
1331
1331
  ? undefined