@cursor/july 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ab.d.ts +8 -95
- package/dist/ab.d.ts.map +1 -1
- package/dist/ab.js +9 -150
- package/dist/bin/agent-serve.js +41 -8
- package/dist/channels/slack/post-update-delivery.d.ts +85 -0
- package/dist/channels/slack/post-update-delivery.d.ts.map +1 -0
- package/dist/docs/404.html +2 -2
- package/dist/docs/ab.html +4 -4
- package/dist/docs/assets/{app.DabPG-io.js → app.COTN7wgo.js} +1 -1
- package/dist/docs/assets/chunks/@localSearchIndexroot.B7UcKvIn.js +1 -0
- package/dist/docs/assets/chunks/{VPLocalSearchBox.jmyr0bU0.js → VPLocalSearchBox.BW3TBdT0.js} +1 -1
- package/dist/docs/assets/chunks/{theme.DysN9-VN.js → theme.BEJW0vE7.js} +2 -2
- package/dist/docs/assets/deployment.md.BtfEsc9S.js +55 -0
- package/dist/docs/assets/deployment.md.BtfEsc9S.lean.js +1 -0
- package/dist/docs/assets/example-agents_approval-buddy.md.8R5phXb5.js +10 -0
- package/dist/docs/assets/example-agents_approval-buddy.md.8R5phXb5.lean.js +1 -0
- package/dist/docs/assets/example-agents_benny.md.B0gjhI-p.js +7 -0
- package/dist/docs/assets/example-agents_benny.md.B0gjhI-p.lean.js +1 -0
- package/dist/docs/assets/example-agents_bugbot.md.DelIdhxB.js +11 -0
- package/dist/docs/assets/example-agents_bugbot.md.DelIdhxB.lean.js +1 -0
- package/dist/docs/assets/example-agents_codebase-wiki.md.DC6sgwn0.js +8 -0
- package/dist/docs/assets/example-agents_codebase-wiki.md.DC6sgwn0.lean.js +1 -0
- package/dist/docs/assets/example-agents_codeowners-review.md.Ku_tG2RY.js +8 -0
- package/dist/docs/assets/example-agents_codeowners-review.md.Ku_tG2RY.lean.js +1 -0
- package/dist/docs/assets/example-agents_concierge.md.4rQTSMXt.js +23 -0
- package/dist/docs/assets/example-agents_concierge.md.4rQTSMXt.lean.js +1 -0
- package/dist/docs/assets/example-agents_fsd.md.CzgUrDfi.js +15 -0
- package/dist/docs/assets/example-agents_fsd.md.CzgUrDfi.lean.js +1 -0
- package/dist/docs/assets/example-agents_index.md.CRqJlnIf.js +2 -0
- package/dist/docs/assets/example-agents_index.md.CRqJlnIf.lean.js +1 -0
- package/dist/docs/assets/example-agents_knowledge-base.md.BPJiVueF.js +11 -0
- package/dist/docs/assets/example-agents_knowledge-base.md.BPJiVueF.lean.js +1 -0
- package/dist/docs/assets/example-agents_security-reviewer.md.D2rtwDTO.js +19 -0
- package/dist/docs/assets/example-agents_security-reviewer.md.D2rtwDTO.lean.js +1 -0
- package/dist/docs/assets/example-agents_slack-agent.md.buLbgvBf.js +5 -0
- package/dist/docs/assets/example-agents_slack-agent.md.buLbgvBf.lean.js +1 -0
- package/dist/docs/assets/example-agents_weather-agent.md.C9Qv-W0o.js +24 -0
- package/dist/docs/assets/example-agents_weather-agent.md.C9Qv-W0o.lean.js +1 -0
- package/dist/docs/assets/index.md.COiu-1jL.js +20 -0
- package/dist/docs/assets/{index.md.Cylk70gg.lean.js → index.md.COiu-1jL.lean.js} +1 -1
- package/dist/docs/assets/reference_cli.md.D189RBCH.js +60 -0
- package/dist/docs/assets/reference_cli.md.D189RBCH.lean.js +1 -0
- package/dist/docs/building-with-agents.html +4 -4
- package/dist/docs/concepts.html +4 -4
- package/dist/docs/deployment.html +58 -17
- package/dist/docs/evals.html +4 -4
- package/dist/docs/example-agents/approval-buddy.html +34 -0
- package/dist/docs/example-agents/benny.html +31 -0
- package/dist/docs/example-agents/bugbot.html +35 -0
- package/dist/docs/example-agents/codebase-wiki.html +32 -0
- package/dist/docs/example-agents/codeowners-review.html +32 -0
- package/dist/docs/example-agents/concierge.html +47 -0
- package/dist/docs/example-agents/fsd.html +39 -0
- package/dist/docs/example-agents/index.html +26 -0
- package/dist/docs/example-agents/knowledge-base.html +35 -0
- package/dist/docs/example-agents/security-reviewer.html +43 -0
- package/dist/docs/example-agents/slack-agent.html +29 -0
- package/dist/docs/example-agents/weather-agent.html +48 -0
- package/dist/docs/guides/agent-to-agent.html +4 -4
- package/dist/docs/guides/cloud-runtime.html +5 -5
- package/dist/docs/guides/github.html +4 -4
- package/dist/docs/guides/human-in-the-loop.html +4 -4
- package/dist/docs/guides/slack.html +4 -4
- package/dist/docs/guides/webhooks.html +4 -4
- package/dist/docs/hashmap.json +1 -1
- package/dist/docs/hillclimbing.html +4 -4
- package/dist/docs/index.html +7 -7
- package/dist/docs/quickstart.html +4 -4
- package/dist/docs/reference/agent-config.html +4 -4
- package/dist/docs/reference/channels.html +4 -4
- package/dist/docs/reference/cli.html +52 -30
- package/dist/docs/reference/connections.html +4 -4
- package/dist/docs/reference/hooks.html +4 -4
- package/dist/docs/reference/http-api.html +4 -4
- package/dist/docs/reference/instructions.html +4 -4
- package/dist/docs/reference/playground.html +4 -4
- package/dist/docs/reference/project-layout.html +4 -4
- package/dist/docs/reference/schedules.html +4 -4
- package/dist/docs/reference/sessions.html +4 -4
- package/dist/docs/reference/skills.html +4 -4
- package/dist/docs/reference/subagents.html +4 -4
- package/dist/docs/reference/tools.html +4 -4
- package/dist/docs/scaffolding-agents.html +4 -4
- package/dist/docs/storage.html +4 -4
- package/dist/docs/troubleshooting.html +4 -4
- package/dist/evals.d.ts +5 -62
- package/dist/evals.d.ts.map +1 -1
- package/dist/evals.js +3 -66
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/internal/ab-collector.d.ts +7 -5
- package/dist/internal/ab-collector.d.ts.map +1 -1
- package/dist/internal/ab-collector.js +3 -14
- package/dist/internal/ab-snapshot.d.ts +2 -4
- package/dist/internal/ab-snapshot.d.ts.map +1 -1
- package/dist/internal/cli-ax.d.ts +33 -5
- package/dist/internal/cli-ax.d.ts.map +1 -1
- package/dist/internal/cli-ax.js +428 -87
- package/dist/internal/cli-deploy.js +1 -1
- package/dist/internal/discovery.js +3 -3
- package/dist/internal/eval-run-store.d.ts +35 -30
- package/dist/internal/eval-run-store.d.ts.map +1 -1
- package/dist/internal/eval-run-store.js +88 -100
- package/dist/internal/evals-client.d.ts +96 -0
- package/dist/internal/evals-client.d.ts.map +1 -0
- package/dist/internal/evals-client.js +262 -0
- package/dist/internal/init-project.d.ts.map +1 -1
- package/dist/internal/init-project.js +1 -0
- package/dist/internal/persistence-coordinator.d.ts +127 -0
- package/dist/internal/persistence-coordinator.d.ts.map +1 -0
- package/dist/internal/playground-proxy.d.ts +5 -5
- package/dist/internal/playground-proxy.js +3 -3
- package/dist/internal/resolve-prod-target.d.ts +30 -0
- package/dist/internal/resolve-prod-target.d.ts.map +1 -1
- package/dist/internal/resolve-prod-target.js +74 -2
- package/dist/internal/server.d.ts.map +1 -1
- package/dist/internal/server.js +16 -5
- package/dist/internal/session-engine.d.ts +1 -2
- package/dist/internal/session-engine.d.ts.map +1 -1
- package/dist/internal/session-engine.js +14 -31
- package/dist/internal/storage-coordinator.d.ts +16 -15
- package/dist/internal/storage-coordinator.d.ts.map +1 -1
- package/dist/internal/storage-coordinator.js +73 -80
- package/dist/persistence.d.ts +184 -0
- package/dist/persistence.d.ts.map +1 -0
- package/dist/playground/assets/cursor-icons-16-CQ50JpfO.woff2 +0 -0
- package/dist/playground/assets/index-72vCOBWO.js +86 -0
- package/dist/playground/assets/index-BjnMwYoR.css +1 -0
- package/dist/playground/index.html +2 -2
- package/dist/storage.d.ts +51 -10
- package/dist/storage.d.ts.map +1 -1
- package/dist/storage.js +27 -10
- package/docs/README.md +34 -5
- package/docs/deployment.md +352 -149
- package/docs/example-agents/approval-buddy.md +270 -0
- package/docs/example-agents/benny.md +186 -0
- package/docs/example-agents/bugbot.md +231 -0
- package/docs/example-agents/codebase-wiki.md +174 -0
- package/docs/example-agents/codeowners-review.md +195 -0
- package/docs/example-agents/concierge.md +205 -0
- package/docs/example-agents/fsd.md +330 -0
- package/docs/example-agents/index.md +102 -0
- package/docs/example-agents/knowledge-base.md +171 -0
- package/docs/example-agents/security-reviewer.md +296 -0
- package/docs/example-agents/slack-agent.md +146 -0
- package/docs/example-agents/weather-agent.md +302 -0
- package/docs/reference/cli.md +546 -147
- package/package.json +1 -1
- package/src/ab.ts +9 -261
- package/src/bin/agent-serve.ts +46 -7
- package/src/evals.ts +5 -119
- package/src/index.ts +2 -0
- package/src/internal/ab-collector.ts +12 -22
- package/src/internal/ab-snapshot.ts +2 -4
- package/src/internal/cli-ax.ts +551 -104
- package/src/internal/cli-deploy.ts +1 -1
- package/src/internal/discovery.ts +2 -2
- package/src/internal/eval-run-store.ts +91 -100
- package/src/internal/evals-client.ts +431 -0
- package/src/internal/init-project.ts +1 -0
- package/src/internal/playground-proxy.ts +5 -5
- package/src/internal/resolve-prod-target.ts +101 -3
- package/src/internal/server.ts +17 -3
- package/src/internal/session-engine.ts +9 -29
- package/src/internal/storage-coordinator.ts +109 -101
- package/src/storage.ts +79 -14
- package/dist/docs/assets/chunks/@localSearchIndexroot.QwK5BtEH.js +0 -1
- package/dist/docs/assets/deployment.md.DTKwE15Z.js +0 -14
- package/dist/docs/assets/deployment.md.DTKwE15Z.lean.js +0 -1
- package/dist/docs/assets/index.md.Cylk70gg.js +0 -20
- package/dist/docs/assets/reference_cli.md.Bv6pOxcF.js +0 -38
- package/dist/docs/assets/reference_cli.md.Bv6pOxcF.lean.js +0 -1
- package/dist/internal/json-dir-store.js +0 -100
- package/dist/playground/assets/cursor-icons-outline-BxTT_FVJ.woff2 +0 -0
- package/dist/playground/assets/index-BEauYlII.css +0 -1
- package/dist/playground/assets/index-BtM0wEGg.js +0 -319
- package/src/internal/json-dir-store.ts +0 -109
package/dist/internal/cli-ax.js
CHANGED
|
@@ -25,12 +25,13 @@ import { resolveDeployAuth } from "./cli-deploy.js";
|
|
|
25
25
|
import { resolveApiKey } from "./cursor/credentials.js";
|
|
26
26
|
import { CLI_COMMAND_NAME } from "./distribution.js";
|
|
27
27
|
import { discoverEvals, filterDiscoveredEvals, runDiscoveredEvals, } from "./eval-runner.js";
|
|
28
|
+
import { getRemoteEvalRun, listRemoteEvalRuns, listRemoteEvals, RemoteEvalError, runRemoteEvals, startRemoteEvalRun, } from "./evals-client.js";
|
|
28
29
|
import { formatInitNextSteps, formatInitScaffoldSummary, initNextCommands, initProject, installScaffoldDependencies, printInitGuide, } from "./init-project.js";
|
|
29
30
|
import { offerInstallCursorSkills } from "./install-cursor-skills.js";
|
|
30
31
|
import { runLogsTail, sanitizeCustomerError } from "./logs-client.js";
|
|
31
32
|
import { openBrowser } from "./open-browser.js";
|
|
32
33
|
import { startPlaygroundProxy } from "./playground-proxy.js";
|
|
33
|
-
import { resolveProdSlug, resolveProdTarget } from "./resolve-prod-target.js";
|
|
34
|
+
import { ProdTargetError, resolveProdAliasPlaygroundUrl, resolveProdSlug, resolveProdTarget, } from "./resolve-prod-target.js";
|
|
34
35
|
import { runSession } from "./run-client.js";
|
|
35
36
|
import { fetchSessionEvents, formatSessionsTable, listSessions, resolvePreviousSessionId, } from "./sessions-client.js";
|
|
36
37
|
import { createStreamProgress } from "./stream-progress.js";
|
|
@@ -475,82 +476,451 @@ export function cmdTrajectory(options) {
|
|
|
475
476
|
return trajectory.ok ? 0 : 1;
|
|
476
477
|
});
|
|
477
478
|
}
|
|
479
|
+
/**
|
|
480
|
+
* Which surface `eval` should target. Local (ephemeral) is the default;
|
|
481
|
+
* `--prod` uses the hosted deployment, `--url` a running server.
|
|
482
|
+
* Exported for tests.
|
|
483
|
+
*/
|
|
484
|
+
export function resolveEvalTargetMode(options) {
|
|
485
|
+
const flags = [
|
|
486
|
+
options.prod === true ? "--prod" : undefined,
|
|
487
|
+
options.url !== undefined ? "--url" : undefined,
|
|
488
|
+
].filter((flag) => flag !== undefined);
|
|
489
|
+
if (flags.length > 1) {
|
|
490
|
+
return { error: `${flags.join(" and ")} are mutually exclusive` };
|
|
491
|
+
}
|
|
492
|
+
if (options.url !== undefined) {
|
|
493
|
+
return "url";
|
|
494
|
+
}
|
|
495
|
+
if (options.prod === true) {
|
|
496
|
+
return "deployment";
|
|
497
|
+
}
|
|
498
|
+
return "local";
|
|
499
|
+
}
|
|
500
|
+
/** Resolve the eval target; prints a CLI-ready error and returns an exit code on failure. */
|
|
501
|
+
function resolveEvalTarget(options) {
|
|
502
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
503
|
+
const mode = resolveEvalTargetMode(options);
|
|
504
|
+
if (typeof mode === "object") {
|
|
505
|
+
process.stderr.write(`${mode.error}\n`);
|
|
506
|
+
return 2;
|
|
507
|
+
}
|
|
508
|
+
if (mode === "url") {
|
|
509
|
+
return { agentUrl: options.url.replace(/\/$/, "") };
|
|
510
|
+
}
|
|
511
|
+
if (mode === "local") {
|
|
512
|
+
return startEphemeral(options);
|
|
513
|
+
}
|
|
514
|
+
try {
|
|
515
|
+
const prod = yield resolveProdTarget({
|
|
516
|
+
dir: options.dir,
|
|
517
|
+
cwd: options.cwd,
|
|
518
|
+
slug: options.slug,
|
|
519
|
+
team: options.team,
|
|
520
|
+
apiKey: options.apiKey,
|
|
521
|
+
fetchImpl: options.fetchImpl,
|
|
522
|
+
});
|
|
523
|
+
return { agentUrl: prod.agentUrl, headers: prod.headers };
|
|
524
|
+
}
|
|
525
|
+
catch (error) {
|
|
526
|
+
if (error instanceof ProdTargetError && error.notFound) {
|
|
527
|
+
// Keep the API error text: a 404 can also mean a wrong API host or a
|
|
528
|
+
// disabled feature gate, and the message distinguishes those from a
|
|
529
|
+
// genuinely missing deployment.
|
|
530
|
+
process.stderr.write(`${sanitizeCustomerError(error.message)}\n` +
|
|
531
|
+
` deploy it first: ${CLI_COMMAND_NAME} deploy --slug ${error.slug}\n` +
|
|
532
|
+
` or omit --prod: ${CLI_COMMAND_NAME} eval --dir ${options.dir}\n`);
|
|
533
|
+
return 1;
|
|
534
|
+
}
|
|
535
|
+
process.stderr.write(`${sanitizeCustomerError(error instanceof Error ? error.message : String(error))}\n` +
|
|
536
|
+
`Pass --prod for the hosted deployment, or omit it to run on an ephemeral local server.\n`);
|
|
537
|
+
return 1;
|
|
538
|
+
}
|
|
539
|
+
});
|
|
540
|
+
}
|
|
478
541
|
export function cmdEval(options) {
|
|
479
542
|
return __awaiter(this, void 0, void 0, function* () {
|
|
480
543
|
var _a;
|
|
481
544
|
const projectRoot = resolve((_a = options.cwd) !== null && _a !== void 0 ? _a : process.cwd(), options.dir);
|
|
545
|
+
const mode = resolveEvalTargetMode(options);
|
|
546
|
+
if (typeof mode === "object") {
|
|
547
|
+
process.stderr.write(`${mode.error}\n`);
|
|
548
|
+
return 2;
|
|
549
|
+
}
|
|
550
|
+
if (options.noWait === true) {
|
|
551
|
+
if (options.list === true) {
|
|
552
|
+
process.stderr.write("--no-wait and --list are mutually exclusive\n");
|
|
553
|
+
return 2;
|
|
554
|
+
}
|
|
555
|
+
if (mode === "local") {
|
|
556
|
+
process.stderr.write("--no-wait requires --prod or --url (not the default local target)\n");
|
|
557
|
+
return 2;
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
const filterIds = options.evalIds === undefined || options.evalIds.length === 0
|
|
561
|
+
? undefined
|
|
562
|
+
: options.evalIds;
|
|
563
|
+
const tags = options.tags === undefined || options.tags.length === 0
|
|
564
|
+
? undefined
|
|
565
|
+
: options.tags;
|
|
482
566
|
if (options.list) {
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
}
|
|
488
|
-
|
|
489
|
-
|
|
567
|
+
// List the same surface a run will execute: local filesystem by default,
|
|
568
|
+
// GET /v1/dev/evals on --prod / --url.
|
|
569
|
+
if (mode === "local") {
|
|
570
|
+
return listLocalEvals(projectRoot, options, filterIds, tags);
|
|
571
|
+
}
|
|
572
|
+
const target = yield resolveEvalTarget(options);
|
|
573
|
+
if (typeof target === "number") {
|
|
574
|
+
return target;
|
|
575
|
+
}
|
|
576
|
+
try {
|
|
577
|
+
const listed = yield listRemoteEvals({
|
|
578
|
+
baseUrl: target.agentUrl,
|
|
579
|
+
auth: {
|
|
580
|
+
bearerToken: options.bearerToken,
|
|
581
|
+
headers: target.headers,
|
|
582
|
+
},
|
|
583
|
+
fetchImpl: options.fetchImpl,
|
|
584
|
+
filterIds,
|
|
585
|
+
tags,
|
|
586
|
+
});
|
|
587
|
+
return printListedEvals(listed.map((e) => { var _a; return ({
|
|
490
588
|
id: e.id,
|
|
491
589
|
fileId: e.fileId,
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
}); }), null, 2));
|
|
590
|
+
description: e.description,
|
|
591
|
+
tags: (_a = e.tags) !== null && _a !== void 0 ? _a : [],
|
|
592
|
+
}); }), options, `No evals on ${target.agentUrl}`);
|
|
496
593
|
}
|
|
497
|
-
|
|
498
|
-
process.
|
|
594
|
+
catch (error) {
|
|
595
|
+
process.stderr.write(`${sanitizeCustomerError(error instanceof Error ? error.message : String(error))}\n`);
|
|
596
|
+
return 1;
|
|
499
597
|
}
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
? ` [${e.definition.tags.join(", ")}]`
|
|
504
|
-
: "";
|
|
505
|
-
const desc = e.definition.description !== undefined
|
|
506
|
-
? ` — ${e.definition.description}`
|
|
507
|
-
: "";
|
|
508
|
-
process.stdout.write(`${e.id}${tags}${desc}\n`);
|
|
598
|
+
finally {
|
|
599
|
+
if (target.close !== undefined) {
|
|
600
|
+
yield target.close();
|
|
509
601
|
}
|
|
510
602
|
}
|
|
511
|
-
return 0;
|
|
512
603
|
}
|
|
513
|
-
const target = yield
|
|
604
|
+
const target = yield resolveEvalTarget(options);
|
|
605
|
+
if (typeof target === "number") {
|
|
606
|
+
return target;
|
|
607
|
+
}
|
|
514
608
|
const stream = shouldStreamProgress(options);
|
|
515
|
-
const progress = stream ? createStreamProgress() : undefined;
|
|
516
609
|
try {
|
|
517
610
|
if (options.verbose || stream) {
|
|
518
611
|
process.stderr.write(`eval target: ${target.agentUrl}\n`);
|
|
519
612
|
}
|
|
520
|
-
|
|
521
|
-
|
|
613
|
+
// Default local: ephemeral in-process harness against local eval sources.
|
|
614
|
+
// --prod / --url: run on the serve process via /v1/dev/evals/runs so the
|
|
615
|
+
// batch lands in playground history and defineStorage evals (when set).
|
|
616
|
+
if (mode === "local") {
|
|
617
|
+
const progress = stream ? createStreamProgress() : undefined;
|
|
618
|
+
try {
|
|
619
|
+
const results = yield runDiscoveredEvals({
|
|
620
|
+
projectRoot,
|
|
621
|
+
baseUrl: target.agentUrl,
|
|
622
|
+
filterIds,
|
|
623
|
+
tags,
|
|
624
|
+
timeoutMs: options.timeoutMs,
|
|
625
|
+
bearerToken: options.bearerToken,
|
|
626
|
+
headers: target.headers,
|
|
627
|
+
verbose: Boolean(options.verbose) || stream,
|
|
628
|
+
onEvent: progress === undefined ? undefined : (e) => progress.onEvent(e),
|
|
629
|
+
});
|
|
630
|
+
if (results.length === 0) {
|
|
631
|
+
process.stderr.write("No matching evals found.\n");
|
|
632
|
+
return 2;
|
|
633
|
+
}
|
|
634
|
+
return printEvalResults(results, options);
|
|
635
|
+
}
|
|
636
|
+
finally {
|
|
637
|
+
progress === null || progress === void 0 ? void 0 : progress.close();
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
try {
|
|
641
|
+
if (options.noWait === true) {
|
|
642
|
+
const started = yield startRemoteEvalRun({
|
|
643
|
+
baseUrl: target.agentUrl,
|
|
644
|
+
filterIds,
|
|
645
|
+
tags,
|
|
646
|
+
timeoutMs: options.timeoutMs,
|
|
647
|
+
verbose: Boolean(options.verbose),
|
|
648
|
+
auth: {
|
|
649
|
+
bearerToken: options.bearerToken,
|
|
650
|
+
headers: target.headers,
|
|
651
|
+
},
|
|
652
|
+
fetchImpl: options.fetchImpl,
|
|
653
|
+
});
|
|
654
|
+
return printEvalAccepted(started, options, target.agentUrl);
|
|
655
|
+
}
|
|
656
|
+
const progress = createRemoteEvalProgress(options.verbose === true || stream);
|
|
657
|
+
const { snapshot, results } = yield runRemoteEvals({
|
|
658
|
+
baseUrl: target.agentUrl,
|
|
659
|
+
filterIds,
|
|
660
|
+
tags,
|
|
661
|
+
timeoutMs: options.timeoutMs,
|
|
662
|
+
verbose: Boolean(options.verbose) || stream,
|
|
663
|
+
bearerToken: options.bearerToken,
|
|
664
|
+
headers: target.headers,
|
|
665
|
+
fetchImpl: options.fetchImpl,
|
|
666
|
+
onSnapshot: progress === undefined ? undefined : progress.onSnapshot,
|
|
667
|
+
});
|
|
668
|
+
if (results.length === 0 && snapshot.status !== "failed") {
|
|
669
|
+
process.stderr.write("No matching evals found.\n");
|
|
670
|
+
return 2;
|
|
671
|
+
}
|
|
672
|
+
return printEvalResults(results, options, {
|
|
673
|
+
runId: snapshot.runId,
|
|
674
|
+
status: snapshot.status,
|
|
675
|
+
error: snapshot.error,
|
|
676
|
+
});
|
|
677
|
+
}
|
|
678
|
+
catch (error) {
|
|
679
|
+
if (error instanceof RemoteEvalError &&
|
|
680
|
+
error.code === "no_matching_evals") {
|
|
681
|
+
process.stderr.write(`${error.message}\n`);
|
|
682
|
+
return 2;
|
|
683
|
+
}
|
|
684
|
+
process.stderr.write(`${sanitizeCustomerError(error instanceof Error ? error.message : String(error))}\n`);
|
|
685
|
+
return 1;
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
finally {
|
|
689
|
+
if (target.close !== undefined) {
|
|
690
|
+
yield target.close();
|
|
691
|
+
}
|
|
692
|
+
}
|
|
693
|
+
});
|
|
694
|
+
}
|
|
695
|
+
/**
|
|
696
|
+
* Flags to re-target the same surface from an `eval --no-wait` hint.
|
|
697
|
+
* Exported for tests.
|
|
698
|
+
*/
|
|
699
|
+
export function formatEvalStatusFlags(options) {
|
|
700
|
+
const parts = [];
|
|
701
|
+
if (options.url !== undefined) {
|
|
702
|
+
parts.push(`--url ${options.url.replace(/\/$/, "")}`);
|
|
703
|
+
}
|
|
704
|
+
else {
|
|
705
|
+
parts.push("--prod");
|
|
706
|
+
if (options.slug !== undefined && options.slug !== "") {
|
|
707
|
+
parts.push(`--slug ${options.slug}`);
|
|
708
|
+
}
|
|
709
|
+
}
|
|
710
|
+
if (options.team !== undefined && options.team !== "") {
|
|
711
|
+
parts.push(`--team ${options.team}`);
|
|
712
|
+
}
|
|
713
|
+
return parts.length === 0 ? "" : ` ${parts.join(" ")}`;
|
|
714
|
+
}
|
|
715
|
+
/** `--no-wait`: print the accepted batch id (and how to poll) then exit. */
|
|
716
|
+
function printEvalAccepted(started, options, agentUrl) {
|
|
717
|
+
const pollPath = `${agentUrl.replace(/\/$/, "")}/v1/dev/evals/runs/${started.runId}`;
|
|
718
|
+
const statusFlags = formatEvalStatusFlags(options);
|
|
719
|
+
if (options.json) {
|
|
720
|
+
console.log(JSON.stringify({
|
|
721
|
+
ok: true,
|
|
722
|
+
accepted: true,
|
|
723
|
+
runId: started.runId,
|
|
724
|
+
status: started.status,
|
|
725
|
+
summary: started.summary,
|
|
726
|
+
pollUrl: pollPath,
|
|
727
|
+
durableRuns: started.config.durableRuns === true,
|
|
728
|
+
}, null, 2));
|
|
729
|
+
}
|
|
730
|
+
else {
|
|
731
|
+
const durable = started.config.durableRuns === true ? " (persisted)" : "";
|
|
732
|
+
process.stdout.write(`Accepted eval run ${started.runId}${durable} — ${started.summary.total} case(s)\n`);
|
|
733
|
+
process.stdout.write(`Status: ${CLI_COMMAND_NAME} eval status ${started.runId}${statusFlags}\n`);
|
|
734
|
+
process.stdout.write(`Poll: ${pollPath}\n`);
|
|
735
|
+
}
|
|
736
|
+
return 0;
|
|
737
|
+
}
|
|
738
|
+
/**
|
|
739
|
+
* `eval status [runId]` — fetch one batch (or list recent) on the deployment /
|
|
740
|
+
* `--url` target. Exit `0` when the batch is completed with all cases passing,
|
|
741
|
+
* `1` when failed / any case failed / unknown, `2` for usage errors, `3` when
|
|
742
|
+
* the batch is still running (for poll loops after `--no-wait`).
|
|
743
|
+
*/
|
|
744
|
+
export function cmdEvalStatus(runId, options) {
|
|
745
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
746
|
+
const mode = resolveEvalTargetMode(options);
|
|
747
|
+
if (typeof mode === "object") {
|
|
748
|
+
process.stderr.write(`${mode.error}\n`);
|
|
749
|
+
return 2;
|
|
750
|
+
}
|
|
751
|
+
if (mode === "local") {
|
|
752
|
+
process.stderr.write("eval status requires --prod or --url (not the default local target)\n");
|
|
753
|
+
return 2;
|
|
754
|
+
}
|
|
755
|
+
if (options.list === true || options.noWait === true) {
|
|
756
|
+
process.stderr.write("eval status does not accept --list or --no-wait\n");
|
|
757
|
+
return 2;
|
|
758
|
+
}
|
|
759
|
+
const target = yield resolveEvalTarget(options);
|
|
760
|
+
if (typeof target === "number") {
|
|
761
|
+
return target;
|
|
762
|
+
}
|
|
763
|
+
const auth = {
|
|
764
|
+
bearerToken: options.bearerToken,
|
|
765
|
+
headers: target.headers,
|
|
766
|
+
};
|
|
767
|
+
try {
|
|
768
|
+
if (runId === undefined || runId === "") {
|
|
769
|
+
const { runs, activeRunId } = yield listRemoteEvalRuns({
|
|
770
|
+
baseUrl: target.agentUrl,
|
|
771
|
+
auth,
|
|
772
|
+
fetchImpl: options.fetchImpl,
|
|
773
|
+
});
|
|
774
|
+
if (options.json) {
|
|
775
|
+
console.log(JSON.stringify({ runs, activeRunId }, null, 2));
|
|
776
|
+
}
|
|
777
|
+
else if (runs.length === 0) {
|
|
778
|
+
process.stdout.write("No eval runs on this agent.\n");
|
|
779
|
+
}
|
|
780
|
+
else {
|
|
781
|
+
if (activeRunId !== undefined) {
|
|
782
|
+
process.stdout.write(`active: ${activeRunId}\n`);
|
|
783
|
+
}
|
|
784
|
+
for (const run of runs) {
|
|
785
|
+
process.stdout.write(`${formatEvalRunSummaryLine(run)}\n`);
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
return 0;
|
|
789
|
+
}
|
|
790
|
+
const snap = yield getRemoteEvalRun({
|
|
522
791
|
baseUrl: target.agentUrl,
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
tags: options.tags === undefined || options.tags.length === 0
|
|
527
|
-
? undefined
|
|
528
|
-
: options.tags,
|
|
529
|
-
timeoutMs: options.timeoutMs,
|
|
530
|
-
bearerToken: options.bearerToken,
|
|
531
|
-
headers: target.headers,
|
|
532
|
-
verbose: Boolean(options.verbose) || stream,
|
|
533
|
-
onEvent: progress === undefined ? undefined : (e) => progress.onEvent(e),
|
|
792
|
+
runId,
|
|
793
|
+
auth,
|
|
794
|
+
fetchImpl: options.fetchImpl,
|
|
534
795
|
});
|
|
535
|
-
if (
|
|
536
|
-
|
|
537
|
-
|
|
796
|
+
if (options.json) {
|
|
797
|
+
console.log(JSON.stringify(snap, null, 2));
|
|
798
|
+
}
|
|
799
|
+
else {
|
|
800
|
+
process.stdout.write(`${formatEvalRunSummaryLine(snap)}\n`);
|
|
801
|
+
if (snap.error !== undefined && snap.error !== "") {
|
|
802
|
+
process.stdout.write(`error: ${snap.error}\n`);
|
|
803
|
+
}
|
|
804
|
+
for (const c of snap.cases) {
|
|
805
|
+
const mark = c.status !== "done" ? c.status : c.ok === true ? "PASS" : "FAIL";
|
|
806
|
+
const dur = c.durationMs === undefined
|
|
807
|
+
? ""
|
|
808
|
+
: ` (${(c.durationMs / 1000).toFixed(1)}s)`;
|
|
809
|
+
process.stdout.write(` ${mark} ${c.id}${dur}\n`);
|
|
810
|
+
if (c.error !== undefined && c.error !== "") {
|
|
811
|
+
process.stdout.write(` error: ${c.error}\n`);
|
|
812
|
+
}
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
if (snap.status === "running") {
|
|
816
|
+
return 3;
|
|
538
817
|
}
|
|
539
|
-
|
|
818
|
+
const failed = snap.status === "failed" || snap.cases.some((c) => c.ok !== true);
|
|
819
|
+
return failed ? 1 : 0;
|
|
820
|
+
}
|
|
821
|
+
catch (error) {
|
|
822
|
+
process.stderr.write(`${sanitizeCustomerError(error instanceof Error ? error.message : String(error))}\n`);
|
|
823
|
+
return 1;
|
|
540
824
|
}
|
|
541
825
|
finally {
|
|
542
|
-
progress === null || progress === void 0 ? void 0 : progress.close();
|
|
543
826
|
if (target.close !== undefined) {
|
|
544
827
|
yield target.close();
|
|
545
828
|
}
|
|
546
829
|
}
|
|
547
830
|
});
|
|
548
831
|
}
|
|
549
|
-
function
|
|
832
|
+
function formatEvalRunSummaryLine(run) {
|
|
833
|
+
const { passed, failed, total, done } = run.summary;
|
|
834
|
+
const durable = run.config.durableRuns === true ? " persisted" : "";
|
|
835
|
+
return `${run.runId} ${run.status} ${done}/${total} done (${passed} passed, ${failed} failed)${durable}`;
|
|
836
|
+
}
|
|
837
|
+
function listLocalEvals(projectRoot, options, filterIds, tags) {
|
|
838
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
839
|
+
const { evals } = yield discoverEvals(projectRoot);
|
|
840
|
+
const listed = filterDiscoveredEvals(evals, { filterIds, tags });
|
|
841
|
+
return printListedEvals(listed.map((e) => { var _a; return ({
|
|
842
|
+
id: e.id,
|
|
843
|
+
fileId: e.fileId,
|
|
844
|
+
path: e.path,
|
|
845
|
+
description: e.definition.description,
|
|
846
|
+
tags: (_a = e.definition.tags) !== null && _a !== void 0 ? _a : [],
|
|
847
|
+
}); }), options, `No evals under ${projectRoot}/evals`);
|
|
848
|
+
});
|
|
849
|
+
}
|
|
850
|
+
function printListedEvals(listed, options, emptyMessage) {
|
|
851
|
+
if (options.json) {
|
|
852
|
+
console.log(JSON.stringify(listed, null, 2));
|
|
853
|
+
}
|
|
854
|
+
else if (listed.length === 0) {
|
|
855
|
+
process.stdout.write(`${emptyMessage}\n`);
|
|
856
|
+
}
|
|
857
|
+
else {
|
|
858
|
+
for (const e of listed) {
|
|
859
|
+
const tagSuffix = e.tags.length > 0 ? ` [${e.tags.join(", ")}]` : "";
|
|
860
|
+
const desc = e.description !== undefined ? ` — ${e.description}` : "";
|
|
861
|
+
process.stdout.write(`${e.id}${tagSuffix}${desc}\n`);
|
|
862
|
+
}
|
|
863
|
+
}
|
|
864
|
+
return 0;
|
|
865
|
+
}
|
|
866
|
+
/**
|
|
867
|
+
* Remote batches don't stream turn events to the CLI; print run/case progress
|
|
868
|
+
* (and case logs once they appear on the snapshot) instead.
|
|
869
|
+
*/
|
|
870
|
+
function createRemoteEvalProgress(enabled) {
|
|
871
|
+
if (!enabled) {
|
|
872
|
+
return undefined;
|
|
873
|
+
}
|
|
874
|
+
let announced = false;
|
|
875
|
+
let lastSummaryKey = "";
|
|
876
|
+
const seenCaseKeys = new Set();
|
|
877
|
+
return {
|
|
878
|
+
onSnapshot(snap) {
|
|
879
|
+
var _a, _b;
|
|
880
|
+
if (!announced) {
|
|
881
|
+
announced = true;
|
|
882
|
+
const durable = snap.config.durableRuns === true ? " (persisted)" : "";
|
|
883
|
+
process.stderr.write(`eval run: ${snap.runId}${durable}\n`);
|
|
884
|
+
}
|
|
885
|
+
const summaryKey = `${snap.status}:${snap.summary.done}:${snap.summary.passed}:${snap.summary.failed}`;
|
|
886
|
+
if (summaryKey !== lastSummaryKey) {
|
|
887
|
+
lastSummaryKey = summaryKey;
|
|
888
|
+
const { done, total, passed, failed } = snap.summary;
|
|
889
|
+
process.stderr.write(`eval progress: ${done}/${total} done (${passed} passed, ${failed} failed) [${snap.status}]\n`);
|
|
890
|
+
}
|
|
891
|
+
for (const c of snap.cases) {
|
|
892
|
+
if (c.status !== "done") {
|
|
893
|
+
continue;
|
|
894
|
+
}
|
|
895
|
+
const caseKey = `${c.id}:${c.ok === true ? "pass" : "fail"}:${(_a = c.durationMs) !== null && _a !== void 0 ? _a : 0}`;
|
|
896
|
+
if (seenCaseKeys.has(caseKey)) {
|
|
897
|
+
continue;
|
|
898
|
+
}
|
|
899
|
+
seenCaseKeys.add(caseKey);
|
|
900
|
+
process.stderr.write(` eval ${c.id}: ${c.ok === true ? "PASS" : "FAIL"}${c.error !== undefined ? ` — ${c.error}` : ""}\n`);
|
|
901
|
+
for (const line of (_b = c.logs) !== null && _b !== void 0 ? _b : []) {
|
|
902
|
+
process.stderr.write(` ${line}\n`);
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
},
|
|
906
|
+
};
|
|
907
|
+
}
|
|
908
|
+
function printEvalResults(results, options, batch) {
|
|
550
909
|
const passed = results.filter((r) => r.ok).length;
|
|
551
910
|
const failed = results.length - passed;
|
|
911
|
+
const batchFailed = (batch === null || batch === void 0 ? void 0 : batch.status) === "failed";
|
|
912
|
+
const ok = failed === 0 && !batchFailed;
|
|
913
|
+
if ((batch === null || batch === void 0 ? void 0 : batch.error) !== undefined && batch.error !== "" && !options.json) {
|
|
914
|
+
// Always surface harness-level failures (not only --verbose).
|
|
915
|
+
process.stderr.write(`eval run error: ${batch.error}\n`);
|
|
916
|
+
}
|
|
552
917
|
if (options.json) {
|
|
553
|
-
console.log(JSON.stringify({ ok
|
|
918
|
+
console.log(JSON.stringify(Object.assign(Object.assign(Object.assign({ ok,
|
|
919
|
+
passed,
|
|
920
|
+
failed,
|
|
921
|
+
results }, ((batch === null || batch === void 0 ? void 0 : batch.runId) === undefined ? {} : { runId: batch.runId })), ((batch === null || batch === void 0 ? void 0 : batch.status) === undefined ? {} : { status: batch.status })), ((batch === null || batch === void 0 ? void 0 : batch.error) === undefined || batch.error === ""
|
|
922
|
+
? {}
|
|
923
|
+
: { error: batch.error })), null, 2));
|
|
554
924
|
}
|
|
555
925
|
else {
|
|
556
926
|
for (const r of results) {
|
|
@@ -571,7 +941,7 @@ function printEvalResults(results, options) {
|
|
|
571
941
|
}
|
|
572
942
|
process.stdout.write(`\n${passed} passed, ${failed} failed, ${results.length} total\n`);
|
|
573
943
|
}
|
|
574
|
-
return
|
|
944
|
+
return ok ? 0 : 1;
|
|
575
945
|
}
|
|
576
946
|
/**
|
|
577
947
|
* `logs` — follow deploy progress (`--prod`) then runtime agent logs.
|
|
@@ -770,9 +1140,9 @@ export function cmdSession(sessionId, options) {
|
|
|
770
1140
|
/**
|
|
771
1141
|
* `playground` — open the web UI against local / `--url` / `--prod`.
|
|
772
1142
|
*
|
|
773
|
-
*
|
|
774
|
-
*
|
|
775
|
-
*
|
|
1143
|
+
* `--prod` opens the stable hosted alias playground URL; the SPA collects
|
|
1144
|
+
* the alias token. `--bearer-token` (and other injected headers) still run a
|
|
1145
|
+
* loopback reverse proxy — Ctrl-C stops that proxy.
|
|
776
1146
|
*/
|
|
777
1147
|
export function cmdPlayground(options) {
|
|
778
1148
|
return __awaiter(this, void 0, void 0, function* () {
|
|
@@ -783,58 +1153,29 @@ export function cmdPlayground(options) {
|
|
|
783
1153
|
const sessionId = options.sessionId !== undefined && options.sessionId.trim() !== ""
|
|
784
1154
|
? options.sessionId.trim()
|
|
785
1155
|
: undefined;
|
|
786
|
-
// Hosted:
|
|
1156
|
+
// Hosted: open the alias playground; SPA owns alias-token sign-in.
|
|
787
1157
|
if (options.prod === true) {
|
|
788
|
-
let
|
|
1158
|
+
let resolved;
|
|
789
1159
|
try {
|
|
790
|
-
|
|
1160
|
+
resolved = yield resolveProdAliasPlaygroundUrl({
|
|
791
1161
|
dir: options.dir,
|
|
792
1162
|
cwd: options.cwd,
|
|
793
1163
|
slug: options.slug,
|
|
794
1164
|
team: options.team,
|
|
795
1165
|
apiKey: options.apiKey,
|
|
796
1166
|
fetchImpl: options.fetchImpl,
|
|
1167
|
+
sessionId,
|
|
797
1168
|
});
|
|
798
1169
|
}
|
|
799
1170
|
catch (error) {
|
|
800
1171
|
process.stderr.write(`${sanitizeCustomerError(error instanceof Error ? error.message : String(error))}\n`);
|
|
801
1172
|
return 1;
|
|
802
1173
|
}
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
upstreamUrl: initial.agentUrl,
|
|
807
|
-
headers: initial.headers,
|
|
808
|
-
expiresAt: initial.expiresAt,
|
|
809
|
-
refreshUpstream: () => __awaiter(this, void 0, void 0, function* () {
|
|
810
|
-
var _a;
|
|
811
|
-
const next = yield resolveProdTarget({
|
|
812
|
-
dir: options.dir,
|
|
813
|
-
cwd: options.cwd,
|
|
814
|
-
slug: (_a = options.slug) !== null && _a !== void 0 ? _a : initial.slug,
|
|
815
|
-
team: options.team,
|
|
816
|
-
apiKey: options.apiKey,
|
|
817
|
-
fetchImpl: options.fetchImpl,
|
|
818
|
-
err: () => { },
|
|
819
|
-
});
|
|
820
|
-
return {
|
|
821
|
-
upstreamUrl: next.agentUrl,
|
|
822
|
-
headers: next.headers,
|
|
823
|
-
expiresAt: next.expiresAt,
|
|
824
|
-
};
|
|
825
|
-
}),
|
|
826
|
-
});
|
|
827
|
-
}
|
|
828
|
-
catch (error) {
|
|
829
|
-
process.stderr.write(`${sanitizeCustomerError(error instanceof Error ? error.message : String(error))}\n`);
|
|
830
|
-
return 1;
|
|
1174
|
+
process.stdout.write(`${resolved.playgroundUrl}\n`);
|
|
1175
|
+
if (options.print !== true) {
|
|
1176
|
+
openBrowser(resolved.playgroundUrl);
|
|
831
1177
|
}
|
|
832
|
-
return
|
|
833
|
-
proxy,
|
|
834
|
-
sessionId,
|
|
835
|
-
print: options.print === true,
|
|
836
|
-
signal: options.signal,
|
|
837
|
-
});
|
|
1178
|
+
return 0;
|
|
838
1179
|
}
|
|
839
1180
|
const target = yield resolveSessionTarget(options);
|
|
840
1181
|
if (typeof target === "number") {
|
|
@@ -464,7 +464,7 @@ function deployOne(ctx, options, target, extras) {
|
|
|
464
464
|
` kind: ${detail.deploymentKind}\n` +
|
|
465
465
|
` alias: ${aliasUrl}\n` +
|
|
466
466
|
` playground: ${aliasUrl}/playground (sign in with the alias token)\n` +
|
|
467
|
-
`
|
|
467
|
+
` open: ${CLI} playground --prod --slug ${slug} --team ${ctx.teamId}\n`);
|
|
468
468
|
return 0;
|
|
469
469
|
});
|
|
470
470
|
}
|
|
@@ -1226,7 +1226,7 @@ function loadSeedFiles(ctx, agentDir) {
|
|
|
1226
1226
|
// Info projection
|
|
1227
1227
|
// ============================================================================
|
|
1228
1228
|
export function projectInfo(project) {
|
|
1229
|
-
var _a, _b, _c;
|
|
1229
|
+
var _a, _b, _c, _d;
|
|
1230
1230
|
const agent = project.agent;
|
|
1231
1231
|
const httpRoutes = [
|
|
1232
1232
|
{ method: "POST", path: "/v1/session", url: "/v1/session" },
|
|
@@ -1324,8 +1324,8 @@ export function projectInfo(project) {
|
|
|
1324
1324
|
: {
|
|
1325
1325
|
abConfig: {
|
|
1326
1326
|
maxPlaygroundSessions: resolveABMaxPlaygroundSessions((_a = project.abConfig) === null || _a === void 0 ? void 0 : _a.maxPlaygroundSessions),
|
|
1327
|
-
durableSamples: ((_b = project.
|
|
1328
|
-
durableSnapshots: ((_c = project.
|
|
1327
|
+
durableSamples: ((_b = project.storage) === null || _b === void 0 ? void 0 : _b.abs) !== undefined,
|
|
1328
|
+
durableSnapshots: ((_d = (_c = project.storage) === null || _c === void 0 ? void 0 : _c.abs) === null || _d === void 0 ? void 0 : _d.putSnapshot) !== undefined,
|
|
1329
1329
|
},
|
|
1330
1330
|
})), { storage: project.storage === undefined
|
|
1331
1331
|
? undefined
|