@testchimp/cli 0.1.30 → 0.1.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/dist/cli/program.js +198 -2
- package/dist/core/schemas.d.ts +97 -0
- package/dist/core/schemas.js +92 -2
- package/dist/core/tools.js +153 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -42,10 +42,17 @@ testchimp get-requirement-coverage --branch-name main --help
|
|
|
42
42
|
testchimp create-user-story --platform-file-path plans/stories/foo.md --title "Checkout"
|
|
43
43
|
testchimp list-screen-states --json-input '{}'
|
|
44
44
|
testchimp upsert-screen-states --json-input '{"screenStates":[{"screen":"Checkout","states":["empty","filled"]}]}'
|
|
45
|
+
testchimp list-perf-runs --testchimp-id TC-123 --kind JOURNEY --limit 20
|
|
46
|
+
testchimp get-perf-run --run-id 01ABC --include-raw
|
|
47
|
+
testchimp promote-perf-baseline --run-id 01ABC --env-class CI
|
|
48
|
+
testchimp compare-perf-to-baseline --run-id 01ABC --max-p95-regression-percent 10
|
|
49
|
+
testchimp list-related-perf-tests --scenario-titles "Checkout,Refund"
|
|
50
|
+
testchimp list-api-operation-interactions --operation-id 01XYZ --interaction-type REAL --limit 100
|
|
45
51
|
```
|
|
46
52
|
|
|
47
53
|
- **stdout:** API response JSON.
|
|
48
54
|
- **stderr:** progress for `provision-ephemeral-environment-and-wait` (“still waiting…” polls).
|
|
55
|
+
- **performance gate:** `compare-perf-to-baseline` still prints its JSON response but exits nonzero when `regressed` is `true` (top-level or under `comparison`).
|
|
49
56
|
- **Flags:** default for each subcommand; **`--json-input '<json>'`** or **`--json-input @file.json`** merges over flags (JSON wins on conflicts). Use JSON for nested bodies (e.g. TrueCoverage scopes).
|
|
50
57
|
|
|
51
58
|
## Migration from `testchimp-mcp-client`
|
package/dist/cli/program.js
CHANGED
|
@@ -8,6 +8,13 @@ import { TOOL_DEFINITIONS } from "../core/tools.js";
|
|
|
8
8
|
import { resolveGitHeadSha } from "../core/gitSha.js";
|
|
9
9
|
import { PACKAGE_VERSION } from "../core/version.js";
|
|
10
10
|
export { PACKAGE_VERSION };
|
|
11
|
+
/** True when ComparePerfToBaselineResponse (or a flat PerfComparison) reports a regression. */
|
|
12
|
+
function isPerfComparisonRegressed(parsed) {
|
|
13
|
+
if (!parsed || typeof parsed !== "object")
|
|
14
|
+
return false;
|
|
15
|
+
const body = parsed;
|
|
16
|
+
return body.regressed === true || body.comparison?.regressed === true;
|
|
17
|
+
}
|
|
11
18
|
function parseRecordTypesCsv(raw) {
|
|
12
19
|
return String(raw)
|
|
13
20
|
.split(",")
|
|
@@ -23,9 +30,11 @@ function parseRecordTypesCsv(raw) {
|
|
|
23
30
|
return "smart_test";
|
|
24
31
|
if (s === "manual")
|
|
25
32
|
return "manual";
|
|
33
|
+
if (s === "perf" || s === "perftest" || s === "perf_test")
|
|
34
|
+
return "perf_test";
|
|
26
35
|
return s;
|
|
27
36
|
})
|
|
28
|
-
.filter((v) => v === "smart_test" || v === "manual");
|
|
37
|
+
.filter((v) => v === "smart_test" || v === "manual" || v === "perf_test");
|
|
29
38
|
}
|
|
30
39
|
function parseJsonInput(raw) {
|
|
31
40
|
if (raw == null || raw.trim() === "")
|
|
@@ -132,8 +141,9 @@ export function buildCliProgram() {
|
|
|
132
141
|
.option("--environment <s>")
|
|
133
142
|
.option("--branch-name <s>")
|
|
134
143
|
.option("--platform <web|ios|android>")
|
|
135
|
-
.option("--record-types <csv>", "coverage sources: smart_test,manual (aliases: automated,smarttest)")
|
|
144
|
+
.option("--record-types <csv>", "coverage sources: smart_test,manual,perf_test (aliases: automated,smarttest,perf)")
|
|
136
145
|
.option("--include-manual", "include manual session coverage in addition to automated SmartTests")
|
|
146
|
+
.option("--include-perf", "include PERF_TEST journey coverage in addition to automated SmartTests")
|
|
137
147
|
.option("--manual-only", "manual-only coverage (no automated)")
|
|
138
148
|
.option("--lifecycle-statuses <csv>", "scenario lifecycle allowlist (e.g. ready or draft,ready)")
|
|
139
149
|
.option("--limit <n>", "top N gaps after filter+rank into rankedScenarios (max 200)", (v) => parseInt(v, 10))
|
|
@@ -160,6 +170,9 @@ export function buildCliProgram() {
|
|
|
160
170
|
recordTypes = Array.from(new Set([...(recordTypes ?? ["smart_test"]), "manual"]));
|
|
161
171
|
if (opts.manualOnly)
|
|
162
172
|
recordTypes = ["manual"];
|
|
173
|
+
if (opts.includePerf) {
|
|
174
|
+
recordTypes = Array.from(new Set([...(recordTypes ?? ["smart_test"]), "perf_test"]));
|
|
175
|
+
}
|
|
163
176
|
if (recordTypes && recordTypes.length > 0)
|
|
164
177
|
body.recordTypes = recordTypes;
|
|
165
178
|
if (opts.lifecycleStatuses) {
|
|
@@ -1240,6 +1253,21 @@ export function buildCliProgram() {
|
|
|
1240
1253
|
}
|
|
1241
1254
|
console.log(await runTool("upsert-plans-support-file", merged, { postMcp }));
|
|
1242
1255
|
});
|
|
1256
|
+
program
|
|
1257
|
+
.command("get-plans-support-file")
|
|
1258
|
+
.description(TOOL_DEFINITIONS.find((t) => t.kebab === "get-plans-support-file").description)
|
|
1259
|
+
.addOption(jsonInputOption())
|
|
1260
|
+
.option("--file-path <path>", "path relative to mapped plans root (e.g. knowledge/workflow_plans/run-qa/<ulid>.plan.md)")
|
|
1261
|
+
.action(async (opts) => {
|
|
1262
|
+
const body = {};
|
|
1263
|
+
if (opts.filePath)
|
|
1264
|
+
body.filePath = String(opts.filePath);
|
|
1265
|
+
const merged = mergeBodies(body, opts.jsonInput);
|
|
1266
|
+
if (!merged.filePath) {
|
|
1267
|
+
throw new Error("Provide --file-path (or full body via --json-input)");
|
|
1268
|
+
}
|
|
1269
|
+
console.log(await runTool("get-plans-support-file", merged, { postMcp }));
|
|
1270
|
+
});
|
|
1243
1271
|
program
|
|
1244
1272
|
.command("list-api-operation-services")
|
|
1245
1273
|
.description(TOOL_DEFINITIONS.find((t) => t.kebab === "list-api-operation-services").description)
|
|
@@ -1302,6 +1330,174 @@ export function buildCliProgram() {
|
|
|
1302
1330
|
const merged = mergeBodies(body, opts.jsonInput);
|
|
1303
1331
|
console.log(await runTool("get-api-operation-detail", merged, { postMcp }));
|
|
1304
1332
|
});
|
|
1333
|
+
program
|
|
1334
|
+
.command("list-perf-runs")
|
|
1335
|
+
.description(TOOL_DEFINITIONS.find((t) => t.kebab === "list-perf-runs").description)
|
|
1336
|
+
.addOption(jsonInputOption())
|
|
1337
|
+
.option("--testchimp-id <id>")
|
|
1338
|
+
.option("--kind <kind>", "JOURNEY | COMPOSITE")
|
|
1339
|
+
.option("--branch-name <name>")
|
|
1340
|
+
.option("--profile <name>")
|
|
1341
|
+
.option("--dataset <name>")
|
|
1342
|
+
.option("--llm-mode <mode>")
|
|
1343
|
+
.option("--environment <name>")
|
|
1344
|
+
.option("--limit <n>", "Maximum results", (v) => Number(v))
|
|
1345
|
+
.option("--offset <n>", "Pagination offset", (v) => Number(v))
|
|
1346
|
+
.action(async (opts) => {
|
|
1347
|
+
const body = {};
|
|
1348
|
+
if (opts.testchimpId)
|
|
1349
|
+
body.testchimpId = String(opts.testchimpId);
|
|
1350
|
+
if (opts.kind)
|
|
1351
|
+
body.kind = String(opts.kind);
|
|
1352
|
+
if (opts.branchName)
|
|
1353
|
+
body.branchName = String(opts.branchName);
|
|
1354
|
+
if (opts.profile)
|
|
1355
|
+
body.profile = String(opts.profile);
|
|
1356
|
+
if (opts.dataset)
|
|
1357
|
+
body.dataset = String(opts.dataset);
|
|
1358
|
+
if (opts.llmMode)
|
|
1359
|
+
body.llmMode = String(opts.llmMode);
|
|
1360
|
+
if (opts.environment)
|
|
1361
|
+
body.environment = String(opts.environment);
|
|
1362
|
+
if (opts.limit != null)
|
|
1363
|
+
body.limit = opts.limit;
|
|
1364
|
+
if (opts.offset != null)
|
|
1365
|
+
body.offset = opts.offset;
|
|
1366
|
+
console.log(await runTool("list-perf-runs", mergeBodies(body, opts.jsonInput), { postMcp }));
|
|
1367
|
+
});
|
|
1368
|
+
program
|
|
1369
|
+
.command("get-perf-run")
|
|
1370
|
+
.description(TOOL_DEFINITIONS.find((t) => t.kebab === "get-perf-run").description)
|
|
1371
|
+
.addOption(jsonInputOption())
|
|
1372
|
+
.option("--run-id <id>")
|
|
1373
|
+
.option("--include-raw", "Include the raw performance payload")
|
|
1374
|
+
.action(async (opts) => {
|
|
1375
|
+
const body = {};
|
|
1376
|
+
if (opts.runId)
|
|
1377
|
+
body.runId = String(opts.runId);
|
|
1378
|
+
if (opts.includeRaw)
|
|
1379
|
+
body.includeRaw = true;
|
|
1380
|
+
console.log(await runTool("get-perf-run", mergeBodies(body, opts.jsonInput), { postMcp }));
|
|
1381
|
+
});
|
|
1382
|
+
program
|
|
1383
|
+
.command("list-perf-baselines")
|
|
1384
|
+
.description(TOOL_DEFINITIONS.find((t) => t.kebab === "list-perf-baselines").description)
|
|
1385
|
+
.addOption(jsonInputOption())
|
|
1386
|
+
.option("--testchimp-id <id>")
|
|
1387
|
+
.option("--limit <n>", "Maximum results", (v) => Number(v))
|
|
1388
|
+
.option("--offset <n>", "Pagination offset", (v) => Number(v))
|
|
1389
|
+
.action(async (opts) => {
|
|
1390
|
+
const body = {};
|
|
1391
|
+
if (opts.testchimpId)
|
|
1392
|
+
body.testchimpId = String(opts.testchimpId);
|
|
1393
|
+
if (opts.limit != null)
|
|
1394
|
+
body.limit = opts.limit;
|
|
1395
|
+
if (opts.offset != null)
|
|
1396
|
+
body.offset = opts.offset;
|
|
1397
|
+
console.log(await runTool("list-perf-baselines", mergeBodies(body, opts.jsonInput), { postMcp }));
|
|
1398
|
+
});
|
|
1399
|
+
addAgentTraceabilityOptions(program
|
|
1400
|
+
.command("promote-perf-baseline")
|
|
1401
|
+
.description(TOOL_DEFINITIONS.find((t) => t.kebab === "promote-perf-baseline").description)
|
|
1402
|
+
.addOption(jsonInputOption())
|
|
1403
|
+
.option("--run-id <id>")
|
|
1404
|
+
.option("--env-class <name>")).action(async (opts) => {
|
|
1405
|
+
const body = {
|
|
1406
|
+
...collectAgentTraceabilityFlags(opts),
|
|
1407
|
+
};
|
|
1408
|
+
if (opts.runId)
|
|
1409
|
+
body.runId = String(opts.runId);
|
|
1410
|
+
if (opts.envClass)
|
|
1411
|
+
body.envClass = String(opts.envClass);
|
|
1412
|
+
console.log(await runTool("promote-perf-baseline", mergeBodies(body, opts.jsonInput), { postMcp }));
|
|
1413
|
+
});
|
|
1414
|
+
program
|
|
1415
|
+
.command("compare-perf-to-baseline")
|
|
1416
|
+
.description(TOOL_DEFINITIONS.find((t) => t.kebab === "compare-perf-to-baseline").description)
|
|
1417
|
+
.addOption(jsonInputOption())
|
|
1418
|
+
.option("--run-id <id>")
|
|
1419
|
+
.option("--testchimp-id <id>")
|
|
1420
|
+
.option("--profile <name>")
|
|
1421
|
+
.option("--dataset <name>")
|
|
1422
|
+
.option("--llm-mode <mode>")
|
|
1423
|
+
.option("--environment <name>")
|
|
1424
|
+
.option("--env-class <name>", "Baseline environment class (required)")
|
|
1425
|
+
.option("--max-p95-regression-percent <n>", "Allowed p95 regression percent", (v) => Number(v))
|
|
1426
|
+
.option("--max-fail-rate-increase <n>", "Allowed fail-rate increase", (v) => Number(v))
|
|
1427
|
+
.action(async (opts) => {
|
|
1428
|
+
const body = {};
|
|
1429
|
+
if (opts.runId)
|
|
1430
|
+
body.runId = String(opts.runId);
|
|
1431
|
+
if (opts.testchimpId)
|
|
1432
|
+
body.testchimpId = String(opts.testchimpId);
|
|
1433
|
+
if (opts.profile)
|
|
1434
|
+
body.profile = String(opts.profile);
|
|
1435
|
+
if (opts.dataset)
|
|
1436
|
+
body.dataset = String(opts.dataset);
|
|
1437
|
+
if (opts.llmMode)
|
|
1438
|
+
body.llmMode = String(opts.llmMode);
|
|
1439
|
+
if (opts.environment)
|
|
1440
|
+
body.environment = String(opts.environment);
|
|
1441
|
+
if (opts.envClass)
|
|
1442
|
+
body.envClass = String(opts.envClass);
|
|
1443
|
+
if (opts.maxP95RegressionPercent != null) {
|
|
1444
|
+
body.maxP95RegressionPercent = opts.maxP95RegressionPercent;
|
|
1445
|
+
}
|
|
1446
|
+
if (opts.maxFailRateIncrease != null)
|
|
1447
|
+
body.maxFailRateIncrease = opts.maxFailRateIncrease;
|
|
1448
|
+
const out = await runTool("compare-perf-to-baseline", mergeBodies(body, opts.jsonInput), { postMcp });
|
|
1449
|
+
// Always print response JSON; gate CI on regressed after stdout flush.
|
|
1450
|
+
console.log(out);
|
|
1451
|
+
try {
|
|
1452
|
+
if (isPerfComparisonRegressed(JSON.parse(out)))
|
|
1453
|
+
process.exitCode = 1;
|
|
1454
|
+
}
|
|
1455
|
+
catch {
|
|
1456
|
+
/* non-JSON responses still printed; leave exit 0 unless runTool threw */
|
|
1457
|
+
}
|
|
1458
|
+
});
|
|
1459
|
+
program
|
|
1460
|
+
.command("list-related-perf-tests")
|
|
1461
|
+
.description(TOOL_DEFINITIONS.find((t) => t.kebab === "list-related-perf-tests").description)
|
|
1462
|
+
.addOption(jsonInputOption())
|
|
1463
|
+
.option("--scenario-titles <csv>", "Comma-separated scenario titles")
|
|
1464
|
+
.option("--testchimp-ids <csv>", "Comma-separated TestChimp ids")
|
|
1465
|
+
.option("--no-include-composites", "Exclude COMPOSITE tests")
|
|
1466
|
+
.option("--limit <n>", "Maximum results (max 100)", (v) => Number(v))
|
|
1467
|
+
.action(async (opts) => {
|
|
1468
|
+
const body = {
|
|
1469
|
+
includeComposites: opts.includeComposites,
|
|
1470
|
+
};
|
|
1471
|
+
if (opts.scenarioTitles) {
|
|
1472
|
+
body.scenarioTitles = String(opts.scenarioTitles).split(",").map((s) => s.trim()).filter(Boolean);
|
|
1473
|
+
}
|
|
1474
|
+
if (opts.testchimpIds) {
|
|
1475
|
+
body.testchimpIds = String(opts.testchimpIds).split(",").map((s) => s.trim()).filter(Boolean);
|
|
1476
|
+
}
|
|
1477
|
+
if (opts.limit != null)
|
|
1478
|
+
body.limit = opts.limit;
|
|
1479
|
+
console.log(await runTool("list-related-perf-tests", mergeBodies(body, opts.jsonInput), { postMcp }));
|
|
1480
|
+
});
|
|
1481
|
+
program
|
|
1482
|
+
.command("list-api-operation-interactions")
|
|
1483
|
+
.description(TOOL_DEFINITIONS.find((t) => t.kebab === "list-api-operation-interactions").description)
|
|
1484
|
+
.addOption(jsonInputOption())
|
|
1485
|
+
.option("--test-id <id>")
|
|
1486
|
+
.option("--operation-id <id>")
|
|
1487
|
+
.option("--interaction-type <type>", "REAL | MOCKED", "REAL")
|
|
1488
|
+
.option("--limit <n>", "Maximum results (max 100)", (v) => Number(v))
|
|
1489
|
+
.action(async (opts) => {
|
|
1490
|
+
const body = {
|
|
1491
|
+
interactionType: opts.interactionType,
|
|
1492
|
+
};
|
|
1493
|
+
if (opts.testId)
|
|
1494
|
+
body.testId = String(opts.testId);
|
|
1495
|
+
if (opts.operationId)
|
|
1496
|
+
body.operationId = String(opts.operationId);
|
|
1497
|
+
if (opts.limit != null)
|
|
1498
|
+
body.limit = opts.limit;
|
|
1499
|
+
console.log(await runTool("list-api-operation-interactions", mergeBodies(body, opts.jsonInput), { postMcp }));
|
|
1500
|
+
});
|
|
1305
1501
|
program.on("--help", () => {
|
|
1306
1502
|
/* default */
|
|
1307
1503
|
});
|
package/dist/core/schemas.d.ts
CHANGED
|
@@ -25,8 +25,10 @@ export declare const listCoverageInput: z.ZodObject<{
|
|
|
25
25
|
recordTypes: z.ZodOptional<z.ZodArray<z.ZodEnum<{
|
|
26
26
|
smart_test: "smart_test";
|
|
27
27
|
manual: "manual";
|
|
28
|
+
perf_test: "perf_test";
|
|
28
29
|
SMART_TEST: "SMART_TEST";
|
|
29
30
|
MANUAL: "MANUAL";
|
|
31
|
+
PERF_TEST: "PERF_TEST";
|
|
30
32
|
}>>>;
|
|
31
33
|
scenarioLifecycleStatuses: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
32
34
|
limit: z.ZodOptional<z.ZodNumber>;
|
|
@@ -1476,6 +1478,9 @@ export declare const upsertPlansSupportFileInput: z.ZodObject<{
|
|
|
1476
1478
|
filePath: z.ZodString;
|
|
1477
1479
|
content: z.ZodString;
|
|
1478
1480
|
}, z.core.$strip>;
|
|
1481
|
+
export declare const getPlansSupportFileInput: z.ZodObject<{
|
|
1482
|
+
filePath: z.ZodString;
|
|
1483
|
+
}, z.core.$strip>;
|
|
1479
1484
|
export declare const listWorkflowCatalogInput: z.ZodObject<{}, z.core.$strip>;
|
|
1480
1485
|
/** API operation coverage (OpenAPI ops + denorm coverage) — CLI ≥ 0.1.28 */
|
|
1481
1486
|
export declare const listApiOperationServicesInput: z.ZodObject<{}, z.core.$strip>;
|
|
@@ -1495,3 +1500,95 @@ export declare const getApiOperationDetailInput: z.ZodObject<{
|
|
|
1495
1500
|
includeManual: z.ZodOptional<z.ZodBoolean>;
|
|
1496
1501
|
includeRemoved: z.ZodOptional<z.ZodBoolean>;
|
|
1497
1502
|
}, z.core.$strip>;
|
|
1503
|
+
/** Performance run kind persisted by the Phase 2 performance API. */
|
|
1504
|
+
export declare const perfRunKindSchema: z.ZodEnum<{
|
|
1505
|
+
JOURNEY: "JOURNEY";
|
|
1506
|
+
COMPOSITE: "COMPOSITE";
|
|
1507
|
+
}>;
|
|
1508
|
+
export declare const listPerfRunsInput: z.ZodObject<{
|
|
1509
|
+
testchimpId: z.ZodOptional<z.ZodString>;
|
|
1510
|
+
kind: z.ZodOptional<z.ZodEnum<{
|
|
1511
|
+
JOURNEY: "JOURNEY";
|
|
1512
|
+
COMPOSITE: "COMPOSITE";
|
|
1513
|
+
}>>;
|
|
1514
|
+
branchName: z.ZodOptional<z.ZodString>;
|
|
1515
|
+
profile: z.ZodOptional<z.ZodString>;
|
|
1516
|
+
dataset: z.ZodOptional<z.ZodString>;
|
|
1517
|
+
llmMode: z.ZodOptional<z.ZodString>;
|
|
1518
|
+
environment: z.ZodOptional<z.ZodString>;
|
|
1519
|
+
limit: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
|
|
1520
|
+
offset: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
|
|
1521
|
+
}, z.core.$strip>;
|
|
1522
|
+
export declare const getPerfRunInput: z.ZodObject<{
|
|
1523
|
+
runId: z.ZodString;
|
|
1524
|
+
includeRaw: z.ZodOptional<z.ZodBoolean>;
|
|
1525
|
+
}, z.core.$strip>;
|
|
1526
|
+
export declare const listPerfBaselinesInput: z.ZodObject<{
|
|
1527
|
+
testchimpId: z.ZodOptional<z.ZodString>;
|
|
1528
|
+
limit: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
|
|
1529
|
+
offset: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
|
|
1530
|
+
}, z.core.$strip>;
|
|
1531
|
+
export declare const promotePerfBaselineInput: z.ZodObject<{
|
|
1532
|
+
runId: z.ZodString;
|
|
1533
|
+
envClass: z.ZodString;
|
|
1534
|
+
workflowId: z.ZodOptional<z.ZodString>;
|
|
1535
|
+
workflowExecutionId: z.ZodOptional<z.ZodString>;
|
|
1536
|
+
policyFile: z.ZodOptional<z.ZodString>;
|
|
1537
|
+
policyVersion: z.ZodOptional<z.ZodString>;
|
|
1538
|
+
gitSha: z.ZodOptional<z.ZodString>;
|
|
1539
|
+
actorType: z.ZodOptional<z.ZodEnum<{
|
|
1540
|
+
LOCAL_AGENT: "LOCAL_AGENT";
|
|
1541
|
+
CLOUD_AGENT: "CLOUD_AGENT";
|
|
1542
|
+
"local-agent": "local-agent";
|
|
1543
|
+
"cloud-agent": "cloud-agent";
|
|
1544
|
+
}>>;
|
|
1545
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
1546
|
+
branchName: z.ZodOptional<z.ZodString>;
|
|
1547
|
+
agentModel: z.ZodOptional<z.ZodString>;
|
|
1548
|
+
skillVersion: z.ZodOptional<z.ZodString>;
|
|
1549
|
+
cliVersion: z.ZodOptional<z.ZodString>;
|
|
1550
|
+
agentTraceability: z.ZodOptional<z.ZodObject<{
|
|
1551
|
+
workflowId: z.ZodOptional<z.ZodString>;
|
|
1552
|
+
workflowExecutionId: z.ZodOptional<z.ZodString>;
|
|
1553
|
+
policyFile: z.ZodOptional<z.ZodString>;
|
|
1554
|
+
policyVersion: z.ZodOptional<z.ZodString>;
|
|
1555
|
+
gitSha: z.ZodOptional<z.ZodString>;
|
|
1556
|
+
actorType: z.ZodOptional<z.ZodEnum<{
|
|
1557
|
+
LOCAL_AGENT: "LOCAL_AGENT";
|
|
1558
|
+
CLOUD_AGENT: "CLOUD_AGENT";
|
|
1559
|
+
"local-agent": "local-agent";
|
|
1560
|
+
"cloud-agent": "cloud-agent";
|
|
1561
|
+
}>>;
|
|
1562
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
1563
|
+
branchName: z.ZodOptional<z.ZodString>;
|
|
1564
|
+
agentModel: z.ZodOptional<z.ZodString>;
|
|
1565
|
+
skillVersion: z.ZodOptional<z.ZodString>;
|
|
1566
|
+
cliVersion: z.ZodOptional<z.ZodString>;
|
|
1567
|
+
}, z.core.$strict>>;
|
|
1568
|
+
}, z.core.$strip>;
|
|
1569
|
+
export declare const comparePerfToBaselineInput: z.ZodObject<{
|
|
1570
|
+
runId: z.ZodOptional<z.ZodString>;
|
|
1571
|
+
testchimpId: z.ZodOptional<z.ZodString>;
|
|
1572
|
+
profile: z.ZodOptional<z.ZodString>;
|
|
1573
|
+
dataset: z.ZodOptional<z.ZodString>;
|
|
1574
|
+
llmMode: z.ZodOptional<z.ZodString>;
|
|
1575
|
+
environment: z.ZodOptional<z.ZodString>;
|
|
1576
|
+
envClass: z.ZodString;
|
|
1577
|
+
maxP95RegressionPercent: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
|
|
1578
|
+
maxFailRateIncrease: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
|
|
1579
|
+
}, z.core.$strip>;
|
|
1580
|
+
export declare const listRelatedPerfTestsInput: z.ZodObject<{
|
|
1581
|
+
scenarioTitles: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
1582
|
+
testchimpIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
1583
|
+
includeComposites: z.ZodDefault<z.ZodBoolean>;
|
|
1584
|
+
limit: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
|
|
1585
|
+
}, z.core.$strip>;
|
|
1586
|
+
export declare const listApiOperationInteractionsInput: z.ZodObject<{
|
|
1587
|
+
testId: z.ZodOptional<z.ZodString>;
|
|
1588
|
+
operationId: z.ZodOptional<z.ZodString>;
|
|
1589
|
+
interactionType: z.ZodDefault<z.ZodEnum<{
|
|
1590
|
+
REAL: "REAL";
|
|
1591
|
+
MOCKED: "MOCKED";
|
|
1592
|
+
}>>;
|
|
1593
|
+
limit: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
|
|
1594
|
+
}, z.core.$strip>;
|
package/dist/core/schemas.js
CHANGED
|
@@ -6,7 +6,14 @@ export const scopeSchema = z
|
|
|
6
6
|
})
|
|
7
7
|
.optional();
|
|
8
8
|
const executionPlatformSchema = z.enum(["web", "ios", "android"]);
|
|
9
|
-
const requirementCoverageRecordTypeSchema = z.enum([
|
|
9
|
+
const requirementCoverageRecordTypeSchema = z.enum([
|
|
10
|
+
"smart_test",
|
|
11
|
+
"manual",
|
|
12
|
+
"perf_test",
|
|
13
|
+
"SMART_TEST",
|
|
14
|
+
"MANUAL",
|
|
15
|
+
"PERF_TEST",
|
|
16
|
+
]);
|
|
10
17
|
export const executionJobDimensionFilterSchema = z.object({
|
|
11
18
|
dimension: z.string().min(1),
|
|
12
19
|
values: z.array(z.string()).min(1),
|
|
@@ -23,7 +30,8 @@ export const listCoverageInput = z.object({
|
|
|
23
30
|
* Which coverage sources to include.
|
|
24
31
|
*
|
|
25
32
|
* Omit for legacy default: SMART_TEST only.
|
|
26
|
-
* When provided, send proto enum names ("SMART_TEST", "MANUAL"
|
|
33
|
+
* When provided, send proto enum names ("SMART_TEST", "MANUAL", "PERF_TEST")
|
|
34
|
+
* or CLI-friendly aliases ("smart_test", "manual", "perf_test").
|
|
27
35
|
*/
|
|
28
36
|
recordTypes: z.array(requirementCoverageRecordTypeSchema).optional(),
|
|
29
37
|
/** Allowlist of scenario lifecycle statuses (e.g. ["ready"] or ["draft","ready"]). Empty/omit = no status filter. */
|
|
@@ -803,6 +811,10 @@ export const upsertPlansSupportFileInput = z.object({
|
|
|
803
811
|
filePath: z.string().min(1),
|
|
804
812
|
content: z.string().min(1),
|
|
805
813
|
});
|
|
814
|
+
export const getPlansSupportFileInput = z.object({
|
|
815
|
+
/** Path relative to mapped plans root (e.g. knowledge/workflow_plans/run-qa/<ulid>.plan.md). */
|
|
816
|
+
filePath: z.string().min(1),
|
|
817
|
+
});
|
|
806
818
|
export const listWorkflowCatalogInput = z.object({});
|
|
807
819
|
/** API operation coverage (OpenAPI ops + denorm coverage) — CLI ≥ 0.1.28 */
|
|
808
820
|
export const listApiOperationServicesInput = z.object({});
|
|
@@ -848,3 +860,81 @@ export const getApiOperationDetailInput = z
|
|
|
848
860
|
"or --root-file-path/--service-key with --http-method and --path-template",
|
|
849
861
|
});
|
|
850
862
|
});
|
|
863
|
+
/** Performance run kind persisted by the Phase 2 performance API. */
|
|
864
|
+
export const perfRunKindSchema = z.enum(["JOURNEY", "COMPOSITE"]);
|
|
865
|
+
export const listPerfRunsInput = z.object({
|
|
866
|
+
testchimpId: z.string().min(1).optional(),
|
|
867
|
+
kind: perfRunKindSchema.optional(),
|
|
868
|
+
branchName: z.string().min(1).optional(),
|
|
869
|
+
profile: z.string().min(1).optional(),
|
|
870
|
+
dataset: z.string().min(1).optional(),
|
|
871
|
+
llmMode: z.string().min(1).optional(),
|
|
872
|
+
environment: z.string().min(1).optional(),
|
|
873
|
+
limit: z.coerce.number().int().positive().max(100).optional(),
|
|
874
|
+
offset: z.coerce.number().int().nonnegative().optional(),
|
|
875
|
+
});
|
|
876
|
+
export const getPerfRunInput = z.object({
|
|
877
|
+
runId: z.string().min(1),
|
|
878
|
+
includeRaw: z.boolean().optional(),
|
|
879
|
+
});
|
|
880
|
+
export const listPerfBaselinesInput = z.object({
|
|
881
|
+
testchimpId: z.string().min(1).optional(),
|
|
882
|
+
limit: z.coerce.number().int().positive().max(100).optional(),
|
|
883
|
+
offset: z.coerce.number().int().nonnegative().optional(),
|
|
884
|
+
});
|
|
885
|
+
export const promotePerfBaselineInput = z
|
|
886
|
+
.object({
|
|
887
|
+
runId: z.string().min(1),
|
|
888
|
+
envClass: z.string().min(1),
|
|
889
|
+
})
|
|
890
|
+
.merge(agentTraceabilityFieldsSchema);
|
|
891
|
+
export const comparePerfToBaselineInput = z
|
|
892
|
+
.object({
|
|
893
|
+
runId: z.string().min(1).optional(),
|
|
894
|
+
testchimpId: z.string().min(1).optional(),
|
|
895
|
+
profile: z.string().min(1).optional(),
|
|
896
|
+
dataset: z.string().min(1).optional(),
|
|
897
|
+
llmMode: z.string().min(1).optional(),
|
|
898
|
+
environment: z.string().min(1).optional(),
|
|
899
|
+
envClass: z.string().min(1),
|
|
900
|
+
maxP95RegressionPercent: z.coerce.number().nonnegative().optional(),
|
|
901
|
+
maxFailRateIncrease: z.coerce.number().nonnegative().optional(),
|
|
902
|
+
})
|
|
903
|
+
.superRefine((v, ctx) => {
|
|
904
|
+
if (!(v.runId ?? "").trim() && !(v.testchimpId ?? "").trim()) {
|
|
905
|
+
ctx.addIssue({
|
|
906
|
+
code: z.ZodIssueCode.custom,
|
|
907
|
+
message: "Provide runId or testchimpId",
|
|
908
|
+
});
|
|
909
|
+
}
|
|
910
|
+
});
|
|
911
|
+
export const listRelatedPerfTestsInput = z
|
|
912
|
+
.object({
|
|
913
|
+
scenarioTitles: z.array(z.string().min(1)).min(1).optional(),
|
|
914
|
+
testchimpIds: z.array(z.string().min(1)).min(1).optional(),
|
|
915
|
+
includeComposites: z.boolean().default(true),
|
|
916
|
+
limit: z.coerce.number().int().positive().max(100).optional(),
|
|
917
|
+
})
|
|
918
|
+
.superRefine((v, ctx) => {
|
|
919
|
+
if (!v.scenarioTitles?.length && !v.testchimpIds?.length) {
|
|
920
|
+
ctx.addIssue({
|
|
921
|
+
code: z.ZodIssueCode.custom,
|
|
922
|
+
message: "Provide scenarioTitles and/or testchimpIds",
|
|
923
|
+
});
|
|
924
|
+
}
|
|
925
|
+
});
|
|
926
|
+
export const listApiOperationInteractionsInput = z
|
|
927
|
+
.object({
|
|
928
|
+
testId: z.string().min(1).optional(),
|
|
929
|
+
operationId: z.string().min(1).optional(),
|
|
930
|
+
interactionType: z.enum(["REAL", "MOCKED"]).default("REAL"),
|
|
931
|
+
limit: z.coerce.number().int().positive().max(100).optional(),
|
|
932
|
+
})
|
|
933
|
+
.superRefine((v, ctx) => {
|
|
934
|
+
if (!(v.testId ?? "").trim() && !(v.operationId ?? "").trim()) {
|
|
935
|
+
ctx.addIssue({
|
|
936
|
+
code: z.ZodIssueCode.custom,
|
|
937
|
+
message: "Provide testId and/or operationId",
|
|
938
|
+
});
|
|
939
|
+
}
|
|
940
|
+
});
|
package/dist/core/tools.js
CHANGED
|
@@ -51,6 +51,8 @@ function listCoverageBody(args) {
|
|
|
51
51
|
return "MANUAL";
|
|
52
52
|
if (raw === "smart_test")
|
|
53
53
|
return "SMART_TEST";
|
|
54
|
+
if (raw === "perf_test")
|
|
55
|
+
return "PERF_TEST";
|
|
54
56
|
return raw;
|
|
55
57
|
});
|
|
56
58
|
body.recordTypes = normalized;
|
|
@@ -1181,6 +1183,20 @@ export const TOOL_DEFINITIONS = [
|
|
|
1181
1183
|
});
|
|
1182
1184
|
},
|
|
1183
1185
|
},
|
|
1186
|
+
{
|
|
1187
|
+
kebab: "get-plans-support-file",
|
|
1188
|
+
description: "Fetch a file under the mapped plans root from the platform by relative path (no git required). " +
|
|
1189
|
+
"Primary use: load a workflow execution plan named in a Continue Locally / implement prompt before falling back to the repo copy. " +
|
|
1190
|
+
"filePath is relative to the plans mapped root (leading plans/ is stripped). Under workflow_plans/, filenames are coerced to *.plan.md. " +
|
|
1191
|
+
"Response: found (false if missing), supportFileId, filePath (canonical), filetype, content.",
|
|
1192
|
+
inputSchema: S.getPlansSupportFileInput,
|
|
1193
|
+
execute: async (args, { postMcp }) => {
|
|
1194
|
+
const a = args;
|
|
1195
|
+
return postMcp("/api/mcp/get_plans_support_file", {
|
|
1196
|
+
filePath: a.filePath,
|
|
1197
|
+
});
|
|
1198
|
+
},
|
|
1199
|
+
},
|
|
1184
1200
|
{
|
|
1185
1201
|
kebab: "list-workflow-catalog",
|
|
1186
1202
|
description: "List supported TestChimp workflows with Active / Disabled / Missing Config status for the project.",
|
|
@@ -1241,6 +1257,143 @@ export const TOOL_DEFINITIONS = [
|
|
|
1241
1257
|
return postMcp("/api/mcp/get_api_operation_detail", body);
|
|
1242
1258
|
},
|
|
1243
1259
|
},
|
|
1260
|
+
{
|
|
1261
|
+
kebab: "list-perf-runs",
|
|
1262
|
+
description: "List performance runs, optionally filtered by TestChimp id, JOURNEY/COMPOSITE kind, branch, profile, dataset, LLM mode, or environment.",
|
|
1263
|
+
inputSchema: S.listPerfRunsInput,
|
|
1264
|
+
execute: async (args, { postMcp }) => {
|
|
1265
|
+
const a = args;
|
|
1266
|
+
const body = {};
|
|
1267
|
+
if (a.testchimpId)
|
|
1268
|
+
body.testchimpId = a.testchimpId;
|
|
1269
|
+
if (a.kind)
|
|
1270
|
+
body.kind = a.kind;
|
|
1271
|
+
if (a.branchName)
|
|
1272
|
+
body.branchName = a.branchName;
|
|
1273
|
+
if (a.profile)
|
|
1274
|
+
body.profile = a.profile;
|
|
1275
|
+
if (a.dataset)
|
|
1276
|
+
body.dataset = a.dataset;
|
|
1277
|
+
if (a.llmMode)
|
|
1278
|
+
body.llmMode = a.llmMode;
|
|
1279
|
+
if (a.environment)
|
|
1280
|
+
body.environment = a.environment;
|
|
1281
|
+
if (a.limit != null)
|
|
1282
|
+
body.limit = a.limit;
|
|
1283
|
+
if (a.offset != null)
|
|
1284
|
+
body.offset = a.offset;
|
|
1285
|
+
return postMcp("/api/mcp/list_perf_runs", body);
|
|
1286
|
+
},
|
|
1287
|
+
},
|
|
1288
|
+
{
|
|
1289
|
+
kebab: "get-perf-run",
|
|
1290
|
+
description: "Fetch one performance run by runId; set includeRaw to include its raw payload.",
|
|
1291
|
+
inputSchema: S.getPerfRunInput,
|
|
1292
|
+
execute: async (args, { postMcp }) => {
|
|
1293
|
+
const a = args;
|
|
1294
|
+
return postMcp("/api/mcp/get_perf_run", {
|
|
1295
|
+
runId: a.runId,
|
|
1296
|
+
...(a.includeRaw != null ? { includeRaw: a.includeRaw } : {}),
|
|
1297
|
+
});
|
|
1298
|
+
},
|
|
1299
|
+
},
|
|
1300
|
+
{
|
|
1301
|
+
kebab: "list-perf-baselines",
|
|
1302
|
+
description: "List promoted performance baselines, optionally filtered by TestChimp id.",
|
|
1303
|
+
inputSchema: S.listPerfBaselinesInput,
|
|
1304
|
+
execute: async (args, { postMcp }) => {
|
|
1305
|
+
const a = args;
|
|
1306
|
+
const body = {};
|
|
1307
|
+
if (a.testchimpId)
|
|
1308
|
+
body.testchimpId = a.testchimpId;
|
|
1309
|
+
if (a.limit != null)
|
|
1310
|
+
body.limit = a.limit;
|
|
1311
|
+
if (a.offset != null)
|
|
1312
|
+
body.offset = a.offset;
|
|
1313
|
+
return postMcp("/api/mcp/list_perf_baselines", body);
|
|
1314
|
+
},
|
|
1315
|
+
},
|
|
1316
|
+
{
|
|
1317
|
+
kebab: "promote-perf-baseline",
|
|
1318
|
+
description: "Promote a performance run as the baseline for an environment class. Optional agent traceability records the mutation.",
|
|
1319
|
+
inputSchema: S.promotePerfBaselineInput,
|
|
1320
|
+
execute: async (args, { postMcp }) => {
|
|
1321
|
+
const a = args;
|
|
1322
|
+
const body = {
|
|
1323
|
+
runId: a.runId,
|
|
1324
|
+
envClass: a.envClass,
|
|
1325
|
+
};
|
|
1326
|
+
const trace = buildAgentTraceabilityPayload(a);
|
|
1327
|
+
if (trace)
|
|
1328
|
+
body.agentTraceability = trace;
|
|
1329
|
+
return postMcp("/api/mcp/promote_perf_baseline", body);
|
|
1330
|
+
},
|
|
1331
|
+
},
|
|
1332
|
+
{
|
|
1333
|
+
kebab: "compare-perf-to-baseline",
|
|
1334
|
+
description: "Compare a run (runId) or filtered target (testchimpId plus optional dimensions) to its promoted baseline. " +
|
|
1335
|
+
"envClass is required. Optional thresholds override max p95 regression percent and maximum fail-rate increase.",
|
|
1336
|
+
inputSchema: S.comparePerfToBaselineInput,
|
|
1337
|
+
execute: async (args, { postMcp }) => {
|
|
1338
|
+
const a = args;
|
|
1339
|
+
const body = {};
|
|
1340
|
+
if (a.runId)
|
|
1341
|
+
body.runId = a.runId;
|
|
1342
|
+
if (a.testchimpId)
|
|
1343
|
+
body.testchimpId = a.testchimpId;
|
|
1344
|
+
if (a.profile)
|
|
1345
|
+
body.profile = a.profile;
|
|
1346
|
+
if (a.dataset)
|
|
1347
|
+
body.dataset = a.dataset;
|
|
1348
|
+
if (a.llmMode)
|
|
1349
|
+
body.llmMode = a.llmMode;
|
|
1350
|
+
if (a.environment)
|
|
1351
|
+
body.environment = a.environment;
|
|
1352
|
+
body.envClass = a.envClass;
|
|
1353
|
+
if (a.maxP95RegressionPercent != null)
|
|
1354
|
+
body.maxP95RegressionPercent = a.maxP95RegressionPercent;
|
|
1355
|
+
if (a.maxFailRateIncrease != null)
|
|
1356
|
+
body.maxFailRateIncrease = a.maxFailRateIncrease;
|
|
1357
|
+
return postMcp("/api/mcp/compare_perf_to_baseline", body);
|
|
1358
|
+
},
|
|
1359
|
+
},
|
|
1360
|
+
{
|
|
1361
|
+
kebab: "list-related-perf-tests",
|
|
1362
|
+
description: "Find JOURNEY and, by default, COMPOSITE performance tests related to scenario titles and/or TestChimp ids.",
|
|
1363
|
+
inputSchema: S.listRelatedPerfTestsInput,
|
|
1364
|
+
execute: async (args, { postMcp }) => {
|
|
1365
|
+
const a = args;
|
|
1366
|
+
const body = {
|
|
1367
|
+
includeComposites: a.includeComposites,
|
|
1368
|
+
};
|
|
1369
|
+
if (a.scenarioTitles?.length)
|
|
1370
|
+
body.scenarioTitles = a.scenarioTitles;
|
|
1371
|
+
if (a.testchimpIds?.length)
|
|
1372
|
+
body.testchimpIds = a.testchimpIds;
|
|
1373
|
+
if (a.limit != null)
|
|
1374
|
+
body.limit = a.limit;
|
|
1375
|
+
return postMcp("/api/mcp/list_related_perf_tests", body);
|
|
1376
|
+
},
|
|
1377
|
+
},
|
|
1378
|
+
{
|
|
1379
|
+
kebab: "list-api-operation-interactions",
|
|
1380
|
+
description: "List recorded API operation interactions. Requires testId and/or operationId. " +
|
|
1381
|
+
"Defaults to REAL interactions; limit is capped at 100.",
|
|
1382
|
+
inputSchema: S.listApiOperationInteractionsInput,
|
|
1383
|
+
execute: async (args, { postMcp }) => {
|
|
1384
|
+
const a = args;
|
|
1385
|
+
const body = {
|
|
1386
|
+
interactionType: a.interactionType,
|
|
1387
|
+
};
|
|
1388
|
+
if (a.testId)
|
|
1389
|
+
body.testId = a.testId;
|
|
1390
|
+
if (a.operationId)
|
|
1391
|
+
body.operationId = a.operationId;
|
|
1392
|
+
if (a.limit != null)
|
|
1393
|
+
body.limit = a.limit;
|
|
1394
|
+
return postMcp("/api/mcp/list_api_operation_interactions", body);
|
|
1395
|
+
},
|
|
1396
|
+
},
|
|
1244
1397
|
];
|
|
1245
1398
|
const TOOL_BY_KEBAB = new Map(TOOL_DEFINITIONS.map((t) => [t.kebab, t]));
|
|
1246
1399
|
export function getToolDefinition(kebab) {
|
package/package.json
CHANGED