@plune-ai/cli 0.34.0 → 0.34.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -117,7 +117,7 @@ the last resort, used only when `PLUNE_TOKEN` is set nowhere else. Nothing write
117
117
  | ------- | ------- |
118
118
  | `plune run` | Run the suite. Flags: `--dry-run` (price the run from the model's rates — no provider key, no network), `--only <id\|tag>` (repeatable), `--bail`, `--no-cache`, `--cache-strict`, `--cache-max-age <duration>`, `--concurrency <n>`, `--format console\|json\|markdown`, `-o, --output <file>`. |
119
119
  | `plune report` | Re-render the most recent run. Flags: `--format`, `-o`. |
120
- | `plune diff <baseline> <current>` | Compare two `plune run --format json` outputs and report pass→fail regressions. Flags: `--fail-on-regression`, `--format`, `-o`. |
120
+ | `plune diff <baseline> <current>` | Compare two `plune run --format json` outputs and report pass→fail regressions. Flags: `--fail-on-regression` (exit `1` on a regression, `3` when none but an eval was not measured), `--format`, `-o`. |
121
121
  | `plune init` | Scaffold `plune.yaml`, a sample dataset, and `.env.example`. Flags: `--yes` (non-interactive), `--force`. |
122
122
  | `plune login` | Save a [Plune platform](https://plune.ai/platform) API token so `sync` and `ingest` can reach it. The token is **checked against the API before it is saved**, so a wrong one fails here rather than two commands later. Get one at `https://beta.plune.ai` → Account → API tokens. Flags: `--token <token>` (omit to paste it or pipe it via stdin), `--skip-verify` (save without checking, for offline setup). In CI, skip this step: every platform command reads `PLUNE_TOKEN` from the environment first, and the saved login only when it is not set. |
123
123
  | `plune logout` | Remove the saved token. |
@@ -631,6 +631,16 @@ once; the rest it finds with `plune_discover_tools`. Every write tool takes `dry
631
631
  100 items, and the agent is held to the platform's agent rules: a proposed case lands in the review
632
632
  queue, a bulk decision needs a confirm, and nothing is deleted through MCP.
633
633
 
634
+ `plune_cases_generate` asks the project's AI for case proposals from pasted text (a requirement, a
635
+ story, a ticket's text; 20 words to 12 000 characters, up to 12 proposals, 8 by default). It spends the
636
+ project's AI budget, and its proposals wait in the review queue like any other until a person approves
637
+ them. The run is queued: the tool answers with its id, and `plune_cases_generation_get` reads where it
638
+ stands, what it produced and what it cost. A Jira ticket by its key is not taken over MCP: paste its
639
+ text. The platform's refusals (409 `duplicate_source` until `confirmDuplicates: true`, 422 for text
640
+ too short or too long, 429 limits, 503 / 409 when the project's AI is not set up) come back with
641
+ their `code` as the platform sent them. Both tools work in the server's `--project` unless the call
642
+ names a `projectId`.
643
+
634
644
  Claude Code:
635
645
 
636
646
  ```bash
package/dist/cli.cjs CHANGED
@@ -2010,7 +2010,8 @@ async function runOrchestration(config, options, deps) {
2010
2010
  limit,
2011
2011
  (row) => runRow({
2012
2012
  template,
2013
- assertions: ev.assertions,
2013
+ // `defaults.assertions` apply to every eval, ahead of its own (#2415).
2014
+ assertions: [...config.defaults?.assertions ?? [], ...ev.assertions],
2014
2015
  row,
2015
2016
  providerConfig,
2016
2017
  provider,
@@ -2887,13 +2888,13 @@ var init_fallback = __esm({
2887
2888
  function failureOf(attempt, home = (0, import_node_os2.homedir)()) {
2888
2889
  const error = chosen(attempt);
2889
2890
  const line = error === void 0 ? void 0 : firstLine(machineless(error.text, attempt.repoRoot, home));
2890
- const headline = line !== void 0 && jsonBytes(line) <= HEADLINE_LIMIT ? line : void 0;
2891
+ const headline2 = line !== void 0 && jsonBytes(line) <= HEADLINE_LIMIT ? line : void 0;
2891
2892
  const steps = chainOf(attempt.steps);
2892
2893
  const location = error === void 0 ? void 0 : fromRoot(placeOf(error, attempt.testFile), attempt.repoRoot);
2893
2894
  const artifacts = attempt.attachments.filter((a) => a.path !== void 0 && !a.name.startsWith("_") && a.contentType !== METADATA_TYPE).map(({ name, contentType }) => ({ name: fileNameOf(name), ...contentType ? { contentType } : {} })).filter((a) => a.name !== "");
2894
2895
  const ciUrl = webLink(attempt.buildHref);
2895
2896
  const failure = {
2896
- ...headline !== void 0 ? { headline } : {},
2897
+ ...headline2 !== void 0 ? { headline: headline2 } : {},
2897
2898
  ...steps.length > 0 ? { steps } : {},
2898
2899
  ...location !== void 0 ? { location } : {},
2899
2900
  ...artifacts.length > 0 ? { artifacts } : {},
@@ -6241,7 +6242,8 @@ function classify2(baseline, current) {
6241
6242
  if (baseline === "absent") return current === "passed" ? "new-pass" : "new-fail";
6242
6243
  if (current === "absent") return "removed";
6243
6244
  if (baseline === "passed") return current === "passed" ? "stable-pass" : "regression";
6244
- return current === "passed" ? "improvement" : "pre-existing-fail";
6245
+ if (current === "passed") return "improvement";
6246
+ return baseline === "errored" ? "unmeasured" : "pre-existing-fail";
6245
6247
  }
6246
6248
  function diffRuns(baseline, current) {
6247
6249
  const baseMap = new Map(baseline.evals.map((e) => [e.id, evalStatus(e)]));
@@ -6262,6 +6264,7 @@ function diffRuns(baseline, current) {
6262
6264
  stablePasses: count2("stable-pass"),
6263
6265
  errored: count2("errored"),
6264
6266
  removed: count2("removed"),
6267
+ unmeasured: count2("errored") + count2("unmeasured"),
6265
6268
  hasRegression: count2("regression") > 0
6266
6269
  };
6267
6270
  return { evals, summary };
@@ -6332,6 +6335,11 @@ __export(diff_exports2, {
6332
6335
  renderDiff: () => renderDiff,
6333
6336
  renderDiffMarkdown: () => renderDiffMarkdown
6334
6337
  });
6338
+ function headline(d) {
6339
+ if (d.summary.hasRegression) return { mark: "\u274C", text: `${d.summary.regressions} regression(s)` };
6340
+ if (d.summary.unmeasured > 0) return { mark: "\u26A0\uFE0F", text: `${d.summary.unmeasured} eval(s) not measured` };
6341
+ return null;
6342
+ }
6335
6343
  function summaryLine(d) {
6336
6344
  const s = d.summary;
6337
6345
  return [
@@ -6341,14 +6349,14 @@ function summaryLine(d) {
6341
6349
  `${s.preExistingFails} pre-existing-fail`,
6342
6350
  `${s.errored} errored`,
6343
6351
  `${s.removed} removed`,
6352
+ `${s.unmeasured} not measured`,
6344
6353
  `${s.stablePasses} stable`
6345
6354
  ].join(" \xB7 ");
6346
6355
  }
6347
6356
  function renderDiffMarkdown(d) {
6348
6357
  const out = [STICKY_MARKER, "## Plune eval diff", ""];
6349
- out.push(
6350
- d.summary.hasRegression ? `### \u274C ${d.summary.regressions} regression(s)` : "### \u2705 No regressions"
6351
- );
6358
+ const head = headline(d);
6359
+ out.push(head ? `### ${head.mark} ${head.text}` : "### \u2705 No regressions");
6352
6360
  out.push("");
6353
6361
  const notable = d.evals.filter((e) => e.status !== "stable-pass");
6354
6362
  if (notable.length > 0) {
@@ -6363,9 +6371,8 @@ function renderDiffMarkdown(d) {
6363
6371
  return out.join("\n");
6364
6372
  }
6365
6373
  function renderDiffConsole(d) {
6366
- const out = [
6367
- d.summary.hasRegression ? `Plune eval diff: ${d.summary.regressions} regression(s)` : "Plune eval diff: no regressions"
6368
- ];
6374
+ const head = headline(d);
6375
+ const out = [`Plune eval diff: ${head ? head.text : "no regressions"}`];
6369
6376
  for (const e of d.evals) {
6370
6377
  if (e.status === "stable-pass") continue;
6371
6378
  out.push(` ${e.id}: ${e.baseline} -> ${e.current} [${e.status}]`);
@@ -6395,6 +6402,7 @@ var init_diff3 = __esm({
6395
6402
  "new-pass": "\u{1F7E2} new pass",
6396
6403
  "stable-pass": "\u2705 stable",
6397
6404
  errored: "\u26A0\uFE0F errored",
6405
+ unmeasured: "\u26A0\uFE0F not measured (baseline errored)",
6398
6406
  removed: "\u2796 removed"
6399
6407
  };
6400
6408
  }
@@ -6690,7 +6698,7 @@ var init_logout = __esm({
6690
6698
  });
6691
6699
 
6692
6700
  // src/mcp-tools/schemas.ts
6693
- var import_zod4, BATCH_MAX2, isHttp, httpLink, id, dryRun, MISSION_STATES, MISSION_DOCUMENTATION, MISSION_LINK_KINDS, missionName, missionSummary, missionUrl, missionFields, sourceTitle, sourceVersion, linkSource, fileSource, ACCESS_KINDS, questionText, questionInput, STRATEGY_SOURCE_KINDS, TEST_LEVELS, QUALITY_CHARACTERISTICS, TEST_TECHNIQUES, RISK_SCALE, itemFields, strategyItem, items, unique, linkIds, share, strategyContent, SOURCE_REF_KINDS, proposedCase, API_METHODS, proposedEndpoint, REVIEW_STATES, REVIEW_KINDS, REJECTION_CATEGORIES, entity, plannedRequest, writeOutput, batchOutput, bulkPreview;
6701
+ var import_zod4, BATCH_MAX2, isHttp, httpLink, id, dryRun, MISSION_STATES, MISSION_DOCUMENTATION, MISSION_LINK_KINDS, missionName, missionSummary, missionUrl, missionFields, sourceTitle, sourceVersion, linkSource, fileSource, ACCESS_KINDS, questionText, questionInput, STRATEGY_SOURCE_KINDS, TEST_LEVELS, QUALITY_CHARACTERISTICS, TEST_TECHNIQUES, RISK_SCALE, itemFields, strategyItem, items, unique, linkIds, share, strategyContent, SOURCE_REF_KINDS, proposedCase, CASE_GENERATION_TITLE_MAX, CASE_GENERATION_MAX_CASES, CASE_GENERATION_DEFAULT_CASES, CASE_GENERATION_MIN_WORDS, CASE_GENERATION_TEXT_MAX, caseGenerationBody, API_METHODS, proposedEndpoint, REVIEW_STATES, REVIEW_KINDS, REJECTION_CATEGORIES, entity, plannedRequest, writeOutput, batchOutput, bulkPreview;
6694
6702
  var init_schemas = __esm({
6695
6703
  "src/mcp-tools/schemas.ts"() {
6696
6704
  "use strict";
@@ -6820,6 +6828,21 @@ var init_schemas = __esm({
6820
6828
  source: import_zod4.z.strictObject({ kind: import_zod4.z.enum(SOURCE_REF_KINDS), value: import_zod4.z.string().min(1).max(100) }).optional().describe("What you read to design it"),
6821
6829
  rationale: import_zod4.z.string().min(1).max(400).optional().describe("Why the case is worth having")
6822
6830
  });
6831
+ CASE_GENERATION_TITLE_MAX = 300;
6832
+ CASE_GENERATION_MAX_CASES = 12;
6833
+ CASE_GENERATION_DEFAULT_CASES = 8;
6834
+ CASE_GENERATION_MIN_WORDS = 20;
6835
+ CASE_GENERATION_TEXT_MAX = 12e3;
6836
+ caseGenerationBody = {
6837
+ input: import_zod4.z.strictObject({
6838
+ kind: import_zod4.z.literal("pasted-text"),
6839
+ text: import_zod4.z.string().describe(`The requirement, story or ticket text, at least ${CASE_GENERATION_MIN_WORDS} words and at most ${CASE_GENERATION_TEXT_MAX} characters`),
6840
+ title: import_zod4.z.string().trim().max(CASE_GENERATION_TITLE_MAX).optional()
6841
+ }),
6842
+ maxCases: import_zod4.z.number().int().min(1).max(CASE_GENERATION_MAX_CASES).optional().describe(`How many proposals at most; ${CASE_GENERATION_DEFAULT_CASES} when left out`),
6843
+ language: import_zod4.z.string().trim().min(2).max(35).optional().describe("The cases' language; the project's AI language when left out"),
6844
+ confirmDuplicates: import_zod4.z.boolean().optional().describe("true: go ahead although cases or proposals from this text exist (the platform answers 409 duplicate_source first)")
6845
+ };
6823
6846
  API_METHODS = ["GET", "PUT", "POST", "DELETE", "OPTIONS", "HEAD", "PATCH", "TRACE"];
6824
6847
  proposedEndpoint = import_zod4.z.strictObject({
6825
6848
  method: import_zod4.z.enum(API_METHODS),
@@ -6882,7 +6905,7 @@ function isPluneToolName(name) {
6882
6905
  function defaultVisibleTools(scope) {
6883
6906
  return PLUNE_TOOLS.filter((t) => t.visibility === "core" && !(scope === "read" && t.write)).map((t) => t.name);
6884
6907
  }
6885
- var import_zod5, MAX_VISIBLE_TOOLS, define, listPage, missionId, PLUNE_TOOLS, PLUNE_TOOL_NAMES, BY_NAME;
6908
+ var import_zod5, MAX_VISIBLE_TOOLS, define, listPage, missionId, projectId, PLUNE_TOOLS, PLUNE_TOOL_NAMES, BY_NAME;
6886
6909
  var init_contract = __esm({
6887
6910
  "src/mcp-tools/contract.ts"() {
6888
6911
  "use strict";
@@ -6896,6 +6919,7 @@ var init_contract = __esm({
6896
6919
  cursor: import_zod5.z.string().min(1).optional().describe("nextCursor of the previous page")
6897
6920
  };
6898
6921
  missionId = id("The mission id");
6922
+ projectId = import_zod5.z.string().min(1).optional().describe("The project; the one the server was started with when left out");
6899
6923
  PLUNE_TOOLS = [
6900
6924
  define({
6901
6925
  name: "plune_project_context",
@@ -6991,6 +7015,26 @@ var init_contract = __esm({
6991
7015
  visibility: "core",
6992
7016
  endpoints: ["POST /v1/ingest/cairn"]
6993
7017
  }),
7018
+ define({
7019
+ name: "plune_cases_generate",
7020
+ title: "Generate cases from pasted text",
7021
+ description: "Generate test case proposals from pasted text \u2014 a requirement, a user story, the text of a ticket \u2014 with the project's AI. It spends the project's AI budget. The proposals land in the review queue and nothing becomes a case until a person approves it. The run is queued: keep its id and poll it with plune_cases_generation_get until it is done. A Jira ticket by its key is not taken here: paste its text.",
7022
+ input: import_zod5.z.strictObject({ projectId, ...caseGenerationBody, dry_run: dryRun }),
7023
+ output: writeOutput,
7024
+ write: true,
7025
+ visibility: "core",
7026
+ endpoints: ["POST /v1/projects/:id/case-generations"]
7027
+ }),
7028
+ define({
7029
+ name: "plune_cases_generation_get",
7030
+ title: "Read a case generation",
7031
+ description: "Check where a run of cases from pasted text stands \u2014 queued, running, done, failed or cancelled \u2014 with how many proposals it made, how many it skipped, and what it cost.",
7032
+ input: import_zod5.z.strictObject({ projectId, generationId: id("The id the start answered with") }),
7033
+ output: import_zod5.z.object({ generation: entity }),
7034
+ write: false,
7035
+ visibility: "core",
7036
+ endpoints: ["GET /v1/projects/:id/case-generations/:generationId"]
7037
+ }),
6994
7038
  define({
6995
7039
  name: "plune_mission_create",
6996
7040
  title: "Create a mission",
@@ -7356,10 +7400,10 @@ function createPluneToolExecutor(opts) {
7356
7400
  ]);
7357
7401
  const projectList = asArray(asRecord(projects.body)["projects"]);
7358
7402
  const only = projectList.length === 1 ? asRecord(projectList[0])["id"] : void 0;
7359
- const projectId = opts.projectId ?? (typeof only === "string" ? only : null);
7403
+ const projectId2 = opts.projectId ?? (typeof only === "string" ? only : null);
7360
7404
  const total = asRecord(cases.body)["total"];
7361
7405
  return {
7362
- projectId,
7406
+ projectId: projectId2,
7363
7407
  projects: projectList,
7364
7408
  suites: asArray(asRecord(suites.body)["suites"]),
7365
7409
  cases: { total: typeof total === "number" ? total : 0 },
@@ -7428,6 +7472,18 @@ function createPluneToolExecutor(opts) {
7428
7472
  const proposed = summary["proposed"];
7429
7473
  return { dryRun: false, requested: cases.length, applied: typeof proposed === "number" ? proposed : 0, report: summary };
7430
7474
  },
7475
+ async plune_cases_generate(i) {
7476
+ const { dry_run, projectId: projectId2, ...body } = i;
7477
+ const project = projectId2 ?? opts.projectId;
7478
+ if (project === void 0) return fail("invalid_input", NO_PROJECT);
7479
+ return write({ method: "POST", path: path19("/v1/projects/:id/case-generations", project), body: withoutUndefined(body) }, dry_run);
7480
+ },
7481
+ async plune_cases_generation_get(i) {
7482
+ const project = i["projectId"] ?? opts.projectId;
7483
+ if (project === void 0) return fail("invalid_input", NO_PROJECT);
7484
+ const res = await call({ method: "GET", path: path19("/v1/projects/:id/case-generations/:generationId", project, str2(i["generationId"])) });
7485
+ return { generation: asRecord(res.body) };
7486
+ },
7431
7487
  async plune_mission_create(i) {
7432
7488
  const { dry_run, ...body } = i;
7433
7489
  return write({ method: "POST", path: "/v1/missions", body: withoutUndefined(body) }, dry_run);
@@ -7571,7 +7627,7 @@ function createPluneToolExecutor(opts) {
7571
7627
  }
7572
7628
  };
7573
7629
  }
7574
- var import_node_crypto5, CONFIRM_TTL_MS, READ_ONLY_REFUSAL, readScopeMessage, fail, zodMessage, path19, withoutUndefined, asRecord, asArray, cursorOf, rawWrites;
7630
+ var import_node_crypto5, CONFIRM_TTL_MS, READ_ONLY_REFUSAL, NO_PROJECT, readScopeMessage, fail, zodMessage, path19, withoutUndefined, asRecord, asArray, cursorOf, rawWrites;
7575
7631
  var init_execute = __esm({
7576
7632
  "src/mcp-tools/execute.ts"() {
7577
7633
  "use strict";
@@ -7582,6 +7638,7 @@ var init_execute = __esm({
7582
7638
  init_schemas();
7583
7639
  CONFIRM_TTL_MS = 5 * 60 * 1e3;
7584
7640
  READ_ONLY_REFUSAL = "read-only token";
7641
+ NO_PROJECT = "projectId is needed: the server was started without a project (--project or PLUNE_PROJECT). plune_project_context lists the projects.";
7585
7642
  readScopeMessage = (what) => `This token has the read scope, so ${what}. Use a token with the write scope (Plune \u2192 Account \u2192 API tokens) and restart the server.`;
7586
7643
  fail = (code, message, extra = {}) => ({
7587
7644
  ok: false,
@@ -7696,13 +7753,13 @@ async function handleMcpServe(flags, deps) {
7696
7753
  const token = (deps.loadToken ?? loadToken)();
7697
7754
  if (token === null) throw new NotLoggedInError();
7698
7755
  const apiUrl = resolveApiUrl(deps.apiUrl);
7699
- const projectId = flags.project?.trim() || process.env["PLUNE_PROJECT"]?.trim() || void 0;
7756
+ const projectId2 = flags.project?.trim() || process.env["PLUNE_PROJECT"]?.trim() || void 0;
7700
7757
  const serve = deps.serve ?? (await Promise.resolve().then(() => (init_server(), server_exports))).serveStdio;
7701
7758
  await serve({
7702
7759
  token,
7703
7760
  apiUrl,
7704
7761
  version: deps.version,
7705
- ...projectId !== void 0 ? { projectId } : {},
7762
+ ...projectId2 !== void 0 ? { projectId: projectId2 } : {},
7706
7763
  ...flags.readOnly === true ? { scope: "read" } : {}
7707
7764
  });
7708
7765
  }
@@ -8127,7 +8184,7 @@ function readMarkdown(cwd, file) {
8127
8184
  if (text2.trim() === "") usage(`${target} is empty.`);
8128
8185
  return text2;
8129
8186
  }
8130
- function refusal(out, apiUrl, projectId) {
8187
+ function refusal(out, apiUrl, projectId2) {
8131
8188
  const d = out.detail;
8132
8189
  const said = d === "" ? "" : ` (${d})`;
8133
8190
  switch (out.status) {
@@ -8138,7 +8195,7 @@ function refusal(out, apiUrl, projectId) {
8138
8195
  case 403:
8139
8196
  return new RequirementsImportError(`The platform refused: ${d || "read-only member"}. Importing needs an owner or a member of the project.`, 1);
8140
8197
  case 404:
8141
- return /^project '.*' not found$/.test(d) ? new RequirementsImportError(`Project ${projectId} was not found for this token. A token only works in the project it was minted in; pass --project <id> or use a token from that project.`, 1) : new RequirementsImportError(SUPPORT, 1);
8198
+ return /^project '.*' not found$/.test(d) ? new RequirementsImportError(`Project ${projectId2} was not found for this token. A token only works in the project it was minted in; pass --project <id> or use a token from that project.`, 1) : new RequirementsImportError(SUPPORT, 1);
8142
8199
  case 409:
8143
8200
  return new RequirementsImportError(`A requirement key was taken by another change while the import ran. Nothing was written; run it again${said}.`, 1);
8144
8201
  case 413:
@@ -8168,15 +8225,15 @@ async function handleRequirementsImport(deps) {
8168
8225
  const token = (deps.loadToken ?? loadToken)();
8169
8226
  if (token === null) throw new RequirementsImportError('Not logged in. Run "plune login" first, or set PLUNE_TOKEN.', 2);
8170
8227
  const client = createClient({ apiUrl, token, ...deps.fetchImpl !== void 0 ? { fetchImpl: deps.fetchImpl } : {} });
8171
- const projectId = await resolveProject(client, deps.project, apiUrl);
8172
- const base = `/v1/projects/${encodeURIComponent(projectId)}/requirements/import`;
8228
+ const projectId2 = await resolveProject(client, deps.project, apiUrl);
8229
+ const base = `/v1/projects/${encodeURIComponent(projectId2)}/requirements/import`;
8173
8230
  if (mapping === null) {
8174
8231
  const out2 = await client.post(`${base}/discover`, { markdown });
8175
- if (!out2.ok) throw refusal(out2, apiUrl, projectId);
8232
+ if (!out2.ok) throw refusal(out2, apiUrl, projectId2);
8176
8233
  return { step: "discover", result: out2.body };
8177
8234
  }
8178
8235
  const out = await client.post(base, { markdown, dryRun: deps.yes !== true, ...mapping });
8179
- if (!out.ok) throw refusal(out, apiUrl, projectId);
8236
+ if (!out.ok) throw refusal(out, apiUrl, projectId2);
8180
8237
  return { step: "import", report: out.body };
8181
8238
  }
8182
8239
  function formatDiscover(file, r) {
@@ -8332,15 +8389,15 @@ async function handleRequirementsLinks(deps) {
8332
8389
  const token = (deps.loadToken ?? loadToken)();
8333
8390
  if (token === null) throw new RequirementsImportError('Not logged in. Run "plune login" first, or set PLUNE_TOKEN.', 2);
8334
8391
  const client = createClient({ apiUrl, token, ...deps.fetchImpl !== void 0 ? { fetchImpl: deps.fetchImpl } : {} });
8335
- const projectId = await resolveProject(client, deps.project, apiUrl);
8336
- const base = `/v1/projects/${encodeURIComponent(projectId)}/requirements/import`;
8392
+ const projectId2 = await resolveProject(client, deps.project, apiUrl);
8393
+ const base = `/v1/projects/${encodeURIComponent(projectId2)}/requirements/import`;
8337
8394
  if (mapping === null) {
8338
8395
  const out2 = await client.post(`${base}/discover`, { markdown });
8339
- if (!out2.ok) throw refusal(out2, apiUrl, projectId);
8396
+ if (!out2.ok) throw refusal(out2, apiUrl, projectId2);
8340
8397
  return { step: "discover", result: out2.body };
8341
8398
  }
8342
8399
  const out = await client.post(`${base}/links`, { markdown, dryRun: deps.yes !== true, ...mapping });
8343
- if (!out.ok) throw refusal(out, apiUrl, projectId);
8400
+ if (!out.ok) throw refusal(out, apiUrl, projectId2);
8344
8401
  return { step: "links", report: out.body };
8345
8402
  }
8346
8403
  function formatLinksDiscover(file, r) {
@@ -9014,7 +9071,11 @@ ${missed.map((m) => ` - ${m}`).join("\n")}
9014
9071
  failUnexpected(err, globals.verbose === true);
9015
9072
  }
9016
9073
  });
9017
- program.command("diff").description("Compare two `plune run --format json` outputs and report regressions").argument("<baseline>", "Path to the baseline run JSON (e.g. from main)").argument("<current>", "Path to the current run JSON (e.g. from the PR)").option("--format <fmt>", "console | json | markdown", "console").option("--fail-on-regression", "Exit 1 when a pass\u2192fail regression is detected", false).option("-o, --output <path>", "Write the diff to a file instead of stdout").action(
9074
+ program.command("diff").description("Compare two `plune run --format json` outputs and report regressions").argument("<baseline>", "Path to the baseline run JSON (e.g. from main)").argument("<current>", "Path to the current run JSON (e.g. from the PR)").option("--format <fmt>", "console | json | markdown", "console").option(
9075
+ "--fail-on-regression",
9076
+ "Exit 1 when a pass\u2192fail regression is detected, 3 when none is but an eval was not measured",
9077
+ false
9078
+ ).option("-o, --output <path>", "Write the diff to a file instead of stdout").action(
9018
9079
  async (baseline, current, options, command) => {
9019
9080
  const verbose = command.optsWithGlobals().verbose === true;
9020
9081
  const [{ handleDiff: handleDiff2, DiffInputError: DiffInputError2 }, { renderDiff: renderDiff2 }] = await Promise.all([
@@ -9024,7 +9085,9 @@ ${missed.map((m) => ` - ${m}`).join("\n")}
9024
9085
  try {
9025
9086
  const diff = handleDiff2({ baselinePath: baseline, currentPath: current });
9026
9087
  writeReport(renderDiff2(diff, toFormat(options.format)), options.output);
9027
- process.exitCode = options.failOnRegression && diff.summary.hasRegression ? 1 : 0;
9088
+ process.exitCode = 0;
9089
+ if (options.failOnRegression && diff.summary.hasRegression) process.exitCode = 1;
9090
+ else if (options.failOnRegression && diff.summary.unmeasured > 0) process.exitCode = 3;
9028
9091
  } catch (err) {
9029
9092
  if (err instanceof DiffInputError2) {
9030
9093
  process.stderr.write(err.message + "\n");
package/dist/index.cjs CHANGED
@@ -1647,7 +1647,8 @@ async function runOrchestration(config, options, deps) {
1647
1647
  limit,
1648
1648
  (row) => runRow({
1649
1649
  template,
1650
- assertions: ev.assertions,
1650
+ // `defaults.assertions` apply to every eval, ahead of its own (#2415).
1651
+ assertions: [...config.defaults?.assertions ?? [], ...ev.assertions],
1651
1652
  row,
1652
1653
  providerConfig,
1653
1654
  provider,
package/dist/index.js CHANGED
@@ -1587,7 +1587,8 @@ async function runOrchestration(config, options, deps) {
1587
1587
  limit,
1588
1588
  (row) => runRow({
1589
1589
  template,
1590
- assertions: ev.assertions,
1590
+ // `defaults.assertions` apply to every eval, ahead of its own (#2415).
1591
+ assertions: [...config.defaults?.assertions ?? [], ...ev.assertions],
1591
1592
  row,
1592
1593
  providerConfig,
1593
1594
  provider,
@@ -163,6 +163,21 @@ var proposedCase = import_zod.z.strictObject({
163
163
  source: import_zod.z.strictObject({ kind: import_zod.z.enum(SOURCE_REF_KINDS), value: import_zod.z.string().min(1).max(100) }).optional().describe("What you read to design it"),
164
164
  rationale: import_zod.z.string().min(1).max(400).optional().describe("Why the case is worth having")
165
165
  });
166
+ var CASE_GENERATION_TITLE_MAX = 300;
167
+ var CASE_GENERATION_MAX_CASES = 12;
168
+ var CASE_GENERATION_DEFAULT_CASES = 8;
169
+ var CASE_GENERATION_MIN_WORDS = 20;
170
+ var CASE_GENERATION_TEXT_MAX = 12e3;
171
+ var caseGenerationBody = {
172
+ input: import_zod.z.strictObject({
173
+ kind: import_zod.z.literal("pasted-text"),
174
+ text: import_zod.z.string().describe(`The requirement, story or ticket text, at least ${CASE_GENERATION_MIN_WORDS} words and at most ${CASE_GENERATION_TEXT_MAX} characters`),
175
+ title: import_zod.z.string().trim().max(CASE_GENERATION_TITLE_MAX).optional()
176
+ }),
177
+ maxCases: import_zod.z.number().int().min(1).max(CASE_GENERATION_MAX_CASES).optional().describe(`How many proposals at most; ${CASE_GENERATION_DEFAULT_CASES} when left out`),
178
+ language: import_zod.z.string().trim().min(2).max(35).optional().describe("The cases' language; the project's AI language when left out"),
179
+ confirmDuplicates: import_zod.z.boolean().optional().describe("true: go ahead although cases or proposals from this text exist (the platform answers 409 duplicate_source first)")
180
+ };
166
181
  var API_METHODS = ["GET", "PUT", "POST", "DELETE", "OPTIONS", "HEAD", "PATCH", "TRACE"];
167
182
  var proposedEndpoint = import_zod.z.strictObject({
168
183
  method: import_zod.z.enum(API_METHODS),
@@ -219,6 +234,7 @@ var listPage = {
219
234
  cursor: import_zod2.z.string().min(1).optional().describe("nextCursor of the previous page")
220
235
  };
221
236
  var missionId = id("The mission id");
237
+ var projectId = import_zod2.z.string().min(1).optional().describe("The project; the one the server was started with when left out");
222
238
  var PLUNE_TOOLS = [
223
239
  define({
224
240
  name: "plune_project_context",
@@ -314,6 +330,26 @@ var PLUNE_TOOLS = [
314
330
  visibility: "core",
315
331
  endpoints: ["POST /v1/ingest/cairn"]
316
332
  }),
333
+ define({
334
+ name: "plune_cases_generate",
335
+ title: "Generate cases from pasted text",
336
+ description: "Generate test case proposals from pasted text \u2014 a requirement, a user story, the text of a ticket \u2014 with the project's AI. It spends the project's AI budget. The proposals land in the review queue and nothing becomes a case until a person approves it. The run is queued: keep its id and poll it with plune_cases_generation_get until it is done. A Jira ticket by its key is not taken here: paste its text.",
337
+ input: import_zod2.z.strictObject({ projectId, ...caseGenerationBody, dry_run: dryRun }),
338
+ output: writeOutput,
339
+ write: true,
340
+ visibility: "core",
341
+ endpoints: ["POST /v1/projects/:id/case-generations"]
342
+ }),
343
+ define({
344
+ name: "plune_cases_generation_get",
345
+ title: "Read a case generation",
346
+ description: "Check where a run of cases from pasted text stands \u2014 queued, running, done, failed or cancelled \u2014 with how many proposals it made, how many it skipped, and what it cost.",
347
+ input: import_zod2.z.strictObject({ projectId, generationId: id("The id the start answered with") }),
348
+ output: import_zod2.z.object({ generation: entity }),
349
+ write: false,
350
+ visibility: "core",
351
+ endpoints: ["GET /v1/projects/:id/case-generations/:generationId"]
352
+ }),
317
353
  define({
318
354
  name: "plune_mission_create",
319
355
  title: "Create a mission",
@@ -607,6 +643,7 @@ function createPluneApi(opts) {
607
643
  // src/mcp-tools/execute.ts
608
644
  var CONFIRM_TTL_MS = 5 * 60 * 1e3;
609
645
  var READ_ONLY_REFUSAL = "read-only token";
646
+ var NO_PROJECT = "projectId is needed: the server was started without a project (--project or PLUNE_PROJECT). plune_project_context lists the projects.";
610
647
  var readScopeMessage = (what) => `This token has the read scope, so ${what}. Use a token with the write scope (Plune \u2192 Account \u2192 API tokens) and restart the server.`;
611
648
  var fail = (code, message, extra = {}) => ({
612
649
  ok: false,
@@ -701,10 +738,10 @@ function createPluneToolExecutor(opts) {
701
738
  ]);
702
739
  const projectList = asArray(asRecord(projects.body)["projects"]);
703
740
  const only = projectList.length === 1 ? asRecord(projectList[0])["id"] : void 0;
704
- const projectId = opts.projectId ?? (typeof only === "string" ? only : null);
741
+ const projectId2 = opts.projectId ?? (typeof only === "string" ? only : null);
705
742
  const total = asRecord(cases.body)["total"];
706
743
  return {
707
- projectId,
744
+ projectId: projectId2,
708
745
  projects: projectList,
709
746
  suites: asArray(asRecord(suites.body)["suites"]),
710
747
  cases: { total: typeof total === "number" ? total : 0 },
@@ -773,6 +810,18 @@ function createPluneToolExecutor(opts) {
773
810
  const proposed = summary["proposed"];
774
811
  return { dryRun: false, requested: cases.length, applied: typeof proposed === "number" ? proposed : 0, report: summary };
775
812
  },
813
+ async plune_cases_generate(i) {
814
+ const { dry_run, projectId: projectId2, ...body } = i;
815
+ const project = projectId2 ?? opts.projectId;
816
+ if (project === void 0) return fail("invalid_input", NO_PROJECT);
817
+ return write({ method: "POST", path: path("/v1/projects/:id/case-generations", project), body: withoutUndefined(body) }, dry_run);
818
+ },
819
+ async plune_cases_generation_get(i) {
820
+ const project = i["projectId"] ?? opts.projectId;
821
+ if (project === void 0) return fail("invalid_input", NO_PROJECT);
822
+ const res = await call({ method: "GET", path: path("/v1/projects/:id/case-generations/:generationId", project, str(i["generationId"])) });
823
+ return { generation: asRecord(res.body) };
824
+ },
776
825
  async plune_mission_create(i) {
777
826
  const { dry_run, ...body } = i;
778
827
  return write({ method: "POST", path: "/v1/missions", body: withoutUndefined(body) }, dry_run);
@@ -208,6 +208,53 @@ declare const PLUNE_TOOLS: readonly [{
208
208
  write: true;
209
209
  visibility: "core";
210
210
  endpoints: string[];
211
+ }, {
212
+ name: string;
213
+ title: string;
214
+ description: string;
215
+ input: z.ZodObject<{
216
+ dry_run: z.ZodOptional<z.ZodBoolean>;
217
+ input: z.ZodObject<{
218
+ kind: z.ZodLiteral<"pasted-text">;
219
+ text: z.ZodString;
220
+ title: z.ZodOptional<z.ZodString>;
221
+ }, z.core.$strict>;
222
+ maxCases: z.ZodOptional<z.ZodNumber>;
223
+ language: z.ZodOptional<z.ZodString>;
224
+ confirmDuplicates: z.ZodOptional<z.ZodBoolean>;
225
+ projectId: z.ZodOptional<z.ZodString>;
226
+ }, z.core.$strict>;
227
+ output: z.ZodObject<{
228
+ dryRun: z.ZodBoolean;
229
+ request: z.ZodOptional<z.ZodObject<{
230
+ method: z.ZodString;
231
+ path: z.ZodString;
232
+ query: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
233
+ body: z.ZodOptional<z.ZodUnknown>;
234
+ }, z.core.$strip>>;
235
+ result: z.ZodOptional<z.ZodObject<{
236
+ id: z.ZodOptional<z.ZodString>;
237
+ }, z.core.$loose>>;
238
+ }, z.core.$strip>;
239
+ write: true;
240
+ visibility: "core";
241
+ endpoints: string[];
242
+ }, {
243
+ name: string;
244
+ title: string;
245
+ description: string;
246
+ input: z.ZodObject<{
247
+ projectId: z.ZodOptional<z.ZodString>;
248
+ generationId: z.ZodString;
249
+ }, z.core.$strict>;
250
+ output: z.ZodObject<{
251
+ generation: z.ZodObject<{
252
+ id: z.ZodOptional<z.ZodString>;
253
+ }, z.core.$loose>;
254
+ }, z.core.$strip>;
255
+ write: false;
256
+ visibility: "core";
257
+ endpoints: string[];
211
258
  }, {
212
259
  name: string;
213
260
  title: string;
@@ -208,6 +208,53 @@ declare const PLUNE_TOOLS: readonly [{
208
208
  write: true;
209
209
  visibility: "core";
210
210
  endpoints: string[];
211
+ }, {
212
+ name: string;
213
+ title: string;
214
+ description: string;
215
+ input: z.ZodObject<{
216
+ dry_run: z.ZodOptional<z.ZodBoolean>;
217
+ input: z.ZodObject<{
218
+ kind: z.ZodLiteral<"pasted-text">;
219
+ text: z.ZodString;
220
+ title: z.ZodOptional<z.ZodString>;
221
+ }, z.core.$strict>;
222
+ maxCases: z.ZodOptional<z.ZodNumber>;
223
+ language: z.ZodOptional<z.ZodString>;
224
+ confirmDuplicates: z.ZodOptional<z.ZodBoolean>;
225
+ projectId: z.ZodOptional<z.ZodString>;
226
+ }, z.core.$strict>;
227
+ output: z.ZodObject<{
228
+ dryRun: z.ZodBoolean;
229
+ request: z.ZodOptional<z.ZodObject<{
230
+ method: z.ZodString;
231
+ path: z.ZodString;
232
+ query: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
233
+ body: z.ZodOptional<z.ZodUnknown>;
234
+ }, z.core.$strip>>;
235
+ result: z.ZodOptional<z.ZodObject<{
236
+ id: z.ZodOptional<z.ZodString>;
237
+ }, z.core.$loose>>;
238
+ }, z.core.$strip>;
239
+ write: true;
240
+ visibility: "core";
241
+ endpoints: string[];
242
+ }, {
243
+ name: string;
244
+ title: string;
245
+ description: string;
246
+ input: z.ZodObject<{
247
+ projectId: z.ZodOptional<z.ZodString>;
248
+ generationId: z.ZodString;
249
+ }, z.core.$strict>;
250
+ output: z.ZodObject<{
251
+ generation: z.ZodObject<{
252
+ id: z.ZodOptional<z.ZodString>;
253
+ }, z.core.$loose>;
254
+ }, z.core.$strip>;
255
+ write: false;
256
+ visibility: "core";
257
+ endpoints: string[];
211
258
  }, {
212
259
  name: string;
213
260
  title: string;
package/dist/mcp-tools.js CHANGED
@@ -127,6 +127,21 @@ var proposedCase = z.strictObject({
127
127
  source: z.strictObject({ kind: z.enum(SOURCE_REF_KINDS), value: z.string().min(1).max(100) }).optional().describe("What you read to design it"),
128
128
  rationale: z.string().min(1).max(400).optional().describe("Why the case is worth having")
129
129
  });
130
+ var CASE_GENERATION_TITLE_MAX = 300;
131
+ var CASE_GENERATION_MAX_CASES = 12;
132
+ var CASE_GENERATION_DEFAULT_CASES = 8;
133
+ var CASE_GENERATION_MIN_WORDS = 20;
134
+ var CASE_GENERATION_TEXT_MAX = 12e3;
135
+ var caseGenerationBody = {
136
+ input: z.strictObject({
137
+ kind: z.literal("pasted-text"),
138
+ text: z.string().describe(`The requirement, story or ticket text, at least ${CASE_GENERATION_MIN_WORDS} words and at most ${CASE_GENERATION_TEXT_MAX} characters`),
139
+ title: z.string().trim().max(CASE_GENERATION_TITLE_MAX).optional()
140
+ }),
141
+ maxCases: z.number().int().min(1).max(CASE_GENERATION_MAX_CASES).optional().describe(`How many proposals at most; ${CASE_GENERATION_DEFAULT_CASES} when left out`),
142
+ language: z.string().trim().min(2).max(35).optional().describe("The cases' language; the project's AI language when left out"),
143
+ confirmDuplicates: z.boolean().optional().describe("true: go ahead although cases or proposals from this text exist (the platform answers 409 duplicate_source first)")
144
+ };
130
145
  var API_METHODS = ["GET", "PUT", "POST", "DELETE", "OPTIONS", "HEAD", "PATCH", "TRACE"];
131
146
  var proposedEndpoint = z.strictObject({
132
147
  method: z.enum(API_METHODS),
@@ -183,6 +198,7 @@ var listPage = {
183
198
  cursor: z2.string().min(1).optional().describe("nextCursor of the previous page")
184
199
  };
185
200
  var missionId = id("The mission id");
201
+ var projectId = z2.string().min(1).optional().describe("The project; the one the server was started with when left out");
186
202
  var PLUNE_TOOLS = [
187
203
  define({
188
204
  name: "plune_project_context",
@@ -278,6 +294,26 @@ var PLUNE_TOOLS = [
278
294
  visibility: "core",
279
295
  endpoints: ["POST /v1/ingest/cairn"]
280
296
  }),
297
+ define({
298
+ name: "plune_cases_generate",
299
+ title: "Generate cases from pasted text",
300
+ description: "Generate test case proposals from pasted text \u2014 a requirement, a user story, the text of a ticket \u2014 with the project's AI. It spends the project's AI budget. The proposals land in the review queue and nothing becomes a case until a person approves it. The run is queued: keep its id and poll it with plune_cases_generation_get until it is done. A Jira ticket by its key is not taken here: paste its text.",
301
+ input: z2.strictObject({ projectId, ...caseGenerationBody, dry_run: dryRun }),
302
+ output: writeOutput,
303
+ write: true,
304
+ visibility: "core",
305
+ endpoints: ["POST /v1/projects/:id/case-generations"]
306
+ }),
307
+ define({
308
+ name: "plune_cases_generation_get",
309
+ title: "Read a case generation",
310
+ description: "Check where a run of cases from pasted text stands \u2014 queued, running, done, failed or cancelled \u2014 with how many proposals it made, how many it skipped, and what it cost.",
311
+ input: z2.strictObject({ projectId, generationId: id("The id the start answered with") }),
312
+ output: z2.object({ generation: entity }),
313
+ write: false,
314
+ visibility: "core",
315
+ endpoints: ["GET /v1/projects/:id/case-generations/:generationId"]
316
+ }),
281
317
  define({
282
318
  name: "plune_mission_create",
283
319
  title: "Create a mission",
@@ -571,6 +607,7 @@ function createPluneApi(opts) {
571
607
  // src/mcp-tools/execute.ts
572
608
  var CONFIRM_TTL_MS = 5 * 60 * 1e3;
573
609
  var READ_ONLY_REFUSAL = "read-only token";
610
+ var NO_PROJECT = "projectId is needed: the server was started without a project (--project or PLUNE_PROJECT). plune_project_context lists the projects.";
574
611
  var readScopeMessage = (what) => `This token has the read scope, so ${what}. Use a token with the write scope (Plune \u2192 Account \u2192 API tokens) and restart the server.`;
575
612
  var fail = (code, message, extra = {}) => ({
576
613
  ok: false,
@@ -665,10 +702,10 @@ function createPluneToolExecutor(opts) {
665
702
  ]);
666
703
  const projectList = asArray(asRecord(projects.body)["projects"]);
667
704
  const only = projectList.length === 1 ? asRecord(projectList[0])["id"] : void 0;
668
- const projectId = opts.projectId ?? (typeof only === "string" ? only : null);
705
+ const projectId2 = opts.projectId ?? (typeof only === "string" ? only : null);
669
706
  const total = asRecord(cases.body)["total"];
670
707
  return {
671
- projectId,
708
+ projectId: projectId2,
672
709
  projects: projectList,
673
710
  suites: asArray(asRecord(suites.body)["suites"]),
674
711
  cases: { total: typeof total === "number" ? total : 0 },
@@ -737,6 +774,18 @@ function createPluneToolExecutor(opts) {
737
774
  const proposed = summary["proposed"];
738
775
  return { dryRun: false, requested: cases.length, applied: typeof proposed === "number" ? proposed : 0, report: summary };
739
776
  },
777
+ async plune_cases_generate(i) {
778
+ const { dry_run, projectId: projectId2, ...body } = i;
779
+ const project = projectId2 ?? opts.projectId;
780
+ if (project === void 0) return fail("invalid_input", NO_PROJECT);
781
+ return write({ method: "POST", path: path("/v1/projects/:id/case-generations", project), body: withoutUndefined(body) }, dry_run);
782
+ },
783
+ async plune_cases_generation_get(i) {
784
+ const project = i["projectId"] ?? opts.projectId;
785
+ if (project === void 0) return fail("invalid_input", NO_PROJECT);
786
+ const res = await call({ method: "GET", path: path("/v1/projects/:id/case-generations/:generationId", project, str(i["generationId"])) });
787
+ return { generation: asRecord(res.body) };
788
+ },
740
789
  async plune_mission_create(i) {
741
790
  const { dry_run, ...body } = i;
742
791
  return write({ method: "POST", path: "/v1/missions", body: withoutUndefined(body) }, dry_run);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@plune-ai/cli",
3
- "version": "0.34.0",
3
+ "version": "0.34.2",
4
4
  "description": "AI-powered assertion testing for LLM apps — write assertions, run them against any provider, get a pass/fail report (local, CI, or diff).",
5
5
  "keywords": [
6
6
  "ai",