bamboo-agents 0.1.7-dev.87.b63312a → 0.1.7-dev.88.66a1ec6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/VERSION CHANGED
@@ -1 +1 @@
1
- 0.1.7-dev.87.b63312a
1
+ 0.1.7-dev.88.66a1ec6
package/dist/index.js CHANGED
@@ -3037,7 +3037,7 @@ var VERSION;
3037
3037
  var init_version = __esm({
3038
3038
  "src/version.ts"() {
3039
3039
  "use strict";
3040
- VERSION = "0.1.7-dev.87.b63312a";
3040
+ VERSION = "0.1.7-dev.88.66a1ec6";
3041
3041
  }
3042
3042
  });
3043
3043
 
@@ -13035,6 +13035,46 @@ var init_episodes2 = __esm({
13035
13035
  }
13036
13036
  });
13037
13037
 
13038
+ // ../../src/lib/api-contracts/cli/eval-baseline-promote.ts
13039
+ var Reason, PrUrl, EvalBaselinePromoteRunSchema, EvalBaselineDemoteSchema, EvalBaselinePromoteRequestSchema, EvalBaselinePromoteResponseSchema;
13040
+ var init_eval_baseline_promote = __esm({
13041
+ "../../src/lib/api-contracts/cli/eval-baseline-promote.ts"() {
13042
+ init_zod();
13043
+ Reason = external_exports.string().trim().min(1).max(2e3).describe("Why this run is the reference now. Recorded on the row.");
13044
+ PrUrl = external_exports.string().url().optional().describe("The PR this decision was made on, recorded as the row's source_url.");
13045
+ EvalBaselinePromoteRunSchema = external_exports.object({
13046
+ run_id: external_exports.string().uuid().describe("A completed qrels run in the caller's org \u2014 the id the correctness gate printed."),
13047
+ reason: Reason,
13048
+ pr_url: PrUrl,
13049
+ // A body that says `demote: true` but fails the demote shape must not fall
13050
+ // through to a promotion: objects strip unknown keys, so name it here.
13051
+ demote: external_exports.literal(false).optional()
13052
+ });
13053
+ EvalBaselineDemoteSchema = external_exports.object({
13054
+ demote: external_exports.literal(true),
13055
+ brand_id: external_exports.string().uuid(),
13056
+ qrels_version: external_exports.string().min(1).describe("The key to empty: its current baseline is tombstoned."),
13057
+ reason: Reason,
13058
+ pr_url: PrUrl
13059
+ });
13060
+ EvalBaselinePromoteRequestSchema = external_exports.union([
13061
+ EvalBaselineDemoteSchema,
13062
+ EvalBaselinePromoteRunSchema
13063
+ ]);
13064
+ EvalBaselinePromoteResponseSchema = external_exports.object({
13065
+ promotion_id: external_exports.string().uuid(),
13066
+ kind: external_exports.enum(["promote", "demote"]),
13067
+ brand_id: external_exports.string().uuid(),
13068
+ qrels_version: external_exports.string(),
13069
+ /** Null after a demotion. */
13070
+ run_id: external_exports.string().uuid().nullable(),
13071
+ /** What the head pointed at before this row; null when the version had no baseline. */
13072
+ previous_run_id: external_exports.string().uuid().nullable(),
13073
+ promoted_at: external_exports.string()
13074
+ });
13075
+ }
13076
+ });
13077
+
13038
13078
  // ../../src/lib/api-contracts/eval-roadmap/evidence-kinds.ts
13039
13079
  var EvidenceKinds, EvidenceKindSchema;
13040
13080
  var init_evidence_kinds = __esm({
@@ -13596,6 +13636,19 @@ async function startAbTest(bambooSession, body) {
13596
13636
  const json = await res.json();
13597
13637
  return EvalAbResponseSchema.parse(json.data);
13598
13638
  }
13639
+ async function promoteBaseline(bambooSession, body) {
13640
+ const res = await cliApiFetch(`${apiBase()}/api/v1/eval/baseline-promotions`, {
13641
+ method: "POST",
13642
+ headers: {
13643
+ Authorization: `Bearer ${bambooSession}`,
13644
+ "Content-Type": "application/json"
13645
+ },
13646
+ body: JSON.stringify(body)
13647
+ });
13648
+ if (!res.ok) throw await parseError(res);
13649
+ const json = await res.json();
13650
+ return EvalBaselinePromoteResponseSchema.parse(json.data);
13651
+ }
13599
13652
  async function createEvalTestSet(bambooSession, body) {
13600
13653
  const res = await cliApiFetch(`${apiBase()}/api/v1/eval/test-sets`, {
13601
13654
  method: "POST",
@@ -13674,6 +13727,7 @@ var init_api = __esm({
13674
13727
  init_usage();
13675
13728
  init_episodes2();
13676
13729
  init_eval_ab();
13730
+ init_eval_baseline_promote();
13677
13731
  init_eval_test_sets();
13678
13732
  init_version();
13679
13733
  PROD_DEFAULT_BASE = "https://withbamboo.com";
@@ -19434,13 +19488,117 @@ var init_eval_ab2 = __esm({
19434
19488
  }
19435
19489
  });
19436
19490
 
19491
+ // src/commands/eval-promote.ts
19492
+ var eval_promote_exports = {};
19493
+ __export(eval_promote_exports, {
19494
+ runEvalPromote: () => runEvalPromote
19495
+ });
19496
+ async function requireSession13(opts) {
19497
+ const creds = await readCredentials();
19498
+ if (!creds) {
19499
+ emitError(
19500
+ {
19501
+ code: "not_signed_in",
19502
+ message: "not signed in.",
19503
+ hint: "Run `bamboo-agents login` first."
19504
+ },
19505
+ opts,
19506
+ 1
19507
+ );
19508
+ throw new Error("unreachable");
19509
+ }
19510
+ return creds.bamboo_session;
19511
+ }
19512
+ function short(version) {
19513
+ return version.slice(0, 12);
19514
+ }
19515
+ function render(data, opts) {
19516
+ human("", opts);
19517
+ if (data.kind === "demote") {
19518
+ human(
19519
+ ` Baseline demoted for qrels_version ${short(data.qrels_version)} (was run ${data.previous_run_id}).`,
19520
+ opts
19521
+ );
19522
+ human(
19523
+ " The gate reports baseline_demoted for that version until a human promotes a run.",
19524
+ opts
19525
+ );
19526
+ } else {
19527
+ human(
19528
+ ` Baseline promoted: run ${data.run_id} is the reference for qrels_version ${short(data.qrels_version)}` + (data.previous_run_id ? ` (was ${data.previous_run_id}).` : " (the version had none)."),
19529
+ opts
19530
+ );
19531
+ human(" Re-run the PR's retrieval-evals check: it compares flat against its own run.", opts);
19532
+ }
19533
+ human("", opts);
19534
+ }
19535
+ async function runEvalPromote(options) {
19536
+ const opts = { json: options.json, fields: options.fields, quiet: options.quiet };
19537
+ let body;
19538
+ if (options.demote) {
19539
+ if (!options.brand || !options.qrelsVersion) {
19540
+ emitError(
19541
+ {
19542
+ code: "missing_key",
19543
+ message: "--demote needs --brand <id> and --qrels-version <hash>.",
19544
+ hint: "A tombstone names the key it empties; both are on the gate report."
19545
+ },
19546
+ opts,
19547
+ 1
19548
+ );
19549
+ return;
19550
+ }
19551
+ body = {
19552
+ demote: true,
19553
+ brand_id: options.brand,
19554
+ qrels_version: options.qrelsVersion,
19555
+ reason: options.reason,
19556
+ ...options.pr ? { pr_url: options.pr } : {}
19557
+ };
19558
+ } else {
19559
+ if (!options.runId) {
19560
+ emitError(
19561
+ {
19562
+ code: "missing_run_id",
19563
+ message: "Pass the run id the correctness gate printed.",
19564
+ hint: 'bamboo-agents eval promote <run-id> --reason "why this drop is intentional"'
19565
+ },
19566
+ opts,
19567
+ 1
19568
+ );
19569
+ return;
19570
+ }
19571
+ body = {
19572
+ run_id: options.runId,
19573
+ reason: options.reason,
19574
+ ...options.pr ? { pr_url: options.pr } : {}
19575
+ };
19576
+ }
19577
+ const session = await requireSession13(opts);
19578
+ try {
19579
+ const data = await promoteBaseline(session, body);
19580
+ emitSuccess(data, opts, (d) => render(d, opts));
19581
+ } catch (err) {
19582
+ const e = err;
19583
+ emitError({ code: e.errorCode ?? "promote_failed", message: e.message }, opts, 1);
19584
+ }
19585
+ }
19586
+ var init_eval_promote = __esm({
19587
+ "src/commands/eval-promote.ts"() {
19588
+ "use strict";
19589
+ init_credentials();
19590
+ init_api();
19591
+ init_output();
19592
+ }
19593
+ });
19594
+
19437
19595
  // src/commands/eval-test-sets.ts
19438
19596
  var eval_test_sets_exports = {};
19439
19597
  __export(eval_test_sets_exports, {
19440
19598
  runEvalSetsCreate: () => runEvalSetsCreate,
19441
19599
  slugifyTestSetName: () => slugifyTestSetName
19442
19600
  });
19443
- async function requireSession13(opts) {
19601
+ async function requireSession14(opts) {
19444
19602
  const creds = await readCredentials();
19445
19603
  if (!creds) {
19446
19604
  emitError(
@@ -19480,7 +19638,7 @@ function printCreated(data, opts) {
19480
19638
  }
19481
19639
  async function runEvalSetsCreate(options) {
19482
19640
  const opts = options;
19483
- const session = await requireSession13(opts);
19641
+ const session = await requireSession14(opts);
19484
19642
  const brandId = await resolveBrandId(
19485
19643
  session,
19486
19644
  options.brand,
@@ -19549,7 +19707,7 @@ __export(eval_set_cases_exports, {
19549
19707
  runEvalSetsAddCases: () => runEvalSetsAddCases,
19550
19708
  unionCaseIds: () => unionCaseIds
19551
19709
  });
19552
- async function requireSession14(opts) {
19710
+ async function requireSession15(opts) {
19553
19711
  const creds = await readCredentials();
19554
19712
  if (!creds) {
19555
19713
  emitError(
@@ -19580,7 +19738,7 @@ function unionCaseIds(existing, incoming) {
19580
19738
  }
19581
19739
  async function runEvalSetsAddCases(options) {
19582
19740
  const opts = options;
19583
- const session = await requireSession14(opts);
19741
+ const session = await requireSession15(opts);
19584
19742
  const incoming = parseCaseIds(options.cases);
19585
19743
  if (incoming.length === 0) {
19586
19744
  emitError(
@@ -20151,6 +20309,14 @@ evals.command("ablate").description(
20151
20309
  await runEvalAblate2(options);
20152
20310
  }
20153
20311
  );
20312
+ evals.command("promote").description(
20313
+ "Promote a correctness run as the baseline for its qrels version, or --demote the current one"
20314
+ ).argument("[run-id]", "Run id printed by the correctness gate (omit with --demote)").requiredOption("--reason <text>", "Why this run is the reference now \u2014 recorded on the row").option("--pr <url>", "PR URL to record as the promotion's source").option("--demote", "Write a tombstone: the version has no baseline until a human promotes one").option("--brand <id>", "Brand id (with --demote)").option("--qrels-version <hash>", "qrels version to demote (with --demote)").option("--json", "Emit a JSON envelope for agent consumption").option("--fields <list>", "Project only these dotted field paths from data (comma-separated)").option("--quiet", "Suppress human chatter (data/errors only)").option("--no-sync", "Skip the auto-sync git pull on this command").action(
20315
+ async (runId, options) => {
20316
+ const { runEvalPromote: runEvalPromote2 } = await Promise.resolve().then(() => (init_eval_promote(), eval_promote_exports));
20317
+ await runEvalPromote2({ runId, ...options });
20318
+ }
20319
+ );
20154
20320
  var evalSets = evals.command("sets").description("Test sets \u2014 the definition a run executes (cases attach separately)");
20155
20321
  evalSets.command("create").description("Create an empty test set of a given evidence kind").requiredOption("--name <text>", 'Human name, e.g. "Connector parity"').requiredOption(
20156
20322
  "--kind <kind>",