bamboo-agents 0.1.6-dev.81.7694f7f → 0.1.6-dev.83.8a89ee2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/VERSION CHANGED
@@ -1 +1 @@
1
- 0.1.6-dev.81.7694f7f
1
+ 0.1.6-dev.83.8a89ee2
package/dist/index.js CHANGED
@@ -3037,7 +3037,7 @@ var VERSION;
3037
3037
  var init_version = __esm({
3038
3038
  "src/version.ts"() {
3039
3039
  "use strict";
3040
- VERSION = "0.1.6-dev.81.7694f7f";
3040
+ VERSION = "0.1.6-dev.83.8a89ee2";
3041
3041
  }
3042
3042
  });
3043
3043
 
@@ -7619,6 +7619,9 @@ var init_eval_ab = __esm({
7619
7619
  ),
7620
7620
  brand_id: external_exports.string().uuid().optional().describe(
7621
7621
  "Only needed to disambiguate an agent assigned to more than one brand. Omitted = the agent's single brand."
7622
+ ),
7623
+ set_id: external_exports.string().uuid().optional().describe(
7624
+ "BAM-4556: run THIS named test set's cases instead of the skill-attributed holdout. The set runs AS AUTHORED \u2014 cases outside the holdout split are reported on the response (`non_holdout_case_count`), never silently dropped and never silently kept quiet. Omitted = skill attribution, the default and SkillOpt's only path."
7622
7625
  )
7623
7626
  });
7624
7627
  EvalAbResponseSchema = external_exports.object({
@@ -7629,8 +7632,31 @@ var init_eval_ab = __esm({
7629
7632
  skill_name: external_exports.string(),
7630
7633
  brand_id: external_exports.string().uuid(),
7631
7634
  agent_id: external_exports.string().uuid(),
7632
- /** Holdout cases the cycle will run over — chosen server-side, never passed in. */
7635
+ /** Cases the cycle will run over — chosen server-side, never passed in. */
7633
7636
  case_count: external_exports.number().int().positive(),
7637
+ /**
7638
+ * BAM-4556 — how those cases were chosen, and what it cost.
7639
+ *
7640
+ * `non_holdout_case_count` is the contamination signal and it is ALWAYS
7641
+ * present, 0 on the attribution path (where holdout is enforced by the query
7642
+ * that picked the cases). A named set runs as authored, so a non-zero value
7643
+ * means this verdict was measured partly on cases the skill may have been
7644
+ * tuned against — reported here, in `plan`, and persisted on every queue row's
7645
+ * payload, because a contaminated result that looks clean is the failure mode
7646
+ * this field exists to prevent.
7647
+ */
7648
+ selection: external_exports.object({
7649
+ /** `attribution` (the default) or `named_set`. */
7650
+ strategy: external_exports.enum(["attribution", "named_set"]),
7651
+ /** The set that was run. Null on the attribution path. */
7652
+ set_id: external_exports.string().uuid().nullable(),
7653
+ /** The set's slug — the same token the `selection_policy` stamp carries. */
7654
+ set_slug: external_exports.string().nullable(),
7655
+ /** The `eval_run_queue.selection_policy` stamp. Null on the attribution path. */
7656
+ selection_policy: external_exports.string().nullable(),
7657
+ /** Cases in the batch whose split is NOT `holdout`. 0 on the attribution path. */
7658
+ non_holdout_case_count: external_exports.number().int().nonnegative()
7659
+ }),
7634
7660
  /** Reps per ARM (not per case). `total_runs = case_count × reps_per_arm × 2`. */
7635
7661
  reps_per_arm: external_exports.number().int().positive(),
7636
7662
  /** Queue rows staged, across BOTH arms. */
@@ -13015,7 +13041,7 @@ var init_evidence_kinds = __esm({
13015
13041
  });
13016
13042
 
13017
13043
  // ../../src/lib/api-contracts/cli/eval-test-sets.ts
13018
- var CliEvalTestSetCreateRequestSchema, CliEvalTestSetCreateResponseSchema;
13044
+ var CliEvalTestSetCreateRequestSchema, CliEvalTestSetCreateResponseSchema, CliRoadmapSetMembershipSchema, CliRoadmapMembershipResponseSchema, CliEvalTestSetCasesRequestSchema;
13019
13045
  var init_eval_test_sets = __esm({
13020
13046
  "../../src/lib/api-contracts/cli/eval-test-sets.ts"() {
13021
13047
  init_zod();
@@ -13047,6 +13073,17 @@ var init_eval_test_sets = __esm({
13047
13073
  priority: external_exports.number().int(),
13048
13074
  hypothesis: external_exports.string().nullable()
13049
13075
  });
13076
+ CliRoadmapSetMembershipSchema = external_exports.object({
13077
+ id: external_exports.string().uuid(),
13078
+ cases: external_exports.array(external_exports.object({ id: external_exports.string().uuid() })).default([])
13079
+ });
13080
+ CliRoadmapMembershipResponseSchema = external_exports.array(
13081
+ CliRoadmapSetMembershipSchema
13082
+ );
13083
+ CliEvalTestSetCasesRequestSchema = external_exports.object({
13084
+ brand_id: external_exports.string().uuid(),
13085
+ case_ids: external_exports.array(external_exports.string().uuid())
13086
+ });
13050
13087
  }
13051
13088
  });
13052
13089
 
@@ -13577,6 +13614,29 @@ async function createEvalTestSet(bambooSession, body) {
13577
13614
  const json = await res.json();
13578
13615
  return CliEvalTestSetCreateResponseSchema.parse(json.data);
13579
13616
  }
13617
+ async function getEvalTestSetMemberships(bambooSession, brandId) {
13618
+ const url = `${apiBase()}/api/v1/eval/test-roadmap?brand_id=${encodeURIComponent(brandId)}`;
13619
+ const res = await cliApiFetch(url, {
13620
+ headers: { Authorization: `Bearer ${bambooSession}` }
13621
+ });
13622
+ if (!res.ok) throw await parseError(res);
13623
+ const json = await res.json();
13624
+ return CliRoadmapMembershipResponseSchema.parse(json.data ?? []);
13625
+ }
13626
+ async function updateEvalTestSetCases(bambooSession, setId, body) {
13627
+ const res = await cliApiFetch(
13628
+ `${apiBase()}/api/v1/eval/test-sets/${encodeURIComponent(setId)}`,
13629
+ {
13630
+ method: "PATCH",
13631
+ headers: {
13632
+ Authorization: `Bearer ${bambooSession}`,
13633
+ "Content-Type": "application/json"
13634
+ },
13635
+ body: JSON.stringify(body)
13636
+ }
13637
+ );
13638
+ if (!res.ok) throw await parseError(res);
13639
+ }
13580
13640
  function absoluteUrl(relativePath) {
13581
13641
  return `${apiBase()}${relativePath}`;
13582
13642
  }
@@ -19338,7 +19398,21 @@ function printStaged(data, opts) {
19338
19398
  ` ${data.case_count} cases \xD7 ${data.reps_per_arm} reps \xD7 2 arms = ${data.total_runs} queued runs.`,
19339
19399
  opts
19340
19400
  );
19401
+ if (data.selection.non_holdout_case_count > 0) {
19402
+ human("", opts);
19403
+ human(
19404
+ ` \u26A0 ${data.selection.non_holdout_case_count} of these cases are NOT in the holdout split.`,
19405
+ opts
19406
+ );
19407
+ human(
19408
+ " This verdict is measured partly on cases the skill may have been tuned against.",
19409
+ opts
19410
+ );
19411
+ }
19341
19412
  human("", opts);
19413
+ if (data.selection.set_slug) {
19414
+ human(` Set: ${data.selection.set_slug}`, opts);
19415
+ }
19342
19416
  human(` Batch: ${data.run_batch_id}`, opts);
19343
19417
  human(` Results: ${absoluteUrl(data.deep_link)}`, opts);
19344
19418
  human("", opts);
@@ -19385,7 +19459,8 @@ async function runEvalAblate(options) {
19385
19459
  skill_name: options.skill,
19386
19460
  hypothesis: options.hypothesis,
19387
19461
  candidate_body: candidateBody,
19388
- ...options.brand ? { brand_id: options.brand } : {}
19462
+ ...options.brand ? { brand_id: options.brand } : {},
19463
+ ...options.set ? { set_id: options.set } : {}
19389
19464
  });
19390
19465
  } catch (err) {
19391
19466
  const e = err;
@@ -19515,6 +19590,145 @@ var init_eval_test_sets2 = __esm({
19515
19590
  }
19516
19591
  });
19517
19592
 
19593
+ // src/commands/eval-set-cases.ts
19594
+ var eval_set_cases_exports = {};
19595
+ __export(eval_set_cases_exports, {
19596
+ parseCaseIds: () => parseCaseIds,
19597
+ runEvalSetsAddCases: () => runEvalSetsAddCases,
19598
+ unionCaseIds: () => unionCaseIds
19599
+ });
19600
+ async function requireSession14(opts) {
19601
+ const creds = await readCredentials();
19602
+ if (!creds) {
19603
+ emitError(
19604
+ {
19605
+ code: "not_signed_in",
19606
+ message: "not signed in.",
19607
+ hint: "Run `bamboo-agents login` first."
19608
+ },
19609
+ opts,
19610
+ 1
19611
+ );
19612
+ throw new Error("unreachable");
19613
+ }
19614
+ return creds.bamboo_session;
19615
+ }
19616
+ function parseCaseIds(raw) {
19617
+ return raw.split(",").map((s) => s.trim()).filter((s) => s.length > 0);
19618
+ }
19619
+ function unionCaseIds(existing, incoming) {
19620
+ const seen = /* @__PURE__ */ new Set();
19621
+ const out = [];
19622
+ for (const id of [...existing, ...incoming]) {
19623
+ if (seen.has(id)) continue;
19624
+ seen.add(id);
19625
+ out.push(id);
19626
+ }
19627
+ return out;
19628
+ }
19629
+ async function runEvalSetsAddCases(options) {
19630
+ const opts = options;
19631
+ const session = await requireSession14(opts);
19632
+ const incoming = parseCaseIds(options.cases);
19633
+ if (incoming.length === 0) {
19634
+ emitError(
19635
+ { code: "no_cases", message: "--cases was empty \u2014 nothing to attach." },
19636
+ opts,
19637
+ 1
19638
+ );
19639
+ throw new Error("unreachable");
19640
+ }
19641
+ const brandId = await resolveBrandId(
19642
+ session,
19643
+ options.brand,
19644
+ opts,
19645
+ "Specify the brand the test set belongs to with --brand <id>."
19646
+ );
19647
+ let finalIds;
19648
+ let existingCount = 0;
19649
+ if (options.replace) {
19650
+ finalIds = unionCaseIds([], incoming);
19651
+ } else {
19652
+ let sets;
19653
+ try {
19654
+ sets = await getEvalTestSetMemberships(session, brandId);
19655
+ } catch (err) {
19656
+ const e = err;
19657
+ emitError(
19658
+ {
19659
+ code: e.code ?? "roadmap_read_failed",
19660
+ message: `couldn't read the set's current cases: ${e.message ?? "unknown error"}`,
19661
+ hint: "Pass --replace to write the list without reading first."
19662
+ },
19663
+ opts,
19664
+ 1
19665
+ );
19666
+ throw new Error("unreachable");
19667
+ }
19668
+ const target = sets.find((s) => s.id === options.set);
19669
+ if (!target) {
19670
+ emitError(
19671
+ {
19672
+ code: "set_not_found",
19673
+ message: `no test set ${options.set} in brand ${brandId}.`,
19674
+ hint: "Check the set id, or pass --brand if the set is in another brand."
19675
+ },
19676
+ opts,
19677
+ 1
19678
+ );
19679
+ throw new Error("unreachable");
19680
+ }
19681
+ const existing = target.cases.map((c) => c.id);
19682
+ existingCount = existing.length;
19683
+ finalIds = unionCaseIds(existing, incoming);
19684
+ }
19685
+ try {
19686
+ await updateEvalTestSetCases(session, options.set, {
19687
+ brand_id: brandId,
19688
+ case_ids: finalIds
19689
+ });
19690
+ } catch (err) {
19691
+ const e = err;
19692
+ emitError(
19693
+ {
19694
+ code: e.code ?? "attach_failed",
19695
+ message: e.message ?? "failed to attach the cases."
19696
+ },
19697
+ opts,
19698
+ 1
19699
+ );
19700
+ throw new Error("unreachable");
19701
+ }
19702
+ const added = finalIds.length - existingCount;
19703
+ human("", opts);
19704
+ if (options.replace) {
19705
+ human(` Set ${options.set} now holds exactly ${finalIds.length} cases.`, opts);
19706
+ } else {
19707
+ human(
19708
+ ` Added ${added} of ${incoming.length} to ${options.set} \u2014 ${finalIds.length} cases total.`,
19709
+ opts
19710
+ );
19711
+ const skipped = incoming.length - added;
19712
+ if (skipped > 0) {
19713
+ human(` ${skipped} ${skipped === 1 ? "was" : "were"} already in the set.`, opts);
19714
+ }
19715
+ }
19716
+ human("", opts);
19717
+ emitSuccess(
19718
+ { set_id: options.set, case_ids: finalIds, total: finalIds.length, added },
19719
+ opts
19720
+ );
19721
+ }
19722
+ var init_eval_set_cases = __esm({
19723
+ "src/commands/eval-set-cases.ts"() {
19724
+ "use strict";
19725
+ init_credentials();
19726
+ init_api();
19727
+ init_resolve_brand();
19728
+ init_output();
19729
+ }
19730
+ });
19731
+
19518
19732
  // node_modules/commander/esm.mjs
19519
19733
  var import_index = __toESM(require_commander(), 1);
19520
19734
  var {
@@ -19982,7 +20196,10 @@ evals.command("ablate").description(
19982
20196
  ).requiredOption("--agent <id>", "Agent id whose skill is under test").requiredOption("--skill <name>", "kebab-case skill name (must be one of this agent's own skills)").requiredOption(
19983
20197
  "--hypothesis <text>",
19984
20198
  "Plain-language claim the edit is meant to produce (an experiment is a claim, not a diff)"
19985
- ).requiredOption("--body <path>", "Path to the candidate SKILL.md").option("--brand <id>", "Brand id \u2014 only needed when the agent spans more than one brand").option("--json", "Emit a JSON envelope for agent consumption").option("--fields <list>", "Project only these dotted field paths from data (comma-separated)").option("--quiet", "Suppress human chatter (data/errors only)").option("--no-sync", "Skip the auto-sync git pull on this command").action(
20199
+ ).requiredOption("--body <path>", "Path to the candidate SKILL.md").option("--brand <id>", "Brand id \u2014 only needed when the agent spans more than one brand").option(
20200
+ "--set <id>",
20201
+ "Test set id \u2014 run THIS set's cases instead of the skill-attributed holdout"
20202
+ ).option("--json", "Emit a JSON envelope for agent consumption").option("--fields <list>", "Project only these dotted field paths from data (comma-separated)").option("--quiet", "Suppress human chatter (data/errors only)").option("--no-sync", "Skip the auto-sync git pull on this command").action(
19986
20203
  async (options) => {
19987
20204
  const { runEvalAblate: runEvalAblate2 } = await Promise.resolve().then(() => (init_eval_ab2(), eval_ab_exports2));
19988
20205
  await runEvalAblate2(options);
@@ -20001,6 +20218,15 @@ evalSets.command("create").description("Create an empty test set of a given evid
20001
20218
  await runEvalSetsCreate2(options);
20002
20219
  }
20003
20220
  );
20221
+ evalSets.command("add-cases").description("Attach existing cases to a test set (adds by default; --replace overwrites)").requiredOption("--set <id>", "Test set id").requiredOption("--cases <ids>", "Comma-separated eval case ids").option(
20222
+ "--replace",
20223
+ "Send exactly these cases, dropping any others. Without it, the current cases are read and merged"
20224
+ ).option("--brand <id>", "Brand id \u2014 only needed when the org has more than one brand").option("--json", "Emit a JSON envelope for agent consumption").option("--fields <list>", "Project only these dotted field paths from data (comma-separated)").option("--quiet", "Suppress human chatter (data/errors only)").option("--no-sync", "Skip the auto-sync git pull on this command").action(
20225
+ async (options) => {
20226
+ const { runEvalSetsAddCases: runEvalSetsAddCases2 } = await Promise.resolve().then(() => (init_eval_set_cases(), eval_set_cases_exports));
20227
+ await runEvalSetsAddCases2(options);
20228
+ }
20229
+ );
20004
20230
  (async () => {
20005
20231
  const rawArgs = process.argv.slice(2);
20006
20232
  const { operands } = program2.parseOptions(rawArgs);