bamboo-agents 0.1.6-dev.82.ffce4c6 → 0.1.6-dev.83.8a89ee2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/VERSION CHANGED
@@ -1 +1 @@
1
- 0.1.6-dev.82.ffce4c6
1
+ 0.1.6-dev.83.8a89ee2
package/dist/index.js CHANGED
@@ -3037,7 +3037,7 @@ var VERSION;
3037
3037
  var init_version = __esm({
3038
3038
  "src/version.ts"() {
3039
3039
  "use strict";
3040
- VERSION = "0.1.6-dev.82.ffce4c6";
3040
+ VERSION = "0.1.6-dev.83.8a89ee2";
3041
3041
  }
3042
3042
  });
3043
3043
 
@@ -7619,6 +7619,9 @@ var init_eval_ab = __esm({
7619
7619
  ),
7620
7620
  brand_id: external_exports.string().uuid().optional().describe(
7621
7621
  "Only needed to disambiguate an agent assigned to more than one brand. Omitted = the agent's single brand."
7622
+ ),
7623
+ set_id: external_exports.string().uuid().optional().describe(
7624
+ "BAM-4556: run THIS named test set's cases instead of the skill-attributed holdout. The set runs AS AUTHORED \u2014 cases outside the holdout split are reported on the response (`non_holdout_case_count`), never silently dropped and never silently kept quiet. Omitted = skill attribution, the default and SkillOpt's only path."
7622
7625
  )
7623
7626
  });
7624
7627
  EvalAbResponseSchema = external_exports.object({
@@ -7629,8 +7632,31 @@ var init_eval_ab = __esm({
7629
7632
  skill_name: external_exports.string(),
7630
7633
  brand_id: external_exports.string().uuid(),
7631
7634
  agent_id: external_exports.string().uuid(),
7632
- /** Holdout cases the cycle will run over — chosen server-side, never passed in. */
7635
+ /** Cases the cycle will run over — chosen server-side, never passed in. */
7633
7636
  case_count: external_exports.number().int().positive(),
7637
+ /**
7638
+ * BAM-4556 — how those cases were chosen, and what it cost.
7639
+ *
7640
+ * `non_holdout_case_count` is the contamination signal and it is ALWAYS
7641
+ * present, 0 on the attribution path (where holdout is enforced by the query
7642
+ * that picked the cases). A named set runs as authored, so a non-zero value
7643
+ * means this verdict was measured partly on cases the skill may have been
7644
+ * tuned against — reported here, in `plan`, and persisted on every queue row's
7645
+ * payload, because a contaminated result that looks clean is the failure mode
7646
+ * this field exists to prevent.
7647
+ */
7648
+ selection: external_exports.object({
7649
+ /** `attribution` (the default) or `named_set`. */
7650
+ strategy: external_exports.enum(["attribution", "named_set"]),
7651
+ /** The set that was run. Null on the attribution path. */
7652
+ set_id: external_exports.string().uuid().nullable(),
7653
+ /** The set's slug — the same token the `selection_policy` stamp carries. */
7654
+ set_slug: external_exports.string().nullable(),
7655
+ /** The `eval_run_queue.selection_policy` stamp. Null on the attribution path. */
7656
+ selection_policy: external_exports.string().nullable(),
7657
+ /** Cases in the batch whose split is NOT `holdout`. 0 on the attribution path. */
7658
+ non_holdout_case_count: external_exports.number().int().nonnegative()
7659
+ }),
7634
7660
  /** Reps per ARM (not per case). `total_runs = case_count × reps_per_arm × 2`. */
7635
7661
  reps_per_arm: external_exports.number().int().positive(),
7636
7662
  /** Queue rows staged, across BOTH arms. */
@@ -19372,7 +19398,21 @@ function printStaged(data, opts) {
19372
19398
  ` ${data.case_count} cases \xD7 ${data.reps_per_arm} reps \xD7 2 arms = ${data.total_runs} queued runs.`,
19373
19399
  opts
19374
19400
  );
19401
+ if (data.selection.non_holdout_case_count > 0) {
19402
+ human("", opts);
19403
+ human(
19404
+ ` \u26A0 ${data.selection.non_holdout_case_count} of these cases are NOT in the holdout split.`,
19405
+ opts
19406
+ );
19407
+ human(
19408
+ " This verdict is measured partly on cases the skill may have been tuned against.",
19409
+ opts
19410
+ );
19411
+ }
19375
19412
  human("", opts);
19413
+ if (data.selection.set_slug) {
19414
+ human(` Set: ${data.selection.set_slug}`, opts);
19415
+ }
19376
19416
  human(` Batch: ${data.run_batch_id}`, opts);
19377
19417
  human(` Results: ${absoluteUrl(data.deep_link)}`, opts);
19378
19418
  human("", opts);
@@ -19419,7 +19459,8 @@ async function runEvalAblate(options) {
19419
19459
  skill_name: options.skill,
19420
19460
  hypothesis: options.hypothesis,
19421
19461
  candidate_body: candidateBody,
19422
- ...options.brand ? { brand_id: options.brand } : {}
19462
+ ...options.brand ? { brand_id: options.brand } : {},
19463
+ ...options.set ? { set_id: options.set } : {}
19423
19464
  });
19424
19465
  } catch (err) {
19425
19466
  const e = err;
@@ -20155,7 +20196,10 @@ evals.command("ablate").description(
20155
20196
  ).requiredOption("--agent <id>", "Agent id whose skill is under test").requiredOption("--skill <name>", "kebab-case skill name (must be one of this agent's own skills)").requiredOption(
20156
20197
  "--hypothesis <text>",
20157
20198
  "Plain-language claim the edit is meant to produce (an experiment is a claim, not a diff)"
20158
- ).requiredOption("--body <path>", "Path to the candidate SKILL.md").option("--brand <id>", "Brand id \u2014 only needed when the agent spans more than one brand").option("--json", "Emit a JSON envelope for agent consumption").option("--fields <list>", "Project only these dotted field paths from data (comma-separated)").option("--quiet", "Suppress human chatter (data/errors only)").option("--no-sync", "Skip the auto-sync git pull on this command").action(
20199
+ ).requiredOption("--body <path>", "Path to the candidate SKILL.md").option("--brand <id>", "Brand id \u2014 only needed when the agent spans more than one brand").option(
20200
+ "--set <id>",
20201
+ "Test set id \u2014 run THIS set's cases instead of the skill-attributed holdout"
20202
+ ).option("--json", "Emit a JSON envelope for agent consumption").option("--fields <list>", "Project only these dotted field paths from data (comma-separated)").option("--quiet", "Suppress human chatter (data/errors only)").option("--no-sync", "Skip the auto-sync git pull on this command").action(
20159
20203
  async (options) => {
20160
20204
  const { runEvalAblate: runEvalAblate2 } = await Promise.resolve().then(() => (init_eval_ab2(), eval_ab_exports2));
20161
20205
  await runEvalAblate2(options);