bamboo-agents 0.1.6-dev.82.ffce4c6 → 0.1.6-dev.83.8a89ee2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/VERSION +1 -1
- package/dist/index.js +48 -4
- package/dist/index.js.map +2 -2
- package/package.json +1 -1
package/dist/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
0.1.6-dev.
|
|
1
|
+
0.1.6-dev.83.8a89ee2
|
package/dist/index.js
CHANGED
|
@@ -3037,7 +3037,7 @@ var VERSION;
|
|
|
3037
3037
|
var init_version = __esm({
|
|
3038
3038
|
"src/version.ts"() {
|
|
3039
3039
|
"use strict";
|
|
3040
|
-
VERSION = "0.1.6-dev.
|
|
3040
|
+
VERSION = "0.1.6-dev.83.8a89ee2";
|
|
3041
3041
|
}
|
|
3042
3042
|
});
|
|
3043
3043
|
|
|
@@ -7619,6 +7619,9 @@ var init_eval_ab = __esm({
|
|
|
7619
7619
|
),
|
|
7620
7620
|
brand_id: external_exports.string().uuid().optional().describe(
|
|
7621
7621
|
"Only needed to disambiguate an agent assigned to more than one brand. Omitted = the agent's single brand."
|
|
7622
|
+
),
|
|
7623
|
+
set_id: external_exports.string().uuid().optional().describe(
|
|
7624
|
+
"BAM-4556: run THIS named test set's cases instead of the skill-attributed holdout. The set runs AS AUTHORED \u2014 cases outside the holdout split are reported on the response (`non_holdout_case_count`), never silently dropped and never silently kept quiet. Omitted = skill attribution, the default and SkillOpt's only path."
|
|
7622
7625
|
)
|
|
7623
7626
|
});
|
|
7624
7627
|
EvalAbResponseSchema = external_exports.object({
|
|
@@ -7629,8 +7632,31 @@ var init_eval_ab = __esm({
|
|
|
7629
7632
|
skill_name: external_exports.string(),
|
|
7630
7633
|
brand_id: external_exports.string().uuid(),
|
|
7631
7634
|
agent_id: external_exports.string().uuid(),
|
|
7632
|
-
/**
|
|
7635
|
+
/** Cases the cycle will run over — chosen server-side, never passed in. */
|
|
7633
7636
|
case_count: external_exports.number().int().positive(),
|
|
7637
|
+
/**
|
|
7638
|
+
* BAM-4556 — how those cases were chosen, and what it cost.
|
|
7639
|
+
*
|
|
7640
|
+
* `non_holdout_case_count` is the contamination signal and it is ALWAYS
|
|
7641
|
+
* present, 0 on the attribution path (where holdout is enforced by the query
|
|
7642
|
+
* that picked the cases). A named set runs as authored, so a non-zero value
|
|
7643
|
+
* means this verdict was measured partly on cases the skill may have been
|
|
7644
|
+
* tuned against — reported here, in `plan`, and persisted on every queue row's
|
|
7645
|
+
* payload, because a contaminated result that looks clean is the failure mode
|
|
7646
|
+
* this field exists to prevent.
|
|
7647
|
+
*/
|
|
7648
|
+
selection: external_exports.object({
|
|
7649
|
+
/** `attribution` (the default) or `named_set`. */
|
|
7650
|
+
strategy: external_exports.enum(["attribution", "named_set"]),
|
|
7651
|
+
/** The set that was run. Null on the attribution path. */
|
|
7652
|
+
set_id: external_exports.string().uuid().nullable(),
|
|
7653
|
+
/** The set's slug — the same token the `selection_policy` stamp carries. */
|
|
7654
|
+
set_slug: external_exports.string().nullable(),
|
|
7655
|
+
/** The `eval_run_queue.selection_policy` stamp. Null on the attribution path. */
|
|
7656
|
+
selection_policy: external_exports.string().nullable(),
|
|
7657
|
+
/** Cases in the batch whose split is NOT `holdout`. 0 on the attribution path. */
|
|
7658
|
+
non_holdout_case_count: external_exports.number().int().nonnegative()
|
|
7659
|
+
}),
|
|
7634
7660
|
/** Reps per ARM (not per case). `total_runs = case_count × reps_per_arm × 2`. */
|
|
7635
7661
|
reps_per_arm: external_exports.number().int().positive(),
|
|
7636
7662
|
/** Queue rows staged, across BOTH arms. */
|
|
@@ -19372,7 +19398,21 @@ function printStaged(data, opts) {
|
|
|
19372
19398
|
` ${data.case_count} cases \xD7 ${data.reps_per_arm} reps \xD7 2 arms = ${data.total_runs} queued runs.`,
|
|
19373
19399
|
opts
|
|
19374
19400
|
);
|
|
19401
|
+
if (data.selection.non_holdout_case_count > 0) {
|
|
19402
|
+
human("", opts);
|
|
19403
|
+
human(
|
|
19404
|
+
` \u26A0 ${data.selection.non_holdout_case_count} of these cases are NOT in the holdout split.`,
|
|
19405
|
+
opts
|
|
19406
|
+
);
|
|
19407
|
+
human(
|
|
19408
|
+
" This verdict is measured partly on cases the skill may have been tuned against.",
|
|
19409
|
+
opts
|
|
19410
|
+
);
|
|
19411
|
+
}
|
|
19375
19412
|
human("", opts);
|
|
19413
|
+
if (data.selection.set_slug) {
|
|
19414
|
+
human(` Set: ${data.selection.set_slug}`, opts);
|
|
19415
|
+
}
|
|
19376
19416
|
human(` Batch: ${data.run_batch_id}`, opts);
|
|
19377
19417
|
human(` Results: ${absoluteUrl(data.deep_link)}`, opts);
|
|
19378
19418
|
human("", opts);
|
|
@@ -19419,7 +19459,8 @@ async function runEvalAblate(options) {
|
|
|
19419
19459
|
skill_name: options.skill,
|
|
19420
19460
|
hypothesis: options.hypothesis,
|
|
19421
19461
|
candidate_body: candidateBody,
|
|
19422
|
-
...options.brand ? { brand_id: options.brand } : {}
|
|
19462
|
+
...options.brand ? { brand_id: options.brand } : {},
|
|
19463
|
+
...options.set ? { set_id: options.set } : {}
|
|
19423
19464
|
});
|
|
19424
19465
|
} catch (err) {
|
|
19425
19466
|
const e = err;
|
|
@@ -20155,7 +20196,10 @@ evals.command("ablate").description(
|
|
|
20155
20196
|
).requiredOption("--agent <id>", "Agent id whose skill is under test").requiredOption("--skill <name>", "kebab-case skill name (must be one of this agent's own skills)").requiredOption(
|
|
20156
20197
|
"--hypothesis <text>",
|
|
20157
20198
|
"Plain-language claim the edit is meant to produce (an experiment is a claim, not a diff)"
|
|
20158
|
-
).requiredOption("--body <path>", "Path to the candidate SKILL.md").option("--brand <id>", "Brand id \u2014 only needed when the agent spans more than one brand").option(
|
|
20199
|
+
).requiredOption("--body <path>", "Path to the candidate SKILL.md").option("--brand <id>", "Brand id \u2014 only needed when the agent spans more than one brand").option(
|
|
20200
|
+
"--set <id>",
|
|
20201
|
+
"Test set id \u2014 run THIS set's cases instead of the skill-attributed holdout"
|
|
20202
|
+
).option("--json", "Emit a JSON envelope for agent consumption").option("--fields <list>", "Project only these dotted field paths from data (comma-separated)").option("--quiet", "Suppress human chatter (data/errors only)").option("--no-sync", "Skip the auto-sync git pull on this command").action(
|
|
20159
20203
|
async (options) => {
|
|
20160
20204
|
const { runEvalAblate: runEvalAblate2 } = await Promise.resolve().then(() => (init_eval_ab2(), eval_ab_exports2));
|
|
20161
20205
|
await runEvalAblate2(options);
|