bamboo-agents 0.1.6-dev.81.7694f7f → 0.1.6-dev.83.8a89ee2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/VERSION +1 -1
- package/dist/index.js +231 -5
- package/dist/index.js.map +4 -4
- package/package.json +1 -1
package/dist/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
0.1.6-dev.
|
|
1
|
+
0.1.6-dev.83.8a89ee2
|
package/dist/index.js
CHANGED
|
@@ -3037,7 +3037,7 @@ var VERSION;
|
|
|
3037
3037
|
var init_version = __esm({
|
|
3038
3038
|
"src/version.ts"() {
|
|
3039
3039
|
"use strict";
|
|
3040
|
-
VERSION = "0.1.6-dev.
|
|
3040
|
+
VERSION = "0.1.6-dev.83.8a89ee2";
|
|
3041
3041
|
}
|
|
3042
3042
|
});
|
|
3043
3043
|
|
|
@@ -7619,6 +7619,9 @@ var init_eval_ab = __esm({
|
|
|
7619
7619
|
),
|
|
7620
7620
|
brand_id: external_exports.string().uuid().optional().describe(
|
|
7621
7621
|
"Only needed to disambiguate an agent assigned to more than one brand. Omitted = the agent's single brand."
|
|
7622
|
+
),
|
|
7623
|
+
set_id: external_exports.string().uuid().optional().describe(
|
|
7624
|
+
"BAM-4556: run THIS named test set's cases instead of the skill-attributed holdout. The set runs AS AUTHORED \u2014 cases outside the holdout split are reported on the response (`non_holdout_case_count`), never silently dropped and never silently kept quiet. Omitted = skill attribution, the default and SkillOpt's only path."
|
|
7622
7625
|
)
|
|
7623
7626
|
});
|
|
7624
7627
|
EvalAbResponseSchema = external_exports.object({
|
|
@@ -7629,8 +7632,31 @@ var init_eval_ab = __esm({
|
|
|
7629
7632
|
skill_name: external_exports.string(),
|
|
7630
7633
|
brand_id: external_exports.string().uuid(),
|
|
7631
7634
|
agent_id: external_exports.string().uuid(),
|
|
7632
|
-
/**
|
|
7635
|
+
/** Cases the cycle will run over — chosen server-side, never passed in. */
|
|
7633
7636
|
case_count: external_exports.number().int().positive(),
|
|
7637
|
+
/**
|
|
7638
|
+
* BAM-4556 — how those cases were chosen, and what it cost.
|
|
7639
|
+
*
|
|
7640
|
+
* `non_holdout_case_count` is the contamination signal and it is ALWAYS
|
|
7641
|
+
* present, 0 on the attribution path (where holdout is enforced by the query
|
|
7642
|
+
* that picked the cases). A named set runs as authored, so a non-zero value
|
|
7643
|
+
* means this verdict was measured partly on cases the skill may have been
|
|
7644
|
+
* tuned against — reported here, in `plan`, and persisted on every queue row's
|
|
7645
|
+
* payload, because a contaminated result that looks clean is the failure mode
|
|
7646
|
+
* this field exists to prevent.
|
|
7647
|
+
*/
|
|
7648
|
+
selection: external_exports.object({
|
|
7649
|
+
/** `attribution` (the default) or `named_set`. */
|
|
7650
|
+
strategy: external_exports.enum(["attribution", "named_set"]),
|
|
7651
|
+
/** The set that was run. Null on the attribution path. */
|
|
7652
|
+
set_id: external_exports.string().uuid().nullable(),
|
|
7653
|
+
/** The set's slug — the same token the `selection_policy` stamp carries. */
|
|
7654
|
+
set_slug: external_exports.string().nullable(),
|
|
7655
|
+
/** The `eval_run_queue.selection_policy` stamp. Null on the attribution path. */
|
|
7656
|
+
selection_policy: external_exports.string().nullable(),
|
|
7657
|
+
/** Cases in the batch whose split is NOT `holdout`. 0 on the attribution path. */
|
|
7658
|
+
non_holdout_case_count: external_exports.number().int().nonnegative()
|
|
7659
|
+
}),
|
|
7634
7660
|
/** Reps per ARM (not per case). `total_runs = case_count × reps_per_arm × 2`. */
|
|
7635
7661
|
reps_per_arm: external_exports.number().int().positive(),
|
|
7636
7662
|
/** Queue rows staged, across BOTH arms. */
|
|
@@ -13015,7 +13041,7 @@ var init_evidence_kinds = __esm({
|
|
|
13015
13041
|
});
|
|
13016
13042
|
|
|
13017
13043
|
// ../../src/lib/api-contracts/cli/eval-test-sets.ts
|
|
13018
|
-
var CliEvalTestSetCreateRequestSchema, CliEvalTestSetCreateResponseSchema;
|
|
13044
|
+
var CliEvalTestSetCreateRequestSchema, CliEvalTestSetCreateResponseSchema, CliRoadmapSetMembershipSchema, CliRoadmapMembershipResponseSchema, CliEvalTestSetCasesRequestSchema;
|
|
13019
13045
|
var init_eval_test_sets = __esm({
|
|
13020
13046
|
"../../src/lib/api-contracts/cli/eval-test-sets.ts"() {
|
|
13021
13047
|
init_zod();
|
|
@@ -13047,6 +13073,17 @@ var init_eval_test_sets = __esm({
|
|
|
13047
13073
|
priority: external_exports.number().int(),
|
|
13048
13074
|
hypothesis: external_exports.string().nullable()
|
|
13049
13075
|
});
|
|
13076
|
+
CliRoadmapSetMembershipSchema = external_exports.object({
|
|
13077
|
+
id: external_exports.string().uuid(),
|
|
13078
|
+
cases: external_exports.array(external_exports.object({ id: external_exports.string().uuid() })).default([])
|
|
13079
|
+
});
|
|
13080
|
+
CliRoadmapMembershipResponseSchema = external_exports.array(
|
|
13081
|
+
CliRoadmapSetMembershipSchema
|
|
13082
|
+
);
|
|
13083
|
+
CliEvalTestSetCasesRequestSchema = external_exports.object({
|
|
13084
|
+
brand_id: external_exports.string().uuid(),
|
|
13085
|
+
case_ids: external_exports.array(external_exports.string().uuid())
|
|
13086
|
+
});
|
|
13050
13087
|
}
|
|
13051
13088
|
});
|
|
13052
13089
|
|
|
@@ -13577,6 +13614,29 @@ async function createEvalTestSet(bambooSession, body) {
|
|
|
13577
13614
|
const json = await res.json();
|
|
13578
13615
|
return CliEvalTestSetCreateResponseSchema.parse(json.data);
|
|
13579
13616
|
}
|
|
13617
|
+
async function getEvalTestSetMemberships(bambooSession, brandId) {
|
|
13618
|
+
const url = `${apiBase()}/api/v1/eval/test-roadmap?brand_id=${encodeURIComponent(brandId)}`;
|
|
13619
|
+
const res = await cliApiFetch(url, {
|
|
13620
|
+
headers: { Authorization: `Bearer ${bambooSession}` }
|
|
13621
|
+
});
|
|
13622
|
+
if (!res.ok) throw await parseError(res);
|
|
13623
|
+
const json = await res.json();
|
|
13624
|
+
return CliRoadmapMembershipResponseSchema.parse(json.data ?? []);
|
|
13625
|
+
}
|
|
13626
|
+
async function updateEvalTestSetCases(bambooSession, setId, body) {
|
|
13627
|
+
const res = await cliApiFetch(
|
|
13628
|
+
`${apiBase()}/api/v1/eval/test-sets/${encodeURIComponent(setId)}`,
|
|
13629
|
+
{
|
|
13630
|
+
method: "PATCH",
|
|
13631
|
+
headers: {
|
|
13632
|
+
Authorization: `Bearer ${bambooSession}`,
|
|
13633
|
+
"Content-Type": "application/json"
|
|
13634
|
+
},
|
|
13635
|
+
body: JSON.stringify(body)
|
|
13636
|
+
}
|
|
13637
|
+
);
|
|
13638
|
+
if (!res.ok) throw await parseError(res);
|
|
13639
|
+
}
|
|
13580
13640
|
function absoluteUrl(relativePath) {
|
|
13581
13641
|
return `${apiBase()}${relativePath}`;
|
|
13582
13642
|
}
|
|
@@ -19338,7 +19398,21 @@ function printStaged(data, opts) {
|
|
|
19338
19398
|
` ${data.case_count} cases \xD7 ${data.reps_per_arm} reps \xD7 2 arms = ${data.total_runs} queued runs.`,
|
|
19339
19399
|
opts
|
|
19340
19400
|
);
|
|
19401
|
+
if (data.selection.non_holdout_case_count > 0) {
|
|
19402
|
+
human("", opts);
|
|
19403
|
+
human(
|
|
19404
|
+
` \u26A0 ${data.selection.non_holdout_case_count} of these cases are NOT in the holdout split.`,
|
|
19405
|
+
opts
|
|
19406
|
+
);
|
|
19407
|
+
human(
|
|
19408
|
+
" This verdict is measured partly on cases the skill may have been tuned against.",
|
|
19409
|
+
opts
|
|
19410
|
+
);
|
|
19411
|
+
}
|
|
19341
19412
|
human("", opts);
|
|
19413
|
+
if (data.selection.set_slug) {
|
|
19414
|
+
human(` Set: ${data.selection.set_slug}`, opts);
|
|
19415
|
+
}
|
|
19342
19416
|
human(` Batch: ${data.run_batch_id}`, opts);
|
|
19343
19417
|
human(` Results: ${absoluteUrl(data.deep_link)}`, opts);
|
|
19344
19418
|
human("", opts);
|
|
@@ -19385,7 +19459,8 @@ async function runEvalAblate(options) {
|
|
|
19385
19459
|
skill_name: options.skill,
|
|
19386
19460
|
hypothesis: options.hypothesis,
|
|
19387
19461
|
candidate_body: candidateBody,
|
|
19388
|
-
...options.brand ? { brand_id: options.brand } : {}
|
|
19462
|
+
...options.brand ? { brand_id: options.brand } : {},
|
|
19463
|
+
...options.set ? { set_id: options.set } : {}
|
|
19389
19464
|
});
|
|
19390
19465
|
} catch (err) {
|
|
19391
19466
|
const e = err;
|
|
@@ -19515,6 +19590,145 @@ var init_eval_test_sets2 = __esm({
|
|
|
19515
19590
|
}
|
|
19516
19591
|
});
|
|
19517
19592
|
|
|
19593
|
+
// src/commands/eval-set-cases.ts
|
|
19594
|
+
var eval_set_cases_exports = {};
|
|
19595
|
+
__export(eval_set_cases_exports, {
|
|
19596
|
+
parseCaseIds: () => parseCaseIds,
|
|
19597
|
+
runEvalSetsAddCases: () => runEvalSetsAddCases,
|
|
19598
|
+
unionCaseIds: () => unionCaseIds
|
|
19599
|
+
});
|
|
19600
|
+
async function requireSession14(opts) {
|
|
19601
|
+
const creds = await readCredentials();
|
|
19602
|
+
if (!creds) {
|
|
19603
|
+
emitError(
|
|
19604
|
+
{
|
|
19605
|
+
code: "not_signed_in",
|
|
19606
|
+
message: "not signed in.",
|
|
19607
|
+
hint: "Run `bamboo-agents login` first."
|
|
19608
|
+
},
|
|
19609
|
+
opts,
|
|
19610
|
+
1
|
|
19611
|
+
);
|
|
19612
|
+
throw new Error("unreachable");
|
|
19613
|
+
}
|
|
19614
|
+
return creds.bamboo_session;
|
|
19615
|
+
}
|
|
19616
|
+
function parseCaseIds(raw) {
|
|
19617
|
+
return raw.split(",").map((s) => s.trim()).filter((s) => s.length > 0);
|
|
19618
|
+
}
|
|
19619
|
+
function unionCaseIds(existing, incoming) {
|
|
19620
|
+
const seen = /* @__PURE__ */ new Set();
|
|
19621
|
+
const out = [];
|
|
19622
|
+
for (const id of [...existing, ...incoming]) {
|
|
19623
|
+
if (seen.has(id)) continue;
|
|
19624
|
+
seen.add(id);
|
|
19625
|
+
out.push(id);
|
|
19626
|
+
}
|
|
19627
|
+
return out;
|
|
19628
|
+
}
|
|
19629
|
+
async function runEvalSetsAddCases(options) {
|
|
19630
|
+
const opts = options;
|
|
19631
|
+
const session = await requireSession14(opts);
|
|
19632
|
+
const incoming = parseCaseIds(options.cases);
|
|
19633
|
+
if (incoming.length === 0) {
|
|
19634
|
+
emitError(
|
|
19635
|
+
{ code: "no_cases", message: "--cases was empty \u2014 nothing to attach." },
|
|
19636
|
+
opts,
|
|
19637
|
+
1
|
|
19638
|
+
);
|
|
19639
|
+
throw new Error("unreachable");
|
|
19640
|
+
}
|
|
19641
|
+
const brandId = await resolveBrandId(
|
|
19642
|
+
session,
|
|
19643
|
+
options.brand,
|
|
19644
|
+
opts,
|
|
19645
|
+
"Specify the brand the test set belongs to with --brand <id>."
|
|
19646
|
+
);
|
|
19647
|
+
let finalIds;
|
|
19648
|
+
let existingCount = 0;
|
|
19649
|
+
if (options.replace) {
|
|
19650
|
+
finalIds = unionCaseIds([], incoming);
|
|
19651
|
+
} else {
|
|
19652
|
+
let sets;
|
|
19653
|
+
try {
|
|
19654
|
+
sets = await getEvalTestSetMemberships(session, brandId);
|
|
19655
|
+
} catch (err) {
|
|
19656
|
+
const e = err;
|
|
19657
|
+
emitError(
|
|
19658
|
+
{
|
|
19659
|
+
code: e.code ?? "roadmap_read_failed",
|
|
19660
|
+
message: `couldn't read the set's current cases: ${e.message ?? "unknown error"}`,
|
|
19661
|
+
hint: "Pass --replace to write the list without reading first."
|
|
19662
|
+
},
|
|
19663
|
+
opts,
|
|
19664
|
+
1
|
|
19665
|
+
);
|
|
19666
|
+
throw new Error("unreachable");
|
|
19667
|
+
}
|
|
19668
|
+
const target = sets.find((s) => s.id === options.set);
|
|
19669
|
+
if (!target) {
|
|
19670
|
+
emitError(
|
|
19671
|
+
{
|
|
19672
|
+
code: "set_not_found",
|
|
19673
|
+
message: `no test set ${options.set} in brand ${brandId}.`,
|
|
19674
|
+
hint: "Check the set id, or pass --brand if the set is in another brand."
|
|
19675
|
+
},
|
|
19676
|
+
opts,
|
|
19677
|
+
1
|
|
19678
|
+
);
|
|
19679
|
+
throw new Error("unreachable");
|
|
19680
|
+
}
|
|
19681
|
+
const existing = target.cases.map((c) => c.id);
|
|
19682
|
+
existingCount = existing.length;
|
|
19683
|
+
finalIds = unionCaseIds(existing, incoming);
|
|
19684
|
+
}
|
|
19685
|
+
try {
|
|
19686
|
+
await updateEvalTestSetCases(session, options.set, {
|
|
19687
|
+
brand_id: brandId,
|
|
19688
|
+
case_ids: finalIds
|
|
19689
|
+
});
|
|
19690
|
+
} catch (err) {
|
|
19691
|
+
const e = err;
|
|
19692
|
+
emitError(
|
|
19693
|
+
{
|
|
19694
|
+
code: e.code ?? "attach_failed",
|
|
19695
|
+
message: e.message ?? "failed to attach the cases."
|
|
19696
|
+
},
|
|
19697
|
+
opts,
|
|
19698
|
+
1
|
|
19699
|
+
);
|
|
19700
|
+
throw new Error("unreachable");
|
|
19701
|
+
}
|
|
19702
|
+
const added = finalIds.length - existingCount;
|
|
19703
|
+
human("", opts);
|
|
19704
|
+
if (options.replace) {
|
|
19705
|
+
human(` Set ${options.set} now holds exactly ${finalIds.length} cases.`, opts);
|
|
19706
|
+
} else {
|
|
19707
|
+
human(
|
|
19708
|
+
` Added ${added} of ${incoming.length} to ${options.set} \u2014 ${finalIds.length} cases total.`,
|
|
19709
|
+
opts
|
|
19710
|
+
);
|
|
19711
|
+
const skipped = incoming.length - added;
|
|
19712
|
+
if (skipped > 0) {
|
|
19713
|
+
human(` ${skipped} ${skipped === 1 ? "was" : "were"} already in the set.`, opts);
|
|
19714
|
+
}
|
|
19715
|
+
}
|
|
19716
|
+
human("", opts);
|
|
19717
|
+
emitSuccess(
|
|
19718
|
+
{ set_id: options.set, case_ids: finalIds, total: finalIds.length, added },
|
|
19719
|
+
opts
|
|
19720
|
+
);
|
|
19721
|
+
}
|
|
19722
|
+
var init_eval_set_cases = __esm({
|
|
19723
|
+
"src/commands/eval-set-cases.ts"() {
|
|
19724
|
+
"use strict";
|
|
19725
|
+
init_credentials();
|
|
19726
|
+
init_api();
|
|
19727
|
+
init_resolve_brand();
|
|
19728
|
+
init_output();
|
|
19729
|
+
}
|
|
19730
|
+
});
|
|
19731
|
+
|
|
19518
19732
|
// node_modules/commander/esm.mjs
|
|
19519
19733
|
var import_index = __toESM(require_commander(), 1);
|
|
19520
19734
|
var {
|
|
@@ -19982,7 +20196,10 @@ evals.command("ablate").description(
|
|
|
19982
20196
|
).requiredOption("--agent <id>", "Agent id whose skill is under test").requiredOption("--skill <name>", "kebab-case skill name (must be one of this agent's own skills)").requiredOption(
|
|
19983
20197
|
"--hypothesis <text>",
|
|
19984
20198
|
"Plain-language claim the edit is meant to produce (an experiment is a claim, not a diff)"
|
|
19985
|
-
).requiredOption("--body <path>", "Path to the candidate SKILL.md").option("--brand <id>", "Brand id \u2014 only needed when the agent spans more than one brand").option(
|
|
20199
|
+
).requiredOption("--body <path>", "Path to the candidate SKILL.md").option("--brand <id>", "Brand id \u2014 only needed when the agent spans more than one brand").option(
|
|
20200
|
+
"--set <id>",
|
|
20201
|
+
"Test set id \u2014 run THIS set's cases instead of the skill-attributed holdout"
|
|
20202
|
+
).option("--json", "Emit a JSON envelope for agent consumption").option("--fields <list>", "Project only these dotted field paths from data (comma-separated)").option("--quiet", "Suppress human chatter (data/errors only)").option("--no-sync", "Skip the auto-sync git pull on this command").action(
|
|
19986
20203
|
async (options) => {
|
|
19987
20204
|
const { runEvalAblate: runEvalAblate2 } = await Promise.resolve().then(() => (init_eval_ab2(), eval_ab_exports2));
|
|
19988
20205
|
await runEvalAblate2(options);
|
|
@@ -20001,6 +20218,15 @@ evalSets.command("create").description("Create an empty test set of a given evid
|
|
|
20001
20218
|
await runEvalSetsCreate2(options);
|
|
20002
20219
|
}
|
|
20003
20220
|
);
|
|
20221
|
+
evalSets.command("add-cases").description("Attach existing cases to a test set (adds by default; --replace overwrites)").requiredOption("--set <id>", "Test set id").requiredOption("--cases <ids>", "Comma-separated eval case ids").option(
|
|
20222
|
+
"--replace",
|
|
20223
|
+
"Send exactly these cases, dropping any others. Without it, the current cases are read and merged"
|
|
20224
|
+
).option("--brand <id>", "Brand id \u2014 only needed when the org has more than one brand").option("--json", "Emit a JSON envelope for agent consumption").option("--fields <list>", "Project only these dotted field paths from data (comma-separated)").option("--quiet", "Suppress human chatter (data/errors only)").option("--no-sync", "Skip the auto-sync git pull on this command").action(
|
|
20225
|
+
async (options) => {
|
|
20226
|
+
const { runEvalSetsAddCases: runEvalSetsAddCases2 } = await Promise.resolve().then(() => (init_eval_set_cases(), eval_set_cases_exports));
|
|
20227
|
+
await runEvalSetsAddCases2(options);
|
|
20228
|
+
}
|
|
20229
|
+
);
|
|
20004
20230
|
(async () => {
|
|
20005
20231
|
const rawArgs = process.argv.slice(2);
|
|
20006
20232
|
const { operands } = program2.parseOptions(rawArgs);
|