bamboo-agents 0.1.6 → 0.1.7-dev.87.b63312a

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -3037,7 +3037,7 @@ var VERSION;
3037
3037
  var init_version = __esm({
3038
3038
  "src/version.ts"() {
3039
3039
  "use strict";
3040
- VERSION = "0.1.6";
3040
+ VERSION = "0.1.7-dev.87.b63312a";
3041
3041
  }
3042
3042
  });
3043
3043
 
@@ -7619,6 +7619,9 @@ var init_eval_ab = __esm({
7619
7619
  ),
7620
7620
  brand_id: external_exports.string().uuid().optional().describe(
7621
7621
  "Only needed to disambiguate an agent assigned to more than one brand. Omitted = the agent's single brand."
7622
+ ),
7623
+ set_id: external_exports.string().uuid().optional().describe(
7624
+ "BAM-4556: run THIS named test set's cases instead of the skill-attributed holdout. The set runs AS AUTHORED \u2014 cases outside the holdout split are reported on the response (`non_holdout_case_count`), never silently dropped and never silently kept quiet. Omitted = skill attribution, the default and SkillOpt's only path."
7622
7625
  )
7623
7626
  });
7624
7627
  EvalAbResponseSchema = external_exports.object({
@@ -7629,8 +7632,31 @@ var init_eval_ab = __esm({
7629
7632
  skill_name: external_exports.string(),
7630
7633
  brand_id: external_exports.string().uuid(),
7631
7634
  agent_id: external_exports.string().uuid(),
7632
- /** Holdout cases the cycle will run over — chosen server-side, never passed in. */
7635
+ /** Cases the cycle will run over — chosen server-side, never passed in. */
7633
7636
  case_count: external_exports.number().int().positive(),
7637
+ /**
7638
+ * BAM-4556 — how those cases were chosen, and what it cost.
7639
+ *
7640
+ * `non_holdout_case_count` is the contamination signal and it is ALWAYS
7641
+ * present, 0 on the attribution path (where holdout is enforced by the query
7642
+ * that picked the cases). A named set runs as authored, so a non-zero value
7643
+ * means this verdict was measured partly on cases the skill may have been
7644
+ * tuned against — reported here, in `plan`, and persisted on every queue row's
7645
+ * payload, because a contaminated result that looks clean is the failure mode
7646
+ * this field exists to prevent.
7647
+ */
7648
+ selection: external_exports.object({
7649
+ /** `attribution` (the default) or `named_set`. */
7650
+ strategy: external_exports.enum(["attribution", "named_set"]),
7651
+ /** The set that was run. Null on the attribution path. */
7652
+ set_id: external_exports.string().uuid().nullable(),
7653
+ /** The set's slug — the same token the `selection_policy` stamp carries. */
7654
+ set_slug: external_exports.string().nullable(),
7655
+ /** The `eval_run_queue.selection_policy` stamp. Null on the attribution path. */
7656
+ selection_policy: external_exports.string().nullable(),
7657
+ /** Cases in the batch whose split is NOT `holdout`. 0 on the attribution path. */
7658
+ non_holdout_case_count: external_exports.number().int().nonnegative()
7659
+ }),
7634
7660
  /** Reps per ARM (not per case). `total_runs = case_count × reps_per_arm × 2`. */
7635
7661
  reps_per_arm: external_exports.number().int().positive(),
7636
7662
  /** Queue rows staged, across BOTH arms. */
@@ -7773,8 +7799,18 @@ function bambooDir() {
7773
7799
  function configPath() {
7774
7800
  return join(bambooDir(), "config.json");
7775
7801
  }
7802
+ function credentialsFileName(version = VERSION, baseUrl = process.env.BAMBOO_BASE_URL) {
7803
+ if (baseUrl) {
7804
+ try {
7805
+ return new URL(baseUrl).hostname === "dev.withbamboo.com" ? "credentials.dev.json" : "credentials.json";
7806
+ } catch {
7807
+ return "credentials.json";
7808
+ }
7809
+ }
7810
+ return version.includes("-dev.") ? "credentials.dev.json" : "credentials.json";
7811
+ }
7776
7812
  function credentialsPath() {
7777
- return join(bambooDir(), "credentials.json");
7813
+ return join(bambooDir(), credentialsFileName());
7778
7814
  }
7779
7815
  function bambooStateDir() {
7780
7816
  if (process.env.BAMBOO_STATE_HOME) return process.env.BAMBOO_STATE_HOME;
@@ -7788,6 +7824,7 @@ function statePath() {
7788
7824
  var init_paths = __esm({
7789
7825
  "src/lib/paths.ts"() {
7790
7826
  "use strict";
7827
+ init_version();
7791
7828
  }
7792
7829
  });
7793
7830
 
@@ -12998,6 +13035,69 @@ var init_episodes2 = __esm({
12998
13035
  }
12999
13036
  });
13000
13037
 
13038
+ // ../../src/lib/api-contracts/eval-roadmap/evidence-kinds.ts
13039
+ var EvidenceKinds, EvidenceKindSchema;
13040
+ var init_evidence_kinds = __esm({
13041
+ "../../src/lib/api-contracts/eval-roadmap/evidence-kinds.ts"() {
13042
+ init_zod();
13043
+ EvidenceKinds = [
13044
+ "golden_set",
13045
+ "quiz",
13046
+ "regression",
13047
+ "probe",
13048
+ "compare"
13049
+ ];
13050
+ EvidenceKindSchema = external_exports.enum(EvidenceKinds);
13051
+ }
13052
+ });
13053
+
13054
+ // ../../src/lib/api-contracts/cli/eval-test-sets.ts
13055
+ var CliEvalTestSetCreateRequestSchema, CliEvalTestSetCreateResponseSchema, CliRoadmapSetMembershipSchema, CliRoadmapMembershipResponseSchema, CliEvalTestSetCasesRequestSchema;
13056
+ var init_eval_test_sets = __esm({
13057
+ "../../src/lib/api-contracts/cli/eval-test-sets.ts"() {
13058
+ init_zod();
13059
+ init_evidence_kinds();
13060
+ CliEvalTestSetCreateRequestSchema = external_exports.object({
13061
+ brand_id: external_exports.string().uuid(),
13062
+ agent_id: external_exports.string().uuid().nullable().optional(),
13063
+ name: external_exports.string().trim().min(1).max(200),
13064
+ slug: external_exports.string().trim().min(1).max(120),
13065
+ hypothesis: external_exports.string().trim().min(1).max(5e3),
13066
+ /**
13067
+ * Required here, optional server-side. A set created without a kind is
13068
+ * `unclassified`, and this verb exists so an agent can create a set *of a
13069
+ * given kind* — leaving it out would silently produce the one outcome the
13070
+ * caller did not ask for.
13071
+ */
13072
+ evidence_kind: EvidenceKindSchema,
13073
+ lifecycle: external_exports.enum(["draft", "active", "paused", "retired"]).default("draft"),
13074
+ priority: external_exports.number().int().default(0)
13075
+ });
13076
+ CliEvalTestSetCreateResponseSchema = external_exports.object({
13077
+ id: external_exports.string().uuid(),
13078
+ name: external_exports.string(),
13079
+ slug: external_exports.string(),
13080
+ brand_id: external_exports.string().uuid(),
13081
+ agent_id: external_exports.string().uuid().nullable(),
13082
+ evidence_kind: EvidenceKindSchema.nullable(),
13083
+ lifecycle: external_exports.string(),
13084
+ priority: external_exports.number().int(),
13085
+ hypothesis: external_exports.string().nullable()
13086
+ });
13087
+ CliRoadmapSetMembershipSchema = external_exports.object({
13088
+ id: external_exports.string().uuid(),
13089
+ cases: external_exports.array(external_exports.object({ id: external_exports.string().uuid() })).default([])
13090
+ });
13091
+ CliRoadmapMembershipResponseSchema = external_exports.array(
13092
+ CliRoadmapSetMembershipSchema
13093
+ );
13094
+ CliEvalTestSetCasesRequestSchema = external_exports.object({
13095
+ brand_id: external_exports.string().uuid(),
13096
+ case_ids: external_exports.array(external_exports.string().uuid())
13097
+ });
13098
+ }
13099
+ });
13100
+
13001
13101
  // src/lib/api.ts
13002
13102
  import { platform as platform2 } from "node:os";
13003
13103
  function defaultApiBase(version) {
@@ -13415,22 +13515,6 @@ async function updateSkillFromRegistry(bambooSession, agentId, skillName) {
13415
13515
  const json = await res.json();
13416
13516
  return SkillSubscriptionsResponseSchema.parse(json.data);
13417
13517
  }
13418
- async function scaffoldSkill(bambooSession, args) {
13419
- const res = await cliApiFetch(
13420
- `${apiBase()}/api/v1/orgs/me/skills/scaffold`,
13421
- {
13422
- method: "POST",
13423
- headers: {
13424
- Authorization: `Bearer ${bambooSession}`,
13425
- "Content-Type": "application/json"
13426
- },
13427
- body: JSON.stringify(args)
13428
- }
13429
- );
13430
- if (!res.ok) throw await parseError(res);
13431
- const json = await res.json();
13432
- return ScaffoldSkillResponseSchema.parse(json.data);
13433
- }
13434
13518
  async function patchBrand(bambooSession, brandId, websiteUrl) {
13435
13519
  const res = await cliApiFetch(
13436
13520
  `${apiBase()}/api/v1/cli/brands/${encodeURIComponent(brandId)}`,
@@ -13512,6 +13596,42 @@ async function startAbTest(bambooSession, body) {
13512
13596
  const json = await res.json();
13513
13597
  return EvalAbResponseSchema.parse(json.data);
13514
13598
  }
13599
+ async function createEvalTestSet(bambooSession, body) {
13600
+ const res = await cliApiFetch(`${apiBase()}/api/v1/eval/test-sets`, {
13601
+ method: "POST",
13602
+ headers: {
13603
+ Authorization: `Bearer ${bambooSession}`,
13604
+ "Content-Type": "application/json"
13605
+ },
13606
+ body: JSON.stringify(body)
13607
+ });
13608
+ if (!res.ok) throw await parseError(res);
13609
+ const json = await res.json();
13610
+ return CliEvalTestSetCreateResponseSchema.parse(json.data);
13611
+ }
13612
+ async function getEvalTestSetMemberships(bambooSession, brandId) {
13613
+ const url = `${apiBase()}/api/v1/eval/test-roadmap?brand_id=${encodeURIComponent(brandId)}`;
13614
+ const res = await cliApiFetch(url, {
13615
+ headers: { Authorization: `Bearer ${bambooSession}` }
13616
+ });
13617
+ if (!res.ok) throw await parseError(res);
13618
+ const json = await res.json();
13619
+ return CliRoadmapMembershipResponseSchema.parse(json.data ?? []);
13620
+ }
13621
+ async function updateEvalTestSetCases(bambooSession, setId, body) {
13622
+ const res = await cliApiFetch(
13623
+ `${apiBase()}/api/v1/eval/test-sets/${encodeURIComponent(setId)}`,
13624
+ {
13625
+ method: "PATCH",
13626
+ headers: {
13627
+ Authorization: `Bearer ${bambooSession}`,
13628
+ "Content-Type": "application/json"
13629
+ },
13630
+ body: JSON.stringify(body)
13631
+ }
13632
+ );
13633
+ if (!res.ok) throw await parseError(res);
13634
+ }
13515
13635
  function absoluteUrl(relativePath) {
13516
13636
  return `${apiBase()}${relativePath}`;
13517
13637
  }
@@ -13528,7 +13648,7 @@ async function postRecEpisode(bambooSession, body) {
13528
13648
  const json = await res.json();
13529
13649
  return CliRecEpisodeResponseSchema.parse(json);
13530
13650
  }
13531
- var PROD_DEFAULT_BASE, DEV_DEFAULT_BASE, REQUEST_TIMEOUT_MS, ApiError, OAuthRelinkStartResponseSchema, ProvisionWorkspaceResponseSchema, FirstAutoInstalledAgentResponseSchema, SkillCatalogEntrySchema, SkillCatalogResponseSchema, SkillSubscriptionsResponseSchema, ScaffoldSkillResponseSchema;
13651
+ var PROD_DEFAULT_BASE, DEV_DEFAULT_BASE, REQUEST_TIMEOUT_MS, ApiError, OAuthRelinkStartResponseSchema, ProvisionWorkspaceResponseSchema, FirstAutoInstalledAgentResponseSchema, SkillCatalogEntrySchema, SkillCatalogResponseSchema, SkillSubscriptionsResponseSchema;
13532
13652
  var init_api = __esm({
13533
13653
  "src/lib/api.ts"() {
13534
13654
  "use strict";
@@ -13554,6 +13674,7 @@ var init_api = __esm({
13554
13674
  init_usage();
13555
13675
  init_episodes2();
13556
13676
  init_eval_ab();
13677
+ init_eval_test_sets();
13557
13678
  init_version();
13558
13679
  PROD_DEFAULT_BASE = "https://withbamboo.com";
13559
13680
  DEV_DEFAULT_BASE = "https://dev.withbamboo.com";
@@ -13598,9 +13719,6 @@ var init_api = __esm({
13598
13719
  SkillSubscriptionsResponseSchema = external_exports.object({
13599
13720
  subscriptions: external_exports.array(external_exports.string())
13600
13721
  });
13601
- ScaffoldSkillResponseSchema = external_exports.object({
13602
- skill_name: external_exports.string()
13603
- });
13604
13722
  }
13605
13723
  });
13606
13724
 
@@ -17382,7 +17500,6 @@ var init_skills_local = __esm({
17382
17500
  var skills_exports = {};
17383
17501
  __export(skills_exports, {
17384
17502
  runSkillsApply: () => runSkillsApply,
17385
- runSkillsCreate: () => runSkillsCreate,
17386
17503
  runSkillsGet: () => runSkillsGet,
17387
17504
  runSkillsList: () => runSkillsList,
17388
17505
  runSkillsSubscribe: () => runSkillsSubscribe,
@@ -17646,45 +17763,6 @@ async function runSubscribeOp(action, agentIdentifier, skillName) {
17646
17763
  console.log(" git -C $(bamboo-agents status) pull");
17647
17764
  console.log("");
17648
17765
  }
17649
- async function runSkillsCreate(skillName, options = {}) {
17650
- return withFreshWorkspace(async () => runSkillsCreateImpl(skillName, options));
17651
- }
17652
- async function runSkillsCreateImpl(skillName, options = {}) {
17653
- if (!skillName || !skillName.trim()) {
17654
- console.error("bamboo-agents: skills create requires a skill name.");
17655
- console.error("Example: bamboo-agents skills create analyze-creative");
17656
- process.exit(2);
17657
- }
17658
- if (!/^[a-z0-9][a-z0-9-]*$/.test(skillName.trim())) {
17659
- console.error(
17660
- "bamboo-agents: skill name must be kebab-case (lowercase + dashes)."
17661
- );
17662
- process.exit(2);
17663
- }
17664
- const bambooSession = await requireSession2();
17665
- let result;
17666
- try {
17667
- result = await scaffoldSkill(bambooSession, {
17668
- skill_name: skillName.trim(),
17669
- category: options.category,
17670
- description: options.description
17671
- });
17672
- } catch (err) {
17673
- console.error(
17674
- `bamboo-agents: could not scaffold ${skillName}: ${err.message}`
17675
- );
17676
- process.exit(1);
17677
- }
17678
- console.log("");
17679
- console.log(` Scaffolded ${result.skill_name}.`);
17680
- console.log("");
17681
- console.log(
17682
- ` Edit skills/${result.skill_name}/SKILL.md in your workspace clone:`
17683
- );
17684
- console.log(" git -C $(bamboo-agents status) pull");
17685
- console.log(` $EDITOR skills/${result.skill_name}/SKILL.md`);
17686
- console.log("");
17687
- }
17688
17766
  async function runSkillsUpdate(agentIdentifier, skillName) {
17689
17767
  return withFreshWorkspace(
17690
17768
  async () => runSkillsUpdateImpl(agentIdentifier, skillName)
@@ -18701,6 +18779,40 @@ var init_brand2 = __esm({
18701
18779
  }
18702
18780
  });
18703
18781
 
18782
+ // src/lib/resolve-brand.ts
18783
+ async function resolveBrandId(session, explicit, opts, hint = "Specify the brand the agent belongs to with --brand <id>.") {
18784
+ if (explicit) return explicit;
18785
+ let brands;
18786
+ try {
18787
+ brands = (await listCliBrands(session)).brands;
18788
+ } catch (err) {
18789
+ emitError(
18790
+ { code: "brands_list_failed", message: `couldn't list brands: ${err.message}` },
18791
+ opts,
18792
+ 1
18793
+ );
18794
+ throw new Error("unreachable");
18795
+ }
18796
+ if (brands.length === 1) return brands[0].id;
18797
+ emitError(
18798
+ {
18799
+ code: "brand_required",
18800
+ message: brands.length === 0 ? "no brands found for your org \u2014 create a brand first." : `multiple brands \u2014 pass --brand <id>. Options: ${brands.map((b) => `${b.name} (${b.id})`).join(", ")}.`,
18801
+ hint
18802
+ },
18803
+ opts,
18804
+ 1
18805
+ );
18806
+ throw new Error("unreachable");
18807
+ }
18808
+ var init_resolve_brand = __esm({
18809
+ "src/lib/resolve-brand.ts"() {
18810
+ "use strict";
18811
+ init_api();
18812
+ init_output();
18813
+ }
18814
+ });
18815
+
18704
18816
  // src/commands/agents.ts
18705
18817
  var agents_exports2 = {};
18706
18818
  __export(agents_exports2, {
@@ -18843,31 +18955,6 @@ async function runAgentsGet(agentRef, options) {
18843
18955
  }
18844
18956
  emitSuccess(agent, opts, (a) => printDetail(a, opts));
18845
18957
  }
18846
- async function resolveBrandId(session, explicit, opts) {
18847
- if (explicit) return explicit;
18848
- let brands;
18849
- try {
18850
- brands = (await listCliBrands(session)).brands;
18851
- } catch (err) {
18852
- emitError(
18853
- { code: "brands_list_failed", message: `couldn't list brands: ${err.message}` },
18854
- opts,
18855
- 1
18856
- );
18857
- throw new Error("unreachable");
18858
- }
18859
- if (brands.length === 1) return brands[0].id;
18860
- emitError(
18861
- {
18862
- code: "brand_required",
18863
- message: brands.length === 0 ? "no brands found for your org \u2014 create a brand first." : `multiple brands \u2014 pass --brand <id>. Options: ${brands.map((b) => `${b.name} (${b.id})`).join(", ")}.`,
18864
- hint: "Specify the brand the agent belongs to with --brand <id>."
18865
- },
18866
- opts,
18867
- 1
18868
- );
18869
- throw new Error("unreachable");
18870
- }
18871
18958
  async function runAgentsProvision(name, options) {
18872
18959
  const opts = options;
18873
18960
  const session = await requireSession10(opts);
@@ -19081,6 +19168,7 @@ var init_agents2 = __esm({
19081
19168
  "src/commands/agents.ts"() {
19082
19169
  "use strict";
19083
19170
  init_credentials();
19171
+ init_resolve_brand();
19084
19172
  init_api();
19085
19173
  init_output();
19086
19174
  }
@@ -19262,7 +19350,21 @@ function printStaged(data, opts) {
19262
19350
  ` ${data.case_count} cases \xD7 ${data.reps_per_arm} reps \xD7 2 arms = ${data.total_runs} queued runs.`,
19263
19351
  opts
19264
19352
  );
19353
+ if (data.selection.non_holdout_case_count > 0) {
19354
+ human("", opts);
19355
+ human(
19356
+ ` \u26A0 ${data.selection.non_holdout_case_count} of these cases are NOT in the holdout split.`,
19357
+ opts
19358
+ );
19359
+ human(
19360
+ " This verdict is measured partly on cases the skill may have been tuned against.",
19361
+ opts
19362
+ );
19363
+ }
19265
19364
  human("", opts);
19365
+ if (data.selection.set_slug) {
19366
+ human(` Set: ${data.selection.set_slug}`, opts);
19367
+ }
19266
19368
  human(` Batch: ${data.run_batch_id}`, opts);
19267
19369
  human(` Results: ${absoluteUrl(data.deep_link)}`, opts);
19268
19370
  human("", opts);
@@ -19309,7 +19411,8 @@ async function runEvalAblate(options) {
19309
19411
  skill_name: options.skill,
19310
19412
  hypothesis: options.hypothesis,
19311
19413
  candidate_body: candidateBody,
19312
- ...options.brand ? { brand_id: options.brand } : {}
19414
+ ...options.brand ? { brand_id: options.brand } : {},
19415
+ ...options.set ? { set_id: options.set } : {}
19313
19416
  });
19314
19417
  } catch (err) {
19315
19418
  const e = err;
@@ -19331,6 +19434,253 @@ var init_eval_ab2 = __esm({
19331
19434
  }
19332
19435
  });
19333
19436
 
19437
+ // src/commands/eval-test-sets.ts
19438
+ var eval_test_sets_exports = {};
19439
+ __export(eval_test_sets_exports, {
19440
+ runEvalSetsCreate: () => runEvalSetsCreate,
19441
+ slugifyTestSetName: () => slugifyTestSetName
19442
+ });
19443
+ async function requireSession13(opts) {
19444
+ const creds = await readCredentials();
19445
+ if (!creds) {
19446
+ emitError(
19447
+ {
19448
+ code: "not_signed_in",
19449
+ message: "not signed in.",
19450
+ hint: "Run `bamboo-agents login` first."
19451
+ },
19452
+ opts,
19453
+ 1
19454
+ );
19455
+ throw new Error("unreachable");
19456
+ }
19457
+ return creds.bamboo_session;
19458
+ }
19459
+ function slugifyTestSetName(name) {
19460
+ return name.trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 120);
19461
+ }
19462
+ function printCreated(data, opts) {
19463
+ human("", opts);
19464
+ human(` Created "${data.name}" (${data.evidence_kind ?? "unclassified"}).`, opts);
19465
+ human("", opts);
19466
+ human(` Set: ${data.id}`, opts);
19467
+ human(` Slug: ${data.slug}`, opts);
19468
+ human(` Lifecycle: ${data.lifecycle}`, opts);
19469
+ human("", opts);
19470
+ human(" No cases yet \u2014 a set with none will not run. Add them next.", opts);
19471
+ if (data.evidence_kind === "compare") {
19472
+ human("", opts);
19473
+ human(
19474
+ " This classifies the set \u2014 it does not start a comparison.",
19475
+ opts
19476
+ );
19477
+ human(" Ablations run per skill: `bamboo-agents eval ablate --help`.", opts);
19478
+ }
19479
+ human("", opts);
19480
+ }
19481
+ async function runEvalSetsCreate(options) {
19482
+ const opts = options;
19483
+ const session = await requireSession13(opts);
19484
+ const brandId = await resolveBrandId(
19485
+ session,
19486
+ options.brand,
19487
+ opts,
19488
+ "Specify the brand this test set belongs to with --brand <id>."
19489
+ );
19490
+ let priority = 0;
19491
+ if (options.priority !== void 0) {
19492
+ priority = Number.parseInt(options.priority, 10);
19493
+ if (!Number.isFinite(priority)) {
19494
+ emitError(
19495
+ {
19496
+ code: "invalid_priority",
19497
+ message: `--priority must be an integer, got "${options.priority}".`
19498
+ },
19499
+ opts,
19500
+ 1
19501
+ );
19502
+ throw new Error("unreachable");
19503
+ }
19504
+ }
19505
+ let created;
19506
+ try {
19507
+ created = await createEvalTestSet(session, {
19508
+ brand_id: brandId,
19509
+ agent_id: options.agent ?? null,
19510
+ name: options.name,
19511
+ slug: options.slug?.trim() || slugifyTestSetName(options.name),
19512
+ hypothesis: options.hypothesis,
19513
+ // Cast, not validate: the server's schema is the authority on the kind
19514
+ // vocabulary, and a second check here would be the fork the parity test
19515
+ // exists to prevent. An unknown kind comes back as a 422 naming it.
19516
+ evidence_kind: options.kind,
19517
+ lifecycle: options.lifecycle ?? "draft",
19518
+ priority
19519
+ });
19520
+ } catch (err) {
19521
+ const e = err;
19522
+ emitError(
19523
+ {
19524
+ code: e.code ?? "create_failed",
19525
+ message: e.message ?? "failed to create the test set."
19526
+ },
19527
+ opts,
19528
+ 1
19529
+ );
19530
+ throw new Error("unreachable");
19531
+ }
19532
+ printCreated(created, opts);
19533
+ emitSuccess(created, opts);
19534
+ }
19535
+ var init_eval_test_sets2 = __esm({
19536
+ "src/commands/eval-test-sets.ts"() {
19537
+ "use strict";
19538
+ init_credentials();
19539
+ init_api();
19540
+ init_resolve_brand();
19541
+ init_output();
19542
+ }
19543
+ });
19544
+
19545
+ // src/commands/eval-set-cases.ts
19546
+ var eval_set_cases_exports = {};
19547
+ __export(eval_set_cases_exports, {
19548
+ parseCaseIds: () => parseCaseIds,
19549
+ runEvalSetsAddCases: () => runEvalSetsAddCases,
19550
+ unionCaseIds: () => unionCaseIds
19551
+ });
19552
+ async function requireSession14(opts) {
19553
+ const creds = await readCredentials();
19554
+ if (!creds) {
19555
+ emitError(
19556
+ {
19557
+ code: "not_signed_in",
19558
+ message: "not signed in.",
19559
+ hint: "Run `bamboo-agents login` first."
19560
+ },
19561
+ opts,
19562
+ 1
19563
+ );
19564
+ throw new Error("unreachable");
19565
+ }
19566
+ return creds.bamboo_session;
19567
+ }
19568
+ function parseCaseIds(raw) {
19569
+ return raw.split(",").map((s) => s.trim()).filter((s) => s.length > 0);
19570
+ }
19571
+ function unionCaseIds(existing, incoming) {
19572
+ const seen = /* @__PURE__ */ new Set();
19573
+ const out = [];
19574
+ for (const id of [...existing, ...incoming]) {
19575
+ if (seen.has(id)) continue;
19576
+ seen.add(id);
19577
+ out.push(id);
19578
+ }
19579
+ return out;
19580
+ }
19581
+ async function runEvalSetsAddCases(options) {
19582
+ const opts = options;
19583
+ const session = await requireSession14(opts);
19584
+ const incoming = parseCaseIds(options.cases);
19585
+ if (incoming.length === 0) {
19586
+ emitError(
19587
+ { code: "no_cases", message: "--cases was empty \u2014 nothing to attach." },
19588
+ opts,
19589
+ 1
19590
+ );
19591
+ throw new Error("unreachable");
19592
+ }
19593
+ const brandId = await resolveBrandId(
19594
+ session,
19595
+ options.brand,
19596
+ opts,
19597
+ "Specify the brand the test set belongs to with --brand <id>."
19598
+ );
19599
+ let finalIds;
19600
+ let existingCount = 0;
19601
+ if (options.replace) {
19602
+ finalIds = unionCaseIds([], incoming);
19603
+ } else {
19604
+ let sets;
19605
+ try {
19606
+ sets = await getEvalTestSetMemberships(session, brandId);
19607
+ } catch (err) {
19608
+ const e = err;
19609
+ emitError(
19610
+ {
19611
+ code: e.code ?? "roadmap_read_failed",
19612
+ message: `couldn't read the set's current cases: ${e.message ?? "unknown error"}`,
19613
+ hint: "Pass --replace to write the list without reading first."
19614
+ },
19615
+ opts,
19616
+ 1
19617
+ );
19618
+ throw new Error("unreachable");
19619
+ }
19620
+ const target = sets.find((s) => s.id === options.set);
19621
+ if (!target) {
19622
+ emitError(
19623
+ {
19624
+ code: "set_not_found",
19625
+ message: `no test set ${options.set} in brand ${brandId}.`,
19626
+ hint: "Check the set id, or pass --brand if the set is in another brand."
19627
+ },
19628
+ opts,
19629
+ 1
19630
+ );
19631
+ throw new Error("unreachable");
19632
+ }
19633
+ const existing = target.cases.map((c) => c.id);
19634
+ existingCount = existing.length;
19635
+ finalIds = unionCaseIds(existing, incoming);
19636
+ }
19637
+ try {
19638
+ await updateEvalTestSetCases(session, options.set, {
19639
+ brand_id: brandId,
19640
+ case_ids: finalIds
19641
+ });
19642
+ } catch (err) {
19643
+ const e = err;
19644
+ emitError(
19645
+ {
19646
+ code: e.code ?? "attach_failed",
19647
+ message: e.message ?? "failed to attach the cases."
19648
+ },
19649
+ opts,
19650
+ 1
19651
+ );
19652
+ throw new Error("unreachable");
19653
+ }
19654
+ const added = finalIds.length - existingCount;
19655
+ human("", opts);
19656
+ if (options.replace) {
19657
+ human(` Set ${options.set} now holds exactly ${finalIds.length} cases.`, opts);
19658
+ } else {
19659
+ human(
19660
+ ` Added ${added} of ${incoming.length} to ${options.set} \u2014 ${finalIds.length} cases total.`,
19661
+ opts
19662
+ );
19663
+ const skipped = incoming.length - added;
19664
+ if (skipped > 0) {
19665
+ human(` ${skipped} ${skipped === 1 ? "was" : "were"} already in the set.`, opts);
19666
+ }
19667
+ }
19668
+ human("", opts);
19669
+ emitSuccess(
19670
+ { set_id: options.set, case_ids: finalIds, total: finalIds.length, added },
19671
+ opts
19672
+ );
19673
+ }
19674
+ var init_eval_set_cases = __esm({
19675
+ "src/commands/eval-set-cases.ts"() {
19676
+ "use strict";
19677
+ init_credentials();
19678
+ init_api();
19679
+ init_resolve_brand();
19680
+ init_output();
19681
+ }
19682
+ });
19683
+
19334
19684
  // node_modules/commander/esm.mjs
19335
19685
  var import_index = __toESM(require_commander(), 1);
19336
19686
  var {
@@ -19648,12 +19998,6 @@ skills.command("unsubscribe <agent> <skill-name>").description("Unsubscribe an a
19648
19998
  const { runSkillsUnsubscribe: runSkillsUnsubscribe2 } = await Promise.resolve().then(() => (init_skills(), skills_exports));
19649
19999
  await runSkillsUnsubscribe2(agent, skillName);
19650
20000
  });
19651
- skills.command("create <skill-name>").description("Scaffold a new org-custom skill in your workspace").option("--category <category>", "Optional category label (e.g. analysis)").option("--description <description>", "Optional one-line description").option("--no-sync", "Skip the auto-sync git pull on this command").action(
19652
- async (skillName, options) => {
19653
- const { runSkillsCreate: runSkillsCreate2 } = await Promise.resolve().then(() => (init_skills(), skills_exports));
19654
- await runSkillsCreate2(skillName, options);
19655
- }
19656
- );
19657
20001
  skills.command("update <agent> <skill-name>").description("Update an installed skill from the bamboo registry").option("--no-sync", "Skip the auto-sync git pull on this command").action(async (agent, skillName) => {
19658
20002
  const { runSkillsUpdate: runSkillsUpdate2 } = await Promise.resolve().then(() => (init_skills(), skills_exports));
19659
20003
  await runSkillsUpdate2(agent, skillName);
@@ -19798,12 +20142,37 @@ evals.command("ablate").description(
19798
20142
  ).requiredOption("--agent <id>", "Agent id whose skill is under test").requiredOption("--skill <name>", "kebab-case skill name (must be one of this agent's own skills)").requiredOption(
19799
20143
  "--hypothesis <text>",
19800
20144
  "Plain-language claim the edit is meant to produce (an experiment is a claim, not a diff)"
19801
- ).requiredOption("--body <path>", "Path to the candidate SKILL.md").option("--brand <id>", "Brand id \u2014 only needed when the agent spans more than one brand").option("--json", "Emit a JSON envelope for agent consumption").option("--fields <list>", "Project only these dotted field paths from data (comma-separated)").option("--quiet", "Suppress human chatter (data/errors only)").option("--no-sync", "Skip the auto-sync git pull on this command").action(
20145
+ ).requiredOption("--body <path>", "Path to the candidate SKILL.md").option("--brand <id>", "Brand id \u2014 only needed when the agent spans more than one brand").option(
20146
+ "--set <id>",
20147
+ "Test set id \u2014 run THIS set's cases instead of the skill-attributed holdout"
20148
+ ).option("--json", "Emit a JSON envelope for agent consumption").option("--fields <list>", "Project only these dotted field paths from data (comma-separated)").option("--quiet", "Suppress human chatter (data/errors only)").option("--no-sync", "Skip the auto-sync git pull on this command").action(
19802
20149
  async (options) => {
19803
20150
  const { runEvalAblate: runEvalAblate2 } = await Promise.resolve().then(() => (init_eval_ab2(), eval_ab_exports2));
19804
20151
  await runEvalAblate2(options);
19805
20152
  }
19806
20153
  );
20154
+ var evalSets = evals.command("sets").description("Test sets \u2014 the definition a run executes (cases attach separately)");
20155
+ evalSets.command("create").description("Create an empty test set of a given evidence kind").requiredOption("--name <text>", 'Human name, e.g. "Connector parity"').requiredOption(
20156
+ "--kind <kind>",
20157
+ "Evidence kind: golden_set | quiz | regression | probe | compare"
20158
+ ).requiredOption(
20159
+ "--hypothesis <text>",
20160
+ "The claim this set exists to settle \u2014 what the kind makes a good hypothesis differs"
20161
+ ).option("--slug <slug>", "URL slug (derived from --name when omitted)").option("--agent <id>", "Scope the set to one agent (omit for brand-wide)").option("--brand <id>", "Brand id \u2014 only needed when the org has more than one brand").option("--lifecycle <state>", "draft | active | paused | retired (default: draft)").option("--priority <n>", "Integer ordering hint (default: 0)").option("--json", "Emit a JSON envelope for agent consumption").option("--fields <list>", "Project only these dotted field paths from data (comma-separated)").option("--quiet", "Suppress human chatter (data/errors only)").option("--no-sync", "Skip the auto-sync git pull on this command").action(
20162
+ async (options) => {
20163
+ const { runEvalSetsCreate: runEvalSetsCreate2 } = await Promise.resolve().then(() => (init_eval_test_sets2(), eval_test_sets_exports));
20164
+ await runEvalSetsCreate2(options);
20165
+ }
20166
+ );
20167
+ evalSets.command("add-cases").description("Attach existing cases to a test set (adds by default; --replace overwrites)").requiredOption("--set <id>", "Test set id").requiredOption("--cases <ids>", "Comma-separated eval case ids").option(
20168
+ "--replace",
20169
+ "Send exactly these cases, dropping any others. Without it, the current cases are read and merged"
20170
+ ).option("--brand <id>", "Brand id \u2014 only needed when the org has more than one brand").option("--json", "Emit a JSON envelope for agent consumption").option("--fields <list>", "Project only these dotted field paths from data (comma-separated)").option("--quiet", "Suppress human chatter (data/errors only)").option("--no-sync", "Skip the auto-sync git pull on this command").action(
20171
+ async (options) => {
20172
+ const { runEvalSetsAddCases: runEvalSetsAddCases2 } = await Promise.resolve().then(() => (init_eval_set_cases(), eval_set_cases_exports));
20173
+ await runEvalSetsAddCases2(options);
20174
+ }
20175
+ );
19807
20176
  (async () => {
19808
20177
  const rawArgs = process.argv.slice(2);
19809
20178
  const { operands } = program2.parseOptions(rawArgs);