bamboo-agents 0.1.6 → 0.1.7-dev.85.44cc015

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/VERSION CHANGED
@@ -1 +1 @@
1
- 0.1.6
1
+ 0.1.7-dev.85.44cc015
package/dist/index.js CHANGED
@@ -3037,7 +3037,7 @@ var VERSION;
3037
3037
  var init_version = __esm({
3038
3038
  "src/version.ts"() {
3039
3039
  "use strict";
3040
- VERSION = "0.1.6";
3040
+ VERSION = "0.1.7-dev.85.44cc015";
3041
3041
  }
3042
3042
  });
3043
3043
 
@@ -7619,6 +7619,9 @@ var init_eval_ab = __esm({
7619
7619
  ),
7620
7620
  brand_id: external_exports.string().uuid().optional().describe(
7621
7621
  "Only needed to disambiguate an agent assigned to more than one brand. Omitted = the agent's single brand."
7622
+ ),
7623
+ set_id: external_exports.string().uuid().optional().describe(
7624
+ "BAM-4556: run THIS named test set's cases instead of the skill-attributed holdout. The set runs AS AUTHORED \u2014 cases outside the holdout split are reported on the response (`non_holdout_case_count`), never silently dropped and never silently kept quiet. Omitted = skill attribution, the default and SkillOpt's only path."
7622
7625
  )
7623
7626
  });
7624
7627
  EvalAbResponseSchema = external_exports.object({
@@ -7629,8 +7632,31 @@ var init_eval_ab = __esm({
7629
7632
  skill_name: external_exports.string(),
7630
7633
  brand_id: external_exports.string().uuid(),
7631
7634
  agent_id: external_exports.string().uuid(),
7632
- /** Holdout cases the cycle will run over — chosen server-side, never passed in. */
7635
+ /** Cases the cycle will run over — chosen server-side, never passed in. */
7633
7636
  case_count: external_exports.number().int().positive(),
7637
+ /**
7638
+ * BAM-4556 — how those cases were chosen, and what it cost.
7639
+ *
7640
+ * `non_holdout_case_count` is the contamination signal and it is ALWAYS
7641
+ * present, 0 on the attribution path (where holdout is enforced by the query
7642
+ * that picked the cases). A named set runs as authored, so a non-zero value
7643
+ * means this verdict was measured partly on cases the skill may have been
7644
+ * tuned against — reported here, in `plan`, and persisted on every queue row's
7645
+ * payload, because a contaminated result that looks clean is the failure mode
7646
+ * this field exists to prevent.
7647
+ */
7648
+ selection: external_exports.object({
7649
+ /** `attribution` (the default) or `named_set`. */
7650
+ strategy: external_exports.enum(["attribution", "named_set"]),
7651
+ /** The set that was run. Null on the attribution path. */
7652
+ set_id: external_exports.string().uuid().nullable(),
7653
+ /** The set's slug — the same token the `selection_policy` stamp carries. */
7654
+ set_slug: external_exports.string().nullable(),
7655
+ /** The `eval_run_queue.selection_policy` stamp. Null on the attribution path. */
7656
+ selection_policy: external_exports.string().nullable(),
7657
+ /** Cases in the batch whose split is NOT `holdout`. 0 on the attribution path. */
7658
+ non_holdout_case_count: external_exports.number().int().nonnegative()
7659
+ }),
7634
7660
  /** Reps per ARM (not per case). `total_runs = case_count × reps_per_arm × 2`. */
7635
7661
  reps_per_arm: external_exports.number().int().positive(),
7636
7662
  /** Queue rows staged, across BOTH arms. */
@@ -7773,8 +7799,18 @@ function bambooDir() {
7773
7799
  function configPath() {
7774
7800
  return join(bambooDir(), "config.json");
7775
7801
  }
7802
+ function credentialsFileName(version = VERSION, baseUrl = process.env.BAMBOO_BASE_URL) {
7803
+ if (baseUrl) {
7804
+ try {
7805
+ return new URL(baseUrl).hostname === "dev.withbamboo.com" ? "credentials.dev.json" : "credentials.json";
7806
+ } catch {
7807
+ return "credentials.json";
7808
+ }
7809
+ }
7810
+ return version.includes("-dev.") ? "credentials.dev.json" : "credentials.json";
7811
+ }
7776
7812
  function credentialsPath() {
7777
- return join(bambooDir(), "credentials.json");
7813
+ return join(bambooDir(), credentialsFileName());
7778
7814
  }
7779
7815
  function bambooStateDir() {
7780
7816
  if (process.env.BAMBOO_STATE_HOME) return process.env.BAMBOO_STATE_HOME;
@@ -7788,6 +7824,7 @@ function statePath() {
7788
7824
  var init_paths = __esm({
7789
7825
  "src/lib/paths.ts"() {
7790
7826
  "use strict";
7827
+ init_version();
7791
7828
  }
7792
7829
  });
7793
7830
 
@@ -12998,6 +13035,69 @@ var init_episodes2 = __esm({
12998
13035
  }
12999
13036
  });
13000
13037
 
13038
+ // ../../src/lib/api-contracts/eval-roadmap/evidence-kinds.ts
13039
+ var EvidenceKinds, EvidenceKindSchema;
13040
+ var init_evidence_kinds = __esm({
13041
+ "../../src/lib/api-contracts/eval-roadmap/evidence-kinds.ts"() {
13042
+ init_zod();
13043
+ EvidenceKinds = [
13044
+ "golden_set",
13045
+ "quiz",
13046
+ "regression",
13047
+ "probe",
13048
+ "compare"
13049
+ ];
13050
+ EvidenceKindSchema = external_exports.enum(EvidenceKinds);
13051
+ }
13052
+ });
13053
+
13054
+ // ../../src/lib/api-contracts/cli/eval-test-sets.ts
13055
+ var CliEvalTestSetCreateRequestSchema, CliEvalTestSetCreateResponseSchema, CliRoadmapSetMembershipSchema, CliRoadmapMembershipResponseSchema, CliEvalTestSetCasesRequestSchema;
13056
+ var init_eval_test_sets = __esm({
13057
+ "../../src/lib/api-contracts/cli/eval-test-sets.ts"() {
13058
+ init_zod();
13059
+ init_evidence_kinds();
13060
+ CliEvalTestSetCreateRequestSchema = external_exports.object({
13061
+ brand_id: external_exports.string().uuid(),
13062
+ agent_id: external_exports.string().uuid().nullable().optional(),
13063
+ name: external_exports.string().trim().min(1).max(200),
13064
+ slug: external_exports.string().trim().min(1).max(120),
13065
+ hypothesis: external_exports.string().trim().min(1).max(5e3),
13066
+ /**
13067
+ * Required here, optional server-side. A set created without a kind is
13068
+ * `unclassified`, and this verb exists so an agent can create a set *of a
13069
+ * given kind* — leaving it out would silently produce the one outcome the
13070
+ * caller did not ask for.
13071
+ */
13072
+ evidence_kind: EvidenceKindSchema,
13073
+ lifecycle: external_exports.enum(["draft", "active", "paused", "retired"]).default("draft"),
13074
+ priority: external_exports.number().int().default(0)
13075
+ });
13076
+ CliEvalTestSetCreateResponseSchema = external_exports.object({
13077
+ id: external_exports.string().uuid(),
13078
+ name: external_exports.string(),
13079
+ slug: external_exports.string(),
13080
+ brand_id: external_exports.string().uuid(),
13081
+ agent_id: external_exports.string().uuid().nullable(),
13082
+ evidence_kind: EvidenceKindSchema.nullable(),
13083
+ lifecycle: external_exports.string(),
13084
+ priority: external_exports.number().int(),
13085
+ hypothesis: external_exports.string().nullable()
13086
+ });
13087
+ CliRoadmapSetMembershipSchema = external_exports.object({
13088
+ id: external_exports.string().uuid(),
13089
+ cases: external_exports.array(external_exports.object({ id: external_exports.string().uuid() })).default([])
13090
+ });
13091
+ CliRoadmapMembershipResponseSchema = external_exports.array(
13092
+ CliRoadmapSetMembershipSchema
13093
+ );
13094
+ CliEvalTestSetCasesRequestSchema = external_exports.object({
13095
+ brand_id: external_exports.string().uuid(),
13096
+ case_ids: external_exports.array(external_exports.string().uuid())
13097
+ });
13098
+ }
13099
+ });
13100
+
13001
13101
  // src/lib/api.ts
13002
13102
  import { platform as platform2 } from "node:os";
13003
13103
  function defaultApiBase(version) {
@@ -13512,6 +13612,42 @@ async function startAbTest(bambooSession, body) {
13512
13612
  const json = await res.json();
13513
13613
  return EvalAbResponseSchema.parse(json.data);
13514
13614
  }
13615
+ async function createEvalTestSet(bambooSession, body) {
13616
+ const res = await cliApiFetch(`${apiBase()}/api/v1/eval/test-sets`, {
13617
+ method: "POST",
13618
+ headers: {
13619
+ Authorization: `Bearer ${bambooSession}`,
13620
+ "Content-Type": "application/json"
13621
+ },
13622
+ body: JSON.stringify(body)
13623
+ });
13624
+ if (!res.ok) throw await parseError(res);
13625
+ const json = await res.json();
13626
+ return CliEvalTestSetCreateResponseSchema.parse(json.data);
13627
+ }
13628
+ async function getEvalTestSetMemberships(bambooSession, brandId) {
13629
+ const url = `${apiBase()}/api/v1/eval/test-roadmap?brand_id=${encodeURIComponent(brandId)}`;
13630
+ const res = await cliApiFetch(url, {
13631
+ headers: { Authorization: `Bearer ${bambooSession}` }
13632
+ });
13633
+ if (!res.ok) throw await parseError(res);
13634
+ const json = await res.json();
13635
+ return CliRoadmapMembershipResponseSchema.parse(json.data ?? []);
13636
+ }
13637
+ async function updateEvalTestSetCases(bambooSession, setId, body) {
13638
+ const res = await cliApiFetch(
13639
+ `${apiBase()}/api/v1/eval/test-sets/${encodeURIComponent(setId)}`,
13640
+ {
13641
+ method: "PATCH",
13642
+ headers: {
13643
+ Authorization: `Bearer ${bambooSession}`,
13644
+ "Content-Type": "application/json"
13645
+ },
13646
+ body: JSON.stringify(body)
13647
+ }
13648
+ );
13649
+ if (!res.ok) throw await parseError(res);
13650
+ }
13515
13651
  function absoluteUrl(relativePath) {
13516
13652
  return `${apiBase()}${relativePath}`;
13517
13653
  }
@@ -13554,6 +13690,7 @@ var init_api = __esm({
13554
13690
  init_usage();
13555
13691
  init_episodes2();
13556
13692
  init_eval_ab();
13693
+ init_eval_test_sets();
13557
13694
  init_version();
13558
13695
  PROD_DEFAULT_BASE = "https://withbamboo.com";
13559
13696
  DEV_DEFAULT_BASE = "https://dev.withbamboo.com";
@@ -18701,6 +18838,40 @@ var init_brand2 = __esm({
18701
18838
  }
18702
18839
  });
18703
18840
 
18841
+ // src/lib/resolve-brand.ts
18842
+ async function resolveBrandId(session, explicit, opts, hint = "Specify the brand the agent belongs to with --brand <id>.") {
18843
+ if (explicit) return explicit;
18844
+ let brands;
18845
+ try {
18846
+ brands = (await listCliBrands(session)).brands;
18847
+ } catch (err) {
18848
+ emitError(
18849
+ { code: "brands_list_failed", message: `couldn't list brands: ${err.message}` },
18850
+ opts,
18851
+ 1
18852
+ );
18853
+ throw new Error("unreachable");
18854
+ }
18855
+ if (brands.length === 1) return brands[0].id;
18856
+ emitError(
18857
+ {
18858
+ code: "brand_required",
18859
+ message: brands.length === 0 ? "no brands found for your org \u2014 create a brand first." : `multiple brands \u2014 pass --brand <id>. Options: ${brands.map((b) => `${b.name} (${b.id})`).join(", ")}.`,
18860
+ hint
18861
+ },
18862
+ opts,
18863
+ 1
18864
+ );
18865
+ throw new Error("unreachable");
18866
+ }
18867
+ var init_resolve_brand = __esm({
18868
+ "src/lib/resolve-brand.ts"() {
18869
+ "use strict";
18870
+ init_api();
18871
+ init_output();
18872
+ }
18873
+ });
18874
+
18704
18875
  // src/commands/agents.ts
18705
18876
  var agents_exports2 = {};
18706
18877
  __export(agents_exports2, {
@@ -18843,31 +19014,6 @@ async function runAgentsGet(agentRef, options) {
18843
19014
  }
18844
19015
  emitSuccess(agent, opts, (a) => printDetail(a, opts));
18845
19016
  }
18846
- async function resolveBrandId(session, explicit, opts) {
18847
- if (explicit) return explicit;
18848
- let brands;
18849
- try {
18850
- brands = (await listCliBrands(session)).brands;
18851
- } catch (err) {
18852
- emitError(
18853
- { code: "brands_list_failed", message: `couldn't list brands: ${err.message}` },
18854
- opts,
18855
- 1
18856
- );
18857
- throw new Error("unreachable");
18858
- }
18859
- if (brands.length === 1) return brands[0].id;
18860
- emitError(
18861
- {
18862
- code: "brand_required",
18863
- message: brands.length === 0 ? "no brands found for your org \u2014 create a brand first." : `multiple brands \u2014 pass --brand <id>. Options: ${brands.map((b) => `${b.name} (${b.id})`).join(", ")}.`,
18864
- hint: "Specify the brand the agent belongs to with --brand <id>."
18865
- },
18866
- opts,
18867
- 1
18868
- );
18869
- throw new Error("unreachable");
18870
- }
18871
19017
  async function runAgentsProvision(name, options) {
18872
19018
  const opts = options;
18873
19019
  const session = await requireSession10(opts);
@@ -19081,6 +19227,7 @@ var init_agents2 = __esm({
19081
19227
  "src/commands/agents.ts"() {
19082
19228
  "use strict";
19083
19229
  init_credentials();
19230
+ init_resolve_brand();
19084
19231
  init_api();
19085
19232
  init_output();
19086
19233
  }
@@ -19262,7 +19409,21 @@ function printStaged(data, opts) {
19262
19409
  ` ${data.case_count} cases \xD7 ${data.reps_per_arm} reps \xD7 2 arms = ${data.total_runs} queued runs.`,
19263
19410
  opts
19264
19411
  );
19412
+ if (data.selection.non_holdout_case_count > 0) {
19413
+ human("", opts);
19414
+ human(
19415
+ ` \u26A0 ${data.selection.non_holdout_case_count} of these cases are NOT in the holdout split.`,
19416
+ opts
19417
+ );
19418
+ human(
19419
+ " This verdict is measured partly on cases the skill may have been tuned against.",
19420
+ opts
19421
+ );
19422
+ }
19265
19423
  human("", opts);
19424
+ if (data.selection.set_slug) {
19425
+ human(` Set: ${data.selection.set_slug}`, opts);
19426
+ }
19266
19427
  human(` Batch: ${data.run_batch_id}`, opts);
19267
19428
  human(` Results: ${absoluteUrl(data.deep_link)}`, opts);
19268
19429
  human("", opts);
@@ -19309,7 +19470,8 @@ async function runEvalAblate(options) {
19309
19470
  skill_name: options.skill,
19310
19471
  hypothesis: options.hypothesis,
19311
19472
  candidate_body: candidateBody,
19312
- ...options.brand ? { brand_id: options.brand } : {}
19473
+ ...options.brand ? { brand_id: options.brand } : {},
19474
+ ...options.set ? { set_id: options.set } : {}
19313
19475
  });
19314
19476
  } catch (err) {
19315
19477
  const e = err;
@@ -19331,6 +19493,253 @@ var init_eval_ab2 = __esm({
19331
19493
  }
19332
19494
  });
19333
19495
 
19496
+ // src/commands/eval-test-sets.ts
19497
+ var eval_test_sets_exports = {};
19498
+ __export(eval_test_sets_exports, {
19499
+ runEvalSetsCreate: () => runEvalSetsCreate,
19500
+ slugifyTestSetName: () => slugifyTestSetName
19501
+ });
19502
+ async function requireSession13(opts) {
19503
+ const creds = await readCredentials();
19504
+ if (!creds) {
19505
+ emitError(
19506
+ {
19507
+ code: "not_signed_in",
19508
+ message: "not signed in.",
19509
+ hint: "Run `bamboo-agents login` first."
19510
+ },
19511
+ opts,
19512
+ 1
19513
+ );
19514
+ throw new Error("unreachable");
19515
+ }
19516
+ return creds.bamboo_session;
19517
+ }
19518
+ function slugifyTestSetName(name) {
19519
+ return name.trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 120);
19520
+ }
19521
+ function printCreated(data, opts) {
19522
+ human("", opts);
19523
+ human(` Created "${data.name}" (${data.evidence_kind ?? "unclassified"}).`, opts);
19524
+ human("", opts);
19525
+ human(` Set: ${data.id}`, opts);
19526
+ human(` Slug: ${data.slug}`, opts);
19527
+ human(` Lifecycle: ${data.lifecycle}`, opts);
19528
+ human("", opts);
19529
+ human(" No cases yet \u2014 a set with none will not run. Add them next.", opts);
19530
+ if (data.evidence_kind === "compare") {
19531
+ human("", opts);
19532
+ human(
19533
+ " This classifies the set \u2014 it does not start a comparison.",
19534
+ opts
19535
+ );
19536
+ human(" Ablations run per skill: `bamboo-agents eval ablate --help`.", opts);
19537
+ }
19538
+ human("", opts);
19539
+ }
19540
+ async function runEvalSetsCreate(options) {
19541
+ const opts = options;
19542
+ const session = await requireSession13(opts);
19543
+ const brandId = await resolveBrandId(
19544
+ session,
19545
+ options.brand,
19546
+ opts,
19547
+ "Specify the brand this test set belongs to with --brand <id>."
19548
+ );
19549
+ let priority = 0;
19550
+ if (options.priority !== void 0) {
19551
+ priority = Number.parseInt(options.priority, 10);
19552
+ if (!Number.isFinite(priority)) {
19553
+ emitError(
19554
+ {
19555
+ code: "invalid_priority",
19556
+ message: `--priority must be an integer, got "${options.priority}".`
19557
+ },
19558
+ opts,
19559
+ 1
19560
+ );
19561
+ throw new Error("unreachable");
19562
+ }
19563
+ }
19564
+ let created;
19565
+ try {
19566
+ created = await createEvalTestSet(session, {
19567
+ brand_id: brandId,
19568
+ agent_id: options.agent ?? null,
19569
+ name: options.name,
19570
+ slug: options.slug?.trim() || slugifyTestSetName(options.name),
19571
+ hypothesis: options.hypothesis,
19572
+ // Cast, not validate: the server's schema is the authority on the kind
19573
+ // vocabulary, and a second check here would be the fork the parity test
19574
+ // exists to prevent. An unknown kind comes back as a 422 naming it.
19575
+ evidence_kind: options.kind,
19576
+ lifecycle: options.lifecycle ?? "draft",
19577
+ priority
19578
+ });
19579
+ } catch (err) {
19580
+ const e = err;
19581
+ emitError(
19582
+ {
19583
+ code: e.code ?? "create_failed",
19584
+ message: e.message ?? "failed to create the test set."
19585
+ },
19586
+ opts,
19587
+ 1
19588
+ );
19589
+ throw new Error("unreachable");
19590
+ }
19591
+ printCreated(created, opts);
19592
+ emitSuccess(created, opts);
19593
+ }
19594
+ var init_eval_test_sets2 = __esm({
19595
+ "src/commands/eval-test-sets.ts"() {
19596
+ "use strict";
19597
+ init_credentials();
19598
+ init_api();
19599
+ init_resolve_brand();
19600
+ init_output();
19601
+ }
19602
+ });
19603
+
19604
+ // src/commands/eval-set-cases.ts
19605
+ var eval_set_cases_exports = {};
19606
+ __export(eval_set_cases_exports, {
19607
+ parseCaseIds: () => parseCaseIds,
19608
+ runEvalSetsAddCases: () => runEvalSetsAddCases,
19609
+ unionCaseIds: () => unionCaseIds
19610
+ });
19611
+ async function requireSession14(opts) {
19612
+ const creds = await readCredentials();
19613
+ if (!creds) {
19614
+ emitError(
19615
+ {
19616
+ code: "not_signed_in",
19617
+ message: "not signed in.",
19618
+ hint: "Run `bamboo-agents login` first."
19619
+ },
19620
+ opts,
19621
+ 1
19622
+ );
19623
+ throw new Error("unreachable");
19624
+ }
19625
+ return creds.bamboo_session;
19626
+ }
19627
+ function parseCaseIds(raw) {
19628
+ return raw.split(",").map((s) => s.trim()).filter((s) => s.length > 0);
19629
+ }
19630
+ function unionCaseIds(existing, incoming) {
19631
+ const seen = /* @__PURE__ */ new Set();
19632
+ const out = [];
19633
+ for (const id of [...existing, ...incoming]) {
19634
+ if (seen.has(id)) continue;
19635
+ seen.add(id);
19636
+ out.push(id);
19637
+ }
19638
+ return out;
19639
+ }
19640
+ async function runEvalSetsAddCases(options) {
19641
+ const opts = options;
19642
+ const session = await requireSession14(opts);
19643
+ const incoming = parseCaseIds(options.cases);
19644
+ if (incoming.length === 0) {
19645
+ emitError(
19646
+ { code: "no_cases", message: "--cases was empty \u2014 nothing to attach." },
19647
+ opts,
19648
+ 1
19649
+ );
19650
+ throw new Error("unreachable");
19651
+ }
19652
+ const brandId = await resolveBrandId(
19653
+ session,
19654
+ options.brand,
19655
+ opts,
19656
+ "Specify the brand the test set belongs to with --brand <id>."
19657
+ );
19658
+ let finalIds;
19659
+ let existingCount = 0;
19660
+ if (options.replace) {
19661
+ finalIds = unionCaseIds([], incoming);
19662
+ } else {
19663
+ let sets;
19664
+ try {
19665
+ sets = await getEvalTestSetMemberships(session, brandId);
19666
+ } catch (err) {
19667
+ const e = err;
19668
+ emitError(
19669
+ {
19670
+ code: e.code ?? "roadmap_read_failed",
19671
+ message: `couldn't read the set's current cases: ${e.message ?? "unknown error"}`,
19672
+ hint: "Pass --replace to write the list without reading first."
19673
+ },
19674
+ opts,
19675
+ 1
19676
+ );
19677
+ throw new Error("unreachable");
19678
+ }
19679
+ const target = sets.find((s) => s.id === options.set);
19680
+ if (!target) {
19681
+ emitError(
19682
+ {
19683
+ code: "set_not_found",
19684
+ message: `no test set ${options.set} in brand ${brandId}.`,
19685
+ hint: "Check the set id, or pass --brand if the set is in another brand."
19686
+ },
19687
+ opts,
19688
+ 1
19689
+ );
19690
+ throw new Error("unreachable");
19691
+ }
19692
+ const existing = target.cases.map((c) => c.id);
19693
+ existingCount = existing.length;
19694
+ finalIds = unionCaseIds(existing, incoming);
19695
+ }
19696
+ try {
19697
+ await updateEvalTestSetCases(session, options.set, {
19698
+ brand_id: brandId,
19699
+ case_ids: finalIds
19700
+ });
19701
+ } catch (err) {
19702
+ const e = err;
19703
+ emitError(
19704
+ {
19705
+ code: e.code ?? "attach_failed",
19706
+ message: e.message ?? "failed to attach the cases."
19707
+ },
19708
+ opts,
19709
+ 1
19710
+ );
19711
+ throw new Error("unreachable");
19712
+ }
19713
+ const added = finalIds.length - existingCount;
19714
+ human("", opts);
19715
+ if (options.replace) {
19716
+ human(` Set ${options.set} now holds exactly ${finalIds.length} cases.`, opts);
19717
+ } else {
19718
+ human(
19719
+ ` Added ${added} of ${incoming.length} to ${options.set} \u2014 ${finalIds.length} cases total.`,
19720
+ opts
19721
+ );
19722
+ const skipped = incoming.length - added;
19723
+ if (skipped > 0) {
19724
+ human(` ${skipped} ${skipped === 1 ? "was" : "were"} already in the set.`, opts);
19725
+ }
19726
+ }
19727
+ human("", opts);
19728
+ emitSuccess(
19729
+ { set_id: options.set, case_ids: finalIds, total: finalIds.length, added },
19730
+ opts
19731
+ );
19732
+ }
19733
+ var init_eval_set_cases = __esm({
19734
+ "src/commands/eval-set-cases.ts"() {
19735
+ "use strict";
19736
+ init_credentials();
19737
+ init_api();
19738
+ init_resolve_brand();
19739
+ init_output();
19740
+ }
19741
+ });
19742
+
19334
19743
  // node_modules/commander/esm.mjs
19335
19744
  var import_index = __toESM(require_commander(), 1);
19336
19745
  var {
@@ -19798,12 +20207,37 @@ evals.command("ablate").description(
19798
20207
  ).requiredOption("--agent <id>", "Agent id whose skill is under test").requiredOption("--skill <name>", "kebab-case skill name (must be one of this agent's own skills)").requiredOption(
19799
20208
  "--hypothesis <text>",
19800
20209
  "Plain-language claim the edit is meant to produce (an experiment is a claim, not a diff)"
19801
- ).requiredOption("--body <path>", "Path to the candidate SKILL.md").option("--brand <id>", "Brand id \u2014 only needed when the agent spans more than one brand").option("--json", "Emit a JSON envelope for agent consumption").option("--fields <list>", "Project only these dotted field paths from data (comma-separated)").option("--quiet", "Suppress human chatter (data/errors only)").option("--no-sync", "Skip the auto-sync git pull on this command").action(
20210
+ ).requiredOption("--body <path>", "Path to the candidate SKILL.md").option("--brand <id>", "Brand id \u2014 only needed when the agent spans more than one brand").option(
20211
+ "--set <id>",
20212
+ "Test set id \u2014 run THIS set's cases instead of the skill-attributed holdout"
20213
+ ).option("--json", "Emit a JSON envelope for agent consumption").option("--fields <list>", "Project only these dotted field paths from data (comma-separated)").option("--quiet", "Suppress human chatter (data/errors only)").option("--no-sync", "Skip the auto-sync git pull on this command").action(
19802
20214
  async (options) => {
19803
20215
  const { runEvalAblate: runEvalAblate2 } = await Promise.resolve().then(() => (init_eval_ab2(), eval_ab_exports2));
19804
20216
  await runEvalAblate2(options);
19805
20217
  }
19806
20218
  );
20219
+ var evalSets = evals.command("sets").description("Test sets \u2014 the definition a run executes (cases attach separately)");
20220
+ evalSets.command("create").description("Create an empty test set of a given evidence kind").requiredOption("--name <text>", 'Human name, e.g. "Connector parity"').requiredOption(
20221
+ "--kind <kind>",
20222
+ "Evidence kind: golden_set | quiz | regression | probe | compare"
20223
+ ).requiredOption(
20224
+ "--hypothesis <text>",
20225
+ "The claim this set exists to settle \u2014 what the kind makes a good hypothesis differs"
20226
+ ).option("--slug <slug>", "URL slug (derived from --name when omitted)").option("--agent <id>", "Scope the set to one agent (omit for brand-wide)").option("--brand <id>", "Brand id \u2014 only needed when the org has more than one brand").option("--lifecycle <state>", "draft | active | paused | retired (default: draft)").option("--priority <n>", "Integer ordering hint (default: 0)").option("--json", "Emit a JSON envelope for agent consumption").option("--fields <list>", "Project only these dotted field paths from data (comma-separated)").option("--quiet", "Suppress human chatter (data/errors only)").option("--no-sync", "Skip the auto-sync git pull on this command").action(
20227
+ async (options) => {
20228
+ const { runEvalSetsCreate: runEvalSetsCreate2 } = await Promise.resolve().then(() => (init_eval_test_sets2(), eval_test_sets_exports));
20229
+ await runEvalSetsCreate2(options);
20230
+ }
20231
+ );
20232
+ evalSets.command("add-cases").description("Attach existing cases to a test set (adds by default; --replace overwrites)").requiredOption("--set <id>", "Test set id").requiredOption("--cases <ids>", "Comma-separated eval case ids").option(
20233
+ "--replace",
20234
+ "Send exactly these cases, dropping any others. Without it, the current cases are read and merged"
20235
+ ).option("--brand <id>", "Brand id \u2014 only needed when the org has more than one brand").option("--json", "Emit a JSON envelope for agent consumption").option("--fields <list>", "Project only these dotted field paths from data (comma-separated)").option("--quiet", "Suppress human chatter (data/errors only)").option("--no-sync", "Skip the auto-sync git pull on this command").action(
20236
+ async (options) => {
20237
+ const { runEvalSetsAddCases: runEvalSetsAddCases2 } = await Promise.resolve().then(() => (init_eval_set_cases(), eval_set_cases_exports));
20238
+ await runEvalSetsAddCases2(options);
20239
+ }
20240
+ );
19807
20241
  (async () => {
19808
20242
  const rawArgs = process.argv.slice(2);
19809
20243
  const { operands } = program2.parseOptions(rawArgs);