bamboo-agents 0.1.6-dev.78.33bdebe → 0.1.6-dev.81.7694f7f

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/VERSION CHANGED
@@ -1 +1 @@
1
- 0.1.6-dev.78.33bdebe
1
+ 0.1.6-dev.81.7694f7f
package/dist/index.js CHANGED
@@ -3037,7 +3037,7 @@ var VERSION;
3037
3037
  var init_version = __esm({
3038
3038
  "src/version.ts"() {
3039
3039
  "use strict";
3040
- VERSION = "0.1.6-dev.78.33bdebe";
3040
+ VERSION = "0.1.6-dev.81.7694f7f";
3041
3041
  }
3042
3042
  });
3043
3043
 
@@ -12998,6 +12998,58 @@ var init_episodes2 = __esm({
12998
12998
  }
12999
12999
  });
13000
13000
 
13001
+ // ../../src/lib/api-contracts/eval-roadmap/evidence-kinds.ts
13002
+ var EvidenceKinds, EvidenceKindSchema;
13003
+ var init_evidence_kinds = __esm({
13004
+ "../../src/lib/api-contracts/eval-roadmap/evidence-kinds.ts"() {
13005
+ init_zod();
13006
+ EvidenceKinds = [
13007
+ "golden_set",
13008
+ "quiz",
13009
+ "regression",
13010
+ "probe",
13011
+ "compare"
13012
+ ];
13013
+ EvidenceKindSchema = external_exports.enum(EvidenceKinds);
13014
+ }
13015
+ });
13016
+
13017
+ // ../../src/lib/api-contracts/cli/eval-test-sets.ts
13018
+ var CliEvalTestSetCreateRequestSchema, CliEvalTestSetCreateResponseSchema;
13019
+ var init_eval_test_sets = __esm({
13020
+ "../../src/lib/api-contracts/cli/eval-test-sets.ts"() {
13021
+ init_zod();
13022
+ init_evidence_kinds();
13023
+ CliEvalTestSetCreateRequestSchema = external_exports.object({
13024
+ brand_id: external_exports.string().uuid(),
13025
+ agent_id: external_exports.string().uuid().nullable().optional(),
13026
+ name: external_exports.string().trim().min(1).max(200),
13027
+ slug: external_exports.string().trim().min(1).max(120),
13028
+ hypothesis: external_exports.string().trim().min(1).max(5e3),
13029
+ /**
13030
+ * Required here, optional server-side. A set created without a kind is
13031
+ * `unclassified`, and this verb exists so an agent can create a set *of a
13032
+ * given kind* — leaving it out would silently produce the one outcome the
13033
+ * caller did not ask for.
13034
+ */
13035
+ evidence_kind: EvidenceKindSchema,
13036
+ lifecycle: external_exports.enum(["draft", "active", "paused", "retired"]).default("draft"),
13037
+ priority: external_exports.number().int().default(0)
13038
+ });
13039
+ CliEvalTestSetCreateResponseSchema = external_exports.object({
13040
+ id: external_exports.string().uuid(),
13041
+ name: external_exports.string(),
13042
+ slug: external_exports.string(),
13043
+ brand_id: external_exports.string().uuid(),
13044
+ agent_id: external_exports.string().uuid().nullable(),
13045
+ evidence_kind: EvidenceKindSchema.nullable(),
13046
+ lifecycle: external_exports.string(),
13047
+ priority: external_exports.number().int(),
13048
+ hypothesis: external_exports.string().nullable()
13049
+ });
13050
+ }
13051
+ });
13052
+
13001
13053
  // src/lib/api.ts
13002
13054
  import { platform as platform2 } from "node:os";
13003
13055
  function defaultApiBase(version) {
@@ -13512,6 +13564,19 @@ async function startAbTest(bambooSession, body) {
13512
13564
  const json = await res.json();
13513
13565
  return EvalAbResponseSchema.parse(json.data);
13514
13566
  }
13567
+ async function createEvalTestSet(bambooSession, body) {
13568
+ const res = await cliApiFetch(`${apiBase()}/api/v1/eval/test-sets`, {
13569
+ method: "POST",
13570
+ headers: {
13571
+ Authorization: `Bearer ${bambooSession}`,
13572
+ "Content-Type": "application/json"
13573
+ },
13574
+ body: JSON.stringify(body)
13575
+ });
13576
+ if (!res.ok) throw await parseError(res);
13577
+ const json = await res.json();
13578
+ return CliEvalTestSetCreateResponseSchema.parse(json.data);
13579
+ }
13515
13580
  function absoluteUrl(relativePath) {
13516
13581
  return `${apiBase()}${relativePath}`;
13517
13582
  }
@@ -13554,6 +13619,7 @@ var init_api = __esm({
13554
13619
  init_usage();
13555
13620
  init_episodes2();
13556
13621
  init_eval_ab();
13622
+ init_eval_test_sets();
13557
13623
  init_version();
13558
13624
  PROD_DEFAULT_BASE = "https://withbamboo.com";
13559
13625
  DEV_DEFAULT_BASE = "https://dev.withbamboo.com";
@@ -18701,6 +18767,40 @@ var init_brand2 = __esm({
18701
18767
  }
18702
18768
  });
18703
18769
 
18770
+ // src/lib/resolve-brand.ts
18771
+ async function resolveBrandId(session, explicit, opts, hint = "Specify the brand the agent belongs to with --brand <id>.") {
18772
+ if (explicit) return explicit;
18773
+ let brands;
18774
+ try {
18775
+ brands = (await listCliBrands(session)).brands;
18776
+ } catch (err) {
18777
+ emitError(
18778
+ { code: "brands_list_failed", message: `couldn't list brands: ${err.message}` },
18779
+ opts,
18780
+ 1
18781
+ );
18782
+ throw new Error("unreachable");
18783
+ }
18784
+ if (brands.length === 1) return brands[0].id;
18785
+ emitError(
18786
+ {
18787
+ code: "brand_required",
18788
+ message: brands.length === 0 ? "no brands found for your org \u2014 create a brand first." : `multiple brands \u2014 pass --brand <id>. Options: ${brands.map((b) => `${b.name} (${b.id})`).join(", ")}.`,
18789
+ hint
18790
+ },
18791
+ opts,
18792
+ 1
18793
+ );
18794
+ throw new Error("unreachable");
18795
+ }
18796
+ var init_resolve_brand = __esm({
18797
+ "src/lib/resolve-brand.ts"() {
18798
+ "use strict";
18799
+ init_api();
18800
+ init_output();
18801
+ }
18802
+ });
18803
+
18704
18804
  // src/commands/agents.ts
18705
18805
  var agents_exports2 = {};
18706
18806
  __export(agents_exports2, {
@@ -18843,31 +18943,6 @@ async function runAgentsGet(agentRef, options) {
18843
18943
  }
18844
18944
  emitSuccess(agent, opts, (a) => printDetail(a, opts));
18845
18945
  }
18846
- async function resolveBrandId(session, explicit, opts) {
18847
- if (explicit) return explicit;
18848
- let brands;
18849
- try {
18850
- brands = (await listCliBrands(session)).brands;
18851
- } catch (err) {
18852
- emitError(
18853
- { code: "brands_list_failed", message: `couldn't list brands: ${err.message}` },
18854
- opts,
18855
- 1
18856
- );
18857
- throw new Error("unreachable");
18858
- }
18859
- if (brands.length === 1) return brands[0].id;
18860
- emitError(
18861
- {
18862
- code: "brand_required",
18863
- message: brands.length === 0 ? "no brands found for your org \u2014 create a brand first." : `multiple brands \u2014 pass --brand <id>. Options: ${brands.map((b) => `${b.name} (${b.id})`).join(", ")}.`,
18864
- hint: "Specify the brand the agent belongs to with --brand <id>."
18865
- },
18866
- opts,
18867
- 1
18868
- );
18869
- throw new Error("unreachable");
18870
- }
18871
18946
  async function runAgentsProvision(name, options) {
18872
18947
  const opts = options;
18873
18948
  const session = await requireSession10(opts);
@@ -19081,6 +19156,7 @@ var init_agents2 = __esm({
19081
19156
  "src/commands/agents.ts"() {
19082
19157
  "use strict";
19083
19158
  init_credentials();
19159
+ init_resolve_brand();
19084
19160
  init_api();
19085
19161
  init_output();
19086
19162
  }
@@ -19331,6 +19407,114 @@ var init_eval_ab2 = __esm({
19331
19407
  }
19332
19408
  });
19333
19409
 
19410
+ // src/commands/eval-test-sets.ts
19411
+ var eval_test_sets_exports = {};
19412
+ __export(eval_test_sets_exports, {
19413
+ runEvalSetsCreate: () => runEvalSetsCreate,
19414
+ slugifyTestSetName: () => slugifyTestSetName
19415
+ });
19416
+ async function requireSession13(opts) {
19417
+ const creds = await readCredentials();
19418
+ if (!creds) {
19419
+ emitError(
19420
+ {
19421
+ code: "not_signed_in",
19422
+ message: "not signed in.",
19423
+ hint: "Run `bamboo-agents login` first."
19424
+ },
19425
+ opts,
19426
+ 1
19427
+ );
19428
+ throw new Error("unreachable");
19429
+ }
19430
+ return creds.bamboo_session;
19431
+ }
19432
+ function slugifyTestSetName(name) {
19433
+ return name.trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 120);
19434
+ }
19435
+ function printCreated(data, opts) {
19436
+ human("", opts);
19437
+ human(` Created "${data.name}" (${data.evidence_kind ?? "unclassified"}).`, opts);
19438
+ human("", opts);
19439
+ human(` Set: ${data.id}`, opts);
19440
+ human(` Slug: ${data.slug}`, opts);
19441
+ human(` Lifecycle: ${data.lifecycle}`, opts);
19442
+ human("", opts);
19443
+ human(" No cases yet \u2014 a set with none will not run. Add them next.", opts);
19444
+ if (data.evidence_kind === "compare") {
19445
+ human("", opts);
19446
+ human(
19447
+ " This classifies the set \u2014 it does not start a comparison.",
19448
+ opts
19449
+ );
19450
+ human(" Ablations run per skill: `bamboo-agents eval ablate --help`.", opts);
19451
+ }
19452
+ human("", opts);
19453
+ }
19454
+ async function runEvalSetsCreate(options) {
19455
+ const opts = options;
19456
+ const session = await requireSession13(opts);
19457
+ const brandId = await resolveBrandId(
19458
+ session,
19459
+ options.brand,
19460
+ opts,
19461
+ "Specify the brand this test set belongs to with --brand <id>."
19462
+ );
19463
+ let priority = 0;
19464
+ if (options.priority !== void 0) {
19465
+ priority = Number.parseInt(options.priority, 10);
19466
+ if (!Number.isFinite(priority)) {
19467
+ emitError(
19468
+ {
19469
+ code: "invalid_priority",
19470
+ message: `--priority must be an integer, got "${options.priority}".`
19471
+ },
19472
+ opts,
19473
+ 1
19474
+ );
19475
+ throw new Error("unreachable");
19476
+ }
19477
+ }
19478
+ let created;
19479
+ try {
19480
+ created = await createEvalTestSet(session, {
19481
+ brand_id: brandId,
19482
+ agent_id: options.agent ?? null,
19483
+ name: options.name,
19484
+ slug: options.slug?.trim() || slugifyTestSetName(options.name),
19485
+ hypothesis: options.hypothesis,
19486
+ // Cast, not validate: the server's schema is the authority on the kind
19487
+ // vocabulary, and a second check here would be the fork the parity test
19488
+ // exists to prevent. An unknown kind comes back as a 422 naming it.
19489
+ evidence_kind: options.kind,
19490
+ lifecycle: options.lifecycle ?? "draft",
19491
+ priority
19492
+ });
19493
+ } catch (err) {
19494
+ const e = err;
19495
+ emitError(
19496
+ {
19497
+ code: e.code ?? "create_failed",
19498
+ message: e.message ?? "failed to create the test set."
19499
+ },
19500
+ opts,
19501
+ 1
19502
+ );
19503
+ throw new Error("unreachable");
19504
+ }
19505
+ printCreated(created, opts);
19506
+ emitSuccess(created, opts);
19507
+ }
19508
+ var init_eval_test_sets2 = __esm({
19509
+ "src/commands/eval-test-sets.ts"() {
19510
+ "use strict";
19511
+ init_credentials();
19512
+ init_api();
19513
+ init_resolve_brand();
19514
+ init_output();
19515
+ }
19516
+ });
19517
+
19334
19518
  // node_modules/commander/esm.mjs
19335
19519
  var import_index = __toESM(require_commander(), 1);
19336
19520
  var {
@@ -19804,6 +19988,19 @@ evals.command("ablate").description(
19804
19988
  await runEvalAblate2(options);
19805
19989
  }
19806
19990
  );
19991
+ var evalSets = evals.command("sets").description("Test sets \u2014 the definition a run executes (cases attach separately)");
19992
+ evalSets.command("create").description("Create an empty test set of a given evidence kind").requiredOption("--name <text>", 'Human name, e.g. "Connector parity"').requiredOption(
19993
+ "--kind <kind>",
19994
+ "Evidence kind: golden_set | quiz | regression | probe | compare"
19995
+ ).requiredOption(
19996
+ "--hypothesis <text>",
19997
+ "The claim this set exists to settle \u2014 what the kind makes a good hypothesis differs"
19998
+ ).option("--slug <slug>", "URL slug (derived from --name when omitted)").option("--agent <id>", "Scope the set to one agent (omit for brand-wide)").option("--brand <id>", "Brand id \u2014 only needed when the org has more than one brand").option("--lifecycle <state>", "draft | active | paused | retired (default: draft)").option("--priority <n>", "Integer ordering hint (default: 0)").option("--json", "Emit a JSON envelope for agent consumption").option("--fields <list>", "Project only these dotted field paths from data (comma-separated)").option("--quiet", "Suppress human chatter (data/errors only)").option("--no-sync", "Skip the auto-sync git pull on this command").action(
19999
+ async (options) => {
20000
+ const { runEvalSetsCreate: runEvalSetsCreate2 } = await Promise.resolve().then(() => (init_eval_test_sets2(), eval_test_sets_exports));
20001
+ await runEvalSetsCreate2(options);
20002
+ }
20003
+ );
19807
20004
  (async () => {
19808
20005
  const rawArgs = process.argv.slice(2);
19809
20006
  const { operands } = program2.parseOptions(rawArgs);