bamboo-agents 0.1.6-dev.78.33bdebe → 0.1.6-dev.80.3b0f3f6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/VERSION +1 -1
- package/dist/index.js +223 -26
- package/dist/index.js.map +4 -4
- package/package.json +1 -1
package/dist/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
0.1.6-dev.
|
|
1
|
+
0.1.6-dev.80.3b0f3f6
|
package/dist/index.js
CHANGED
|
@@ -3037,7 +3037,7 @@ var VERSION;
|
|
|
3037
3037
|
var init_version = __esm({
|
|
3038
3038
|
"src/version.ts"() {
|
|
3039
3039
|
"use strict";
|
|
3040
|
-
VERSION = "0.1.6-dev.
|
|
3040
|
+
VERSION = "0.1.6-dev.80.3b0f3f6";
|
|
3041
3041
|
}
|
|
3042
3042
|
});
|
|
3043
3043
|
|
|
@@ -12998,6 +12998,58 @@ var init_episodes2 = __esm({
|
|
|
12998
12998
|
}
|
|
12999
12999
|
});
|
|
13000
13000
|
|
|
13001
|
+
// ../../src/lib/api-contracts/eval-roadmap/evidence-kinds.ts
|
|
13002
|
+
var EvidenceKinds, EvidenceKindSchema;
|
|
13003
|
+
var init_evidence_kinds = __esm({
|
|
13004
|
+
"../../src/lib/api-contracts/eval-roadmap/evidence-kinds.ts"() {
|
|
13005
|
+
init_zod();
|
|
13006
|
+
EvidenceKinds = [
|
|
13007
|
+
"golden_set",
|
|
13008
|
+
"quiz",
|
|
13009
|
+
"regression",
|
|
13010
|
+
"probe",
|
|
13011
|
+
"compare"
|
|
13012
|
+
];
|
|
13013
|
+
EvidenceKindSchema = external_exports.enum(EvidenceKinds);
|
|
13014
|
+
}
|
|
13015
|
+
});
|
|
13016
|
+
|
|
13017
|
+
// ../../src/lib/api-contracts/cli/eval-test-sets.ts
|
|
13018
|
+
var CliEvalTestSetCreateRequestSchema, CliEvalTestSetCreateResponseSchema;
|
|
13019
|
+
var init_eval_test_sets = __esm({
|
|
13020
|
+
"../../src/lib/api-contracts/cli/eval-test-sets.ts"() {
|
|
13021
|
+
init_zod();
|
|
13022
|
+
init_evidence_kinds();
|
|
13023
|
+
CliEvalTestSetCreateRequestSchema = external_exports.object({
|
|
13024
|
+
brand_id: external_exports.string().uuid(),
|
|
13025
|
+
agent_id: external_exports.string().uuid().nullable().optional(),
|
|
13026
|
+
name: external_exports.string().trim().min(1).max(200),
|
|
13027
|
+
slug: external_exports.string().trim().min(1).max(120),
|
|
13028
|
+
hypothesis: external_exports.string().trim().min(1).max(5e3),
|
|
13029
|
+
/**
|
|
13030
|
+
* Required here, optional server-side. A set created without a kind is
|
|
13031
|
+
* `unclassified`, and this verb exists so an agent can create a set *of a
|
|
13032
|
+
* given kind* — leaving it out would silently produce the one outcome the
|
|
13033
|
+
* caller did not ask for.
|
|
13034
|
+
*/
|
|
13035
|
+
evidence_kind: EvidenceKindSchema,
|
|
13036
|
+
lifecycle: external_exports.enum(["draft", "active", "paused", "retired"]).default("draft"),
|
|
13037
|
+
priority: external_exports.number().int().default(0)
|
|
13038
|
+
});
|
|
13039
|
+
CliEvalTestSetCreateResponseSchema = external_exports.object({
|
|
13040
|
+
id: external_exports.string().uuid(),
|
|
13041
|
+
name: external_exports.string(),
|
|
13042
|
+
slug: external_exports.string(),
|
|
13043
|
+
brand_id: external_exports.string().uuid(),
|
|
13044
|
+
agent_id: external_exports.string().uuid().nullable(),
|
|
13045
|
+
evidence_kind: EvidenceKindSchema.nullable(),
|
|
13046
|
+
lifecycle: external_exports.string(),
|
|
13047
|
+
priority: external_exports.number().int(),
|
|
13048
|
+
hypothesis: external_exports.string().nullable()
|
|
13049
|
+
});
|
|
13050
|
+
}
|
|
13051
|
+
});
|
|
13052
|
+
|
|
13001
13053
|
// src/lib/api.ts
|
|
13002
13054
|
import { platform as platform2 } from "node:os";
|
|
13003
13055
|
function defaultApiBase(version) {
|
|
@@ -13512,6 +13564,19 @@ async function startAbTest(bambooSession, body) {
|
|
|
13512
13564
|
const json = await res.json();
|
|
13513
13565
|
return EvalAbResponseSchema.parse(json.data);
|
|
13514
13566
|
}
|
|
13567
|
+
async function createEvalTestSet(bambooSession, body) {
|
|
13568
|
+
const res = await cliApiFetch(`${apiBase()}/api/v1/eval/test-sets`, {
|
|
13569
|
+
method: "POST",
|
|
13570
|
+
headers: {
|
|
13571
|
+
Authorization: `Bearer ${bambooSession}`,
|
|
13572
|
+
"Content-Type": "application/json"
|
|
13573
|
+
},
|
|
13574
|
+
body: JSON.stringify(body)
|
|
13575
|
+
});
|
|
13576
|
+
if (!res.ok) throw await parseError(res);
|
|
13577
|
+
const json = await res.json();
|
|
13578
|
+
return CliEvalTestSetCreateResponseSchema.parse(json.data);
|
|
13579
|
+
}
|
|
13515
13580
|
function absoluteUrl(relativePath) {
|
|
13516
13581
|
return `${apiBase()}${relativePath}`;
|
|
13517
13582
|
}
|
|
@@ -13554,6 +13619,7 @@ var init_api = __esm({
|
|
|
13554
13619
|
init_usage();
|
|
13555
13620
|
init_episodes2();
|
|
13556
13621
|
init_eval_ab();
|
|
13622
|
+
init_eval_test_sets();
|
|
13557
13623
|
init_version();
|
|
13558
13624
|
PROD_DEFAULT_BASE = "https://withbamboo.com";
|
|
13559
13625
|
DEV_DEFAULT_BASE = "https://dev.withbamboo.com";
|
|
@@ -18701,6 +18767,40 @@ var init_brand2 = __esm({
|
|
|
18701
18767
|
}
|
|
18702
18768
|
});
|
|
18703
18769
|
|
|
18770
|
+
// src/lib/resolve-brand.ts
|
|
18771
|
+
async function resolveBrandId(session, explicit, opts, hint = "Specify the brand the agent belongs to with --brand <id>.") {
|
|
18772
|
+
if (explicit) return explicit;
|
|
18773
|
+
let brands;
|
|
18774
|
+
try {
|
|
18775
|
+
brands = (await listCliBrands(session)).brands;
|
|
18776
|
+
} catch (err) {
|
|
18777
|
+
emitError(
|
|
18778
|
+
{ code: "brands_list_failed", message: `couldn't list brands: ${err.message}` },
|
|
18779
|
+
opts,
|
|
18780
|
+
1
|
|
18781
|
+
);
|
|
18782
|
+
throw new Error("unreachable");
|
|
18783
|
+
}
|
|
18784
|
+
if (brands.length === 1) return brands[0].id;
|
|
18785
|
+
emitError(
|
|
18786
|
+
{
|
|
18787
|
+
code: "brand_required",
|
|
18788
|
+
message: brands.length === 0 ? "no brands found for your org \u2014 create a brand first." : `multiple brands \u2014 pass --brand <id>. Options: ${brands.map((b) => `${b.name} (${b.id})`).join(", ")}.`,
|
|
18789
|
+
hint
|
|
18790
|
+
},
|
|
18791
|
+
opts,
|
|
18792
|
+
1
|
|
18793
|
+
);
|
|
18794
|
+
throw new Error("unreachable");
|
|
18795
|
+
}
|
|
18796
|
+
var init_resolve_brand = __esm({
|
|
18797
|
+
"src/lib/resolve-brand.ts"() {
|
|
18798
|
+
"use strict";
|
|
18799
|
+
init_api();
|
|
18800
|
+
init_output();
|
|
18801
|
+
}
|
|
18802
|
+
});
|
|
18803
|
+
|
|
18704
18804
|
// src/commands/agents.ts
|
|
18705
18805
|
var agents_exports2 = {};
|
|
18706
18806
|
__export(agents_exports2, {
|
|
@@ -18843,31 +18943,6 @@ async function runAgentsGet(agentRef, options) {
|
|
|
18843
18943
|
}
|
|
18844
18944
|
emitSuccess(agent, opts, (a) => printDetail(a, opts));
|
|
18845
18945
|
}
|
|
18846
|
-
async function resolveBrandId(session, explicit, opts) {
|
|
18847
|
-
if (explicit) return explicit;
|
|
18848
|
-
let brands;
|
|
18849
|
-
try {
|
|
18850
|
-
brands = (await listCliBrands(session)).brands;
|
|
18851
|
-
} catch (err) {
|
|
18852
|
-
emitError(
|
|
18853
|
-
{ code: "brands_list_failed", message: `couldn't list brands: ${err.message}` },
|
|
18854
|
-
opts,
|
|
18855
|
-
1
|
|
18856
|
-
);
|
|
18857
|
-
throw new Error("unreachable");
|
|
18858
|
-
}
|
|
18859
|
-
if (brands.length === 1) return brands[0].id;
|
|
18860
|
-
emitError(
|
|
18861
|
-
{
|
|
18862
|
-
code: "brand_required",
|
|
18863
|
-
message: brands.length === 0 ? "no brands found for your org \u2014 create a brand first." : `multiple brands \u2014 pass --brand <id>. Options: ${brands.map((b) => `${b.name} (${b.id})`).join(", ")}.`,
|
|
18864
|
-
hint: "Specify the brand the agent belongs to with --brand <id>."
|
|
18865
|
-
},
|
|
18866
|
-
opts,
|
|
18867
|
-
1
|
|
18868
|
-
);
|
|
18869
|
-
throw new Error("unreachable");
|
|
18870
|
-
}
|
|
18871
18946
|
async function runAgentsProvision(name, options) {
|
|
18872
18947
|
const opts = options;
|
|
18873
18948
|
const session = await requireSession10(opts);
|
|
@@ -19081,6 +19156,7 @@ var init_agents2 = __esm({
|
|
|
19081
19156
|
"src/commands/agents.ts"() {
|
|
19082
19157
|
"use strict";
|
|
19083
19158
|
init_credentials();
|
|
19159
|
+
init_resolve_brand();
|
|
19084
19160
|
init_api();
|
|
19085
19161
|
init_output();
|
|
19086
19162
|
}
|
|
@@ -19331,6 +19407,114 @@ var init_eval_ab2 = __esm({
|
|
|
19331
19407
|
}
|
|
19332
19408
|
});
|
|
19333
19409
|
|
|
19410
|
+
// src/commands/eval-test-sets.ts
|
|
19411
|
+
var eval_test_sets_exports = {};
|
|
19412
|
+
__export(eval_test_sets_exports, {
|
|
19413
|
+
runEvalSetsCreate: () => runEvalSetsCreate,
|
|
19414
|
+
slugifyTestSetName: () => slugifyTestSetName
|
|
19415
|
+
});
|
|
19416
|
+
async function requireSession13(opts) {
|
|
19417
|
+
const creds = await readCredentials();
|
|
19418
|
+
if (!creds) {
|
|
19419
|
+
emitError(
|
|
19420
|
+
{
|
|
19421
|
+
code: "not_signed_in",
|
|
19422
|
+
message: "not signed in.",
|
|
19423
|
+
hint: "Run `bamboo-agents login` first."
|
|
19424
|
+
},
|
|
19425
|
+
opts,
|
|
19426
|
+
1
|
|
19427
|
+
);
|
|
19428
|
+
throw new Error("unreachable");
|
|
19429
|
+
}
|
|
19430
|
+
return creds.bamboo_session;
|
|
19431
|
+
}
|
|
19432
|
+
function slugifyTestSetName(name) {
|
|
19433
|
+
return name.trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 120);
|
|
19434
|
+
}
|
|
19435
|
+
function printCreated(data, opts) {
|
|
19436
|
+
human("", opts);
|
|
19437
|
+
human(` Created "${data.name}" (${data.evidence_kind ?? "unclassified"}).`, opts);
|
|
19438
|
+
human("", opts);
|
|
19439
|
+
human(` Set: ${data.id}`, opts);
|
|
19440
|
+
human(` Slug: ${data.slug}`, opts);
|
|
19441
|
+
human(` Lifecycle: ${data.lifecycle}`, opts);
|
|
19442
|
+
human("", opts);
|
|
19443
|
+
human(" No cases yet \u2014 a set with none will not run. Add them next.", opts);
|
|
19444
|
+
if (data.evidence_kind === "compare") {
|
|
19445
|
+
human("", opts);
|
|
19446
|
+
human(
|
|
19447
|
+
" This classifies the set \u2014 it does not start a comparison.",
|
|
19448
|
+
opts
|
|
19449
|
+
);
|
|
19450
|
+
human(" Ablations run per skill: `bamboo-agents eval ablate --help`.", opts);
|
|
19451
|
+
}
|
|
19452
|
+
human("", opts);
|
|
19453
|
+
}
|
|
19454
|
+
async function runEvalSetsCreate(options) {
|
|
19455
|
+
const opts = options;
|
|
19456
|
+
const session = await requireSession13(opts);
|
|
19457
|
+
const brandId = await resolveBrandId(
|
|
19458
|
+
session,
|
|
19459
|
+
options.brand,
|
|
19460
|
+
opts,
|
|
19461
|
+
"Specify the brand this test set belongs to with --brand <id>."
|
|
19462
|
+
);
|
|
19463
|
+
let priority = 0;
|
|
19464
|
+
if (options.priority !== void 0) {
|
|
19465
|
+
priority = Number.parseInt(options.priority, 10);
|
|
19466
|
+
if (!Number.isFinite(priority)) {
|
|
19467
|
+
emitError(
|
|
19468
|
+
{
|
|
19469
|
+
code: "invalid_priority",
|
|
19470
|
+
message: `--priority must be an integer, got "${options.priority}".`
|
|
19471
|
+
},
|
|
19472
|
+
opts,
|
|
19473
|
+
1
|
|
19474
|
+
);
|
|
19475
|
+
throw new Error("unreachable");
|
|
19476
|
+
}
|
|
19477
|
+
}
|
|
19478
|
+
let created;
|
|
19479
|
+
try {
|
|
19480
|
+
created = await createEvalTestSet(session, {
|
|
19481
|
+
brand_id: brandId,
|
|
19482
|
+
agent_id: options.agent ?? null,
|
|
19483
|
+
name: options.name,
|
|
19484
|
+
slug: options.slug?.trim() || slugifyTestSetName(options.name),
|
|
19485
|
+
hypothesis: options.hypothesis,
|
|
19486
|
+
// Cast, not validate: the server's schema is the authority on the kind
|
|
19487
|
+
// vocabulary, and a second check here would be the fork the parity test
|
|
19488
|
+
// exists to prevent. An unknown kind comes back as a 422 naming it.
|
|
19489
|
+
evidence_kind: options.kind,
|
|
19490
|
+
lifecycle: options.lifecycle ?? "draft",
|
|
19491
|
+
priority
|
|
19492
|
+
});
|
|
19493
|
+
} catch (err) {
|
|
19494
|
+
const e = err;
|
|
19495
|
+
emitError(
|
|
19496
|
+
{
|
|
19497
|
+
code: e.code ?? "create_failed",
|
|
19498
|
+
message: e.message ?? "failed to create the test set."
|
|
19499
|
+
},
|
|
19500
|
+
opts,
|
|
19501
|
+
1
|
|
19502
|
+
);
|
|
19503
|
+
throw new Error("unreachable");
|
|
19504
|
+
}
|
|
19505
|
+
printCreated(created, opts);
|
|
19506
|
+
emitSuccess(created, opts);
|
|
19507
|
+
}
|
|
19508
|
+
var init_eval_test_sets2 = __esm({
|
|
19509
|
+
"src/commands/eval-test-sets.ts"() {
|
|
19510
|
+
"use strict";
|
|
19511
|
+
init_credentials();
|
|
19512
|
+
init_api();
|
|
19513
|
+
init_resolve_brand();
|
|
19514
|
+
init_output();
|
|
19515
|
+
}
|
|
19516
|
+
});
|
|
19517
|
+
|
|
19334
19518
|
// node_modules/commander/esm.mjs
|
|
19335
19519
|
var import_index = __toESM(require_commander(), 1);
|
|
19336
19520
|
var {
|
|
@@ -19804,6 +19988,19 @@ evals.command("ablate").description(
|
|
|
19804
19988
|
await runEvalAblate2(options);
|
|
19805
19989
|
}
|
|
19806
19990
|
);
|
|
19991
|
+
var evalSets = evals.command("sets").description("Test sets \u2014 the definition a run executes (cases attach separately)");
|
|
19992
|
+
evalSets.command("create").description("Create an empty test set of a given evidence kind").requiredOption("--name <text>", 'Human name, e.g. "Connector parity"').requiredOption(
|
|
19993
|
+
"--kind <kind>",
|
|
19994
|
+
"Evidence kind: golden_set | quiz | regression | probe | compare"
|
|
19995
|
+
).requiredOption(
|
|
19996
|
+
"--hypothesis <text>",
|
|
19997
|
+
"The claim this set exists to settle \u2014 what the kind makes a good hypothesis differs"
|
|
19998
|
+
).option("--slug <slug>", "URL slug (derived from --name when omitted)").option("--agent <id>", "Scope the set to one agent (omit for brand-wide)").option("--brand <id>", "Brand id \u2014 only needed when the org has more than one brand").option("--lifecycle <state>", "draft | active | paused | retired (default: draft)").option("--priority <n>", "Integer ordering hint (default: 0)").option("--json", "Emit a JSON envelope for agent consumption").option("--fields <list>", "Project only these dotted field paths from data (comma-separated)").option("--quiet", "Suppress human chatter (data/errors only)").option("--no-sync", "Skip the auto-sync git pull on this command").action(
|
|
19999
|
+
async (options) => {
|
|
20000
|
+
const { runEvalSetsCreate: runEvalSetsCreate2 } = await Promise.resolve().then(() => (init_eval_test_sets2(), eval_test_sets_exports));
|
|
20001
|
+
await runEvalSetsCreate2(options);
|
|
20002
|
+
}
|
|
20003
|
+
);
|
|
19807
20004
|
(async () => {
|
|
19808
20005
|
const rawArgs = process.argv.slice(2);
|
|
19809
20006
|
const { operands } = program2.parseOptions(rawArgs);
|