bamboo-agents 0.1.7-dev.85.44cc015 → 0.1.7-dev.88.66a1ec6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/VERSION +1 -1
- package/dist/index.js +172 -71
- package/dist/index.js.map +4 -4
- package/package.json +1 -1
package/dist/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
0.1.7-dev.
|
|
1
|
+
0.1.7-dev.88.66a1ec6
|
package/dist/index.js
CHANGED
|
@@ -3037,7 +3037,7 @@ var VERSION;
|
|
|
3037
3037
|
var init_version = __esm({
|
|
3038
3038
|
"src/version.ts"() {
|
|
3039
3039
|
"use strict";
|
|
3040
|
-
VERSION = "0.1.7-dev.
|
|
3040
|
+
VERSION = "0.1.7-dev.88.66a1ec6";
|
|
3041
3041
|
}
|
|
3042
3042
|
});
|
|
3043
3043
|
|
|
@@ -13035,6 +13035,46 @@ var init_episodes2 = __esm({
|
|
|
13035
13035
|
}
|
|
13036
13036
|
});
|
|
13037
13037
|
|
|
13038
|
+
// ../../src/lib/api-contracts/cli/eval-baseline-promote.ts
|
|
13039
|
+
var Reason, PrUrl, EvalBaselinePromoteRunSchema, EvalBaselineDemoteSchema, EvalBaselinePromoteRequestSchema, EvalBaselinePromoteResponseSchema;
|
|
13040
|
+
var init_eval_baseline_promote = __esm({
|
|
13041
|
+
"../../src/lib/api-contracts/cli/eval-baseline-promote.ts"() {
|
|
13042
|
+
init_zod();
|
|
13043
|
+
Reason = external_exports.string().trim().min(1).max(2e3).describe("Why this run is the reference now. Recorded on the row.");
|
|
13044
|
+
PrUrl = external_exports.string().url().optional().describe("The PR this decision was made on, recorded as the row's source_url.");
|
|
13045
|
+
EvalBaselinePromoteRunSchema = external_exports.object({
|
|
13046
|
+
run_id: external_exports.string().uuid().describe("A completed qrels run in the caller's org \u2014 the id the correctness gate printed."),
|
|
13047
|
+
reason: Reason,
|
|
13048
|
+
pr_url: PrUrl,
|
|
13049
|
+
// A body that says `demote: true` but fails the demote shape must not fall
|
|
13050
|
+
// through to a promotion: objects strip unknown keys, so name it here.
|
|
13051
|
+
demote: external_exports.literal(false).optional()
|
|
13052
|
+
});
|
|
13053
|
+
EvalBaselineDemoteSchema = external_exports.object({
|
|
13054
|
+
demote: external_exports.literal(true),
|
|
13055
|
+
brand_id: external_exports.string().uuid(),
|
|
13056
|
+
qrels_version: external_exports.string().min(1).describe("The key to empty: its current baseline is tombstoned."),
|
|
13057
|
+
reason: Reason,
|
|
13058
|
+
pr_url: PrUrl
|
|
13059
|
+
});
|
|
13060
|
+
EvalBaselinePromoteRequestSchema = external_exports.union([
|
|
13061
|
+
EvalBaselineDemoteSchema,
|
|
13062
|
+
EvalBaselinePromoteRunSchema
|
|
13063
|
+
]);
|
|
13064
|
+
EvalBaselinePromoteResponseSchema = external_exports.object({
|
|
13065
|
+
promotion_id: external_exports.string().uuid(),
|
|
13066
|
+
kind: external_exports.enum(["promote", "demote"]),
|
|
13067
|
+
brand_id: external_exports.string().uuid(),
|
|
13068
|
+
qrels_version: external_exports.string(),
|
|
13069
|
+
/** Null after a demotion. */
|
|
13070
|
+
run_id: external_exports.string().uuid().nullable(),
|
|
13071
|
+
/** What the head pointed at before this row; null when the version had no baseline. */
|
|
13072
|
+
previous_run_id: external_exports.string().uuid().nullable(),
|
|
13073
|
+
promoted_at: external_exports.string()
|
|
13074
|
+
});
|
|
13075
|
+
}
|
|
13076
|
+
});
|
|
13077
|
+
|
|
13038
13078
|
// ../../src/lib/api-contracts/eval-roadmap/evidence-kinds.ts
|
|
13039
13079
|
var EvidenceKinds, EvidenceKindSchema;
|
|
13040
13080
|
var init_evidence_kinds = __esm({
|
|
@@ -13515,22 +13555,6 @@ async function updateSkillFromRegistry(bambooSession, agentId, skillName) {
|
|
|
13515
13555
|
const json = await res.json();
|
|
13516
13556
|
return SkillSubscriptionsResponseSchema.parse(json.data);
|
|
13517
13557
|
}
|
|
13518
|
-
async function scaffoldSkill(bambooSession, args) {
|
|
13519
|
-
const res = await cliApiFetch(
|
|
13520
|
-
`${apiBase()}/api/v1/orgs/me/skills/scaffold`,
|
|
13521
|
-
{
|
|
13522
|
-
method: "POST",
|
|
13523
|
-
headers: {
|
|
13524
|
-
Authorization: `Bearer ${bambooSession}`,
|
|
13525
|
-
"Content-Type": "application/json"
|
|
13526
|
-
},
|
|
13527
|
-
body: JSON.stringify(args)
|
|
13528
|
-
}
|
|
13529
|
-
);
|
|
13530
|
-
if (!res.ok) throw await parseError(res);
|
|
13531
|
-
const json = await res.json();
|
|
13532
|
-
return ScaffoldSkillResponseSchema.parse(json.data);
|
|
13533
|
-
}
|
|
13534
13558
|
async function patchBrand(bambooSession, brandId, websiteUrl) {
|
|
13535
13559
|
const res = await cliApiFetch(
|
|
13536
13560
|
`${apiBase()}/api/v1/cli/brands/${encodeURIComponent(brandId)}`,
|
|
@@ -13612,6 +13636,19 @@ async function startAbTest(bambooSession, body) {
|
|
|
13612
13636
|
const json = await res.json();
|
|
13613
13637
|
return EvalAbResponseSchema.parse(json.data);
|
|
13614
13638
|
}
|
|
13639
|
+
async function promoteBaseline(bambooSession, body) {
|
|
13640
|
+
const res = await cliApiFetch(`${apiBase()}/api/v1/eval/baseline-promotions`, {
|
|
13641
|
+
method: "POST",
|
|
13642
|
+
headers: {
|
|
13643
|
+
Authorization: `Bearer ${bambooSession}`,
|
|
13644
|
+
"Content-Type": "application/json"
|
|
13645
|
+
},
|
|
13646
|
+
body: JSON.stringify(body)
|
|
13647
|
+
});
|
|
13648
|
+
if (!res.ok) throw await parseError(res);
|
|
13649
|
+
const json = await res.json();
|
|
13650
|
+
return EvalBaselinePromoteResponseSchema.parse(json.data);
|
|
13651
|
+
}
|
|
13615
13652
|
async function createEvalTestSet(bambooSession, body) {
|
|
13616
13653
|
const res = await cliApiFetch(`${apiBase()}/api/v1/eval/test-sets`, {
|
|
13617
13654
|
method: "POST",
|
|
@@ -13664,7 +13701,7 @@ async function postRecEpisode(bambooSession, body) {
|
|
|
13664
13701
|
const json = await res.json();
|
|
13665
13702
|
return CliRecEpisodeResponseSchema.parse(json);
|
|
13666
13703
|
}
|
|
13667
|
-
var PROD_DEFAULT_BASE, DEV_DEFAULT_BASE, REQUEST_TIMEOUT_MS, ApiError, OAuthRelinkStartResponseSchema, ProvisionWorkspaceResponseSchema, FirstAutoInstalledAgentResponseSchema, SkillCatalogEntrySchema, SkillCatalogResponseSchema, SkillSubscriptionsResponseSchema
|
|
13704
|
+
var PROD_DEFAULT_BASE, DEV_DEFAULT_BASE, REQUEST_TIMEOUT_MS, ApiError, OAuthRelinkStartResponseSchema, ProvisionWorkspaceResponseSchema, FirstAutoInstalledAgentResponseSchema, SkillCatalogEntrySchema, SkillCatalogResponseSchema, SkillSubscriptionsResponseSchema;
|
|
13668
13705
|
var init_api = __esm({
|
|
13669
13706
|
"src/lib/api.ts"() {
|
|
13670
13707
|
"use strict";
|
|
@@ -13690,6 +13727,7 @@ var init_api = __esm({
|
|
|
13690
13727
|
init_usage();
|
|
13691
13728
|
init_episodes2();
|
|
13692
13729
|
init_eval_ab();
|
|
13730
|
+
init_eval_baseline_promote();
|
|
13693
13731
|
init_eval_test_sets();
|
|
13694
13732
|
init_version();
|
|
13695
13733
|
PROD_DEFAULT_BASE = "https://withbamboo.com";
|
|
@@ -13735,9 +13773,6 @@ var init_api = __esm({
|
|
|
13735
13773
|
SkillSubscriptionsResponseSchema = external_exports.object({
|
|
13736
13774
|
subscriptions: external_exports.array(external_exports.string())
|
|
13737
13775
|
});
|
|
13738
|
-
ScaffoldSkillResponseSchema = external_exports.object({
|
|
13739
|
-
skill_name: external_exports.string()
|
|
13740
|
-
});
|
|
13741
13776
|
}
|
|
13742
13777
|
});
|
|
13743
13778
|
|
|
@@ -17519,7 +17554,6 @@ var init_skills_local = __esm({
|
|
|
17519
17554
|
var skills_exports = {};
|
|
17520
17555
|
__export(skills_exports, {
|
|
17521
17556
|
runSkillsApply: () => runSkillsApply,
|
|
17522
|
-
runSkillsCreate: () => runSkillsCreate,
|
|
17523
17557
|
runSkillsGet: () => runSkillsGet,
|
|
17524
17558
|
runSkillsList: () => runSkillsList,
|
|
17525
17559
|
runSkillsSubscribe: () => runSkillsSubscribe,
|
|
@@ -17783,45 +17817,6 @@ async function runSubscribeOp(action, agentIdentifier, skillName) {
|
|
|
17783
17817
|
console.log(" git -C $(bamboo-agents status) pull");
|
|
17784
17818
|
console.log("");
|
|
17785
17819
|
}
|
|
17786
|
-
async function runSkillsCreate(skillName, options = {}) {
|
|
17787
|
-
return withFreshWorkspace(async () => runSkillsCreateImpl(skillName, options));
|
|
17788
|
-
}
|
|
17789
|
-
async function runSkillsCreateImpl(skillName, options = {}) {
|
|
17790
|
-
if (!skillName || !skillName.trim()) {
|
|
17791
|
-
console.error("bamboo-agents: skills create requires a skill name.");
|
|
17792
|
-
console.error("Example: bamboo-agents skills create analyze-creative");
|
|
17793
|
-
process.exit(2);
|
|
17794
|
-
}
|
|
17795
|
-
if (!/^[a-z0-9][a-z0-9-]*$/.test(skillName.trim())) {
|
|
17796
|
-
console.error(
|
|
17797
|
-
"bamboo-agents: skill name must be kebab-case (lowercase + dashes)."
|
|
17798
|
-
);
|
|
17799
|
-
process.exit(2);
|
|
17800
|
-
}
|
|
17801
|
-
const bambooSession = await requireSession2();
|
|
17802
|
-
let result;
|
|
17803
|
-
try {
|
|
17804
|
-
result = await scaffoldSkill(bambooSession, {
|
|
17805
|
-
skill_name: skillName.trim(),
|
|
17806
|
-
category: options.category,
|
|
17807
|
-
description: options.description
|
|
17808
|
-
});
|
|
17809
|
-
} catch (err) {
|
|
17810
|
-
console.error(
|
|
17811
|
-
`bamboo-agents: could not scaffold ${skillName}: ${err.message}`
|
|
17812
|
-
);
|
|
17813
|
-
process.exit(1);
|
|
17814
|
-
}
|
|
17815
|
-
console.log("");
|
|
17816
|
-
console.log(` Scaffolded ${result.skill_name}.`);
|
|
17817
|
-
console.log("");
|
|
17818
|
-
console.log(
|
|
17819
|
-
` Edit skills/${result.skill_name}/SKILL.md in your workspace clone:`
|
|
17820
|
-
);
|
|
17821
|
-
console.log(" git -C $(bamboo-agents status) pull");
|
|
17822
|
-
console.log(` $EDITOR skills/${result.skill_name}/SKILL.md`);
|
|
17823
|
-
console.log("");
|
|
17824
|
-
}
|
|
17825
17820
|
async function runSkillsUpdate(agentIdentifier, skillName) {
|
|
17826
17821
|
return withFreshWorkspace(
|
|
17827
17822
|
async () => runSkillsUpdateImpl(agentIdentifier, skillName)
|
|
@@ -19493,13 +19488,117 @@ var init_eval_ab2 = __esm({
|
|
|
19493
19488
|
}
|
|
19494
19489
|
});
|
|
19495
19490
|
|
|
19491
|
+
// src/commands/eval-promote.ts
|
|
19492
|
+
var eval_promote_exports = {};
|
|
19493
|
+
__export(eval_promote_exports, {
|
|
19494
|
+
runEvalPromote: () => runEvalPromote
|
|
19495
|
+
});
|
|
19496
|
+
async function requireSession13(opts) {
|
|
19497
|
+
const creds = await readCredentials();
|
|
19498
|
+
if (!creds) {
|
|
19499
|
+
emitError(
|
|
19500
|
+
{
|
|
19501
|
+
code: "not_signed_in",
|
|
19502
|
+
message: "not signed in.",
|
|
19503
|
+
hint: "Run `bamboo-agents login` first."
|
|
19504
|
+
},
|
|
19505
|
+
opts,
|
|
19506
|
+
1
|
|
19507
|
+
);
|
|
19508
|
+
throw new Error("unreachable");
|
|
19509
|
+
}
|
|
19510
|
+
return creds.bamboo_session;
|
|
19511
|
+
}
|
|
19512
|
+
function short(version) {
|
|
19513
|
+
return version.slice(0, 12);
|
|
19514
|
+
}
|
|
19515
|
+
function render(data, opts) {
|
|
19516
|
+
human("", opts);
|
|
19517
|
+
if (data.kind === "demote") {
|
|
19518
|
+
human(
|
|
19519
|
+
` Baseline demoted for qrels_version ${short(data.qrels_version)} (was run ${data.previous_run_id}).`,
|
|
19520
|
+
opts
|
|
19521
|
+
);
|
|
19522
|
+
human(
|
|
19523
|
+
" The gate reports baseline_demoted for that version until a human promotes a run.",
|
|
19524
|
+
opts
|
|
19525
|
+
);
|
|
19526
|
+
} else {
|
|
19527
|
+
human(
|
|
19528
|
+
` Baseline promoted: run ${data.run_id} is the reference for qrels_version ${short(data.qrels_version)}` + (data.previous_run_id ? ` (was ${data.previous_run_id}).` : " (the version had none)."),
|
|
19529
|
+
opts
|
|
19530
|
+
);
|
|
19531
|
+
human(" Re-run the PR's retrieval-evals check: it compares flat against its own run.", opts);
|
|
19532
|
+
}
|
|
19533
|
+
human("", opts);
|
|
19534
|
+
}
|
|
19535
|
+
async function runEvalPromote(options) {
|
|
19536
|
+
const opts = { json: options.json, fields: options.fields, quiet: options.quiet };
|
|
19537
|
+
let body;
|
|
19538
|
+
if (options.demote) {
|
|
19539
|
+
if (!options.brand || !options.qrelsVersion) {
|
|
19540
|
+
emitError(
|
|
19541
|
+
{
|
|
19542
|
+
code: "missing_key",
|
|
19543
|
+
message: "--demote needs --brand <id> and --qrels-version <hash>.",
|
|
19544
|
+
hint: "A tombstone names the key it empties; both are on the gate report."
|
|
19545
|
+
},
|
|
19546
|
+
opts,
|
|
19547
|
+
1
|
|
19548
|
+
);
|
|
19549
|
+
return;
|
|
19550
|
+
}
|
|
19551
|
+
body = {
|
|
19552
|
+
demote: true,
|
|
19553
|
+
brand_id: options.brand,
|
|
19554
|
+
qrels_version: options.qrelsVersion,
|
|
19555
|
+
reason: options.reason,
|
|
19556
|
+
...options.pr ? { pr_url: options.pr } : {}
|
|
19557
|
+
};
|
|
19558
|
+
} else {
|
|
19559
|
+
if (!options.runId) {
|
|
19560
|
+
emitError(
|
|
19561
|
+
{
|
|
19562
|
+
code: "missing_run_id",
|
|
19563
|
+
message: "Pass the run id the correctness gate printed.",
|
|
19564
|
+
hint: 'bamboo-agents eval promote <run-id> --reason "why this drop is intentional"'
|
|
19565
|
+
},
|
|
19566
|
+
opts,
|
|
19567
|
+
1
|
|
19568
|
+
);
|
|
19569
|
+
return;
|
|
19570
|
+
}
|
|
19571
|
+
body = {
|
|
19572
|
+
run_id: options.runId,
|
|
19573
|
+
reason: options.reason,
|
|
19574
|
+
...options.pr ? { pr_url: options.pr } : {}
|
|
19575
|
+
};
|
|
19576
|
+
}
|
|
19577
|
+
const session = await requireSession13(opts);
|
|
19578
|
+
try {
|
|
19579
|
+
const data = await promoteBaseline(session, body);
|
|
19580
|
+
emitSuccess(data, opts, (d) => render(d, opts));
|
|
19581
|
+
} catch (err) {
|
|
19582
|
+
const e = err;
|
|
19583
|
+
emitError({ code: e.errorCode ?? "promote_failed", message: e.message }, opts, 1);
|
|
19584
|
+
}
|
|
19585
|
+
}
|
|
19586
|
+
var init_eval_promote = __esm({
|
|
19587
|
+
"src/commands/eval-promote.ts"() {
|
|
19588
|
+
"use strict";
|
|
19589
|
+
init_credentials();
|
|
19590
|
+
init_api();
|
|
19591
|
+
init_output();
|
|
19592
|
+
}
|
|
19593
|
+
});
|
|
19594
|
+
|
|
19496
19595
|
// src/commands/eval-test-sets.ts
|
|
19497
19596
|
var eval_test_sets_exports = {};
|
|
19498
19597
|
__export(eval_test_sets_exports, {
|
|
19499
19598
|
runEvalSetsCreate: () => runEvalSetsCreate,
|
|
19500
19599
|
slugifyTestSetName: () => slugifyTestSetName
|
|
19501
19600
|
});
|
|
19502
|
-
async function
|
|
19601
|
+
async function requireSession14(opts) {
|
|
19503
19602
|
const creds = await readCredentials();
|
|
19504
19603
|
if (!creds) {
|
|
19505
19604
|
emitError(
|
|
@@ -19539,7 +19638,7 @@ function printCreated(data, opts) {
|
|
|
19539
19638
|
}
|
|
19540
19639
|
async function runEvalSetsCreate(options) {
|
|
19541
19640
|
const opts = options;
|
|
19542
|
-
const session = await
|
|
19641
|
+
const session = await requireSession14(opts);
|
|
19543
19642
|
const brandId = await resolveBrandId(
|
|
19544
19643
|
session,
|
|
19545
19644
|
options.brand,
|
|
@@ -19608,7 +19707,7 @@ __export(eval_set_cases_exports, {
|
|
|
19608
19707
|
runEvalSetsAddCases: () => runEvalSetsAddCases,
|
|
19609
19708
|
unionCaseIds: () => unionCaseIds
|
|
19610
19709
|
});
|
|
19611
|
-
async function
|
|
19710
|
+
async function requireSession15(opts) {
|
|
19612
19711
|
const creds = await readCredentials();
|
|
19613
19712
|
if (!creds) {
|
|
19614
19713
|
emitError(
|
|
@@ -19639,7 +19738,7 @@ function unionCaseIds(existing, incoming) {
|
|
|
19639
19738
|
}
|
|
19640
19739
|
async function runEvalSetsAddCases(options) {
|
|
19641
19740
|
const opts = options;
|
|
19642
|
-
const session = await
|
|
19741
|
+
const session = await requireSession15(opts);
|
|
19643
19742
|
const incoming = parseCaseIds(options.cases);
|
|
19644
19743
|
if (incoming.length === 0) {
|
|
19645
19744
|
emitError(
|
|
@@ -20057,12 +20156,6 @@ skills.command("unsubscribe <agent> <skill-name>").description("Unsubscribe an a
|
|
|
20057
20156
|
const { runSkillsUnsubscribe: runSkillsUnsubscribe2 } = await Promise.resolve().then(() => (init_skills(), skills_exports));
|
|
20058
20157
|
await runSkillsUnsubscribe2(agent, skillName);
|
|
20059
20158
|
});
|
|
20060
|
-
skills.command("create <skill-name>").description("Scaffold a new org-custom skill in your workspace").option("--category <category>", "Optional category label (e.g. analysis)").option("--description <description>", "Optional one-line description").option("--no-sync", "Skip the auto-sync git pull on this command").action(
|
|
20061
|
-
async (skillName, options) => {
|
|
20062
|
-
const { runSkillsCreate: runSkillsCreate2 } = await Promise.resolve().then(() => (init_skills(), skills_exports));
|
|
20063
|
-
await runSkillsCreate2(skillName, options);
|
|
20064
|
-
}
|
|
20065
|
-
);
|
|
20066
20159
|
skills.command("update <agent> <skill-name>").description("Update an installed skill from the bamboo registry").option("--no-sync", "Skip the auto-sync git pull on this command").action(async (agent, skillName) => {
|
|
20067
20160
|
const { runSkillsUpdate: runSkillsUpdate2 } = await Promise.resolve().then(() => (init_skills(), skills_exports));
|
|
20068
20161
|
await runSkillsUpdate2(agent, skillName);
|
|
@@ -20216,6 +20309,14 @@ evals.command("ablate").description(
|
|
|
20216
20309
|
await runEvalAblate2(options);
|
|
20217
20310
|
}
|
|
20218
20311
|
);
|
|
20312
|
+
evals.command("promote").description(
|
|
20313
|
+
"Promote a correctness run as the baseline for its qrels version, or --demote the current one"
|
|
20314
|
+
).argument("[run-id]", "Run id printed by the correctness gate (omit with --demote)").requiredOption("--reason <text>", "Why this run is the reference now \u2014 recorded on the row").option("--pr <url>", "PR URL to record as the promotion's source").option("--demote", "Write a tombstone: the version has no baseline until a human promotes one").option("--brand <id>", "Brand id (with --demote)").option("--qrels-version <hash>", "qrels version to demote (with --demote)").option("--json", "Emit a JSON envelope for agent consumption").option("--fields <list>", "Project only these dotted field paths from data (comma-separated)").option("--quiet", "Suppress human chatter (data/errors only)").option("--no-sync", "Skip the auto-sync git pull on this command").action(
|
|
20315
|
+
async (runId, options) => {
|
|
20316
|
+
const { runEvalPromote: runEvalPromote2 } = await Promise.resolve().then(() => (init_eval_promote(), eval_promote_exports));
|
|
20317
|
+
await runEvalPromote2({ runId, ...options });
|
|
20318
|
+
}
|
|
20319
|
+
);
|
|
20219
20320
|
var evalSets = evals.command("sets").description("Test sets \u2014 the definition a run executes (cases attach separately)");
|
|
20220
20321
|
evalSets.command("create").description("Create an empty test set of a given evidence kind").requiredOption("--name <text>", 'Human name, e.g. "Connector parity"').requiredOption(
|
|
20221
20322
|
"--kind <kind>",
|