bamboo-agents 0.1.7-dev.87.b63312a → 0.1.7-dev.88.66a1ec6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/VERSION +1 -1
- package/dist/index.js +171 -5
- package/dist/index.js.map +4 -4
- package/package.json +1 -1
package/dist/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
0.1.7-dev.
|
|
1
|
+
0.1.7-dev.88.66a1ec6
|
package/dist/index.js
CHANGED
|
@@ -3037,7 +3037,7 @@ var VERSION;
|
|
|
3037
3037
|
var init_version = __esm({
|
|
3038
3038
|
"src/version.ts"() {
|
|
3039
3039
|
"use strict";
|
|
3040
|
-
VERSION = "0.1.7-dev.
|
|
3040
|
+
VERSION = "0.1.7-dev.88.66a1ec6";
|
|
3041
3041
|
}
|
|
3042
3042
|
});
|
|
3043
3043
|
|
|
@@ -13035,6 +13035,46 @@ var init_episodes2 = __esm({
|
|
|
13035
13035
|
}
|
|
13036
13036
|
});
|
|
13037
13037
|
|
|
13038
|
+
// ../../src/lib/api-contracts/cli/eval-baseline-promote.ts
|
|
13039
|
+
var Reason, PrUrl, EvalBaselinePromoteRunSchema, EvalBaselineDemoteSchema, EvalBaselinePromoteRequestSchema, EvalBaselinePromoteResponseSchema;
|
|
13040
|
+
var init_eval_baseline_promote = __esm({
|
|
13041
|
+
"../../src/lib/api-contracts/cli/eval-baseline-promote.ts"() {
|
|
13042
|
+
init_zod();
|
|
13043
|
+
Reason = external_exports.string().trim().min(1).max(2e3).describe("Why this run is the reference now. Recorded on the row.");
|
|
13044
|
+
PrUrl = external_exports.string().url().optional().describe("The PR this decision was made on, recorded as the row's source_url.");
|
|
13045
|
+
EvalBaselinePromoteRunSchema = external_exports.object({
|
|
13046
|
+
run_id: external_exports.string().uuid().describe("A completed qrels run in the caller's org \u2014 the id the correctness gate printed."),
|
|
13047
|
+
reason: Reason,
|
|
13048
|
+
pr_url: PrUrl,
|
|
13049
|
+
// A body that says `demote: true` but fails the demote shape must not fall
|
|
13050
|
+
// through to a promotion: objects strip unknown keys, so name it here.
|
|
13051
|
+
demote: external_exports.literal(false).optional()
|
|
13052
|
+
});
|
|
13053
|
+
EvalBaselineDemoteSchema = external_exports.object({
|
|
13054
|
+
demote: external_exports.literal(true),
|
|
13055
|
+
brand_id: external_exports.string().uuid(),
|
|
13056
|
+
qrels_version: external_exports.string().min(1).describe("The key to empty: its current baseline is tombstoned."),
|
|
13057
|
+
reason: Reason,
|
|
13058
|
+
pr_url: PrUrl
|
|
13059
|
+
});
|
|
13060
|
+
EvalBaselinePromoteRequestSchema = external_exports.union([
|
|
13061
|
+
EvalBaselineDemoteSchema,
|
|
13062
|
+
EvalBaselinePromoteRunSchema
|
|
13063
|
+
]);
|
|
13064
|
+
EvalBaselinePromoteResponseSchema = external_exports.object({
|
|
13065
|
+
promotion_id: external_exports.string().uuid(),
|
|
13066
|
+
kind: external_exports.enum(["promote", "demote"]),
|
|
13067
|
+
brand_id: external_exports.string().uuid(),
|
|
13068
|
+
qrels_version: external_exports.string(),
|
|
13069
|
+
/** Null after a demotion. */
|
|
13070
|
+
run_id: external_exports.string().uuid().nullable(),
|
|
13071
|
+
/** What the head pointed at before this row; null when the version had no baseline. */
|
|
13072
|
+
previous_run_id: external_exports.string().uuid().nullable(),
|
|
13073
|
+
promoted_at: external_exports.string()
|
|
13074
|
+
});
|
|
13075
|
+
}
|
|
13076
|
+
});
|
|
13077
|
+
|
|
13038
13078
|
// ../../src/lib/api-contracts/eval-roadmap/evidence-kinds.ts
|
|
13039
13079
|
var EvidenceKinds, EvidenceKindSchema;
|
|
13040
13080
|
var init_evidence_kinds = __esm({
|
|
@@ -13596,6 +13636,19 @@ async function startAbTest(bambooSession, body) {
|
|
|
13596
13636
|
const json = await res.json();
|
|
13597
13637
|
return EvalAbResponseSchema.parse(json.data);
|
|
13598
13638
|
}
|
|
13639
|
+
async function promoteBaseline(bambooSession, body) {
|
|
13640
|
+
const res = await cliApiFetch(`${apiBase()}/api/v1/eval/baseline-promotions`, {
|
|
13641
|
+
method: "POST",
|
|
13642
|
+
headers: {
|
|
13643
|
+
Authorization: `Bearer ${bambooSession}`,
|
|
13644
|
+
"Content-Type": "application/json"
|
|
13645
|
+
},
|
|
13646
|
+
body: JSON.stringify(body)
|
|
13647
|
+
});
|
|
13648
|
+
if (!res.ok) throw await parseError(res);
|
|
13649
|
+
const json = await res.json();
|
|
13650
|
+
return EvalBaselinePromoteResponseSchema.parse(json.data);
|
|
13651
|
+
}
|
|
13599
13652
|
async function createEvalTestSet(bambooSession, body) {
|
|
13600
13653
|
const res = await cliApiFetch(`${apiBase()}/api/v1/eval/test-sets`, {
|
|
13601
13654
|
method: "POST",
|
|
@@ -13674,6 +13727,7 @@ var init_api = __esm({
|
|
|
13674
13727
|
init_usage();
|
|
13675
13728
|
init_episodes2();
|
|
13676
13729
|
init_eval_ab();
|
|
13730
|
+
init_eval_baseline_promote();
|
|
13677
13731
|
init_eval_test_sets();
|
|
13678
13732
|
init_version();
|
|
13679
13733
|
PROD_DEFAULT_BASE = "https://withbamboo.com";
|
|
@@ -19434,13 +19488,117 @@ var init_eval_ab2 = __esm({
|
|
|
19434
19488
|
}
|
|
19435
19489
|
});
|
|
19436
19490
|
|
|
19491
|
+
// src/commands/eval-promote.ts
|
|
19492
|
+
var eval_promote_exports = {};
|
|
19493
|
+
__export(eval_promote_exports, {
|
|
19494
|
+
runEvalPromote: () => runEvalPromote
|
|
19495
|
+
});
|
|
19496
|
+
async function requireSession13(opts) {
|
|
19497
|
+
const creds = await readCredentials();
|
|
19498
|
+
if (!creds) {
|
|
19499
|
+
emitError(
|
|
19500
|
+
{
|
|
19501
|
+
code: "not_signed_in",
|
|
19502
|
+
message: "not signed in.",
|
|
19503
|
+
hint: "Run `bamboo-agents login` first."
|
|
19504
|
+
},
|
|
19505
|
+
opts,
|
|
19506
|
+
1
|
|
19507
|
+
);
|
|
19508
|
+
throw new Error("unreachable");
|
|
19509
|
+
}
|
|
19510
|
+
return creds.bamboo_session;
|
|
19511
|
+
}
|
|
19512
|
+
function short(version) {
|
|
19513
|
+
return version.slice(0, 12);
|
|
19514
|
+
}
|
|
19515
|
+
function render(data, opts) {
|
|
19516
|
+
human("", opts);
|
|
19517
|
+
if (data.kind === "demote") {
|
|
19518
|
+
human(
|
|
19519
|
+
` Baseline demoted for qrels_version ${short(data.qrels_version)} (was run ${data.previous_run_id}).`,
|
|
19520
|
+
opts
|
|
19521
|
+
);
|
|
19522
|
+
human(
|
|
19523
|
+
" The gate reports baseline_demoted for that version until a human promotes a run.",
|
|
19524
|
+
opts
|
|
19525
|
+
);
|
|
19526
|
+
} else {
|
|
19527
|
+
human(
|
|
19528
|
+
` Baseline promoted: run ${data.run_id} is the reference for qrels_version ${short(data.qrels_version)}` + (data.previous_run_id ? ` (was ${data.previous_run_id}).` : " (the version had none)."),
|
|
19529
|
+
opts
|
|
19530
|
+
);
|
|
19531
|
+
human(" Re-run the PR's retrieval-evals check: it compares flat against its own run.", opts);
|
|
19532
|
+
}
|
|
19533
|
+
human("", opts);
|
|
19534
|
+
}
|
|
19535
|
+
async function runEvalPromote(options) {
|
|
19536
|
+
const opts = { json: options.json, fields: options.fields, quiet: options.quiet };
|
|
19537
|
+
let body;
|
|
19538
|
+
if (options.demote) {
|
|
19539
|
+
if (!options.brand || !options.qrelsVersion) {
|
|
19540
|
+
emitError(
|
|
19541
|
+
{
|
|
19542
|
+
code: "missing_key",
|
|
19543
|
+
message: "--demote needs --brand <id> and --qrels-version <hash>.",
|
|
19544
|
+
hint: "A tombstone names the key it empties; both are on the gate report."
|
|
19545
|
+
},
|
|
19546
|
+
opts,
|
|
19547
|
+
1
|
|
19548
|
+
);
|
|
19549
|
+
return;
|
|
19550
|
+
}
|
|
19551
|
+
body = {
|
|
19552
|
+
demote: true,
|
|
19553
|
+
brand_id: options.brand,
|
|
19554
|
+
qrels_version: options.qrelsVersion,
|
|
19555
|
+
reason: options.reason,
|
|
19556
|
+
...options.pr ? { pr_url: options.pr } : {}
|
|
19557
|
+
};
|
|
19558
|
+
} else {
|
|
19559
|
+
if (!options.runId) {
|
|
19560
|
+
emitError(
|
|
19561
|
+
{
|
|
19562
|
+
code: "missing_run_id",
|
|
19563
|
+
message: "Pass the run id the correctness gate printed.",
|
|
19564
|
+
hint: 'bamboo-agents eval promote <run-id> --reason "why this drop is intentional"'
|
|
19565
|
+
},
|
|
19566
|
+
opts,
|
|
19567
|
+
1
|
|
19568
|
+
);
|
|
19569
|
+
return;
|
|
19570
|
+
}
|
|
19571
|
+
body = {
|
|
19572
|
+
run_id: options.runId,
|
|
19573
|
+
reason: options.reason,
|
|
19574
|
+
...options.pr ? { pr_url: options.pr } : {}
|
|
19575
|
+
};
|
|
19576
|
+
}
|
|
19577
|
+
const session = await requireSession13(opts);
|
|
19578
|
+
try {
|
|
19579
|
+
const data = await promoteBaseline(session, body);
|
|
19580
|
+
emitSuccess(data, opts, (d) => render(d, opts));
|
|
19581
|
+
} catch (err) {
|
|
19582
|
+
const e = err;
|
|
19583
|
+
emitError({ code: e.errorCode ?? "promote_failed", message: e.message }, opts, 1);
|
|
19584
|
+
}
|
|
19585
|
+
}
|
|
19586
|
+
var init_eval_promote = __esm({
|
|
19587
|
+
"src/commands/eval-promote.ts"() {
|
|
19588
|
+
"use strict";
|
|
19589
|
+
init_credentials();
|
|
19590
|
+
init_api();
|
|
19591
|
+
init_output();
|
|
19592
|
+
}
|
|
19593
|
+
});
|
|
19594
|
+
|
|
19437
19595
|
// src/commands/eval-test-sets.ts
|
|
19438
19596
|
var eval_test_sets_exports = {};
|
|
19439
19597
|
__export(eval_test_sets_exports, {
|
|
19440
19598
|
runEvalSetsCreate: () => runEvalSetsCreate,
|
|
19441
19599
|
slugifyTestSetName: () => slugifyTestSetName
|
|
19442
19600
|
});
|
|
19443
|
-
async function
|
|
19601
|
+
async function requireSession14(opts) {
|
|
19444
19602
|
const creds = await readCredentials();
|
|
19445
19603
|
if (!creds) {
|
|
19446
19604
|
emitError(
|
|
@@ -19480,7 +19638,7 @@ function printCreated(data, opts) {
|
|
|
19480
19638
|
}
|
|
19481
19639
|
async function runEvalSetsCreate(options) {
|
|
19482
19640
|
const opts = options;
|
|
19483
|
-
const session = await
|
|
19641
|
+
const session = await requireSession14(opts);
|
|
19484
19642
|
const brandId = await resolveBrandId(
|
|
19485
19643
|
session,
|
|
19486
19644
|
options.brand,
|
|
@@ -19549,7 +19707,7 @@ __export(eval_set_cases_exports, {
|
|
|
19549
19707
|
runEvalSetsAddCases: () => runEvalSetsAddCases,
|
|
19550
19708
|
unionCaseIds: () => unionCaseIds
|
|
19551
19709
|
});
|
|
19552
|
-
async function
|
|
19710
|
+
async function requireSession15(opts) {
|
|
19553
19711
|
const creds = await readCredentials();
|
|
19554
19712
|
if (!creds) {
|
|
19555
19713
|
emitError(
|
|
@@ -19580,7 +19738,7 @@ function unionCaseIds(existing, incoming) {
|
|
|
19580
19738
|
}
|
|
19581
19739
|
async function runEvalSetsAddCases(options) {
|
|
19582
19740
|
const opts = options;
|
|
19583
|
-
const session = await
|
|
19741
|
+
const session = await requireSession15(opts);
|
|
19584
19742
|
const incoming = parseCaseIds(options.cases);
|
|
19585
19743
|
if (incoming.length === 0) {
|
|
19586
19744
|
emitError(
|
|
@@ -20151,6 +20309,14 @@ evals.command("ablate").description(
|
|
|
20151
20309
|
await runEvalAblate2(options);
|
|
20152
20310
|
}
|
|
20153
20311
|
);
|
|
20312
|
+
evals.command("promote").description(
|
|
20313
|
+
"Promote a correctness run as the baseline for its qrels version, or --demote the current one"
|
|
20314
|
+
).argument("[run-id]", "Run id printed by the correctness gate (omit with --demote)").requiredOption("--reason <text>", "Why this run is the reference now \u2014 recorded on the row").option("--pr <url>", "PR URL to record as the promotion's source").option("--demote", "Write a tombstone: the version has no baseline until a human promotes one").option("--brand <id>", "Brand id (with --demote)").option("--qrels-version <hash>", "qrels version to demote (with --demote)").option("--json", "Emit a JSON envelope for agent consumption").option("--fields <list>", "Project only these dotted field paths from data (comma-separated)").option("--quiet", "Suppress human chatter (data/errors only)").option("--no-sync", "Skip the auto-sync git pull on this command").action(
|
|
20315
|
+
async (runId, options) => {
|
|
20316
|
+
const { runEvalPromote: runEvalPromote2 } = await Promise.resolve().then(() => (init_eval_promote(), eval_promote_exports));
|
|
20317
|
+
await runEvalPromote2({ runId, ...options });
|
|
20318
|
+
}
|
|
20319
|
+
);
|
|
20154
20320
|
var evalSets = evals.command("sets").description("Test sets \u2014 the definition a run executes (cases attach separately)");
|
|
20155
20321
|
evalSets.command("create").description("Create an empty test set of a given evidence kind").requiredOption("--name <text>", 'Human name, e.g. "Connector parity"').requiredOption(
|
|
20156
20322
|
"--kind <kind>",
|