bamboo-agents 0.1.7 → 0.1.8-dev.90.eaeaaa5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/VERSION +1 -1
- package/dist/index.js +656 -76
- package/dist/index.js.map +4 -4
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -3037,7 +3037,7 @@ var VERSION;
|
|
|
3037
3037
|
var init_version = __esm({
|
|
3038
3038
|
"src/version.ts"() {
|
|
3039
3039
|
"use strict";
|
|
3040
|
-
VERSION = "0.1.
|
|
3040
|
+
VERSION = "0.1.8-dev.90.eaeaaa5";
|
|
3041
3041
|
}
|
|
3042
3042
|
});
|
|
3043
3043
|
|
|
@@ -13035,6 +13035,46 @@ var init_episodes2 = __esm({
|
|
|
13035
13035
|
}
|
|
13036
13036
|
});
|
|
13037
13037
|
|
|
13038
|
+
// ../../src/lib/api-contracts/cli/eval-baseline-promote.ts
|
|
13039
|
+
var Reason, PrUrl, EvalBaselinePromoteRunSchema, EvalBaselineDemoteSchema, EvalBaselinePromoteRequestSchema, EvalBaselinePromoteResponseSchema;
|
|
13040
|
+
var init_eval_baseline_promote = __esm({
|
|
13041
|
+
"../../src/lib/api-contracts/cli/eval-baseline-promote.ts"() {
|
|
13042
|
+
init_zod();
|
|
13043
|
+
Reason = external_exports.string().trim().min(1).max(2e3).describe("Why this run is the reference now. Recorded on the row.");
|
|
13044
|
+
PrUrl = external_exports.string().url().optional().describe("The PR this decision was made on, recorded as the row's source_url.");
|
|
13045
|
+
EvalBaselinePromoteRunSchema = external_exports.object({
|
|
13046
|
+
run_id: external_exports.string().uuid().describe("A completed qrels run in the caller's org \u2014 the id the correctness gate printed."),
|
|
13047
|
+
reason: Reason,
|
|
13048
|
+
pr_url: PrUrl,
|
|
13049
|
+
// A body that says `demote: true` but fails the demote shape must not fall
|
|
13050
|
+
// through to a promotion: objects strip unknown keys, so name it here.
|
|
13051
|
+
demote: external_exports.literal(false).optional()
|
|
13052
|
+
});
|
|
13053
|
+
EvalBaselineDemoteSchema = external_exports.object({
|
|
13054
|
+
demote: external_exports.literal(true),
|
|
13055
|
+
brand_id: external_exports.string().uuid(),
|
|
13056
|
+
qrels_version: external_exports.string().min(1).describe("The key to empty: its current baseline is tombstoned."),
|
|
13057
|
+
reason: Reason,
|
|
13058
|
+
pr_url: PrUrl
|
|
13059
|
+
});
|
|
13060
|
+
EvalBaselinePromoteRequestSchema = external_exports.union([
|
|
13061
|
+
EvalBaselineDemoteSchema,
|
|
13062
|
+
EvalBaselinePromoteRunSchema
|
|
13063
|
+
]);
|
|
13064
|
+
EvalBaselinePromoteResponseSchema = external_exports.object({
|
|
13065
|
+
promotion_id: external_exports.string().uuid(),
|
|
13066
|
+
kind: external_exports.enum(["promote", "demote"]),
|
|
13067
|
+
brand_id: external_exports.string().uuid(),
|
|
13068
|
+
qrels_version: external_exports.string(),
|
|
13069
|
+
/** Null after a demotion. */
|
|
13070
|
+
run_id: external_exports.string().uuid().nullable(),
|
|
13071
|
+
/** What the head pointed at before this row; null when the version had no baseline. */
|
|
13072
|
+
previous_run_id: external_exports.string().uuid().nullable(),
|
|
13073
|
+
promoted_at: external_exports.string()
|
|
13074
|
+
});
|
|
13075
|
+
}
|
|
13076
|
+
});
|
|
13077
|
+
|
|
13038
13078
|
// ../../src/lib/api-contracts/eval-roadmap/evidence-kinds.ts
|
|
13039
13079
|
var EvidenceKinds, EvidenceKindSchema;
|
|
13040
13080
|
var init_evidence_kinds = __esm({
|
|
@@ -13098,6 +13138,312 @@ var init_eval_test_sets = __esm({
|
|
|
13098
13138
|
}
|
|
13099
13139
|
});
|
|
13100
13140
|
|
|
13141
|
+
// ../../src/lib/api-contracts/cli/eval-management.ts
|
|
13142
|
+
var EvalManagementOperationSchema, EvalManagementBindingSchema, EvalManagementInputHelpSchema, EVAL_MANAGEMENT_INPUTS, common, EVAL_MANAGEMENT_BINDINGS, EvalManagementCatalogResponseSchema, EvalManagementCallSchema, EvalManagementResultSchema;
|
|
13143
|
+
var init_eval_management = __esm({
|
|
13144
|
+
"../../src/lib/api-contracts/cli/eval-management.ts"() {
|
|
13145
|
+
init_zod();
|
|
13146
|
+
EvalManagementOperationSchema = external_exports.enum([
|
|
13147
|
+
"investigation.list",
|
|
13148
|
+
"investigation.get",
|
|
13149
|
+
"investigation.create",
|
|
13150
|
+
"investigation.update",
|
|
13151
|
+
"investigation.hold",
|
|
13152
|
+
"investigation.resume",
|
|
13153
|
+
"investigation.cancel",
|
|
13154
|
+
"investigation.conclude_claim",
|
|
13155
|
+
"investigation.close",
|
|
13156
|
+
"investigation.reopen",
|
|
13157
|
+
"investigation.retire",
|
|
13158
|
+
"wave.list",
|
|
13159
|
+
"wave.get",
|
|
13160
|
+
"wave.create",
|
|
13161
|
+
"wave.update",
|
|
13162
|
+
"wave.schedule"
|
|
13163
|
+
]);
|
|
13164
|
+
EvalManagementBindingSchema = external_exports.object({
|
|
13165
|
+
operation: EvalManagementOperationSchema,
|
|
13166
|
+
description: external_exports.string(),
|
|
13167
|
+
method: external_exports.enum(["GET", "POST", "PATCH"]),
|
|
13168
|
+
path: external_exports.string(),
|
|
13169
|
+
cli_command: external_exports.literal("bamboo-agents eval manage"),
|
|
13170
|
+
supported_runtimes: external_exports.array(external_exports.enum(["cli", "api"])),
|
|
13171
|
+
authority: external_exports.string(),
|
|
13172
|
+
side_effect: external_exports.string(),
|
|
13173
|
+
receipt: external_exports.string(),
|
|
13174
|
+
minimum_cli_version: external_exports.string()
|
|
13175
|
+
});
|
|
13176
|
+
EvalManagementInputHelpSchema = external_exports.object({
|
|
13177
|
+
required: external_exports.array(external_exports.string()),
|
|
13178
|
+
optional: external_exports.array(external_exports.string()),
|
|
13179
|
+
server_injected: external_exports.record(external_exports.string(), external_exports.string()).optional()
|
|
13180
|
+
});
|
|
13181
|
+
EVAL_MANAGEMENT_INPUTS = {
|
|
13182
|
+
"investigation.list": {
|
|
13183
|
+
required: [],
|
|
13184
|
+
optional: [
|
|
13185
|
+
"brand_id",
|
|
13186
|
+
"agent_id",
|
|
13187
|
+
"lifecycle",
|
|
13188
|
+
"search",
|
|
13189
|
+
"limit",
|
|
13190
|
+
"cursor"
|
|
13191
|
+
]
|
|
13192
|
+
},
|
|
13193
|
+
"investigation.get": { required: ["id"], optional: [] },
|
|
13194
|
+
"investigation.create": {
|
|
13195
|
+
required: ["brand_id", "agent_id", "title"],
|
|
13196
|
+
optional: ["hypothesis", "provenance", "play_type", "claims"]
|
|
13197
|
+
},
|
|
13198
|
+
"investigation.update": {
|
|
13199
|
+
required: ["id", "brand_id"],
|
|
13200
|
+
optional: [
|
|
13201
|
+
"title",
|
|
13202
|
+
"hypothesis",
|
|
13203
|
+
"play_type",
|
|
13204
|
+
"provenance",
|
|
13205
|
+
"claims",
|
|
13206
|
+
"expected_source_sha"
|
|
13207
|
+
]
|
|
13208
|
+
},
|
|
13209
|
+
"investigation.hold": {
|
|
13210
|
+
required: ["id", "brand_id", "expected_revision", "idempotency_key"],
|
|
13211
|
+
optional: [],
|
|
13212
|
+
server_injected: { operation: "hold" }
|
|
13213
|
+
},
|
|
13214
|
+
"investigation.resume": {
|
|
13215
|
+
required: ["id", "brand_id", "expected_revision", "idempotency_key"],
|
|
13216
|
+
optional: [],
|
|
13217
|
+
server_injected: { operation: "resume" }
|
|
13218
|
+
},
|
|
13219
|
+
"investigation.cancel": {
|
|
13220
|
+
required: ["id", "brand_id", "expected_revision", "idempotency_key"],
|
|
13221
|
+
optional: [],
|
|
13222
|
+
server_injected: { operation: "cancel" }
|
|
13223
|
+
},
|
|
13224
|
+
"investigation.conclude_claim": {
|
|
13225
|
+
required: [
|
|
13226
|
+
"id",
|
|
13227
|
+
"brand_id",
|
|
13228
|
+
"command_id",
|
|
13229
|
+
"claim_key",
|
|
13230
|
+
"conclusion",
|
|
13231
|
+
"verdict_tier",
|
|
13232
|
+
"evidence_refs",
|
|
13233
|
+
"body"
|
|
13234
|
+
],
|
|
13235
|
+
optional: ["decision", "next_action", "expected_source_sha"],
|
|
13236
|
+
server_injected: { operation: "conclude_claim" }
|
|
13237
|
+
},
|
|
13238
|
+
"investigation.close": {
|
|
13239
|
+
required: ["id", "brand_id", "command_id", "body"],
|
|
13240
|
+
optional: ["expected_source_sha"],
|
|
13241
|
+
server_injected: { operation: "close" }
|
|
13242
|
+
},
|
|
13243
|
+
"investigation.reopen": {
|
|
13244
|
+
required: ["id", "brand_id", "command_id", "body"],
|
|
13245
|
+
optional: ["expected_source_sha"],
|
|
13246
|
+
server_injected: { operation: "reopen" }
|
|
13247
|
+
},
|
|
13248
|
+
"investigation.retire": {
|
|
13249
|
+
required: ["id", "brand_id", "command_id", "body"],
|
|
13250
|
+
optional: ["expected_source_sha"],
|
|
13251
|
+
server_injected: { operation: "retire" }
|
|
13252
|
+
},
|
|
13253
|
+
"wave.list": { required: ["brand_id"], optional: [] },
|
|
13254
|
+
"wave.get": { required: ["id", "brand_id"], optional: [] },
|
|
13255
|
+
"wave.create": {
|
|
13256
|
+
required: ["brand_id", "name"],
|
|
13257
|
+
optional: ["timezone", "not_before"]
|
|
13258
|
+
},
|
|
13259
|
+
"wave.update": {
|
|
13260
|
+
required: ["id", "brand_id", "expected_revision"],
|
|
13261
|
+
optional: [
|
|
13262
|
+
"name",
|
|
13263
|
+
"status",
|
|
13264
|
+
"timezone",
|
|
13265
|
+
"not_before",
|
|
13266
|
+
"add_investigation_ids",
|
|
13267
|
+
"remove_investigation_ids"
|
|
13268
|
+
]
|
|
13269
|
+
},
|
|
13270
|
+
"wave.schedule": {
|
|
13271
|
+
required: ["id", "brand_id", "agent_id", "idempotency_key"],
|
|
13272
|
+
optional: ["not_before"]
|
|
13273
|
+
}
|
|
13274
|
+
};
|
|
13275
|
+
common = {
|
|
13276
|
+
cli_command: "bamboo-agents eval manage",
|
|
13277
|
+
supported_runtimes: ["cli", "api"],
|
|
13278
|
+
authority: "Authenticated human or attributed agent with access to the target brand",
|
|
13279
|
+
minimum_cli_version: "0.1.8"
|
|
13280
|
+
};
|
|
13281
|
+
EVAL_MANAGEMENT_BINDINGS = [
|
|
13282
|
+
{
|
|
13283
|
+
...common,
|
|
13284
|
+
operation: "investigation.list",
|
|
13285
|
+
description: "List current or historical investigations",
|
|
13286
|
+
method: "GET",
|
|
13287
|
+
path: "/api/v1/plays",
|
|
13288
|
+
side_effect: "none",
|
|
13289
|
+
receipt: "investigation read model"
|
|
13290
|
+
},
|
|
13291
|
+
{
|
|
13292
|
+
...common,
|
|
13293
|
+
operation: "investigation.get",
|
|
13294
|
+
description: "Read one investigation with its claims and evidence",
|
|
13295
|
+
method: "GET",
|
|
13296
|
+
path: "/api/v1/plays/{id}",
|
|
13297
|
+
side_effect: "none",
|
|
13298
|
+
receipt: "investigation read model"
|
|
13299
|
+
},
|
|
13300
|
+
{
|
|
13301
|
+
...common,
|
|
13302
|
+
operation: "investigation.create",
|
|
13303
|
+
description: "Open an authored investigation; does not queue execution",
|
|
13304
|
+
method: "POST",
|
|
13305
|
+
path: "/api/v1/plays",
|
|
13306
|
+
side_effect: "repo-authored investigation revision",
|
|
13307
|
+
receipt: "source path, source SHA, and investigation id"
|
|
13308
|
+
},
|
|
13309
|
+
{
|
|
13310
|
+
...common,
|
|
13311
|
+
operation: "investigation.update",
|
|
13312
|
+
description: "Revise an authored investigation with optimistic source concurrency",
|
|
13313
|
+
method: "PATCH",
|
|
13314
|
+
path: "/api/v1/plays/{id}",
|
|
13315
|
+
side_effect: "repo-authored investigation revision",
|
|
13316
|
+
receipt: "source SHA and publication result"
|
|
13317
|
+
},
|
|
13318
|
+
{
|
|
13319
|
+
...common,
|
|
13320
|
+
operation: "investigation.hold",
|
|
13321
|
+
description: "Stop future queue claims without discarding pending work",
|
|
13322
|
+
method: "POST",
|
|
13323
|
+
path: "/api/v1/eval/investigations/{id}/control",
|
|
13324
|
+
side_effect: "execution gate becomes held",
|
|
13325
|
+
receipt: "control receipt and resulting revision"
|
|
13326
|
+
},
|
|
13327
|
+
{
|
|
13328
|
+
...common,
|
|
13329
|
+
operation: "investigation.resume",
|
|
13330
|
+
description: "Allow pending work to be claimed again",
|
|
13331
|
+
method: "POST",
|
|
13332
|
+
path: "/api/v1/eval/investigations/{id}/control",
|
|
13333
|
+
side_effect: "execution gate becomes active",
|
|
13334
|
+
receipt: "control receipt and resulting revision"
|
|
13335
|
+
},
|
|
13336
|
+
{
|
|
13337
|
+
...common,
|
|
13338
|
+
operation: "investigation.cancel",
|
|
13339
|
+
description: "Cancel pending work without deleting served evidence",
|
|
13340
|
+
method: "POST",
|
|
13341
|
+
path: "/api/v1/eval/investigations/{id}/control",
|
|
13342
|
+
side_effect: "pending queue items are canceled",
|
|
13343
|
+
receipt: "control receipt with affected count and served ids"
|
|
13344
|
+
},
|
|
13345
|
+
{
|
|
13346
|
+
...common,
|
|
13347
|
+
operation: "investigation.conclude_claim",
|
|
13348
|
+
description: "Record an evidence-linked claim conclusion",
|
|
13349
|
+
method: "POST",
|
|
13350
|
+
path: "/api/v1/eval/investigations/{id}/decisions",
|
|
13351
|
+
side_effect: "append-only judgment plus repo publication",
|
|
13352
|
+
receipt: "decision, judgment, publication, and evidence references"
|
|
13353
|
+
},
|
|
13354
|
+
{
|
|
13355
|
+
...common,
|
|
13356
|
+
operation: "investigation.close",
|
|
13357
|
+
description: "Close an investigation through an evidence-linked decision",
|
|
13358
|
+
method: "POST",
|
|
13359
|
+
path: "/api/v1/eval/investigations/{id}/decisions",
|
|
13360
|
+
side_effect: "repo lifecycle closes and pending work is canceled",
|
|
13361
|
+
receipt: "decision, publication, cancellation count, and served ids"
|
|
13362
|
+
},
|
|
13363
|
+
{
|
|
13364
|
+
...common,
|
|
13365
|
+
operation: "investigation.reopen",
|
|
13366
|
+
description: "Reopen while retaining prior decisions and runs",
|
|
13367
|
+
method: "POST",
|
|
13368
|
+
path: "/api/v1/eval/investigations/{id}/decisions",
|
|
13369
|
+
side_effect: "repo lifecycle opens and held execution resumes",
|
|
13370
|
+
receipt: "decision and publication receipt"
|
|
13371
|
+
},
|
|
13372
|
+
{
|
|
13373
|
+
...common,
|
|
13374
|
+
operation: "investigation.retire",
|
|
13375
|
+
description: "Retire an investigation without deleting its history",
|
|
13376
|
+
method: "POST",
|
|
13377
|
+
path: "/api/v1/eval/investigations/{id}/decisions",
|
|
13378
|
+
side_effect: "repo lifecycle retires and pending work is canceled",
|
|
13379
|
+
receipt: "decision, publication, cancellation count, and served ids"
|
|
13380
|
+
},
|
|
13381
|
+
{
|
|
13382
|
+
...common,
|
|
13383
|
+
operation: "wave.list",
|
|
13384
|
+
description: "List coordinated investigation waves",
|
|
13385
|
+
method: "GET",
|
|
13386
|
+
path: "/api/v1/eval/waves",
|
|
13387
|
+
side_effect: "none",
|
|
13388
|
+
receipt: "wave read model"
|
|
13389
|
+
},
|
|
13390
|
+
{
|
|
13391
|
+
...common,
|
|
13392
|
+
operation: "wave.get",
|
|
13393
|
+
description: "Read membership revisions and admission receipts for one wave",
|
|
13394
|
+
method: "GET",
|
|
13395
|
+
path: "/api/v1/eval/waves/{id}",
|
|
13396
|
+
side_effect: "none",
|
|
13397
|
+
receipt: "wave, members, and request receipts"
|
|
13398
|
+
},
|
|
13399
|
+
{
|
|
13400
|
+
...common,
|
|
13401
|
+
operation: "wave.create",
|
|
13402
|
+
description: "Create a coordination container; does not schedule it",
|
|
13403
|
+
method: "POST",
|
|
13404
|
+
path: "/api/v1/eval/waves",
|
|
13405
|
+
side_effect: "operational wave row",
|
|
13406
|
+
receipt: "wave id and membership revision"
|
|
13407
|
+
},
|
|
13408
|
+
{
|
|
13409
|
+
...common,
|
|
13410
|
+
operation: "wave.update",
|
|
13411
|
+
description: "Revise wave metadata or membership atomically",
|
|
13412
|
+
method: "PATCH",
|
|
13413
|
+
path: "/api/v1/eval/waves/{id}",
|
|
13414
|
+
side_effect: "wave metadata or membership revision",
|
|
13415
|
+
receipt: "wave and resulting membership revision"
|
|
13416
|
+
},
|
|
13417
|
+
{
|
|
13418
|
+
...common,
|
|
13419
|
+
operation: "wave.schedule",
|
|
13420
|
+
description: "Freeze wave membership and admit each investigation idempotently",
|
|
13421
|
+
method: "POST",
|
|
13422
|
+
path: "/api/v1/eval/waves/{id}/schedule",
|
|
13423
|
+
side_effect: "durable run and queue admission",
|
|
13424
|
+
receipt: "wave request and per-investigation run receipts"
|
|
13425
|
+
}
|
|
13426
|
+
];
|
|
13427
|
+
EvalManagementCatalogResponseSchema = external_exports.object({
|
|
13428
|
+
contract_version: external_exports.literal(1),
|
|
13429
|
+
bindings: external_exports.array(EvalManagementBindingSchema),
|
|
13430
|
+
inputs: external_exports.record(
|
|
13431
|
+
EvalManagementOperationSchema,
|
|
13432
|
+
EvalManagementInputHelpSchema
|
|
13433
|
+
),
|
|
13434
|
+
related_cli_commands: external_exports.array(external_exports.string())
|
|
13435
|
+
});
|
|
13436
|
+
EvalManagementCallSchema = external_exports.object({
|
|
13437
|
+
operation: EvalManagementOperationSchema,
|
|
13438
|
+
input: external_exports.record(external_exports.string(), external_exports.unknown())
|
|
13439
|
+
});
|
|
13440
|
+
EvalManagementResultSchema = external_exports.object({
|
|
13441
|
+
operation: EvalManagementOperationSchema,
|
|
13442
|
+
data: external_exports.unknown()
|
|
13443
|
+
});
|
|
13444
|
+
}
|
|
13445
|
+
});
|
|
13446
|
+
|
|
13101
13447
|
// src/lib/api.ts
|
|
13102
13448
|
import { platform as platform2 } from "node:os";
|
|
13103
13449
|
function defaultApiBase(version) {
|
|
@@ -13515,22 +13861,6 @@ async function updateSkillFromRegistry(bambooSession, agentId, skillName) {
|
|
|
13515
13861
|
const json = await res.json();
|
|
13516
13862
|
return SkillSubscriptionsResponseSchema.parse(json.data);
|
|
13517
13863
|
}
|
|
13518
|
-
async function scaffoldSkill(bambooSession, args) {
|
|
13519
|
-
const res = await cliApiFetch(
|
|
13520
|
-
`${apiBase()}/api/v1/orgs/me/skills/scaffold`,
|
|
13521
|
-
{
|
|
13522
|
-
method: "POST",
|
|
13523
|
-
headers: {
|
|
13524
|
-
Authorization: `Bearer ${bambooSession}`,
|
|
13525
|
-
"Content-Type": "application/json"
|
|
13526
|
-
},
|
|
13527
|
-
body: JSON.stringify(args)
|
|
13528
|
-
}
|
|
13529
|
-
);
|
|
13530
|
-
if (!res.ok) throw await parseError(res);
|
|
13531
|
-
const json = await res.json();
|
|
13532
|
-
return ScaffoldSkillResponseSchema.parse(json.data);
|
|
13533
|
-
}
|
|
13534
13864
|
async function patchBrand(bambooSession, brandId, websiteUrl) {
|
|
13535
13865
|
const res = await cliApiFetch(
|
|
13536
13866
|
`${apiBase()}/api/v1/cli/brands/${encodeURIComponent(brandId)}`,
|
|
@@ -13612,6 +13942,19 @@ async function startAbTest(bambooSession, body) {
|
|
|
13612
13942
|
const json = await res.json();
|
|
13613
13943
|
return EvalAbResponseSchema.parse(json.data);
|
|
13614
13944
|
}
|
|
13945
|
+
async function promoteBaseline(bambooSession, body) {
|
|
13946
|
+
const res = await cliApiFetch(`${apiBase()}/api/v1/eval/baseline-promotions`, {
|
|
13947
|
+
method: "POST",
|
|
13948
|
+
headers: {
|
|
13949
|
+
Authorization: `Bearer ${bambooSession}`,
|
|
13950
|
+
"Content-Type": "application/json"
|
|
13951
|
+
},
|
|
13952
|
+
body: JSON.stringify(body)
|
|
13953
|
+
});
|
|
13954
|
+
if (!res.ok) throw await parseError(res);
|
|
13955
|
+
const json = await res.json();
|
|
13956
|
+
return EvalBaselinePromoteResponseSchema.parse(json.data);
|
|
13957
|
+
}
|
|
13615
13958
|
async function createEvalTestSet(bambooSession, body) {
|
|
13616
13959
|
const res = await cliApiFetch(`${apiBase()}/api/v1/eval/test-sets`, {
|
|
13617
13960
|
method: "POST",
|
|
@@ -13625,6 +13968,68 @@ async function createEvalTestSet(bambooSession, body) {
|
|
|
13625
13968
|
const json = await res.json();
|
|
13626
13969
|
return CliEvalTestSetCreateResponseSchema.parse(json.data);
|
|
13627
13970
|
}
|
|
13971
|
+
async function getEvalManagementOperations(bambooSession) {
|
|
13972
|
+
const res = await cliApiFetch(`${apiBase()}/api/v1/eval/operations`, {
|
|
13973
|
+
headers: { Authorization: `Bearer ${bambooSession}` }
|
|
13974
|
+
});
|
|
13975
|
+
if (!res.ok) throw await parseError(res);
|
|
13976
|
+
const json = await res.json();
|
|
13977
|
+
return EvalManagementCatalogResponseSchema.parse(json.data);
|
|
13978
|
+
}
|
|
13979
|
+
async function callEvalManagement(bambooSession, call) {
|
|
13980
|
+
const binding = EVAL_MANAGEMENT_BINDINGS.find((candidate) => candidate.operation === call.operation);
|
|
13981
|
+
if (!binding) throw new ApiError(`Unsupported operation: ${call.operation}`, 400, "unsupported_operation");
|
|
13982
|
+
const help = EVAL_MANAGEMENT_INPUTS[call.operation];
|
|
13983
|
+
const missing = help.required.filter(
|
|
13984
|
+
(field) => call.input[field] === void 0 || call.input[field] === null || call.input[field] === ""
|
|
13985
|
+
);
|
|
13986
|
+
if (missing.length) {
|
|
13987
|
+
throw new ApiError(`${call.operation} requires input fields: ${missing.join(", ")}`, 400, "missing_required_input");
|
|
13988
|
+
}
|
|
13989
|
+
const allowed = /* @__PURE__ */ new Set([...help.required, ...help.optional]);
|
|
13990
|
+
const unknown = Object.keys(call.input).filter((field) => !allowed.has(field));
|
|
13991
|
+
if (unknown.length) {
|
|
13992
|
+
throw new ApiError(`${call.operation} does not accept input fields: ${unknown.join(", ")}`, 400, "unknown_input");
|
|
13993
|
+
}
|
|
13994
|
+
const input = { ...call.input };
|
|
13995
|
+
let path2 = binding.path;
|
|
13996
|
+
if (path2.includes("{id}")) {
|
|
13997
|
+
const id = input.id;
|
|
13998
|
+
if (typeof id !== "string" || id.length === 0) {
|
|
13999
|
+
throw new ApiError(`${call.operation} requires input.id`, 400, "missing_resource_id");
|
|
14000
|
+
}
|
|
14001
|
+
path2 = path2.replace("{id}", encodeURIComponent(id));
|
|
14002
|
+
delete input.id;
|
|
14003
|
+
}
|
|
14004
|
+
const action = call.operation.split(".")[1] ?? "";
|
|
14005
|
+
if (call.operation.startsWith("investigation.") && ["hold", "resume", "cancel", "conclude_claim", "close", "reopen", "retire"].includes(action)) {
|
|
14006
|
+
input.operation = action;
|
|
14007
|
+
}
|
|
14008
|
+
let url = `${apiBase()}${path2}`;
|
|
14009
|
+
const init = {
|
|
14010
|
+
method: binding.method,
|
|
14011
|
+
headers: { Authorization: `Bearer ${bambooSession}` }
|
|
14012
|
+
};
|
|
14013
|
+
if (binding.method === "GET") {
|
|
14014
|
+
const query = new URLSearchParams();
|
|
14015
|
+
for (const [key, value] of Object.entries(input)) {
|
|
14016
|
+
if (value !== void 0 && value !== null) query.set(key, String(value));
|
|
14017
|
+
}
|
|
14018
|
+
const encoded = query.toString();
|
|
14019
|
+
if (encoded) url += `?${encoded}`;
|
|
14020
|
+
} else {
|
|
14021
|
+
init.headers["Content-Type"] = "application/json";
|
|
14022
|
+
init.body = JSON.stringify(input);
|
|
14023
|
+
}
|
|
14024
|
+
const res = await cliApiFetch(url, init);
|
|
14025
|
+
if (!res.ok) throw await parseError(res);
|
|
14026
|
+
const json = await res.json();
|
|
14027
|
+
const keys = Object.keys(json);
|
|
14028
|
+
return EvalManagementResultSchema.parse({
|
|
14029
|
+
operation: call.operation,
|
|
14030
|
+
data: keys.length === 1 && keys[0] === "data" ? json.data : json
|
|
14031
|
+
});
|
|
14032
|
+
}
|
|
13628
14033
|
async function getEvalTestSetMemberships(bambooSession, brandId) {
|
|
13629
14034
|
const url = `${apiBase()}/api/v1/eval/test-roadmap?brand_id=${encodeURIComponent(brandId)}`;
|
|
13630
14035
|
const res = await cliApiFetch(url, {
|
|
@@ -13664,7 +14069,7 @@ async function postRecEpisode(bambooSession, body) {
|
|
|
13664
14069
|
const json = await res.json();
|
|
13665
14070
|
return CliRecEpisodeResponseSchema.parse(json);
|
|
13666
14071
|
}
|
|
13667
|
-
var PROD_DEFAULT_BASE, DEV_DEFAULT_BASE, REQUEST_TIMEOUT_MS, ApiError, OAuthRelinkStartResponseSchema, ProvisionWorkspaceResponseSchema, FirstAutoInstalledAgentResponseSchema, SkillCatalogEntrySchema, SkillCatalogResponseSchema, SkillSubscriptionsResponseSchema
|
|
14072
|
+
var PROD_DEFAULT_BASE, DEV_DEFAULT_BASE, REQUEST_TIMEOUT_MS, ApiError, OAuthRelinkStartResponseSchema, ProvisionWorkspaceResponseSchema, FirstAutoInstalledAgentResponseSchema, SkillCatalogEntrySchema, SkillCatalogResponseSchema, SkillSubscriptionsResponseSchema;
|
|
13668
14073
|
var init_api = __esm({
|
|
13669
14074
|
"src/lib/api.ts"() {
|
|
13670
14075
|
"use strict";
|
|
@@ -13690,7 +14095,9 @@ var init_api = __esm({
|
|
|
13690
14095
|
init_usage();
|
|
13691
14096
|
init_episodes2();
|
|
13692
14097
|
init_eval_ab();
|
|
14098
|
+
init_eval_baseline_promote();
|
|
13693
14099
|
init_eval_test_sets();
|
|
14100
|
+
init_eval_management();
|
|
13694
14101
|
init_version();
|
|
13695
14102
|
PROD_DEFAULT_BASE = "https://withbamboo.com";
|
|
13696
14103
|
DEV_DEFAULT_BASE = "https://dev.withbamboo.com";
|
|
@@ -13716,7 +14123,7 @@ var init_api = __esm({
|
|
|
13716
14123
|
name: external_exports.string(),
|
|
13717
14124
|
display_name: external_exports.string(),
|
|
13718
14125
|
description: external_exports.string(),
|
|
13719
|
-
tier: external_exports.enum(["system", "agent"]),
|
|
14126
|
+
tier: external_exports.enum(["system", "agent", "shared"]),
|
|
13720
14127
|
source: external_exports.enum(["bamboo", "org", "community"]),
|
|
13721
14128
|
category: external_exports.string().nullable(),
|
|
13722
14129
|
recommended_for: external_exports.array(external_exports.string()),
|
|
@@ -13735,9 +14142,6 @@ var init_api = __esm({
|
|
|
13735
14142
|
SkillSubscriptionsResponseSchema = external_exports.object({
|
|
13736
14143
|
subscriptions: external_exports.array(external_exports.string())
|
|
13737
14144
|
});
|
|
13738
|
-
ScaffoldSkillResponseSchema = external_exports.object({
|
|
13739
|
-
skill_name: external_exports.string()
|
|
13740
|
-
});
|
|
13741
14145
|
}
|
|
13742
14146
|
});
|
|
13743
14147
|
|
|
@@ -17519,7 +17923,6 @@ var init_skills_local = __esm({
|
|
|
17519
17923
|
var skills_exports = {};
|
|
17520
17924
|
__export(skills_exports, {
|
|
17521
17925
|
runSkillsApply: () => runSkillsApply,
|
|
17522
|
-
runSkillsCreate: () => runSkillsCreate,
|
|
17523
17926
|
runSkillsGet: () => runSkillsGet,
|
|
17524
17927
|
runSkillsList: () => runSkillsList,
|
|
17525
17928
|
runSkillsSubscribe: () => runSkillsSubscribe,
|
|
@@ -17783,45 +18186,6 @@ async function runSubscribeOp(action, agentIdentifier, skillName) {
|
|
|
17783
18186
|
console.log(" git -C $(bamboo-agents status) pull");
|
|
17784
18187
|
console.log("");
|
|
17785
18188
|
}
|
|
17786
|
-
async function runSkillsCreate(skillName, options = {}) {
|
|
17787
|
-
return withFreshWorkspace(async () => runSkillsCreateImpl(skillName, options));
|
|
17788
|
-
}
|
|
17789
|
-
async function runSkillsCreateImpl(skillName, options = {}) {
|
|
17790
|
-
if (!skillName || !skillName.trim()) {
|
|
17791
|
-
console.error("bamboo-agents: skills create requires a skill name.");
|
|
17792
|
-
console.error("Example: bamboo-agents skills create analyze-creative");
|
|
17793
|
-
process.exit(2);
|
|
17794
|
-
}
|
|
17795
|
-
if (!/^[a-z0-9][a-z0-9-]*$/.test(skillName.trim())) {
|
|
17796
|
-
console.error(
|
|
17797
|
-
"bamboo-agents: skill name must be kebab-case (lowercase + dashes)."
|
|
17798
|
-
);
|
|
17799
|
-
process.exit(2);
|
|
17800
|
-
}
|
|
17801
|
-
const bambooSession = await requireSession2();
|
|
17802
|
-
let result;
|
|
17803
|
-
try {
|
|
17804
|
-
result = await scaffoldSkill(bambooSession, {
|
|
17805
|
-
skill_name: skillName.trim(),
|
|
17806
|
-
category: options.category,
|
|
17807
|
-
description: options.description
|
|
17808
|
-
});
|
|
17809
|
-
} catch (err) {
|
|
17810
|
-
console.error(
|
|
17811
|
-
`bamboo-agents: could not scaffold ${skillName}: ${err.message}`
|
|
17812
|
-
);
|
|
17813
|
-
process.exit(1);
|
|
17814
|
-
}
|
|
17815
|
-
console.log("");
|
|
17816
|
-
console.log(` Scaffolded ${result.skill_name}.`);
|
|
17817
|
-
console.log("");
|
|
17818
|
-
console.log(
|
|
17819
|
-
` Edit skills/${result.skill_name}/SKILL.md in your workspace clone:`
|
|
17820
|
-
);
|
|
17821
|
-
console.log(" git -C $(bamboo-agents status) pull");
|
|
17822
|
-
console.log(` $EDITOR skills/${result.skill_name}/SKILL.md`);
|
|
17823
|
-
console.log("");
|
|
17824
|
-
}
|
|
17825
18189
|
async function runSkillsUpdate(agentIdentifier, skillName) {
|
|
17826
18190
|
return withFreshWorkspace(
|
|
17827
18191
|
async () => runSkillsUpdateImpl(agentIdentifier, skillName)
|
|
@@ -19377,14 +19741,116 @@ var init_episode = __esm({
|
|
|
19377
19741
|
}
|
|
19378
19742
|
});
|
|
19379
19743
|
|
|
19744
|
+
// src/commands/eval-management.ts
|
|
19745
|
+
var eval_management_exports = {};
|
|
19746
|
+
__export(eval_management_exports, {
|
|
19747
|
+
runEvalManage: () => runEvalManage,
|
|
19748
|
+
runEvalOperations: () => runEvalOperations
|
|
19749
|
+
});
|
|
19750
|
+
import { readFile as readFile9 } from "node:fs/promises";
|
|
19751
|
+
async function requireSession12(opts) {
|
|
19752
|
+
const creds = await readCredentials();
|
|
19753
|
+
if (!creds) {
|
|
19754
|
+
emitError(
|
|
19755
|
+
{
|
|
19756
|
+
code: "not_signed_in",
|
|
19757
|
+
message: "not signed in.",
|
|
19758
|
+
hint: "Run `bamboo-agents login` first."
|
|
19759
|
+
},
|
|
19760
|
+
opts,
|
|
19761
|
+
1
|
|
19762
|
+
);
|
|
19763
|
+
throw new Error("unreachable");
|
|
19764
|
+
}
|
|
19765
|
+
return creds.bamboo_session;
|
|
19766
|
+
}
|
|
19767
|
+
async function parseInput(value) {
|
|
19768
|
+
const raw = value.startsWith("@") ? await readFile9(value.slice(1), "utf8") : value;
|
|
19769
|
+
const parsed = JSON.parse(raw);
|
|
19770
|
+
if (!parsed || Array.isArray(parsed) || typeof parsed !== "object") {
|
|
19771
|
+
throw new Error("--input must be a JSON object");
|
|
19772
|
+
}
|
|
19773
|
+
return parsed;
|
|
19774
|
+
}
|
|
19775
|
+
async function runEvalOperations(options) {
|
|
19776
|
+
const session = await requireSession12(options);
|
|
19777
|
+
try {
|
|
19778
|
+
const catalog = await getEvalManagementOperations(session);
|
|
19779
|
+
human("", options);
|
|
19780
|
+
for (const binding of catalog.bindings) {
|
|
19781
|
+
human(
|
|
19782
|
+
` ${binding.operation.padEnd(31)} ${binding.description}`,
|
|
19783
|
+
options
|
|
19784
|
+
);
|
|
19785
|
+
const input = catalog.inputs[binding.operation] ?? {
|
|
19786
|
+
required: [],
|
|
19787
|
+
optional: []
|
|
19788
|
+
};
|
|
19789
|
+
human(` required: ${input.required.join(", ") || "none"}`, options);
|
|
19790
|
+
if (input.optional.length)
|
|
19791
|
+
human(` optional: ${input.optional.join(", ")}`, options);
|
|
19792
|
+
}
|
|
19793
|
+
human("", options);
|
|
19794
|
+
human(" Existing specialized commands:", options);
|
|
19795
|
+
for (const command of catalog.related_cli_commands)
|
|
19796
|
+
human(` - ${command}`, options);
|
|
19797
|
+
human("", options);
|
|
19798
|
+
emitSuccess(catalog, options);
|
|
19799
|
+
} catch (error) {
|
|
19800
|
+
const value = error;
|
|
19801
|
+
emitError(
|
|
19802
|
+
{
|
|
19803
|
+
code: value.code ?? "operations_failed",
|
|
19804
|
+
message: value.message ?? "Could not load operations."
|
|
19805
|
+
},
|
|
19806
|
+
options,
|
|
19807
|
+
1
|
|
19808
|
+
);
|
|
19809
|
+
}
|
|
19810
|
+
}
|
|
19811
|
+
async function runEvalManage(options) {
|
|
19812
|
+
const output = options;
|
|
19813
|
+
const session = await requireSession12(output);
|
|
19814
|
+
try {
|
|
19815
|
+
const input = await parseInput(options.input);
|
|
19816
|
+
const call = EvalManagementCallSchema.parse({
|
|
19817
|
+
operation: options.operation,
|
|
19818
|
+
input
|
|
19819
|
+
});
|
|
19820
|
+
const result = await callEvalManagement(session, call);
|
|
19821
|
+
human(`Applied ${result.operation}.`, output);
|
|
19822
|
+
emitSuccess(result, output);
|
|
19823
|
+
} catch (error) {
|
|
19824
|
+
const value = error;
|
|
19825
|
+
emitError(
|
|
19826
|
+
{
|
|
19827
|
+
code: value.code ?? "management_failed",
|
|
19828
|
+
message: value.message ?? "Management operation failed.",
|
|
19829
|
+
hint: "Run `bamboo-agents eval operations --json` for supported names and bindings."
|
|
19830
|
+
},
|
|
19831
|
+
output,
|
|
19832
|
+
1
|
|
19833
|
+
);
|
|
19834
|
+
}
|
|
19835
|
+
}
|
|
19836
|
+
var init_eval_management2 = __esm({
|
|
19837
|
+
"src/commands/eval-management.ts"() {
|
|
19838
|
+
"use strict";
|
|
19839
|
+
init_eval_management();
|
|
19840
|
+
init_credentials();
|
|
19841
|
+
init_api();
|
|
19842
|
+
init_output();
|
|
19843
|
+
}
|
|
19844
|
+
});
|
|
19845
|
+
|
|
19380
19846
|
// src/commands/eval-ab.ts
|
|
19381
19847
|
var eval_ab_exports2 = {};
|
|
19382
19848
|
__export(eval_ab_exports2, {
|
|
19383
19849
|
runEvalAblate: () => runEvalAblate
|
|
19384
19850
|
});
|
|
19385
|
-
import { readFile as
|
|
19851
|
+
import { readFile as readFile10 } from "node:fs/promises";
|
|
19386
19852
|
import { resolve as resolve2 } from "node:path";
|
|
19387
|
-
async function
|
|
19853
|
+
async function requireSession13(opts) {
|
|
19388
19854
|
const creds = await readCredentials();
|
|
19389
19855
|
if (!creds) {
|
|
19390
19856
|
emitError(
|
|
@@ -19435,11 +19901,11 @@ function printStaged(data, opts) {
|
|
|
19435
19901
|
}
|
|
19436
19902
|
async function runEvalAblate(options) {
|
|
19437
19903
|
const opts = options;
|
|
19438
|
-
const session = await
|
|
19904
|
+
const session = await requireSession13(opts);
|
|
19439
19905
|
let candidateBody;
|
|
19440
19906
|
const bodyPath = resolve2(process.cwd(), options.body);
|
|
19441
19907
|
try {
|
|
19442
|
-
candidateBody = await
|
|
19908
|
+
candidateBody = await readFile10(bodyPath, "utf-8");
|
|
19443
19909
|
} catch (err) {
|
|
19444
19910
|
emitError(
|
|
19445
19911
|
{
|
|
@@ -19493,13 +19959,117 @@ var init_eval_ab2 = __esm({
|
|
|
19493
19959
|
}
|
|
19494
19960
|
});
|
|
19495
19961
|
|
|
19962
|
+
// src/commands/eval-promote.ts
|
|
19963
|
+
var eval_promote_exports = {};
|
|
19964
|
+
__export(eval_promote_exports, {
|
|
19965
|
+
runEvalPromote: () => runEvalPromote
|
|
19966
|
+
});
|
|
19967
|
+
async function requireSession14(opts) {
|
|
19968
|
+
const creds = await readCredentials();
|
|
19969
|
+
if (!creds) {
|
|
19970
|
+
emitError(
|
|
19971
|
+
{
|
|
19972
|
+
code: "not_signed_in",
|
|
19973
|
+
message: "not signed in.",
|
|
19974
|
+
hint: "Run `bamboo-agents login` first."
|
|
19975
|
+
},
|
|
19976
|
+
opts,
|
|
19977
|
+
1
|
|
19978
|
+
);
|
|
19979
|
+
throw new Error("unreachable");
|
|
19980
|
+
}
|
|
19981
|
+
return creds.bamboo_session;
|
|
19982
|
+
}
|
|
19983
|
+
function short(version) {
|
|
19984
|
+
return version.slice(0, 12);
|
|
19985
|
+
}
|
|
19986
|
+
function render(data, opts) {
|
|
19987
|
+
human("", opts);
|
|
19988
|
+
if (data.kind === "demote") {
|
|
19989
|
+
human(
|
|
19990
|
+
` Baseline demoted for qrels_version ${short(data.qrels_version)} (was run ${data.previous_run_id}).`,
|
|
19991
|
+
opts
|
|
19992
|
+
);
|
|
19993
|
+
human(
|
|
19994
|
+
" The gate reports baseline_demoted for that version until a human promotes a run.",
|
|
19995
|
+
opts
|
|
19996
|
+
);
|
|
19997
|
+
} else {
|
|
19998
|
+
human(
|
|
19999
|
+
` Baseline promoted: run ${data.run_id} is the reference for qrels_version ${short(data.qrels_version)}` + (data.previous_run_id ? ` (was ${data.previous_run_id}).` : " (the version had none)."),
|
|
20000
|
+
opts
|
|
20001
|
+
);
|
|
20002
|
+
human(" Re-run the PR's retrieval-evals check: it compares flat against its own run.", opts);
|
|
20003
|
+
}
|
|
20004
|
+
human("", opts);
|
|
20005
|
+
}
|
|
20006
|
+
async function runEvalPromote(options) {
|
|
20007
|
+
const opts = { json: options.json, fields: options.fields, quiet: options.quiet };
|
|
20008
|
+
let body;
|
|
20009
|
+
if (options.demote) {
|
|
20010
|
+
if (!options.brand || !options.qrelsVersion) {
|
|
20011
|
+
emitError(
|
|
20012
|
+
{
|
|
20013
|
+
code: "missing_key",
|
|
20014
|
+
message: "--demote needs --brand <id> and --qrels-version <hash>.",
|
|
20015
|
+
hint: "A tombstone names the key it empties; both are on the gate report."
|
|
20016
|
+
},
|
|
20017
|
+
opts,
|
|
20018
|
+
1
|
|
20019
|
+
);
|
|
20020
|
+
return;
|
|
20021
|
+
}
|
|
20022
|
+
body = {
|
|
20023
|
+
demote: true,
|
|
20024
|
+
brand_id: options.brand,
|
|
20025
|
+
qrels_version: options.qrelsVersion,
|
|
20026
|
+
reason: options.reason,
|
|
20027
|
+
...options.pr ? { pr_url: options.pr } : {}
|
|
20028
|
+
};
|
|
20029
|
+
} else {
|
|
20030
|
+
if (!options.runId) {
|
|
20031
|
+
emitError(
|
|
20032
|
+
{
|
|
20033
|
+
code: "missing_run_id",
|
|
20034
|
+
message: "Pass the run id the correctness gate printed.",
|
|
20035
|
+
hint: 'bamboo-agents eval promote <run-id> --reason "why this drop is intentional"'
|
|
20036
|
+
},
|
|
20037
|
+
opts,
|
|
20038
|
+
1
|
|
20039
|
+
);
|
|
20040
|
+
return;
|
|
20041
|
+
}
|
|
20042
|
+
body = {
|
|
20043
|
+
run_id: options.runId,
|
|
20044
|
+
reason: options.reason,
|
|
20045
|
+
...options.pr ? { pr_url: options.pr } : {}
|
|
20046
|
+
};
|
|
20047
|
+
}
|
|
20048
|
+
const session = await requireSession14(opts);
|
|
20049
|
+
try {
|
|
20050
|
+
const data = await promoteBaseline(session, body);
|
|
20051
|
+
emitSuccess(data, opts, (d) => render(d, opts));
|
|
20052
|
+
} catch (err) {
|
|
20053
|
+
const e = err;
|
|
20054
|
+
emitError({ code: e.errorCode ?? "promote_failed", message: e.message }, opts, 1);
|
|
20055
|
+
}
|
|
20056
|
+
}
|
|
20057
|
+
var init_eval_promote = __esm({
|
|
20058
|
+
"src/commands/eval-promote.ts"() {
|
|
20059
|
+
"use strict";
|
|
20060
|
+
init_credentials();
|
|
20061
|
+
init_api();
|
|
20062
|
+
init_output();
|
|
20063
|
+
}
|
|
20064
|
+
});
|
|
20065
|
+
|
|
19496
20066
|
// src/commands/eval-test-sets.ts
|
|
19497
20067
|
var eval_test_sets_exports = {};
|
|
19498
20068
|
__export(eval_test_sets_exports, {
|
|
19499
20069
|
runEvalSetsCreate: () => runEvalSetsCreate,
|
|
19500
20070
|
slugifyTestSetName: () => slugifyTestSetName
|
|
19501
20071
|
});
|
|
19502
|
-
async function
|
|
20072
|
+
async function requireSession15(opts) {
|
|
19503
20073
|
const creds = await readCredentials();
|
|
19504
20074
|
if (!creds) {
|
|
19505
20075
|
emitError(
|
|
@@ -19539,7 +20109,7 @@ function printCreated(data, opts) {
|
|
|
19539
20109
|
}
|
|
19540
20110
|
async function runEvalSetsCreate(options) {
|
|
19541
20111
|
const opts = options;
|
|
19542
|
-
const session = await
|
|
20112
|
+
const session = await requireSession15(opts);
|
|
19543
20113
|
const brandId = await resolveBrandId(
|
|
19544
20114
|
session,
|
|
19545
20115
|
options.brand,
|
|
@@ -19608,7 +20178,7 @@ __export(eval_set_cases_exports, {
|
|
|
19608
20178
|
runEvalSetsAddCases: () => runEvalSetsAddCases,
|
|
19609
20179
|
unionCaseIds: () => unionCaseIds
|
|
19610
20180
|
});
|
|
19611
|
-
async function
|
|
20181
|
+
async function requireSession16(opts) {
|
|
19612
20182
|
const creds = await readCredentials();
|
|
19613
20183
|
if (!creds) {
|
|
19614
20184
|
emitError(
|
|
@@ -19639,7 +20209,7 @@ function unionCaseIds(existing, incoming) {
|
|
|
19639
20209
|
}
|
|
19640
20210
|
async function runEvalSetsAddCases(options) {
|
|
19641
20211
|
const opts = options;
|
|
19642
|
-
const session = await
|
|
20212
|
+
const session = await requireSession16(opts);
|
|
19643
20213
|
const incoming = parseCaseIds(options.cases);
|
|
19644
20214
|
if (incoming.length === 0) {
|
|
19645
20215
|
emitError(
|
|
@@ -20057,12 +20627,6 @@ skills.command("unsubscribe <agent> <skill-name>").description("Unsubscribe an a
|
|
|
20057
20627
|
const { runSkillsUnsubscribe: runSkillsUnsubscribe2 } = await Promise.resolve().then(() => (init_skills(), skills_exports));
|
|
20058
20628
|
await runSkillsUnsubscribe2(agent, skillName);
|
|
20059
20629
|
});
|
|
20060
|
-
skills.command("create <skill-name>").description("Scaffold a new org-custom skill in your workspace").option("--category <category>", "Optional category label (e.g. analysis)").option("--description <description>", "Optional one-line description").option("--no-sync", "Skip the auto-sync git pull on this command").action(
|
|
20061
|
-
async (skillName, options) => {
|
|
20062
|
-
const { runSkillsCreate: runSkillsCreate2 } = await Promise.resolve().then(() => (init_skills(), skills_exports));
|
|
20063
|
-
await runSkillsCreate2(skillName, options);
|
|
20064
|
-
}
|
|
20065
|
-
);
|
|
20066
20630
|
skills.command("update <agent> <skill-name>").description("Update an installed skill from the bamboo registry").option("--no-sync", "Skip the auto-sync git pull on this command").action(async (agent, skillName) => {
|
|
20067
20631
|
const { runSkillsUpdate: runSkillsUpdate2 } = await Promise.resolve().then(() => (init_skills(), skills_exports));
|
|
20068
20632
|
await runSkillsUpdate2(agent, skillName);
|
|
@@ -20202,6 +20766,14 @@ episode.command("log-rec").description(
|
|
|
20202
20766
|
await runEpisodeLogRec2(options);
|
|
20203
20767
|
});
|
|
20204
20768
|
var evals = program2.command("eval").description("Evaluation surface \u2014 A/B a candidate skill against its baseline");
|
|
20769
|
+
evals.command("operations").description("List the exact supported investigation and wave management operations").option("--json", "Emit a JSON envelope for agent consumption").option("--fields <list>", "Project only these dotted field paths from data (comma-separated)").option("--quiet", "Suppress human chatter (data/errors only)").action(async (options) => {
|
|
20770
|
+
const { runEvalOperations: runEvalOperations2 } = await Promise.resolve().then(() => (init_eval_management2(), eval_management_exports));
|
|
20771
|
+
await runEvalOperations2(options);
|
|
20772
|
+
});
|
|
20773
|
+
evals.command("manage").description("Call one canonical investigation or wave operation").argument("<operation>", "Exact name from `eval operations`").requiredOption("--input <json-or-@file>", "JSON object, inline or loaded from @path").option("--json", "Emit a JSON envelope for agent consumption").option("--fields <list>", "Project only these dotted field paths from data (comma-separated)").option("--quiet", "Suppress human chatter (data/errors only)").option("--no-sync", "Skip the auto-sync git pull on this command").action(async (operation, options) => {
|
|
20774
|
+
const { runEvalManage: runEvalManage2 } = await Promise.resolve().then(() => (init_eval_management2(), eval_management_exports));
|
|
20775
|
+
await runEvalManage2({ operation, ...options });
|
|
20776
|
+
});
|
|
20205
20777
|
evals.command("ablate").description(
|
|
20206
20778
|
"Start an A/B test: stage a candidate skill body against the published baseline"
|
|
20207
20779
|
).requiredOption("--agent <id>", "Agent id whose skill is under test").requiredOption("--skill <name>", "kebab-case skill name (must be one of this agent's own skills)").requiredOption(
|
|
@@ -20216,6 +20788,14 @@ evals.command("ablate").description(
|
|
|
20216
20788
|
await runEvalAblate2(options);
|
|
20217
20789
|
}
|
|
20218
20790
|
);
|
|
20791
|
+
evals.command("promote").description(
|
|
20792
|
+
"Promote a correctness run as the baseline for its qrels version, or --demote the current one"
|
|
20793
|
+
).argument("[run-id]", "Run id printed by the correctness gate (omit with --demote)").requiredOption("--reason <text>", "Why this run is the reference now \u2014 recorded on the row").option("--pr <url>", "PR URL to record as the promotion's source").option("--demote", "Write a tombstone: the version has no baseline until a human promotes one").option("--brand <id>", "Brand id (with --demote)").option("--qrels-version <hash>", "qrels version to demote (with --demote)").option("--json", "Emit a JSON envelope for agent consumption").option("--fields <list>", "Project only these dotted field paths from data (comma-separated)").option("--quiet", "Suppress human chatter (data/errors only)").option("--no-sync", "Skip the auto-sync git pull on this command").action(
|
|
20794
|
+
async (runId, options) => {
|
|
20795
|
+
const { runEvalPromote: runEvalPromote2 } = await Promise.resolve().then(() => (init_eval_promote(), eval_promote_exports));
|
|
20796
|
+
await runEvalPromote2({ runId, ...options });
|
|
20797
|
+
}
|
|
20798
|
+
);
|
|
20219
20799
|
var evalSets = evals.command("sets").description("Test sets \u2014 the definition a run executes (cases attach separately)");
|
|
20220
20800
|
evalSets.command("create").description("Create an empty test set of a given evidence kind").requiredOption("--name <text>", 'Human name, e.g. "Connector parity"').requiredOption(
|
|
20221
20801
|
"--kind <kind>",
|