@brainbase-labs/cli 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +62 -0
- package/dist/index.js +2067 -108
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -35141,9 +35141,9 @@ var require_dist2 = __commonJS((exports, module) => {
|
|
|
35141
35141
|
});
|
|
35142
35142
|
|
|
35143
35143
|
// src/index.ts
|
|
35144
|
-
var
|
|
35144
|
+
var import_picocolors49 = __toESM(require_picocolors(), 1);
|
|
35145
35145
|
import process14 from "node:process";
|
|
35146
|
-
import
|
|
35146
|
+
import fs81 from "node:fs";
|
|
35147
35147
|
|
|
35148
35148
|
// src/cli/template.ts
|
|
35149
35149
|
var import_picocolors12 = __toESM(require_picocolors(), 1);
|
|
@@ -36008,7 +36008,7 @@ function padStart(s, n) {
|
|
|
36008
36008
|
// package.json
|
|
36009
36009
|
var package_default = {
|
|
36010
36010
|
name: "@brainbase-labs/cli",
|
|
36011
|
-
version: "0.
|
|
36011
|
+
version: "0.20.0",
|
|
36012
36012
|
description: "Pack, share, and install agent templates across harnesses (Claude Code, Codex, ...).",
|
|
36013
36013
|
type: "module",
|
|
36014
36014
|
bin: {
|
|
@@ -40969,6 +40969,9 @@ function writeAuthUnlocked(s) {
|
|
|
40969
40969
|
fs6.chmodSync(AUTH_FILE, 384);
|
|
40970
40970
|
} catch {}
|
|
40971
40971
|
}
|
|
40972
|
+
function writeAuth(s) {
|
|
40973
|
+
withAuthLock(() => writeAuthUnlocked(s));
|
|
40974
|
+
}
|
|
40972
40975
|
function clearAuth() {
|
|
40973
40976
|
withAuthLock(() => {
|
|
40974
40977
|
if (exists(AUTH_FILE))
|
|
@@ -58078,45 +58081,59 @@ async function extract(opts) {
|
|
|
58078
58081
|
const maxCount = opts.maxFileCount ?? MAX_FILE_COUNT;
|
|
58079
58082
|
const maxRatio = opts.maxRatio ?? 100;
|
|
58080
58083
|
let totalBytes = 0;
|
|
58081
|
-
let
|
|
58084
|
+
let entryCount = 0;
|
|
58082
58085
|
const written = [];
|
|
58086
|
+
let filterError;
|
|
58083
58087
|
await co({
|
|
58084
58088
|
file: opts.tarFile,
|
|
58085
58089
|
cwd: out,
|
|
58086
58090
|
strict: true,
|
|
58087
58091
|
filter: (entryPath, entryAny) => {
|
|
58088
|
-
const entry = entryAny;
|
|
58089
|
-
const t = entry.type;
|
|
58090
|
-
if (t !== "File" && t !== "OldFile" && t !== "ContiguousFile" && t !== "Directory" && t !== "GNUDumpDir") {
|
|
58091
|
-
throw new TarballError(`refusing entry type "${t}" for ${entryPath}`);
|
|
58092
|
-
}
|
|
58093
58092
|
try {
|
|
58094
|
-
|
|
58095
|
-
|
|
58096
|
-
|
|
58097
|
-
|
|
58098
|
-
|
|
58099
|
-
|
|
58100
|
-
|
|
58101
|
-
|
|
58102
|
-
|
|
58093
|
+
if (filterError)
|
|
58094
|
+
return false;
|
|
58095
|
+
const entry = entryAny;
|
|
58096
|
+
const t = entry.type;
|
|
58097
|
+
if (t !== "File" && t !== "OldFile" && t !== "ContiguousFile" && t !== "Directory" && t !== "GNUDumpDir") {
|
|
58098
|
+
throw new TarballError(`refusing entry type "${t}" for ${entryPath}`);
|
|
58099
|
+
}
|
|
58100
|
+
let normalizedPath;
|
|
58101
|
+
try {
|
|
58102
|
+
normalizedPath = safeRelPath(entryPath);
|
|
58103
|
+
} catch (err) {
|
|
58104
|
+
throw new TarballError(err.message);
|
|
58105
|
+
}
|
|
58106
|
+
if (normalizedPath.split("/").length > 64) {
|
|
58107
|
+
throw new TarballError(`entry path exceeds 64 segments: ${entryPath}`);
|
|
58108
|
+
}
|
|
58109
|
+
const size2 = entry.size ?? 0;
|
|
58110
|
+
if (size2 > maxBytes) {
|
|
58111
|
+
throw new TarballError(`entry ${entryPath} size ${size2} exceeds cap ${maxBytes}`);
|
|
58112
|
+
}
|
|
58113
|
+
entryCount++;
|
|
58114
|
+
if (entryCount > maxCount) {
|
|
58115
|
+
throw new TarballError(`too many entries (>${maxCount})`);
|
|
58116
|
+
}
|
|
58117
|
+
if (t === "Directory" || t === "GNUDumpDir")
|
|
58118
|
+
return true;
|
|
58119
|
+
totalBytes += size2;
|
|
58120
|
+
if (totalBytes > maxBytes) {
|
|
58121
|
+
throw new TarballError(`total uncompressed bytes ${totalBytes} exceeds cap ${maxBytes}`);
|
|
58122
|
+
}
|
|
58123
|
+
if (totalBytes > compressed * maxRatio) {
|
|
58124
|
+
throw new TarballError(`decompression ratio exceeds ${maxRatio}× (${totalBytes}B from ${compressed}B)`);
|
|
58125
|
+
}
|
|
58126
|
+
written.push(entryPath);
|
|
58103
58127
|
return true;
|
|
58104
|
-
|
|
58105
|
-
|
|
58106
|
-
|
|
58107
|
-
}
|
|
58108
|
-
totalBytes += size2;
|
|
58109
|
-
if (totalBytes > maxBytes) {
|
|
58110
|
-
throw new TarballError(`total uncompressed bytes ${totalBytes} exceeds cap ${maxBytes}`);
|
|
58111
|
-
}
|
|
58112
|
-
if (totalBytes > compressed * maxRatio) {
|
|
58113
|
-
throw new TarballError(`decompression ratio exceeds ${maxRatio}× (${totalBytes}B from ${compressed}B)`);
|
|
58128
|
+
} catch (error) {
|
|
58129
|
+
filterError = error instanceof TarballError ? error : new TarballError(error instanceof Error ? error.message : "invalid archive entry");
|
|
58130
|
+
return false;
|
|
58114
58131
|
}
|
|
58115
|
-
written.push(entryPath);
|
|
58116
|
-
return true;
|
|
58117
58132
|
},
|
|
58118
58133
|
preserveOwner: false
|
|
58119
58134
|
});
|
|
58135
|
+
if (filterError)
|
|
58136
|
+
throw filterError;
|
|
58120
58137
|
return written;
|
|
58121
58138
|
}
|
|
58122
58139
|
async function sha256OfFile(filePath) {
|
|
@@ -61424,6 +61441,8 @@ function isInsecureWebBase(webBase) {
|
|
|
61424
61441
|
return !["127.0.0.1", "localhost", "[::1]", "::1"].includes(url.hostname);
|
|
61425
61442
|
}
|
|
61426
61443
|
function openInBrowser(url) {
|
|
61444
|
+
if (process.env.BRAINBASE_NON_INTERACTIVE)
|
|
61445
|
+
return;
|
|
61427
61446
|
const [cmd, args] = process.platform === "darwin" ? ["open", [url]] : process.platform === "win32" ? ["rundll32.exe", ["url.dll,FileProtocolHandler", url]] : ["xdg-open", [url]];
|
|
61428
61447
|
try {
|
|
61429
61448
|
const child = spawn(cmd, args, {
|
|
@@ -61522,6 +61541,7 @@ async function runLogin(_cwd, args) {
|
|
|
61522
61541
|
spinning = true;
|
|
61523
61542
|
const session = await withCancelOnInterrupt(() => awaitDeviceSession({ webBase, authorization }));
|
|
61524
61543
|
const stored = toAuthSession(session, webBase);
|
|
61544
|
+
writeAuth(stored);
|
|
61525
61545
|
spinner.stop("Authorized.");
|
|
61526
61546
|
spinning = false;
|
|
61527
61547
|
$e("Welcome to brainbase.");
|
|
@@ -77024,6 +77044,1934 @@ function printHelp4() {
|
|
|
77024
77044
|
`));
|
|
77025
77045
|
}
|
|
77026
77046
|
|
|
77047
|
+
// src/cli/benchmark.ts
|
|
77048
|
+
var import_picocolors48 = __toESM(require_picocolors(), 1);
|
|
77049
|
+
import {
|
|
77050
|
+
execFileSync as execFileSync3,
|
|
77051
|
+
spawn as spawn5
|
|
77052
|
+
} from "node:child_process";
|
|
77053
|
+
import crypto7 from "node:crypto";
|
|
77054
|
+
import fs80 from "node:fs";
|
|
77055
|
+
import os17 from "node:os";
|
|
77056
|
+
import path88 from "node:path";
|
|
77057
|
+
|
|
77058
|
+
// src/core/benchmark-phase.ts
|
|
77059
|
+
import {
|
|
77060
|
+
execFileSync as execFileSync2,
|
|
77061
|
+
spawn as spawn4
|
|
77062
|
+
} from "node:child_process";
|
|
77063
|
+
import crypto6 from "node:crypto";
|
|
77064
|
+
import fs79 from "node:fs";
|
|
77065
|
+
import os16 from "node:os";
|
|
77066
|
+
import path87 from "node:path";
|
|
77067
|
+
import { pipeline as pipeline2 } from "node:stream/promises";
|
|
77068
|
+
var SCHEMA_VERSION = "1";
|
|
77069
|
+
var SHA256_RE = /^[a-f0-9]{64}$/i;
|
|
77070
|
+
var ID_RE = /^[a-zA-Z0-9][a-zA-Z0-9._-]{0,127}$/;
|
|
77071
|
+
var ENV_NAME_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
77072
|
+
var MAX_SPEC_BYTES = 20 * 1024 * 1024;
|
|
77073
|
+
var MAX_FINAL_OUTPUT_BYTES = 10 * 1024 * 1024;
|
|
77074
|
+
var MAX_TRAJECTORY_BYTES = 100 * 1024 * 1024;
|
|
77075
|
+
var RESERVED_WORKSPACE_PATHS = new Set([
|
|
77076
|
+
".brainbase",
|
|
77077
|
+
".git",
|
|
77078
|
+
"brainbase.agent.yaml"
|
|
77079
|
+
]);
|
|
77080
|
+
var BASE_ENV_NAMES = [
|
|
77081
|
+
"HOME",
|
|
77082
|
+
"LANG",
|
|
77083
|
+
"LC_ALL",
|
|
77084
|
+
"LOGNAME",
|
|
77085
|
+
"PATH",
|
|
77086
|
+
"SHELL",
|
|
77087
|
+
"TMPDIR",
|
|
77088
|
+
"USER"
|
|
77089
|
+
];
|
|
77090
|
+
var SENSITIVE_ENV_NAME_RE = /(?:^|_)(?:TOKEN|SECRET|PASSWORD|PASSWD|API_KEY|PRIVATE_KEY|ACCESS_KEY|PAT|CREDENTIAL)(?:$|_)|^(?:PGPASSWORD|DATABASE_URL|REDIS_URL|MONGODB_URI)$/i;
|
|
77091
|
+
var AbsolutePathSchema = exports_external.string().min(1).refine(path87.isAbsolute, {
|
|
77092
|
+
message: "must be an absolute path"
|
|
77093
|
+
});
|
|
77094
|
+
var Sha256Schema = exports_external.string().regex(SHA256_RE).transform((value) => value.toLowerCase());
|
|
77095
|
+
var IdSchema = exports_external.string().regex(ID_RE);
|
|
77096
|
+
var EnvNameSchema = exports_external.string().regex(ENV_NAME_RE);
|
|
77097
|
+
var NonSecretEnvironmentValueSchema = exports_external.object({
|
|
77098
|
+
value: exports_external.string(),
|
|
77099
|
+
sensitive: exports_external.literal(false)
|
|
77100
|
+
}).strict();
|
|
77101
|
+
var RootRelativePathSchema = exports_external.string().min(1).max(1024);
|
|
77102
|
+
var BudgetSchema = exports_external.object({
|
|
77103
|
+
timeout_ms: exports_external.number().int().min(100).max(3600000),
|
|
77104
|
+
max_output_bytes: exports_external.number().int().min(1024).max(100 * 1024 * 1024)
|
|
77105
|
+
}).strict();
|
|
77106
|
+
var MaterialSchema = exports_external.object({
|
|
77107
|
+
id: IdSchema,
|
|
77108
|
+
kind: exports_external.enum(["file", "tar_gz"]),
|
|
77109
|
+
source: RootRelativePathSchema,
|
|
77110
|
+
destination: RootRelativePathSchema,
|
|
77111
|
+
sha256: Sha256Schema,
|
|
77112
|
+
size_bytes: exports_external.number().int().min(0).max(2 * 1024 * 1024 * 1024),
|
|
77113
|
+
mode: exports_external.number().int().min(0).max(511).optional(),
|
|
77114
|
+
max_unpacked_bytes: exports_external.number().int().min(1).max(2 * 1024 * 1024 * 1024).optional(),
|
|
77115
|
+
max_file_count: exports_external.number().int().min(1).max(1e5).optional()
|
|
77116
|
+
}).strict();
|
|
77117
|
+
var CommandSchema = exports_external.object({
|
|
77118
|
+
id: IdSchema,
|
|
77119
|
+
argv: exports_external.array(exports_external.string().min(1)).min(1).max(128),
|
|
77120
|
+
cwd: RootRelativePathSchema.default("."),
|
|
77121
|
+
timeout_ms: exports_external.number().int().min(100).max(3600000).optional(),
|
|
77122
|
+
secret_env: exports_external.array(EnvNameSchema).max(100).default([])
|
|
77123
|
+
}).strict();
|
|
77124
|
+
var BaseSpecSchema = exports_external.object({
|
|
77125
|
+
schema_version: exports_external.literal(SCHEMA_VERSION),
|
|
77126
|
+
attempt_id: exports_external.string().uuid(),
|
|
77127
|
+
phase_id: IdSchema,
|
|
77128
|
+
workspace_root: AbsolutePathSchema,
|
|
77129
|
+
staging_root: AbsolutePathSchema,
|
|
77130
|
+
logs_root: AbsolutePathSchema,
|
|
77131
|
+
budget: BudgetSchema,
|
|
77132
|
+
environment: exports_external.record(EnvNameSchema, NonSecretEnvironmentValueSchema).default({}),
|
|
77133
|
+
secret_env: exports_external.array(EnvNameSchema).max(100).default([])
|
|
77134
|
+
}).strict();
|
|
77135
|
+
var OutputAssertionSchema = exports_external.discriminatedUnion("operator", [
|
|
77136
|
+
exports_external.object({ operator: exports_external.literal("exact"), expected: exports_external.string() }).strict(),
|
|
77137
|
+
exports_external.object({ operator: exports_external.literal("contains"), expected: exports_external.string() }).strict(),
|
|
77138
|
+
exports_external.object({
|
|
77139
|
+
operator: exports_external.literal("regex"),
|
|
77140
|
+
pattern: exports_external.string().max(4096),
|
|
77141
|
+
flags: exports_external.string().regex(/^[imsu]*$/).default("")
|
|
77142
|
+
}).strict()
|
|
77143
|
+
]);
|
|
77144
|
+
var WorkspaceAssertionSchema = exports_external.discriminatedUnion("operator", [
|
|
77145
|
+
exports_external.object({ operator: exports_external.literal("exists") }).strict(),
|
|
77146
|
+
exports_external.object({ operator: exports_external.literal("not_exists") }).strict(),
|
|
77147
|
+
exports_external.object({ operator: exports_external.literal("sha256"), expected: Sha256Schema }).strict(),
|
|
77148
|
+
exports_external.object({ operator: exports_external.literal("contains"), expected: exports_external.string() }).strict()
|
|
77149
|
+
]);
|
|
77150
|
+
var EvaluatorBase = {
|
|
77151
|
+
id: IdSchema,
|
|
77152
|
+
required: exports_external.boolean().default(true),
|
|
77153
|
+
primary: exports_external.boolean().default(false)
|
|
77154
|
+
};
|
|
77155
|
+
var EvaluatorSchema = exports_external.discriminatedUnion("type", [
|
|
77156
|
+
exports_external.object({
|
|
77157
|
+
...EvaluatorBase,
|
|
77158
|
+
type: exports_external.literal("output_assertion"),
|
|
77159
|
+
assertion: OutputAssertionSchema
|
|
77160
|
+
}).strict(),
|
|
77161
|
+
exports_external.object({
|
|
77162
|
+
...EvaluatorBase,
|
|
77163
|
+
type: exports_external.literal("trajectory_assertion"),
|
|
77164
|
+
event_type: exports_external.string().min(1).max(256),
|
|
77165
|
+
min_count: exports_external.number().int().min(0).default(1),
|
|
77166
|
+
max_count: exports_external.number().int().min(0).optional()
|
|
77167
|
+
}).strict(),
|
|
77168
|
+
exports_external.object({
|
|
77169
|
+
...EvaluatorBase,
|
|
77170
|
+
type: exports_external.literal("workspace_assertion"),
|
|
77171
|
+
path: RootRelativePathSchema,
|
|
77172
|
+
assertion: WorkspaceAssertionSchema
|
|
77173
|
+
}).strict(),
|
|
77174
|
+
exports_external.object({
|
|
77175
|
+
...EvaluatorBase,
|
|
77176
|
+
type: exports_external.literal("sandbox_command"),
|
|
77177
|
+
command: CommandSchema.omit({ id: true }),
|
|
77178
|
+
root: exports_external.enum(["workspace", "tests"]).default("tests")
|
|
77179
|
+
}).strict()
|
|
77180
|
+
]);
|
|
77181
|
+
var EvidenceFileSchema = exports_external.object({
|
|
77182
|
+
source: RootRelativePathSchema,
|
|
77183
|
+
sha256: Sha256Schema,
|
|
77184
|
+
size_bytes: exports_external.number().int().min(0).max(2 * 1024 * 1024 * 1024)
|
|
77185
|
+
}).strict();
|
|
77186
|
+
var FinalOutputEvidenceSchema = EvidenceFileSchema.extend({
|
|
77187
|
+
size_bytes: exports_external.number().int().min(0).max(MAX_FINAL_OUTPUT_BYTES)
|
|
77188
|
+
}).strict();
|
|
77189
|
+
var TrajectoryEvidenceSchema = EvidenceFileSchema.extend({
|
|
77190
|
+
size_bytes: exports_external.number().int().min(0).max(MAX_TRAJECTORY_BYTES)
|
|
77191
|
+
}).strict();
|
|
77192
|
+
var HydrateSpecSchema = BaseSpecSchema.extend({
|
|
77193
|
+
phase: exports_external.literal("hydrate"),
|
|
77194
|
+
materials: exports_external.array(MaterialSchema).max(1e4).default([]),
|
|
77195
|
+
setup_commands: exports_external.array(CommandSchema).max(128).default([])
|
|
77196
|
+
}).strict();
|
|
77197
|
+
var EvaluateSpecSchema = BaseSpecSchema.extend({
|
|
77198
|
+
phase: exports_external.literal("evaluate"),
|
|
77199
|
+
tests_root: AbsolutePathSchema,
|
|
77200
|
+
evidence: exports_external.object({
|
|
77201
|
+
final_output: FinalOutputEvidenceSchema,
|
|
77202
|
+
trajectory: TrajectoryEvidenceSchema
|
|
77203
|
+
}).strict(),
|
|
77204
|
+
references: exports_external.array(MaterialSchema).max(1e4).default([]),
|
|
77205
|
+
evaluators: exports_external.array(EvaluatorSchema).min(1).max(1000),
|
|
77206
|
+
candidate_artifacts: exports_external.array(RootRelativePathSchema).max(1000).default([]),
|
|
77207
|
+
capture_workspace_archive: exports_external.boolean().default(false),
|
|
77208
|
+
workspace_limits: exports_external.object({
|
|
77209
|
+
max_file_count: exports_external.number().int().min(1).max(1e6).default(1e5),
|
|
77210
|
+
max_total_bytes: exports_external.number().int().min(1).max(20 * 1024 * 1024 * 1024).default(2 * 1024 * 1024 * 1024)
|
|
77211
|
+
}).strict().default({})
|
|
77212
|
+
}).strict();
|
|
77213
|
+
var BenchmarkSpecSchema = exports_external.discriminatedUnion("phase", [
|
|
77214
|
+
HydrateSpecSchema,
|
|
77215
|
+
EvaluateSpecSchema
|
|
77216
|
+
]).superRefine((value, context) => {
|
|
77217
|
+
for (const name of Object.keys(value.environment)) {
|
|
77218
|
+
if (SENSITIVE_ENV_NAME_RE.test(name)) {
|
|
77219
|
+
context.addIssue({
|
|
77220
|
+
code: exports_external.ZodIssueCode.custom,
|
|
77221
|
+
path: ["environment", name],
|
|
77222
|
+
message: "credential-like values must be supplied through secret_env"
|
|
77223
|
+
});
|
|
77224
|
+
}
|
|
77225
|
+
}
|
|
77226
|
+
const declaredSecrets = new Set(value.secret_env);
|
|
77227
|
+
const commands = value.phase === "hydrate" ? value.setup_commands.map((command, index) => ({
|
|
77228
|
+
command,
|
|
77229
|
+
path: ["setup_commands", index, "secret_env"]
|
|
77230
|
+
})) : value.evaluators.flatMap((evaluator, index) => evaluator.type === "sandbox_command" ? [{
|
|
77231
|
+
command: evaluator.command,
|
|
77232
|
+
path: ["evaluators", index, "command", "secret_env"]
|
|
77233
|
+
}] : []);
|
|
77234
|
+
for (const { command, path: issuePath } of commands) {
|
|
77235
|
+
for (const name of command.secret_env) {
|
|
77236
|
+
if (!declaredSecrets.has(name)) {
|
|
77237
|
+
context.addIssue({
|
|
77238
|
+
code: exports_external.ZodIssueCode.custom,
|
|
77239
|
+
path: issuePath,
|
|
77240
|
+
message: `command secret_env references undeclared binding: ${name}`
|
|
77241
|
+
});
|
|
77242
|
+
}
|
|
77243
|
+
}
|
|
77244
|
+
}
|
|
77245
|
+
const uniqueLists = value.phase === "hydrate" ? [
|
|
77246
|
+
{ items: value.materials, path: "materials" },
|
|
77247
|
+
{ items: value.setup_commands, path: "setup_commands" }
|
|
77248
|
+
] : [
|
|
77249
|
+
{ items: value.references, path: "references" },
|
|
77250
|
+
{ items: value.evaluators, path: "evaluators" }
|
|
77251
|
+
];
|
|
77252
|
+
for (const { items, path: issuePath } of uniqueLists) {
|
|
77253
|
+
const ids = items.map((item) => item.id);
|
|
77254
|
+
if (new Set(ids).size !== ids.length) {
|
|
77255
|
+
context.addIssue({
|
|
77256
|
+
code: exports_external.ZodIssueCode.custom,
|
|
77257
|
+
path: [issuePath],
|
|
77258
|
+
message: `${issuePath} ids must be unique`
|
|
77259
|
+
});
|
|
77260
|
+
}
|
|
77261
|
+
}
|
|
77262
|
+
if (value.phase === "evaluate" && value.evaluators.filter((evaluator) => evaluator.type === "sandbox_command").length > 1) {
|
|
77263
|
+
context.addIssue({
|
|
77264
|
+
code: exports_external.ZodIssueCode.custom,
|
|
77265
|
+
path: ["evaluators"],
|
|
77266
|
+
message: "schema version 1 supports at most one sandbox_command evaluator"
|
|
77267
|
+
});
|
|
77268
|
+
}
|
|
77269
|
+
});
|
|
77270
|
+
var BENCHMARK_CAPABILITIES = {
|
|
77271
|
+
contract: "brainbase.benchmark.phase",
|
|
77272
|
+
cli_version: VERSION,
|
|
77273
|
+
schema_versions: [SCHEMA_VERSION],
|
|
77274
|
+
phases: ["hydrate", "evaluate"],
|
|
77275
|
+
evaluator_types: [
|
|
77276
|
+
"output_assertion",
|
|
77277
|
+
"trajectory_assertion",
|
|
77278
|
+
"workspace_assertion",
|
|
77279
|
+
"sandbox_command"
|
|
77280
|
+
],
|
|
77281
|
+
limits: {
|
|
77282
|
+
max_secret_bindings: 100,
|
|
77283
|
+
max_evaluators: 1000,
|
|
77284
|
+
max_sandbox_commands: 1
|
|
77285
|
+
}
|
|
77286
|
+
};
|
|
77287
|
+
|
|
77288
|
+
class BenchmarkPhaseError extends Error {
|
|
77289
|
+
code;
|
|
77290
|
+
details;
|
|
77291
|
+
constructor(code, message, details) {
|
|
77292
|
+
super(message);
|
|
77293
|
+
this.code = code;
|
|
77294
|
+
this.details = details;
|
|
77295
|
+
this.name = "BenchmarkPhaseError";
|
|
77296
|
+
}
|
|
77297
|
+
}
|
|
77298
|
+
function nowIso() {
|
|
77299
|
+
return new Date().toISOString();
|
|
77300
|
+
}
|
|
77301
|
+
function normalizedRootRelative(input) {
|
|
77302
|
+
if (input === ".")
|
|
77303
|
+
return ".";
|
|
77304
|
+
return safeRelPath(input);
|
|
77305
|
+
}
|
|
77306
|
+
function isWithin(root, candidate) {
|
|
77307
|
+
const relative = path87.relative(path87.resolve(root), path87.resolve(candidate));
|
|
77308
|
+
return relative === "" || !relative.startsWith("..") && !path87.isAbsolute(relative);
|
|
77309
|
+
}
|
|
77310
|
+
function canonicalFuturePath(input) {
|
|
77311
|
+
const resolved = path87.resolve(input);
|
|
77312
|
+
const suffix = [];
|
|
77313
|
+
let current = resolved;
|
|
77314
|
+
while (!fs79.existsSync(current)) {
|
|
77315
|
+
const parent = path87.dirname(current);
|
|
77316
|
+
if (parent === current)
|
|
77317
|
+
break;
|
|
77318
|
+
suffix.unshift(path87.basename(current));
|
|
77319
|
+
current = parent;
|
|
77320
|
+
}
|
|
77321
|
+
const canonicalBase = fs79.realpathSync(current);
|
|
77322
|
+
return path87.join(canonicalBase, ...suffix);
|
|
77323
|
+
}
|
|
77324
|
+
function validateRoots(spec) {
|
|
77325
|
+
const workspace = path87.resolve(spec.workspace_root);
|
|
77326
|
+
if (!fs79.existsSync(workspace) || fs79.lstatSync(workspace).isSymbolicLink() || !fs79.lstatSync(workspace).isDirectory()) {
|
|
77327
|
+
throw new BenchmarkPhaseError("invalid_workspace_root", "workspace_root must be an existing real directory");
|
|
77328
|
+
}
|
|
77329
|
+
const canonicalWorkspace = canonicalFuturePath(workspace);
|
|
77330
|
+
const staging = path87.resolve(spec.staging_root);
|
|
77331
|
+
const expectedStaging = path87.join(workspace, ".brainbase", "benchmark", spec.attempt_id, "incoming");
|
|
77332
|
+
if (staging !== expectedStaging) {
|
|
77333
|
+
throw new BenchmarkPhaseError("invalid_staging_root", `staging_root must be ${expectedStaging}`);
|
|
77334
|
+
}
|
|
77335
|
+
if (!fs79.existsSync(staging) || fs79.lstatSync(staging).isSymbolicLink() || !fs79.lstatSync(staging).isDirectory() || fs79.realpathSync(staging) !== path87.join(canonicalWorkspace, ".brainbase", "benchmark", spec.attempt_id, "incoming")) {
|
|
77336
|
+
throw new BenchmarkPhaseError("invalid_staging_root", "staging_root must be a real directory under workspace_root");
|
|
77337
|
+
}
|
|
77338
|
+
const canonicalLogs = validateExternalRoot("logs_root", spec.logs_root, canonicalWorkspace);
|
|
77339
|
+
if (spec.phase === "evaluate") {
|
|
77340
|
+
const canonicalTests = validateExternalRoot("tests_root", spec.tests_root, canonicalWorkspace);
|
|
77341
|
+
if (isWithin(canonicalTests, canonicalLogs) || isWithin(canonicalLogs, canonicalTests)) {
|
|
77342
|
+
throw new BenchmarkPhaseError("invalid_tests_root", "tests_root and logs_root must be disjoint");
|
|
77343
|
+
}
|
|
77344
|
+
}
|
|
77345
|
+
}
|
|
77346
|
+
function validateExternalRoot(label, input, canonicalWorkspace) {
|
|
77347
|
+
const candidate = path87.resolve(input);
|
|
77348
|
+
if (candidate === path87.parse(candidate).root) {
|
|
77349
|
+
throw new BenchmarkPhaseError(`invalid_${label}`, `${label} cannot be a filesystem root`);
|
|
77350
|
+
}
|
|
77351
|
+
if (fs79.existsSync(candidate) && fs79.lstatSync(candidate).isSymbolicLink()) {
|
|
77352
|
+
throw new BenchmarkPhaseError(`invalid_${label}`, `${label} cannot be a symlink`);
|
|
77353
|
+
}
|
|
77354
|
+
const canonicalCandidate = canonicalFuturePath(candidate);
|
|
77355
|
+
if (isWithin(canonicalWorkspace, canonicalCandidate)) {
|
|
77356
|
+
throw new BenchmarkPhaseError(`invalid_${label}`, `${label} must be outside workspace_root`);
|
|
77357
|
+
}
|
|
77358
|
+
if (isWithin(canonicalCandidate, canonicalWorkspace)) {
|
|
77359
|
+
throw new BenchmarkPhaseError(`invalid_${label}`, `${label} cannot contain workspace_root`);
|
|
77360
|
+
}
|
|
77361
|
+
return canonicalCandidate;
|
|
77362
|
+
}
|
|
77363
|
+
function workspaceRel(input, allowRoot = false) {
|
|
77364
|
+
const rel = normalizedRootRelative(input);
|
|
77365
|
+
if (rel === "." && !allowRoot) {
|
|
77366
|
+
throw new BenchmarkPhaseError("unsafe_path", "destination cannot be the workspace root");
|
|
77367
|
+
}
|
|
77368
|
+
const first = rel.split("/")[0].toLowerCase();
|
|
77369
|
+
if (RESERVED_WORKSPACE_PATHS.has(first)) {
|
|
77370
|
+
throw new BenchmarkPhaseError("reserved_path", `benchmark input cannot replace ${first}`);
|
|
77371
|
+
}
|
|
77372
|
+
return rel;
|
|
77373
|
+
}
|
|
77374
|
+
function assertNoSymlinkTraversal(root, relative) {
|
|
77375
|
+
const rel = normalizedRootRelative(relative);
|
|
77376
|
+
if (rel === ".")
|
|
77377
|
+
return;
|
|
77378
|
+
let current = path87.resolve(root);
|
|
77379
|
+
for (const segment of rel.split("/").slice(0, -1)) {
|
|
77380
|
+
current = path87.join(current, segment);
|
|
77381
|
+
if (!fs79.existsSync(current))
|
|
77382
|
+
continue;
|
|
77383
|
+
if (fs79.lstatSync(current).isSymbolicLink()) {
|
|
77384
|
+
throw new BenchmarkPhaseError("unsafe_path", `path traverses symlink: ${relative}`);
|
|
77385
|
+
}
|
|
77386
|
+
}
|
|
77387
|
+
}
|
|
77388
|
+
function assertOpenedFileWithinRoot(root, filePath, openedStat, label) {
|
|
77389
|
+
let canonicalRoot;
|
|
77390
|
+
let canonicalFile;
|
|
77391
|
+
let currentStat;
|
|
77392
|
+
try {
|
|
77393
|
+
canonicalRoot = fs79.realpathSync(root);
|
|
77394
|
+
canonicalFile = fs79.realpathSync(filePath);
|
|
77395
|
+
currentStat = fs79.statSync(filePath);
|
|
77396
|
+
} catch {
|
|
77397
|
+
throw new BenchmarkPhaseError("unsafe_path", `${label} changed while it was opened`);
|
|
77398
|
+
}
|
|
77399
|
+
if (!isWithin(canonicalRoot, canonicalFile) || currentStat.dev !== openedStat.dev || currentStat.ino !== openedStat.ino) {
|
|
77400
|
+
throw new BenchmarkPhaseError("unsafe_path", `${label} escapes its declared root`);
|
|
77401
|
+
}
|
|
77402
|
+
}
|
|
77403
|
+
function openRegularFileNoFollow(filePath, label, root) {
|
|
77404
|
+
const noFollow = typeof fs79.constants.O_NOFOLLOW === "number" ? fs79.constants.O_NOFOLLOW : 0;
|
|
77405
|
+
let fd;
|
|
77406
|
+
try {
|
|
77407
|
+
fd = fs79.openSync(filePath, fs79.constants.O_RDONLY | noFollow);
|
|
77408
|
+
} catch (error2) {
|
|
77409
|
+
const code = error2.code;
|
|
77410
|
+
if (code === "ELOOP") {
|
|
77411
|
+
throw new BenchmarkPhaseError("unsafe_path", `${label} cannot be a symlink`);
|
|
77412
|
+
}
|
|
77413
|
+
throw error2;
|
|
77414
|
+
}
|
|
77415
|
+
const stat = fs79.fstatSync(fd);
|
|
77416
|
+
if (!stat.isFile()) {
|
|
77417
|
+
fs79.closeSync(fd);
|
|
77418
|
+
throw new BenchmarkPhaseError("invalid_input", `${label} must be a regular file`);
|
|
77419
|
+
}
|
|
77420
|
+
if (root) {
|
|
77421
|
+
try {
|
|
77422
|
+
assertOpenedFileWithinRoot(root, filePath, stat, label);
|
|
77423
|
+
} catch (error2) {
|
|
77424
|
+
fs79.closeSync(fd);
|
|
77425
|
+
throw error2;
|
|
77426
|
+
}
|
|
77427
|
+
}
|
|
77428
|
+
return { fd, stat };
|
|
77429
|
+
}
|
|
77430
|
+
async function sha256OfDescriptor(fd) {
|
|
77431
|
+
const hash = crypto6.createHash("sha256");
|
|
77432
|
+
const stream = fs79.createReadStream("", {
|
|
77433
|
+
fd,
|
|
77434
|
+
autoClose: false,
|
|
77435
|
+
start: 0
|
|
77436
|
+
});
|
|
77437
|
+
for await (const chunk2 of stream) {
|
|
77438
|
+
hash.update(chunk2);
|
|
77439
|
+
}
|
|
77440
|
+
return hash.digest("hex");
|
|
77441
|
+
}
|
|
77442
|
+
function readDescriptor(fd) {
|
|
77443
|
+
return fs79.readFileSync(fd);
|
|
77444
|
+
}
|
|
77445
|
+
function assertWritableDestination(root, relative) {
|
|
77446
|
+
const rel = normalizedRootRelative(relative);
|
|
77447
|
+
if (rel === ".")
|
|
77448
|
+
return;
|
|
77449
|
+
let current = path87.resolve(root);
|
|
77450
|
+
const segments = rel.split("/");
|
|
77451
|
+
for (const segment of segments.slice(0, -1)) {
|
|
77452
|
+
current = path87.join(current, segment);
|
|
77453
|
+
if (!fs79.existsSync(current))
|
|
77454
|
+
continue;
|
|
77455
|
+
const stat = fs79.lstatSync(current);
|
|
77456
|
+
if (stat.isSymbolicLink()) {
|
|
77457
|
+
throw new BenchmarkPhaseError("unsafe_path", `path traverses symlink: ${relative}`);
|
|
77458
|
+
}
|
|
77459
|
+
if (!stat.isDirectory()) {
|
|
77460
|
+
throw new BenchmarkPhaseError("destination_conflict", `destination parent is not a directory: ${relative}`);
|
|
77461
|
+
}
|
|
77462
|
+
}
|
|
77463
|
+
const destination = path87.resolve(root, rel);
|
|
77464
|
+
if (fs79.existsSync(destination) && fs79.lstatSync(destination).isDirectory()) {
|
|
77465
|
+
throw new BenchmarkPhaseError("destination_conflict", `file destination is an existing directory: ${relative}`);
|
|
77466
|
+
}
|
|
77467
|
+
}
|
|
77468
|
+
function validateDestinationGraph(paths) {
|
|
77469
|
+
const portablePaths = new Map;
|
|
77470
|
+
for (const candidate of paths) {
|
|
77471
|
+
const key2 = candidate.toLowerCase();
|
|
77472
|
+
const existing = portablePaths.get(key2);
|
|
77473
|
+
if (existing && existing !== candidate) {
|
|
77474
|
+
throw new BenchmarkPhaseError("destination_conflict", `destination ${candidate} conflicts by case with ${existing}`);
|
|
77475
|
+
}
|
|
77476
|
+
portablePaths.set(key2, candidate);
|
|
77477
|
+
}
|
|
77478
|
+
const sorted = [...portablePaths.keys()].sort();
|
|
77479
|
+
for (let index = 0;index < sorted.length - 1; index += 1) {
|
|
77480
|
+
const current = sorted[index];
|
|
77481
|
+
const next = sorted[index + 1];
|
|
77482
|
+
if (next.startsWith(`${current}/`)) {
|
|
77483
|
+
throw new BenchmarkPhaseError("destination_conflict", `destination ${current} conflicts with descendant ${next}`);
|
|
77484
|
+
}
|
|
77485
|
+
}
|
|
77486
|
+
}
|
|
77487
|
+
function sourcePath(stagingRoot, relative) {
|
|
77488
|
+
const rel = safeRelPath(relative);
|
|
77489
|
+
const source = path87.resolve(stagingRoot, rel);
|
|
77490
|
+
if (!isWithin(stagingRoot, source)) {
|
|
77491
|
+
throw new BenchmarkPhaseError("unsafe_path", `source escapes staging root: ${relative}`);
|
|
77492
|
+
}
|
|
77493
|
+
assertNoSymlinkTraversal(stagingRoot, rel);
|
|
77494
|
+
let stat;
|
|
77495
|
+
try {
|
|
77496
|
+
stat = fs79.lstatSync(source);
|
|
77497
|
+
} catch {
|
|
77498
|
+
throw new BenchmarkPhaseError("missing_input", `staged input does not exist: ${relative}`);
|
|
77499
|
+
}
|
|
77500
|
+
if (!stat.isFile()) {
|
|
77501
|
+
throw new BenchmarkPhaseError("invalid_input", `staged input must be a regular file: ${relative}`);
|
|
77502
|
+
}
|
|
77503
|
+
return source;
|
|
77504
|
+
}
|
|
77505
|
+
function rootForRecord(spec, rootName) {
|
|
77506
|
+
if (rootName === "workspace")
|
|
77507
|
+
return spec.workspace_root;
|
|
77508
|
+
if (rootName === "staging")
|
|
77509
|
+
return spec.staging_root;
|
|
77510
|
+
if (rootName === "logs")
|
|
77511
|
+
return spec.logs_root;
|
|
77512
|
+
if (rootName === "tests" && spec.phase === "evaluate")
|
|
77513
|
+
return spec.tests_root;
|
|
77514
|
+
return null;
|
|
77515
|
+
}
|
|
77516
|
+
async function verifyRecordsUnchanged(records, spec) {
|
|
77517
|
+
for (const record3 of records) {
|
|
77518
|
+
const root = rootForRecord(spec, record3.root);
|
|
77519
|
+
if (!root) {
|
|
77520
|
+
throw new BenchmarkPhaseError("evidence_tampered", `unknown evidence root: ${record3.root}`);
|
|
77521
|
+
}
|
|
77522
|
+
const relative = safeRelPath(record3.path);
|
|
77523
|
+
assertNoSymlinkTraversal(root, relative);
|
|
77524
|
+
const candidate = path87.resolve(root, relative);
|
|
77525
|
+
if (!isWithin(root, candidate) || !fs79.existsSync(candidate)) {
|
|
77526
|
+
throw new BenchmarkPhaseError("evidence_tampered", `evidence was removed during evaluation: ${record3.root}:${record3.path}`);
|
|
77527
|
+
}
|
|
77528
|
+
const stat = fs79.lstatSync(candidate);
|
|
77529
|
+
if (record3.kind === "symlink") {
|
|
77530
|
+
const target = stat.isSymbolicLink() ? fs79.readlinkSync(candidate) : null;
|
|
77531
|
+
if (target === null || Buffer.byteLength(target) !== record3.size || sha256(target) !== record3.sha256) {
|
|
77532
|
+
throw new BenchmarkPhaseError("evidence_tampered", `evidence changed during evaluation: ${record3.root}:${record3.path}`);
|
|
77533
|
+
}
|
|
77534
|
+
continue;
|
|
77535
|
+
}
|
|
77536
|
+
if (!stat.isFile()) {
|
|
77537
|
+
throw new BenchmarkPhaseError("evidence_tampered", `evidence changed during evaluation: ${record3.root}:${record3.path}`);
|
|
77538
|
+
}
|
|
77539
|
+
const opened = openRegularFileNoFollow(candidate, `evidence ${record3.root}:${record3.path}`, root);
|
|
77540
|
+
try {
|
|
77541
|
+
if (opened.stat.size !== record3.size || await sha256OfDescriptor(opened.fd) !== record3.sha256) {
|
|
77542
|
+
throw new BenchmarkPhaseError("evidence_tampered", `evidence changed during evaluation: ${record3.root}:${record3.path}`);
|
|
77543
|
+
}
|
|
77544
|
+
} finally {
|
|
77545
|
+
fs79.closeSync(opened.fd);
|
|
77546
|
+
}
|
|
77547
|
+
}
|
|
77548
|
+
}
|
|
77549
|
+
async function verifyInput(stagingRoot, material) {
|
|
77550
|
+
const source = sourcePath(stagingRoot, material.source);
|
|
77551
|
+
const opened = openRegularFileNoFollow(source, `staged input ${material.source}`, stagingRoot);
|
|
77552
|
+
try {
|
|
77553
|
+
if (opened.stat.size !== material.size_bytes) {
|
|
77554
|
+
throw new BenchmarkPhaseError("size_mismatch", `size mismatch for ${material.source}`, { expected: material.size_bytes, actual: opened.stat.size });
|
|
77555
|
+
}
|
|
77556
|
+
const actual = await sha256OfDescriptor(opened.fd);
|
|
77557
|
+
if (actual !== material.sha256) {
|
|
77558
|
+
throw new BenchmarkPhaseError("digest_mismatch", `checksum mismatch for ${material.source}`, { expected: material.sha256, actual });
|
|
77559
|
+
}
|
|
77560
|
+
} finally {
|
|
77561
|
+
fs79.closeSync(opened.fd);
|
|
77562
|
+
}
|
|
77563
|
+
return source;
|
|
77564
|
+
}
|
|
77565
|
+
async function atomicCopy(source, destination, mode, sourceRoot) {
|
|
77566
|
+
fs79.mkdirSync(path87.dirname(destination), { recursive: true });
|
|
77567
|
+
const temporary = `${destination}.${process.pid}.${crypto6.randomBytes(6).toString("hex")}.tmp`;
|
|
77568
|
+
const opened = openRegularFileNoFollow(source, `copy source ${source}`, sourceRoot);
|
|
77569
|
+
try {
|
|
77570
|
+
await pipeline2(fs79.createReadStream("", {
|
|
77571
|
+
fd: opened.fd,
|
|
77572
|
+
autoClose: false,
|
|
77573
|
+
start: 0
|
|
77574
|
+
}), fs79.createWriteStream(temporary, {
|
|
77575
|
+
flags: "wx",
|
|
77576
|
+
mode: 384
|
|
77577
|
+
}));
|
|
77578
|
+
fs79.chmodSync(temporary, mode ?? opened.stat.mode & 511);
|
|
77579
|
+
fs79.renameSync(temporary, destination);
|
|
77580
|
+
} finally {
|
|
77581
|
+
fs79.closeSync(opened.fd);
|
|
77582
|
+
fs79.rmSync(temporary, { force: true });
|
|
77583
|
+
}
|
|
77584
|
+
}
|
|
77585
|
+
async function recordFile(root, filePath, rootName, kind = "file") {
|
|
77586
|
+
const relative = path87.relative(root, filePath).replace(/\\/g, "/");
|
|
77587
|
+
if (kind === "symlink") {
|
|
77588
|
+
const stat = fs79.lstatSync(filePath);
|
|
77589
|
+
const target = fs79.readlinkSync(filePath);
|
|
77590
|
+
return {
|
|
77591
|
+
root: rootName,
|
|
77592
|
+
path: relative,
|
|
77593
|
+
sha256: sha256(target),
|
|
77594
|
+
size: Buffer.byteLength(target),
|
|
77595
|
+
mode: stat.mode & 511,
|
|
77596
|
+
kind
|
|
77597
|
+
};
|
|
77598
|
+
}
|
|
77599
|
+
const opened = openRegularFileNoFollow(filePath, `${rootName} file ${relative}`, root);
|
|
77600
|
+
try {
|
|
77601
|
+
return {
|
|
77602
|
+
root: rootName,
|
|
77603
|
+
path: relative,
|
|
77604
|
+
sha256: await sha256OfDescriptor(opened.fd),
|
|
77605
|
+
size: opened.stat.size,
|
|
77606
|
+
mode: opened.stat.mode & 511
|
|
77607
|
+
};
|
|
77608
|
+
} finally {
|
|
77609
|
+
fs79.closeSync(opened.fd);
|
|
77610
|
+
}
|
|
77611
|
+
}
|
|
77612
|
+
async function copyMaterial(material, sourceRoot, destinationRoot, destinationRootName, protectWorkspace) {
|
|
77613
|
+
const source = await verifyInput(sourceRoot, material);
|
|
77614
|
+
const destinationRel = protectWorkspace ? workspaceRel(material.destination, material.kind === "tar_gz") : normalizedRootRelative(material.destination);
|
|
77615
|
+
if (!protectWorkspace && destinationRel.toLowerCase() === ".brainbase-benchmark-owner.json") {
|
|
77616
|
+
throw new BenchmarkPhaseError("reserved_path", "benchmark input cannot replace the phase ownership marker");
|
|
77617
|
+
}
|
|
77618
|
+
if (material.kind === "file") {
|
|
77619
|
+
if (destinationRel === ".") {
|
|
77620
|
+
throw new BenchmarkPhaseError("unsafe_path", "file destination cannot be a directory root");
|
|
77621
|
+
}
|
|
77622
|
+
assertWritableDestination(destinationRoot, destinationRel);
|
|
77623
|
+
const destination = path87.resolve(destinationRoot, destinationRel);
|
|
77624
|
+
await atomicCopy(source, destination, material.mode, sourceRoot);
|
|
77625
|
+
const record3 = await recordFile(destinationRoot, destination, destinationRootName);
|
|
77626
|
+
if (record3.sha256 !== material.sha256 || record3.size !== material.size_bytes) {
|
|
77627
|
+
throw new BenchmarkPhaseError("evidence_tampered", `material changed while it was copied: ${material.source}`);
|
|
77628
|
+
}
|
|
77629
|
+
return [record3];
|
|
77630
|
+
}
|
|
77631
|
+
const temporary = fs79.mkdtempSync(path87.join(os16.tmpdir(), "brainbase-benchmark-"));
|
|
77632
|
+
try {
|
|
77633
|
+
const verifiedArchive = path87.join(temporary, "material.tar.gz");
|
|
77634
|
+
await atomicCopy(source, verifiedArchive, 384, sourceRoot);
|
|
77635
|
+
const archiveRecord = await recordFile(temporary, verifiedArchive, "staging");
|
|
77636
|
+
if (archiveRecord.sha256 !== material.sha256 || archiveRecord.size !== material.size_bytes) {
|
|
77637
|
+
throw new BenchmarkPhaseError("evidence_tampered", `material changed while it was copied: ${material.source}`);
|
|
77638
|
+
}
|
|
77639
|
+
const extractedRoot = path87.join(temporary, "extracted");
|
|
77640
|
+
const extracted = await extract({
|
|
77641
|
+
tarFile: verifiedArchive,
|
|
77642
|
+
outDir: extractedRoot,
|
|
77643
|
+
maxTotalBytes: material.max_unpacked_bytes,
|
|
77644
|
+
maxFileCount: material.max_file_count
|
|
77645
|
+
});
|
|
77646
|
+
const outputs = [];
|
|
77647
|
+
for (const extractedRel of extracted.sort()) {
|
|
77648
|
+
const sourceFile = path87.resolve(extractedRoot, safeRelPath(extractedRel));
|
|
77649
|
+
const stat = fs79.lstatSync(sourceFile);
|
|
77650
|
+
if (!stat.isFile())
|
|
77651
|
+
continue;
|
|
77652
|
+
const combined = destinationRel === "." ? safeRelPath(extractedRel) : safeRelPath(path87.posix.join(destinationRel, extractedRel));
|
|
77653
|
+
const checked = protectWorkspace ? workspaceRel(combined) : combined;
|
|
77654
|
+
if (!protectWorkspace && checked.toLowerCase() === ".brainbase-benchmark-owner.json") {
|
|
77655
|
+
throw new BenchmarkPhaseError("reserved_path", "benchmark input cannot replace the phase ownership marker");
|
|
77656
|
+
}
|
|
77657
|
+
assertWritableDestination(destinationRoot, checked);
|
|
77658
|
+
const destination = path87.resolve(destinationRoot, checked);
|
|
77659
|
+
await atomicCopy(sourceFile, destination, material.mode, extractedRoot);
|
|
77660
|
+
outputs.push(await recordFile(destinationRoot, destination, destinationRootName));
|
|
77661
|
+
}
|
|
77662
|
+
return outputs;
|
|
77663
|
+
} finally {
|
|
77664
|
+
fs79.rmSync(temporary, { recursive: true, force: true });
|
|
77665
|
+
}
|
|
77666
|
+
}
|
|
77667
|
+
async function preflightMaterial(material, sourceRoot, destinationRoot, protectWorkspace) {
|
|
77668
|
+
const source = await verifyInput(sourceRoot, material);
|
|
77669
|
+
const destinationRel = protectWorkspace ? workspaceRel(material.destination, material.kind === "tar_gz") : normalizedRootRelative(material.destination);
|
|
77670
|
+
if (!protectWorkspace && destinationRel.toLowerCase() === ".brainbase-benchmark-owner.json") {
|
|
77671
|
+
throw new BenchmarkPhaseError("reserved_path", "benchmark input cannot replace the phase ownership marker");
|
|
77672
|
+
}
|
|
77673
|
+
if (material.kind === "file") {
|
|
77674
|
+
if (destinationRel === ".") {
|
|
77675
|
+
throw new BenchmarkPhaseError("unsafe_path", "file destination cannot be a directory root");
|
|
77676
|
+
}
|
|
77677
|
+
assertWritableDestination(destinationRoot, destinationRel);
|
|
77678
|
+
return [destinationRel];
|
|
77679
|
+
}
|
|
77680
|
+
const temporary = fs79.mkdtempSync(path87.join(os16.tmpdir(), "brainbase-benchmark-preflight-"));
|
|
77681
|
+
try {
|
|
77682
|
+
const verifiedArchive = path87.join(temporary, "material.tar.gz");
|
|
77683
|
+
await atomicCopy(source, verifiedArchive, 384, sourceRoot);
|
|
77684
|
+
const archiveRecord = await recordFile(temporary, verifiedArchive, "staging");
|
|
77685
|
+
if (archiveRecord.sha256 !== material.sha256 || archiveRecord.size !== material.size_bytes) {
|
|
77686
|
+
throw new BenchmarkPhaseError("evidence_tampered", `material changed while it was copied: ${material.source}`);
|
|
77687
|
+
}
|
|
77688
|
+
const extractedRoot = path87.join(temporary, "extracted");
|
|
77689
|
+
const extracted = await extract({
|
|
77690
|
+
tarFile: verifiedArchive,
|
|
77691
|
+
outDir: extractedRoot,
|
|
77692
|
+
maxTotalBytes: material.max_unpacked_bytes,
|
|
77693
|
+
maxFileCount: material.max_file_count
|
|
77694
|
+
});
|
|
77695
|
+
const planned = [];
|
|
77696
|
+
for (const extractedRel of extracted) {
|
|
77697
|
+
const combined = destinationRel === "." ? safeRelPath(extractedRel) : safeRelPath(path87.posix.join(destinationRel, extractedRel));
|
|
77698
|
+
const checked = protectWorkspace ? workspaceRel(combined) : combined;
|
|
77699
|
+
if (!protectWorkspace && checked.toLowerCase() === ".brainbase-benchmark-owner.json") {
|
|
77700
|
+
throw new BenchmarkPhaseError("reserved_path", "benchmark input cannot replace the phase ownership marker");
|
|
77701
|
+
}
|
|
77702
|
+
assertWritableDestination(destinationRoot, checked);
|
|
77703
|
+
planned.push(checked);
|
|
77704
|
+
}
|
|
77705
|
+
return planned;
|
|
77706
|
+
} finally {
|
|
77707
|
+
fs79.rmSync(temporary, { recursive: true, force: true });
|
|
77708
|
+
}
|
|
77709
|
+
}
|
|
77710
|
+
function ownerMarker(root) {
|
|
77711
|
+
return path87.join(root, ".brainbase-benchmark-owner.json");
|
|
77712
|
+
}
|
|
77713
|
+
function verifyOwnedDirectory(root, role, spec) {
|
|
77714
|
+
if (!fs79.existsSync(root) || fs79.lstatSync(root).isSymbolicLink())
|
|
77715
|
+
return false;
|
|
77716
|
+
try {
|
|
77717
|
+
const marker = JSON.parse(fs79.readFileSync(ownerMarker(root), "utf8"));
|
|
77718
|
+
return marker.attempt_id === spec.attempt_id && marker.phase === spec.phase && marker.phase_id === spec.phase_id && marker.role === role;
|
|
77719
|
+
} catch {
|
|
77720
|
+
return false;
|
|
77721
|
+
}
|
|
77722
|
+
}
|
|
77723
|
+
function prepareOwnedDirectory(root, role, spec) {
|
|
77724
|
+
if (fs79.existsSync(root)) {
|
|
77725
|
+
if (!verifyOwnedDirectory(root, role, spec)) {
|
|
77726
|
+
const stat = fs79.lstatSync(root);
|
|
77727
|
+
if (!stat.isDirectory() || fs79.readdirSync(root).length > 0) {
|
|
77728
|
+
throw new BenchmarkPhaseError(`unowned_${role}_root`, `${role}_root exists without a matching attempt ownership marker`);
|
|
77729
|
+
}
|
|
77730
|
+
} else {
|
|
77731
|
+
fs79.rmSync(root, { recursive: true, force: true });
|
|
77732
|
+
}
|
|
77733
|
+
}
|
|
77734
|
+
fs79.mkdirSync(root, { recursive: true, mode: 448 });
|
|
77735
|
+
writeJsonAtomic(ownerMarker(root), {
|
|
77736
|
+
schema_version: SCHEMA_VERSION,
|
|
77737
|
+
attempt_id: spec.attempt_id,
|
|
77738
|
+
phase: spec.phase,
|
|
77739
|
+
phase_id: spec.phase_id,
|
|
77740
|
+
role
|
|
77741
|
+
});
|
|
77742
|
+
}
|
|
77743
|
+
function buildEnvironment(spec, secretNames, additions = {}) {
|
|
77744
|
+
const env3 = {};
|
|
77745
|
+
for (const name of BASE_ENV_NAMES) {
|
|
77746
|
+
if (process.env[name] !== undefined)
|
|
77747
|
+
env3[name] = process.env[name];
|
|
77748
|
+
}
|
|
77749
|
+
for (const [name, declared] of Object.entries(spec.environment)) {
|
|
77750
|
+
env3[name] = declared.value;
|
|
77751
|
+
}
|
|
77752
|
+
for (const name of secretNames) {
|
|
77753
|
+
const value = process.env[name];
|
|
77754
|
+
if (value === undefined) {
|
|
77755
|
+
throw new BenchmarkPhaseError("missing_environment", `required environment variable is missing: ${name}`);
|
|
77756
|
+
}
|
|
77757
|
+
env3[name] = value;
|
|
77758
|
+
}
|
|
77759
|
+
Object.assign(env3, additions);
|
|
77760
|
+
return env3;
|
|
77761
|
+
}
|
|
77762
|
+
function redactCommandOutput(data, spec) {
|
|
77763
|
+
let value = data.toString("utf8");
|
|
77764
|
+
for (const name of spec.secret_env) {
|
|
77765
|
+
const secret = process.env[name];
|
|
77766
|
+
if (secret)
|
|
77767
|
+
value = value.split(secret).join("[REDACTED]");
|
|
77768
|
+
}
|
|
77769
|
+
return Buffer.from(value);
|
|
77770
|
+
}
|
|
77771
|
+
function descendantPids(parentPid) {
|
|
77772
|
+
if (process.platform === "win32")
|
|
77773
|
+
return [];
|
|
77774
|
+
try {
|
|
77775
|
+
const output = execFileSync2("ps", ["-eo", "pid=,ppid="], {
|
|
77776
|
+
encoding: "utf8",
|
|
77777
|
+
stdio: ["ignore", "pipe", "ignore"]
|
|
77778
|
+
});
|
|
77779
|
+
const children = new Map;
|
|
77780
|
+
for (const line of output.split(`
|
|
77781
|
+
`)) {
|
|
77782
|
+
const [pidRaw, parentRaw] = line.trim().split(/\s+/);
|
|
77783
|
+
const pid = Number(pidRaw);
|
|
77784
|
+
const parent = Number(parentRaw);
|
|
77785
|
+
if (!Number.isInteger(pid) || !Number.isInteger(parent))
|
|
77786
|
+
continue;
|
|
77787
|
+
const current = children.get(parent) ?? [];
|
|
77788
|
+
current.push(pid);
|
|
77789
|
+
children.set(parent, current);
|
|
77790
|
+
}
|
|
77791
|
+
const descendants = [];
|
|
77792
|
+
const stack = [...children.get(parentPid) ?? []];
|
|
77793
|
+
while (stack.length > 0) {
|
|
77794
|
+
const pid = stack.pop();
|
|
77795
|
+
descendants.push(pid);
|
|
77796
|
+
stack.push(...children.get(pid) ?? []);
|
|
77797
|
+
}
|
|
77798
|
+
return descendants;
|
|
77799
|
+
} catch {
|
|
77800
|
+
return [];
|
|
77801
|
+
}
|
|
77802
|
+
}
|
|
77803
|
+
function terminate(child) {
|
|
77804
|
+
if (child.pid === undefined)
|
|
77805
|
+
return;
|
|
77806
|
+
for (const pid of descendantPids(child.pid).reverse()) {
|
|
77807
|
+
try {
|
|
77808
|
+
process.kill(pid, "SIGKILL");
|
|
77809
|
+
} catch {}
|
|
77810
|
+
}
|
|
77811
|
+
try {
|
|
77812
|
+
if (process.platform !== "win32")
|
|
77813
|
+
process.kill(-child.pid, "SIGKILL");
|
|
77814
|
+
else
|
|
77815
|
+
child.kill("SIGKILL");
|
|
77816
|
+
} catch {
|
|
77817
|
+
child.kill("SIGKILL");
|
|
77818
|
+
}
|
|
77819
|
+
}
|
|
77820
|
+
async function runCommand(command, root, spec, context, additions = {}) {
|
|
77821
|
+
const cwdRel = normalizedRootRelative(command.cwd);
|
|
77822
|
+
assertNoSymlinkTraversal(root, cwdRel);
|
|
77823
|
+
const cwd2 = path87.resolve(root, cwdRel);
|
|
77824
|
+
let cwdStat;
|
|
77825
|
+
try {
|
|
77826
|
+
cwdStat = fs79.lstatSync(cwd2);
|
|
77827
|
+
} catch {
|
|
77828
|
+
throw new BenchmarkPhaseError("invalid_command_cwd", `command cwd is invalid: ${command.cwd}`);
|
|
77829
|
+
}
|
|
77830
|
+
if (!isWithin(root, cwd2) || cwdStat.isSymbolicLink() || !cwdStat.isDirectory()) {
|
|
77831
|
+
throw new BenchmarkPhaseError("invalid_command_cwd", `command cwd is invalid: ${command.cwd}`);
|
|
77832
|
+
}
|
|
77833
|
+
const remainingMs = context.deadline - Date.now();
|
|
77834
|
+
if (remainingMs <= 0)
|
|
77835
|
+
throw new BenchmarkPhaseError("phase_timeout", "phase budget expired");
|
|
77836
|
+
const timeoutMs2 = Math.min(command.timeout_ms ?? remainingMs, remainingMs);
|
|
77837
|
+
const started = Date.now();
|
|
77838
|
+
return await new Promise((resolve, reject2) => {
|
|
77839
|
+
const child = spawn4(command.argv[0], command.argv.slice(1), {
|
|
77840
|
+
cwd: cwd2,
|
|
77841
|
+
env: buildEnvironment(spec, command.secret_env, additions),
|
|
77842
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
77843
|
+
detached: process.platform !== "win32"
|
|
77844
|
+
});
|
|
77845
|
+
const stdout = [];
|
|
77846
|
+
const stderr = [];
|
|
77847
|
+
let captured = 0;
|
|
77848
|
+
let settled = false;
|
|
77849
|
+
let timer;
|
|
77850
|
+
const fail = (error2) => {
|
|
77851
|
+
if (settled)
|
|
77852
|
+
return;
|
|
77853
|
+
settled = true;
|
|
77854
|
+
if (timer)
|
|
77855
|
+
clearTimeout(timer);
|
|
77856
|
+
terminate(child);
|
|
77857
|
+
reject2(error2);
|
|
77858
|
+
};
|
|
77859
|
+
const capture = (target, chunk2) => {
|
|
77860
|
+
if (settled)
|
|
77861
|
+
return;
|
|
77862
|
+
captured += chunk2.length;
|
|
77863
|
+
context.remainingOutputBytes -= chunk2.length;
|
|
77864
|
+
if (captured > spec.budget.max_output_bytes || context.remainingOutputBytes < 0) {
|
|
77865
|
+
fail(new BenchmarkPhaseError("output_limit_exceeded", `command output exceeded budget: ${command.id}`));
|
|
77866
|
+
return;
|
|
77867
|
+
}
|
|
77868
|
+
target.push(chunk2);
|
|
77869
|
+
};
|
|
77870
|
+
child.stdout?.on("data", (chunk2) => capture(stdout, chunk2));
|
|
77871
|
+
child.stderr?.on("data", (chunk2) => capture(stderr, chunk2));
|
|
77872
|
+
child.on("error", (error2) => {
|
|
77873
|
+
fail(new BenchmarkPhaseError("command_start_failed", `failed to start ${command.id}: ${error2.message}`));
|
|
77874
|
+
});
|
|
77875
|
+
timer = setTimeout(() => {
|
|
77876
|
+
fail(new BenchmarkPhaseError("command_timeout", `command timed out: ${command.id}`));
|
|
77877
|
+
}, timeoutMs2);
|
|
77878
|
+
child.on("close", (code, signal) => {
|
|
77879
|
+
if (settled)
|
|
77880
|
+
return;
|
|
77881
|
+
settled = true;
|
|
77882
|
+
clearTimeout(timer);
|
|
77883
|
+
if (code === null) {
|
|
77884
|
+
reject2(new BenchmarkPhaseError("command_terminated", `command was terminated by ${signal ?? "an unknown signal"}: ${command.id}`));
|
|
77885
|
+
return;
|
|
77886
|
+
}
|
|
77887
|
+
resolve({
|
|
77888
|
+
exitCode: code,
|
|
77889
|
+
stdout: Buffer.concat(stdout),
|
|
77890
|
+
stderr: Buffer.concat(stderr),
|
|
77891
|
+
durationMs: Date.now() - started
|
|
77892
|
+
});
|
|
77893
|
+
});
|
|
77894
|
+
});
|
|
77895
|
+
}
|
|
77896
|
+
async function writeLog(root, name, data, spec) {
|
|
77897
|
+
const destination = path87.join(root, name);
|
|
77898
|
+
fs79.mkdirSync(path87.dirname(destination), { recursive: true });
|
|
77899
|
+
const temporary = `${destination}.${process.pid}.${crypto6.randomBytes(6).toString("hex")}.tmp`;
|
|
77900
|
+
try {
|
|
77901
|
+
fs79.writeFileSync(temporary, redactCommandOutput(data, spec), {
|
|
77902
|
+
flag: "wx",
|
|
77903
|
+
mode: 384
|
|
77904
|
+
});
|
|
77905
|
+
fs79.renameSync(temporary, destination);
|
|
77906
|
+
} finally {
|
|
77907
|
+
fs79.rmSync(temporary, { force: true });
|
|
77908
|
+
}
|
|
77909
|
+
return await recordFile(root, destination, "logs");
|
|
77910
|
+
}
|
|
77911
|
+
function writeBufferAtomic(destination, data) {
|
|
77912
|
+
fs79.mkdirSync(path87.dirname(destination), { recursive: true });
|
|
77913
|
+
const temporary = `${destination}.${process.pid}.${crypto6.randomBytes(6).toString("hex")}.tmp`;
|
|
77914
|
+
try {
|
|
77915
|
+
fs79.writeFileSync(temporary, data, { flag: "wx", mode: 384 });
|
|
77916
|
+
fs79.renameSync(temporary, destination);
|
|
77917
|
+
} finally {
|
|
77918
|
+
fs79.rmSync(temporary, { force: true });
|
|
77919
|
+
}
|
|
77920
|
+
}
|
|
77921
|
+
function assertBudget(context) {
|
|
77922
|
+
if (Date.now() > context.deadline) {
|
|
77923
|
+
throw new BenchmarkPhaseError("phase_timeout", "phase budget expired");
|
|
77924
|
+
}
|
|
77925
|
+
}
|
|
77926
|
+
async function executeHydrate(spec, context) {
|
|
77927
|
+
fs79.mkdirSync(spec.workspace_root, { recursive: true });
|
|
77928
|
+
prepareOwnedDirectory(spec.logs_root, "logs", spec);
|
|
77929
|
+
context.logsOwned = true;
|
|
77930
|
+
const outputs = [];
|
|
77931
|
+
const plannedDestinations = [];
|
|
77932
|
+
for (const material of spec.materials) {
|
|
77933
|
+
assertBudget(context);
|
|
77934
|
+
plannedDestinations.push(...await preflightMaterial(material, spec.staging_root, spec.workspace_root, true));
|
|
77935
|
+
const source = await verifyInput(spec.staging_root, material);
|
|
77936
|
+
context.inputs.push(await recordFile(spec.staging_root, source, "staging"));
|
|
77937
|
+
}
|
|
77938
|
+
validateDestinationGraph(plannedDestinations);
|
|
77939
|
+
for (const material of spec.materials) {
|
|
77940
|
+
assertBudget(context);
|
|
77941
|
+
const started = Date.now();
|
|
77942
|
+
try {
|
|
77943
|
+
const records = await copyMaterial(material, spec.staging_root, spec.workspace_root, "workspace", true);
|
|
77944
|
+
outputs.push(...records);
|
|
77945
|
+
context.outputs.push(...records);
|
|
77946
|
+
context.steps.push({
|
|
77947
|
+
id: material.id,
|
|
77948
|
+
type: `material:${material.kind}`,
|
|
77949
|
+
status: "passed",
|
|
77950
|
+
duration_ms: Date.now() - started
|
|
77951
|
+
});
|
|
77952
|
+
} catch (error2) {
|
|
77953
|
+
context.steps.push({
|
|
77954
|
+
id: material.id,
|
|
77955
|
+
type: `material:${material.kind}`,
|
|
77956
|
+
status: "failed",
|
|
77957
|
+
duration_ms: Date.now() - started
|
|
77958
|
+
});
|
|
77959
|
+
throw error2;
|
|
77960
|
+
}
|
|
77961
|
+
}
|
|
77962
|
+
for (const command of spec.setup_commands) {
|
|
77963
|
+
assertBudget(context);
|
|
77964
|
+
let result2;
|
|
77965
|
+
try {
|
|
77966
|
+
result2 = await runCommand(command, spec.workspace_root, spec, context);
|
|
77967
|
+
} catch (error2) {
|
|
77968
|
+
context.steps.push({
|
|
77969
|
+
id: command.id,
|
|
77970
|
+
type: "setup_command",
|
|
77971
|
+
status: "failed",
|
|
77972
|
+
duration_ms: 0
|
|
77973
|
+
});
|
|
77974
|
+
throw error2;
|
|
77975
|
+
}
|
|
77976
|
+
const stdout = await writeLog(spec.logs_root, `${command.id}.stdout.log`, result2.stdout, spec);
|
|
77977
|
+
const stderr = await writeLog(spec.logs_root, `${command.id}.stderr.log`, result2.stderr, spec);
|
|
77978
|
+
context.steps.push({
|
|
77979
|
+
id: command.id,
|
|
77980
|
+
type: "setup_command",
|
|
77981
|
+
status: result2.exitCode === 0 ? "passed" : "failed",
|
|
77982
|
+
duration_ms: result2.durationMs,
|
|
77983
|
+
exit_code: result2.exitCode,
|
|
77984
|
+
stdout,
|
|
77985
|
+
stderr
|
|
77986
|
+
});
|
|
77987
|
+
outputs.push(stdout, stderr);
|
|
77988
|
+
context.outputs.push(stdout, stderr);
|
|
77989
|
+
if (result2.exitCode !== 0) {
|
|
77990
|
+
throw new BenchmarkPhaseError("setup_failed", `setup command failed: ${command.id}`, {
|
|
77991
|
+
exit_code: result2.exitCode
|
|
77992
|
+
});
|
|
77993
|
+
}
|
|
77994
|
+
}
|
|
77995
|
+
const finalOutputs = [];
|
|
77996
|
+
const seenOutputs = new Set;
|
|
77997
|
+
for (const output of outputs) {
|
|
77998
|
+
const key2 = `${output.root}:${output.path}`;
|
|
77999
|
+
if (seenOutputs.has(key2))
|
|
78000
|
+
continue;
|
|
78001
|
+
seenOutputs.add(key2);
|
|
78002
|
+
if (output.root !== "workspace") {
|
|
78003
|
+
finalOutputs.push(output);
|
|
78004
|
+
continue;
|
|
78005
|
+
}
|
|
78006
|
+
const candidate = path87.resolve(spec.workspace_root, safeRelPath(output.path));
|
|
78007
|
+
assertNoSymlinkTraversal(spec.workspace_root, output.path);
|
|
78008
|
+
if (!fs79.existsSync(candidate))
|
|
78009
|
+
continue;
|
|
78010
|
+
const stat = fs79.lstatSync(candidate);
|
|
78011
|
+
if (!stat.isFile() && !stat.isSymbolicLink())
|
|
78012
|
+
continue;
|
|
78013
|
+
finalOutputs.push(await recordFile(spec.workspace_root, candidate, "workspace", stat.isSymbolicLink() ? "symlink" : "file"));
|
|
78014
|
+
}
|
|
78015
|
+
outputs.splice(0, outputs.length, ...finalOutputs);
|
|
78016
|
+
context.outputs.splice(0, context.outputs.length, ...finalOutputs);
|
|
78017
|
+
assertBudget(context);
|
|
78018
|
+
return outputs;
|
|
78019
|
+
}
|
|
78020
|
+
async function readEvidence(stagingRoot, evidence) {
|
|
78021
|
+
const filePath = sourcePath(stagingRoot, evidence.source);
|
|
78022
|
+
const opened = openRegularFileNoFollow(filePath, `evidence ${evidence.source}`, stagingRoot);
|
|
78023
|
+
try {
|
|
78024
|
+
const buffer = readDescriptor(opened.fd);
|
|
78025
|
+
if (opened.stat.size !== evidence.size_bytes) {
|
|
78026
|
+
throw new BenchmarkPhaseError("size_mismatch", `size mismatch for ${evidence.source}`, { expected: evidence.size_bytes, actual: opened.stat.size });
|
|
78027
|
+
}
|
|
78028
|
+
const actual = sha256(buffer);
|
|
78029
|
+
if (actual !== evidence.sha256) {
|
|
78030
|
+
throw new BenchmarkPhaseError("digest_mismatch", `checksum mismatch for ${evidence.source}`, { expected: evidence.sha256, actual });
|
|
78031
|
+
}
|
|
78032
|
+
return {
|
|
78033
|
+
path: filePath,
|
|
78034
|
+
buffer,
|
|
78035
|
+
record: {
|
|
78036
|
+
root: "staging",
|
|
78037
|
+
path: path87.relative(stagingRoot, filePath).replace(/\\/g, "/"),
|
|
78038
|
+
sha256: evidence.sha256,
|
|
78039
|
+
size: opened.stat.size,
|
|
78040
|
+
mode: opened.stat.mode & 511
|
|
78041
|
+
}
|
|
78042
|
+
};
|
|
78043
|
+
} finally {
|
|
78044
|
+
fs79.closeSync(opened.fd);
|
|
78045
|
+
}
|
|
78046
|
+
}
|
|
78047
|
+
async function workspaceManifest(spec, context) {
|
|
78048
|
+
const records = [];
|
|
78049
|
+
let totalBytes = 0;
|
|
78050
|
+
const stack = [path87.resolve(spec.workspace_root)];
|
|
78051
|
+
while (stack.length > 0) {
|
|
78052
|
+
const directory = stack.pop();
|
|
78053
|
+
const entries = fs79.readdirSync(directory, { withFileTypes: true }).sort((a3, b4) => a3.name.localeCompare(b4.name));
|
|
78054
|
+
for (const entry of entries) {
|
|
78055
|
+
assertBudget(context);
|
|
78056
|
+
const full = path87.join(directory, entry.name);
|
|
78057
|
+
const relative = path87.relative(spec.workspace_root, full).replace(/\\/g, "/");
|
|
78058
|
+
if (relative === ".brainbase" || relative.startsWith(".brainbase/"))
|
|
78059
|
+
continue;
|
|
78060
|
+
if (relative === ".git" || relative.startsWith(".git/"))
|
|
78061
|
+
continue;
|
|
78062
|
+
if (entry.isDirectory()) {
|
|
78063
|
+
stack.push(full);
|
|
78064
|
+
continue;
|
|
78065
|
+
}
|
|
78066
|
+
if (!entry.isFile() && !entry.isSymbolicLink())
|
|
78067
|
+
continue;
|
|
78068
|
+
const record3 = await recordFile(spec.workspace_root, full, "workspace", entry.isSymbolicLink() ? "symlink" : "file");
|
|
78069
|
+
records.push(record3);
|
|
78070
|
+
totalBytes += record3.size;
|
|
78071
|
+
if (records.length > spec.workspace_limits.max_file_count) {
|
|
78072
|
+
throw new BenchmarkPhaseError("workspace_limit_exceeded", "workspace file count exceeds budget");
|
|
78073
|
+
}
|
|
78074
|
+
if (totalBytes > spec.workspace_limits.max_total_bytes) {
|
|
78075
|
+
throw new BenchmarkPhaseError("workspace_limit_exceeded", "workspace bytes exceed budget");
|
|
78076
|
+
}
|
|
78077
|
+
}
|
|
78078
|
+
}
|
|
78079
|
+
return records.sort((a3, b4) => a3.path.localeCompare(b4.path));
|
|
78080
|
+
}
|
|
78081
|
+
function trajectoryEvents(value) {
|
|
78082
|
+
if (Array.isArray(value))
|
|
78083
|
+
return value;
|
|
78084
|
+
if (value && typeof value === "object" && Array.isArray(value.events)) {
|
|
78085
|
+
return value.events;
|
|
78086
|
+
}
|
|
78087
|
+
throw new BenchmarkPhaseError("invalid_trajectory", "trajectory evidence must be an array or an object with events");
|
|
78088
|
+
}
|
|
78089
|
+
function eventType(event) {
|
|
78090
|
+
if (!event || typeof event !== "object")
|
|
78091
|
+
return;
|
|
78092
|
+
const record3 = event;
|
|
78093
|
+
for (const key2 of ["event_type", "type", "kind"]) {
|
|
78094
|
+
if (typeof record3[key2] === "string")
|
|
78095
|
+
return record3[key2];
|
|
78096
|
+
}
|
|
78097
|
+
return;
|
|
78098
|
+
}
|
|
78099
|
+
async function evaluateOne(evaluator, spec, finalOutput, trajectory, frozenEvidence, context) {
|
|
78100
|
+
const started = Date.now();
|
|
78101
|
+
const base2 = {
|
|
78102
|
+
id: evaluator.id,
|
|
78103
|
+
type: evaluator.type,
|
|
78104
|
+
required: evaluator.required,
|
|
78105
|
+
primary: evaluator.primary,
|
|
78106
|
+
engine: "brainbase-cli",
|
|
78107
|
+
engine_version: VERSION
|
|
78108
|
+
};
|
|
78109
|
+
if (evaluator.type === "output_assertion") {
|
|
78110
|
+
const assertion = evaluator.assertion;
|
|
78111
|
+
let verdict2 = false;
|
|
78112
|
+
if (assertion.operator === "exact")
|
|
78113
|
+
verdict2 = finalOutput === assertion.expected;
|
|
78114
|
+
if (assertion.operator === "contains")
|
|
78115
|
+
verdict2 = finalOutput.includes(assertion.expected);
|
|
78116
|
+
if (assertion.operator === "regex") {
|
|
78117
|
+
const regexResult = await runCommand({
|
|
78118
|
+
id: `${evaluator.id}.regex`,
|
|
78119
|
+
argv: [
|
|
78120
|
+
process.execPath,
|
|
78121
|
+
"-e",
|
|
78122
|
+
[
|
|
78123
|
+
'const fs=require("node:fs");',
|
|
78124
|
+
"try {",
|
|
78125
|
+
'const pattern=Buffer.from(process.env.BB_REGEX_PATTERN_B64,"base64").toString("utf8");',
|
|
78126
|
+
'const value=fs.readFileSync(process.env.BB_REGEX_INPUT,"utf8");',
|
|
78127
|
+
"process.exit(new RegExp(pattern,process.env.BB_REGEX_FLAGS).test(value)?0:1);",
|
|
78128
|
+
"} catch { process.exit(2); }"
|
|
78129
|
+
].join("")
|
|
78130
|
+
],
|
|
78131
|
+
cwd: ".",
|
|
78132
|
+
secret_env: []
|
|
78133
|
+
}, spec.tests_root, spec, context, {
|
|
78134
|
+
BB_REGEX_INPUT: frozenEvidence.finalOutputPath,
|
|
78135
|
+
BB_REGEX_PATTERN_B64: Buffer.from(assertion.pattern).toString("base64"),
|
|
78136
|
+
BB_REGEX_FLAGS: assertion.flags
|
|
78137
|
+
});
|
|
78138
|
+
if (regexResult.exitCode === 2) {
|
|
78139
|
+
throw new BenchmarkPhaseError("invalid_evaluator", `invalid regex in evaluator: ${evaluator.id}`);
|
|
78140
|
+
}
|
|
78141
|
+
verdict2 = regexResult.exitCode === 0;
|
|
78142
|
+
}
|
|
78143
|
+
return { ...base2, status: verdict2 ? "passed" : "failed", verdict: verdict2, duration_ms: Date.now() - started };
|
|
78144
|
+
}
|
|
78145
|
+
if (evaluator.type === "trajectory_assertion") {
|
|
78146
|
+
const count = trajectory.filter((event) => eventType(event) === evaluator.event_type).length;
|
|
78147
|
+
const verdict2 = count >= evaluator.min_count && (evaluator.max_count === undefined || count <= evaluator.max_count);
|
|
78148
|
+
return {
|
|
78149
|
+
...base2,
|
|
78150
|
+
status: verdict2 ? "passed" : "failed",
|
|
78151
|
+
verdict: verdict2,
|
|
78152
|
+
duration_ms: Date.now() - started,
|
|
78153
|
+
details: { count, min_count: evaluator.min_count, max_count: evaluator.max_count }
|
|
78154
|
+
};
|
|
78155
|
+
}
|
|
78156
|
+
if (evaluator.type === "workspace_assertion") {
|
|
78157
|
+
const relative = workspaceRel(evaluator.path);
|
|
78158
|
+
assertNoSymlinkTraversal(spec.workspace_root, relative);
|
|
78159
|
+
const candidate = path87.resolve(spec.workspace_root, relative);
|
|
78160
|
+
let stat = null;
|
|
78161
|
+
try {
|
|
78162
|
+
stat = fs79.lstatSync(candidate);
|
|
78163
|
+
} catch (error2) {
|
|
78164
|
+
const code = error2.code;
|
|
78165
|
+
if (code !== "ENOENT" && code !== "ENOTDIR")
|
|
78166
|
+
throw error2;
|
|
78167
|
+
}
|
|
78168
|
+
if (stat?.isSymbolicLink()) {
|
|
78169
|
+
throw new BenchmarkPhaseError("unsafe_path", `workspace assertion cannot target a symlink: ${relative}`);
|
|
78170
|
+
}
|
|
78171
|
+
const exists2 = stat !== null;
|
|
78172
|
+
let verdict2 = false;
|
|
78173
|
+
if (evaluator.assertion.operator === "exists")
|
|
78174
|
+
verdict2 = exists2;
|
|
78175
|
+
if (evaluator.assertion.operator === "not_exists")
|
|
78176
|
+
verdict2 = !exists2;
|
|
78177
|
+
if (evaluator.assertion.operator === "sha256") {
|
|
78178
|
+
if (stat?.isFile()) {
|
|
78179
|
+
const opened = openRegularFileNoFollow(candidate, `workspace assertion ${relative}`, spec.workspace_root);
|
|
78180
|
+
try {
|
|
78181
|
+
verdict2 = await sha256OfDescriptor(opened.fd) === evaluator.assertion.expected;
|
|
78182
|
+
} finally {
|
|
78183
|
+
fs79.closeSync(opened.fd);
|
|
78184
|
+
}
|
|
78185
|
+
}
|
|
78186
|
+
}
|
|
78187
|
+
if (evaluator.assertion.operator === "contains") {
|
|
78188
|
+
if (stat?.isFile()) {
|
|
78189
|
+
const opened = openRegularFileNoFollow(candidate, `workspace assertion ${relative}`, spec.workspace_root);
|
|
78190
|
+
try {
|
|
78191
|
+
verdict2 = readDescriptor(opened.fd).toString("utf8").includes(evaluator.assertion.expected);
|
|
78192
|
+
} finally {
|
|
78193
|
+
fs79.closeSync(opened.fd);
|
|
78194
|
+
}
|
|
78195
|
+
}
|
|
78196
|
+
}
|
|
78197
|
+
return { ...base2, status: verdict2 ? "passed" : "failed", verdict: verdict2, duration_ms: Date.now() - started };
|
|
78198
|
+
}
|
|
78199
|
+
const commandRoot = evaluator.root === "workspace" ? spec.workspace_root : spec.tests_root;
|
|
78200
|
+
const command = { id: evaluator.id, ...evaluator.command };
|
|
78201
|
+
const result2 = await runCommand(command, commandRoot, spec, context, {
|
|
78202
|
+
BRAINBASE_BENCHMARK_WORKSPACE: spec.workspace_root,
|
|
78203
|
+
BRAINBASE_BENCHMARK_TESTS: spec.tests_root,
|
|
78204
|
+
BRAINBASE_BENCHMARK_LOGS: spec.logs_root,
|
|
78205
|
+
BRAINBASE_BENCHMARK_FINAL_OUTPUT: frozenEvidence.finalOutputPath,
|
|
78206
|
+
BRAINBASE_BENCHMARK_TRAJECTORY: frozenEvidence.trajectoryPath
|
|
78207
|
+
});
|
|
78208
|
+
const stdout = await writeLog(spec.logs_root, `${evaluator.id}.stdout.log`, result2.stdout, spec);
|
|
78209
|
+
const stderr = await writeLog(spec.logs_root, `${evaluator.id}.stderr.log`, result2.stderr, spec);
|
|
78210
|
+
const verdict = result2.exitCode === 0;
|
|
78211
|
+
return {
|
|
78212
|
+
...base2,
|
|
78213
|
+
status: verdict ? "passed" : "failed",
|
|
78214
|
+
verdict,
|
|
78215
|
+
duration_ms: result2.durationMs,
|
|
78216
|
+
details: { exit_code: result2.exitCode },
|
|
78217
|
+
stdout,
|
|
78218
|
+
stderr
|
|
78219
|
+
};
|
|
78220
|
+
}
|
|
78221
|
+
async function executeEvaluate(spec, context) {
|
|
78222
|
+
const finalOutput = await readEvidence(spec.staging_root, spec.evidence.final_output);
|
|
78223
|
+
const trajectoryEvidence = await readEvidence(spec.staging_root, spec.evidence.trajectory);
|
|
78224
|
+
context.inputs.push(finalOutput.record, trajectoryEvidence.record);
|
|
78225
|
+
let parsedTrajectory;
|
|
78226
|
+
try {
|
|
78227
|
+
parsedTrajectory = JSON.parse(trajectoryEvidence.buffer.toString("utf8"));
|
|
78228
|
+
} catch {
|
|
78229
|
+
throw new BenchmarkPhaseError("invalid_trajectory", "trajectory evidence is not valid JSON");
|
|
78230
|
+
}
|
|
78231
|
+
const trajectory = trajectoryEvents(parsedTrajectory);
|
|
78232
|
+
const plannedReferences = [];
|
|
78233
|
+
for (const reference of spec.references) {
|
|
78234
|
+
assertBudget(context);
|
|
78235
|
+
plannedReferences.push(...await preflightMaterial(reference, spec.staging_root, spec.tests_root, false));
|
|
78236
|
+
const source = await verifyInput(spec.staging_root, reference);
|
|
78237
|
+
context.inputs.push(await recordFile(spec.staging_root, source, "staging"));
|
|
78238
|
+
}
|
|
78239
|
+
validateDestinationGraph(plannedReferences);
|
|
78240
|
+
assertBudget(context);
|
|
78241
|
+
validateRoots(spec);
|
|
78242
|
+
prepareOwnedDirectory(spec.tests_root, "tests", spec);
|
|
78243
|
+
prepareOwnedDirectory(spec.logs_root, "logs", spec);
|
|
78244
|
+
context.logsOwned = true;
|
|
78245
|
+
validateRoots(spec);
|
|
78246
|
+
const outputs = [];
|
|
78247
|
+
const finalOutputPath = path87.join(spec.logs_root, "candidate-evidence", "final-output");
|
|
78248
|
+
const trajectoryPath = path87.join(spec.logs_root, "candidate-evidence", "trajectory.json");
|
|
78249
|
+
writeBufferAtomic(finalOutputPath, finalOutput.buffer);
|
|
78250
|
+
writeBufferAtomic(trajectoryPath, trajectoryEvidence.buffer);
|
|
78251
|
+
const frozenEvidenceRecords = [
|
|
78252
|
+
await recordFile(spec.logs_root, finalOutputPath, "logs"),
|
|
78253
|
+
await recordFile(spec.logs_root, trajectoryPath, "logs")
|
|
78254
|
+
];
|
|
78255
|
+
outputs.push(...frozenEvidenceRecords);
|
|
78256
|
+
context.outputs.push(...frozenEvidenceRecords);
|
|
78257
|
+
assertBudget(context);
|
|
78258
|
+
const manifest = await workspaceManifest(spec, context);
|
|
78259
|
+
const manifestPath2 = path87.join(spec.logs_root, "candidate-workspace-manifest.json");
|
|
78260
|
+
writeJsonAtomic(manifestPath2, {
|
|
78261
|
+
schema_version: SCHEMA_VERSION,
|
|
78262
|
+
attempt_id: spec.attempt_id,
|
|
78263
|
+
phase_id: spec.phase_id,
|
|
78264
|
+
files: manifest
|
|
78265
|
+
});
|
|
78266
|
+
const manifestRecord = await recordFile(spec.logs_root, manifestPath2, "logs");
|
|
78267
|
+
outputs.push(manifestRecord);
|
|
78268
|
+
context.outputs.push(manifestRecord);
|
|
78269
|
+
for (const artifactRelInput of spec.candidate_artifacts) {
|
|
78270
|
+
const artifactRel = workspaceRel(artifactRelInput);
|
|
78271
|
+
assertNoSymlinkTraversal(spec.workspace_root, artifactRel);
|
|
78272
|
+
const source = path87.resolve(spec.workspace_root, artifactRel);
|
|
78273
|
+
const frozenArtifact = manifest.find((entry) => entry.path === artifactRel && entry.kind !== "symlink");
|
|
78274
|
+
if (!frozenArtifact || !fs79.existsSync(source) || !fs79.lstatSync(source).isFile()) {
|
|
78275
|
+
throw new BenchmarkPhaseError("missing_artifact", `candidate artifact is missing: ${artifactRel}`);
|
|
78276
|
+
}
|
|
78277
|
+
const destination = path87.resolve(spec.logs_root, "candidate-artifacts", artifactRel);
|
|
78278
|
+
await atomicCopy(source, destination, undefined, spec.workspace_root);
|
|
78279
|
+
const artifact = await recordFile(spec.logs_root, destination, "logs");
|
|
78280
|
+
if (artifact.sha256 !== frozenArtifact.sha256 || artifact.size !== frozenArtifact.size || artifact.mode !== frozenArtifact.mode) {
|
|
78281
|
+
throw new BenchmarkPhaseError("evidence_tampered", `candidate artifact changed after the workspace freeze: ${artifactRel}`);
|
|
78282
|
+
}
|
|
78283
|
+
outputs.push(artifact);
|
|
78284
|
+
context.outputs.push(artifact);
|
|
78285
|
+
}
|
|
78286
|
+
if (spec.capture_workspace_archive) {
|
|
78287
|
+
const regularFiles = manifest.filter((entry) => entry.kind !== "symlink").map((entry) => entry.path);
|
|
78288
|
+
const archive = path87.join(spec.logs_root, "candidate-workspace.tar.gz");
|
|
78289
|
+
const temporary = `${archive}.${process.pid}.${crypto6.randomBytes(6).toString("hex")}.tmp`;
|
|
78290
|
+
try {
|
|
78291
|
+
await pack({ rootDir: spec.workspace_root, outFile: temporary, files: regularFiles });
|
|
78292
|
+
fs79.renameSync(temporary, archive);
|
|
78293
|
+
} finally {
|
|
78294
|
+
fs79.rmSync(temporary, { force: true });
|
|
78295
|
+
}
|
|
78296
|
+
const archiveRecord = await recordFile(spec.logs_root, archive, "logs");
|
|
78297
|
+
outputs.push(archiveRecord);
|
|
78298
|
+
context.outputs.push(archiveRecord);
|
|
78299
|
+
}
|
|
78300
|
+
await verifyRecordsUnchanged(manifest, spec);
|
|
78301
|
+
for (const reference of spec.references) {
|
|
78302
|
+
const referenceOutputs = await copyMaterial(reference, spec.staging_root, spec.tests_root, "tests", false);
|
|
78303
|
+
outputs.push(...referenceOutputs);
|
|
78304
|
+
context.outputs.push(...referenceOutputs);
|
|
78305
|
+
}
|
|
78306
|
+
const frozenOutputCount = context.outputs.length;
|
|
78307
|
+
const evaluators = [];
|
|
78308
|
+
const evaluatorOrder = new Map(spec.evaluators.map((evaluator, index) => [evaluator.id, index]));
|
|
78309
|
+
const executionOrder = [...spec.evaluators].sort((left, right) => Number(left.type === "sandbox_command") - Number(right.type === "sandbox_command"));
|
|
78310
|
+
for (const evaluator of executionOrder) {
|
|
78311
|
+
assertBudget(context);
|
|
78312
|
+
const started = Date.now();
|
|
78313
|
+
try {
|
|
78314
|
+
const evaluated = await evaluateOne(evaluator, spec, finalOutput.buffer.toString("utf8"), trajectory, { finalOutputPath, trajectoryPath }, context);
|
|
78315
|
+
assertBudget(context);
|
|
78316
|
+
evaluators.push(evaluated);
|
|
78317
|
+
context.evaluators.push(evaluated);
|
|
78318
|
+
if (evaluated.stdout) {
|
|
78319
|
+
outputs.push(evaluated.stdout);
|
|
78320
|
+
context.outputs.push(evaluated.stdout);
|
|
78321
|
+
}
|
|
78322
|
+
if (evaluated.stderr) {
|
|
78323
|
+
outputs.push(evaluated.stderr);
|
|
78324
|
+
context.outputs.push(evaluated.stderr);
|
|
78325
|
+
}
|
|
78326
|
+
} catch (error2) {
|
|
78327
|
+
const normalized = stableError(error2);
|
|
78328
|
+
const errored = {
|
|
78329
|
+
id: evaluator.id,
|
|
78330
|
+
type: evaluator.type,
|
|
78331
|
+
required: evaluator.required,
|
|
78332
|
+
primary: evaluator.primary,
|
|
78333
|
+
status: "errored",
|
|
78334
|
+
verdict: null,
|
|
78335
|
+
engine: "brainbase-cli",
|
|
78336
|
+
engine_version: VERSION,
|
|
78337
|
+
duration_ms: Date.now() - started,
|
|
78338
|
+
details: {
|
|
78339
|
+
error_code: normalized?.code ?? "phase_failed",
|
|
78340
|
+
error_message: normalized?.message ?? "evaluator execution failed"
|
|
78341
|
+
}
|
|
78342
|
+
};
|
|
78343
|
+
evaluators.push(errored);
|
|
78344
|
+
context.evaluators.push(errored);
|
|
78345
|
+
if (evaluator.required)
|
|
78346
|
+
throw error2;
|
|
78347
|
+
assertBudget(context);
|
|
78348
|
+
}
|
|
78349
|
+
}
|
|
78350
|
+
await verifyRecordsUnchanged([...context.inputs, ...context.outputs.slice(0, frozenOutputCount)], spec);
|
|
78351
|
+
if (!verifyOwnedDirectory(spec.tests_root, "tests", spec) || !verifyOwnedDirectory(spec.logs_root, "logs", spec)) {
|
|
78352
|
+
throw new BenchmarkPhaseError("evidence_tampered", "benchmark evaluator changed an owned phase directory marker");
|
|
78353
|
+
}
|
|
78354
|
+
assertBudget(context);
|
|
78355
|
+
evaluators.sort((left, right) => evaluatorOrder.get(left.id) - evaluatorOrder.get(right.id));
|
|
78356
|
+
return { outputs, evaluators };
|
|
78357
|
+
}
|
|
78358
|
+
function stableError(error2) {
|
|
78359
|
+
if (error2 instanceof BenchmarkPhaseError) {
|
|
78360
|
+
return { code: error2.code, message: error2.message, details: error2.details };
|
|
78361
|
+
}
|
|
78362
|
+
if (error2 instanceof TarballError) {
|
|
78363
|
+
return { code: "invalid_archive", message: error2.message };
|
|
78364
|
+
}
|
|
78365
|
+
if (error2 instanceof exports_external.ZodError) {
|
|
78366
|
+
return {
|
|
78367
|
+
code: "invalid_spec",
|
|
78368
|
+
message: "benchmark phase spec is invalid",
|
|
78369
|
+
details: error2.issues.map((issue2) => ({
|
|
78370
|
+
path: issue2.path.join("."),
|
|
78371
|
+
code: issue2.code,
|
|
78372
|
+
message: issue2.message
|
|
78373
|
+
}))
|
|
78374
|
+
};
|
|
78375
|
+
}
|
|
78376
|
+
return {
|
|
78377
|
+
code: "phase_failed",
|
|
78378
|
+
message: error2 instanceof Error ? error2.message : "unknown benchmark phase failure"
|
|
78379
|
+
};
|
|
78380
|
+
}
|
|
78381
|
+
function rawIdentity(value) {
|
|
78382
|
+
if (!value || typeof value !== "object") {
|
|
78383
|
+
return { phase: "unknown", attemptId: null, phaseId: null };
|
|
78384
|
+
}
|
|
78385
|
+
const record3 = value;
|
|
78386
|
+
return {
|
|
78387
|
+
phase: record3.phase === "hydrate" || record3.phase === "evaluate" ? record3.phase : "unknown",
|
|
78388
|
+
attemptId: typeof record3.attempt_id === "string" ? record3.attempt_id : null,
|
|
78389
|
+
phaseId: typeof record3.phase_id === "string" ? record3.phase_id : null
|
|
78390
|
+
};
|
|
78391
|
+
}
|
|
78392
|
+
function readSpecBytes(specPathInput) {
|
|
78393
|
+
const specPath = path87.resolve(specPathInput);
|
|
78394
|
+
const noFollow = typeof fs79.constants.O_NOFOLLOW === "number" ? fs79.constants.O_NOFOLLOW : 0;
|
|
78395
|
+
let fd;
|
|
78396
|
+
try {
|
|
78397
|
+
fd = fs79.openSync(specPath, fs79.constants.O_RDONLY | noFollow);
|
|
78398
|
+
} catch {
|
|
78399
|
+
throw new BenchmarkPhaseError("spec_read_failed", "spec file could not be read");
|
|
78400
|
+
}
|
|
78401
|
+
try {
|
|
78402
|
+
const stat = fs79.fstatSync(fd);
|
|
78403
|
+
if (!stat.isFile() || stat.size > MAX_SPEC_BYTES) {
|
|
78404
|
+
throw new BenchmarkPhaseError("invalid_spec_file", "spec must be a regular JSON file no larger than 20 MiB");
|
|
78405
|
+
}
|
|
78406
|
+
return readDescriptor(fd);
|
|
78407
|
+
} finally {
|
|
78408
|
+
fs79.closeSync(fd);
|
|
78409
|
+
}
|
|
78410
|
+
}
|
|
78411
|
+
function validateBenchmarkInvocationBytes(bytes, resultPathInput, expectedPhase) {
|
|
78412
|
+
const digest = sha256(bytes);
|
|
78413
|
+
let raw;
|
|
78414
|
+
try {
|
|
78415
|
+
raw = JSON.parse(bytes.toString("utf8"));
|
|
78416
|
+
} catch {
|
|
78417
|
+
throw new BenchmarkPhaseError("invalid_spec_json", "spec is not valid JSON");
|
|
78418
|
+
}
|
|
78419
|
+
const spec = BenchmarkSpecSchema.parse(raw);
|
|
78420
|
+
if (spec.phase !== expectedPhase) {
|
|
78421
|
+
throw new BenchmarkPhaseError("phase_mismatch", `the ${expectedPhase} command cannot execute a ${spec.phase} spec`);
|
|
78422
|
+
}
|
|
78423
|
+
validateRoots(spec);
|
|
78424
|
+
const resultPath = path87.resolve(resultPathInput);
|
|
78425
|
+
const expectedResultPath = path87.join(path87.resolve(spec.logs_root), "result.json");
|
|
78426
|
+
if (resultPath !== expectedResultPath) {
|
|
78427
|
+
throw new BenchmarkPhaseError("invalid_result_path", `result path must be ${expectedResultPath}`);
|
|
78428
|
+
}
|
|
78429
|
+
return { spec, digest, resultPath };
|
|
78430
|
+
}
|
|
78431
|
+
function prepareBenchmarkPhaseInvocation(specPathInput, resultPathInput, expectedPhase, startedAt, started) {
|
|
78432
|
+
let raw = undefined;
|
|
78433
|
+
let digest = null;
|
|
78434
|
+
let identity2 = rawIdentity(raw);
|
|
78435
|
+
try {
|
|
78436
|
+
const bytes = readSpecBytes(specPathInput);
|
|
78437
|
+
digest = sha256(bytes);
|
|
78438
|
+
try {
|
|
78439
|
+
raw = JSON.parse(bytes.toString("utf8"));
|
|
78440
|
+
} catch {
|
|
78441
|
+
throw new BenchmarkPhaseError("invalid_spec_json", "spec is not valid JSON");
|
|
78442
|
+
}
|
|
78443
|
+
identity2 = rawIdentity(raw);
|
|
78444
|
+
const { spec, resultPath } = validateBenchmarkInvocationBytes(bytes, resultPathInput, expectedPhase);
|
|
78445
|
+
const invocation = {
|
|
78446
|
+
phase: spec.phase,
|
|
78447
|
+
attempt_id: spec.attempt_id,
|
|
78448
|
+
phase_id: spec.phase_id,
|
|
78449
|
+
spec_digest: digest,
|
|
78450
|
+
timeout_ms: spec.budget.timeout_ms
|
|
78451
|
+
};
|
|
78452
|
+
if (fs79.existsSync(resultPath)) {
|
|
78453
|
+
try {
|
|
78454
|
+
const cached2 = JSON.parse(fs79.readFileSync(resultPath, "utf8"));
|
|
78455
|
+
if (cached2.schema_version === SCHEMA_VERSION && cached2.phase === spec.phase && cached2.attempt_id === spec.attempt_id && cached2.phase_id === spec.phase_id && cached2.spec_digest === digest && cached2.status === "succeeded") {
|
|
78456
|
+
return {
|
|
78457
|
+
ok: true,
|
|
78458
|
+
invocation,
|
|
78459
|
+
spec_bytes: bytes,
|
|
78460
|
+
cached_result: cached2
|
|
78461
|
+
};
|
|
78462
|
+
}
|
|
78463
|
+
} catch {}
|
|
78464
|
+
}
|
|
78465
|
+
return { ok: true, invocation, spec_bytes: bytes };
|
|
78466
|
+
} catch (error2) {
|
|
78467
|
+
return {
|
|
78468
|
+
ok: false,
|
|
78469
|
+
result: {
|
|
78470
|
+
schema_version: SCHEMA_VERSION,
|
|
78471
|
+
cli_version: VERSION,
|
|
78472
|
+
phase: identity2.phase === "unknown" ? expectedPhase : identity2.phase,
|
|
78473
|
+
attempt_id: identity2.attemptId,
|
|
78474
|
+
phase_id: identity2.phaseId,
|
|
78475
|
+
spec_digest: digest,
|
|
78476
|
+
status: "failed",
|
|
78477
|
+
started_at: startedAt,
|
|
78478
|
+
completed_at: nowIso(),
|
|
78479
|
+
duration_ms: Date.now() - started,
|
|
78480
|
+
steps: [],
|
|
78481
|
+
inputs: [],
|
|
78482
|
+
outputs: [],
|
|
78483
|
+
error: stableError(error2)
|
|
78484
|
+
}
|
|
78485
|
+
};
|
|
78486
|
+
}
|
|
78487
|
+
}
|
|
78488
|
+
function writeBenchmarkPhaseTimeoutResult(specBytes, resultPathInput, expectedPhase, startedAt, started) {
|
|
78489
|
+
const { spec, digest, resultPath } = validateBenchmarkInvocationBytes(specBytes, resultPathInput, expectedPhase);
|
|
78490
|
+
const result2 = {
|
|
78491
|
+
schema_version: SCHEMA_VERSION,
|
|
78492
|
+
cli_version: VERSION,
|
|
78493
|
+
phase: spec.phase,
|
|
78494
|
+
attempt_id: spec.attempt_id,
|
|
78495
|
+
phase_id: spec.phase_id,
|
|
78496
|
+
spec_digest: digest,
|
|
78497
|
+
status: "failed",
|
|
78498
|
+
started_at: startedAt,
|
|
78499
|
+
completed_at: nowIso(),
|
|
78500
|
+
duration_ms: Date.now() - started,
|
|
78501
|
+
steps: [],
|
|
78502
|
+
inputs: [],
|
|
78503
|
+
outputs: [],
|
|
78504
|
+
error: {
|
|
78505
|
+
code: "phase_timeout",
|
|
78506
|
+
message: "phase budget expired"
|
|
78507
|
+
}
|
|
78508
|
+
};
|
|
78509
|
+
try {
|
|
78510
|
+
prepareOwnedDirectory(spec.logs_root, "logs", spec);
|
|
78511
|
+
writeJsonAtomic(resultPath, result2);
|
|
78512
|
+
} catch (error2) {
|
|
78513
|
+
result2.error.details = {
|
|
78514
|
+
result_write_error: error2 instanceof Error ? error2.message : "failed to write timeout result"
|
|
78515
|
+
};
|
|
78516
|
+
}
|
|
78517
|
+
return result2;
|
|
78518
|
+
}
|
|
78519
|
+
async function runBenchmarkPhase(specPathInput, resultPathInput, expectedPhase, immutableSpecBytes) {
|
|
78520
|
+
const startedAt = nowIso();
|
|
78521
|
+
const started = Date.now();
|
|
78522
|
+
const resultPath = path87.resolve(resultPathInput);
|
|
78523
|
+
let raw = undefined;
|
|
78524
|
+
let digest = null;
|
|
78525
|
+
let identity2 = rawIdentity(raw);
|
|
78526
|
+
const steps = [];
|
|
78527
|
+
let inputs = [];
|
|
78528
|
+
let outputs = [];
|
|
78529
|
+
let evaluators;
|
|
78530
|
+
let status = "failed";
|
|
78531
|
+
let error2;
|
|
78532
|
+
let context;
|
|
78533
|
+
let resultPathValidated = false;
|
|
78534
|
+
try {
|
|
78535
|
+
let bytes;
|
|
78536
|
+
if (immutableSpecBytes) {
|
|
78537
|
+
bytes = immutableSpecBytes;
|
|
78538
|
+
if (bytes.length > MAX_SPEC_BYTES) {
|
|
78539
|
+
throw new BenchmarkPhaseError("invalid_spec_file", "spec must be a regular JSON file no larger than 20 MiB");
|
|
78540
|
+
}
|
|
78541
|
+
} else {
|
|
78542
|
+
bytes = readSpecBytes(specPathInput);
|
|
78543
|
+
}
|
|
78544
|
+
digest = sha256(bytes);
|
|
78545
|
+
try {
|
|
78546
|
+
raw = JSON.parse(bytes.toString("utf8"));
|
|
78547
|
+
} catch {
|
|
78548
|
+
throw new BenchmarkPhaseError("invalid_spec_json", "spec is not valid JSON");
|
|
78549
|
+
}
|
|
78550
|
+
identity2 = rawIdentity(raw);
|
|
78551
|
+
const spec = BenchmarkSpecSchema.parse(raw);
|
|
78552
|
+
if (expectedPhase && spec.phase !== expectedPhase) {
|
|
78553
|
+
throw new BenchmarkPhaseError("phase_mismatch", `the ${expectedPhase} command cannot execute a ${spec.phase} spec`);
|
|
78554
|
+
}
|
|
78555
|
+
validateRoots(spec);
|
|
78556
|
+
const expectedResultPath = path87.join(path87.resolve(spec.logs_root), "result.json");
|
|
78557
|
+
if (resultPath !== expectedResultPath) {
|
|
78558
|
+
throw new BenchmarkPhaseError("invalid_result_path", `result path must be ${expectedResultPath}`);
|
|
78559
|
+
}
|
|
78560
|
+
resultPathValidated = true;
|
|
78561
|
+
if (fs79.existsSync(resultPath)) {
|
|
78562
|
+
try {
|
|
78563
|
+
const cached2 = JSON.parse(fs79.readFileSync(resultPath, "utf8"));
|
|
78564
|
+
if (cached2.schema_version === SCHEMA_VERSION && cached2.phase === spec.phase && cached2.attempt_id === spec.attempt_id && cached2.phase_id === spec.phase_id && cached2.spec_digest === digest && cached2.status === "succeeded") {
|
|
78565
|
+
return { exitCode: 0, result: cached2 };
|
|
78566
|
+
}
|
|
78567
|
+
} catch {}
|
|
78568
|
+
}
|
|
78569
|
+
context = {
|
|
78570
|
+
deadline: started + spec.budget.timeout_ms,
|
|
78571
|
+
remainingOutputBytes: spec.budget.max_output_bytes,
|
|
78572
|
+
steps,
|
|
78573
|
+
evaluators: [],
|
|
78574
|
+
inputs: [],
|
|
78575
|
+
outputs: [],
|
|
78576
|
+
logsOwned: false
|
|
78577
|
+
};
|
|
78578
|
+
if (spec.phase === "hydrate") {
|
|
78579
|
+
outputs = await executeHydrate(spec, context);
|
|
78580
|
+
} else {
|
|
78581
|
+
const evaluated = await executeEvaluate(spec, context);
|
|
78582
|
+
outputs = evaluated.outputs;
|
|
78583
|
+
evaluators = evaluated.evaluators;
|
|
78584
|
+
}
|
|
78585
|
+
inputs = context.inputs;
|
|
78586
|
+
status = "succeeded";
|
|
78587
|
+
} catch (caught) {
|
|
78588
|
+
error2 = stableError(caught);
|
|
78589
|
+
if (context) {
|
|
78590
|
+
inputs = context.inputs;
|
|
78591
|
+
outputs = context.outputs;
|
|
78592
|
+
if (context.evaluators.length)
|
|
78593
|
+
evaluators = context.evaluators;
|
|
78594
|
+
}
|
|
78595
|
+
}
|
|
78596
|
+
const result2 = {
|
|
78597
|
+
schema_version: SCHEMA_VERSION,
|
|
78598
|
+
cli_version: VERSION,
|
|
78599
|
+
phase: identity2.phase === "unknown" && expectedPhase ? expectedPhase : identity2.phase,
|
|
78600
|
+
attempt_id: identity2.attemptId,
|
|
78601
|
+
phase_id: identity2.phaseId,
|
|
78602
|
+
spec_digest: digest,
|
|
78603
|
+
status,
|
|
78604
|
+
started_at: startedAt,
|
|
78605
|
+
completed_at: nowIso(),
|
|
78606
|
+
duration_ms: Date.now() - started,
|
|
78607
|
+
steps,
|
|
78608
|
+
inputs,
|
|
78609
|
+
outputs,
|
|
78610
|
+
...evaluators ? { evaluators } : {},
|
|
78611
|
+
...error2 ? { error: error2 } : {}
|
|
78612
|
+
};
|
|
78613
|
+
if (!resultPathValidated || !context?.logsOwned) {
|
|
78614
|
+
return { exitCode: status === "succeeded" ? 0 : 1, result: result2 };
|
|
78615
|
+
}
|
|
78616
|
+
try {
|
|
78617
|
+
writeJsonAtomic(resultPath, result2);
|
|
78618
|
+
} catch (writeError) {
|
|
78619
|
+
result2.status = "failed";
|
|
78620
|
+
result2.error = {
|
|
78621
|
+
code: "result_write_failed",
|
|
78622
|
+
message: writeError instanceof Error ? writeError.message : "failed to write result"
|
|
78623
|
+
};
|
|
78624
|
+
return { exitCode: 1, result: result2 };
|
|
78625
|
+
}
|
|
78626
|
+
return { exitCode: status === "succeeded" ? 0 : 1, result: result2 };
|
|
78627
|
+
}
|
|
78628
|
+
|
|
78629
|
+
// src/cli/benchmark.ts
|
|
78630
|
+
function parsePhaseArgs(args) {
|
|
78631
|
+
let spec;
|
|
78632
|
+
let result2;
|
|
78633
|
+
let json = false;
|
|
78634
|
+
const seen = new Set;
|
|
78635
|
+
for (let index = 0;index < args.length; index += 1) {
|
|
78636
|
+
const arg = args[index];
|
|
78637
|
+
if (arg === "--json") {
|
|
78638
|
+
if (seen.has(arg))
|
|
78639
|
+
throw new Error(`Duplicate benchmark option: ${arg}`);
|
|
78640
|
+
seen.add(arg);
|
|
78641
|
+
json = true;
|
|
78642
|
+
continue;
|
|
78643
|
+
}
|
|
78644
|
+
if (arg === "--spec" || arg === "--result") {
|
|
78645
|
+
if (seen.has(arg))
|
|
78646
|
+
throw new Error(`Duplicate benchmark option: ${arg}`);
|
|
78647
|
+
seen.add(arg);
|
|
78648
|
+
const value = args[index + 1];
|
|
78649
|
+
if (!value || value.startsWith("--")) {
|
|
78650
|
+
throw new Error(`${arg} requires a path`);
|
|
78651
|
+
}
|
|
78652
|
+
if (arg === "--spec")
|
|
78653
|
+
spec = value;
|
|
78654
|
+
else
|
|
78655
|
+
result2 = value;
|
|
78656
|
+
index += 1;
|
|
78657
|
+
continue;
|
|
78658
|
+
}
|
|
78659
|
+
throw new Error(`Unknown benchmark argument: ${arg}`);
|
|
78660
|
+
}
|
|
78661
|
+
if (!spec)
|
|
78662
|
+
throw new Error("--spec is required");
|
|
78663
|
+
if (!result2)
|
|
78664
|
+
throw new Error("--result is required");
|
|
78665
|
+
if (!json)
|
|
78666
|
+
throw new Error("--json is required for benchmark phase commands");
|
|
78667
|
+
return { spec, result: result2, json: true };
|
|
78668
|
+
}
|
|
78669
|
+
function argumentFailure(phase, message) {
|
|
78670
|
+
const timestamp = new Date().toISOString();
|
|
78671
|
+
return {
|
|
78672
|
+
schema_version: "1",
|
|
78673
|
+
cli_version: VERSION,
|
|
78674
|
+
phase,
|
|
78675
|
+
attempt_id: null,
|
|
78676
|
+
phase_id: null,
|
|
78677
|
+
spec_digest: null,
|
|
78678
|
+
status: "failed",
|
|
78679
|
+
started_at: timestamp,
|
|
78680
|
+
completed_at: timestamp,
|
|
78681
|
+
duration_ms: 0,
|
|
78682
|
+
steps: [],
|
|
78683
|
+
inputs: [],
|
|
78684
|
+
outputs: [],
|
|
78685
|
+
error: {
|
|
78686
|
+
code: "invalid_arguments",
|
|
78687
|
+
message
|
|
78688
|
+
}
|
|
78689
|
+
};
|
|
78690
|
+
}
|
|
78691
|
+
function phaseFailure(invocation, startedAt, started, code, message) {
|
|
78692
|
+
return {
|
|
78693
|
+
schema_version: "1",
|
|
78694
|
+
cli_version: VERSION,
|
|
78695
|
+
phase: invocation.phase,
|
|
78696
|
+
attempt_id: invocation.attempt_id,
|
|
78697
|
+
phase_id: invocation.phase_id,
|
|
78698
|
+
spec_digest: invocation.spec_digest,
|
|
78699
|
+
status: "failed",
|
|
78700
|
+
started_at: startedAt,
|
|
78701
|
+
completed_at: new Date().toISOString(),
|
|
78702
|
+
duration_ms: Date.now() - started,
|
|
78703
|
+
steps: [],
|
|
78704
|
+
inputs: [],
|
|
78705
|
+
outputs: [],
|
|
78706
|
+
error: { code, message }
|
|
78707
|
+
};
|
|
78708
|
+
}
|
|
78709
|
+
function descendantPids2(parentPid) {
|
|
78710
|
+
if (process.platform === "win32")
|
|
78711
|
+
return [];
|
|
78712
|
+
try {
|
|
78713
|
+
const output = execFileSync3("ps", ["-eo", "pid=,ppid="], {
|
|
78714
|
+
encoding: "utf8",
|
|
78715
|
+
stdio: ["ignore", "pipe", "ignore"]
|
|
78716
|
+
});
|
|
78717
|
+
const children = new Map;
|
|
78718
|
+
for (const line of output.split(`
|
|
78719
|
+
`)) {
|
|
78720
|
+
const [pidRaw, parentRaw] = line.trim().split(/\s+/);
|
|
78721
|
+
const pid = Number(pidRaw);
|
|
78722
|
+
const parent = Number(parentRaw);
|
|
78723
|
+
if (!Number.isInteger(pid) || !Number.isInteger(parent))
|
|
78724
|
+
continue;
|
|
78725
|
+
const current = children.get(parent) ?? [];
|
|
78726
|
+
current.push(pid);
|
|
78727
|
+
children.set(parent, current);
|
|
78728
|
+
}
|
|
78729
|
+
const descendants = [];
|
|
78730
|
+
const stack = [...children.get(parentPid) ?? []];
|
|
78731
|
+
while (stack.length > 0) {
|
|
78732
|
+
const pid = stack.pop();
|
|
78733
|
+
descendants.push(pid);
|
|
78734
|
+
stack.push(...children.get(pid) ?? []);
|
|
78735
|
+
}
|
|
78736
|
+
return descendants;
|
|
78737
|
+
} catch {
|
|
78738
|
+
return [];
|
|
78739
|
+
}
|
|
78740
|
+
}
|
|
78741
|
+
function terminatePhase(child) {
|
|
78742
|
+
if (child.pid === undefined)
|
|
78743
|
+
return;
|
|
78744
|
+
for (const pid of descendantPids2(child.pid).reverse()) {
|
|
78745
|
+
try {
|
|
78746
|
+
process.kill(pid, "SIGKILL");
|
|
78747
|
+
} catch {}
|
|
78748
|
+
}
|
|
78749
|
+
try {
|
|
78750
|
+
if (process.platform !== "win32")
|
|
78751
|
+
process.kill(-child.pid, "SIGKILL");
|
|
78752
|
+
else
|
|
78753
|
+
child.kill("SIGKILL");
|
|
78754
|
+
} catch {
|
|
78755
|
+
child.kill("SIGKILL");
|
|
78756
|
+
}
|
|
78757
|
+
}
|
|
78758
|
+
function createAnonymousSpecFd(bytes) {
|
|
78759
|
+
const temporary = path88.join(os17.tmpdir(), `brainbase-benchmark-spec-${process.pid}-${crypto7.randomBytes(12).toString("hex")}`);
|
|
78760
|
+
fs80.writeFileSync(temporary, bytes, { flag: "wx", mode: 384 });
|
|
78761
|
+
try {
|
|
78762
|
+
const fd = fs80.openSync(temporary, "r");
|
|
78763
|
+
fs80.unlinkSync(temporary);
|
|
78764
|
+
return fd;
|
|
78765
|
+
} catch (error2) {
|
|
78766
|
+
fs80.rmSync(temporary, { force: true });
|
|
78767
|
+
throw error2;
|
|
78768
|
+
}
|
|
78769
|
+
}
|
|
78770
|
+
async function runSupervisedPhase(phase, parsed, write) {
|
|
78771
|
+
const startedAt = new Date().toISOString();
|
|
78772
|
+
const started = Date.now();
|
|
78773
|
+
const prepared = prepareBenchmarkPhaseInvocation(parsed.spec, parsed.result, phase, startedAt, started);
|
|
78774
|
+
if (!prepared.ok) {
|
|
78775
|
+
write(`${JSON.stringify(prepared.result)}
|
|
78776
|
+
`);
|
|
78777
|
+
return 1;
|
|
78778
|
+
}
|
|
78779
|
+
if (prepared.cached_result) {
|
|
78780
|
+
write(`${JSON.stringify(prepared.cached_result)}
|
|
78781
|
+
`);
|
|
78782
|
+
return 0;
|
|
78783
|
+
}
|
|
78784
|
+
const { invocation, spec_bytes: specBytes } = prepared;
|
|
78785
|
+
const deadline = started + invocation.timeout_ms;
|
|
78786
|
+
const entrypoint = process.argv[1];
|
|
78787
|
+
if (!entrypoint) {
|
|
78788
|
+
const failure = phaseFailure(invocation, startedAt, started, "phase_supervisor_failed", "benchmark phase supervisor could not resolve the CLI entrypoint");
|
|
78789
|
+
write(`${JSON.stringify(failure)}
|
|
78790
|
+
`);
|
|
78791
|
+
return 1;
|
|
78792
|
+
}
|
|
78793
|
+
const remainingMs = deadline - Date.now();
|
|
78794
|
+
if (remainingMs <= 0) {
|
|
78795
|
+
let failure = phaseFailure(invocation, startedAt, started, "phase_timeout", "phase budget expired");
|
|
78796
|
+
try {
|
|
78797
|
+
failure = writeBenchmarkPhaseTimeoutResult(specBytes, parsed.result, phase, startedAt, started);
|
|
78798
|
+
} catch {}
|
|
78799
|
+
write(`${JSON.stringify(failure)}
|
|
78800
|
+
`);
|
|
78801
|
+
return 1;
|
|
78802
|
+
}
|
|
78803
|
+
const childToken = crypto7.randomBytes(32).toString("hex");
|
|
78804
|
+
let specFd;
|
|
78805
|
+
try {
|
|
78806
|
+
specFd = createAnonymousSpecFd(specBytes);
|
|
78807
|
+
} catch {
|
|
78808
|
+
const failure = phaseFailure(invocation, startedAt, started, "phase_supervisor_failed", "benchmark phase supervisor could not snapshot the validated spec");
|
|
78809
|
+
write(`${JSON.stringify(failure)}
|
|
78810
|
+
`);
|
|
78811
|
+
return 1;
|
|
78812
|
+
}
|
|
78813
|
+
let child;
|
|
78814
|
+
try {
|
|
78815
|
+
child = spawn5(process.execPath, [
|
|
78816
|
+
entrypoint,
|
|
78817
|
+
"benchmark",
|
|
78818
|
+
"__phase-child",
|
|
78819
|
+
phase,
|
|
78820
|
+
"--result",
|
|
78821
|
+
parsed.result,
|
|
78822
|
+
"--spec-fd",
|
|
78823
|
+
"3",
|
|
78824
|
+
"--token",
|
|
78825
|
+
childToken
|
|
78826
|
+
], {
|
|
78827
|
+
env: {
|
|
78828
|
+
...process.env,
|
|
78829
|
+
BRAINBASE_BENCHMARK_PHASE_CHILD_TOKEN: childToken
|
|
78830
|
+
},
|
|
78831
|
+
stdio: ["ignore", "pipe", "pipe", specFd],
|
|
78832
|
+
detached: process.platform !== "win32"
|
|
78833
|
+
});
|
|
78834
|
+
} catch {
|
|
78835
|
+
fs80.closeSync(specFd);
|
|
78836
|
+
const failure = phaseFailure(invocation, startedAt, started, "phase_supervisor_failed", "benchmark phase child process could not be started");
|
|
78837
|
+
write(`${JSON.stringify(failure)}
|
|
78838
|
+
`);
|
|
78839
|
+
return 1;
|
|
78840
|
+
}
|
|
78841
|
+
fs80.closeSync(specFd);
|
|
78842
|
+
return await new Promise((resolve) => {
|
|
78843
|
+
const stdout = [];
|
|
78844
|
+
let settled = false;
|
|
78845
|
+
let timedOut = false;
|
|
78846
|
+
let timer;
|
|
78847
|
+
const expire = () => {
|
|
78848
|
+
if (settled || timedOut)
|
|
78849
|
+
return;
|
|
78850
|
+
timedOut = true;
|
|
78851
|
+
terminatePhase(child);
|
|
78852
|
+
};
|
|
78853
|
+
const remainingAfterSpawn = deadline - Date.now();
|
|
78854
|
+
if (remainingAfterSpawn <= 0)
|
|
78855
|
+
queueMicrotask(expire);
|
|
78856
|
+
else
|
|
78857
|
+
timer = setTimeout(expire, remainingAfterSpawn);
|
|
78858
|
+
child.stdout?.on("data", (chunk2) => {
|
|
78859
|
+
if (!timedOut)
|
|
78860
|
+
stdout.push(chunk2);
|
|
78861
|
+
});
|
|
78862
|
+
child.stderr?.resume();
|
|
78863
|
+
child.on("error", () => {
|
|
78864
|
+
if (settled)
|
|
78865
|
+
return;
|
|
78866
|
+
settled = true;
|
|
78867
|
+
if (timer)
|
|
78868
|
+
clearTimeout(timer);
|
|
78869
|
+
const failure = phaseFailure(invocation, startedAt, started, "phase_supervisor_failed", "benchmark phase child process could not be started");
|
|
78870
|
+
write(`${JSON.stringify(failure)}
|
|
78871
|
+
`);
|
|
78872
|
+
resolve(1);
|
|
78873
|
+
});
|
|
78874
|
+
child.on("close", (code) => {
|
|
78875
|
+
if (settled)
|
|
78876
|
+
return;
|
|
78877
|
+
settled = true;
|
|
78878
|
+
if (timer)
|
|
78879
|
+
clearTimeout(timer);
|
|
78880
|
+
if (Date.now() >= deadline)
|
|
78881
|
+
timedOut = true;
|
|
78882
|
+
if (timedOut) {
|
|
78883
|
+
let failure = phaseFailure(invocation, startedAt, started, "phase_timeout", "phase budget expired");
|
|
78884
|
+
try {
|
|
78885
|
+
failure = writeBenchmarkPhaseTimeoutResult(specBytes, parsed.result, phase, startedAt, started);
|
|
78886
|
+
} catch {}
|
|
78887
|
+
write(`${JSON.stringify(failure)}
|
|
78888
|
+
`);
|
|
78889
|
+
resolve(1);
|
|
78890
|
+
return;
|
|
78891
|
+
}
|
|
78892
|
+
const output = Buffer.concat(stdout).toString("utf8");
|
|
78893
|
+
if (!output) {
|
|
78894
|
+
const failure = phaseFailure(invocation, startedAt, started, "phase_supervisor_failed", "benchmark phase child exited without a result");
|
|
78895
|
+
write(`${JSON.stringify(failure)}
|
|
78896
|
+
`);
|
|
78897
|
+
resolve(1);
|
|
78898
|
+
return;
|
|
78899
|
+
}
|
|
78900
|
+
write(output.endsWith(`
|
|
78901
|
+
`) ? output : `${output}
|
|
78902
|
+
`);
|
|
78903
|
+
resolve(code === 0 ? 0 : 1);
|
|
78904
|
+
});
|
|
78905
|
+
});
|
|
78906
|
+
}
|
|
78907
|
+
async function runBenchmark(sub, args, write = (value) => process.stdout.write(value)) {
|
|
78908
|
+
switch (sub) {
|
|
78909
|
+
case "__phase-child": {
|
|
78910
|
+
const [
|
|
78911
|
+
phase,
|
|
78912
|
+
resultFlag,
|
|
78913
|
+
resultPath,
|
|
78914
|
+
specFdFlag,
|
|
78915
|
+
specFdRaw,
|
|
78916
|
+
tokenFlag,
|
|
78917
|
+
token
|
|
78918
|
+
] = args;
|
|
78919
|
+
const specFd = Number(specFdRaw);
|
|
78920
|
+
if (phase !== "hydrate" && phase !== "evaluate" || resultFlag !== "--result" || !resultPath || specFdFlag !== "--spec-fd" || !Number.isInteger(specFd) || specFd < 3 || tokenFlag !== "--token" || !token || token !== process.env.BRAINBASE_BENCHMARK_PHASE_CHILD_TOKEN) {
|
|
78921
|
+
throw new Error("Invalid internal benchmark phase invocation");
|
|
78922
|
+
}
|
|
78923
|
+
const specBytes = fs80.readFileSync(specFd);
|
|
78924
|
+
const { exitCode, result: result2 } = await runBenchmarkPhase("", resultPath, phase, specBytes);
|
|
78925
|
+
write(`${JSON.stringify(result2)}
|
|
78926
|
+
`);
|
|
78927
|
+
return exitCode;
|
|
78928
|
+
}
|
|
78929
|
+
case "hydrate":
|
|
78930
|
+
case "evaluate": {
|
|
78931
|
+
let parsed;
|
|
78932
|
+
try {
|
|
78933
|
+
parsed = parsePhaseArgs(args);
|
|
78934
|
+
} catch (error2) {
|
|
78935
|
+
const failure = argumentFailure(sub, error2 instanceof Error ? error2.message : "invalid benchmark arguments");
|
|
78936
|
+
write(`${JSON.stringify(failure)}
|
|
78937
|
+
`);
|
|
78938
|
+
return 1;
|
|
78939
|
+
}
|
|
78940
|
+
return await runSupervisedPhase(sub, parsed, write);
|
|
78941
|
+
}
|
|
78942
|
+
case "capabilities": {
|
|
78943
|
+
if (args.length !== 1 || args[0] !== "--json") {
|
|
78944
|
+
throw new Error("Usage: brainbase benchmark capabilities --json");
|
|
78945
|
+
}
|
|
78946
|
+
write(`${JSON.stringify(BENCHMARK_CAPABILITIES)}
|
|
78947
|
+
`);
|
|
78948
|
+
return 0;
|
|
78949
|
+
}
|
|
78950
|
+
case undefined:
|
|
78951
|
+
case "help":
|
|
78952
|
+
case "-h":
|
|
78953
|
+
case "--help":
|
|
78954
|
+
printHelp5();
|
|
78955
|
+
return 0;
|
|
78956
|
+
default:
|
|
78957
|
+
throw new Error(`Unknown benchmark subcommand: ${sub}`);
|
|
78958
|
+
}
|
|
78959
|
+
}
|
|
78960
|
+
function printHelp5() {
|
|
78961
|
+
const out = [];
|
|
78962
|
+
out.push("");
|
|
78963
|
+
out.push(` ${import_picocolors48.default.bold("brainbase benchmark")} ${import_picocolors48.default.dim("<sub> [options]")}`);
|
|
78964
|
+
out.push("");
|
|
78965
|
+
out.push(` ${import_picocolors48.default.cyan("hydrate")} ${import_picocolors48.default.dim("--spec <path> --result <path> --json")}`);
|
|
78966
|
+
out.push(` ${import_picocolors48.default.cyan("evaluate")} ${import_picocolors48.default.dim("--spec <path> --result <path> --json")}`);
|
|
78967
|
+
out.push(` ${import_picocolors48.default.cyan("capabilities")} ${import_picocolors48.default.dim("--json")}`);
|
|
78968
|
+
out.push("");
|
|
78969
|
+
out.push(` ${import_picocolors48.default.dim("These machine-only commands execute versioned benchmark phase specs inside a task sandbox.")}`);
|
|
78970
|
+
out.push("");
|
|
78971
|
+
console.log(out.join(`
|
|
78972
|
+
`));
|
|
78973
|
+
}
|
|
78974
|
+
|
|
77027
78975
|
// src/index.ts
|
|
77028
78976
|
var PROTECTED = new Set([
|
|
77029
78977
|
"template",
|
|
@@ -77044,115 +78992,121 @@ var STORED_PAT_COMMANDS = new Set([
|
|
|
77044
78992
|
function help() {
|
|
77045
78993
|
const out = [];
|
|
77046
78994
|
out.push("");
|
|
77047
|
-
out.push(` ${brandTint("◆")} ${
|
|
77048
|
-
out.push(` ${
|
|
78995
|
+
out.push(` ${brandTint("◆")} ${import_picocolors49.default.bold("brainbase")} ${import_picocolors49.default.dim(`v${VERSION}`)}`);
|
|
78996
|
+
out.push(` ${import_picocolors49.default.dim("connect your local agent to the brainbase platform")}`);
|
|
77049
78997
|
out.push("");
|
|
77050
78998
|
out.push(divider("USAGE"));
|
|
77051
78999
|
out.push("");
|
|
77052
|
-
out.push(` ${
|
|
79000
|
+
out.push(` ${import_picocolors49.default.bold("brainbase")} ${import_picocolors49.default.dim("<command> [options]")}`);
|
|
77053
79001
|
out.push("");
|
|
77054
79002
|
out.push(divider("AUTH"));
|
|
77055
79003
|
out.push("");
|
|
77056
|
-
out.push(` ${
|
|
77057
|
-
out.push(` ${
|
|
77058
|
-
out.push(` ${
|
|
79004
|
+
out.push(` ${import_picocolors49.default.cyan("login")} ${import_picocolors49.default.dim(" open the web app and connect this device")}`);
|
|
79005
|
+
out.push(` ${import_picocolors49.default.cyan("logout")} ${import_picocolors49.default.dim(" clear the local session")}`);
|
|
79006
|
+
out.push(` ${import_picocolors49.default.cyan("whoami")} ${import_picocolors49.default.dim(" show the current user")}`);
|
|
77059
79007
|
out.push("");
|
|
77060
79008
|
out.push(divider("DISCOVERY"));
|
|
77061
79009
|
out.push("");
|
|
77062
|
-
out.push(` ${
|
|
77063
|
-
out.push(` ${
|
|
79010
|
+
out.push(` ${import_picocolors49.default.cyan("team list")} ${import_picocolors49.default.dim("show the teams you can create agents in")}`);
|
|
79011
|
+
out.push(` ${import_picocolors49.default.cyan("agent list")} ${import_picocolors49.default.dim("show a team's agents and their ids")}`);
|
|
77064
79012
|
out.push("");
|
|
77065
79013
|
out.push(divider("LINKED AGENT"));
|
|
77066
79014
|
out.push("");
|
|
77067
|
-
out.push(` ${
|
|
77068
|
-
out.push(` ${
|
|
77069
|
-
out.push(` ${
|
|
77070
|
-
out.push(` ${
|
|
77071
|
-
out.push(` ${
|
|
77072
|
-
out.push(` ${
|
|
77073
|
-
out.push(` ${
|
|
77074
|
-
out.push(` ${
|
|
77075
|
-
out.push(` ${
|
|
77076
|
-
out.push(` ${
|
|
79015
|
+
out.push(` ${import_picocolors49.default.cyan("agent create")} ${import_picocolors49.default.dim("claim an unclaimed brainbase.agent.yaml and create the cloud agent")}`);
|
|
79016
|
+
out.push(` ${import_picocolors49.default.cyan("agent pull")} ${import_picocolors49.default.dim("[<id>]")} ${import_picocolors49.default.dim("bring cloud changes into this folder (--force to override; --run-entrypoint to also execute the agent entrypoint)")}`);
|
|
79017
|
+
out.push(` ${import_picocolors49.default.cyan("agent push")} ${import_picocolors49.default.dim("send local changes to the cloud (--force to overwrite cloud-side conflicts with local)")}`);
|
|
79018
|
+
out.push(` ${import_picocolors49.default.cyan("agent unpack")} ${import_picocolors49.default.dim("install the claimed agent into a harness layout")}`);
|
|
79019
|
+
out.push(` ${import_picocolors49.default.cyan("link")} ${import_picocolors49.default.dim("attach this folder to an existing agent")}`);
|
|
79020
|
+
out.push(` ${import_picocolors49.default.cyan("agent status")} ${import_picocolors49.default.dim("show what would pull and what would push")}`);
|
|
79021
|
+
out.push(` ${import_picocolors49.default.cyan("agent env")} ${import_picocolors49.default.dim("print export lines for `eval $(brainbase agent env)`")}`);
|
|
79022
|
+
out.push(` ${import_picocolors49.default.cyan("run")} ${import_picocolors49.default.dim("<cmd> [args...]")} ${import_picocolors49.default.dim("run <cmd> with secrets.env loaded into env")}`);
|
|
79023
|
+
out.push(` ${import_picocolors49.default.cyan("status")} ${import_picocolors49.default.dim("show what this folder is linked to")}`);
|
|
79024
|
+
out.push(` ${import_picocolors49.default.cyan("unlink")} ${import_picocolors49.default.dim("disconnect this folder")}`);
|
|
77077
79025
|
out.push("");
|
|
77078
79026
|
out.push(divider("TASKS"));
|
|
77079
79027
|
out.push("");
|
|
77080
|
-
out.push(` ${
|
|
79028
|
+
out.push(` ${import_picocolors49.default.cyan("task create")} ${import_picocolors49.default.dim("--message <text>")} ${import_picocolors49.default.dim("create a managed task and start its first run")}`);
|
|
79029
|
+
out.push("");
|
|
79030
|
+
out.push(divider("BENCHMARK RUNTIME"));
|
|
79031
|
+
out.push("");
|
|
79032
|
+
out.push(` ${import_picocolors49.default.cyan("benchmark hydrate")} ${import_picocolors49.default.dim("--spec <path> --result <path> --json")}`);
|
|
79033
|
+
out.push(` ${import_picocolors49.default.cyan("benchmark evaluate")} ${import_picocolors49.default.dim("--spec <path> --result <path> --json")}`);
|
|
79034
|
+
out.push(` ${import_picocolors49.default.cyan("benchmark capabilities")} ${import_picocolors49.default.dim("--json")}`);
|
|
77081
79035
|
out.push("");
|
|
77082
79036
|
out.push(divider("ORCHESTRATIONS"));
|
|
77083
79037
|
out.push("");
|
|
77084
|
-
out.push(` ${
|
|
77085
|
-
out.push(` ${
|
|
77086
|
-
out.push(` ${
|
|
77087
|
-
out.push(` ${
|
|
77088
|
-
out.push(` ${
|
|
79038
|
+
out.push(` ${import_picocolors49.default.cyan("orchestration create")} ${import_picocolors49.default.dim("claim a local orchestration manifest and create it in the cloud")}`);
|
|
79039
|
+
out.push(` ${import_picocolors49.default.cyan("orchestration list")} ${import_picocolors49.default.dim("list orchestrations under a team")}`);
|
|
79040
|
+
out.push(` ${import_picocolors49.default.cyan("orchestration pull")} ${import_picocolors49.default.dim("<id>")} ${import_picocolors49.default.dim("recursively fetch an orchestration + every member agent")}`);
|
|
79041
|
+
out.push(` ${import_picocolors49.default.cyan("orchestration push")} ${import_picocolors49.default.dim("recursively push each member, then update the graph")}`);
|
|
79042
|
+
out.push(` ${import_picocolors49.default.cyan("orchestration status")} ${import_picocolors49.default.dim("show what would push and what would pull")}`);
|
|
77089
79043
|
out.push("");
|
|
77090
79044
|
out.push(divider("TEMPLATES"));
|
|
77091
79045
|
out.push("");
|
|
77092
|
-
out.push(` ${
|
|
77093
|
-
out.push(` ${
|
|
77094
|
-
out.push(` ${
|
|
77095
|
-
out.push(` ${
|
|
77096
|
-
out.push(` ${
|
|
77097
|
-
out.push(` ${
|
|
77098
|
-
out.push(` ${
|
|
79046
|
+
out.push(` ${import_picocolors49.default.cyan("template pack")} ${import_picocolors49.default.dim("bundle the current agent into a template")}`);
|
|
79047
|
+
out.push(` ${import_picocolors49.default.cyan("template publish")} ${import_picocolors49.default.dim("upload a template to the registry")}`);
|
|
79048
|
+
out.push(` ${import_picocolors49.default.cyan("template search")} ${import_picocolors49.default.dim("[query]")} ${import_picocolors49.default.dim("search the registry")}`);
|
|
79049
|
+
out.push(` ${import_picocolors49.default.cyan("template info")} ${import_picocolors49.default.dim("<creator/slug>")} ${import_picocolors49.default.dim("show registry details for a template")}`);
|
|
79050
|
+
out.push(` ${import_picocolors49.default.cyan("template onboard")} ${import_picocolors49.default.dim("<creator/slug>")} ${import_picocolors49.default.dim("install (or refresh) a template")}`);
|
|
79051
|
+
out.push(` ${import_picocolors49.default.cyan("template list")} ${import_picocolors49.default.dim("show installed templates")}`);
|
|
79052
|
+
out.push(` ${import_picocolors49.default.cyan("template remove")} ${import_picocolors49.default.dim("<creator/slug>")} ${import_picocolors49.default.dim("uninstall a template")}`);
|
|
77099
79053
|
out.push("");
|
|
77100
79054
|
out.push(divider("SKILLS"));
|
|
77101
79055
|
out.push("");
|
|
77102
|
-
out.push(` ${
|
|
77103
|
-
out.push(` ${
|
|
77104
|
-
out.push(` ${
|
|
77105
|
-
out.push(` ${
|
|
77106
|
-
out.push(` ${
|
|
77107
|
-
out.push(` ${
|
|
77108
|
-
out.push(` ${
|
|
79056
|
+
out.push(` ${import_picocolors49.default.cyan("skill add")} ${import_picocolors49.default.dim("<source>")} ${import_picocolors49.default.dim("install a skill (github / git / brainbase)")}`);
|
|
79057
|
+
out.push(` ${import_picocolors49.default.cyan("skill list")} ${import_picocolors49.default.dim("show locally installed skills + their source")}`);
|
|
79058
|
+
out.push(` ${import_picocolors49.default.cyan("skill update")} ${import_picocolors49.default.dim("<slug>")} ${import_picocolors49.default.dim("re-fetch a skill from its recorded source")}`);
|
|
79059
|
+
out.push(` ${import_picocolors49.default.cyan("skill remove")} ${import_picocolors49.default.dim("<slug>")} ${import_picocolors49.default.dim("uninstall a skill")}`);
|
|
79060
|
+
out.push(` ${import_picocolors49.default.cyan("skill search")} ${import_picocolors49.default.dim("[query]")} ${import_picocolors49.default.dim("search the brainbase skill registry")}`);
|
|
79061
|
+
out.push(` ${import_picocolors49.default.cyan("skill info")} ${import_picocolors49.default.dim("<creator/slug>")} ${import_picocolors49.default.dim("show registry details for a skill")}`);
|
|
79062
|
+
out.push(` ${import_picocolors49.default.cyan("skill publish")} ${import_picocolors49.default.dim("[dir]")} ${import_picocolors49.default.dim("publish a SKILL.md folder (defaults to .)")}`);
|
|
77109
79063
|
out.push("");
|
|
77110
79064
|
out.push(divider("CLI TOKENS"));
|
|
77111
79065
|
out.push("");
|
|
77112
|
-
out.push(` ${
|
|
77113
|
-
out.push(` ${
|
|
77114
|
-
out.push(` ${
|
|
79066
|
+
out.push(` ${import_picocolors49.default.cyan("token create")} ${import_picocolors49.default.dim("issue a long-lived CLI key for CI / scripts")}`);
|
|
79067
|
+
out.push(` ${import_picocolors49.default.cyan("token list")} ${import_picocolors49.default.dim("show your active tokens")}`);
|
|
79068
|
+
out.push(` ${import_picocolors49.default.cyan("token revoke")} ${import_picocolors49.default.dim("<id>")} ${import_picocolors49.default.dim("revoke a token")}`);
|
|
77115
79069
|
out.push("");
|
|
77116
79070
|
out.push(divider("MCP"));
|
|
77117
79071
|
out.push("");
|
|
77118
|
-
out.push(` ${
|
|
79072
|
+
out.push(` ${import_picocolors49.default.cyan("mcp check")} ${import_picocolors49.default.dim("[--json]")} ${import_picocolors49.default.dim("verify MCP server connectivity through the brainbase proxy (runs at sandbox bootstrap)")}`);
|
|
77119
79073
|
out.push("");
|
|
77120
79074
|
out.push(divider("FLAGS"));
|
|
77121
79075
|
out.push("");
|
|
77122
|
-
out.push(` ${
|
|
77123
|
-
out.push(` ${
|
|
77124
|
-
out.push(` ${
|
|
77125
|
-
out.push(` ${
|
|
77126
|
-
out.push(` ${
|
|
77127
|
-
out.push(` ${
|
|
77128
|
-
out.push(` ${
|
|
77129
|
-
out.push(` ${
|
|
77130
|
-
out.push(` ${
|
|
77131
|
-
out.push(` ${
|
|
77132
|
-
out.push(` ${
|
|
77133
|
-
out.push(` ${
|
|
77134
|
-
out.push(` ${
|
|
77135
|
-
out.push(` ${
|
|
77136
|
-
out.push(` ${
|
|
79076
|
+
out.push(` ${import_picocolors49.default.dim("--harness <id>")} force harness for onboard / sync (e.g. claude-code)`);
|
|
79077
|
+
out.push(` ${import_picocolors49.default.dim("--scope <s>")} force scope: global | project`);
|
|
79078
|
+
out.push(` ${import_picocolors49.default.dim("--yes, -y")} skip confirmations / auto-overwrite`);
|
|
79079
|
+
out.push(` ${import_picocolors49.default.dim("--agent <id>")} for link/task create: use this agent id explicitly`);
|
|
79080
|
+
out.push(` ${import_picocolors49.default.dim("--message <text>")} for task create: required first user message`);
|
|
79081
|
+
out.push(` ${import_picocolors49.default.dim("--title <text>")} for task create: optional task title`);
|
|
79082
|
+
out.push(` ${import_picocolors49.default.dim("--model <id>")} for task create: optional model override`);
|
|
79083
|
+
out.push(` ${import_picocolors49.default.dim("--org <id-or-slug>")} pick the organization (team/agent list, agent create, orchestration create/list)`);
|
|
79084
|
+
out.push(` ${import_picocolors49.default.dim("--team <id>")} pick the team, same commands (works without --org)`);
|
|
79085
|
+
out.push(` ${import_picocolors49.default.dim("--json")} machine-readable output for supported commands`);
|
|
79086
|
+
out.push(` ${import_picocolors49.default.dim("--no-tracking")} for link: skip routing LLM traffic through brainbase`);
|
|
79087
|
+
out.push(` ${import_picocolors49.default.dim("--track")} for agent create: enable tracking non-interactively (off without a TTY)`);
|
|
79088
|
+
out.push(` ${import_picocolors49.default.dim("--shell <sh|fish>")} for agent env: pick output format (auto-detected from $SHELL)`);
|
|
79089
|
+
out.push(` ${import_picocolors49.default.dim("--all")} for template list: include installs from other folders`);
|
|
79090
|
+
out.push(` ${import_picocolors49.default.dim("--web <url>")} for login: web app URL (default https://app.brainbaselabs.com)`);
|
|
77137
79091
|
out.push("");
|
|
77138
79092
|
out.push(divider("ENV"));
|
|
77139
79093
|
out.push("");
|
|
77140
|
-
out.push(` ${
|
|
77141
|
-
out.push(` ${
|
|
77142
|
-
out.push(` ${
|
|
77143
|
-
out.push(` ${
|
|
77144
|
-
out.push(` ${
|
|
77145
|
-
out.push(` ${
|
|
77146
|
-
out.push(` ${
|
|
77147
|
-
out.push(` ${
|
|
77148
|
-
out.push(` ${
|
|
77149
|
-
out.push(` ${
|
|
79094
|
+
out.push(` ${import_picocolors49.default.dim("BRAINBASE_HOME")} override the local config dir (default ~/.brainbase)`);
|
|
79095
|
+
out.push(` ${import_picocolors49.default.dim("BRAINBASE_WEB_URL")} override the web app URL used by login`);
|
|
79096
|
+
out.push(` ${import_picocolors49.default.dim("BRAINBASE_CONTROL_PLANE_URL")} override the MAS host (/v2/cli; task create uses /v2/tasks)`);
|
|
79097
|
+
out.push(` ${import_picocolors49.default.dim("BRAINBASE_API_URL")} legacy KLS host override (uses /api/cli; proxy/registry fallback)`);
|
|
79098
|
+
out.push(` ${import_picocolors49.default.dim("BRAINBASE_PROXY_URL")} override the model-proxy URL used by harness traffic (default https://api.v1.brainbaselabs.com)`);
|
|
79099
|
+
out.push(` ${import_picocolors49.default.dim("BRAINBASE_REGISTRY_URL")} override the registry API URL (default https://api.v1.brainbaselabs.com)`);
|
|
79100
|
+
out.push(` ${import_picocolors49.default.dim("BRAINBASE_TOKEN")} long-lived CLI PAT (overrides token.json)`);
|
|
79101
|
+
out.push(` ${import_picocolors49.default.dim("BRAINBASE_SKIP_AUTH")} bypass the auth gate for development`);
|
|
79102
|
+
out.push(` ${import_picocolors49.default.dim("BRAINBASE_NON_INTERACTIVE")} force non-interactive mode — skip/auto-default prompts (CI & agents)`);
|
|
79103
|
+
out.push(` ${import_picocolors49.default.dim("BRAINBASE_RUN_ENTRYPOINT")} =1 → agent pull executes the agent entrypoint (sandbox boots; or pass --run-entrypoint)`);
|
|
77150
79104
|
out.push("");
|
|
77151
79105
|
out.push(divider("HARNESSES"));
|
|
77152
79106
|
out.push("");
|
|
77153
|
-
out.push(` ${
|
|
77154
|
-
out.push(` ${
|
|
77155
|
-
out.push(` ${
|
|
79107
|
+
out.push(` ${import_picocolors49.default.dim("•")} ${import_picocolors49.default.bold("claude-code")} ${import_picocolors49.default.dim("skills, mcps, agents, commands, playbooks, instructions, files")}`);
|
|
79108
|
+
out.push(` ${import_picocolors49.default.dim("•")} ${import_picocolors49.default.bold("codex")} ${import_picocolors49.default.dim("skills, mcps, commands, playbooks, instructions, files")}`);
|
|
79109
|
+
out.push(` ${import_picocolors49.default.dim("•")} ${import_picocolors49.default.bold("kafka")} ${import_picocolors49.default.dim("skills, mcps, agents, commands, playbooks, instructions, files")}`);
|
|
77156
79110
|
out.push("");
|
|
77157
79111
|
console.log(out.join(`
|
|
77158
79112
|
`));
|
|
@@ -77216,13 +79170,13 @@ async function requireAuth(cmd) {
|
|
|
77216
79170
|
if (STORED_PAT_COMMANDS.has(cmd) && readToken())
|
|
77217
79171
|
return;
|
|
77218
79172
|
console.error("");
|
|
77219
|
-
console.error(` ${brandTint("◆")} ${
|
|
79173
|
+
console.error(` ${brandTint("◆")} ${import_picocolors49.default.bold("brainbase")}`);
|
|
77220
79174
|
console.error("");
|
|
77221
|
-
console.error(` ${
|
|
79175
|
+
console.error(` ${import_picocolors49.default.red("✗")} You need to sign in to use ${import_picocolors49.default.bold("brainbase " + cmd)}.`);
|
|
77222
79176
|
if (status.reason)
|
|
77223
|
-
console.error(` ${
|
|
79177
|
+
console.error(` ${import_picocolors49.default.dim(status.reason)}`);
|
|
77224
79178
|
console.error("");
|
|
77225
|
-
console.error(` Run ${
|
|
79179
|
+
console.error(` Run ${import_picocolors49.default.cyan("brainbase login")} to connect this device.`);
|
|
77226
79180
|
console.error("");
|
|
77227
79181
|
process14.exit(1);
|
|
77228
79182
|
}
|
|
@@ -77232,7 +79186,7 @@ async function main() {
|
|
|
77232
79186
|
const rawCwd = process14.cwd();
|
|
77233
79187
|
const cwd2 = (() => {
|
|
77234
79188
|
try {
|
|
77235
|
-
return
|
|
79189
|
+
return fs81.realpathSync(rawCwd);
|
|
77236
79190
|
} catch {
|
|
77237
79191
|
return rawCwd;
|
|
77238
79192
|
}
|
|
@@ -77245,7 +79199,7 @@ async function main() {
|
|
|
77245
79199
|
await runRun(cwd2, argv);
|
|
77246
79200
|
return;
|
|
77247
79201
|
}
|
|
77248
|
-
const sharedArgs = cmd === "task" ? [] : argv;
|
|
79202
|
+
const sharedArgs = cmd === "task" || cmd === "benchmark" ? [] : argv;
|
|
77249
79203
|
const yes = hasFlag2(sharedArgs, "--yes", "-y");
|
|
77250
79204
|
const all = hasFlag2(sharedArgs, "--all");
|
|
77251
79205
|
const harness = getFlag(sharedArgs, "--harness");
|
|
@@ -77374,6 +79328,11 @@ async function main() {
|
|
|
77374
79328
|
await runTask(cwd2, sub, argv);
|
|
77375
79329
|
break;
|
|
77376
79330
|
}
|
|
79331
|
+
case "benchmark": {
|
|
79332
|
+
const sub = argv.shift();
|
|
79333
|
+
process14.exitCode = await runBenchmark(sub, argv);
|
|
79334
|
+
break;
|
|
79335
|
+
}
|
|
77377
79336
|
case "orchestration":
|
|
77378
79337
|
case "orch": {
|
|
77379
79338
|
const sub = argv.shift();
|
|
@@ -77413,10 +79372,10 @@ async function main() {
|
|
|
77413
79372
|
process14.exit(1);
|
|
77414
79373
|
}
|
|
77415
79374
|
} catch (err) {
|
|
77416
|
-
console.error(
|
|
79375
|
+
console.error(import_picocolors49.default.red(`
|
|
77417
79376
|
${err.message}`));
|
|
77418
79377
|
if (err instanceof ApiError && err.status === 401) {
|
|
77419
|
-
console.error(` Run ${
|
|
79378
|
+
console.error(` Run ${import_picocolors49.default.cyan("brainbase login")} to connect this device.`);
|
|
77420
79379
|
}
|
|
77421
79380
|
if (process14.env.BRAINBASE_DEBUG)
|
|
77422
79381
|
console.error(err.stack);
|