@mutagent/evaluator 0.1.0-alpha.4 → 0.1.0-alpha.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/mutagent-evaluator/SKILL.md +2 -1
- package/.claude/skills/mutagent-evaluator/assets/agents/evaluator.md +64 -0
- package/.claude/skills/mutagent-evaluator/references/operation-inventory.md +1 -0
- package/.claude/skills/mutagent-evaluator/scripts/cli/audit-run.ts +6 -3
- package/.claude/skills/mutagent-evaluator/scripts/code-quality-verdict.ts +285 -0
- package/.claude/skills/mutagent-evaluator/scripts/persist-eval-criteria.ts +232 -0
- package/.claude/skills/mutagent-evaluator/scripts/route-failures.ts +12 -2
- package/.claude/skills/mutagent-evaluator/scripts/run-deterministic.ts +114 -18
- package/.claude/skills/mutagent-evaluator/scripts/sync-eval-criteria.ts +244 -0
- package/bin/mutagent-cli.mjs +650 -3
- package/package.json +2 -2
package/bin/mutagent-cli.mjs
CHANGED
|
@@ -9255,8 +9255,645 @@ function runSelect(flags, io) {
|
|
|
9255
9255
|
return 0;
|
|
9256
9256
|
}
|
|
9257
9257
|
|
|
9258
|
-
// ../mutagent-tools/src/cli/
|
|
9258
|
+
// ../mutagent-tools/src/cli/apply.ts
|
|
9259
|
+
import { spawnSync } from "child_process";
|
|
9260
|
+
import { createHash as createHash3, randomUUID } from "crypto";
|
|
9261
|
+
import {
|
|
9262
|
+
existsSync as existsSync4,
|
|
9263
|
+
mkdirSync as mkdirSync2,
|
|
9264
|
+
readFileSync as readFileSync4,
|
|
9265
|
+
rmSync,
|
|
9266
|
+
writeFileSync as writeFileSync2
|
|
9267
|
+
} from "fs";
|
|
9268
|
+
import { dirname as dirname2 } from "path";
|
|
9269
|
+
|
|
9270
|
+
// ../mutagent-tools/src/apply/registry.ts
|
|
9271
|
+
var REGISTRY2 = new Map;
|
|
9272
|
+
function registerTargetAdapter(adapter) {
|
|
9273
|
+
REGISTRY2.set(adapter.id, adapter);
|
|
9274
|
+
}
|
|
9275
|
+
function getTargetAdapter(id) {
|
|
9276
|
+
const a = REGISTRY2.get(id);
|
|
9277
|
+
if (!a) {
|
|
9278
|
+
const known = [...REGISTRY2.keys()].sort().join(", ") || "(none registered)";
|
|
9279
|
+
throw new Error(`no target adapter registered for id '${id}'. Registered: ${known}.`);
|
|
9280
|
+
}
|
|
9281
|
+
return a;
|
|
9282
|
+
}
|
|
9283
|
+
function clearTargetAdapters() {
|
|
9284
|
+
REGISTRY2.clear();
|
|
9285
|
+
}
|
|
9286
|
+
|
|
9287
|
+
// ../mutagent-tools/src/apply/types.ts
|
|
9288
|
+
var AdapterId = {
|
|
9289
|
+
WorktreePr: "worktree-pr",
|
|
9290
|
+
Rest: "rest",
|
|
9291
|
+
VendorCli: "vendor-cli",
|
|
9292
|
+
ReportOnly: "report-only"
|
|
9293
|
+
};
|
|
9294
|
+
var ApplyKind = {
|
|
9295
|
+
CodePr: "code-pr",
|
|
9296
|
+
Markdown: "markdown",
|
|
9297
|
+
CloudDeploy: "cloud-deploy",
|
|
9298
|
+
ReportOnly: "report-only"
|
|
9299
|
+
};
|
|
9300
|
+
|
|
9301
|
+
// ../mutagent-tools/src/apply/binding.ts
|
|
9302
|
+
function adapterIdForKind(kind, opts) {
|
|
9303
|
+
switch (kind) {
|
|
9304
|
+
case ApplyKind.CodePr:
|
|
9305
|
+
case ApplyKind.Markdown:
|
|
9306
|
+
return AdapterId.WorktreePr;
|
|
9307
|
+
case ApplyKind.CloudDeploy:
|
|
9308
|
+
return opts?.platformTransport === "vendor-cli" ? AdapterId.VendorCli : AdapterId.Rest;
|
|
9309
|
+
case ApplyKind.ReportOnly:
|
|
9310
|
+
return AdapterId.ReportOnly;
|
|
9311
|
+
default: {
|
|
9312
|
+
const _never = kind;
|
|
9313
|
+
throw new Error(`no adapter bound for apply.kind '${String(_never)}'`);
|
|
9314
|
+
}
|
|
9315
|
+
}
|
|
9316
|
+
}
|
|
9317
|
+
function kindRunsBuild(kind) {
|
|
9318
|
+
return kind === ApplyKind.CodePr;
|
|
9319
|
+
}
|
|
9320
|
+
|
|
9321
|
+
// ../mutagent-tools/src/apply/audit.ts
|
|
9322
|
+
function auditPath(projectRoot, applyId) {
|
|
9323
|
+
return `${projectRoot}/.mutagent/apply/${applyId}/apply-audit.json`;
|
|
9324
|
+
}
|
|
9325
|
+
var REQUIRED_FIELDS = [
|
|
9326
|
+
"applyId",
|
|
9327
|
+
"transport",
|
|
9328
|
+
"kind",
|
|
9329
|
+
"mode",
|
|
9330
|
+
"fromRev",
|
|
9331
|
+
"toRev",
|
|
9332
|
+
"staleHash",
|
|
9333
|
+
"revertToken",
|
|
9334
|
+
"producedAt"
|
|
9335
|
+
];
|
|
9336
|
+
function missingAuditFields(record) {
|
|
9337
|
+
return REQUIRED_FIELDS.filter((f) => {
|
|
9338
|
+
const v = record[f];
|
|
9339
|
+
return v === undefined || v === null || v === "";
|
|
9340
|
+
});
|
|
9341
|
+
}
|
|
9342
|
+
function writeAudit(record, ctx) {
|
|
9343
|
+
const missing = missingAuditFields(record);
|
|
9344
|
+
if (missing.length > 0) {
|
|
9345
|
+
throw new Error(`apply-audit is incomplete \u2014 missing: ${missing.join(", ")} (0-blind-writes contract, V-O6)`);
|
|
9346
|
+
}
|
|
9347
|
+
const path = auditPath(ctx.projectRoot, record.applyId);
|
|
9348
|
+
ctx.io.mkdirp(`${ctx.projectRoot}/.mutagent/apply/${record.applyId}`);
|
|
9349
|
+
ctx.io.writeFile(path, JSON.stringify(record, null, 2) + `
|
|
9350
|
+
`);
|
|
9351
|
+
}
|
|
9352
|
+
|
|
9353
|
+
// ../mutagent-tools/src/apply/diff.ts
|
|
9354
|
+
function edorderKey(edits) {
|
|
9355
|
+
return [...edits].sort((a, b) => a.path.localeCompare(b.path));
|
|
9356
|
+
}
|
|
9357
|
+
function concatCurrent(edits, read) {
|
|
9358
|
+
return edorderKey(edits).map((e) => `=== ${e.path} ===
|
|
9359
|
+
${read(e.path) ?? ""}`).join(`
|
|
9360
|
+
`);
|
|
9361
|
+
}
|
|
9362
|
+
function concatProposed(edits) {
|
|
9363
|
+
return edorderKey(edits).map((e) => `=== ${e.path} ===
|
|
9364
|
+
${e.contents ?? "<deleted>"}`).join(`
|
|
9365
|
+
`);
|
|
9366
|
+
}
|
|
9367
|
+
function lineDiff(current, proposed) {
|
|
9368
|
+
const a = current.split(`
|
|
9369
|
+
`);
|
|
9370
|
+
const b = proposed.split(`
|
|
9371
|
+
`);
|
|
9372
|
+
const out = [];
|
|
9373
|
+
const max = Math.max(a.length, b.length);
|
|
9374
|
+
for (let i = 0;i < max; i++) {
|
|
9375
|
+
const cur = a[i];
|
|
9376
|
+
const prop = b[i];
|
|
9377
|
+
if (cur === prop)
|
|
9378
|
+
continue;
|
|
9379
|
+
if (cur !== undefined)
|
|
9380
|
+
out.push(`- ${cur}`);
|
|
9381
|
+
if (prop !== undefined)
|
|
9382
|
+
out.push(`+ ${prop}`);
|
|
9383
|
+
}
|
|
9384
|
+
return out.join(`
|
|
9385
|
+
`);
|
|
9386
|
+
}
|
|
9387
|
+
function fileChangeDiff(change, read) {
|
|
9388
|
+
const edits = change.edits ?? [];
|
|
9389
|
+
const current = concatCurrent(edits, read);
|
|
9390
|
+
const proposed = concatProposed(edits);
|
|
9391
|
+
return { current, proposed, diff: lineDiff(current, proposed) };
|
|
9392
|
+
}
|
|
9393
|
+
|
|
9394
|
+
// ../mutagent-tools/src/apply/adapters/report-only.ts
|
|
9395
|
+
var reportOnlyAdapter = {
|
|
9396
|
+
id: AdapterId.ReportOnly,
|
|
9397
|
+
live: true,
|
|
9398
|
+
async read(change, ctx) {
|
|
9399
|
+
const content = (change.edits ?? []).map((e) => ctx.io.readFile(`${change.targetRoot}/${e.path}`) ?? "").join(`
|
|
9400
|
+
`);
|
|
9401
|
+
return { rev: "report-only", content, hash: ctx.io.hash(content), exists: true };
|
|
9402
|
+
},
|
|
9403
|
+
async dryRun(change, current, ctx) {
|
|
9404
|
+
const { diff } = fileChangeDiff(change, (p) => ctx.io.readFile(`${change.targetRoot}/${p}`));
|
|
9405
|
+
return {
|
|
9406
|
+
diff,
|
|
9407
|
+
staleHash: current.hash,
|
|
9408
|
+
wouldWrite: false,
|
|
9409
|
+
notes: ["report-only: no write will be performed"]
|
|
9410
|
+
};
|
|
9411
|
+
},
|
|
9412
|
+
async apply(change, current, ctx) {
|
|
9413
|
+
const applyId = ctx.io.uid();
|
|
9414
|
+
return { applyId, newRev: current.rev, notes: ["report-only: no write performed"] };
|
|
9415
|
+
},
|
|
9416
|
+
async rollback(audit) {
|
|
9417
|
+
return {
|
|
9418
|
+
applyId: audit.applyId,
|
|
9419
|
+
restoredRev: audit.fromRev,
|
|
9420
|
+
notes: ["report-only: nothing to roll back"]
|
|
9421
|
+
};
|
|
9422
|
+
},
|
|
9423
|
+
emitAudit(record, ctx) {
|
|
9424
|
+
writeAudit(record, ctx);
|
|
9425
|
+
}
|
|
9426
|
+
};
|
|
9427
|
+
|
|
9428
|
+
// ../mutagent-tools/src/apply/adapters/worktree-pr.ts
|
|
9429
|
+
function sh(ctx, cmd, args, cwd, input) {
|
|
9430
|
+
const r = ctx.io.shell(cmd, args, { cwd, input });
|
|
9431
|
+
if (r.code !== 0) {
|
|
9432
|
+
throw new Error(`\`${cmd} ${args.join(" ")}\` failed (exit ${r.code}): ${r.stderr.trim() || r.stdout.trim()}`);
|
|
9433
|
+
}
|
|
9434
|
+
return r;
|
|
9435
|
+
}
|
|
9436
|
+
function headRev(ctx, cwd) {
|
|
9437
|
+
const r = ctx.io.shell("git", ["rev-parse", "HEAD"], { cwd });
|
|
9438
|
+
return r.code === 0 ? r.stdout.trim() : "unknown";
|
|
9439
|
+
}
|
|
9440
|
+
function parsePrUrl(stdout) {
|
|
9441
|
+
const m = stdout.match(/https?:\/\/\S+/);
|
|
9442
|
+
return m ? m[0] : undefined;
|
|
9443
|
+
}
|
|
9444
|
+
var worktreePrAdapter = {
|
|
9445
|
+
id: AdapterId.WorktreePr,
|
|
9446
|
+
live: true,
|
|
9447
|
+
async read(change, ctx) {
|
|
9448
|
+
const edits = change.edits ?? [];
|
|
9449
|
+
const content = concatCurrent(edits, (p) => ctx.io.readFile(`${change.targetRoot}/${p}`));
|
|
9450
|
+
const exists = edits.some((e) => ctx.io.exists(`${change.targetRoot}/${e.path}`));
|
|
9451
|
+
return { rev: headRev(ctx, change.targetRoot), content, hash: ctx.io.hash(content), exists };
|
|
9452
|
+
},
|
|
9453
|
+
async dryRun(change, current, ctx) {
|
|
9454
|
+
const { diff } = fileChangeDiff(change, (p) => ctx.io.readFile(`${change.targetRoot}/${p}`));
|
|
9455
|
+
const proposed = concatProposed(change.edits ?? []);
|
|
9456
|
+
return {
|
|
9457
|
+
diff,
|
|
9458
|
+
staleHash: current.hash,
|
|
9459
|
+
wouldWrite: proposed !== current.content,
|
|
9460
|
+
notes: kindRunsBuild(change.kind) ? ["code-pr: lint + typecheck will run in the worktree"] : ["markdown: no build step"]
|
|
9461
|
+
};
|
|
9462
|
+
},
|
|
9463
|
+
async apply(change, current, ctx) {
|
|
9464
|
+
const edits = change.edits ?? [];
|
|
9465
|
+
if (edits.length === 0)
|
|
9466
|
+
throw new Error("worktree-PR apply: no edits provided");
|
|
9467
|
+
const now = concatCurrent(edits, (p) => ctx.io.readFile(`${change.targetRoot}/${p}`));
|
|
9468
|
+
if (ctx.io.hash(now) !== current.hash) {
|
|
9469
|
+
throw new Error("worktree-PR apply: target changed since read (stale-hash mismatch) \u2014 re-run dryRun");
|
|
9470
|
+
}
|
|
9471
|
+
const applyId = ctx.io.uid();
|
|
9472
|
+
const branch = `apply/${applyId}`;
|
|
9473
|
+
const wtdir = `${ctx.projectRoot}/.mutagent/apply/${applyId}/worktree`;
|
|
9474
|
+
ctx.io.mkdirp(`${ctx.projectRoot}/.mutagent/apply/${applyId}`);
|
|
9475
|
+
sh(ctx, "git", ["worktree", "add", "-b", branch, wtdir, "HEAD"], change.targetRoot);
|
|
9476
|
+
try {
|
|
9477
|
+
for (const e of edits) {
|
|
9478
|
+
const target = `${wtdir}/${e.path}`;
|
|
9479
|
+
if (e.contents === null)
|
|
9480
|
+
ctx.io.deleteFile(target);
|
|
9481
|
+
else {
|
|
9482
|
+
ctx.io.mkdirp(dirOf(target));
|
|
9483
|
+
ctx.io.writeFile(target, e.contents);
|
|
9484
|
+
}
|
|
9485
|
+
}
|
|
9486
|
+
if (kindRunsBuild(change.kind)) {
|
|
9487
|
+
sh(ctx, "bun", ["run", "lint"], wtdir);
|
|
9488
|
+
sh(ctx, "bun", ["run", "typecheck"], wtdir);
|
|
9489
|
+
}
|
|
9490
|
+
sh(ctx, "git", ["add", "-A"], wtdir);
|
|
9491
|
+
sh(ctx, "git", ["commit", "-m", change.title], wtdir);
|
|
9492
|
+
const newRev = headRev(ctx, wtdir);
|
|
9493
|
+
sh(ctx, "git", ["push", "-u", "origin", branch], wtdir);
|
|
9494
|
+
const pr = sh(ctx, "gh", ["pr", "create", "--base", "main", "--title", change.title, "--body", change.body ?? change.title], wtdir);
|
|
9495
|
+
return { applyId, newRev, ref: parsePrUrl(pr.stdout), notes: [`branch ${branch}`] };
|
|
9496
|
+
} finally {
|
|
9497
|
+
ctx.io.shell("git", ["worktree", "remove", "--force", wtdir], { cwd: change.targetRoot });
|
|
9498
|
+
}
|
|
9499
|
+
},
|
|
9500
|
+
async rollback(audit, ctx) {
|
|
9501
|
+
const branch = `rollback/${audit.applyId}`;
|
|
9502
|
+
const repoRoot = ctx.projectRoot;
|
|
9503
|
+
const wtdir = `${ctx.projectRoot}/.mutagent/apply/${audit.applyId}/rollback-worktree`;
|
|
9504
|
+
ctx.io.mkdirp(`${ctx.projectRoot}/.mutagent/apply/${audit.applyId}`);
|
|
9505
|
+
sh(ctx, "git", ["worktree", "add", "-b", branch, wtdir, "HEAD"], repoRoot);
|
|
9506
|
+
try {
|
|
9507
|
+
sh(ctx, "git", ["revert", "--no-edit", audit.revertToken], wtdir);
|
|
9508
|
+
const restoredRev = headRev(ctx, wtdir);
|
|
9509
|
+
sh(ctx, "git", ["push", "-u", "origin", branch], wtdir);
|
|
9510
|
+
const pr = sh(ctx, "gh", ["pr", "create", "--base", "main", "--title", `Revert ${audit.applyId}`, "--body", `Reverts ${audit.toRev}`], wtdir);
|
|
9511
|
+
return { applyId: audit.applyId, restoredRev, ref: parsePrUrl(pr.stdout), notes: [`revert of ${audit.toRev}`] };
|
|
9512
|
+
} finally {
|
|
9513
|
+
ctx.io.shell("git", ["worktree", "remove", "--force", wtdir], { cwd: repoRoot });
|
|
9514
|
+
}
|
|
9515
|
+
},
|
|
9516
|
+
emitAudit(record, ctx) {
|
|
9517
|
+
writeAudit(record, ctx);
|
|
9518
|
+
}
|
|
9519
|
+
};
|
|
9520
|
+
function dirOf(path) {
|
|
9521
|
+
const i = path.lastIndexOf("/");
|
|
9522
|
+
return i <= 0 ? path : path.slice(0, i);
|
|
9523
|
+
}
|
|
9524
|
+
|
|
9525
|
+
// ../mutagent-tools/src/apply/adapters/rest.ts
|
|
9526
|
+
function authHeaders(change) {
|
|
9527
|
+
const h = { "content-type": "application/json" };
|
|
9528
|
+
if (change.credentialToken)
|
|
9529
|
+
h["authorization"] = `Bearer ${change.credentialToken}`;
|
|
9530
|
+
return h;
|
|
9531
|
+
}
|
|
9532
|
+
function assertOk(r, what) {
|
|
9533
|
+
if (r.status < 200 || r.status >= 300) {
|
|
9534
|
+
throw new Error(`REST ${what} failed (HTTP ${r.status}): ${r.body.slice(0, 200)}`);
|
|
9535
|
+
}
|
|
9536
|
+
return r;
|
|
9537
|
+
}
|
|
9538
|
+
function extractRev(body) {
|
|
9539
|
+
try {
|
|
9540
|
+
const j = JSON.parse(body);
|
|
9541
|
+
for (const k of ["activeRev", "rev", "revision", "version", "id"]) {
|
|
9542
|
+
const v = j[k];
|
|
9543
|
+
if (typeof v === "string" && v)
|
|
9544
|
+
return v;
|
|
9545
|
+
if (typeof v === "number")
|
|
9546
|
+
return String(v);
|
|
9547
|
+
}
|
|
9548
|
+
} catch {}
|
|
9549
|
+
return "";
|
|
9550
|
+
}
|
|
9551
|
+
var restAdapter = {
|
|
9552
|
+
id: AdapterId.Rest,
|
|
9553
|
+
live: true,
|
|
9554
|
+
async read(change, ctx) {
|
|
9555
|
+
const r = assertOk(await ctx.io.http("GET", change.targetRoot, { headers: authHeaders(change) }), "GET");
|
|
9556
|
+
const rev = extractRev(r.body) || "unknown";
|
|
9557
|
+
return { rev, content: r.body, hash: ctx.io.hash(r.body), exists: r.status !== 404 };
|
|
9558
|
+
},
|
|
9559
|
+
async dryRun(change, current, ctx) {
|
|
9560
|
+
const proposed = JSON.stringify(change.config ?? {}, null, 2);
|
|
9561
|
+
return {
|
|
9562
|
+
diff: lineDiff(current.content, proposed),
|
|
9563
|
+
staleHash: current.hash,
|
|
9564
|
+
wouldWrite: ctx.io.hash(proposed) !== current.hash,
|
|
9565
|
+
notes: ["cloud-rest: non-destructive create-rev + activate"]
|
|
9566
|
+
};
|
|
9567
|
+
},
|
|
9568
|
+
async apply(change, current, ctx) {
|
|
9569
|
+
const applyId = ctx.io.uid();
|
|
9570
|
+
const created = assertOk(await ctx.io.http("POST", `${change.targetRoot}/revisions`, {
|
|
9571
|
+
headers: authHeaders(change),
|
|
9572
|
+
body: JSON.stringify(change.config ?? {})
|
|
9573
|
+
}), "POST create-rev");
|
|
9574
|
+
const newRev = extractRev(created.body);
|
|
9575
|
+
if (!newRev)
|
|
9576
|
+
throw new Error("REST apply: create-rev returned no rev id");
|
|
9577
|
+
assertOk(await ctx.io.http("PATCH", change.targetRoot, {
|
|
9578
|
+
headers: authHeaders(change),
|
|
9579
|
+
body: JSON.stringify({ activeRev: newRev })
|
|
9580
|
+
}), "PATCH activate");
|
|
9581
|
+
return { applyId, newRev, ref: newRev, notes: [`activated ${newRev} (prior ${current.rev} retained)`] };
|
|
9582
|
+
},
|
|
9583
|
+
async rollback(audit, ctx) {
|
|
9584
|
+
if (!audit.target)
|
|
9585
|
+
throw new Error("REST rollback: audit has no target url");
|
|
9586
|
+
assertOk(await ctx.io.http("PATCH", audit.target, {
|
|
9587
|
+
body: JSON.stringify({ activeRev: audit.revertToken }),
|
|
9588
|
+
headers: { "content-type": "application/json" }
|
|
9589
|
+
}), "PATCH re-activate");
|
|
9590
|
+
return {
|
|
9591
|
+
applyId: audit.applyId,
|
|
9592
|
+
restoredRev: audit.revertToken,
|
|
9593
|
+
ref: audit.revertToken,
|
|
9594
|
+
notes: [`re-activated ${audit.revertToken}; ${audit.toRev} retained`]
|
|
9595
|
+
};
|
|
9596
|
+
},
|
|
9597
|
+
emitAudit(record, ctx) {
|
|
9598
|
+
writeAudit(record, ctx);
|
|
9599
|
+
}
|
|
9600
|
+
};
|
|
9601
|
+
|
|
9602
|
+
// ../mutagent-tools/src/apply/adapters/vendor-cli.ts
|
|
9603
|
+
function sh2(ctx, cmd, args) {
|
|
9604
|
+
const r = ctx.io.shell(cmd, args);
|
|
9605
|
+
if (r.code !== 0) {
|
|
9606
|
+
throw new Error(`\`${cmd} ${args.join(" ")}\` failed (exit ${r.code}): ${r.stderr.trim() || r.stdout.trim()}`);
|
|
9607
|
+
}
|
|
9608
|
+
return r;
|
|
9609
|
+
}
|
|
9610
|
+
function parseHandle(stdout) {
|
|
9611
|
+
const m = stdout.match(/(?:version|rev|revision|handle)[:=\s]+(\S+)/i) ?? stdout.match(/\bv\d+\b/) ?? stdout.trim().split(`
|
|
9612
|
+
`).pop()?.match(/(\S+)$/);
|
|
9613
|
+
return (Array.isArray(m) ? m[1] ?? m[0] : "") || "";
|
|
9614
|
+
}
|
|
9615
|
+
function vendorBin(change) {
|
|
9616
|
+
if (!change.vendorBin)
|
|
9617
|
+
throw new Error("vendor-CLI: change.vendorBin is required (the deploy binary name)");
|
|
9618
|
+
return change.vendorBin;
|
|
9619
|
+
}
|
|
9620
|
+
var vendorCliAdapter = {
|
|
9621
|
+
id: AdapterId.VendorCli,
|
|
9622
|
+
live: true,
|
|
9623
|
+
async read(change, ctx) {
|
|
9624
|
+
const bin = vendorBin(change);
|
|
9625
|
+
const r = ctx.io.shell(bin, ["get", change.targetRoot]);
|
|
9626
|
+
const content = r.stdout;
|
|
9627
|
+
return { rev: parseHandle(content) || "unknown", content, hash: ctx.io.hash(content), exists: r.code === 0 };
|
|
9628
|
+
},
|
|
9629
|
+
async dryRun(change, current, ctx) {
|
|
9630
|
+
const proposed = JSON.stringify(change.config ?? {}, null, 2);
|
|
9631
|
+
return {
|
|
9632
|
+
diff: lineDiff(current.content, proposed),
|
|
9633
|
+
staleHash: current.hash,
|
|
9634
|
+
wouldWrite: ctx.io.hash(proposed) !== current.hash,
|
|
9635
|
+
notes: [`vendor-CLI (${change.vendorBin ?? "?"}): shell deploy \u2192 new version handle`]
|
|
9636
|
+
};
|
|
9637
|
+
},
|
|
9638
|
+
async apply(change, current, ctx) {
|
|
9639
|
+
const applyId = ctx.io.uid();
|
|
9640
|
+
const bin = vendorBin(change);
|
|
9641
|
+
const cfgPath = `${ctx.projectRoot}/.mutagent/apply/${applyId}/deploy-config.json`;
|
|
9642
|
+
ctx.io.mkdirp(`${ctx.projectRoot}/.mutagent/apply/${applyId}`);
|
|
9643
|
+
ctx.io.writeFile(cfgPath, JSON.stringify(change.config ?? {}, null, 2));
|
|
9644
|
+
const r = sh2(ctx, bin, ["deploy", change.targetRoot, "--config", cfgPath]);
|
|
9645
|
+
const newRev = parseHandle(r.stdout);
|
|
9646
|
+
if (!newRev)
|
|
9647
|
+
throw new Error("vendor-CLI apply: deploy returned no version handle");
|
|
9648
|
+
return { applyId, newRev, ref: newRev, notes: [`deployed ${newRev} (prior ${current.rev} retained)`] };
|
|
9649
|
+
},
|
|
9650
|
+
async rollback(audit, ctx) {
|
|
9651
|
+
const bin = (audit.note ?? "").match(/vendor-CLI \(([^)]+)\)/)?.[1];
|
|
9652
|
+
const [binName, target] = (audit.target ?? "").split("|");
|
|
9653
|
+
const resolvedBin = bin && bin !== "?" ? bin : binName;
|
|
9654
|
+
if (!resolvedBin || !target)
|
|
9655
|
+
throw new Error("vendor-CLI rollback: audit.target must be 'bin|url'");
|
|
9656
|
+
sh2(ctx, resolvedBin, ["rollback", target, "--to", audit.revertToken]);
|
|
9657
|
+
return {
|
|
9658
|
+
applyId: audit.applyId,
|
|
9659
|
+
restoredRev: audit.revertToken,
|
|
9660
|
+
ref: audit.revertToken,
|
|
9661
|
+
notes: [`vendor rollback to ${audit.revertToken}; ${audit.toRev} retained`]
|
|
9662
|
+
};
|
|
9663
|
+
},
|
|
9664
|
+
emitAudit(record, ctx) {
|
|
9665
|
+
writeAudit(record, ctx);
|
|
9666
|
+
}
|
|
9667
|
+
};
|
|
9668
|
+
// ../mutagent-tools/src/apply/target-select.ts
|
|
9669
|
+
var TargetSelectionOutcome = {
|
|
9670
|
+
Resolved: "resolved",
|
|
9671
|
+
NeedsDisambiguation: "needs-disambiguation",
|
|
9672
|
+
NoTarget: "no-target"
|
|
9673
|
+
};
|
|
9674
|
+
function selectApplyTarget(candidates, pick) {
|
|
9675
|
+
const uniq = [...new Set(candidates.filter((c) => c.trim().length > 0))];
|
|
9676
|
+
if (uniq.length === 0) {
|
|
9677
|
+
return { outcome: TargetSelectionOutcome.NoTarget };
|
|
9678
|
+
}
|
|
9679
|
+
if (uniq.length === 1) {
|
|
9680
|
+
return { outcome: TargetSelectionOutcome.Resolved, target: uniq[0] };
|
|
9681
|
+
}
|
|
9682
|
+
if (pick !== undefined && uniq.includes(pick)) {
|
|
9683
|
+
return { outcome: TargetSelectionOutcome.Resolved, target: pick };
|
|
9684
|
+
}
|
|
9685
|
+
return {
|
|
9686
|
+
outcome: TargetSelectionOutcome.NeedsDisambiguation,
|
|
9687
|
+
candidates: uniq,
|
|
9688
|
+
reason: pick !== undefined ? `--pick '${pick}' is not one of the ${uniq.length} candidate targets` : `${uniq.length} candidate targets present and no explicit --pick \u2014 refusing to default`
|
|
9689
|
+
};
|
|
9690
|
+
}
|
|
9691
|
+
|
|
9692
|
+
// ../mutagent-tools/src/apply/index.ts
|
|
9693
|
+
function registerBuiltinAdapters2() {
|
|
9694
|
+
clearTargetAdapters();
|
|
9695
|
+
registerTargetAdapter(reportOnlyAdapter);
|
|
9696
|
+
registerTargetAdapter(worktreePrAdapter);
|
|
9697
|
+
registerTargetAdapter(restAdapter);
|
|
9698
|
+
registerTargetAdapter(vendorCliAdapter);
|
|
9699
|
+
}
|
|
9700
|
+
async function runApplyFlow(change, ctx, mode) {
|
|
9701
|
+
const adapterId = adapterIdForKind(change.kind, { platformTransport: change.platformTransport });
|
|
9702
|
+
const adapter = getTargetAdapter(adapterId);
|
|
9703
|
+
const read = await adapter.read(change, ctx);
|
|
9704
|
+
const dryRun = await adapter.dryRun(change, read, ctx);
|
|
9705
|
+
if (mode === "dry-run") {
|
|
9706
|
+
return { adapterId, read, dryRun };
|
|
9707
|
+
}
|
|
9708
|
+
const applyResult = await adapter.apply(change, read, ctx);
|
|
9709
|
+
const audit = {
|
|
9710
|
+
applyId: applyResult.applyId,
|
|
9711
|
+
transport: adapterId,
|
|
9712
|
+
kind: change.kind,
|
|
9713
|
+
subject: change.subject,
|
|
9714
|
+
target: change.targetRoot,
|
|
9715
|
+
mode: change.kind === ApplyKind.ReportOnly ? "report-only" : "apply",
|
|
9716
|
+
fromRev: read.rev,
|
|
9717
|
+
toRev: applyResult.newRev,
|
|
9718
|
+
staleHash: dryRun.staleHash,
|
|
9719
|
+
revertToken: read.rev,
|
|
9720
|
+
ref: applyResult.ref,
|
|
9721
|
+
producedAt: ctx.io.producedAt,
|
|
9722
|
+
note: (applyResult.notes ?? []).join("; ") || undefined
|
|
9723
|
+
};
|
|
9724
|
+
adapter.emitAudit(audit, ctx);
|
|
9725
|
+
return { adapterId, read, dryRun, applyResult, audit };
|
|
9726
|
+
}
|
|
9727
|
+
async function runRollbackFlow(audit, ctx) {
|
|
9728
|
+
const adapter = getTargetAdapter(audit.transport);
|
|
9729
|
+
const rollback = await adapter.rollback(audit, ctx);
|
|
9730
|
+
const record = {
|
|
9731
|
+
applyId: `${audit.applyId}-rollback`,
|
|
9732
|
+
transport: audit.transport,
|
|
9733
|
+
kind: audit.kind,
|
|
9734
|
+
subject: audit.subject,
|
|
9735
|
+
target: audit.target,
|
|
9736
|
+
mode: "rollback",
|
|
9737
|
+
fromRev: audit.toRev,
|
|
9738
|
+
toRev: rollback.restoredRev,
|
|
9739
|
+
staleHash: audit.staleHash,
|
|
9740
|
+
revertToken: audit.toRev,
|
|
9741
|
+
ref: rollback.ref,
|
|
9742
|
+
producedAt: ctx.io.producedAt,
|
|
9743
|
+
note: (rollback.notes ?? []).join("; ") || undefined
|
|
9744
|
+
};
|
|
9745
|
+
adapter.emitAudit(record, ctx);
|
|
9746
|
+
return { rollback, audit: record };
|
|
9747
|
+
}
|
|
9748
|
+
|
|
9749
|
+
// ../mutagent-tools/src/cli/apply.ts
|
|
9259
9750
|
function usage(err) {
|
|
9751
|
+
err("mutagent-cli apply \u2014 shared write transport (worktree-PR \xB7 REST \xB7 vendor-CLI)");
|
|
9752
|
+
err("");
|
|
9753
|
+
err("usage: mutagent-cli apply --kind <code-pr|markdown|cloud-deploy|report-only> --target <root>");
|
|
9754
|
+
err(" (--edit <relpath>=<srcfile>)... | --edits-json <file> # file transports");
|
|
9755
|
+
err(" [--config-json <file>] # platform transports");
|
|
9756
|
+
err(" --title <t> [--body <b>] [--subject <name>]");
|
|
9757
|
+
err(" [--platform-transport rest|vendor-cli] [--vendor-bin <bin>] [--token-ref <ENV>]");
|
|
9758
|
+
err(" [--dry-run | --commit] # default: --dry-run");
|
|
9759
|
+
err(" mutagent-cli apply --rollback <apply-audit.json>");
|
|
9760
|
+
err("");
|
|
9761
|
+
err(" read() + dryRun() are always safe. apply() writes ONLY on --commit, AFTER the caller's gate.");
|
|
9762
|
+
}
|
|
9763
|
+
function buildApplyIo(base) {
|
|
9764
|
+
return {
|
|
9765
|
+
now: base.now,
|
|
9766
|
+
producedAt: base.producedAt,
|
|
9767
|
+
out: base.out,
|
|
9768
|
+
err: base.err,
|
|
9769
|
+
readFile: (p) => existsSync4(p) ? readFileSync4(p, "utf8") : undefined,
|
|
9770
|
+
writeFile: (p, c) => {
|
|
9771
|
+
mkdirSync2(dirname2(p), { recursive: true });
|
|
9772
|
+
writeFileSync2(p, c);
|
|
9773
|
+
},
|
|
9774
|
+
deleteFile: (p) => {
|
|
9775
|
+
if (existsSync4(p))
|
|
9776
|
+
rmSync(p, { force: true });
|
|
9777
|
+
},
|
|
9778
|
+
exists: (p) => existsSync4(p),
|
|
9779
|
+
mkdirp: (p) => mkdirSync2(p, { recursive: true }),
|
|
9780
|
+
shell: (cmd, args, opts) => {
|
|
9781
|
+
const r = spawnSync(cmd, args, {
|
|
9782
|
+
cwd: opts?.cwd,
|
|
9783
|
+
input: opts?.input,
|
|
9784
|
+
encoding: "utf8"
|
|
9785
|
+
});
|
|
9786
|
+
return {
|
|
9787
|
+
stdout: r.stdout ?? "",
|
|
9788
|
+
stderr: r.stderr ?? (r.error ? r.error.message : ""),
|
|
9789
|
+
code: r.status ?? (r.error ? 1 : 0)
|
|
9790
|
+
};
|
|
9791
|
+
},
|
|
9792
|
+
http: async (method, url, opts) => {
|
|
9793
|
+
const res = await fetch(url, { method, headers: opts?.headers, body: opts?.body });
|
|
9794
|
+
const body = await res.text();
|
|
9795
|
+
return { status: res.status, body };
|
|
9796
|
+
},
|
|
9797
|
+
hash: (s) => createHash3("sha256").update(s).digest("hex").slice(0, 16),
|
|
9798
|
+
uid: () => randomUUID().slice(0, 8)
|
|
9799
|
+
};
|
|
9800
|
+
}
|
|
9801
|
+
function buildEdits(flags, cwd) {
|
|
9802
|
+
const editsJson = getStr(flags, "edits-json");
|
|
9803
|
+
if (editsJson) {
|
|
9804
|
+
const raw = readFileSync4(resolveRel(editsJson, cwd), "utf8");
|
|
9805
|
+
const parsed = JSON.parse(raw);
|
|
9806
|
+
if (!Array.isArray(parsed))
|
|
9807
|
+
throw new Error("--edits-json must be a JSON array of {path, contents}");
|
|
9808
|
+
return parsed;
|
|
9809
|
+
}
|
|
9810
|
+
const specs = getList(flags, "edit") ?? [];
|
|
9811
|
+
return specs.map((spec) => {
|
|
9812
|
+
const eq = spec.indexOf("=");
|
|
9813
|
+
if (eq < 0)
|
|
9814
|
+
throw new Error(`--edit expects '<relpath>=<srcfile>' (got '${spec}')`);
|
|
9815
|
+
const path = spec.slice(0, eq);
|
|
9816
|
+
const src = spec.slice(eq + 1);
|
|
9817
|
+
return { path, contents: readFileSync4(resolveRel(src, cwd), "utf8") };
|
|
9818
|
+
});
|
|
9819
|
+
}
|
|
9820
|
+
function resolveRel(p, cwd) {
|
|
9821
|
+
return p.startsWith("/") ? p : `${cwd}/${p}`;
|
|
9822
|
+
}
|
|
9823
|
+
var VALID_KINDS = new Set(Object.values(ApplyKind));
|
|
9824
|
+
async function runApply(argv, base, cwd) {
|
|
9825
|
+
const { flags } = parseArgs(argv);
|
|
9826
|
+
if (getBool(flags, "help") || argv.length === 0) {
|
|
9827
|
+
usage(base.err);
|
|
9828
|
+
return argv.length === 0 ? 1 : 0;
|
|
9829
|
+
}
|
|
9830
|
+
registerBuiltinAdapters2();
|
|
9831
|
+
const io = buildApplyIo(base);
|
|
9832
|
+
const ctx = { io, projectRoot: cwd };
|
|
9833
|
+
const rollbackPath = getStr(flags, "rollback");
|
|
9834
|
+
if (rollbackPath) {
|
|
9835
|
+
const auditRaw = readFileSync4(resolveRel(rollbackPath, cwd), "utf8");
|
|
9836
|
+
const audit = JSON.parse(auditRaw);
|
|
9837
|
+
const { rollback, audit: record } = await runRollbackFlow(audit, ctx);
|
|
9838
|
+
base.out(`\u2705 rollback: restored ${rollback.restoredRev}${rollback.ref ? ` (${rollback.ref})` : ""}`);
|
|
9839
|
+
base.out(` audit: .mutagent/apply/${record.applyId}/apply-audit.json`);
|
|
9840
|
+
return 0;
|
|
9841
|
+
}
|
|
9842
|
+
const kind = getStr(flags, "kind");
|
|
9843
|
+
if (!kind || !VALID_KINDS.has(kind)) {
|
|
9844
|
+
base.err(`apply: --kind must be one of ${[...VALID_KINDS].join(" | ")}`);
|
|
9845
|
+
return 1;
|
|
9846
|
+
}
|
|
9847
|
+
const candidates = getList(flags, "target") ?? [];
|
|
9848
|
+
const selection = selectApplyTarget(candidates, getStr(flags, "pick"));
|
|
9849
|
+
if (selection.outcome === TargetSelectionOutcome.NoTarget) {
|
|
9850
|
+
base.err("apply: --target is required");
|
|
9851
|
+
return 1;
|
|
9852
|
+
}
|
|
9853
|
+
if (selection.outcome === TargetSelectionOutcome.NeedsDisambiguation) {
|
|
9854
|
+
base.err(`apply: ambiguous target \u2014 ${selection.reason}`);
|
|
9855
|
+
base.err(` candidates: ${selection.candidates.join(", ")}`);
|
|
9856
|
+
base.err(" re-run with an explicit --pick <target> (the session must confirm which target to bind)");
|
|
9857
|
+
return 2;
|
|
9858
|
+
}
|
|
9859
|
+
const target = selection.target;
|
|
9860
|
+
const title = getStr(flags, "title") ?? "mutagent apply";
|
|
9861
|
+
const tokenRef = getStr(flags, "token-ref");
|
|
9862
|
+
const change = {
|
|
9863
|
+
kind,
|
|
9864
|
+
targetRoot: target,
|
|
9865
|
+
title,
|
|
9866
|
+
body: getStr(flags, "body"),
|
|
9867
|
+
subject: getStr(flags, "subject"),
|
|
9868
|
+
platformTransport: getStr(flags, "platform-transport"),
|
|
9869
|
+
vendorBin: getStr(flags, "vendor-bin"),
|
|
9870
|
+
credentialToken: tokenRef ? process.env[tokenRef] : undefined
|
|
9871
|
+
};
|
|
9872
|
+
if (kind === ApplyKind.CodePr || kind === ApplyKind.Markdown || kind === ApplyKind.ReportOnly) {
|
|
9873
|
+
change.edits = buildEdits(flags, cwd);
|
|
9874
|
+
}
|
|
9875
|
+
const configJson = getStr(flags, "config-json");
|
|
9876
|
+
if (configJson)
|
|
9877
|
+
change.config = JSON.parse(readFileSync4(resolveRel(configJson, cwd), "utf8"));
|
|
9878
|
+
const mode = getBool(flags, "commit") ? "commit" : "dry-run";
|
|
9879
|
+
const result = await runApplyFlow(change, ctx, mode);
|
|
9880
|
+
base.out(`transport: ${result.adapterId} \xB7 kind: ${kind} \xB7 from-rev: ${result.read.rev}`);
|
|
9881
|
+
base.out(`dry-run: ${result.dryRun.wouldWrite ? "WOULD write" : "no change"} \xB7 stale-hash: ${result.dryRun.staleHash}`);
|
|
9882
|
+
if (result.dryRun.diff)
|
|
9883
|
+
base.out(`--- diff ---
|
|
9884
|
+
${result.dryRun.diff}`);
|
|
9885
|
+
if (mode === "dry-run") {
|
|
9886
|
+
base.out("(dry-run \u2014 no write; re-run with --commit after the caller's gate)");
|
|
9887
|
+
return 0;
|
|
9888
|
+
}
|
|
9889
|
+
const a = result.audit;
|
|
9890
|
+
base.out(`\u2705 applied: ${a?.fromRev} \u2192 ${a?.toRev}${result.applyResult?.ref ? ` (${result.applyResult.ref})` : ""}`);
|
|
9891
|
+
base.out(` audit: .mutagent/apply/${a?.applyId}/apply-audit.json (revert-token: ${a?.revertToken})`);
|
|
9892
|
+
return 0;
|
|
9893
|
+
}
|
|
9894
|
+
|
|
9895
|
+
// ../mutagent-tools/src/cli/index.ts
|
|
9896
|
+
function usage2(err) {
|
|
9260
9897
|
err("mutagent-cli \u2014 MutagenT infra CLI");
|
|
9261
9898
|
err("");
|
|
9262
9899
|
err("usage: mutagent-cli <command> [args]");
|
|
@@ -9273,6 +9910,13 @@ function usage(err) {
|
|
|
9273
9910
|
err(" trace select <--in <unitf.jsonl>|-> [--select <preset>] [--signal <flags>]");
|
|
9274
9911
|
err(" [--signal-mode and|or] [--min-worthiness <0..1>] [--export <path.jsonl>]");
|
|
9275
9912
|
err("");
|
|
9913
|
+
err(" apply --kind <code-pr|markdown|cloud-deploy|report-only> --target <root>");
|
|
9914
|
+
err(" (--edit <relpath>=<src>)... | --edits-json <f> | --config-json <f>");
|
|
9915
|
+
err(" --title <t> [--body <b>] [--subject <n>] [--platform-transport rest|vendor-cli]");
|
|
9916
|
+
err(" [--vendor-bin <bin>] [--token-ref <ENV>] [--dry-run | --commit]");
|
|
9917
|
+
err(" apply --rollback <apply-audit.json>");
|
|
9918
|
+
err(" the SHARED write transport \u2014 the caller gates BEFORE --commit (no approval logic here).");
|
|
9919
|
+
err("");
|
|
9276
9920
|
err(" see references/TRACES-CLI.md for the full contract + UniTF format.");
|
|
9277
9921
|
}
|
|
9278
9922
|
async function main() {
|
|
@@ -9290,14 +9934,17 @@ async function main() {
|
|
|
9290
9934
|
err
|
|
9291
9935
|
};
|
|
9292
9936
|
if (command === undefined || command === "--help" || command === "-h" || command === "help") {
|
|
9293
|
-
|
|
9937
|
+
usage2(err);
|
|
9294
9938
|
return command === undefined ? 1 : 0;
|
|
9295
9939
|
}
|
|
9296
9940
|
if (command === "trace") {
|
|
9297
9941
|
return runTrace(rest, io, process.cwd());
|
|
9298
9942
|
}
|
|
9943
|
+
if (command === "apply") {
|
|
9944
|
+
return runApply(rest, io, process.cwd());
|
|
9945
|
+
}
|
|
9299
9946
|
err(`mutagent-cli: unknown command '${command}'`);
|
|
9300
|
-
|
|
9947
|
+
usage2(err);
|
|
9301
9948
|
return 1;
|
|
9302
9949
|
}
|
|
9303
9950
|
main().then((code) => process.exit(code)).catch((e) => {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mutagent/evaluator",
|
|
3
|
-
"version": "0.1.0-alpha.
|
|
3
|
+
"version": "0.1.0-alpha.5",
|
|
4
4
|
"description": "mutagent-evaluator: a generic, subject-agnostic AI-agent auditor — reviewer, never executor. Audits any skill/agent against a generated subject profile and emits a 4-tab master-audit report.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"type": "module",
|
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
"lint": "eslint .claude/skills/mutagent-evaluator/scripts scripts/release --ext .ts --max-warnings 0",
|
|
23
23
|
"typecheck": "tsc --noEmit",
|
|
24
24
|
"build": "tsc -p tsconfig.build.json",
|
|
25
|
-
"test": "bun test ./.claude/skills/mutagent-evaluator/scripts ./.claude/skills/mutagent-evaluator/tests",
|
|
25
|
+
"test": "node scripts/release/stale-markers.mjs --self-test && bun test ./.claude/skills/mutagent-evaluator/scripts ./.claude/skills/mutagent-evaluator/tests",
|
|
26
26
|
"init": "bun run .claude/skills/mutagent-evaluator/scripts/cli/init.ts",
|
|
27
27
|
"bundle:cli": "bun build ../mutagent-tools/src/cli/index.ts --target=bun --outfile=bin/mutagent-cli.mjs",
|
|
28
28
|
"prepublishOnly": "bun run bundle:cli && node ./scripts/release/prepublish-guard.mjs"
|