codeharness 0.25.9 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -3207,7 +3207,7 @@ function generateDockerfileTemplate(projectDir, stackOrDetections) {
|
|
|
3207
3207
|
}
|
|
3208
3208
|
|
|
3209
3209
|
// src/modules/infra/init-project.ts
|
|
3210
|
-
var HARNESS_VERSION = true ? "0.
|
|
3210
|
+
var HARNESS_VERSION = true ? "0.26.0" : "0.0.0-dev";
|
|
3211
3211
|
function failResult(opts, error) {
|
|
3212
3212
|
return {
|
|
3213
3213
|
status: "fail",
|
package/dist/index.js
CHANGED
|
@@ -51,7 +51,7 @@ import {
|
|
|
51
51
|
validateDockerfile,
|
|
52
52
|
warn,
|
|
53
53
|
writeState
|
|
54
|
-
} from "./chunk-
|
|
54
|
+
} from "./chunk-HJLEFHXN.js";
|
|
55
55
|
|
|
56
56
|
// src/index.ts
|
|
57
57
|
import { Command } from "commander";
|
|
@@ -1893,12 +1893,12 @@ function LastThought({ text }) {
|
|
|
1893
1893
|
/* @__PURE__ */ jsx(Text, { dimColor: true, children: text })
|
|
1894
1894
|
] });
|
|
1895
1895
|
}
|
|
1896
|
-
function RetryNotice({ info:
|
|
1896
|
+
function RetryNotice({ info: info3 }) {
|
|
1897
1897
|
return /* @__PURE__ */ jsxs(Text, { color: "yellow", children: [
|
|
1898
1898
|
"\u23F3 API retry ",
|
|
1899
|
-
|
|
1899
|
+
info3.attempt,
|
|
1900
1900
|
" (waiting ",
|
|
1901
|
-
|
|
1901
|
+
info3.delay,
|
|
1902
1902
|
"ms)"
|
|
1903
1903
|
] });
|
|
1904
1904
|
}
|
|
@@ -1934,33 +1934,33 @@ function shortKey(key) {
|
|
|
1934
1934
|
function formatCost(cost) {
|
|
1935
1935
|
return `$${cost.toFixed(2)}`;
|
|
1936
1936
|
}
|
|
1937
|
-
function Header({ info:
|
|
1938
|
-
if (!
|
|
1937
|
+
function Header({ info: info3 }) {
|
|
1938
|
+
if (!info3) return null;
|
|
1939
1939
|
const parts = ["codeharness run"];
|
|
1940
|
-
if (
|
|
1941
|
-
parts.push(`iteration ${
|
|
1940
|
+
if (info3.iterationCount != null) {
|
|
1941
|
+
parts.push(`iteration ${info3.iterationCount}`);
|
|
1942
1942
|
}
|
|
1943
|
-
if (
|
|
1944
|
-
parts.push(`${
|
|
1943
|
+
if (info3.elapsed) {
|
|
1944
|
+
parts.push(`${info3.elapsed} elapsed`);
|
|
1945
1945
|
}
|
|
1946
|
-
if (
|
|
1947
|
-
parts.push(`${formatCost(
|
|
1946
|
+
if (info3.totalCost != null) {
|
|
1947
|
+
parts.push(`${formatCost(info3.totalCost)} spent`);
|
|
1948
1948
|
}
|
|
1949
1949
|
const headerLine = parts.join(" | ");
|
|
1950
1950
|
let phaseLine = "";
|
|
1951
|
-
if (
|
|
1952
|
-
phaseLine = `Phase: ${
|
|
1953
|
-
if (
|
|
1954
|
-
phaseLine += ` \u2192 AC ${
|
|
1951
|
+
if (info3.phase) {
|
|
1952
|
+
phaseLine = `Phase: ${info3.phase}`;
|
|
1953
|
+
if (info3.acProgress) {
|
|
1954
|
+
phaseLine += ` \u2192 AC ${info3.acProgress}`;
|
|
1955
1955
|
}
|
|
1956
|
-
if (
|
|
1957
|
-
phaseLine += ` (${
|
|
1956
|
+
if (info3.currentCommand) {
|
|
1957
|
+
phaseLine += ` (${info3.currentCommand})`;
|
|
1958
1958
|
}
|
|
1959
1959
|
}
|
|
1960
1960
|
return /* @__PURE__ */ jsxs3(Box3, { flexDirection: "column", children: [
|
|
1961
1961
|
/* @__PURE__ */ jsx3(Text2, { children: headerLine }),
|
|
1962
1962
|
/* @__PURE__ */ jsx3(Separator, {}),
|
|
1963
|
-
/* @__PURE__ */ jsx3(Text2, { children: `Story: ${
|
|
1963
|
+
/* @__PURE__ */ jsx3(Text2, { children: `Story: ${info3.storyKey || "(waiting)"}` }),
|
|
1964
1964
|
phaseLine && /* @__PURE__ */ jsx3(Text2, { children: phaseLine })
|
|
1965
1965
|
] });
|
|
1966
1966
|
}
|
|
@@ -7587,7 +7587,7 @@ function registerTeardownCommand(program) {
|
|
|
7587
7587
|
} else if (otlpMode === "remote-routed") {
|
|
7588
7588
|
if (!options.keepDocker) {
|
|
7589
7589
|
try {
|
|
7590
|
-
const { stopCollectorOnly: stopCollectorOnly2 } = await import("./docker-
|
|
7590
|
+
const { stopCollectorOnly: stopCollectorOnly2 } = await import("./docker-PFJYXP5D.js");
|
|
7591
7591
|
stopCollectorOnly2();
|
|
7592
7592
|
result.docker.stopped = true;
|
|
7593
7593
|
if (!isJson) {
|
|
@@ -7619,7 +7619,7 @@ function registerTeardownCommand(program) {
|
|
|
7619
7619
|
info("Shared stack: kept running (other projects may use it)");
|
|
7620
7620
|
}
|
|
7621
7621
|
} else if (isLegacyStack) {
|
|
7622
|
-
const { isStackRunning: isStackRunning2, stopStack } = await import("./docker-
|
|
7622
|
+
const { isStackRunning: isStackRunning2, stopStack } = await import("./docker-PFJYXP5D.js");
|
|
7623
7623
|
let stackRunning = false;
|
|
7624
7624
|
try {
|
|
7625
7625
|
stackRunning = isStackRunning2(composeFile);
|
|
@@ -9517,8 +9517,216 @@ function registerAuditCommand(program) {
|
|
|
9517
9517
|
});
|
|
9518
9518
|
}
|
|
9519
9519
|
|
|
9520
|
+
// src/commands/stats.ts
|
|
9521
|
+
import { existsSync as existsSync31, readdirSync as readdirSync8, readFileSync as readFileSync25, writeFileSync as writeFileSync15 } from "fs";
|
|
9522
|
+
import { join as join30 } from "path";
|
|
9523
|
+
var RATES = {
|
|
9524
|
+
input: 15,
|
|
9525
|
+
output: 75,
|
|
9526
|
+
cacheRead: 1.5,
|
|
9527
|
+
cacheWrite: 18.75
|
|
9528
|
+
};
|
|
9529
|
+
function emptyBucket() {
|
|
9530
|
+
return { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, calls: 0 };
|
|
9531
|
+
}
|
|
9532
|
+
function bucketCost(b) {
|
|
9533
|
+
return (b.input * RATES.input + b.output * RATES.output + b.cacheRead * RATES.cacheRead + b.cacheWrite * RATES.cacheWrite) / 1e6;
|
|
9534
|
+
}
|
|
9535
|
+
function addToBucket(target, input, output, cacheRead, cacheWrite) {
|
|
9536
|
+
target.input += input;
|
|
9537
|
+
target.output += output;
|
|
9538
|
+
target.cacheRead += cacheRead;
|
|
9539
|
+
target.cacheWrite += cacheWrite;
|
|
9540
|
+
target.calls += 1;
|
|
9541
|
+
}
|
|
9542
|
+
function parseLogFile(filePath, report) {
|
|
9543
|
+
const basename2 = filePath.split("/").pop() ?? "";
|
|
9544
|
+
const dateMatch = basename2.match(/(\d{4}-\d{2}-\d{2})/);
|
|
9545
|
+
const date = dateMatch ? dateMatch[1] : "unknown";
|
|
9546
|
+
let currentPhase = "orchestrator";
|
|
9547
|
+
let currentStory = "unknown";
|
|
9548
|
+
let currentTool = "";
|
|
9549
|
+
let msgInput = 0;
|
|
9550
|
+
let msgOutput = 0;
|
|
9551
|
+
let msgCacheRead = 0;
|
|
9552
|
+
let msgCacheWrite = 0;
|
|
9553
|
+
const content = readFileSync25(filePath, "utf-8");
|
|
9554
|
+
for (const line of content.split("\n")) {
|
|
9555
|
+
if (!line.startsWith("{")) continue;
|
|
9556
|
+
let d;
|
|
9557
|
+
try {
|
|
9558
|
+
d = JSON.parse(line);
|
|
9559
|
+
} catch {
|
|
9560
|
+
continue;
|
|
9561
|
+
}
|
|
9562
|
+
const typ = d.type;
|
|
9563
|
+
if (typ === "stream_event") {
|
|
9564
|
+
const evt = d.event;
|
|
9565
|
+
if (!evt) continue;
|
|
9566
|
+
const evtType = evt.type;
|
|
9567
|
+
if (evtType === "message_start") {
|
|
9568
|
+
if (msgInput + msgOutput + msgCacheRead + msgCacheWrite > 0) {
|
|
9569
|
+
report.messages.push({ input: msgInput, output: msgOutput, cacheRead: msgCacheRead, cacheWrite: msgCacheWrite, phase: currentPhase, story: currentStory, tool: currentTool });
|
|
9570
|
+
}
|
|
9571
|
+
const usage = evt.message?.usage ?? {};
|
|
9572
|
+
msgInput = usage.input_tokens ?? 0;
|
|
9573
|
+
msgCacheRead = usage.cache_read_input_tokens ?? 0;
|
|
9574
|
+
msgCacheWrite = usage.cache_creation_input_tokens ?? 0;
|
|
9575
|
+
msgOutput = 0;
|
|
9576
|
+
} else if (evtType === "message_delta") {
|
|
9577
|
+
const usage = evt.usage ?? {};
|
|
9578
|
+
msgOutput += usage.output_tokens ?? 0;
|
|
9579
|
+
} else if (evtType === "content_block_start") {
|
|
9580
|
+
const cb = evt.content_block;
|
|
9581
|
+
if (cb?.type === "tool_use") currentTool = cb.name ?? "";
|
|
9582
|
+
} else if (evtType === "content_block_delta") {
|
|
9583
|
+
const text = evt.delta?.text ?? "";
|
|
9584
|
+
if (text.includes("Step 3a") || text.toLowerCase().includes("create-story")) currentPhase = "create-story";
|
|
9585
|
+
else if (text.includes("Step 3b") || text.toLowerCase().includes("dev-story")) currentPhase = "dev-story";
|
|
9586
|
+
else if (text.includes("Step 3c") || text.toLowerCase().includes("code-review") || text.toLowerCase().includes("code review")) currentPhase = "code-review";
|
|
9587
|
+
else if (text.includes("Step 3d") || text.toLowerCase().includes("verif")) currentPhase = "verify";
|
|
9588
|
+
else if (text.includes("Step 8") || text.toLowerCase().includes("retro")) currentPhase = "retro";
|
|
9589
|
+
const sm = text.match(/(\d+-\d+-[a-z][\w-]*)/);
|
|
9590
|
+
if (sm) currentStory = sm[1];
|
|
9591
|
+
}
|
|
9592
|
+
}
|
|
9593
|
+
if (typ === "system" && d.subtype === "task_started") {
|
|
9594
|
+
const desc = (d.description ?? "").toLowerCase();
|
|
9595
|
+
if (desc.includes("code review") || desc.includes("code-review")) currentPhase = "code-review";
|
|
9596
|
+
else if (desc.includes("dev-story") || desc.includes("dev story") || desc.includes("implement")) currentPhase = "dev-story";
|
|
9597
|
+
else if (desc.includes("create-story") || desc.includes("create story")) currentPhase = "create-story";
|
|
9598
|
+
else if (desc.includes("verif")) currentPhase = "verify";
|
|
9599
|
+
else if (desc.includes("retro")) currentPhase = "retro";
|
|
9600
|
+
}
|
|
9601
|
+
}
|
|
9602
|
+
if (msgInput + msgOutput + msgCacheRead + msgCacheWrite > 0) {
|
|
9603
|
+
report.messages.push({ input: msgInput, output: msgOutput, cacheRead: msgCacheRead, cacheWrite: msgCacheWrite, phase: currentPhase, story: currentStory, tool: currentTool });
|
|
9604
|
+
}
|
|
9605
|
+
}
|
|
9606
|
+
function generateReport3(projectDir) {
|
|
9607
|
+
const logsDir = join30(projectDir, "ralph", "logs");
|
|
9608
|
+
const logFiles = readdirSync8(logsDir).filter((f) => f.startsWith("claude_output_") && f.endsWith(".log")).sort().map((f) => join30(logsDir, f));
|
|
9609
|
+
const report = {
|
|
9610
|
+
byPhase: /* @__PURE__ */ new Map(),
|
|
9611
|
+
byStory: /* @__PURE__ */ new Map(),
|
|
9612
|
+
byTool: /* @__PURE__ */ new Map(),
|
|
9613
|
+
byDate: /* @__PURE__ */ new Map(),
|
|
9614
|
+
messages: []
|
|
9615
|
+
};
|
|
9616
|
+
for (const logFile of logFiles) {
|
|
9617
|
+
parseLogFile(logFile, report);
|
|
9618
|
+
}
|
|
9619
|
+
const byPhase = {};
|
|
9620
|
+
const byStory = {};
|
|
9621
|
+
const byTool = {};
|
|
9622
|
+
const total = emptyBucket();
|
|
9623
|
+
for (const msg of report.messages) {
|
|
9624
|
+
if (!byPhase[msg.phase]) byPhase[msg.phase] = emptyBucket();
|
|
9625
|
+
addToBucket(byPhase[msg.phase], msg.input, msg.output, msg.cacheRead, msg.cacheWrite);
|
|
9626
|
+
if (!byStory[msg.story]) byStory[msg.story] = emptyBucket();
|
|
9627
|
+
addToBucket(byStory[msg.story], msg.input, msg.output, msg.cacheRead, msg.cacheWrite);
|
|
9628
|
+
if (msg.tool) {
|
|
9629
|
+
if (!byTool[msg.tool]) byTool[msg.tool] = emptyBucket();
|
|
9630
|
+
addToBucket(byTool[msg.tool], msg.input, msg.output, msg.cacheRead, msg.cacheWrite);
|
|
9631
|
+
}
|
|
9632
|
+
addToBucket(total, msg.input, msg.output, msg.cacheRead, msg.cacheWrite);
|
|
9633
|
+
}
|
|
9634
|
+
const storyKeys = Object.keys(byStory).filter((k) => /^\d+-\d+-/.test(k));
|
|
9635
|
+
const totalCost = bucketCost(total);
|
|
9636
|
+
const avgCostPerStory = storyKeys.length > 0 ? storyKeys.reduce((sum, k) => sum + bucketCost(byStory[k]), 0) / storyKeys.length : 0;
|
|
9637
|
+
return {
|
|
9638
|
+
totalCost,
|
|
9639
|
+
totalCalls: report.messages.length,
|
|
9640
|
+
byPhase,
|
|
9641
|
+
byStory,
|
|
9642
|
+
byTool,
|
|
9643
|
+
byDate: {},
|
|
9644
|
+
tokenBreakdown: total,
|
|
9645
|
+
avgCostPerStory,
|
|
9646
|
+
storiesTracked: storyKeys.length
|
|
9647
|
+
};
|
|
9648
|
+
}
|
|
9649
|
+
function formatReport2(report) {
|
|
9650
|
+
const lines = [];
|
|
9651
|
+
lines.push("# Harness Cost Report");
|
|
9652
|
+
lines.push("");
|
|
9653
|
+
lines.push(`Total API-equivalent cost: $${report.totalCost.toFixed(2)}`);
|
|
9654
|
+
lines.push(`Total API calls: ${report.totalCalls}`);
|
|
9655
|
+
lines.push(`Average cost per story: $${report.avgCostPerStory.toFixed(2)} (${report.storiesTracked} stories)`);
|
|
9656
|
+
lines.push("");
|
|
9657
|
+
const tb = report.tokenBreakdown;
|
|
9658
|
+
lines.push("## Cost by Token Type");
|
|
9659
|
+
lines.push("");
|
|
9660
|
+
lines.push(`| Type | Tokens | Rate | Cost | % |`);
|
|
9661
|
+
lines.push(`|------|--------|------|------|---|`);
|
|
9662
|
+
const crCost = tb.cacheRead * RATES.cacheRead / 1e6;
|
|
9663
|
+
const cwCost = tb.cacheWrite * RATES.cacheWrite / 1e6;
|
|
9664
|
+
const outCost = tb.output * RATES.output / 1e6;
|
|
9665
|
+
const inpCost = tb.input * RATES.input / 1e6;
|
|
9666
|
+
lines.push(`| Cache reads | ${tb.cacheRead.toLocaleString()} | $1.50/MTok | $${crCost.toFixed(2)} | ${(crCost / report.totalCost * 100).toFixed(0)}% |`);
|
|
9667
|
+
lines.push(`| Cache writes | ${tb.cacheWrite.toLocaleString()} | $18.75/MTok | $${cwCost.toFixed(2)} | ${(cwCost / report.totalCost * 100).toFixed(0)}% |`);
|
|
9668
|
+
lines.push(`| Output | ${tb.output.toLocaleString()} | $75/MTok | $${outCost.toFixed(2)} | ${(outCost / report.totalCost * 100).toFixed(0)}% |`);
|
|
9669
|
+
lines.push(`| Input | ${tb.input.toLocaleString()} | $15/MTok | $${inpCost.toFixed(2)} | ${(inpCost / report.totalCost * 100).toFixed(0)}% |`);
|
|
9670
|
+
lines.push("");
|
|
9671
|
+
lines.push("## Cost by Phase");
|
|
9672
|
+
lines.push("");
|
|
9673
|
+
lines.push(`| Phase | Calls | Cost | % |`);
|
|
9674
|
+
lines.push(`|-------|-------|------|---|`);
|
|
9675
|
+
const sortedPhases = Object.entries(report.byPhase).sort((a, b) => bucketCost(b[1]) - bucketCost(a[1]));
|
|
9676
|
+
for (const [phase, bucket] of sortedPhases) {
|
|
9677
|
+
const c = bucketCost(bucket);
|
|
9678
|
+
lines.push(`| ${phase} | ${bucket.calls} | $${c.toFixed(2)} | ${(c / report.totalCost * 100).toFixed(1)}% |`);
|
|
9679
|
+
}
|
|
9680
|
+
lines.push("");
|
|
9681
|
+
lines.push("## Cost by Tool");
|
|
9682
|
+
lines.push("");
|
|
9683
|
+
lines.push(`| Tool | Calls | Cost | % |`);
|
|
9684
|
+
lines.push(`|------|-------|------|---|`);
|
|
9685
|
+
const sortedTools = Object.entries(report.byTool).sort((a, b) => bucketCost(b[1]) - bucketCost(a[1]));
|
|
9686
|
+
for (const [tool, bucket] of sortedTools.slice(0, 10)) {
|
|
9687
|
+
const c = bucketCost(bucket);
|
|
9688
|
+
lines.push(`| ${tool} | ${bucket.calls} | $${c.toFixed(2)} | ${(c / report.totalCost * 100).toFixed(1)}% |`);
|
|
9689
|
+
}
|
|
9690
|
+
lines.push("");
|
|
9691
|
+
lines.push("## Top 10 Most Expensive Stories");
|
|
9692
|
+
lines.push("");
|
|
9693
|
+
lines.push(`| Story | Calls | Cost | % |`);
|
|
9694
|
+
lines.push(`|-------|-------|------|---|`);
|
|
9695
|
+
const sortedStories = Object.entries(report.byStory).sort((a, b) => bucketCost(b[1]) - bucketCost(a[1]));
|
|
9696
|
+
for (const [story, bucket] of sortedStories.slice(0, 10)) {
|
|
9697
|
+
const c = bucketCost(bucket);
|
|
9698
|
+
lines.push(`| ${story} | ${bucket.calls} | $${c.toFixed(2)} | ${(c / report.totalCost * 100).toFixed(1)}% |`);
|
|
9699
|
+
}
|
|
9700
|
+
return lines.join("\n");
|
|
9701
|
+
}
|
|
9702
|
+
function registerStatsCommand(program) {
|
|
9703
|
+
program.command("stats").description("Analyze token consumption and cost from ralph session logs").option("--save", "Save report to _bmad-output/implementation-artifacts/cost-report.md").action((options, cmd) => {
|
|
9704
|
+
const globalOpts = cmd.optsWithGlobals();
|
|
9705
|
+
const isJson = !!globalOpts.json;
|
|
9706
|
+
const projectDir = process.cwd();
|
|
9707
|
+
const logsDir = join30(projectDir, "ralph", "logs");
|
|
9708
|
+
if (!existsSync31(logsDir)) {
|
|
9709
|
+
fail("No ralph/logs/ directory found \u2014 run codeharness run first");
|
|
9710
|
+
process.exitCode = 1;
|
|
9711
|
+
return;
|
|
9712
|
+
}
|
|
9713
|
+
const report = generateReport3(projectDir);
|
|
9714
|
+
if (isJson) {
|
|
9715
|
+
jsonOutput(report);
|
|
9716
|
+
return;
|
|
9717
|
+
}
|
|
9718
|
+
const formatted = formatReport2(report);
|
|
9719
|
+
console.log(formatted);
|
|
9720
|
+
if (options.save) {
|
|
9721
|
+
const outPath = join30(projectDir, "_bmad-output", "implementation-artifacts", "cost-report.md");
|
|
9722
|
+
writeFileSync15(outPath, formatted, "utf-8");
|
|
9723
|
+
ok(`Report saved to ${outPath}`);
|
|
9724
|
+
}
|
|
9725
|
+
});
|
|
9726
|
+
}
|
|
9727
|
+
|
|
9520
9728
|
// src/index.ts
|
|
9521
|
-
var VERSION = true ? "0.
|
|
9729
|
+
var VERSION = true ? "0.26.0" : "0.0.0-dev";
|
|
9522
9730
|
function createProgram() {
|
|
9523
9731
|
const program = new Command();
|
|
9524
9732
|
program.name("codeharness").description("Makes autonomous coding agents produce software that actually works").version(VERSION).option("--json", "Output in machine-readable JSON format");
|
|
@@ -9545,6 +9753,7 @@ function createProgram() {
|
|
|
9545
9753
|
registerProgressCommand(program);
|
|
9546
9754
|
registerObservabilityGateCommand(program);
|
|
9547
9755
|
registerAuditCommand(program);
|
|
9756
|
+
registerStatsCommand(program);
|
|
9548
9757
|
return program;
|
|
9549
9758
|
}
|
|
9550
9759
|
if (!process.env["VITEST"]) {
|