nairon-bench 0.0.29 → 0.0.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +178 -6
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -14894,7 +14894,128 @@ function parseClaudeTranscript(filePath, fileName) {
|
|
|
14894
14894
|
};
|
|
14895
14895
|
}
|
|
14896
14896
|
function collectOpenCodeSessions2(openCodeDir, since, until) {
|
|
14897
|
-
|
|
14897
|
+
const sessions = [];
|
|
14898
|
+
const sessionDir = join7(openCodeDir, "storage", "session");
|
|
14899
|
+
const messageDir = join7(openCodeDir, "storage", "message");
|
|
14900
|
+
const partDir = join7(openCodeDir, "storage", "part");
|
|
14901
|
+
if (!existsSync7(sessionDir))
|
|
14902
|
+
return sessions;
|
|
14903
|
+
try {
|
|
14904
|
+
const projectDirs = readdirSync3(sessionDir, { withFileTypes: true }).filter((d2) => d2.isDirectory()).map((d2) => join7(sessionDir, d2.name));
|
|
14905
|
+
for (const projectDir of projectDirs) {
|
|
14906
|
+
const sessionFiles = readdirSync3(projectDir).filter((f3) => f3.endsWith(".json"));
|
|
14907
|
+
for (const sessionFile of sessionFiles) {
|
|
14908
|
+
try {
|
|
14909
|
+
const sessionPath = join7(projectDir, sessionFile);
|
|
14910
|
+
const sessionData = JSON.parse(readFileSync6(sessionPath, "utf-8"));
|
|
14911
|
+
const createdAt = new Date(sessionData.time?.created || 0);
|
|
14912
|
+
const updatedAt = new Date(sessionData.time?.updated || sessionData.time?.created || 0);
|
|
14913
|
+
if (updatedAt < since || createdAt > until)
|
|
14914
|
+
continue;
|
|
14915
|
+
const sessionId = sessionData.id;
|
|
14916
|
+
const sessionMsgDir = join7(messageDir, sessionId);
|
|
14917
|
+
const prompts = [];
|
|
14918
|
+
const responses = [];
|
|
14919
|
+
const toolsUsed = new Set;
|
|
14920
|
+
const filesModified = new Set;
|
|
14921
|
+
let totalTokens = 0;
|
|
14922
|
+
let inputTokens = 0;
|
|
14923
|
+
let outputTokens = 0;
|
|
14924
|
+
let model = "unknown";
|
|
14925
|
+
let startTime = createdAt;
|
|
14926
|
+
let endTime = updatedAt;
|
|
14927
|
+
if (existsSync7(sessionMsgDir)) {
|
|
14928
|
+
const msgFiles = readdirSync3(sessionMsgDir).filter((f3) => f3.endsWith(".json")).sort();
|
|
14929
|
+
for (const msgFile of msgFiles) {
|
|
14930
|
+
try {
|
|
14931
|
+
const msgPath = join7(sessionMsgDir, msgFile);
|
|
14932
|
+
const msgData = JSON.parse(readFileSync6(msgPath, "utf-8"));
|
|
14933
|
+
const msgId = msgData.id;
|
|
14934
|
+
if (msgData.model?.modelID) {
|
|
14935
|
+
model = msgData.model.modelID;
|
|
14936
|
+
}
|
|
14937
|
+
const msgTime = new Date(msgData.time?.created || createdAt);
|
|
14938
|
+
if (msgTime < startTime)
|
|
14939
|
+
startTime = msgTime;
|
|
14940
|
+
if (msgTime > endTime)
|
|
14941
|
+
endTime = msgTime;
|
|
14942
|
+
const msgPartDir = join7(partDir, msgId);
|
|
14943
|
+
let messageText = "";
|
|
14944
|
+
if (existsSync7(msgPartDir)) {
|
|
14945
|
+
const partFiles = readdirSync3(msgPartDir).filter((f3) => f3.endsWith(".json")).sort();
|
|
14946
|
+
for (const partFile of partFiles) {
|
|
14947
|
+
try {
|
|
14948
|
+
const partPath = join7(msgPartDir, partFile);
|
|
14949
|
+
const partData = JSON.parse(readFileSync6(partPath, "utf-8"));
|
|
14950
|
+
if (partData.type === "text" && partData.text && !partData.synthetic) {
|
|
14951
|
+
messageText += partData.text + `
|
|
14952
|
+
`;
|
|
14953
|
+
}
|
|
14954
|
+
if (partData.type === "tool-invocation" || partData.type === "tool-result") {
|
|
14955
|
+
const toolName = partData.toolName || partData.name;
|
|
14956
|
+
if (toolName)
|
|
14957
|
+
toolsUsed.add(toolName);
|
|
14958
|
+
}
|
|
14959
|
+
} catch {}
|
|
14960
|
+
}
|
|
14961
|
+
}
|
|
14962
|
+
if (msgData.role === "user" && messageText.trim()) {
|
|
14963
|
+
const prompt2 = analyzePrompt(messageText, sessionId, prompts.length, msgTime);
|
|
14964
|
+
prompts.push(prompt2);
|
|
14965
|
+
}
|
|
14966
|
+
if (msgData.role === "assistant" && messageText.trim()) {
|
|
14967
|
+
const response = analyzeResponse(messageText, sessionId, responses.length, prompts[prompts.length - 1]?.id ?? "", msgTime);
|
|
14968
|
+
responses.push(response);
|
|
14969
|
+
if (messageText.includes("Write ") || messageText.includes("Created ")) {
|
|
14970
|
+
const fileMatches = messageText.match(/(?:Write|Created|Updated)\s+(\S+\.\w+)/g);
|
|
14971
|
+
if (fileMatches) {
|
|
14972
|
+
fileMatches.forEach((m2) => {
|
|
14973
|
+
const file = m2.split(/\s+/)[1];
|
|
14974
|
+
if (file)
|
|
14975
|
+
filesModified.add(file);
|
|
14976
|
+
});
|
|
14977
|
+
}
|
|
14978
|
+
}
|
|
14979
|
+
}
|
|
14980
|
+
if (msgData.usage) {
|
|
14981
|
+
totalTokens += msgData.usage.totalTokens || 0;
|
|
14982
|
+
inputTokens += msgData.usage.inputTokens || 0;
|
|
14983
|
+
outputTokens += msgData.usage.outputTokens || 0;
|
|
14984
|
+
}
|
|
14985
|
+
} catch {}
|
|
14986
|
+
}
|
|
14987
|
+
}
|
|
14988
|
+
if (prompts.length === 0)
|
|
14989
|
+
continue;
|
|
14990
|
+
const correctionCount = prompts.filter((p) => p.isCorrection).length;
|
|
14991
|
+
const frustrationCount = prompts.filter((p) => p.isFrustrated).length;
|
|
14992
|
+
if (sessionData.summary?.files) {
|
|
14993
|
+
filesModified.add(`${sessionData.summary.files} files changed`);
|
|
14994
|
+
}
|
|
14995
|
+
sessions.push({
|
|
14996
|
+
id: sessionId,
|
|
14997
|
+
agent: "opencode",
|
|
14998
|
+
model,
|
|
14999
|
+
startTime,
|
|
15000
|
+
endTime,
|
|
15001
|
+
durationMinutes: Math.round((endTime.getTime() - startTime.getTime()) / 60000),
|
|
15002
|
+
promptCount: prompts.length,
|
|
15003
|
+
totalTokens,
|
|
15004
|
+
inputTokens,
|
|
15005
|
+
outputTokens,
|
|
15006
|
+
prompts,
|
|
15007
|
+
responses,
|
|
15008
|
+
toolsUsed: Array.from(toolsUsed),
|
|
15009
|
+
filesModified: Array.from(filesModified),
|
|
15010
|
+
hadCompaction: false,
|
|
15011
|
+
correctionCount,
|
|
15012
|
+
frustrationCount
|
|
15013
|
+
});
|
|
15014
|
+
} catch {}
|
|
15015
|
+
}
|
|
15016
|
+
}
|
|
15017
|
+
} catch {}
|
|
15018
|
+
return sessions;
|
|
14898
15019
|
}
|
|
14899
15020
|
function analyzePrompt(text, sessionId, index, timestamp) {
|
|
14900
15021
|
const lower = text.toLowerCase();
|
|
@@ -17391,14 +17512,28 @@ function analyzeSkills() {
|
|
|
17391
17512
|
join8(home, ".agents", "skills"),
|
|
17392
17513
|
join8(home, ".config", "claude", "skills")
|
|
17393
17514
|
];
|
|
17394
|
-
|
|
17515
|
+
const projectSkillsDirs = [
|
|
17516
|
+
join8(process.cwd(), ".claude", "skills"),
|
|
17517
|
+
join8(process.cwd(), ".agents", "skills")
|
|
17518
|
+
];
|
|
17519
|
+
const openCodeSkillDirs = [
|
|
17520
|
+
join8(home, ".config", "opencode", "skills"),
|
|
17521
|
+
join8(home, ".local", "share", "opencode", "skills")
|
|
17522
|
+
];
|
|
17523
|
+
const allSkillDirs = [...skillsDirs, ...projectSkillsDirs, ...openCodeSkillDirs];
|
|
17524
|
+
const seenSkills = new Set;
|
|
17525
|
+
for (const skillsDir of allSkillDirs) {
|
|
17395
17526
|
if (existsSync8(skillsDir)) {
|
|
17396
17527
|
try {
|
|
17397
17528
|
const entries = readdirSync4(skillsDir, { withFileTypes: true });
|
|
17398
17529
|
for (const entry of entries) {
|
|
17399
|
-
|
|
17530
|
+
const skillName = entry.name.replace(/\.md$/, "");
|
|
17531
|
+
if (seenSkills.has(skillName))
|
|
17532
|
+
continue;
|
|
17533
|
+
seenSkills.add(skillName);
|
|
17534
|
+
if (entry.isDirectory() || entry.isSymbolicLink() || entry.name.endsWith(".md")) {
|
|
17400
17535
|
skills.push({
|
|
17401
|
-
name:
|
|
17536
|
+
name: skillName,
|
|
17402
17537
|
path: join8(skillsDir, entry.name),
|
|
17403
17538
|
used: false,
|
|
17404
17539
|
usageCount: 0
|
|
@@ -17608,7 +17743,17 @@ function renderToolUtilizationSummary(analysis) {
|
|
|
17608
17743
|
}
|
|
17609
17744
|
lines.push("");
|
|
17610
17745
|
}
|
|
17611
|
-
|
|
17746
|
+
if (analysis.skills.length > 0) {
|
|
17747
|
+
lines.push(` Skills: ${analysis.skillsCount} installed`);
|
|
17748
|
+
for (const skill of analysis.skills.slice(0, 5)) {
|
|
17749
|
+
lines.push(` - ${skill.name}`);
|
|
17750
|
+
}
|
|
17751
|
+
if (analysis.skills.length > 5) {
|
|
17752
|
+
lines.push(` ... and ${analysis.skills.length - 5} more`);
|
|
17753
|
+
}
|
|
17754
|
+
} else {
|
|
17755
|
+
lines.push(` Skills: none (consider installing reusable prompts)`);
|
|
17756
|
+
}
|
|
17612
17757
|
lines.push(` Docs referenced: ${analysis.docsCount} libraries`);
|
|
17613
17758
|
lines.push(` Tests: ${analysis.testCommands.detected ? `${analysis.testCommands.runCount} runs (${analysis.testCommands.framework})` : "none detected"}`);
|
|
17614
17759
|
lines.push("");
|
|
@@ -17639,6 +17784,20 @@ function renderToolUtilizationMarkdown(analysis) {
|
|
|
17639
17784
|
}
|
|
17640
17785
|
}
|
|
17641
17786
|
lines.push("");
|
|
17787
|
+
lines.push("### Skills");
|
|
17788
|
+
lines.push("");
|
|
17789
|
+
if (analysis.skills.length > 0) {
|
|
17790
|
+
lines.push(`**${analysis.skillsCount} skills installed:**`);
|
|
17791
|
+
lines.push("");
|
|
17792
|
+
for (const skill of analysis.skills) {
|
|
17793
|
+
lines.push(`- ${skill.name}`);
|
|
17794
|
+
}
|
|
17795
|
+
} else {
|
|
17796
|
+
lines.push("*No skills installed. Skills provide reusable prompts for common tasks.*");
|
|
17797
|
+
lines.push("");
|
|
17798
|
+
lines.push("Install skills with: `claude skills add anthropics/skills`");
|
|
17799
|
+
}
|
|
17800
|
+
lines.push("");
|
|
17642
17801
|
if (analysis.docsReferenced.length > 0) {
|
|
17643
17802
|
lines.push("### External Docs Referenced");
|
|
17644
17803
|
lines.push("");
|
|
@@ -18376,8 +18535,18 @@ async function generateHackathonReport(projectDir, since, until = new Date, harn
|
|
|
18376
18535
|
const recommendations = generateRecommendations2(data, sdlcAnalysis, promptAnalysis, scores);
|
|
18377
18536
|
const { grade, gradeDescription } = calculateGrade(overallScore, sdlcAnalysis, promptAnalysis);
|
|
18378
18537
|
const summary = buildSummary(data, sdlcAnalysis, promptAnalysis);
|
|
18538
|
+
const agentLabels = {
|
|
18539
|
+
"claude-code": "Claude Code",
|
|
18540
|
+
opencode: "OpenCode",
|
|
18541
|
+
cursor: "Cursor",
|
|
18542
|
+
windsurf: "Windsurf",
|
|
18543
|
+
copilot: "GitHub Copilot",
|
|
18544
|
+
all: "Multiple Agents"
|
|
18545
|
+
};
|
|
18546
|
+
const primaryAgent = agentLabels[harness || "all"] || harness || "Unknown";
|
|
18379
18547
|
return {
|
|
18380
18548
|
projectName: data.projectName,
|
|
18549
|
+
primaryAgent,
|
|
18381
18550
|
generatedAt: new Date,
|
|
18382
18551
|
timeRange: data.timeRange,
|
|
18383
18552
|
overallScore,
|
|
@@ -18530,6 +18699,7 @@ function formatReportAsMarkdown(report) {
|
|
|
18530
18699
|
const lines = [];
|
|
18531
18700
|
lines.push(`# AI-Nativeness Report: ${report.projectName}`);
|
|
18532
18701
|
lines.push("");
|
|
18702
|
+
lines.push(`**Primary Agent:** ${report.primaryAgent}`);
|
|
18533
18703
|
lines.push(`**Generated:** ${report.generatedAt.toLocaleString()}`);
|
|
18534
18704
|
lines.push(`**Time Range:** ${report.timeRange.start.toLocaleString()} - ${report.timeRange.end.toLocaleString()} (${report.timeRange.durationHours} hours)`);
|
|
18535
18705
|
lines.push("");
|
|
@@ -18702,6 +18872,7 @@ function formatReportAsTerminal(report) {
|
|
|
18702
18872
|
lines.push("");
|
|
18703
18873
|
lines.push("═".repeat(width));
|
|
18704
18874
|
lines.push(` AI-NATIVENESS REPORT: ${report.projectName.toUpperCase()}`);
|
|
18875
|
+
lines.push(` Agent: ${report.primaryAgent}`);
|
|
18705
18876
|
lines.push("═".repeat(width));
|
|
18706
18877
|
lines.push("");
|
|
18707
18878
|
const gradeColors = {
|
|
@@ -18796,6 +18967,7 @@ function renderBar2(value, width = 20) {
|
|
|
18796
18967
|
function formatReportAsJSON(report) {
|
|
18797
18968
|
const serializable = {
|
|
18798
18969
|
...report,
|
|
18970
|
+
primaryAgent: report.primaryAgent,
|
|
18799
18971
|
generatedAt: report.generatedAt.toISOString(),
|
|
18800
18972
|
timeRange: {
|
|
18801
18973
|
start: report.timeRange.start.toISOString(),
|
|
@@ -22876,7 +23048,7 @@ var setupCommand = defineCommand2({
|
|
|
22876
23048
|
// package.json
|
|
22877
23049
|
var package_default = {
|
|
22878
23050
|
name: "nairon-bench",
|
|
22879
|
-
version: "0.0.
|
|
23051
|
+
version: "0.0.31",
|
|
22880
23052
|
description: "AI workflow benchmarking CLI",
|
|
22881
23053
|
type: "module",
|
|
22882
23054
|
bin: {
|