@kindlm/cli 0.2.0 → 0.4.0

package/dist/index.cjs

@@ -48,5 +48,5 @@ Interrupted. Exiting...`)),process.exit(130)};process.on("SIGINT",r);try{return
 `).trim()}),r=s[2]?.trim()??"",o=2,a=[]):a.push(i)}return(r||a.length>0)&&n.push({heading:r,headingLevel:o,body:a.join(`
 `).trim()}),n}function kt(e){return e.match(/^# (.+)$/m)?.[1]?.trim()??"KindLM Compliance Report"}function xt(e){return e.match(/SHA-256:\s*`([a-f0-9]+)`/i)?.[1]??null}function De(e){switch(e){case 2:return 18;case 3:return 15;case 4:return 13;default:return 13}}function Pt(e){if(e.length<2)return null;let t=e[0]??"",n=e[1]??"";if(!t.includes("|")||!n.match(/^\s*\|[-:\s|]+\|\s*$/))return null;let r=i=>i.split("|").slice(1,-1).map(s=>s.trim()),o={cells:r(t)},a=[];for(let i=2;i<e.length;i++){let s=e[i]??"";if(!s.includes("|"))break;a.push({cells:r(s)})}return{header:o,rows:a}}function B(e,t){let n=e.page.margins.bottom;e.page.height-n-30-e.y<t&&(e.addPage(),Ue(e))}function Ue(e){let t=new Date().toISOString();e.fontSize(8).font("Helvetica").fillColor("#a8a29e").text("KindLM Compliance Report",60,40),e.text(t,60,40,{align:"right"}),e.moveDown(3)}function It(e){e.fontSize(8).font("Helvetica").fillColor("#a8a29e").text("Generated by KindLM \xB7 kindlm.com",60,e.page.height-50,{align:"center",width:e.page.width-120})}function $t(e,t,n){let r=t.header.cells.length,o=n/r,a=e.page.margins.left,i=18;B(e,i*2);let s=(c,m,u)=>{let f=e.y;u&&(e.save(),e.rect(a,f-2,n,i).fill(u),e.restore());for(let y=0;y<c.length;y++){let d=a+y*o;e.fontSize(8).font(m?"Helvetica-Bold":"Courier").fillColor("#44403c").text(c[y]??"",d+4,f,{width:o-8,height:i,lineBreak:!1})}e.y=f+i};s(t.header.cells,!0,"#f5f5f4");for(let c of t.rows)B(e,i),s(c.cells,!1)}async function Ke(e,t){return await(0,He.mkdir)((0,Be.dirname)(t),{recursive:!0}),new Promise((n,r)=>{let o=new je.default({size:"A4",margins:{top:72,bottom:72,left:60,right:60},info:{Title:"KindLM EU AI Act Compliance Report",Author:"KindLM",Creator:"KindLM CLI"}}),a=(0,Fe.createWriteStream)(t);o.pipe(a);let i=o.page.width-o.page.margins.left-o.page.margins.right,s=kt(e),c=xt(e);o.moveDown(6),o.fontSize(28).font("Helvetica-Bold").fillColor("#1c1917").text(s,{align:"center",width:i}),o.moveDown(.5),o.fontSize(14).font("Helvetica").fillColor("#57534e").text("EU AI Act Annex IV Documentation",{align:"center",width:i}),o.moveDown(1),o.fontSize(10).fillColor("#a8a29e").text(`Generated: ${new Date().toISOString()}`,{align:"center",width:i}),c&&(o.moveDown(.3),o.fontSize(9).font("Courier").fillColor("#78716c").text(`SHA-256: ${c}`,{align:"center",width:i})),o.moveDown(2),o.fontSize(10).font("Helvetica").fillColor("#6366f1").text("kindlm.com",{align:"center",link:"https://kindlm.com",width:i});let m=bt(e);for(let u of m){if(o.addPage(),Ue(o),u.heading){let C=De(u.headingLevel);o.fontSize(C).font("Helvetica-Bold").fillColor("#1c1917").text(u.heading,{width:i}),o.moveDown(.5),o.moveTo(60,o.y).lineTo(60+i,o.y).strokeColor("#e7e5e4").lineWidth(1).stroke(),o.moveDown(.8)}let f=u.body.split(`
 `),y=!1,d=0;for(;d<f.length;){let C=f[d]??"";if(C.startsWith("```")){y=!y,y&&B(o,30),d++;continue}if(y){B(o,14);let l=o.y;o.save(),o.rect(o.page.margins.left,l-2,i,14).fill("#f5f5f4"),o.restore(),o.fontSize(9).font("Courier").fillColor("#44403c").text(C,{width:i}),d++;continue}if(!C.trim()){o.moveDown(.4),d++;continue}let b=f[d+1]??"";if(C.includes("|")&&d+1<f.length&&b.match(/^\s*\|[-:\s|]+\|\s*$/)){let l=[],g=d;for(;g<f.length&&(f[g]??"").includes("|");)l.push(f[g]??""),g++;let I=Pt(l);if(I){$t(o,I,i),d=g;continue}}if(C.match(/^\s*\|[-:]+/)||C.match(/^---+$/)){d++;continue}let h=C.match(/^(#{3,4})\s+(.+)$/);if(h?.[1]&&h[2]){let l=h[1].length,g=De(l);B(o,g+10),o.moveDown(.3),o.fontSize(g).font("Helvetica-Bold").fillColor("#1c1917").text(h[2].trim(),{width:i}),o.moveDown(.3),d++;continue}if(C.match(/^\s*[-*] /)){B(o,14),o.fontSize(10).font("Helvetica").fillColor("#44403c").text(C.trim(),{indent:12,width:i-12}),d++;continue}B(o,14),o.fontSize(10).font("Helvetica").fillColor("#44403c").text(C.trim(),{width:i}),d++}It(o)}o.end(),a.on("finish",()=>n(t)),a.on("error",r)})}function ze(e){e.command("test").description("Run test suites").option("-s, --suite <name>","Run a specific suite").option("--compliance","Generate compliance report").option("--reporter <type>","Output format: pretty, json, junit","pretty").option("--runs <count>","Override run count").option("--gate <percent>","Fail if pass rate below threshold").option("--pdf <path>","Export compliance report as PDF (requires --compliance)").option("-c, --config <path>","Path to config file","kindlm.yaml").action(async t=>{try{let{runnerResult:n,config:r,yamlContent:o}=await V({configPath:t.config,runs:t.runs?parseInt(t.runs,10):void 0,gate:t.gate?parseFloat(t.gate):void 0}),{runResult:a,aggregated:i}=n,s=(0,P.evaluateGates)(r.gates,i),m=Ot(t.reporter).generate(a,s);if(console.log(m.content),t.compliance){let y=(0,P.createComplianceReporter)().generate(a,s);if(console.log(""),console.log(y.content),t.pdf){let d=await Ke(y.content,t.pdf);console.log(""),console.log(w.default.green(`PDF report saved to ${d}`))}}try{Ne({runnerResult:n,suiteName:r.suite.name,configHash:_e(o),timestamp:new Date().toISOString()})}catch{}let u=a.failed===0&&a.errored===0&&s.passed;process.exit(u?0:1)}catch(n){if(n instanceof P.ProviderError){let r=n.code==="TIMEOUT"?"Provider timeout":n.code==="NETWORK_ERROR"?"Network error":n.code==="AUTH_FAILED"?"Authentication failed":n.code==="RATE_LIMITED"?"Rate limited":`Provider error (${n.code})`;console.error(w.default.red(`${r}: ${n.message}`)),n.retryable&&console.error(w.default.yellow("This error may be transient. Try again or increase --timeout."))}else if(At(n)){let o=n.code.startsWith("CONFIG_")?"Config error":"Error";console.error(w.default.red(`${o}: ${n.message}`))}else n instanceof Error&&n.name==="AbortError"?console.error(w.default.red("Request timed out. Check network connectivity or increase timeout.")):console.error(w.default.red(`Error: ${n instanceof Error?n.message:String(n)}`));process.exit(1)}})}var Et={bold:e=>w.default.bold(e),red:e=>w.default.red(e),green:e=>w.default.green(e),yellow:e=>w.default.yellow(e),cyan:e=>w.default.cyan(e),dim:e=>w.default.dim(e),greenBold:e=>w.default.green.bold(e),redBold:e=>w.default.red.bold(e)};function Ot(e){switch(e){case"json":return(0,P.createJsonReporter)();case"junit":return(0,P.createJunitReporter)();default:return(0,P.createPrettyReporter)(Et)}}function At(e){return typeof e=="object"&&e!==null&&"code"in e&&"message"in e&&typeof e.code=="string"&&typeof e.message=="string"}var k=require("path"),Je=require("fs"),p=$(require("chalk"),1),S=require("@kindlm/core");var D=require("fs"),ae=require("path");function Ge(e){return e.replace(/[^a-zA-Z0-9_-]/g,"_")}function ce(e){let t=(0,ae.join)(e,"baselines");return{read(n){let r=(0,ae.join)(t,`${Ge(n)}.json`);try{return{success:!0,data:(0,D.readFileSync)(r,"utf-8")}}catch{return{success:!1,error:{code:"BASELINE_NOT_FOUND",message:`No baseline found for suite "${n}" at ${r}`}}}},write(n,r){try{(0,D.mkdirSync)(t,{recursive:!0});let o=(0,ae.join)(t,`${Ge(n)}.json`);return(0,D.writeFileSync)(o,r,"utf-8"),{success:!0,data:void 0}}catch(o){return{success:!1,error:{code:"UNKNOWN_ERROR",message:`Failed to write baseline: ${o instanceof Error?o.message:String(o)}`}}}},list(){try{return(0,D.mkdirSync)(t,{recursive:!0}),{success:!0,data:(0,D.readdirSync)(t).filter(o=>o.endsWith(".json")).map(o=>o.replace(/\.json$/,""))}}catch(n){return{success:!1,error:{code:"UNKNOWN_ERROR",message:`Failed to list baselines: ${n instanceof Error?n.message:String(n)}`}}}}}}function We(e){let t=e.command("baseline").description("Manage test baselines");t.command("set").description("Save current results as baseline").option("-c, --config <path>","Path to config file","kindlm.yaml").option("--runs <count>","Override run count").action(async n=>{try{let r=(0,k.dirname)((0,k.resolve)(process.cwd(),n.config)),o=(0,k.join)(r,".kindlm"),a=ce(o),{config:i,runnerResult:s}=await V({configPath:n.config,runs:n.runs?parseInt(n.runs,10):void 0}),{aggregated:c}=s,m=(0,S.buildBaselineData)(i.suite.name,c,new Date().toISOString()),u=(0,S.writeBaseline)(m,a);u.success||(console.error(p.default.red(`Failed to save baseline: ${u.error.message}`)),process.exit(1));let f=Object.keys(m.results).length;console.log(""),console.log(p.default.green(`Baseline saved for suite "${i.suite.name}" (${f} test${f===1?"":"s"})`)),console.log(p.default.dim(`  Location: ${o}/baselines/`)),process.exit(0)}catch(r){console.error(p.default.red(`Error: ${r instanceof Error?r.message:String(r)}`)),process.exit(1)}}),t.command("compare").description("Compare latest against baseline").option("-c, --config <path>","Path to config file","kindlm.yaml").option("--runs <count>","Override run count").action(async n=>{try{let r=(0,k.dirname)((0,k.resolve)(process.cwd(),n.config)),o=(0,k.join)(r,".kindlm"),a=ce(o),i=(0,k.resolve)(process.cwd(),n.config),s;try{s=(0,Je.readFileSync)(i,"utf-8")}catch{console.error(p.default.red(`Config file not found: ${i}`)),process.exit(1)}let c=H(),m=(0,S.parseConfig)(s,{configDir:r,fileReader:c});m.success||(console.error(p.default.red(`Config validation failed: ${m.error.message}`)),process.exit(1));let u=m.data.suite.name,f=(0,S.readBaseline)(u,a);f.success||(f.error.code==="BASELINE_NOT_FOUND"?console.error(p.default.red(`No baseline found for suite "${u}". Run \`kindlm baseline set\` first.`)):console.error(p.default.red(`Failed to read baseline: ${f.error.message}`)),process.exit(1));let y=f.data,{runnerResult:d}=await V({configPath:n.config,runs:n.runs?parseInt(n.runs,10):void 0,baselineData:y}),{aggregated:C}=d,b=(0,S.buildBaselineData)(u,C,new Date().toISOString()),h=(0,S.compareBaseline)(y,b.results);if(console.log(""),console.log(p.default.bold(`Baseline comparison for "${u}"`)),console.log(p.default.dim(`  Baseline from: ${y.createdAt}`)),console.log(""),h.regressions.length>0){console.log(p.default.red.bold(`  Regressions (${h.regressions.length}):`));for(let l of h.regressions)console.log(p.default.red(`    ${l.testName}: ${Z(l.baselinePassRate)} \u2192 ${Z(l.currentPassRate)}`)),l.newFailureCodes.length>0&&console.log(p.default.red(`      New failures: ${l.newFailureCodes.join(", ")}`));console.log("")}if(h.improvements.length>0){console.log(p.default.green.bold(`  Improvements (${h.improvements.length}):`));for(let l of h.improvements)console.log(p.default.green(`    ${l.testName}: ${Z(l.baselinePassRate)} \u2192 ${Z(l.currentPassRate)}`));console.log("")}if(h.unchanged.length>0){console.log(p.default.dim(`  Unchanged (${h.unchanged.length}):`));for(let l of h.unchanged)console.log(p.default.dim(`    ${l.testName}: ${Z(l.passRate)}`));console.log("")}if(h.newTests.length>0){console.log(p.default.cyan(`  New tests (${h.newTests.length}):`));for(let l of h.newTests)console.log(p.default.cyan(`    ${l}`));console.log("")}if(h.removedTests.length>0){console.log(p.default.yellow(`  Removed tests (${h.removedTests.length}):`));for(let l of h.removedTests)console.log(p.default.yellow(`    ${l}`));console.log("")}process.exit(h.regressions.length>0?1:0)}catch(r){console.error(p.default.red(`Error: ${r instanceof Error?r.message:String(r)}`)),process.exit(1)}}),t.command("list").description("List saved baselines").option("-c, --config <path>","Path to config file","kindlm.yaml").action(n=>{try{let r=(0,k.dirname)((0,k.resolve)(process.cwd(),n.config)),o=(0,k.join)(r,".kindlm"),a=ce(o),i=(0,S.listBaselines)(a);i.success||(console.error(p.default.red(`Failed to list baselines: ${i.error.message}`)),process.exit(1));let s=i.data;s.length===0&&(console.log(p.default.dim("No baselines saved yet. Run `kindlm baseline set` to create one.")),process.exit(0)),console.log(p.default.bold("Saved baselines:")),console.log("");for(let c of s){let m=a.read(c);if(!m.success){console.log(`  ${c} ${p.default.dim("(unreadable)")}`);continue}let u=(0,S.deserializeBaseline)(m.data);if(!u.success){console.log(`  ${c} ${p.default.dim("(corrupt)")}`);continue}let f=Object.keys(u.data.results).length;console.log(`  ${p.default.cyan(u.data.suiteName)} \u2014 ${f} test${f===1?"":"s"}, saved ${p.default.dim(u.data.createdAt)}`)}process.exit(0)}catch(r){console.error(p.default.red(`Error: ${r instanceof Error?r.message:String(r)}`)),process.exit(1)}})}function Z(e){return`${(e*100).toFixed(1)}%`}var qe=require("readline"),Ve=require("stream"),A=$(require("chalk"),1);var O=require("fs"),pe=require("path"),he=require("os");function ye(){return(0,pe.join)((0,he.homedir)(),".kindlm","credentials")}function le(){try{let e=(0,O.readFileSync)(ye(),"utf-8"),t=JSON.parse(e);return typeof t.token=="string"&&t.token.length>0?t.token:null}catch{return null}}function Ye(e){let t=ye(),n=(0,pe.join)((0,he.homedir)(),".kindlm");(0,O.mkdirSync)(n,{recursive:!0,mode:448});let r={token:e,savedAt:new Date().toISOString()};(0,O.writeFileSync)(t,JSON.stringify(r,null,2),{mode:384}),(0,O.chmodSync)(t,384)}function Xe(){try{(0,O.unlinkSync)(ye())}catch{}}var Mt="https://api.kindlm.com";var j=class extends Error{status;constructor(t,n){super(n),this.name="CloudApiError",this.status=t}};function Y(){let e=process.env.KINDLM_CLOUD_URL??Mt;if(e.startsWith("http://")&&!Nt(e))throw new Error(`Refusing to use insecure HTTP for Cloud API: ${e}. Use HTTPS or target localhost for development.`);return e}function Nt(e){try{let t=new URL(e);return t.hostname==="localhost"||t.hostname==="127.0.0.1"||t.hostname==="::1"}catch{return!1}}function Lt(e){return new Promise(t=>setTimeout(t,e))}function Q(e,t){async function n(r,o,a){let i=`${e}${o}`,s={Authorization:`Bearer ${t}`},c={method:r,headers:s};a!==void 0&&(s["Content-Type"]="application/json",c.body=JSON.stringify(a));let m;for(let u=0;u<=1;u++){u>0&&await Lt(1e3);let f=new AbortController,y=setTimeout(()=>f.abort(),3e4);c.signal=f.signal;try{let d=await fetch(i,c);if(!d.ok){if(d.status>=500&&u<1){m=new j(d.status,`HTTP ${d.status}`);continue}let b=`HTTP ${d.status}`;if((d.headers.get("content-type")??"").includes("application/json"))try{let l=await d.json();l.error&&(b=l.error)}catch{}throw new j(d.status,b)}if(d.status===204)return;let C=d.headers.get("content-type")??"";if(!C.includes("application/json"))throw new j(d.status,`Expected JSON response but got content-type: ${C}`);return await d.json()}catch(d){if(d instanceof j)throw d;if(m=d instanceof Error?d:new Error(String(d)),u<1)continue}finally{clearTimeout(y)}}throw m??new Error("Request failed")}return{baseUrl:e,get:r=>n("GET",r),post:(r,o)=>n("POST",r,o),patch:(r,o)=>n("PATCH",r,o),delete:r=>n("DELETE",r)}}function Ze(e){e.command("login").description("Authenticate with KindLM Cloud").option("-t, --token <token>","API token (skips interactive prompt)").option("--status","Show current authentication status").option("--logout","Remove stored credentials").action(async t=>{try{if(t.logout){Xe(),console.log(A.default.green("Logged out. Credentials removed."));return}if(t.status){await _t();return}let n=t.token??process.env.KINDLM_API_TOKEN??await Dt();n.startsWith("klm_")||(console.error(A.default.red('Invalid token format. KindLM tokens start with "klm_".')),process.exit(1));let r=Q(Y(),n);try{await r.get("/v1/auth/tokens")}catch(o){throw o instanceof j&&o.status===401&&(console.error(A.default.red("Invalid or expired token.")),process.exit(1)),o}Ye(n),console.log(A.default.green("Authenticated successfully. Token saved."))}catch(n){console.error(A.default.red(`Login failed: ${n instanceof Error?n.message:String(n)}`)),process.exit(1)}})}async function _t(){let e=le();if(!e){console.log(A.default.yellow('Not authenticated. Run "kindlm login" to authenticate.'));return}let t=Q(Y(),e);try{await t.get("/v1/auth/tokens"),console.log(A.default.green("Authenticated.")),console.log(`  Cloud URL: ${Y()}`)}catch(n){n instanceof j&&n.status===401?console.log(A.default.yellow('Stored token is invalid or expired. Run "kindlm login" to re-authenticate.')):console.log(A.default.yellow(`Cannot reach Cloud API: ${n instanceof Error?n.message:String(n)}`))}}function Dt(){return new Promise((e,t)=>{let n=new Ve.Writable({write(o,a,i){i()}});process.stderr.write("Paste your KindLM API token: ");let r=(0,qe.createInterface)({input:process.stdin,output:n,terminal:!0});r.question("",o=>{r.close(),process.stderr.write(`
-`);let a=o.trim();if(!a){t(new Error("No token provided"));return}e(a)})})}var nt=require("path"),ot=require("child_process"),me=$(require("chalk"),1);var de=require("child_process");function Qe(){try{let e=(0,de.execSync)("git rev-parse HEAD",{encoding:"utf-8"}).trim()||null,t=(0,de.execSync)("git rev-parse --abbrev-ref HEAD",{encoding:"utf-8"}).trim()||null,r=(0,de.execSync)("git status --porcelain",{encoding:"utf-8"}).trim().length>0;return{commitSha:e,branch:t,dirty:r}}catch{return{commitSha:null,branch:null,dirty:!1}}}function et(){return process.env.GITHUB_ACTIONS?{name:"github_actions",isCI:!0,commitSha:process.env.GITHUB_SHA??null,branch:process.env.GITHUB_REF_NAME??null}:process.env.GITLAB_CI?{name:"gitlab_ci",isCI:!0,commitSha:process.env.CI_COMMIT_SHA??null,branch:process.env.CI_COMMIT_BRANCH??null}:process.env.CI?{name:null,isCI:!0,commitSha:null,branch:null}:{name:null,isCI:!1,commitSha:null,branch:null}}function ee(e){return encodeURIComponent(e)}async function tt(e,t,n){let r=await jt(e,n.projectName),o=await Ft(e,r,n.suiteName,n.configHash),a=await e.post(`/v1/runs/${ee(r)}/runs`,{suiteId:o,commitSha:n.commitSha,branch:n.branch,environment:n.environment,triggeredBy:n.triggeredBy}),i=Ht(t.aggregated),s=50;for(let l=0;l<i.length;l+=s){let g=i.slice(l,l+s);await e.post(`/v1/results/${ee(a.id)}/results`,{results:g})}let{runResult:c}=t,m=c.totalTests>0?c.passed/c.totalTests:0,u=new Set(t.aggregated.map(l=>l.modelId)),f=t.aggregated.map(l=>l.assertionScores.judge?.mean).filter(l=>l!==void 0),y=f.length>0?f.reduce((l,g)=>l+g,0)/f.length:void 0,d=t.aggregated.map(l=>l.latencyAvgMs),C=d.length>0?d.reduce((l,g)=>l+g,0)/d.length:void 0,b=t.aggregated.reduce((l,g)=>l+g.totalCostUsd,0),h=b>0?b:void 0;return await e.patch(`/v1/runs/${ee(a.id)}`,{status:"completed",passRate:m,testCount:c.totalTests,modelCount:u.size,judgeAvgScore:y,latencyAvgMs:C,costEstimateUsd:h,finishedAt:new Date().toISOString()}),{runId:a.id,projectId:r}}async function jt(e,t){let{projects:n}=await e.get("/v1/projects"),r=n.find(a=>a.name===t);return r?r.id:(await e.post("/v1/projects",{name:t})).id}async function Ft(e,t,n,r){let{suites:o}=await e.get(`/v1/suites/${ee(t)}/suites`),a=o.find(s=>s.name===n);return a?a.id:(await e.post(`/v1/suites/${ee(t)}/suites`,{name:n,configHash:r})).id}function Ht(e){return e.map(t=>{let n=t,r=t.runs.length>0?t.runs.flatMap(o=>o.assertions.filter(a=>!a.passed).map(a=>a.failureMessage)).filter(o=>o!==void 0):n.failureMessages??[];return{testCaseName:t.testCaseName,modelId:t.modelId,passed:t.passed?1:0,passRate:t.passRate,runCount:t.runCount,judgeAvg:t.assertionScores.judge?.mean??null,driftScore:t.assertionScores.drift?.mean??null,latencyAvgMs:t.latencyAvgMs??null,costUsd:t.totalCostUsd??null,totalTokens:t.totalTokens??null,failureCodes:t.failureCodes.length>0?JSON.stringify(t.failureCodes):null,failureMessages:r.length>0?JSON.stringify(r):null,assertionScores:Object.keys(t.assertionScores).length>0?JSON.stringify(t.assertionScores):null}})}function rt(e){e.command("upload").description("Push last run results to KindLM Cloud").option("-t, --token <token>","API token (overrides stored token)").option("-p, --project <name>","Project name").action(async t=>{try{let n=t.token??process.env.KINDLM_API_TOKEN??le();n||(console.error(me.default.red('Not authenticated. Run "kindlm login" first or pass --token.')),process.exit(1));let r=Le();r||(console.error(me.default.red('No test run found. Run "kindlm test" first.')),process.exit(1));let o=Qe(),a=et(),i=t.project??Ut(),s=Q(Y(),n),c=G();c.start("Uploading results to KindLM Cloud...");try{let m=await tt(s,r.runnerResult,{projectName:i,suiteName:r.suiteName,configHash:r.configHash,commitSha:a.commitSha??o.commitSha??void 0,branch:a.branch??o.branch??void 0,environment:a.isCI?"ci":"local",triggeredBy:a.name??"local"});c.succeed("Uploaded successfully."),console.log(`  Run ID:  ${m.runId}`),console.log(`  Project: ${i}`),console.log(`  Suite:   ${r.suiteName}`)}catch(m){throw c.fail("Upload failed."),m}}catch(n){console.error(me.default.red(`Upload failed: ${n instanceof Error?n.message:String(n)}`)),process.exit(1)}})}function Bt(e){try{let r=new URL(e).pathname.split("/").filter(Boolean),o=r[r.length-1];if(o)return o.replace(/\.git$/,"")}catch{}let t=e.match(/^[\w.-]+@[\w.-]+:(.+?)(?:\.git)?$/);if(t?.[1]){let n=t[1].split("/");return n[n.length-1]??null}return null}function Ut(){try{let e=(0,ot.execSync)("git remote get-url origin",{encoding:"utf-8"}).trim(),t=Bt(e);if(t)return t}catch{}return(0,nt.basename)(process.cwd())}var ue=require("fs"),fe=require("path"),ct=require("child_process"),x=$(require("chalk"),1),T=require("@kindlm/core");var st=require("http"),it=require("@kindlm/core");function at(e){let t=[],n=null,r=[];function o(){for(let i of r)i()}function a(i,s){if(s.setHeader("Access-Control-Allow-Origin","*"),s.setHeader("Access-Control-Allow-Methods","POST, OPTIONS"),s.setHeader("Access-Control-Allow-Headers","Content-Type"),i.method==="OPTIONS"){s.writeHead(204),s.end();return}if(i.method!=="POST"||i.url!=="/v1/traces"){s.writeHead(404,{"Content-Type":"application/json"}),s.end(JSON.stringify({error:"Not found"}));return}let c=[];i.on("data",m=>c.push(m)),i.on("end",()=>{try{let m=Buffer.concat(c).toString("utf-8"),u=JSON.parse(m),f=(0,it.parseOtlpPayload)(u);f.success?(t.push(...f.data),o(),s.writeHead(200,{"Content-Type":"application/json"}),s.end(JSON.stringify({partialSuccess:{}}))):(s.writeHead(400,{"Content-Type":"application/json"}),s.end(JSON.stringify({error:f.error.message})))}catch{s.writeHead(400,{"Content-Type":"application/json"}),s.end(JSON.stringify({error:"Invalid JSON"}))}})}return{start(){return new Promise((i,s)=>{n=(0,st.createServer)(a),n.on("error",s),n.listen(e,()=>i())})},stop(){return new Promise(i=>{n?n.close(()=>i()):i()})},getSpans(){return[...t]},waitForSpans({timeoutMs:i}){return new Promise(s=>{if(t.length>0){s([...t]);return}let c=setTimeout(()=>{r=r.filter(u=>u!==m),s([...t])},i),m=()=>{clearTimeout(c),r=r.filter(u=>u!==m),setTimeout(()=>s([...t]),500)};r.push(m)})}}}function lt(e){e.command("trace").description("Ingest OpenTelemetry traces and run assertions against them").option("-c, --config <path>","Config file path","kindlm.yaml").option("--port <port>","OTLP HTTP port","4318").option("--command <cmd>","Command to spawn (traces are collected while it runs)").option("--timeout <ms>","Timeout in ms to wait for traces","30000").option("--reporter <type>","Report format: pretty, json, junit","pretty").action(async t=>{let n=G();try{let r=(0,fe.resolve)(process.cwd(),t.config),o=(0,fe.dirname)(r);try{(0,ue.statSync)(r).size>1048576&&(console.error(x.default.red("Config file exceeds 1MB limit")),process.exit(1))}catch{console.error(x.default.red(`Config file not found: ${r}`)),process.exit(1)}let a;try{a=(0,ue.readFileSync)(r,"utf-8")}catch{console.error(x.default.red(`Config file not found: ${r}`)),process.exit(1)}let i=H(),s=(0,T.parseConfig)(a,{configDir:o,fileReader:i});s.success||(console.error(x.default.red(`Config validation failed: ${s.error.message}`)),process.exit(1));let c=s.data,m=c.trace??{port:parseInt(t.port,10),timeoutMs:parseInt(t.timeout,10),spanMapping:{outputTextAttr:"gen_ai.completion.0.content",modelAttr:"gen_ai.response.model",systemAttr:"gen_ai.system",inputTokensAttr:"gen_ai.usage.input_tokens",outputTokensAttr:"gen_ai.usage.output_tokens"}},u=parseInt(t.port,10)||m.port,f=parseInt(t.timeout,10)||m.timeoutMs,y=at(u);await y.start(),n.start(`Listening for OTLP traces on port ${u}...`),t.command&&(0,ct.spawn)("sh",["-c",t.command],{cwd:o,env:{...process.env,OTEL_EXPORTER_OTLP_ENDPOINT:`http://localhost:${u}`,OTEL_EXPORTER_OTLP_PROTOCOL:"http/json"},stdio:"inherit"}).on("error",R=>{n.fail(`Command failed: ${R.message}`)});let d=await y.waitForSpans({timeoutMs:f});await y.stop(),d.length===0&&(n.fail("No traces received"),process.exit(1)),n.succeed(`Received ${d.length} spans`);let C=(0,T.filterSpans)(d,m.spanFilter),b=(0,T.mapSpansToResult)(C,m.spanMapping),h=re(),l=new Map,g=c.providers;for(let[v,R]of Object.entries(g)){if(!R)continue;let M="";if(R.apiKeyEnv){let N=process.env[R.apiKeyEnv];N&&(M=N.trim())}if(!(!M&&v!=="ollama"))try{let N=(0,T.createProvider)(v,h);await N.initialize({apiKey:M,baseUrl:R.baseUrl,organization:R.organization,timeoutMs:c.defaults.timeoutMs,maxRetries:2}),l.set(v,N)}catch{}}let I=c.defaults.judgeModel??c.models[0]?.id,U=c.models.find(v=>v.id===I),X=U?l.get(U.provider):void 0,F=(0,T.buildContextFromTrace)(b,{configDir:o,judgeAdapter:X,judgeModel:U?.model}),K=[];for(let v of c.tests){if(v.skip)continue;let R=(0,T.createAssertionsFromExpect)(v.expect),M=[];for(let N of R){let L=await N.evaluate(F);M.push(...L)}K.push({testName:v.name,assertions:M})}let Ce=K.reduce((v,R)=>v+R.assertions.length,0),ve=K.reduce((v,R)=>v+R.assertions.filter(M=>M.passed).length,0),Re=Ce-ve;console.log(),console.log(x.default.bold("Trace Test Results")),console.log(x.default.dim("\u2500".repeat(50)));for(let{testName:v,assertions:R}of K){let N=R.every(L=>L.passed)?x.default.green("\u2713"):x.default.red("\u2717");console.log(`${N} ${v}`);for(let L of R){let mt=L.passed?x.default.green("  \u2713"):x.default.red("  \u2717"),ut=L.failureMessage?`${L.label}: ${L.failureMessage}`:L.label;console.log(`${mt} ${ut}`)}}console.log(),console.log(`${x.default.bold("Total:")} ${ve} passed, ${Re} failed out of ${Ce} assertions`);let we=(0,T.evaluateGates)(c.gates,[]);if(!we.passed)for(let v of we.gates.filter(R=>!R.passed))console.log(x.default.red(`Gate failed: ${v.message}`));process.exit(Re>0?1:0)}catch(r){n.fail(`Trace command failed: ${r instanceof Error?r.message:String(r)}`),process.exit(1)}})}function Kt(){let e=new dt.Command;return e.name("kindlm").description("AI agent behavioral regression testing").version("0.0.0"),be(e),Ie(e),ze(e),We(e),Ze(e),rt(e),lt(e),e}0&&(module.exports={createProgram});
+`);let a=o.trim();if(!a){t(new Error("No token provided"));return}e(a)})})}var nt=require("path"),ot=require("child_process"),me=$(require("chalk"),1);var de=require("child_process");function Qe(){try{let e=(0,de.execSync)("git rev-parse HEAD",{encoding:"utf-8"}).trim()||null,t=(0,de.execSync)("git rev-parse --abbrev-ref HEAD",{encoding:"utf-8"}).trim()||null,r=(0,de.execSync)("git status --porcelain",{encoding:"utf-8"}).trim().length>0;return{commitSha:e,branch:t,dirty:r}}catch{return{commitSha:null,branch:null,dirty:!1}}}function et(){return process.env.GITHUB_ACTIONS?{name:"github_actions",isCI:!0,commitSha:process.env.GITHUB_SHA??null,branch:process.env.GITHUB_REF_NAME??null}:process.env.GITLAB_CI?{name:"gitlab_ci",isCI:!0,commitSha:process.env.CI_COMMIT_SHA??null,branch:process.env.CI_COMMIT_BRANCH??null}:process.env.CI?{name:null,isCI:!0,commitSha:null,branch:null}:{name:null,isCI:!1,commitSha:null,branch:null}}function ee(e){return encodeURIComponent(e)}async function tt(e,t,n){let r=await jt(e,n.projectName),o=await Ft(e,r,n.suiteName,n.configHash),a=await e.post(`/v1/runs/${ee(r)}/runs`,{suiteId:o,commitSha:n.commitSha,branch:n.branch,environment:n.environment,triggeredBy:n.triggeredBy}),i=Ht(t.aggregated),s=50;for(let l=0;l<i.length;l+=s){let g=i.slice(l,l+s);await e.post(`/v1/results/${ee(a.id)}/results`,{results:g})}let{runResult:c}=t,m=c.totalTests>0?c.passed/c.totalTests:0,u=new Set(t.aggregated.map(l=>l.modelId)),f=t.aggregated.map(l=>l.assertionScores.judge?.mean).filter(l=>l!==void 0),y=f.length>0?f.reduce((l,g)=>l+g,0)/f.length:void 0,d=t.aggregated.map(l=>l.latencyAvgMs),C=d.length>0?d.reduce((l,g)=>l+g,0)/d.length:void 0,b=t.aggregated.reduce((l,g)=>l+g.totalCostUsd,0),h=b>0?b:void 0;return await e.patch(`/v1/runs/${ee(a.id)}`,{status:"completed",passRate:m,testCount:c.totalTests,modelCount:u.size,judgeAvgScore:y,latencyAvgMs:C,costEstimateUsd:h,finishedAt:new Date().toISOString()}),{runId:a.id,projectId:r}}async function jt(e,t){let{projects:n}=await e.get("/v1/projects"),r=n.find(a=>a.name===t);return r?r.id:(await e.post("/v1/projects",{name:t})).id}async function Ft(e,t,n,r){let{suites:o}=await e.get(`/v1/suites/${ee(t)}/suites`),a=o.find(s=>s.name===n);return a?a.id:(await e.post(`/v1/suites/${ee(t)}/suites`,{name:n,configHash:r})).id}function Ht(e){return e.map(t=>{let n=t,r=t.runs.length>0?t.runs.flatMap(o=>o.assertions.filter(a=>!a.passed).map(a=>a.failureMessage)).filter(o=>o!==void 0):n.failureMessages??[];return{testCaseName:t.testCaseName,modelId:t.modelId,passed:t.passed?1:0,passRate:t.passRate,runCount:t.runCount,judgeAvg:t.assertionScores.judge?.mean??null,driftScore:t.assertionScores.drift?.mean??null,latencyAvgMs:t.latencyAvgMs??null,costUsd:t.totalCostUsd??null,totalTokens:t.totalTokens??null,failureCodes:t.failureCodes.length>0?JSON.stringify(t.failureCodes):null,failureMessages:r.length>0?JSON.stringify(r):null,assertionScores:Object.keys(t.assertionScores).length>0?JSON.stringify(t.assertionScores):null}})}function rt(e){e.command("upload").description("Push last run results to KindLM Cloud").option("-t, --token <token>","API token (overrides stored token)").option("-p, --project <name>","Project name").action(async t=>{try{let n=t.token??process.env.KINDLM_API_TOKEN??le();n||(console.error(me.default.red('Not authenticated. Run "kindlm login" first or pass --token.')),process.exit(1));let r=Le();r||(console.error(me.default.red('No test run found. Run "kindlm test" first.')),process.exit(1));let o=Qe(),a=et(),i=t.project??Ut(),s=Q(Y(),n),c=G();c.start("Uploading results to KindLM Cloud...");try{let m=await tt(s,r.runnerResult,{projectName:i,suiteName:r.suiteName,configHash:r.configHash,commitSha:a.commitSha??o.commitSha??void 0,branch:a.branch??o.branch??void 0,environment:a.isCI?"ci":"local",triggeredBy:a.name??"local"});c.succeed("Uploaded successfully."),console.log(`  Run ID:  ${m.runId}`),console.log(`  Project: ${i}`),console.log(`  Suite:   ${r.suiteName}`)}catch(m){throw c.fail("Upload failed."),m}}catch(n){console.error(me.default.red(`Upload failed: ${n instanceof Error?n.message:String(n)}`)),process.exit(1)}})}function Bt(e){try{let r=new URL(e).pathname.split("/").filter(Boolean),o=r[r.length-1];if(o)return o.replace(/\.git$/,"")}catch{}let t=e.match(/^[\w.-]+@[\w.-]+:(.+?)(?:\.git)?$/);if(t?.[1]){let n=t[1].split("/");return n[n.length-1]??null}return null}function Ut(){try{let e=(0,ot.execSync)("git remote get-url origin",{encoding:"utf-8"}).trim(),t=Bt(e);if(t)return t}catch{}return(0,nt.basename)(process.cwd())}var ue=require("fs"),fe=require("path"),ct=require("child_process"),x=$(require("chalk"),1),T=require("@kindlm/core");var st=require("http"),it=require("@kindlm/core");function at(e){let t=[],n=null,r=[];function o(){for(let i of r)i()}function a(i,s){if(s.setHeader("Access-Control-Allow-Origin","*"),s.setHeader("Access-Control-Allow-Methods","POST, OPTIONS"),s.setHeader("Access-Control-Allow-Headers","Content-Type"),i.method==="OPTIONS"){s.writeHead(204),s.end();return}if(i.method!=="POST"||i.url!=="/v1/traces"){s.writeHead(404,{"Content-Type":"application/json"}),s.end(JSON.stringify({error:"Not found"}));return}let c=[];i.on("data",m=>c.push(m)),i.on("end",()=>{try{let m=Buffer.concat(c).toString("utf-8"),u=JSON.parse(m),f=(0,it.parseOtlpPayload)(u);f.success?(t.push(...f.data),o(),s.writeHead(200,{"Content-Type":"application/json"}),s.end(JSON.stringify({partialSuccess:{}}))):(s.writeHead(400,{"Content-Type":"application/json"}),s.end(JSON.stringify({error:f.error.message})))}catch{s.writeHead(400,{"Content-Type":"application/json"}),s.end(JSON.stringify({error:"Invalid JSON"}))}})}return{start(){return new Promise((i,s)=>{n=(0,st.createServer)(a),n.on("error",s),n.listen(e,()=>i())})},stop(){return new Promise(i=>{n?n.close(()=>i()):i()})},getSpans(){return[...t]},waitForSpans({timeoutMs:i}){return new Promise(s=>{if(t.length>0){s([...t]);return}let c=setTimeout(()=>{r=r.filter(u=>u!==m),s([...t])},i),m=()=>{clearTimeout(c),r=r.filter(u=>u!==m),setTimeout(()=>s([...t]),500)};r.push(m)})}}}function lt(e){e.command("trace").description("Ingest OpenTelemetry traces and run assertions against them").option("-c, --config <path>","Config file path","kindlm.yaml").option("--port <port>","OTLP HTTP port","4318").option("--command <cmd>","Command to spawn (traces are collected while it runs)").option("--timeout <ms>","Timeout in ms to wait for traces","30000").option("--reporter <type>","Report format: pretty, json, junit","pretty").action(async t=>{let n=G();try{let r=(0,fe.resolve)(process.cwd(),t.config),o=(0,fe.dirname)(r);try{(0,ue.statSync)(r).size>1048576&&(console.error(x.default.red("Config file exceeds 1MB limit")),process.exit(1))}catch{console.error(x.default.red(`Config file not found: ${r}`)),process.exit(1)}let a;try{a=(0,ue.readFileSync)(r,"utf-8")}catch{console.error(x.default.red(`Config file not found: ${r}`)),process.exit(1)}let i=H(),s=(0,T.parseConfig)(a,{configDir:o,fileReader:i});s.success||(console.error(x.default.red(`Config validation failed: ${s.error.message}`)),process.exit(1));let c=s.data,m=c.trace??{port:parseInt(t.port,10),timeoutMs:parseInt(t.timeout,10),spanMapping:{outputTextAttr:"gen_ai.completion.0.content",modelAttr:"gen_ai.response.model",systemAttr:"gen_ai.system",inputTokensAttr:"gen_ai.usage.input_tokens",outputTokensAttr:"gen_ai.usage.output_tokens"}},u=parseInt(t.port,10)||m.port,f=parseInt(t.timeout,10)||m.timeoutMs,y=at(u);await y.start(),n.start(`Listening for OTLP traces on port ${u}...`),t.command&&(0,ct.spawn)("sh",["-c",t.command],{cwd:o,env:{...process.env,OTEL_EXPORTER_OTLP_ENDPOINT:`http://localhost:${u}`,OTEL_EXPORTER_OTLP_PROTOCOL:"http/json"},stdio:"inherit"}).on("error",R=>{n.fail(`Command failed: ${R.message}`)});let d=await y.waitForSpans({timeoutMs:f});await y.stop(),d.length===0&&(n.fail("No traces received"),process.exit(1)),n.succeed(`Received ${d.length} spans`);let C=(0,T.filterSpans)(d,m.spanFilter),b=(0,T.mapSpansToResult)(C,m.spanMapping),h=re(),l=new Map,g=c.providers;for(let[v,R]of Object.entries(g)){if(!R)continue;let M="";if(R.apiKeyEnv){let N=process.env[R.apiKeyEnv];N&&(M=N.trim())}if(!(!M&&v!=="ollama"))try{let N=(0,T.createProvider)(v,h);await N.initialize({apiKey:M,baseUrl:R.baseUrl,organization:R.organization,timeoutMs:c.defaults.timeoutMs,maxRetries:2}),l.set(v,N)}catch{}}let I=c.defaults.judgeModel??c.models[0]?.id,U=c.models.find(v=>v.id===I),X=U?l.get(U.provider):void 0,F=(0,T.buildContextFromTrace)(b,{configDir:o,judgeAdapter:X,judgeModel:U?.model}),K=[];for(let v of c.tests){if(v.skip)continue;let R=(0,T.createAssertionsFromExpect)(v.expect),M=[];for(let N of R){let L=await N.evaluate(F);M.push(...L)}K.push({testName:v.name,assertions:M})}let Ce=K.reduce((v,R)=>v+R.assertions.length,0),ve=K.reduce((v,R)=>v+R.assertions.filter(M=>M.passed).length,0),Re=Ce-ve;console.log(),console.log(x.default.bold("Trace Test Results")),console.log(x.default.dim("\u2500".repeat(50)));for(let{testName:v,assertions:R}of K){let N=R.every(L=>L.passed)?x.default.green("\u2713"):x.default.red("\u2717");console.log(`${N} ${v}`);for(let L of R){let mt=L.passed?x.default.green("  \u2713"):x.default.red("  \u2717"),ut=L.failureMessage?`${L.label}: ${L.failureMessage}`:L.label;console.log(`${mt} ${ut}`)}}console.log(),console.log(`${x.default.bold("Total:")} ${ve} passed, ${Re} failed out of ${Ce} assertions`);let we=(0,T.evaluateGates)(c.gates,[]);if(!we.passed)for(let v of we.gates.filter(R=>!R.passed))console.log(x.default.red(`Gate failed: ${v.message}`));process.exit(Re>0?1:0)}catch(r){n.fail(`Trace command failed: ${r instanceof Error?r.message:String(r)}`),process.exit(1)}})}function Kt(){let e=new dt.Command;return e.name("kindlm").description("AI agent behavioral regression testing").version("0.4.0"),be(e),Ie(e),ze(e),We(e),Ze(e),rt(e),lt(e),e}0&&(module.exports={createProgram});
 //# sourceMappingURL=index.cjs.map