@kindlm/cli 0.4.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +63 -29
- package/dist/index.cjs +8 -8
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +8 -8
- package/dist/index.js.map +1 -1
- package/dist/kindlm.js +9 -9
- package/dist/kindlm.js.map +1 -1
- package/package.json +4 -4
package/dist/kindlm.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import{Command as
|
|
2
|
+
import{Command as wn}from"commander";import{existsSync as Be,writeFileSync as Ue}from"fs";import{resolve as Ke}from"path";import A from"chalk";var ze=`kindlm: 1
|
|
3
3
|
project: my-project
|
|
4
4
|
|
|
5
5
|
suite:
|
|
@@ -43,15 +43,15 @@ defaults:
|
|
|
43
43
|
repeat: 1
|
|
44
44
|
concurrency: 4
|
|
45
45
|
timeoutMs: 60000
|
|
46
|
-
`;function
|
|
47
|
-
Interrupted. Exiting...`)),process.exit(130)};process.on("SIGINT",
|
|
48
|
-
`),
|
|
49
|
-
`).trim()}),
|
|
50
|
-
`).trim()}),
|
|
51
|
-
`),y=!1,d=0;for(;d<f.length;){let C=f[d]??"";if(C.startsWith("```")){y=!y,y&&N(r,30),d++;continue}if(y){N(r,14);let l=r.y;r.save(),r.rect(r.page.margins.left,l-2,i,14).fill("#f5f5f4"),r.restore(),r.fontSize(9).font("Courier").fillColor("#44403c").text(C,{width:i}),d++;continue}if(!C.trim()){r.moveDown(.4),d++;continue}let S=f[d+1]??"";if(C.includes("|")&&d+1<f.length&&S.match(/^\s*\|[-:\s|]+\|\s*$/)){let l=[],p=d;for(;p<f.length&&(f[p]??"").includes("|");)l.push(f[p]??""),p++;let b=gt(l);if(b){yt(r,b,i),d=p;continue}}if(C.match(/^\s*\|[-:]+/)||C.match(/^---+$/)){d++;continue}let h=C.match(/^(#{3,4})\s+(.+)$/);if(h?.[1]&&h[2]){let l=h[1].length,p=pe(l);N(r,p+10),r.moveDown(.3),r.fontSize(p).font("Helvetica-Bold").fillColor("#1c1917").text(h[2].trim(),{width:i}),r.moveDown(.3),d++;continue}if(C.match(/^\s*[-*] /)){N(r,14),r.fontSize(10).font("Helvetica").fillColor("#44403c").text(C.trim(),{indent:12,width:i-12}),d++;continue}N(r,14),r.fontSize(10).font("Helvetica").fillColor("#44403c").text(C.trim(),{width:i}),d++}ht(r)}r.end(),a.on("finish",()=>n(t)),a.on("error",o)})}function ye(e){e.command("test").description("Run test suites").option("-s, --suite <name>","Run a specific suite").option("--compliance","Generate compliance report").option("--reporter <type>","Output format: pretty, json, junit","pretty").option("--runs <count>","Override run count").option("--gate <percent>","Fail if pass rate below threshold").option("--pdf <path>","Export compliance report as PDF (requires --compliance)").option("-c, --config <path>","Path to config file","kindlm.yaml").action(async t=>{try{let{runnerResult:n,config:o,yamlContent:r}=await B({configPath:t.config,runs:t.runs?parseInt(t.runs,10):void 0,gate:t.gate?parseFloat(t.gate):void 0}),{runResult:a,aggregated:i}=n,s=Ct(o.gates,i),m=xt(t.reporter).generate(a,s);if(console.log(m.content),t.compliance){let y=St().generate(a,s);if(console.log(""),console.log(y.content),t.pdf){let d=await he(y.content,t.pdf);console.log(""),console.log(w.green(`PDF report saved to ${d}`))}}try{me({runnerResult:n,suiteName:o.suite.name,configHash:fe(r),timestamp:new Date().toISOString()})}catch{}let u=a.failed===0&&a.errored===0&&s.passed;process.exit(u?0:1)}catch(n){if(n instanceof Tt){let o=n.code==="TIMEOUT"?"Provider timeout":n.code==="NETWORK_ERROR"?"Network error":n.code==="AUTH_FAILED"?"Authentication failed":n.code==="RATE_LIMITED"?"Rate limited":`Provider error (${n.code})`;console.error(w.red(`${o}: ${n.message}`)),n.retryable&&console.error(w.yellow("This error may be transient. Try again or increase --timeout."))}else if(kt(n)){let r=n.code.startsWith("CONFIG_")?"Config error":"Error";console.error(w.red(`${r}: ${n.message}`))}else n instanceof Error&&n.name==="AbortError"?console.error(w.red("Request timed out. Check network connectivity or increase timeout.")):console.error(w.red(`Error: ${n instanceof Error?n.message:String(n)}`));process.exit(1)}})}var bt={bold:e=>w.bold(e),red:e=>w.red(e),green:e=>w.green(e),yellow:e=>w.yellow(e),cyan:e=>w.cyan(e),dim:e=>w.dim(e),greenBold:e=>w.green.bold(e),redBold:e=>w.red.bold(e)};function xt(e){switch(e){case"json":return Rt();case"junit":return wt();default:return vt(bt)}}function kt(e){return typeof e=="object"&&e!==null&&"code"in e&&"message"in e&&typeof e.code=="string"&&typeof e.message=="string"}import{resolve as W,dirname as q,join as V}from"path";import{readFileSync as It}from"fs";import g from"chalk";import{parseConfig as Ot,readBaseline as At,writeBaseline as Mt,listBaselines as Nt,buildBaselineData as Re,compareBaseline as Lt,deserializeBaseline as _t}from"@kindlm/core";import{readFileSync as Pt,writeFileSync as $t,mkdirSync as Ce,readdirSync as Et}from"fs";import{join as X}from"path";function ve(e){return e.replace(/[^a-zA-Z0-9_-]/g,"_")}function J(e){let t=X(e,"baselines");return{read(n){let o=X(t,`${ve(n)}.json`);try{return{success:!0,data:Pt(o,"utf-8")}}catch{return{success:!1,error:{code:"BASELINE_NOT_FOUND",message:`No baseline found for suite "${n}" at ${o}`}}}},write(n,o){try{Ce(t,{recursive:!0});let r=X(t,`${ve(n)}.json`);return $t(r,o,"utf-8"),{success:!0,data:void 0}}catch(r){return{success:!1,error:{code:"UNKNOWN_ERROR",message:`Failed to write baseline: ${r instanceof Error?r.message:String(r)}`}}}},list(){try{return Ce(t,{recursive:!0}),{success:!0,data:Et(t).filter(r=>r.endsWith(".json")).map(r=>r.replace(/\.json$/,""))}}catch(n){return{success:!1,error:{code:"UNKNOWN_ERROR",message:`Failed to list baselines: ${n instanceof Error?n.message:String(n)}`}}}}}}function we(e){let t=e.command("baseline").description("Manage test baselines");t.command("set").description("Save current results as baseline").option("-c, --config <path>","Path to config file","kindlm.yaml").option("--runs <count>","Override run count").action(async n=>{try{let o=q(W(process.cwd(),n.config)),r=V(o,".kindlm"),a=J(r),{config:i,runnerResult:s}=await B({configPath:n.config,runs:n.runs?parseInt(n.runs,10):void 0}),{aggregated:c}=s,m=Re(i.suite.name,c,new Date().toISOString()),u=Mt(m,a);u.success||(console.error(g.red(`Failed to save baseline: ${u.error.message}`)),process.exit(1));let f=Object.keys(m.results).length;console.log(""),console.log(g.green(`Baseline saved for suite "${i.suite.name}" (${f} test${f===1?"":"s"})`)),console.log(g.dim(` Location: ${r}/baselines/`)),process.exit(0)}catch(o){console.error(g.red(`Error: ${o instanceof Error?o.message:String(o)}`)),process.exit(1)}}),t.command("compare").description("Compare latest against baseline").option("-c, --config <path>","Path to config file","kindlm.yaml").option("--runs <count>","Override run count").action(async n=>{try{let o=q(W(process.cwd(),n.config)),r=V(o,".kindlm"),a=J(r),i=W(process.cwd(),n.config),s;try{s=It(i,"utf-8")}catch{console.error(g.red(`Config file not found: ${i}`)),process.exit(1)}let c=A(),m=Ot(s,{configDir:o,fileReader:c});m.success||(console.error(g.red(`Config validation failed: ${m.error.message}`)),process.exit(1));let u=m.data.suite.name,f=At(u,a);f.success||(f.error.code==="BASELINE_NOT_FOUND"?console.error(g.red(`No baseline found for suite "${u}". Run \`kindlm baseline set\` first.`)):console.error(g.red(`Failed to read baseline: ${f.error.message}`)),process.exit(1));let y=f.data,{runnerResult:d}=await B({configPath:n.config,runs:n.runs?parseInt(n.runs,10):void 0,baselineData:y}),{aggregated:C}=d,S=Re(u,C,new Date().toISOString()),h=Lt(y,S.results);if(console.log(""),console.log(g.bold(`Baseline comparison for "${u}"`)),console.log(g.dim(` Baseline from: ${y.createdAt}`)),console.log(""),h.regressions.length>0){console.log(g.red.bold(` Regressions (${h.regressions.length}):`));for(let l of h.regressions)console.log(g.red(` ${l.testName}: ${U(l.baselinePassRate)} \u2192 ${U(l.currentPassRate)}`)),l.newFailureCodes.length>0&&console.log(g.red(` New failures: ${l.newFailureCodes.join(", ")}`));console.log("")}if(h.improvements.length>0){console.log(g.green.bold(` Improvements (${h.improvements.length}):`));for(let l of h.improvements)console.log(g.green(` ${l.testName}: ${U(l.baselinePassRate)} \u2192 ${U(l.currentPassRate)}`));console.log("")}if(h.unchanged.length>0){console.log(g.dim(` Unchanged (${h.unchanged.length}):`));for(let l of h.unchanged)console.log(g.dim(` ${l.testName}: ${U(l.passRate)}`));console.log("")}if(h.newTests.length>0){console.log(g.cyan(` New tests (${h.newTests.length}):`));for(let l of h.newTests)console.log(g.cyan(` ${l}`));console.log("")}if(h.removedTests.length>0){console.log(g.yellow(` Removed tests (${h.removedTests.length}):`));for(let l of h.removedTests)console.log(g.yellow(` ${l}`));console.log("")}process.exit(h.regressions.length>0?1:0)}catch(o){console.error(g.red(`Error: ${o instanceof Error?o.message:String(o)}`)),process.exit(1)}}),t.command("list").description("List saved baselines").option("-c, --config <path>","Path to config file","kindlm.yaml").action(n=>{try{let o=q(W(process.cwd(),n.config)),r=V(o,".kindlm"),a=J(r),i=Nt(a);i.success||(console.error(g.red(`Failed to list baselines: ${i.error.message}`)),process.exit(1));let s=i.data;s.length===0&&(console.log(g.dim("No baselines saved yet. Run `kindlm baseline set` to create one.")),process.exit(0)),console.log(g.bold("Saved baselines:")),console.log("");for(let c of s){let m=a.read(c);if(!m.success){console.log(` ${c} ${g.dim("(unreadable)")}`);continue}let u=_t(m.data);if(!u.success){console.log(` ${c} ${g.dim("(corrupt)")}`);continue}let f=Object.keys(u.data.results).length;console.log(` ${g.cyan(u.data.suiteName)} \u2014 ${f} test${f===1?"":"s"}, saved ${g.dim(u.data.createdAt)}`)}process.exit(0)}catch(o){console.error(g.red(`Error: ${o instanceof Error?o.message:String(o)}`)),process.exit(1)}})}function U(e){return`${(e*100).toFixed(1)}%`}import{createInterface as Gt}from"readline";import{Writable as Jt}from"stream";import I from"chalk";import{readFileSync as Dt,writeFileSync as jt,mkdirSync as Ft,unlinkSync as Ht,chmodSync as Bt}from"fs";import{join as Se}from"path";import{homedir as Te}from"os";function Z(){return Se(Te(),".kindlm","credentials")}function Y(){try{let e=Dt(Z(),"utf-8"),t=JSON.parse(e);return typeof t.token=="string"&&t.token.length>0?t.token:null}catch{return null}}function be(e){let t=Z(),n=Se(Te(),".kindlm");Ft(n,{recursive:!0,mode:448});let o={token:e,savedAt:new Date().toISOString()};jt(t,JSON.stringify(o,null,2),{mode:384}),Bt(t,384)}function xe(){try{Ht(Z())}catch{}}var Ut="https://api.kindlm.com";var E=class extends Error{status;constructor(t,n){super(n),this.name="CloudApiError",this.status=t}};function j(){let e=process.env.KINDLM_CLOUD_URL??Ut;if(e.startsWith("http://")&&!Kt(e))throw new Error(`Refusing to use insecure HTTP for Cloud API: ${e}. Use HTTPS or target localhost for development.`);return e}function Kt(e){try{let t=new URL(e);return t.hostname==="localhost"||t.hostname==="127.0.0.1"||t.hostname==="::1"}catch{return!1}}function zt(e){return new Promise(t=>setTimeout(t,e))}function K(e,t){async function n(o,r,a){let i=`${e}${r}`,s={Authorization:`Bearer ${t}`},c={method:o,headers:s};a!==void 0&&(s["Content-Type"]="application/json",c.body=JSON.stringify(a));let m;for(let u=0;u<=1;u++){u>0&&await zt(1e3);let f=new AbortController,y=setTimeout(()=>f.abort(),3e4);c.signal=f.signal;try{let d=await fetch(i,c);if(!d.ok){if(d.status>=500&&u<1){m=new E(d.status,`HTTP ${d.status}`);continue}let S=`HTTP ${d.status}`;if((d.headers.get("content-type")??"").includes("application/json"))try{let l=await d.json();l.error&&(S=l.error)}catch{}throw new E(d.status,S)}if(d.status===204)return;let C=d.headers.get("content-type")??"";if(!C.includes("application/json"))throw new E(d.status,`Expected JSON response but got content-type: ${C}`);return await d.json()}catch(d){if(d instanceof E)throw d;if(m=d instanceof Error?d:new Error(String(d)),u<1)continue}finally{clearTimeout(y)}}throw m??new Error("Request failed")}return{baseUrl:e,get:o=>n("GET",o),post:(o,r)=>n("POST",o,r),patch:(o,r)=>n("PATCH",o,r),delete:o=>n("DELETE",o)}}function ke(e){e.command("login").description("Authenticate with KindLM Cloud").option("-t, --token <token>","API token (skips interactive prompt)").option("--status","Show current authentication status").option("--logout","Remove stored credentials").action(async t=>{try{if(t.logout){xe(),console.log(I.green("Logged out. Credentials removed."));return}if(t.status){await Wt();return}let n=t.token??process.env.KINDLM_API_TOKEN??await Yt();n.startsWith("klm_")||(console.error(I.red('Invalid token format. KindLM tokens start with "klm_".')),process.exit(1));let o=K(j(),n);try{await o.get("/v1/auth/tokens")}catch(r){throw r instanceof E&&r.status===401&&(console.error(I.red("Invalid or expired token.")),process.exit(1)),r}be(n),console.log(I.green("Authenticated successfully. Token saved."))}catch(n){console.error(I.red(`Login failed: ${n instanceof Error?n.message:String(n)}`)),process.exit(1)}})}async function Wt(){let e=Y();if(!e){console.log(I.yellow('Not authenticated. Run "kindlm login" to authenticate.'));return}let t=K(j(),e);try{await t.get("/v1/auth/tokens"),console.log(I.green("Authenticated.")),console.log(` Cloud URL: ${j()}`)}catch(n){n instanceof E&&n.status===401?console.log(I.yellow('Stored token is invalid or expired. Run "kindlm login" to re-authenticate.')):console.log(I.yellow(`Cannot reach Cloud API: ${n instanceof Error?n.message:String(n)}`))}}function Yt(){return new Promise((e,t)=>{let n=new Jt({write(r,a,i){i()}});process.stderr.write("Paste your KindLM API token: ");let o=Gt({input:process.stdin,output:n,terminal:!0});o.question("",r=>{o.close(),process.stderr.write(`
|
|
52
|
-
`);let a=r.trim();if(!a){t(new Error("No token provided"));return}e(a)})})}import{basename as
|
|
46
|
+
`;function ae(e){e.command("init").description("Create a kindlm.yaml template").option("--force","Overwrite existing kindlm.yaml").action(t=>{let o=Ke(process.cwd(),"kindlm.yaml");Be(o)&&!t.force&&(console.error(A.red("kindlm.yaml already exists. Use --force to overwrite.")),process.exit(1));try{Ue(o,ze,"utf-8")}catch(n){let r=n instanceof Error&&"code"in n?n.code:void 0;console.error(r==="EACCES"||r==="EROFS"?A.red("Cannot create kindlm.yaml: permission denied"):A.red(`Cannot create kindlm.yaml: ${n instanceof Error?n.message:String(n)}`)),process.exit(1)}console.log(A.green("Created kindlm.yaml")),console.log(""),console.log("Next steps:"),console.log(` 1. Edit ${A.bold("kindlm.yaml")} with your test configuration`),console.log(` 2. Set your API key: ${A.bold("export OPENAI_API_KEY=sk-...")}`),console.log(` 3. Run tests: ${A.bold("kindlm test")}`)})}import{readFileSync as Je}from"fs";import{resolve as We,dirname as Ve}from"path";import O from"chalk";import{parseConfig as Ye}from"@kindlm/core";import{readFileSync as Ge}from"fs";function $(){return{readFile(e){try{return{success:!0,data:Ge(e,"utf-8")}}catch(t){return{success:!1,error:{code:"CONFIG_FILE_REF_ERROR",message:`Cannot read file: ${e}: ${t instanceof Error?t.message:String(t)}`}}}}}}function ce(e){e.command("validate").description("Validate kindlm.yaml configuration").option("-c, --config <path>","Path to config file","kindlm.yaml").action(t=>{let o=We(process.cwd(),t.config),n=Ve(o),r;try{r=Je(o,"utf-8")}catch{console.error(O.red(`Config file not found: ${o}`)),process.exit(1)}let a=$(),s=Ye(r,{configDir:n,fileReader:a});if(!s.success){console.error(O.red("Validation failed:"));let c=s.error.details;if(c&&Array.isArray(c.errors))for(let f of c.errors)console.error(O.red(` - ${f}`));else console.error(O.red(` ${s.error.message}`));process.exit(1)}let i=s.data;console.log(O.green("Config is valid!")),console.log(""),console.log(` Suite: ${O.bold(i.suite.name)}`),console.log(` Tests: ${O.bold(String(i.tests.length))}`),console.log(` Models: ${O.bold(String(i.models.length))}`)})}import L from"chalk";import{evaluateGates as Pt,createComplianceReporter as $t,ProviderError as Ot}from"@kindlm/core";import{readFileSync as et,statSync as tt}from"fs";import{resolve as nt,dirname as rt}from"path";import v from"chalk";import{parseConfig as ot,createProvider as st,createRunner as it}from"@kindlm/core";import{ProviderError as Xe}from"@kindlm/core";function G(){return{async fetch(e,t){let o=new AbortController,n=t.timeoutMs?setTimeout(()=>o.abort(),t.timeoutMs):void 0;try{let r=await globalThis.fetch(e,{method:t.method,headers:t.headers,body:t.body,signal:o.signal});return{ok:r.ok,status:r.status,json:()=>r.json()}}catch(r){throw r instanceof DOMException&&r.name==="AbortError"||r instanceof Error&&r.name==="AbortError"?new Xe("TIMEOUT","Request timed out",408,!0):r}finally{n!==void 0&&clearTimeout(n)}}}}import qe from"ora";function _(){let e;return{start(t){e=qe({text:t,stream:process.stderr}).start()},succeed(t){e?.succeed(t),e=void 0},fail(t){e?.fail(t),e=void 0},stop(){e?.stop(),e=void 0}}}import{spawn as Ze}from"child_process";import{ok as Qe,err as le}from"@kindlm/core";function de(){return{async execute(e,t){return new Promise(o=>{let n=Ze("sh",["-c",e],{cwd:t.cwd,env:{...process.env,...t.env},stdio:["ignore","pipe","pipe"]}),r=[],a=[];n.stdout.on("data",i=>r.push(i)),n.stderr.on("data",i=>a.push(i));let s=setTimeout(()=>{n.kill("SIGTERM"),setTimeout(()=>{n.killed||n.kill("SIGKILL")},1e3)},t.timeoutMs);n.on("close",(i,c)=>{if(clearTimeout(s),c==="SIGTERM"||c==="SIGKILL"){o(le({code:"PROVIDER_TIMEOUT",message:`Command timed out after ${t.timeoutMs}ms`}));return}o(Qe({stdout:Buffer.concat(r).toString("utf-8"),stderr:Buffer.concat(a).toString("utf-8"),exitCode:i??1}))}),n.on("error",i=>{clearTimeout(s),o(le({code:"UNKNOWN_ERROR",message:`Failed to spawn command: ${i.message}`}))})})}}}var at=1048576;async function H(e){let t=_(),o=!1,n=()=>{o&&process.exit(130),o=!0,t.stop(),console.error(v.yellow(`
|
|
47
|
+
Interrupted. Exiting...`)),process.exit(130)};process.on("SIGINT",n);try{return await ct(e,t)}finally{process.removeListener("SIGINT",n)}}async function ct(e,t){let o=nt(process.cwd(),e.configPath),n=rt(o);try{let u=tt(o);u.size>at&&(console.error(v.red(`Config file exceeds 1MB limit (${(u.size/1048576).toFixed(1)}MB): ${o}`)),process.exit(1))}catch{console.error(v.red(`Config file not found: ${o}`)),process.exit(1)}let r;try{r=et(o,"utf-8")}catch{console.error(v.red(`Config file not found: ${o}`)),process.exit(1)}let a=$(),s=ot(r,{configDir:n,fileReader:a});if(!s.success){console.error(v.red("Config validation failed:"));let u=s.error.details;if(u&&Array.isArray(u.errors))for(let S of u.errors)console.error(v.red(` - ${S}`));else console.error(v.red(` ${s.error.message}`));process.exit(1)}let i=s.data;e.suite!==void 0&&i.suite.name!==e.suite&&(console.error(v.red(`Suite "${e.suite}" not found. Available suite: "${i.suite.name}"`)),process.exit(1)),e.runs!==void 0&&((!Number.isInteger(e.runs)||e.runs<1)&&(console.error(v.red(`Invalid --runs value: ${e.runs}. Must be a positive integer (>= 1).`)),process.exit(1)),i.defaults.repeat=e.runs),e.gate!==void 0&&((Number.isNaN(e.gate)||e.gate<0||e.gate>100)&&(console.error(v.red(`Invalid --gate value: ${e.gate}. Must be between 0 and 100.`)),process.exit(1)),e.gate>0&&e.gate<=1&&console.error(v.yellow(`Warning: --gate ${e.gate} looks like a decimal. Did you mean --gate ${Math.round(e.gate*100)}? (--gate uses 0-100 scale)`)),i.gates?i.gates.passRateMin=e.gate/100:i.gates={passRateMin:e.gate/100});let c=G(),f=new Map,m=i.providers;for(let[u,S]of Object.entries(m)){if(!S)continue;let U="";if(S.apiKeyEnv){let I=process.env[S.apiKeyEnv];I||(console.error(v.red(`Missing environment variable: ${S.apiKeyEnv}`)),process.exit(1)),U=I.trim()}else u!=="ollama"&&(console.error(v.red(`Provider "${u}" requires apiKeyEnv to be configured`)),process.exit(1));let M;try{M=st(u,c)}catch(I){let Z=I instanceof Error?I.message:String(I);console.error(v.red(`Failed to create provider "${u}": ${Z}`)),process.exit(1)}await M.initialize({apiKey:U,baseUrl:S.baseUrl,organization:S.organization,timeoutMs:i.defaults.timeoutMs,maxRetries:2}),f.set(u,M)}let p=0,g=lt(i),d=u=>{u.type==="test_start"?t.start(`Running ${u.test} [${u.model}] (${p}/${g})`):u.type==="test_complete"&&p++},w=i.tests.some(u=>u.command)?de():void 0,l=await it(i,{adapters:f,configDir:n,fileReader:a,onProgress:d,baselineData:e.baselineData,commandExecutor:w}).run();return t.stop(),l.success||(console.error(v.red(`Run failed: ${l.error.message}`)),process.exit(1)),{config:i,runnerResult:l.data,configDir:n,yamlContent:r}}function lt(e){let t=0;for(let o of e.tests){if(o.skip)continue;let n=o.repeat??e.defaults.repeat;if(o.command)t+=n;else{let r=o.models?.length??e.models.length;t+=r*n}}return t}import{readFileSync as dt,writeFileSync as mt,mkdirSync as ut}from"fs";import{join as me}from"path";import{createHash as ft}from"crypto";function ue(){return me(process.cwd(),".kindlm","last-run.json")}function fe(e){let t=ue(),o=me(process.cwd(),".kindlm");ut(o,{recursive:!0,mode:448});let n={...e,runnerResult:{...e.runnerResult,aggregated:e.runnerResult.aggregated.map(r=>{let a=r.runs.flatMap(s=>s.assertions.filter(i=>!i.passed).map(i=>i.failureMessage)).filter(s=>s!==void 0);return{...r,failureMessages:a,runs:[]}})}};mt(t,JSON.stringify(n),{mode:384})}function pe(){try{let e=dt(ue(),"utf-8"),t=JSON.parse(e);return t.runnerResult?.runResult&&Array.isArray(t.runnerResult.aggregated)&&typeof t.suiteName=="string"&&typeof t.configHash=="string"&&typeof t.timestamp=="string"?t:null}catch{return null}}function ge(e){return ft("sha256").update(e).digest("hex")}import pt from"pdfkit";import{createWriteStream as gt}from"fs";import{mkdir as ht}from"fs/promises";import{dirname as yt}from"path";function Rt(e){let t=e.split(`
|
|
48
|
+
`),o=[],n="",r=2,a=[];for(let s of t){let i=s.match(/^(#{2,4})\s+(.+)$/);i&&i[1]?.length===2?((n||a.length>0)&&o.push({heading:n,headingLevel:r,body:a.join(`
|
|
49
|
+
`).trim()}),n=i[2]?.trim()??"",r=2,a=[]):a.push(s)}return(n||a.length>0)&&o.push({heading:n,headingLevel:r,body:a.join(`
|
|
50
|
+
`).trim()}),o}function Ct(e){return e.match(/^# (.+)$/m)?.[1]?.trim()??"KindLM Compliance Report"}function vt(e){return e.match(/SHA-256:\s*`([a-f0-9]+)`/i)?.[1]??null}function he(e){switch(e){case 2:return 18;case 3:return 15;case 4:return 13;default:return 13}}function wt(e){if(e.length<2)return null;let t=e[0]??"",o=e[1]??"";if(!t.includes("|")||!o.match(/^\s*\|[-:\s|]+\|\s*$/))return null;let n=s=>s.split("|").slice(1,-1).map(i=>i.trim()),r={cells:n(t)},a=[];for(let s=2;s<e.length;s++){let i=e[s]??"";if(!i.includes("|"))break;a.push({cells:n(i)})}return{header:r,rows:a}}function N(e,t){let o=e.page.margins.bottom;e.page.height-o-30-e.y<t&&(e.addPage(),ye(e))}function ye(e){let t=new Date().toISOString();e.fontSize(8).font("Helvetica").fillColor("#a8a29e").text("KindLM Compliance Report",60,40),e.text(t,60,40,{align:"right"}),e.moveDown(3)}function St(e){e.fontSize(8).font("Helvetica").fillColor("#a8a29e").text("Generated by KindLM \xB7 kindlm.com",60,e.page.height-50,{align:"center",width:e.page.width-120})}function Tt(e,t,o){let n=t.header.cells.length,r=o/n,a=e.page.margins.left,s=18;N(e,s*2);let i=(c,f,m)=>{let p=e.y;m&&(e.save(),e.rect(a,p-2,o,s).fill(m),e.restore());for(let g=0;g<c.length;g++){let d=a+g*r;e.fontSize(8).font(f?"Helvetica-Bold":"Courier").fillColor("#44403c").text(c[g]??"",d+4,p,{width:r-8,height:s,lineBreak:!1})}e.y=p+s};i(t.header.cells,!0,"#f5f5f4");for(let c of t.rows)N(e,s),i(c.cells,!1)}async function Re(e,t){return await ht(yt(t),{recursive:!0}),new Promise((o,n)=>{let r=new pt({size:"A4",margins:{top:72,bottom:72,left:60,right:60},info:{Title:"KindLM EU AI Act Compliance Report",Author:"KindLM",Creator:"KindLM CLI"}}),a=gt(t);r.pipe(a);let s=r.page.width-r.page.margins.left-r.page.margins.right,i=Ct(e),c=vt(e);r.moveDown(6),r.fontSize(28).font("Helvetica-Bold").fillColor("#1c1917").text(i,{align:"center",width:s}),r.moveDown(.5),r.fontSize(14).font("Helvetica").fillColor("#57534e").text("EU AI Act Annex IV Documentation",{align:"center",width:s}),r.moveDown(1),r.fontSize(10).fillColor("#a8a29e").text(`Generated: ${new Date().toISOString()}`,{align:"center",width:s}),c&&(r.moveDown(.3),r.fontSize(9).font("Courier").fillColor("#78716c").text(`SHA-256: ${c}`,{align:"center",width:s})),r.moveDown(2),r.fontSize(10).font("Helvetica").fillColor("#6366f1").text("kindlm.com",{align:"center",link:"https://kindlm.com",width:s});let f=Rt(e);for(let m of f){if(r.addPage(),ye(r),m.heading){let y=he(m.headingLevel);r.fontSize(y).font("Helvetica-Bold").fillColor("#1c1917").text(m.heading,{width:s}),r.moveDown(.5),r.moveTo(60,r.y).lineTo(60+s,r.y).strokeColor("#e7e5e4").lineWidth(1).stroke(),r.moveDown(.8)}let p=m.body.split(`
|
|
51
|
+
`),g=!1,d=0;for(;d<p.length;){let y=p[d]??"";if(y.startsWith("```")){g=!g,g&&N(r,30),d++;continue}if(g){N(r,14);let l=r.y;r.save(),r.rect(r.page.margins.left,l-2,s,14).fill("#f5f5f4"),r.restore(),r.fontSize(9).font("Courier").fillColor("#44403c").text(y,{width:s}),d++;continue}if(!y.trim()){r.moveDown(.4),d++;continue}let w=p[d+1]??"";if(y.includes("|")&&d+1<p.length&&w.match(/^\s*\|[-:\s|]+\|\s*$/)){let l=[],u=d;for(;u<p.length&&(p[u]??"").includes("|");)l.push(p[u]??""),u++;let S=wt(l);if(S){Tt(r,S,s),d=u;continue}}if(y.match(/^\s*\|[-:]+/)||y.match(/^---+$/)){d++;continue}let R=y.match(/^(#{3,4})\s+(.+)$/);if(R?.[1]&&R[2]){let l=R[1].length,u=he(l);N(r,u+10),r.moveDown(.3),r.fontSize(u).font("Helvetica-Bold").fillColor("#1c1917").text(R[2].trim(),{width:s}),r.moveDown(.3),d++;continue}if(y.match(/^\s*[-*] /)){N(r,14),r.fontSize(10).font("Helvetica").fillColor("#44403c").text(y.trim(),{indent:12,width:s-12}),d++;continue}N(r,14),r.fontSize(10).font("Helvetica").fillColor("#44403c").text(y.trim(),{width:s}),d++}St(r)}r.end(),a.on("finish",()=>o(t)),a.on("error",n)})}import k from"chalk";import{createPrettyReporter as bt,createJsonReporter as kt,createJunitReporter as Et}from"@kindlm/core";var xt={bold:e=>k.bold(e),red:e=>k.red(e),green:e=>k.green(e),yellow:e=>k.yellow(e),cyan:e=>k.cyan(e),dim:e=>k.dim(e),greenBold:e=>k.green.bold(e),redBold:e=>k.red.bold(e)},It=["pretty","json","junit"];function J(e){switch(e){case"json":return kt();case"junit":return Et();case"pretty":return bt(xt);default:console.error(k.red(`Unknown reporter: '${e}'. Available: ${It.join(", ")}`)),process.exit(1)}}import{execSync as Q}from"child_process";function W(){try{let e=Q("git rev-parse HEAD",{encoding:"utf-8"}).trim()||null,t=Q("git rev-parse --abbrev-ref HEAD",{encoding:"utf-8"}).trim()||null,n=Q("git status --porcelain",{encoding:"utf-8"}).trim().length>0;return{commitSha:e,branch:t,dirty:n}}catch{return{commitSha:null,branch:null,dirty:!1}}}function Ce(e){e.command("test").description("Run test suites").option("-s, --suite <name>","Run a specific suite").option("--compliance","Generate compliance report").option("--reporter <type>","Output format: pretty, json, junit","pretty").option("--runs <count>","Override run count").option("--gate <percent>","Fail if pass rate below threshold").option("--pdf <path>","Export compliance report as PDF (requires --compliance)").option("-c, --config <path>","Path to config file","kindlm.yaml").action(async t=>{t.pdf&&!t.compliance&&(console.error(L.red("--pdf requires --compliance")),process.exit(1));let o=J(t.reporter);try{let{runnerResult:n,config:r,yamlContent:a}=await H({configPath:t.config,runs:t.runs!==void 0?parseInt(t.runs,10):void 0,gate:t.gate!==void 0?parseFloat(t.gate):void 0,suite:t.suite}),{runResult:s,aggregated:i}=n,c=Pt(r.gates,i),f=await o.generate(s,c);console.log(f.content);let m,p;if(t.compliance){let d=W(),y={runId:crypto.randomUUID(),kindlmVersion:"1.0.0",gitCommitSha:d.commitSha??void 0,modelIds:r.models.map(u=>u.id),...r.compliance?.metadata??{}};if(m=(await $t(y).generate(s,c)).content,p=m.match(/Tamper Evidence Hash \(SHA-256\):\*\* `([a-f0-9]{64})`/)?.[1],t.reporter==="pretty"||!t.reporter?(console.log(""),console.log(m)):(console.error(""),console.error(m)),t.pdf){let u=await Re(m,t.pdf);console.log(""),console.log(L.green(`PDF report saved to ${u}`))}}try{fe({runnerResult:n,suiteName:r.suite.name,configHash:ge(a),timestamp:new Date().toISOString(),complianceReport:m,complianceHash:p})}catch{}let g=s.failed===0&&s.errored===0&&c.passed;process.exit(g?0:1)}catch(n){if(n instanceof Ot){let r=n.code==="TIMEOUT"?"Provider timeout":n.code==="NETWORK_ERROR"?"Network error":n.code==="AUTH_FAILED"?"Authentication failed":n.code==="RATE_LIMITED"?"Rate limited":`Provider error (${n.code})`;console.error(L.red(`${r}: ${n.message}`)),n.retryable&&console.error(L.yellow("This error may be transient. Try again or increase timeoutMs in your kindlm.yaml defaults."))}else if(Mt(n)){let a=n.code.startsWith("CONFIG_")?"Config error":"Error";console.error(L.red(`${a}: ${n.message}`))}else n instanceof Error&&n.name==="AbortError"?console.error(L.red("Request timed out. Check network connectivity or increase timeout.")):console.error(L.red(`Error: ${n instanceof Error?n.message:String(n)}`));process.exit(1)}})}function Mt(e){return typeof e=="object"&&e!==null&&"code"in e&&"message"in e&&typeof e.code=="string"&&typeof e.message=="string"}import{resolve as Y,dirname as te,join as ne}from"path";import{readFileSync as _t}from"fs";import h from"chalk";import{parseConfig as Dt,readBaseline as jt,writeBaseline as Ht,listBaselines as Ft,buildBaselineData as Se,compareBaseline as Bt,deserializeBaseline as Ut}from"@kindlm/core";import{readFileSync as At,writeFileSync as Nt,mkdirSync as ve,readdirSync as Lt}from"fs";import{join as ee}from"path";function we(e){return e.replace(/[^a-zA-Z0-9_-]/g,"_")}function V(e){let t=ee(e,"baselines");return{read(o){let n=ee(t,`${we(o)}.json`);try{return{success:!0,data:At(n,"utf-8")}}catch{return{success:!1,error:{code:"BASELINE_NOT_FOUND",message:`No baseline found for suite "${o}" at ${n}`}}}},write(o,n){try{ve(t,{recursive:!0});let r=ee(t,`${we(o)}.json`);return Nt(r,n,"utf-8"),{success:!0,data:void 0}}catch(r){return{success:!1,error:{code:"UNKNOWN_ERROR",message:`Failed to write baseline: ${r instanceof Error?r.message:String(r)}`}}}},list(){try{return ve(t,{recursive:!0}),{success:!0,data:Lt(t).filter(r=>r.endsWith(".json")).map(r=>r.replace(/\.json$/,""))}}catch(o){return{success:!1,error:{code:"UNKNOWN_ERROR",message:`Failed to list baselines: ${o instanceof Error?o.message:String(o)}`}}}}}}function Te(e){let t=e.command("baseline").description("Manage test baselines");t.command("set").description("Save current results as baseline").option("-c, --config <path>","Path to config file","kindlm.yaml").option("--runs <count>","Override run count").option("--force","Save baseline even if all tests failed").action(async o=>{try{let n=te(Y(process.cwd(),o.config)),r=ne(n,".kindlm"),a=V(r),{config:s,runnerResult:i}=await H({configPath:o.config,runs:o.runs!==void 0?parseInt(o.runs,10):void 0}),{runResult:c,aggregated:f}=i;(c.totalTests>0?c.passed/c.totalTests:0)===0&&!o.force&&(console.error(h.red("All tests failed or errored. Refusing to save a failing baseline.")),console.error(h.yellow("Use --force to save anyway.")),process.exit(1));let p=Se(s.suite.name,f,new Date().toISOString()),g=Ht(p,a);g.success||(console.error(h.red(`Failed to save baseline: ${g.error.message}`)),process.exit(1));let d=Object.keys(p.results).length;console.log(""),console.log(h.green(`Baseline saved for suite "${s.suite.name}" (${d} test${d===1?"":"s"})`)),console.log(h.dim(` Location: ${r}/baselines/`))}catch(n){console.error(h.red(`Error: ${n instanceof Error?n.message:String(n)}`)),process.exit(1)}}),t.command("compare").description("Compare latest against baseline").option("-c, --config <path>","Path to config file","kindlm.yaml").option("--runs <count>","Override run count").action(async o=>{try{let n=te(Y(process.cwd(),o.config)),r=ne(n,".kindlm"),a=V(r),s=Y(process.cwd(),o.config),i;try{i=_t(s,"utf-8")}catch{console.error(h.red(`Config file not found: ${s}`)),process.exit(1)}let c=$(),f=Dt(i,{configDir:n,fileReader:c});f.success||(console.error(h.red(`Config validation failed: ${f.error.message}`)),process.exit(1));let m=f.data.suite.name,p=jt(m,a);p.success||(p.error.code==="BASELINE_NOT_FOUND"?console.error(h.red(`No baseline found for suite "${m}". Run \`kindlm baseline set\` first.`)):console.error(h.red(`Failed to read baseline: ${p.error.message}`)),process.exit(1));let g=p.data,{runnerResult:d}=await H({configPath:o.config,runs:o.runs!==void 0?parseInt(o.runs,10):void 0,baselineData:g}),{aggregated:y}=d,w=Se(m,y,new Date().toISOString()),R=Bt(g,w.results);if(console.log(""),console.log(h.bold(`Baseline comparison for "${m}"`)),console.log(h.dim(` Baseline from: ${g.createdAt}`)),console.log(""),R.regressions.length>0){console.log(h.red.bold(` Regressions (${R.regressions.length}):`));for(let l of R.regressions)console.log(h.red(` ${l.testName}: ${F(l.baselinePassRate)} \u2192 ${F(l.currentPassRate)}`)),l.newFailureCodes.length>0&&console.log(h.red(` New failures: ${l.newFailureCodes.join(", ")}`));console.log("")}if(R.improvements.length>0){console.log(h.green.bold(` Improvements (${R.improvements.length}):`));for(let l of R.improvements)console.log(h.green(` ${l.testName}: ${F(l.baselinePassRate)} \u2192 ${F(l.currentPassRate)}`));console.log("")}if(R.unchanged.length>0){console.log(h.dim(` Unchanged (${R.unchanged.length}):`));for(let l of R.unchanged)console.log(h.dim(` ${l.testName}: ${F(l.passRate)}`));console.log("")}if(R.newTests.length>0){console.log(h.cyan(` New tests (${R.newTests.length}):`));for(let l of R.newTests)console.log(h.cyan(` ${l}`));console.log("")}if(R.removedTests.length>0){console.log(h.yellow(` Removed tests (${R.removedTests.length}):`));for(let l of R.removedTests)console.log(h.yellow(` ${l}`));console.log("")}process.exit(R.regressions.length>0?1:0)}catch(n){console.error(h.red(`Error: ${n instanceof Error?n.message:String(n)}`)),process.exit(1)}}),t.command("list").description("List saved baselines").option("-c, --config <path>","Path to config file","kindlm.yaml").action(o=>{try{let n=te(Y(process.cwd(),o.config)),r=ne(n,".kindlm"),a=V(r),s=Ft(a);s.success||(console.error(h.red(`Failed to list baselines: ${s.error.message}`)),process.exit(1));let i=s.data;if(i.length===0){console.log(h.dim("No baselines saved yet. Run `kindlm baseline set` to create one."));return}console.log(h.bold("Saved baselines:")),console.log("");for(let c of i){let f=a.read(c);if(!f.success){console.log(` ${c} ${h.dim("(unreadable)")}`);continue}let m=Ut(f.data);if(!m.success){console.log(` ${c} ${h.dim("(corrupt)")}`);continue}let p=Object.keys(m.data.results).length;console.log(` ${h.cyan(m.data.suiteName)} \u2014 ${p} test${p===1?"":"s"}, saved ${h.dim(m.data.createdAt)}`)}}catch(n){console.error(h.red(`Error: ${n instanceof Error?n.message:String(n)}`)),process.exit(1)}})}function F(e){return`${(e*100).toFixed(1)}%`}import{createInterface as Xt}from"readline";import{Writable as qt}from"stream";import x from"chalk";import{readFileSync as Kt,writeFileSync as zt,mkdirSync as Gt,unlinkSync as Jt}from"fs";import{join as be}from"path";import{homedir as ke}from"os";function re(){return be(ke(),".kindlm","credentials")}function X(){try{let e=Kt(re(),"utf-8"),t=JSON.parse(e);return typeof t.token=="string"&&t.token.length>0?t.token:null}catch{return null}}function Ee(e){let t=re(),o=be(ke(),".kindlm");Gt(o,{recursive:!0,mode:448});let n={token:e,savedAt:new Date().toISOString()};zt(t,JSON.stringify(n,null,2),{mode:384})}function xe(){try{Jt(re())}catch{}}var Wt="https://api.kindlm.com";var E=class extends Error{status;constructor(t,o){super(o),this.name="CloudApiError",this.status=t}};function D(){let e=process.env.KINDLM_CLOUD_URL??Wt;if(e.startsWith("http://")&&!Vt(e))throw new Error(`Refusing to use insecure HTTP for Cloud API: ${e}. Use HTTPS or target localhost for development.`);return e}function Vt(e){try{let t=new URL(e);return t.hostname==="localhost"||t.hostname==="127.0.0.1"||t.hostname==="::1"}catch{return!1}}function Yt(e){return new Promise(t=>setTimeout(t,e))}function B(e,t){async function o(n,r,a){let s=`${e}${r}`,i={Authorization:`Bearer ${t}`},c={method:n,headers:i};a!==void 0&&(i["Content-Type"]="application/json",c.body=JSON.stringify(a));let f;for(let m=0;m<=1;m++){m>0&&await Yt(1e3);let p=new AbortController,g=setTimeout(()=>p.abort(),3e4);c.signal=p.signal;try{let d=await fetch(s,c);if(!d.ok){if(d.status>=500&&m<1){f=new E(d.status,`HTTP ${d.status}`);continue}let w=`HTTP ${d.status}`;if((d.headers.get("content-type")??"").includes("application/json"))try{let l=await d.json();l.error&&(w=l.error)}catch{}throw new E(d.status,w)}if(d.status===204)return;let y=d.headers.get("content-type")??"";if(!y.includes("application/json"))throw new E(d.status,`Expected JSON response but got content-type: ${y}`);return await d.json()}catch(d){if(d instanceof E)throw d;if(f=d instanceof Error?d:new Error(String(d)),m<1)continue}finally{clearTimeout(g)}}throw f??new Error("Request failed")}return{baseUrl:e,get:n=>o("GET",n),post:(n,r)=>o("POST",n,r),patch:(n,r)=>o("PATCH",n,r),delete:n=>o("DELETE",n)}}function Ie(e){e.command("login").description("Authenticate with KindLM Cloud").option("-t, --token <token>","API token (skips interactive prompt)").option("--status","Show current authentication status").option("--logout","Remove stored credentials").action(async t=>{try{if(t.logout){xe(),console.log(x.green("Logged out. Credentials removed."));return}if(t.status){await Zt();return}let o=t.token??process.env.KINDLM_API_TOKEN??await Qt();o.startsWith("klm_")||(console.error(x.red('Invalid token format. KindLM tokens start with "klm_".')),process.exit(1));let n=B(D(),o);try{await n.get("/v1/auth/tokens")}catch(r){throw r instanceof E&&r.status===401&&(console.error(x.red("Invalid or expired token.")),process.exit(1)),r}Ee(o),console.log(x.green("Authenticated successfully. Token saved."))}catch(o){console.error(x.red(`Login failed: ${o instanceof Error?o.message:String(o)}`)),process.exit(1)}})}async function Zt(){let e=X();if(!e){console.log(x.yellow('Not authenticated. Run "kindlm login" to authenticate.'));return}let t=B(D(),e);try{await t.get("/v1/auth/tokens"),console.log(x.green("Authenticated.")),console.log(` Cloud URL: ${D()}`)}catch(o){o instanceof E&&o.status===401?console.log(x.yellow('Stored token is invalid or expired. Run "kindlm login" to re-authenticate.')):console.log(x.yellow(`Cannot reach Cloud API: ${o instanceof Error?o.message:String(o)}`))}}function Qt(){return new Promise((e,t)=>{let o=new qt({write(r,a,s){s()}});process.stderr.write("Paste your KindLM API token: ");let n=Xt({input:process.stdin,output:o,terminal:!0});n.question("",r=>{n.close(),process.stderr.write(`
|
|
52
|
+
`);let a=r.trim();if(!a){t(new Error("No token provided"));return}e(a)})})}import{basename as rn}from"path";import{execSync as on}from"child_process";import oe from"chalk";function Pe(){return process.env.GITHUB_ACTIONS?{name:"github_actions",isCI:!0,commitSha:process.env.GITHUB_SHA??null,branch:process.env.GITHUB_REF_NAME??null}:process.env.GITLAB_CI?{name:"gitlab_ci",isCI:!0,commitSha:process.env.CI_COMMIT_SHA??null,branch:process.env.CI_COMMIT_BRANCH??null}:process.env.CI?{name:null,isCI:!0,commitSha:null,branch:null}:{name:null,isCI:!1,commitSha:null,branch:null}}function j(e){return encodeURIComponent(e)}async function $e(e,t,o){let n=await en(e,o.projectName),r=await tn(e,n,o.suiteName,o.configHash),a=await e.post(`/v1/projects/${j(n)}/runs`,{suiteId:r,commitSha:o.commitSha,branch:o.branch,environment:o.environment,triggeredBy:o.triggeredBy}),s=nn(t.aggregated),i=50;try{for(let l=0;l<s.length;l+=i){let u=s.slice(l,l+i);await e.post(`/v1/runs/${j(a.id)}/results`,{results:u})}}catch(l){try{await e.patch(`/v1/runs/${j(a.id)}`,{status:"failed"})}catch{}throw l}let{runResult:c}=t,f=c.totalTests>0?c.passed/c.totalTests:0,m=new Set(t.aggregated.map(l=>l.modelId)),p=t.aggregated.map(l=>l.assertionScores.judge?.mean).filter(l=>l!==void 0),g=p.length>0?p.reduce((l,u)=>l+u,0)/p.length:void 0,d=t.aggregated.map(l=>l.latencyAvgMs),y=d.length>0?d.reduce((l,u)=>l+u,0)/d.length:void 0,w=t.aggregated.reduce((l,u)=>l+u.totalCostUsd,0),R=w>0?w:void 0;return await e.patch(`/v1/runs/${j(a.id)}`,{status:"completed",passRate:f,testCount:c.totalTests,modelCount:m.size,judgeAvgScore:g,latencyAvgMs:y,costEstimateUsd:R,complianceReport:o.complianceReport,complianceHash:o.complianceHash,finishedAt:new Date().toISOString()}),{runId:a.id,projectId:n}}async function en(e,t){let{projects:o}=await e.get("/v1/projects"),n=o.find(a=>a.name===t);return n?n.id:(await e.post("/v1/projects",{name:t})).id}async function tn(e,t,o,n){let{suites:r}=await e.get(`/v1/projects/${j(t)}/suites`),a=r.find(i=>i.name===o);return a?a.id:(await e.post(`/v1/projects/${j(t)}/suites`,{name:o,configHash:n})).id}function nn(e){return e.map(t=>{let o=t,n=t.runs.length>0?t.runs.flatMap(r=>r.assertions.filter(a=>!a.passed).map(a=>a.failureMessage)).filter(r=>r!==void 0):o.failureMessages??[];return{testCaseName:t.testCaseName,modelId:t.modelId,passed:t.passed?1:0,passRate:t.passRate,runCount:t.runCount,judgeAvg:t.assertionScores.judge?.mean??null,driftScore:t.assertionScores.drift?.mean??null,latencyAvgMs:t.latencyAvgMs??null,costUsd:t.totalCostUsd??null,totalTokens:t.totalTokens??null,failureCodes:t.failureCodes.length>0?JSON.stringify(t.failureCodes):null,failureMessages:n.length>0?JSON.stringify(n):null,assertionScores:Object.keys(t.assertionScores).length>0?JSON.stringify(t.assertionScores):null}})}function Oe(e){e.command("upload").description("Push last run results to KindLM Cloud").option("-t, --token <token>","API token (overrides stored token)").option("-p, --project <name>","Project name").action(async t=>{try{let o=t.token??process.env.KINDLM_API_TOKEN??X();o||(console.error(oe.red('Not authenticated. Run "kindlm login" first or pass --token.')),process.exit(1));let n=pe();n||(console.error(oe.red('No test run found. Run "kindlm test" first.')),process.exit(1));let r=W(),a=Pe(),s=t.project??an(),i=B(D(),o),c=_();c.start("Uploading results to KindLM Cloud...");try{let f=await $e(i,n.runnerResult,{projectName:s,suiteName:n.suiteName,configHash:n.configHash,commitSha:a.commitSha??r.commitSha??void 0,branch:a.branch??r.branch??void 0,environment:a.isCI?"ci":"local",triggeredBy:a.name??"local",complianceReport:n.complianceReport,complianceHash:n.complianceHash});c.succeed("Uploaded successfully."),console.log(` Run ID: ${f.runId}`),console.log(` Project: ${s}`),console.log(` Suite: ${n.suiteName}`)}catch(f){throw c.fail("Upload failed."),f}}catch(o){console.error(oe.red(`Upload failed: ${o instanceof Error?o.message:String(o)}`)),process.exit(1)}})}function sn(e){try{let n=new URL(e).pathname.split("/").filter(Boolean),r=n[n.length-1];if(r)return r.replace(/\.git$/,"")}catch{}let t=e.match(/^[\w.-]+@[\w.-]+:(.+?)(?:\.git)?$/);if(t?.[1]){let o=t[1].split("/");return o[o.length-1]??null}return null}function an(){try{let e=on("git remote get-url origin",{encoding:"utf-8"}).trim(),t=sn(e);if(t)return t}catch{}return rn(process.cwd())}import{readFileSync as dn,statSync as mn}from"fs";import{resolve as un,dirname as fn}from"path";import{spawn as pn}from"child_process";import q from"chalk";import{parseConfig as gn,createProvider as hn,filterSpans as yn,mapSpansToResult as Rn,buildContextFromTrace as Cn,createAssertionsFromExpect as vn}from"@kindlm/core";import{createServer as cn}from"http";import{parseOtlpPayload as ln}from"@kindlm/core";function Me(e){let t=[],o=null,n=[];function r(){for(let s of n)s()}function a(s,i){if(i.setHeader("Access-Control-Allow-Origin","*"),i.setHeader("Access-Control-Allow-Methods","POST, OPTIONS"),i.setHeader("Access-Control-Allow-Headers","Content-Type"),s.method==="OPTIONS"){i.writeHead(204),i.end();return}if(s.method!=="POST"||s.url!=="/v1/traces"){i.writeHead(404,{"Content-Type":"application/json"}),i.end(JSON.stringify({error:"Not found"}));return}let c=10*1024*1024,f=[],m=0,p=!1;s.on("data",g=>{if(m+=g.length,m>c){p=!0,i.writeHead(413,{"Content-Type":"application/json"}),i.end(JSON.stringify({error:"Payload too large"})),s.destroy();return}f.push(g)}),s.on("end",()=>{if(!p)try{let g=Buffer.concat(f).toString("utf-8"),d=JSON.parse(g),y=ln(d);y.success?(t.push(...y.data),r(),i.writeHead(200,{"Content-Type":"application/json"}),i.end(JSON.stringify({partialSuccess:{}}))):(i.writeHead(400,{"Content-Type":"application/json"}),i.end(JSON.stringify({error:y.error.message})))}catch{i.writeHead(400,{"Content-Type":"application/json"}),i.end(JSON.stringify({error:"Invalid JSON"}))}})}return{start(){return new Promise((s,i)=>{o=cn(a),o.on("error",i),o.listen(e,()=>s())})},stop(){return new Promise(s=>{o?o.close(()=>s()):s()})},getSpans(){return[...t]},waitForSpans({timeoutMs:s}){return new Promise(i=>{if(t.length>0){i([...t]);return}let c=setTimeout(()=>{n=n.filter(m=>m!==f),i([...t])},s),f=()=>{clearTimeout(c),n=n.filter(m=>m!==f),setTimeout(()=>i([...t]),500)};n.push(f)})}}}function Ae(e){e.command("trace").description("Ingest OpenTelemetry traces and run assertions against them").option("-c, --config <path>","Config file path","kindlm.yaml").option("--port <port>","OTLP HTTP port","4318").option("--command <cmd>","Command to spawn (traces are collected while it runs)").option("--timeout <ms>","Timeout in ms to wait for traces","30000").option("--reporter <type>","Report format: pretty, json, junit","pretty").action(async t=>{let o=_();try{let n=un(process.cwd(),t.config),r=fn(n);try{mn(n).size>1048576&&(console.error(q.red("Config file exceeds 1MB limit")),process.exit(1))}catch{console.error(q.red(`Config file not found: ${n}`)),process.exit(1)}let a;try{a=dn(n,"utf-8")}catch{console.error(q.red(`Config file not found: ${n}`)),process.exit(1)}let s=$(),i=gn(a,{configDir:r,fileReader:s});i.success||(console.error(q.red(`Config validation failed: ${i.error.message}`)),process.exit(1));let c=i.data,f=c.trace??{port:parseInt(t.port,10),timeoutMs:parseInt(t.timeout,10),spanMapping:{outputTextAttr:"gen_ai.completion.0.content",modelAttr:"gen_ai.response.model",systemAttr:"gen_ai.system",inputTokensAttr:"gen_ai.usage.input_tokens",outputTokensAttr:"gen_ai.usage.output_tokens"}},m=parseInt(t.port,10)||f.port,p=parseInt(t.timeout,10)||f.timeoutMs,g=Me(m);await g.start();let d;try{o.start(`Listening for OTLP traces on port ${m}...`),t.command&&(d=pn("sh",["-c",t.command],{cwd:r,env:{...process.env,OTEL_EXPORTER_OTLP_ENDPOINT:`http://localhost:${m}`,OTEL_EXPORTER_OTLP_PROTOCOL:"http/json"},stdio:"inherit"}),d.on("error",C=>{o.fail(`Command failed: ${C.message}`)}));let y=await g.waitForSpans({timeoutMs:p});y.length===0&&(o.fail("No traces received"),process.exit(1)),o.succeed(`Received ${y.length} spans`);let w=yn(y,f.spanFilter),R=Rn(w,f.spanMapping),l=G(),u=new Map,S=c.providers;for(let[C,T]of Object.entries(S)){if(!T)continue;let P="";if(T.apiKeyEnv){let b=process.env[T.apiKeyEnv];b&&(P=b.trim())}if(!(!P&&C!=="ollama"))try{let b=hn(C,l);await b.initialize({apiKey:P,baseUrl:T.baseUrl,organization:T.organization,timeoutMs:c.defaults.timeoutMs,maxRetries:2}),u.set(C,b)}catch{}}let U=c.defaults.judgeModel??c.models[0]?.id,M=c.models.find(C=>C.id===U),I=M?u.get(M.provider):void 0,Z=Cn(R,{configDir:r,judgeAdapter:I,judgeModel:M?.model}),se=[];for(let C of c.tests){if(C.skip)continue;let T=vn(C.expect),P=[];for(let b of T){let Fe=await b.evaluate(Z);P.push(...Fe)}se.push({testName:C.name,assertions:P})}let Le=c.tests.filter(C=>C.skip).length,ie=R.latencyMs,K=se.map(({testName:C,assertions:T})=>{let P=T.every(b=>b.passed);return{name:C,modelId:"trace",status:P?"passed":"failed",assertions:T,latencyMs:ie,costUsd:0}}),_e=K.filter(C=>C.status==="passed").length,z=K.filter(C=>C.status==="failed").length,De={suites:[{name:c.suite.name,status:z>0?"failed":"passed",tests:K}],totalTests:K.length,passed:_e,failed:z,errored:0,skipped:Le,durationMs:ie},je={passed:z===0,gates:[]},He=await J(t.reporter).generate(De,je);console.log(He.content),process.exit(z>0?1:0)}finally{d?.kill(),await g.stop()}}catch(n){o.fail(`Trace command failed: ${n instanceof Error?n.message:String(n)}`),process.exit(1)}})}function Ne(){let e=new wn;return e.name("kindlm").description("AI agent behavioral regression testing").version("1.0.0"),ae(e),ce(e),Ce(e),Te(e),Ie(e),Oe(e),Ae(e),e}process.on("unhandledRejection",e=>{let t=e instanceof Error?e.message:String(e);process.stderr.write(`
|
|
53
53
|
Unhandled error: ${t}
|
|
54
54
|
`),process.exit(1)});process.on("uncaughtException",e=>{process.stderr.write(`
|
|
55
55
|
Fatal error: ${e.message}
|
|
56
|
-
`),process.exit(1)});
|
|
56
|
+
`),process.exit(1)});Ne().parse(process.argv);
|
|
57
57
|
//# sourceMappingURL=kindlm.js.map
|