@kindlm/cli 0.4.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +142 -0
- package/dist/index.cjs +8 -8
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +8 -8
- package/dist/index.js.map +1 -1
- package/dist/kindlm.js +9 -9
- package/dist/kindlm.js.map +1 -1
- package/package.json +6 -5
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import{Command as
|
|
1
|
+
import{Command as vn}from"commander";import{existsSync as Fe,writeFileSync as Be}from"fs";import{resolve as Ue}from"path";import A from"chalk";var Ke=`kindlm: 1
|
|
2
2
|
project: my-project
|
|
3
3
|
|
|
4
4
|
suite:
|
|
@@ -42,11 +42,11 @@ defaults:
|
|
|
42
42
|
repeat: 1
|
|
43
43
|
concurrency: 4
|
|
44
44
|
timeoutMs: 60000
|
|
45
|
-
`;function
|
|
46
|
-
Interrupted. Exiting...`)),process.exit(130)};process.on("SIGINT",
|
|
47
|
-
`),
|
|
48
|
-
`).trim()}),
|
|
49
|
-
`).trim()}),
|
|
50
|
-
`),y=!1,d=0;for(;d<f.length;){let C=f[d]??"";if(C.startsWith("```")){y=!y,y&&N(r,30),d++;continue}if(y){N(r,14);let l=r.y;r.save(),r.rect(r.page.margins.left,l-2,i,14).fill("#f5f5f4"),r.restore(),r.fontSize(9).font("Courier").fillColor("#44403c").text(C,{width:i}),d++;continue}if(!C.trim()){r.moveDown(.4),d++;continue}let S=f[d+1]??"";if(C.includes("|")&&d+1<f.length&&S.match(/^\s*\|[-:\s|]+\|\s*$/)){let l=[],g=d;for(;g<f.length&&(f[g]??"").includes("|");)l.push(f[g]??""),g++;let b=gt(l);if(b){ht(r,b,i),d=g;continue}}if(C.match(/^\s*\|[-:]+/)||C.match(/^---+$/)){d++;continue}let h=C.match(/^(#{3,4})\s+(.+)$/);if(h?.[1]&&h[2]){let l=h[1].length,g=ge(l);N(r,g+10),r.moveDown(.3),r.fontSize(g).font("Helvetica-Bold").fillColor("#1c1917").text(h[2].trim(),{width:i}),r.moveDown(.3),d++;continue}if(C.match(/^\s*[-*] /)){N(r,14),r.fontSize(10).font("Helvetica").fillColor("#44403c").text(C.trim(),{indent:12,width:i-12}),d++;continue}N(r,14),r.fontSize(10).font("Helvetica").fillColor("#44403c").text(C.trim(),{width:i}),d++}pt(r)}r.end(),a.on("finish",()=>n(t)),a.on("error",o)})}function ye(e){e.command("test").description("Run test suites").option("-s, --suite <name>","Run a specific suite").option("--compliance","Generate compliance report").option("--reporter <type>","Output format: pretty, json, junit","pretty").option("--runs <count>","Override run count").option("--gate <percent>","Fail if pass rate below threshold").option("--pdf <path>","Export compliance report as PDF (requires --compliance)").option("-c, --config <path>","Path to config file","kindlm.yaml").action(async t=>{try{let{runnerResult:n,config:o,yamlContent:r}=await B({configPath:t.config,runs:t.runs?parseInt(t.runs,10):void 0,gate:t.gate?parseFloat(t.gate):void 0}),{runResult:a,aggregated:i}=n,s=yt(o.gates,i),m=bt(t.reporter).generate(a,s);if(console.log(m.content),t.compliance){let y=wt().generate(a,s);if(console.log(""),console.log(y.content),t.pdf){let d=await he(y.content,t.pdf);console.log(""),console.log(w.green(`PDF report saved to ${d}`))}}try{me({runnerResult:n,suiteName:o.suite.name,configHash:fe(r),timestamp:new Date().toISOString()})}catch{}let u=a.failed===0&&a.errored===0&&s.passed;process.exit(u?0:1)}catch(n){if(n instanceof St){let o=n.code==="TIMEOUT"?"Provider timeout":n.code==="NETWORK_ERROR"?"Network error":n.code==="AUTH_FAILED"?"Authentication failed":n.code==="RATE_LIMITED"?"Rate limited":`Provider error (${n.code})`;console.error(w.red(`${o}: ${n.message}`)),n.retryable&&console.error(w.yellow("This error may be transient. Try again or increase --timeout."))}else if(kt(n)){let r=n.code.startsWith("CONFIG_")?"Config error":"Error";console.error(w.red(`${r}: ${n.message}`))}else n instanceof Error&&n.name==="AbortError"?console.error(w.red("Request timed out. Check network connectivity or increase timeout.")):console.error(w.red(`Error: ${n instanceof Error?n.message:String(n)}`));process.exit(1)}})}var Tt={bold:e=>w.bold(e),red:e=>w.red(e),green:e=>w.green(e),yellow:e=>w.yellow(e),cyan:e=>w.cyan(e),dim:e=>w.dim(e),greenBold:e=>w.green.bold(e),redBold:e=>w.red.bold(e)};function bt(e){switch(e){case"json":return vt();case"junit":return Rt();default:return Ct(Tt)}}function kt(e){return typeof e=="object"&&e!==null&&"code"in e&&"message"in e&&typeof e.code=="string"&&typeof e.message=="string"}import{resolve as W,dirname as q,join as V}from"path";import{readFileSync as $t}from"fs";import p from"chalk";import{parseConfig as Et,readBaseline as Ot,writeBaseline as At,listBaselines as Mt,buildBaselineData as Re,compareBaseline as Nt,deserializeBaseline as Lt}from"@kindlm/core";import{readFileSync as xt,writeFileSync as Pt,mkdirSync as Ce,readdirSync as It}from"fs";import{join as X}from"path";function ve(e){return e.replace(/[^a-zA-Z0-9_-]/g,"_")}function J(e){let t=X(e,"baselines");return{read(n){let o=X(t,`${ve(n)}.json`);try{return{success:!0,data:xt(o,"utf-8")}}catch{return{success:!1,error:{code:"BASELINE_NOT_FOUND",message:`No baseline found for suite "${n}" at ${o}`}}}},write(n,o){try{Ce(t,{recursive:!0});let r=X(t,`${ve(n)}.json`);return Pt(r,o,"utf-8"),{success:!0,data:void 0}}catch(r){return{success:!1,error:{code:"UNKNOWN_ERROR",message:`Failed to write baseline: ${r instanceof Error?r.message:String(r)}`}}}},list(){try{return Ce(t,{recursive:!0}),{success:!0,data:It(t).filter(r=>r.endsWith(".json")).map(r=>r.replace(/\.json$/,""))}}catch(n){return{success:!1,error:{code:"UNKNOWN_ERROR",message:`Failed to list baselines: ${n instanceof Error?n.message:String(n)}`}}}}}}function we(e){let t=e.command("baseline").description("Manage test baselines");t.command("set").description("Save current results as baseline").option("-c, --config <path>","Path to config file","kindlm.yaml").option("--runs <count>","Override run count").action(async n=>{try{let o=q(W(process.cwd(),n.config)),r=V(o,".kindlm"),a=J(r),{config:i,runnerResult:s}=await B({configPath:n.config,runs:n.runs?parseInt(n.runs,10):void 0}),{aggregated:c}=s,m=Re(i.suite.name,c,new Date().toISOString()),u=At(m,a);u.success||(console.error(p.red(`Failed to save baseline: ${u.error.message}`)),process.exit(1));let f=Object.keys(m.results).length;console.log(""),console.log(p.green(`Baseline saved for suite "${i.suite.name}" (${f} test${f===1?"":"s"})`)),console.log(p.dim(` Location: ${r}/baselines/`)),process.exit(0)}catch(o){console.error(p.red(`Error: ${o instanceof Error?o.message:String(o)}`)),process.exit(1)}}),t.command("compare").description("Compare latest against baseline").option("-c, --config <path>","Path to config file","kindlm.yaml").option("--runs <count>","Override run count").action(async n=>{try{let o=q(W(process.cwd(),n.config)),r=V(o,".kindlm"),a=J(r),i=W(process.cwd(),n.config),s;try{s=$t(i,"utf-8")}catch{console.error(p.red(`Config file not found: ${i}`)),process.exit(1)}let c=A(),m=Et(s,{configDir:o,fileReader:c});m.success||(console.error(p.red(`Config validation failed: ${m.error.message}`)),process.exit(1));let u=m.data.suite.name,f=Ot(u,a);f.success||(f.error.code==="BASELINE_NOT_FOUND"?console.error(p.red(`No baseline found for suite "${u}". Run \`kindlm baseline set\` first.`)):console.error(p.red(`Failed to read baseline: ${f.error.message}`)),process.exit(1));let y=f.data,{runnerResult:d}=await B({configPath:n.config,runs:n.runs?parseInt(n.runs,10):void 0,baselineData:y}),{aggregated:C}=d,S=Re(u,C,new Date().toISOString()),h=Nt(y,S.results);if(console.log(""),console.log(p.bold(`Baseline comparison for "${u}"`)),console.log(p.dim(` Baseline from: ${y.createdAt}`)),console.log(""),h.regressions.length>0){console.log(p.red.bold(` Regressions (${h.regressions.length}):`));for(let l of h.regressions)console.log(p.red(` ${l.testName}: ${U(l.baselinePassRate)} \u2192 ${U(l.currentPassRate)}`)),l.newFailureCodes.length>0&&console.log(p.red(` New failures: ${l.newFailureCodes.join(", ")}`));console.log("")}if(h.improvements.length>0){console.log(p.green.bold(` Improvements (${h.improvements.length}):`));for(let l of h.improvements)console.log(p.green(` ${l.testName}: ${U(l.baselinePassRate)} \u2192 ${U(l.currentPassRate)}`));console.log("")}if(h.unchanged.length>0){console.log(p.dim(` Unchanged (${h.unchanged.length}):`));for(let l of h.unchanged)console.log(p.dim(` ${l.testName}: ${U(l.passRate)}`));console.log("")}if(h.newTests.length>0){console.log(p.cyan(` New tests (${h.newTests.length}):`));for(let l of h.newTests)console.log(p.cyan(` ${l}`));console.log("")}if(h.removedTests.length>0){console.log(p.yellow(` Removed tests (${h.removedTests.length}):`));for(let l of h.removedTests)console.log(p.yellow(` ${l}`));console.log("")}process.exit(h.regressions.length>0?1:0)}catch(o){console.error(p.red(`Error: ${o instanceof Error?o.message:String(o)}`)),process.exit(1)}}),t.command("list").description("List saved baselines").option("-c, --config <path>","Path to config file","kindlm.yaml").action(n=>{try{let o=q(W(process.cwd(),n.config)),r=V(o,".kindlm"),a=J(r),i=Mt(a);i.success||(console.error(p.red(`Failed to list baselines: ${i.error.message}`)),process.exit(1));let s=i.data;s.length===0&&(console.log(p.dim("No baselines saved yet. Run `kindlm baseline set` to create one.")),process.exit(0)),console.log(p.bold("Saved baselines:")),console.log("");for(let c of s){let m=a.read(c);if(!m.success){console.log(` ${c} ${p.dim("(unreadable)")}`);continue}let u=Lt(m.data);if(!u.success){console.log(` ${c} ${p.dim("(corrupt)")}`);continue}let f=Object.keys(u.data.results).length;console.log(` ${p.cyan(u.data.suiteName)} \u2014 ${f} test${f===1?"":"s"}, saved ${p.dim(u.data.createdAt)}`)}process.exit(0)}catch(o){console.error(p.red(`Error: ${o instanceof Error?o.message:String(o)}`)),process.exit(1)}})}function U(e){return`${(e*100).toFixed(1)}%`}import{createInterface as zt}from"readline";import{Writable as Gt}from"stream";import E from"chalk";import{readFileSync as _t,writeFileSync as Dt,mkdirSync as jt,unlinkSync as Ft,chmodSync as Ht}from"fs";import{join as Se}from"path";import{homedir as Te}from"os";function Z(){return Se(Te(),".kindlm","credentials")}function Y(){try{let e=_t(Z(),"utf-8"),t=JSON.parse(e);return typeof t.token=="string"&&t.token.length>0?t.token:null}catch{return null}}function be(e){let t=Z(),n=Se(Te(),".kindlm");jt(n,{recursive:!0,mode:448});let o={token:e,savedAt:new Date().toISOString()};Dt(t,JSON.stringify(o,null,2),{mode:384}),Ht(t,384)}function ke(){try{Ft(Z())}catch{}}var Bt="https://api.kindlm.com";var $=class extends Error{status;constructor(t,n){super(n),this.name="CloudApiError",this.status=t}};function j(){let e=process.env.KINDLM_CLOUD_URL??Bt;if(e.startsWith("http://")&&!Ut(e))throw new Error(`Refusing to use insecure HTTP for Cloud API: ${e}. Use HTTPS or target localhost for development.`);return e}function Ut(e){try{let t=new URL(e);return t.hostname==="localhost"||t.hostname==="127.0.0.1"||t.hostname==="::1"}catch{return!1}}function Kt(e){return new Promise(t=>setTimeout(t,e))}function K(e,t){async function n(o,r,a){let i=`${e}${r}`,s={Authorization:`Bearer ${t}`},c={method:o,headers:s};a!==void 0&&(s["Content-Type"]="application/json",c.body=JSON.stringify(a));let m;for(let u=0;u<=1;u++){u>0&&await Kt(1e3);let f=new AbortController,y=setTimeout(()=>f.abort(),3e4);c.signal=f.signal;try{let d=await fetch(i,c);if(!d.ok){if(d.status>=500&&u<1){m=new $(d.status,`HTTP ${d.status}`);continue}let S=`HTTP ${d.status}`;if((d.headers.get("content-type")??"").includes("application/json"))try{let l=await d.json();l.error&&(S=l.error)}catch{}throw new $(d.status,S)}if(d.status===204)return;let C=d.headers.get("content-type")??"";if(!C.includes("application/json"))throw new $(d.status,`Expected JSON response but got content-type: ${C}`);return await d.json()}catch(d){if(d instanceof $)throw d;if(m=d instanceof Error?d:new Error(String(d)),u<1)continue}finally{clearTimeout(y)}}throw m??new Error("Request failed")}return{baseUrl:e,get:o=>n("GET",o),post:(o,r)=>n("POST",o,r),patch:(o,r)=>n("PATCH",o,r),delete:o=>n("DELETE",o)}}function xe(e){e.command("login").description("Authenticate with KindLM Cloud").option("-t, --token <token>","API token (skips interactive prompt)").option("--status","Show current authentication status").option("--logout","Remove stored credentials").action(async t=>{try{if(t.logout){ke(),console.log(E.green("Logged out. Credentials removed."));return}if(t.status){await Jt();return}let n=t.token??process.env.KINDLM_API_TOKEN??await Wt();n.startsWith("klm_")||(console.error(E.red('Invalid token format. KindLM tokens start with "klm_".')),process.exit(1));let o=K(j(),n);try{await o.get("/v1/auth/tokens")}catch(r){throw r instanceof $&&r.status===401&&(console.error(E.red("Invalid or expired token.")),process.exit(1)),r}be(n),console.log(E.green("Authenticated successfully. Token saved."))}catch(n){console.error(E.red(`Login failed: ${n instanceof Error?n.message:String(n)}`)),process.exit(1)}})}async function Jt(){let e=Y();if(!e){console.log(E.yellow('Not authenticated. Run "kindlm login" to authenticate.'));return}let t=K(j(),e);try{await t.get("/v1/auth/tokens"),console.log(E.green("Authenticated.")),console.log(` Cloud URL: ${j()}`)}catch(n){n instanceof $&&n.status===401?console.log(E.yellow('Stored token is invalid or expired. Run "kindlm login" to re-authenticate.')):console.log(E.yellow(`Cannot reach Cloud API: ${n instanceof Error?n.message:String(n)}`))}}function Wt(){return new Promise((e,t)=>{let n=new Gt({write(r,a,i){i()}});process.stderr.write("Paste your KindLM API token: ");let o=zt({input:process.stdin,output:n,terminal:!0});o.question("",r=>{o.close(),process.stderr.write(`
|
|
51
|
-
`);let a=r.trim();if(!a){t(new Error("No token provided"));return}e(a)})})}import{basename as
|
|
45
|
+
`;function ae(e){e.command("init").description("Create a kindlm.yaml template").option("--force","Overwrite existing kindlm.yaml").action(t=>{let o=Ue(process.cwd(),"kindlm.yaml");Fe(o)&&!t.force&&(console.error(A.red("kindlm.yaml already exists. Use --force to overwrite.")),process.exit(1));try{Be(o,Ke,"utf-8")}catch(n){let r=n instanceof Error&&"code"in n?n.code:void 0;console.error(r==="EACCES"||r==="EROFS"?A.red("Cannot create kindlm.yaml: permission denied"):A.red(`Cannot create kindlm.yaml: ${n instanceof Error?n.message:String(n)}`)),process.exit(1)}console.log(A.green("Created kindlm.yaml")),console.log(""),console.log("Next steps:"),console.log(` 1. Edit ${A.bold("kindlm.yaml")} with your test configuration`),console.log(` 2. Set your API key: ${A.bold("export OPENAI_API_KEY=sk-...")}`),console.log(` 3. Run tests: ${A.bold("kindlm test")}`)})}import{readFileSync as Ge}from"fs";import{resolve as Je,dirname as We}from"path";import O from"chalk";import{parseConfig as Ve}from"@kindlm/core";import{readFileSync as ze}from"fs";function $(){return{readFile(e){try{return{success:!0,data:ze(e,"utf-8")}}catch(t){return{success:!1,error:{code:"CONFIG_FILE_REF_ERROR",message:`Cannot read file: ${e}: ${t instanceof Error?t.message:String(t)}`}}}}}}function ce(e){e.command("validate").description("Validate kindlm.yaml configuration").option("-c, --config <path>","Path to config file","kindlm.yaml").action(t=>{let o=Je(process.cwd(),t.config),n=We(o),r;try{r=Ge(o,"utf-8")}catch{console.error(O.red(`Config file not found: ${o}`)),process.exit(1)}let a=$(),s=Ve(r,{configDir:n,fileReader:a});if(!s.success){console.error(O.red("Validation failed:"));let c=s.error.details;if(c&&Array.isArray(c.errors))for(let f of c.errors)console.error(O.red(` - ${f}`));else console.error(O.red(` ${s.error.message}`));process.exit(1)}let i=s.data;console.log(O.green("Config is valid!")),console.log(""),console.log(` Suite: ${O.bold(i.suite.name)}`),console.log(` Tests: ${O.bold(String(i.tests.length))}`),console.log(` Models: ${O.bold(String(i.models.length))}`)})}import L from"chalk";import{evaluateGates as It,createComplianceReporter as Pt,ProviderError as $t}from"@kindlm/core";import{readFileSync as Qe,statSync as et}from"fs";import{resolve as tt,dirname as nt}from"path";import v from"chalk";import{parseConfig as rt,createProvider as ot,createRunner as st}from"@kindlm/core";import{ProviderError as Ye}from"@kindlm/core";function G(){return{async fetch(e,t){let o=new AbortController,n=t.timeoutMs?setTimeout(()=>o.abort(),t.timeoutMs):void 0;try{let r=await globalThis.fetch(e,{method:t.method,headers:t.headers,body:t.body,signal:o.signal});return{ok:r.ok,status:r.status,json:()=>r.json()}}catch(r){throw r instanceof DOMException&&r.name==="AbortError"||r instanceof Error&&r.name==="AbortError"?new Ye("TIMEOUT","Request timed out",408,!0):r}finally{n!==void 0&&clearTimeout(n)}}}}import Xe from"ora";function _(){let e;return{start(t){e=Xe({text:t,stream:process.stderr}).start()},succeed(t){e?.succeed(t),e=void 0},fail(t){e?.fail(t),e=void 0},stop(){e?.stop(),e=void 0}}}import{spawn as qe}from"child_process";import{ok as Ze,err as le}from"@kindlm/core";function de(){return{async execute(e,t){return new Promise(o=>{let n=qe("sh",["-c",e],{cwd:t.cwd,env:{...process.env,...t.env},stdio:["ignore","pipe","pipe"]}),r=[],a=[];n.stdout.on("data",i=>r.push(i)),n.stderr.on("data",i=>a.push(i));let s=setTimeout(()=>{n.kill("SIGTERM"),setTimeout(()=>{n.killed||n.kill("SIGKILL")},1e3)},t.timeoutMs);n.on("close",(i,c)=>{if(clearTimeout(s),c==="SIGTERM"||c==="SIGKILL"){o(le({code:"PROVIDER_TIMEOUT",message:`Command timed out after ${t.timeoutMs}ms`}));return}o(Ze({stdout:Buffer.concat(r).toString("utf-8"),stderr:Buffer.concat(a).toString("utf-8"),exitCode:i??1}))}),n.on("error",i=>{clearTimeout(s),o(le({code:"UNKNOWN_ERROR",message:`Failed to spawn command: ${i.message}`}))})})}}}var it=1048576;async function H(e){let t=_(),o=!1,n=()=>{o&&process.exit(130),o=!0,t.stop(),console.error(v.yellow(`
|
|
46
|
+
Interrupted. Exiting...`)),process.exit(130)};process.on("SIGINT",n);try{return await at(e,t)}finally{process.removeListener("SIGINT",n)}}async function at(e,t){let o=tt(process.cwd(),e.configPath),n=nt(o);try{let u=et(o);u.size>it&&(console.error(v.red(`Config file exceeds 1MB limit (${(u.size/1048576).toFixed(1)}MB): ${o}`)),process.exit(1))}catch{console.error(v.red(`Config file not found: ${o}`)),process.exit(1)}let r;try{r=Qe(o,"utf-8")}catch{console.error(v.red(`Config file not found: ${o}`)),process.exit(1)}let a=$(),s=rt(r,{configDir:n,fileReader:a});if(!s.success){console.error(v.red("Config validation failed:"));let u=s.error.details;if(u&&Array.isArray(u.errors))for(let S of u.errors)console.error(v.red(` - ${S}`));else console.error(v.red(` ${s.error.message}`));process.exit(1)}let i=s.data;e.suite!==void 0&&i.suite.name!==e.suite&&(console.error(v.red(`Suite "${e.suite}" not found. Available suite: "${i.suite.name}"`)),process.exit(1)),e.runs!==void 0&&((!Number.isInteger(e.runs)||e.runs<1)&&(console.error(v.red(`Invalid --runs value: ${e.runs}. Must be a positive integer (>= 1).`)),process.exit(1)),i.defaults.repeat=e.runs),e.gate!==void 0&&((Number.isNaN(e.gate)||e.gate<0||e.gate>100)&&(console.error(v.red(`Invalid --gate value: ${e.gate}. Must be between 0 and 100.`)),process.exit(1)),e.gate>0&&e.gate<=1&&console.error(v.yellow(`Warning: --gate ${e.gate} looks like a decimal. Did you mean --gate ${Math.round(e.gate*100)}? (--gate uses 0-100 scale)`)),i.gates?i.gates.passRateMin=e.gate/100:i.gates={passRateMin:e.gate/100});let c=G(),f=new Map,m=i.providers;for(let[u,S]of Object.entries(m)){if(!S)continue;let U="";if(S.apiKeyEnv){let I=process.env[S.apiKeyEnv];I||(console.error(v.red(`Missing environment variable: ${S.apiKeyEnv}`)),process.exit(1)),U=I.trim()}else u!=="ollama"&&(console.error(v.red(`Provider "${u}" requires apiKeyEnv to be configured`)),process.exit(1));let M;try{M=ot(u,c)}catch(I){let Z=I instanceof Error?I.message:String(I);console.error(v.red(`Failed to create provider "${u}": ${Z}`)),process.exit(1)}await M.initialize({apiKey:U,baseUrl:S.baseUrl,organization:S.organization,timeoutMs:i.defaults.timeoutMs,maxRetries:2}),f.set(u,M)}let p=0,g=ct(i),d=u=>{u.type==="test_start"?t.start(`Running ${u.test} [${u.model}] (${p}/${g})`):u.type==="test_complete"&&p++},w=i.tests.some(u=>u.command)?de():void 0,l=await st(i,{adapters:f,configDir:n,fileReader:a,onProgress:d,baselineData:e.baselineData,commandExecutor:w}).run();return t.stop(),l.success||(console.error(v.red(`Run failed: ${l.error.message}`)),process.exit(1)),{config:i,runnerResult:l.data,configDir:n,yamlContent:r}}function ct(e){let t=0;for(let o of e.tests){if(o.skip)continue;let n=o.repeat??e.defaults.repeat;if(o.command)t+=n;else{let r=o.models?.length??e.models.length;t+=r*n}}return t}import{readFileSync as lt,writeFileSync as dt,mkdirSync as mt}from"fs";import{join as me}from"path";import{createHash as ut}from"crypto";function ue(){return me(process.cwd(),".kindlm","last-run.json")}function fe(e){let t=ue(),o=me(process.cwd(),".kindlm");mt(o,{recursive:!0,mode:448});let n={...e,runnerResult:{...e.runnerResult,aggregated:e.runnerResult.aggregated.map(r=>{let a=r.runs.flatMap(s=>s.assertions.filter(i=>!i.passed).map(i=>i.failureMessage)).filter(s=>s!==void 0);return{...r,failureMessages:a,runs:[]}})}};dt(t,JSON.stringify(n),{mode:384})}function pe(){try{let e=lt(ue(),"utf-8"),t=JSON.parse(e);return t.runnerResult?.runResult&&Array.isArray(t.runnerResult.aggregated)&&typeof t.suiteName=="string"&&typeof t.configHash=="string"&&typeof t.timestamp=="string"?t:null}catch{return null}}function ge(e){return ut("sha256").update(e).digest("hex")}import ft from"pdfkit";import{createWriteStream as pt}from"fs";import{mkdir as gt}from"fs/promises";import{dirname as ht}from"path";function yt(e){let t=e.split(`
|
|
47
|
+
`),o=[],n="",r=2,a=[];for(let s of t){let i=s.match(/^(#{2,4})\s+(.+)$/);i&&i[1]?.length===2?((n||a.length>0)&&o.push({heading:n,headingLevel:r,body:a.join(`
|
|
48
|
+
`).trim()}),n=i[2]?.trim()??"",r=2,a=[]):a.push(s)}return(n||a.length>0)&&o.push({heading:n,headingLevel:r,body:a.join(`
|
|
49
|
+
`).trim()}),o}function Rt(e){return e.match(/^# (.+)$/m)?.[1]?.trim()??"KindLM Compliance Report"}function Ct(e){return e.match(/SHA-256:\s*`([a-f0-9]+)`/i)?.[1]??null}function he(e){switch(e){case 2:return 18;case 3:return 15;case 4:return 13;default:return 13}}function vt(e){if(e.length<2)return null;let t=e[0]??"",o=e[1]??"";if(!t.includes("|")||!o.match(/^\s*\|[-:\s|]+\|\s*$/))return null;let n=s=>s.split("|").slice(1,-1).map(i=>i.trim()),r={cells:n(t)},a=[];for(let s=2;s<e.length;s++){let i=e[s]??"";if(!i.includes("|"))break;a.push({cells:n(i)})}return{header:r,rows:a}}function N(e,t){let o=e.page.margins.bottom;e.page.height-o-30-e.y<t&&(e.addPage(),ye(e))}function ye(e){let t=new Date().toISOString();e.fontSize(8).font("Helvetica").fillColor("#a8a29e").text("KindLM Compliance Report",60,40),e.text(t,60,40,{align:"right"}),e.moveDown(3)}function wt(e){e.fontSize(8).font("Helvetica").fillColor("#a8a29e").text("Generated by KindLM \xB7 kindlm.com",60,e.page.height-50,{align:"center",width:e.page.width-120})}function St(e,t,o){let n=t.header.cells.length,r=o/n,a=e.page.margins.left,s=18;N(e,s*2);let i=(c,f,m)=>{let p=e.y;m&&(e.save(),e.rect(a,p-2,o,s).fill(m),e.restore());for(let g=0;g<c.length;g++){let d=a+g*r;e.fontSize(8).font(f?"Helvetica-Bold":"Courier").fillColor("#44403c").text(c[g]??"",d+4,p,{width:r-8,height:s,lineBreak:!1})}e.y=p+s};i(t.header.cells,!0,"#f5f5f4");for(let c of t.rows)N(e,s),i(c.cells,!1)}async function Re(e,t){return await gt(ht(t),{recursive:!0}),new Promise((o,n)=>{let r=new ft({size:"A4",margins:{top:72,bottom:72,left:60,right:60},info:{Title:"KindLM EU AI Act Compliance Report",Author:"KindLM",Creator:"KindLM CLI"}}),a=pt(t);r.pipe(a);let s=r.page.width-r.page.margins.left-r.page.margins.right,i=Rt(e),c=Ct(e);r.moveDown(6),r.fontSize(28).font("Helvetica-Bold").fillColor("#1c1917").text(i,{align:"center",width:s}),r.moveDown(.5),r.fontSize(14).font("Helvetica").fillColor("#57534e").text("EU AI Act Annex IV Documentation",{align:"center",width:s}),r.moveDown(1),r.fontSize(10).fillColor("#a8a29e").text(`Generated: ${new Date().toISOString()}`,{align:"center",width:s}),c&&(r.moveDown(.3),r.fontSize(9).font("Courier").fillColor("#78716c").text(`SHA-256: ${c}`,{align:"center",width:s})),r.moveDown(2),r.fontSize(10).font("Helvetica").fillColor("#6366f1").text("kindlm.com",{align:"center",link:"https://kindlm.com",width:s});let f=yt(e);for(let m of f){if(r.addPage(),ye(r),m.heading){let y=he(m.headingLevel);r.fontSize(y).font("Helvetica-Bold").fillColor("#1c1917").text(m.heading,{width:s}),r.moveDown(.5),r.moveTo(60,r.y).lineTo(60+s,r.y).strokeColor("#e7e5e4").lineWidth(1).stroke(),r.moveDown(.8)}let p=m.body.split(`
|
|
50
|
+
`),g=!1,d=0;for(;d<p.length;){let y=p[d]??"";if(y.startsWith("```")){g=!g,g&&N(r,30),d++;continue}if(g){N(r,14);let l=r.y;r.save(),r.rect(r.page.margins.left,l-2,s,14).fill("#f5f5f4"),r.restore(),r.fontSize(9).font("Courier").fillColor("#44403c").text(y,{width:s}),d++;continue}if(!y.trim()){r.moveDown(.4),d++;continue}let w=p[d+1]??"";if(y.includes("|")&&d+1<p.length&&w.match(/^\s*\|[-:\s|]+\|\s*$/)){let l=[],u=d;for(;u<p.length&&(p[u]??"").includes("|");)l.push(p[u]??""),u++;let S=vt(l);if(S){St(r,S,s),d=u;continue}}if(y.match(/^\s*\|[-:]+/)||y.match(/^---+$/)){d++;continue}let R=y.match(/^(#{3,4})\s+(.+)$/);if(R?.[1]&&R[2]){let l=R[1].length,u=he(l);N(r,u+10),r.moveDown(.3),r.fontSize(u).font("Helvetica-Bold").fillColor("#1c1917").text(R[2].trim(),{width:s}),r.moveDown(.3),d++;continue}if(y.match(/^\s*[-*] /)){N(r,14),r.fontSize(10).font("Helvetica").fillColor("#44403c").text(y.trim(),{indent:12,width:s-12}),d++;continue}N(r,14),r.fontSize(10).font("Helvetica").fillColor("#44403c").text(y.trim(),{width:s}),d++}wt(r)}r.end(),a.on("finish",()=>o(t)),a.on("error",n)})}import k from"chalk";import{createPrettyReporter as Tt,createJsonReporter as bt,createJunitReporter as kt}from"@kindlm/core";var Et={bold:e=>k.bold(e),red:e=>k.red(e),green:e=>k.green(e),yellow:e=>k.yellow(e),cyan:e=>k.cyan(e),dim:e=>k.dim(e),greenBold:e=>k.green.bold(e),redBold:e=>k.red.bold(e)},xt=["pretty","json","junit"];function J(e){switch(e){case"json":return bt();case"junit":return kt();case"pretty":return Tt(Et);default:console.error(k.red(`Unknown reporter: '${e}'. Available: ${xt.join(", ")}`)),process.exit(1)}}import{execSync as Q}from"child_process";function W(){try{let e=Q("git rev-parse HEAD",{encoding:"utf-8"}).trim()||null,t=Q("git rev-parse --abbrev-ref HEAD",{encoding:"utf-8"}).trim()||null,n=Q("git status --porcelain",{encoding:"utf-8"}).trim().length>0;return{commitSha:e,branch:t,dirty:n}}catch{return{commitSha:null,branch:null,dirty:!1}}}function Ce(e){e.command("test").description("Run test suites").option("-s, --suite <name>","Run a specific suite").option("--compliance","Generate compliance report").option("--reporter <type>","Output format: pretty, json, junit","pretty").option("--runs <count>","Override run count").option("--gate <percent>","Fail if pass rate below threshold").option("--pdf <path>","Export compliance report as PDF (requires --compliance)").option("-c, --config <path>","Path to config file","kindlm.yaml").action(async t=>{t.pdf&&!t.compliance&&(console.error(L.red("--pdf requires --compliance")),process.exit(1));let o=J(t.reporter);try{let{runnerResult:n,config:r,yamlContent:a}=await H({configPath:t.config,runs:t.runs!==void 0?parseInt(t.runs,10):void 0,gate:t.gate!==void 0?parseFloat(t.gate):void 0,suite:t.suite}),{runResult:s,aggregated:i}=n,c=It(r.gates,i),f=await o.generate(s,c);console.log(f.content);let m,p;if(t.compliance){let d=W(),y={runId:crypto.randomUUID(),kindlmVersion:"1.0.0",gitCommitSha:d.commitSha??void 0,modelIds:r.models.map(u=>u.id),...r.compliance?.metadata??{}};if(m=(await Pt(y).generate(s,c)).content,p=m.match(/Tamper Evidence Hash \(SHA-256\):\*\* `([a-f0-9]{64})`/)?.[1],t.reporter==="pretty"||!t.reporter?(console.log(""),console.log(m)):(console.error(""),console.error(m)),t.pdf){let u=await Re(m,t.pdf);console.log(""),console.log(L.green(`PDF report saved to ${u}`))}}try{fe({runnerResult:n,suiteName:r.suite.name,configHash:ge(a),timestamp:new Date().toISOString(),complianceReport:m,complianceHash:p})}catch{}let g=s.failed===0&&s.errored===0&&c.passed;process.exit(g?0:1)}catch(n){if(n instanceof $t){let r=n.code==="TIMEOUT"?"Provider timeout":n.code==="NETWORK_ERROR"?"Network error":n.code==="AUTH_FAILED"?"Authentication failed":n.code==="RATE_LIMITED"?"Rate limited":`Provider error (${n.code})`;console.error(L.red(`${r}: ${n.message}`)),n.retryable&&console.error(L.yellow("This error may be transient. Try again or increase timeoutMs in your kindlm.yaml defaults."))}else if(Ot(n)){let a=n.code.startsWith("CONFIG_")?"Config error":"Error";console.error(L.red(`${a}: ${n.message}`))}else n instanceof Error&&n.name==="AbortError"?console.error(L.red("Request timed out. Check network connectivity or increase timeout.")):console.error(L.red(`Error: ${n instanceof Error?n.message:String(n)}`));process.exit(1)}})}function Ot(e){return typeof e=="object"&&e!==null&&"code"in e&&"message"in e&&typeof e.code=="string"&&typeof e.message=="string"}import{resolve as Y,dirname as te,join as ne}from"path";import{readFileSync as Lt}from"fs";import h from"chalk";import{parseConfig as _t,readBaseline as Dt,writeBaseline as jt,listBaselines as Ht,buildBaselineData as Se,compareBaseline as Ft,deserializeBaseline as Bt}from"@kindlm/core";import{readFileSync as Mt,writeFileSync as At,mkdirSync as ve,readdirSync as Nt}from"fs";import{join as ee}from"path";function we(e){return e.replace(/[^a-zA-Z0-9_-]/g,"_")}function V(e){let t=ee(e,"baselines");return{read(o){let n=ee(t,`${we(o)}.json`);try{return{success:!0,data:Mt(n,"utf-8")}}catch{return{success:!1,error:{code:"BASELINE_NOT_FOUND",message:`No baseline found for suite "${o}" at ${n}`}}}},write(o,n){try{ve(t,{recursive:!0});let r=ee(t,`${we(o)}.json`);return At(r,n,"utf-8"),{success:!0,data:void 0}}catch(r){return{success:!1,error:{code:"UNKNOWN_ERROR",message:`Failed to write baseline: ${r instanceof Error?r.message:String(r)}`}}}},list(){try{return ve(t,{recursive:!0}),{success:!0,data:Nt(t).filter(r=>r.endsWith(".json")).map(r=>r.replace(/\.json$/,""))}}catch(o){return{success:!1,error:{code:"UNKNOWN_ERROR",message:`Failed to list baselines: ${o instanceof Error?o.message:String(o)}`}}}}}}function Te(e){let t=e.command("baseline").description("Manage test baselines");t.command("set").description("Save current results as baseline").option("-c, --config <path>","Path to config file","kindlm.yaml").option("--runs <count>","Override run count").option("--force","Save baseline even if all tests failed").action(async o=>{try{let n=te(Y(process.cwd(),o.config)),r=ne(n,".kindlm"),a=V(r),{config:s,runnerResult:i}=await H({configPath:o.config,runs:o.runs!==void 0?parseInt(o.runs,10):void 0}),{runResult:c,aggregated:f}=i;(c.totalTests>0?c.passed/c.totalTests:0)===0&&!o.force&&(console.error(h.red("All tests failed or errored. Refusing to save a failing baseline.")),console.error(h.yellow("Use --force to save anyway.")),process.exit(1));let p=Se(s.suite.name,f,new Date().toISOString()),g=jt(p,a);g.success||(console.error(h.red(`Failed to save baseline: ${g.error.message}`)),process.exit(1));let d=Object.keys(p.results).length;console.log(""),console.log(h.green(`Baseline saved for suite "${s.suite.name}" (${d} test${d===1?"":"s"})`)),console.log(h.dim(` Location: ${r}/baselines/`))}catch(n){console.error(h.red(`Error: ${n instanceof Error?n.message:String(n)}`)),process.exit(1)}}),t.command("compare").description("Compare latest against baseline").option("-c, --config <path>","Path to config file","kindlm.yaml").option("--runs <count>","Override run count").action(async o=>{try{let n=te(Y(process.cwd(),o.config)),r=ne(n,".kindlm"),a=V(r),s=Y(process.cwd(),o.config),i;try{i=Lt(s,"utf-8")}catch{console.error(h.red(`Config file not found: ${s}`)),process.exit(1)}let c=$(),f=_t(i,{configDir:n,fileReader:c});f.success||(console.error(h.red(`Config validation failed: ${f.error.message}`)),process.exit(1));let m=f.data.suite.name,p=Dt(m,a);p.success||(p.error.code==="BASELINE_NOT_FOUND"?console.error(h.red(`No baseline found for suite "${m}". Run \`kindlm baseline set\` first.`)):console.error(h.red(`Failed to read baseline: ${p.error.message}`)),process.exit(1));let g=p.data,{runnerResult:d}=await H({configPath:o.config,runs:o.runs!==void 0?parseInt(o.runs,10):void 0,baselineData:g}),{aggregated:y}=d,w=Se(m,y,new Date().toISOString()),R=Ft(g,w.results);if(console.log(""),console.log(h.bold(`Baseline comparison for "${m}"`)),console.log(h.dim(` Baseline from: ${g.createdAt}`)),console.log(""),R.regressions.length>0){console.log(h.red.bold(` Regressions (${R.regressions.length}):`));for(let l of R.regressions)console.log(h.red(` ${l.testName}: ${F(l.baselinePassRate)} \u2192 ${F(l.currentPassRate)}`)),l.newFailureCodes.length>0&&console.log(h.red(` New failures: ${l.newFailureCodes.join(", ")}`));console.log("")}if(R.improvements.length>0){console.log(h.green.bold(` Improvements (${R.improvements.length}):`));for(let l of R.improvements)console.log(h.green(` ${l.testName}: ${F(l.baselinePassRate)} \u2192 ${F(l.currentPassRate)}`));console.log("")}if(R.unchanged.length>0){console.log(h.dim(` Unchanged (${R.unchanged.length}):`));for(let l of R.unchanged)console.log(h.dim(` ${l.testName}: ${F(l.passRate)}`));console.log("")}if(R.newTests.length>0){console.log(h.cyan(` New tests (${R.newTests.length}):`));for(let l of R.newTests)console.log(h.cyan(` ${l}`));console.log("")}if(R.removedTests.length>0){console.log(h.yellow(` Removed tests (${R.removedTests.length}):`));for(let l of R.removedTests)console.log(h.yellow(` ${l}`));console.log("")}process.exit(R.regressions.length>0?1:0)}catch(n){console.error(h.red(`Error: ${n instanceof Error?n.message:String(n)}`)),process.exit(1)}}),t.command("list").description("List saved baselines").option("-c, --config <path>","Path to config file","kindlm.yaml").action(o=>{try{let n=te(Y(process.cwd(),o.config)),r=ne(n,".kindlm"),a=V(r),s=Ht(a);s.success||(console.error(h.red(`Failed to list baselines: ${s.error.message}`)),process.exit(1));let i=s.data;if(i.length===0){console.log(h.dim("No baselines saved yet. Run `kindlm baseline set` to create one."));return}console.log(h.bold("Saved baselines:")),console.log("");for(let c of i){let f=a.read(c);if(!f.success){console.log(` ${c} ${h.dim("(unreadable)")}`);continue}let m=Bt(f.data);if(!m.success){console.log(` ${c} ${h.dim("(corrupt)")}`);continue}let p=Object.keys(m.data.results).length;console.log(` ${h.cyan(m.data.suiteName)} \u2014 ${p} test${p===1?"":"s"}, saved ${h.dim(m.data.createdAt)}`)}}catch(n){console.error(h.red(`Error: ${n instanceof Error?n.message:String(n)}`)),process.exit(1)}})}function F(e){return`${(e*100).toFixed(1)}%`}import{createInterface as Yt}from"readline";import{Writable as Xt}from"stream";import x from"chalk";import{readFileSync as Ut,writeFileSync as Kt,mkdirSync as zt,unlinkSync as Gt}from"fs";import{join as be}from"path";import{homedir as ke}from"os";function re(){return be(ke(),".kindlm","credentials")}function X(){try{let e=Ut(re(),"utf-8"),t=JSON.parse(e);return typeof t.token=="string"&&t.token.length>0?t.token:null}catch{return null}}function Ee(e){let t=re(),o=be(ke(),".kindlm");zt(o,{recursive:!0,mode:448});let n={token:e,savedAt:new Date().toISOString()};Kt(t,JSON.stringify(n,null,2),{mode:384})}function xe(){try{Gt(re())}catch{}}var Jt="https://api.kindlm.com";var E=class extends Error{status;constructor(t,o){super(o),this.name="CloudApiError",this.status=t}};function D(){let e=process.env.KINDLM_CLOUD_URL??Jt;if(e.startsWith("http://")&&!Wt(e))throw new Error(`Refusing to use insecure HTTP for Cloud API: ${e}. Use HTTPS or target localhost for development.`);return e}function Wt(e){try{let t=new URL(e);return t.hostname==="localhost"||t.hostname==="127.0.0.1"||t.hostname==="::1"}catch{return!1}}function Vt(e){return new Promise(t=>setTimeout(t,e))}function B(e,t){async function o(n,r,a){let s=`${e}${r}`,i={Authorization:`Bearer ${t}`},c={method:n,headers:i};a!==void 0&&(i["Content-Type"]="application/json",c.body=JSON.stringify(a));let f;for(let m=0;m<=1;m++){m>0&&await Vt(1e3);let p=new AbortController,g=setTimeout(()=>p.abort(),3e4);c.signal=p.signal;try{let d=await fetch(s,c);if(!d.ok){if(d.status>=500&&m<1){f=new E(d.status,`HTTP ${d.status}`);continue}let w=`HTTP ${d.status}`;if((d.headers.get("content-type")??"").includes("application/json"))try{let l=await d.json();l.error&&(w=l.error)}catch{}throw new E(d.status,w)}if(d.status===204)return;let y=d.headers.get("content-type")??"";if(!y.includes("application/json"))throw new E(d.status,`Expected JSON response but got content-type: ${y}`);return await d.json()}catch(d){if(d instanceof E)throw d;if(f=d instanceof Error?d:new Error(String(d)),m<1)continue}finally{clearTimeout(g)}}throw f??new Error("Request failed")}return{baseUrl:e,get:n=>o("GET",n),post:(n,r)=>o("POST",n,r),patch:(n,r)=>o("PATCH",n,r),delete:n=>o("DELETE",n)}}function Ie(e){e.command("login").description("Authenticate with KindLM Cloud").option("-t, --token <token>","API token (skips interactive prompt)").option("--status","Show current authentication status").option("--logout","Remove stored credentials").action(async t=>{try{if(t.logout){xe(),console.log(x.green("Logged out. Credentials removed."));return}if(t.status){await qt();return}let o=t.token??process.env.KINDLM_API_TOKEN??await Zt();o.startsWith("klm_")||(console.error(x.red('Invalid token format. KindLM tokens start with "klm_".')),process.exit(1));let n=B(D(),o);try{await n.get("/v1/auth/tokens")}catch(r){throw r instanceof E&&r.status===401&&(console.error(x.red("Invalid or expired token.")),process.exit(1)),r}Ee(o),console.log(x.green("Authenticated successfully. Token saved."))}catch(o){console.error(x.red(`Login failed: ${o instanceof Error?o.message:String(o)}`)),process.exit(1)}})}async function qt(){let e=X();if(!e){console.log(x.yellow('Not authenticated. Run "kindlm login" to authenticate.'));return}let t=B(D(),e);try{await t.get("/v1/auth/tokens"),console.log(x.green("Authenticated.")),console.log(` Cloud URL: ${D()}`)}catch(o){o instanceof E&&o.status===401?console.log(x.yellow('Stored token is invalid or expired. Run "kindlm login" to re-authenticate.')):console.log(x.yellow(`Cannot reach Cloud API: ${o instanceof Error?o.message:String(o)}`))}}function Zt(){return new Promise((e,t)=>{let o=new Xt({write(r,a,s){s()}});process.stderr.write("Paste your KindLM API token: ");let n=Yt({input:process.stdin,output:o,terminal:!0});n.question("",r=>{n.close(),process.stderr.write(`
|
|
51
|
+
`);let a=r.trim();if(!a){t(new Error("No token provided"));return}e(a)})})}import{basename as nn}from"path";import{execSync as rn}from"child_process";import oe from"chalk";function Pe(){return process.env.GITHUB_ACTIONS?{name:"github_actions",isCI:!0,commitSha:process.env.GITHUB_SHA??null,branch:process.env.GITHUB_REF_NAME??null}:process.env.GITLAB_CI?{name:"gitlab_ci",isCI:!0,commitSha:process.env.CI_COMMIT_SHA??null,branch:process.env.CI_COMMIT_BRANCH??null}:process.env.CI?{name:null,isCI:!0,commitSha:null,branch:null}:{name:null,isCI:!1,commitSha:null,branch:null}}function j(e){return encodeURIComponent(e)}async function $e(e,t,o){let n=await Qt(e,o.projectName),r=await en(e,n,o.suiteName,o.configHash),a=await e.post(`/v1/projects/${j(n)}/runs`,{suiteId:r,commitSha:o.commitSha,branch:o.branch,environment:o.environment,triggeredBy:o.triggeredBy}),s=tn(t.aggregated),i=50;try{for(let l=0;l<s.length;l+=i){let u=s.slice(l,l+i);await e.post(`/v1/runs/${j(a.id)}/results`,{results:u})}}catch(l){try{await e.patch(`/v1/runs/${j(a.id)}`,{status:"failed"})}catch{}throw l}let{runResult:c}=t,f=c.totalTests>0?c.passed/c.totalTests:0,m=new Set(t.aggregated.map(l=>l.modelId)),p=t.aggregated.map(l=>l.assertionScores.judge?.mean).filter(l=>l!==void 0),g=p.length>0?p.reduce((l,u)=>l+u,0)/p.length:void 0,d=t.aggregated.map(l=>l.latencyAvgMs),y=d.length>0?d.reduce((l,u)=>l+u,0)/d.length:void 0,w=t.aggregated.reduce((l,u)=>l+u.totalCostUsd,0),R=w>0?w:void 0;return await e.patch(`/v1/runs/${j(a.id)}`,{status:"completed",passRate:f,testCount:c.totalTests,modelCount:m.size,judgeAvgScore:g,latencyAvgMs:y,costEstimateUsd:R,complianceReport:o.complianceReport,complianceHash:o.complianceHash,finishedAt:new Date().toISOString()}),{runId:a.id,projectId:n}}async function Qt(e,t){let{projects:o}=await e.get("/v1/projects"),n=o.find(a=>a.name===t);return n?n.id:(await e.post("/v1/projects",{name:t})).id}async function en(e,t,o,n){let{suites:r}=await e.get(`/v1/projects/${j(t)}/suites`),a=r.find(i=>i.name===o);return a?a.id:(await e.post(`/v1/projects/${j(t)}/suites`,{name:o,configHash:n})).id}function tn(e){return e.map(t=>{let o=t,n=t.runs.length>0?t.runs.flatMap(r=>r.assertions.filter(a=>!a.passed).map(a=>a.failureMessage)).filter(r=>r!==void 0):o.failureMessages??[];return{testCaseName:t.testCaseName,modelId:t.modelId,passed:t.passed?1:0,passRate:t.passRate,runCount:t.runCount,judgeAvg:t.assertionScores.judge?.mean??null,driftScore:t.assertionScores.drift?.mean??null,latencyAvgMs:t.latencyAvgMs??null,costUsd:t.totalCostUsd??null,totalTokens:t.totalTokens??null,failureCodes:t.failureCodes.length>0?JSON.stringify(t.failureCodes):null,failureMessages:n.length>0?JSON.stringify(n):null,assertionScores:Object.keys(t.assertionScores).length>0?JSON.stringify(t.assertionScores):null}})}function Oe(e){e.command("upload").description("Push last run results to KindLM Cloud").option("-t, --token <token>","API token (overrides stored token)").option("-p, --project <name>","Project name").action(async t=>{try{let o=t.token??process.env.KINDLM_API_TOKEN??X();o||(console.error(oe.red('Not authenticated. Run "kindlm login" first or pass --token.')),process.exit(1));let n=pe();n||(console.error(oe.red('No test run found. Run "kindlm test" first.')),process.exit(1));let r=W(),a=Pe(),s=t.project??sn(),i=B(D(),o),c=_();c.start("Uploading results to KindLM Cloud...");try{let f=await $e(i,n.runnerResult,{projectName:s,suiteName:n.suiteName,configHash:n.configHash,commitSha:a.commitSha??r.commitSha??void 0,branch:a.branch??r.branch??void 0,environment:a.isCI?"ci":"local",triggeredBy:a.name??"local",complianceReport:n.complianceReport,complianceHash:n.complianceHash});c.succeed("Uploaded successfully."),console.log(` Run ID: ${f.runId}`),console.log(` Project: ${s}`),console.log(` Suite: ${n.suiteName}`)}catch(f){throw c.fail("Upload failed."),f}}catch(o){console.error(oe.red(`Upload failed: ${o instanceof Error?o.message:String(o)}`)),process.exit(1)}})}function on(e){try{let n=new URL(e).pathname.split("/").filter(Boolean),r=n[n.length-1];if(r)return r.replace(/\.git$/,"")}catch{}let t=e.match(/^[\w.-]+@[\w.-]+:(.+?)(?:\.git)?$/);if(t?.[1]){let o=t[1].split("/");return o[o.length-1]??null}return null}function sn(){try{let e=rn("git remote get-url origin",{encoding:"utf-8"}).trim(),t=on(e);if(t)return t}catch{}return nn(process.cwd())}import{readFileSync as ln,statSync as dn}from"fs";import{resolve as mn,dirname as un}from"path";import{spawn as fn}from"child_process";import q from"chalk";import{parseConfig as pn,createProvider as gn,filterSpans as hn,mapSpansToResult as yn,buildContextFromTrace as Rn,createAssertionsFromExpect as Cn}from"@kindlm/core";import{createServer as an}from"http";import{parseOtlpPayload as cn}from"@kindlm/core";function Me(e){let t=[],o=null,n=[];function r(){for(let s of n)s()}function a(s,i){if(i.setHeader("Access-Control-Allow-Origin","*"),i.setHeader("Access-Control-Allow-Methods","POST, OPTIONS"),i.setHeader("Access-Control-Allow-Headers","Content-Type"),s.method==="OPTIONS"){i.writeHead(204),i.end();return}if(s.method!=="POST"||s.url!=="/v1/traces"){i.writeHead(404,{"Content-Type":"application/json"}),i.end(JSON.stringify({error:"Not found"}));return}let c=10*1024*1024,f=[],m=0,p=!1;s.on("data",g=>{if(m+=g.length,m>c){p=!0,i.writeHead(413,{"Content-Type":"application/json"}),i.end(JSON.stringify({error:"Payload too large"})),s.destroy();return}f.push(g)}),s.on("end",()=>{if(!p)try{let g=Buffer.concat(f).toString("utf-8"),d=JSON.parse(g),y=cn(d);y.success?(t.push(...y.data),r(),i.writeHead(200,{"Content-Type":"application/json"}),i.end(JSON.stringify({partialSuccess:{}}))):(i.writeHead(400,{"Content-Type":"application/json"}),i.end(JSON.stringify({error:y.error.message})))}catch{i.writeHead(400,{"Content-Type":"application/json"}),i.end(JSON.stringify({error:"Invalid JSON"}))}})}return{start(){return new Promise((s,i)=>{o=an(a),o.on("error",i),o.listen(e,()=>s())})},stop(){return new Promise(s=>{o?o.close(()=>s()):s()})},getSpans(){return[...t]},waitForSpans({timeoutMs:s}){return new Promise(i=>{if(t.length>0){i([...t]);return}let c=setTimeout(()=>{n=n.filter(m=>m!==f),i([...t])},s),f=()=>{clearTimeout(c),n=n.filter(m=>m!==f),setTimeout(()=>i([...t]),500)};n.push(f)})}}}function Ae(e){e.command("trace").description("Ingest OpenTelemetry traces and run assertions against them").option("-c, --config <path>","Config file path","kindlm.yaml").option("--port <port>","OTLP HTTP port","4318").option("--command <cmd>","Command to spawn (traces are collected while it runs)").option("--timeout <ms>","Timeout in ms to wait for traces","30000").option("--reporter <type>","Report format: pretty, json, junit","pretty").action(async t=>{let o=_();try{let n=mn(process.cwd(),t.config),r=un(n);try{dn(n).size>1048576&&(console.error(q.red("Config file exceeds 1MB limit")),process.exit(1))}catch{console.error(q.red(`Config file not found: ${n}`)),process.exit(1)}let a;try{a=ln(n,"utf-8")}catch{console.error(q.red(`Config file not found: ${n}`)),process.exit(1)}let s=$(),i=pn(a,{configDir:r,fileReader:s});i.success||(console.error(q.red(`Config validation failed: ${i.error.message}`)),process.exit(1));let c=i.data,f=c.trace??{port:parseInt(t.port,10),timeoutMs:parseInt(t.timeout,10),spanMapping:{outputTextAttr:"gen_ai.completion.0.content",modelAttr:"gen_ai.response.model",systemAttr:"gen_ai.system",inputTokensAttr:"gen_ai.usage.input_tokens",outputTokensAttr:"gen_ai.usage.output_tokens"}},m=parseInt(t.port,10)||f.port,p=parseInt(t.timeout,10)||f.timeoutMs,g=Me(m);await g.start();let d;try{o.start(`Listening for OTLP traces on port ${m}...`),t.command&&(d=fn("sh",["-c",t.command],{cwd:r,env:{...process.env,OTEL_EXPORTER_OTLP_ENDPOINT:`http://localhost:${m}`,OTEL_EXPORTER_OTLP_PROTOCOL:"http/json"},stdio:"inherit"}),d.on("error",C=>{o.fail(`Command failed: ${C.message}`)}));let y=await g.waitForSpans({timeoutMs:p});y.length===0&&(o.fail("No traces received"),process.exit(1)),o.succeed(`Received ${y.length} spans`);let w=hn(y,f.spanFilter),R=yn(w,f.spanMapping),l=G(),u=new Map,S=c.providers;for(let[C,T]of Object.entries(S)){if(!T)continue;let P="";if(T.apiKeyEnv){let b=process.env[T.apiKeyEnv];b&&(P=b.trim())}if(!(!P&&C!=="ollama"))try{let b=gn(C,l);await b.initialize({apiKey:P,baseUrl:T.baseUrl,organization:T.organization,timeoutMs:c.defaults.timeoutMs,maxRetries:2}),u.set(C,b)}catch{}}let U=c.defaults.judgeModel??c.models[0]?.id,M=c.models.find(C=>C.id===U),I=M?u.get(M.provider):void 0,Z=Rn(R,{configDir:r,judgeAdapter:I,judgeModel:M?.model}),se=[];for(let C of c.tests){if(C.skip)continue;let T=Cn(C.expect),P=[];for(let b of T){let He=await b.evaluate(Z);P.push(...He)}se.push({testName:C.name,assertions:P})}let Ne=c.tests.filter(C=>C.skip).length,ie=R.latencyMs,K=se.map(({testName:C,assertions:T})=>{let P=T.every(b=>b.passed);return{name:C,modelId:"trace",status:P?"passed":"failed",assertions:T,latencyMs:ie,costUsd:0}}),Le=K.filter(C=>C.status==="passed").length,z=K.filter(C=>C.status==="failed").length,_e={suites:[{name:c.suite.name,status:z>0?"failed":"passed",tests:K}],totalTests:K.length,passed:Le,failed:z,errored:0,skipped:Ne,durationMs:ie},De={passed:z===0,gates:[]},je=await J(t.reporter).generate(_e,De);console.log(je.content),process.exit(z>0?1:0)}finally{d?.kill(),await g.stop()}}catch(n){o.fail(`Trace command failed: ${n instanceof Error?n.message:String(n)}`),process.exit(1)}})}function To(){let e=new vn;return e.name("kindlm").description("AI agent behavioral regression testing").version("1.0.0"),ae(e),ce(e),Ce(e),Te(e),Ie(e),Oe(e),Ae(e),e}export{To as createProgram};
|
|
52
52
|
//# sourceMappingURL=index.js.map
|