@sreetej510/pi-shipd-checks 0.8.1 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -103,7 +103,7 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
|
|
|
103
103
|
Required behavior: ${s.requiredBehavior}
|
|
104
104
|
Recommendation: ${s.recommendation}`)].join(`
|
|
105
105
|
`)}import{spawn as rPo}from"node:child_process";import{existsSync as xAe,mkdirSync as oPo}from"node:fs";import{mkdtemp as aPo,readFile as AAe,rm as iPo}from"node:fs/promises";import{createServer as cPo}from"node:net";import{homedir as lPo,tmpdir as uPo}from"node:os";import{dirname as pPo,join as $m}from"node:path";import{Text as dL}from"@earendil-works/pi-tui";import{chromium as CEn}from"playwright-core";var Qee="quality-check",DAe="shipd:link",bAe="shipd_job_link",fEn=$m(lPo(),".pi","agent","shipd-auth","shipd.ai.json"),gEn="https://shipd.ai/",REn=9e5,dPo=18e5,mPo=12e4,fPo=3e5,gPo=9e4,Zy=5e3,EAe=["Test Quality","Solution Quality"],RPo=Ct.Object({});function wC(e){return(e??"").trim().replace(/\s+/g," ")}function SPo(e){return e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}function wm(e){return e!==null&&typeof e=="object"&&!Array.isArray(e)}function mL(e,t){return new Promise((s,a)=>{if(t?.aborted){a(new Error("Cancelled by user."));return}let c=()=>{clearTimeout(l),t?.removeEventListener("abort",c),a(new Error("Cancelled by user."))},l=setTimeout(()=>{t?.removeEventListener("abort",c),s()},e);t?.addEventListener("abort",c,{once:!0})})}function dc(e){if(e?.aborted)throw new Error("Cancelled by user.")}function SEn(){return[process.env.SHIPD_CHROME_PATH,process.env.PROGRAMFILES&&$m(process.env.PROGRAMFILES,"Google","Chrome","Application","chrome.exe"),process.env["PROGRAMFILES(X86)"]&&$m(process.env["PROGRAMFILES(X86)"],"Google","Chrome","Application","chrome.exe"),process.env.LOCALAPPDATA&&$m(process.env.LOCALAPPDATA,"Google","Chrome","Application","chrome.exe"),"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome","/usr/bin/google-chrome","/usr/bin/chromium","/usr/bin/chromium-browser"].filter(t=>!!t).find(t=>xAe(t))}function yPo(){let e=cPo();return new Promise((t,s)=>{e.once("error",s),e.listen(0,"127.0.0.1",()=>{let a=e.address();if(!a||typeof a=="string"){e.close(),s(new Error("Could not allocate a local Chrome debugging port."));return}e.close(c=>c?s(c):t(a.port))})})}async function IPo(e,t,s=15e3){let a=Date.now()+s;for(;Date.now()<a;){if(t.exitCode!==null)throw new Error("Chrome exited before its debugging connection became available.");try{if((await fetch(`http://127.0.0.1:${e}/json/version`)).ok)return}catch{}await mL(250)}throw new Error("Timed out waiting for Chrome to start.")}async function _Po(e,t=3e4){let s=Date.now()+t;for(;Date.now()<s;){let a=e.pages();for(let c of a){if(!kC(c.url()))continue;if(await c.locator('button[aria-label="Open user menu"]').isVisible({timeout:1e3}).catch(()=>!1))return c}for(let c of a)kC(c.url())&&await c.reload({waitUntil:"domcontentloaded",timeout:15e3}).catch(()=>{});await mL(1e3)}}function kC(e){try{let t=new URL(e),s=t.hostname.toLowerCase();return t.protocol==="https:"&&(s==="shipd.ai"||s.endsWith(".shipd.ai"))}catch{return!1}}function hPo(e){let t;for(let s of e){if(!wm(s)||s.type!=="custom"||s.customType!==bAe)continue;let a=s.data;wm(a)&&(typeof a.url=="string"&&kC(a.url)?t=a.url:a.url===null&&(t=void 0))}return t}function TEn(e,t){return e.locator("span").filter({hasText:new RegExp(`^${SPo(t)}$`)}).first()}function PAe(e,t){return TEn(e,t).locator("..")}function vPo(e,t){return e.locator("div.fixed").filter({hasText:t}).last()}function AEn(e){return/(?:starting|queued|pending|running|processing|reviewing|finalizing|building|evaluating|checking|\d+%)/i.test(e)}function yEn(e){return/\bstale\b/i.test(e)}function CPo(e,t){let s=e.trim().replace(/^```(?:json)?\s*/i,"").replace(/\s*```$/i,"").trim(),a;try{a=JSON.parse(s)}catch(c){throw new Error(`${t} raw output was not valid JSON: ${c instanceof Error?c.message:String(c)}`)}if(!wm(a))throw new Error(`${t} raw output was not a JSON object.`);return a}async function TPo(e,t,s){let a=$m(t,"create_patches.sh"),c=$m(t,"agent_prompt.md"),l=$m(t,"test.patch"),u=$m(t,"solution.patch"),p=[["create_patches.sh",a],["agent_prompt.md",c],["test.patch",l],["solution.patch",u]].filter(([,T])=>!xAe(T)).map(([T])=>T);if(p.length)throw new Error(`Missing required file(s) in the working directory: ${p.join(", ")}`);dc(s);let d=await e.exec(OC(),["create_patches.sh"],{cwd:t,timeout:mPo,signal:s});if(d.killed||d.code!==0){let T=(d.stderr||d.stdout||"").trim().slice(-2e3);throw new Error(`create_patches.sh failed${d.killed?" or timed out":` (exit ${d.code})`}.${T?`
|
|
106
|
-
${T}`:""}`)}dc(s);let[m,S,v]=await Promise.all([AAe(c,"utf8"),AAe(l,"utf8"),AAe(u,"utf8")]);return{taskPrompt:m,testPatch:S,solutionPatch:v}}async function APo(e,t,s){dc(s);let a=await e.goto(t,{waitUntil:"domcontentloaded",timeout:6e4});if(dc(s),!kC(e.url()))throw new Error("Shipd authentication or job-link navigation failed. Check the saved job link and auth state.");if(a&&a.status()>=400)throw new Error(`Shipd job-link navigation returned HTTP ${a.status()}.`);await e.waitForTimeout(4e3),dc(s)}async function IEn(e,t,s,a){dc(a);let c=await CEn.launch({executablePath:e,headless:!0});try{let l=await c.newContext({storageState:t});await l.route("**/*",async p=>{let d=p.request().resourceType();d==="image"||d==="font"||d==="media"?await p.abort():await p.continue()});let u=await l.newPage();return await APo(u,s,a),{browser:c,page:u}}catch(l){throw await c.close().catch(()=>{}),l}}async function _En(e,t,s){await t.click({timeout:Zy}),await e.keyboard.press(`${process.platform==="darwin"?"Meta":"Control"}+A`),await e.keyboard.insertText(s)}async function DPo(e,t,s){dc(s);let a=e.locator("#problem-description"),c=e.locator('#problem-test-patch [contenteditable="true"][role="textbox"]'),l=e.locator('#problem-solution-patch [contenteditable="true"][role="textbox"]');for(let u of[a,c,l])if(await u.count()!==1)throw new Error("Shipd challenge form selectors were not found exactly once.");await a.fill(t.taskPrompt),await _En(e,c,t.testPatch),await _En(e,l,t.solutionPatch),await PAe(e,"Test Quality").hover(),await mL(1e3,s)}async function bPo(e,t,s){dc(s);let a=PAe(e,t),c=wC(await a.innerText());if(AEn(c))return{quality:t,status:"already-running",rowTextAfterConfirm:c};let l=a.locator('button[title^="Re-run"]'),u=await l.count();if(u!==1)throw new Error(`${t}: expected one
|
|
106
|
+
${T}`:""}`)}dc(s);let[m,S,v]=await Promise.all([AAe(c,"utf8"),AAe(l,"utf8"),AAe(u,"utf8")]);return{taskPrompt:m,testPatch:S,solutionPatch:v}}async function APo(e,t,s){dc(s);let a=await e.goto(t,{waitUntil:"domcontentloaded",timeout:6e4});if(dc(s),!kC(e.url()))throw new Error("Shipd authentication or job-link navigation failed. Check the saved job link and auth state.");if(a&&a.status()>=400)throw new Error(`Shipd job-link navigation returned HTTP ${a.status()}.`);await e.waitForTimeout(4e3),dc(s)}async function IEn(e,t,s,a){dc(a);let c=await CEn.launch({executablePath:e,headless:!0});try{let l=await c.newContext({storageState:t});await l.route("**/*",async p=>{let d=p.request().resourceType();d==="image"||d==="font"||d==="media"?await p.abort():await p.continue()});let u=await l.newPage();return await APo(u,s,a),{browser:c,page:u}}catch(l){throw await c.close().catch(()=>{}),l}}async function _En(e,t,s){await t.click({timeout:Zy}),await e.keyboard.press(`${process.platform==="darwin"?"Meta":"Control"}+A`),await e.keyboard.insertText(s)}async function DPo(e,t,s){dc(s);let a=e.locator("#problem-description"),c=e.locator('#problem-test-patch [contenteditable="true"][role="textbox"]'),l=e.locator('#problem-solution-patch [contenteditable="true"][role="textbox"]');for(let u of[a,c,l])if(await u.count()!==1)throw new Error("Shipd challenge form selectors were not found exactly once.");await a.fill(t.taskPrompt),await _En(e,c,t.testPatch),await _En(e,l,t.solutionPatch),await PAe(e,"Test Quality").hover(),await mL(1e3,s)}async function bPo(e,t,s){dc(s);let a=PAe(e,t),c=wC(await a.innerText());if(AEn(c))return{quality:t,status:"already-running",rowTextAfterConfirm:c};let l=a.locator('button[title^="Re-run"]').or(a.locator('button[title^="Run"]')),u=await l.count();if(u!==1)throw new Error(`${t}: expected one Run button, found ${u}`);if(!await l.isEnabled())throw new Error(`${t}: Run button is disabled.`);let p=await l.getAttribute("title");await l.click();let d=e.getByRole("dialog").last();await d.waitFor({state:"visible",timeout:Zy});let m=wC(await d.innerText()),S=d.getByRole("button",{name:/confirm/i});if(await S.count()!==1)throw new Error(`${t}: rerun confirmation button was not found.`);return await S.click(),await mL(500,s),{quality:t,status:"started",rerunTitle:p,confirmation:m}}async function hEn(e,t,s=EAe){let a=[];for(let c of s){dc(t);let l=TEn(e,c);if(await l.count()!==1)throw new Error(`${c}: quality status row was not found.`);await l.waitFor({state:"visible",timeout:Zy});let u=wC(await l.locator("..").innerText());a.push({quality:c,rowText:u,busy:AEn(u)})}return a}async function EPo(e,t){await PAe(e,t).hover();let s=vPo(e,t);await s.waitFor({state:"visible",timeout:Zy});let a=s.locator('button[title="Expand to full view"]'),c=await a.count();if(c!==1)throw new Error(`${t}: expected one quality expander, found ${c}`);return a.first()}async function xPo(e,t,s){dc(s),await(await EPo(e,t)).click();let c=e.getByRole("dialog").last();await c.waitFor({state:"visible",timeout:Zy});let l=wC(await c.locator("h2").first().innerText().catch(()=>""));if(l!==t)throw new Error(`${t}: expanded the wrong report dialog (${l||"unknown"}).`);let u=c.locator("pre code").last();if(!await u.isVisible().catch(()=>!1)){let m=c.locator("button").filter({hasText:/raw output/i}).first();try{await m.waitFor({state:"visible",timeout:Zy})}catch{let S=wC(await c.innerText());throw new Error(`${t}: raw output control did not appear; dialog text: ${S.slice(0,1e3)}`)}await m.click()}await u.waitFor({state:"visible",timeout:Zy});let p=await u.innerText(),d=CPo(p,t);return await e.keyboard.press("Escape").catch(()=>{}),{quality:t,parsed:d}}function PPo(e,t){let s=Array.isArray(e.coverageSuggestions)?e.coverageSuggestions:[],a=Array.isArray(e.tests)?e.tests.filter(c=>wm(c)&&c.fairness==="Not fair"):[];return{testQuality:{verdict:e.verdict,completed:e.completed,coverageSuggestions:s,tests:a},solutionQuality:{verdict:t.verdict,completed:t.completed,evaluation:t.evaluation??null}}}function vEn(e,t){if(!wm(e)||!wm(e[t]))return"\u2014";let s=e[t].score,a=e[t].max_score;return typeof s=="number"&&typeof a=="number"?`${s}/${a}`:"\u2014"}function $Po(e,t){let s=wm(e)?e:{},a=wm(s.testQuality)?s.testQuality:{},c=wm(s.solutionQuality)?s.solutionQuality:{},l=Array.isArray(a.tests)?a.tests.length:0,u=Array.isArray(a.coverageSuggestions)?a.coverageSuggestions.length:0,p=c.evaluation;return[` ${t.fg("accent","Test Quality")} ${t.fg("muted",`${l} unfair \xB7 ${u} suggestions`)}`,` ${t.fg("accent","Solution Quality")} ${t.fg("muted",`quality ${vEn(p,"code_quality")} \xB7 comprehensiveness ${vEn(p,"solution_comprehensiveness")}`)}`]}function wPo(e){let t=(e.content??[]).map(s=>s.type==="text"?s.text??"":"").join(" ");return wC(t).slice(0,300)||"Quality checks failed"}function DEn(e){let t;e.on("session_start",(s,a)=>{t=hPo(a.sessionManager.getBranch())}),e.registerCommand("shipd:auth",{description:"Open Shipd in a browser so you can sign in and save authentication.",handler:async(s,a)=>{if(a.mode!=="tui"){a.ui.notify("/shipd:auth requires interactive mode.","error");return}let c=SEn();if(!c){a.ui.notify("Could not find Chrome/Chromium. Set SHIPD_CHROME_PATH and retry.","error");return}let l=process.env.SHIPD_STORAGE_STATE??fEn,u,p,d;try{oPo(pPo(l),{recursive:!0}),d=await aPo($m(uPo(),"shipd-auth-"));let m=await yPo();p=rPo(c,[`--remote-debugging-port=${m}`,`--user-data-dir=${d}`,"--no-first-run","--no-default-browser-check",gEn],{stdio:"ignore",windowsHide:!1}),await IPo(m,p),u=await CEn.connectOverCDP(`http://127.0.0.1:${m}`);let S=u.contexts()[0]??await u.newContext(),v=S.pages()[0]??await S.newPage();if(await v.waitForLoadState("domcontentloaded",{timeout:6e4}).catch(()=>{}),kC(v.url())||await v.goto(gEn,{waitUntil:"domcontentloaded",timeout:6e4}),a.ui.notify("A Shipd browser window is open. Sign in there, then return here.","info"),!await a.ui.confirm("Shipd sign-in","After you finish signing in in the browser, press OK to save the session.")){a.ui.notify("Shipd sign-in cancelled; nothing was saved.","info");return}await S.storageState({path:l}),await _Po(S)?(await S.storageState({path:l}),a.ui.notify(`Shipd authentication saved to ${l}. quality-check is ready to use.`,"info")):a.ui.notify(`Shipd state was saved to ${l}, but the signed-in page could not be verified. Try quality-check; if it reports an authentication error, run /shipd:auth again and wait for Shipd to return before confirming.`,"warning")}catch(m){a.ui.notify(`Shipd authentication failed: ${m instanceof Error?m.message:String(m)}`,"error")}finally{await u?.close().catch(()=>{}),p&&p.exitCode===null&&p.kill(),d&&await iPo(d,{recursive:!0,force:!0}).catch(()=>{})}}}),e.registerCommand(DAe,{description:"Save the Shipd job link for this chat session.",handler:async(s,a)=>{let c=s.trim();if(c.toLowerCase()==="clear"){t=void 0,e.appendEntry(bAe,{url:null}),a.ui.notify("Shipd job link cleared for this chat session.","info");return}if(!kC(c)){a.ui.notify(`Usage: /${DAe} <https://shipd.ai/...>`,"warning");return}t=c,e.appendEntry(bAe,{url:c}),a.ui.notify("Shipd job link saved for this chat session.","info")}}),e.registerTool({name:Qee,label:"Quality Checks",description:'Run create_patches.sh in the current working directory, read agent_prompt.md, test.patch, and solution.patch, fill the Shipd challenge draft fields, rerun only quality checks whose status is Stale, and start Test Quality then Solution Quality in one browser tab. Wait for any started jobs and return agent-focused JSON. details.testQuality contains coverageSuggestions and only tests whose fairness is exactly "Not fair"; details.solutionQuality contains the complete evaluation block. This consumes Shipd tokens and does not click the final challenge-submit button.',promptSnippet:"Submit the working-directory patches to Shipd",promptGuidelines:["Use quality-check when the user explicitly asks to submit or evaluate the current Shipd task.","quality-check has no parameters.","quality-check reruns only checks marked Stale after the draft update, then waits using scheduled browser reopen checks.","Read details.testQuality.coverageSuggestions and details.testQuality.tests for test-quality feedback.","Read details.solutionQuality.evaluation for the complete solution-quality feedback."],parameters:RPo,renderCall(s,a,c){let l=c.state;c.executionStarted&&l.startedAt===void 0&&(l.startedAt=Date.now());let u=c.lastComponent instanceof dL?c.lastComponent:new dL("",0,0);return u.setText(""),u},renderResult(s,a,c,l){let u=l.state;u.startedAt??=Date.now(),u.startedAt!==void 0&&a.isPartial&&!u.interval&&(u.interval=setInterval(()=>l.invalidate(),1e3)),(!a.isPartial||l.isError)&&(u.endedAt??=Date.now(),u.interval&&(clearInterval(u.interval),u.interval=void 0));let p=u.endedAt??Date.now(),d=Hg(p-u.startedAt),m=a.isPartial?`Elapsed ${d}`:`Took ${d}`,S=c.fg("toolTitle",c.bold("Quality Checks"))+c.fg("muted",` ${m}`);return a.isPartial?new dL(`${S}
|
|
107
107
|
${c.fg("muted","Running quality checks...")}`,0,0):l.isError?new dL(`${S}
|
|
108
108
|
${c.fg("error",wPo(s))}`,0,0):new dL([S,...$Po(s.details,c)].join(`
|
|
109
109
|
`),0,0)},async execute(s,a,c,l,u){if(!t)throw new Error(`No Shipd job link is set for this chat session. Provide the job link with /${DAe} <job-link>.`);let p=t,d=process.env.SHIPD_STORAGE_STATE??fEn;if(!xAe(d))throw new Error(`Shipd authentication state not found at ${d}. Run /shipd:auth first.`);let m=SEn();if(!m)throw new Error("Could not find Chrome/Chromium. Set SHIPD_CHROME_PATH and retry.");let S=Number.parseInt(process.env.SHIPD_SUBMIT_TIMEOUT_MS??String(REn),10),v=Math.min(dPo,Math.max(3e4,Number.isFinite(S)?S:REn)),T=Date.now()+v,x,w,L=()=>{x?.close().catch(()=>{})};c?.addEventListener("abort",L,{once:!0});try{dc(c),l?.({content:[{type:"text",text:"Running create_patches.sh..."}],details:void 0});let O=await TPo(e,u.cwd,c);dc(c),l?.({content:[{type:"text",text:"Opening Shipd and filling draft fields..."}],details:void 0});let G=await IEn(m,d,p,c);x=G.browser,w=G.page,await DPo(w,O,c),l?.({content:[{type:"text",text:"Checking which quality checks are stale..."}],details:void 0});let z=await hEn(w,c),ce=z.filter(be=>be.busy||yEn(be.rowText)).map(be=>be.quality),re=[];for(let be of EAe){let Ce=z.find(Je=>Je.quality===be);if(!Ce)throw new Error(`${be}: quality status was not available after updating the draft.`);yEn(Ce.rowText)?re.push(await bPo(w,be,c)):Ce.busy&&re.push({quality:be,status:"already-running",rowTextAfterConfirm:Ce.rowText})}let Le=z.map(({quality:be,rowText:Ce})=>({quality:be,rowText:Ce}));if(ce.length>0){l?.({content:[{type:"text",text:`Quality checks queued (${ce.join(", ")}); closing the browser until the first check in 5 minutes...`}],details:void 0}),await x.close(),x=void 0,w=void 0;let be=Date.now()+fPo;for(;;){let Ce=Math.min(be,T)-Date.now();if(Ce>0&&(l?.({content:[{type:"text",text:`Browser closed; next quality check in ${Hg(Ce)}...`}],details:void 0}),await mL(Ce,c)),dc(c),Date.now()>=T)throw new Error(`Shipd quality checks timed out after ${Hg(v)}.`);l?.({content:[{type:"text",text:"Reopening Shipd to check quality job status..."}],details:void 0});let Je=await IEn(m,d,p,c);x=Je.browser,w=Je.page;let ie=await hEn(w,c,ce);if(ie.every(J=>!J.busy)){Le=[...z.filter(J=>!ce.includes(J.quality)).map(({quality:J,rowText:ur})=>({quality:J,rowText:ur})),...ie.map(({quality:J,rowText:ur})=>({quality:J,rowText:ur}))];break}let M=ie.map(J=>`${J.quality}: ${J.rowText}`).join("; ");l?.({content:[{type:"text",text:`Quality jobs still running (${M}); closing browser until the next check...`}],details:void 0}),await x.close(),x=void 0,w=void 0,be+=gPo}}else l?.({content:[{type:"text",text:"No stale quality checks found; using the current reports."}],details:void 0});if(!w)throw new Error("Shipd quality status page was closed before report extraction.");l?.({content:[{type:"text",text:"Extracting quality reports..."}],details:void 0});let xe=[];for(let be of EAe)xe.push(await xPo(w,be,c));let Ie=xe.find(be=>be.quality==="Test Quality")?.parsed,he=xe.find(be=>be.quality==="Solution Quality")?.parsed;if(!Ie||!he)throw new Error("Shipd did not return both quality reports.");let Be=PPo(Ie,he);return{content:[{type:"text",text:JSON.stringify(Be,null,2)}],details:{...Be,started:re,completed:Le}}}catch(O){throw c?.aborted?new Error("Cancelled by user."):O}finally{c?.removeEventListener("abort",L),await x?.close().catch(()=>{})}}})}var Zee=[CAe,TAe],EEn=["analyze_task_tests","analyze_test_gaps","submit_shipd"],Yee=!1,bEn=!1,Yy=!1;function $Ae(e){return e.getActiveTools().map(s=>typeof s=="string"?s:s.name)}function wAe(){return Yee}function Xee(e){let t=$Ae(e),s=EEn.some(l=>t.includes(l)),a=Zee.every(l=>t.includes(l)),c=Zee.some(l=>t.includes(l));return s||(wAe()?!a:c)}function kPo(e){bEn||(mEn(e),bEn=!0)}function LPo(e,t){let s=$Ae(e);if(t&&s.length===0){Yy=!0;return}let a=s.includes("submit_shipd"),l=s.filter(d=>!EEn.includes(d)).filter(d=>!Zee.includes(d)),u=t?[...l,...Zee]:[...l];a&&!u.includes(Qee)&&u.push(Qee),(u.length!==s.length||u.some((d,m)=>d!==s[m]))&&e.setActiveTools(u),Yy=Xee(e)}function LC(e){let t=wAe();t&&kPo(e),LPo(e,t)}function kAe(e,t){Yee=CL(t),Yy=!0,xEn(e)}function xEn(e,t=0){let s=wAe(),a=$Ae(e);if(s&&a.length===0&&t<8){queueMicrotask(()=>xEn(e,t+1));return}LC(e)}function PEn(e){for(let t of[0,50,150,400])setTimeout(()=>{(Yy||Xee(e))&&LC(e)},t)}function $En(){return Yy}function LAe(e,t,s){Yee=s,TL(t,s),Yy=!0,LC(e)}function wEn(e,t){Yee=CL(t),Yy=!0,LC(e)}var HPo="Ctrl+Shift+X",GEn=[{value:"--config",label:"--config",description:"Configure reviewer, solver, and analysis models"},{value:"--solver-gap-finder",label:"--solver-gap-finder",description:"Use TDD solver attempts to find behavioral gaps"}],jPo=["config","solver-gap-finder"];function zPo(e){let t=e.trimStart(),s=/\s$/.test(t),a=t.trimEnd().split(/\s+/).filter(Boolean),c=s?"":a.at(-1)??"",l=new Set(s?a:a.slice(0,-1)),u=s?t:t.slice(0,t.length-c.length),p=l.has("--config"),d=[...l].some(S=>S!=="--config"),m=GEn.filter(S=>!l.has(S.value)&&!p&&!(d&&S.value==="--config")&&S.value.startsWith(c));return m.length?m.map(S=>({...S,value:`${u}${S.value}`})):null}async function MAe(e,t,s){let a=JAe();if(a.length===0)return e.ui.notify("No enabledModels configured in settings.json.","error"),null;let c=e.modelRegistry.getAll(),l=a.map(m=>{let S=eDe(m);if(!S)return null;let v=c.find(w=>w.provider===S.provider&&w.id===S.modelId),T=t?.provider===S.provider&&t?.modelId===S.modelId,x=v?`${v.name} (${m})`:m;return{ref:m,parsed:S,display:T?`${x} [current]`:x,isCurrent:T}}).filter(m=>m!==null);if(l.length===0)return e.ui.notify("Could not resolve any enabledModels entries.","error"),null;let u=[...l].sort((m,S)=>m.isCurrent===S.isCurrent?0:m.isCurrent?-1:1),p=await e.ui.select(s,u.map(m=>m.display));if(!p)return null;let d=l.find(m=>m.display===p);return d?{provider:d.parsed.provider,modelId:d.parsed.modelId}:null}function MC(e,t,s){let a=e.modelRegistry.getAll().find(c=>c.provider===t&&c.id===s);return WAe(a)}var Jee={provider:"",modelId:"",thinkingLevel:"off",timeoutMinutes:tte,solverCount:ste,saveArtifacts:rte},KPo={provider:"",modelId:"",thinkingLevel:"off",timeoutMinutes:rI};function ete(e){let t=e?.analyzeGap,s=t?.timeoutMinutes,a=typeof s=="number"&&Number.isFinite(s)?Math.min(hL,Math.max(_L,s)):rI;return{...KPo,...t,timeoutMinutes:a}}function MEn(e,t,s,a){let c=t?MC(e,t.provider,t.modelId):["off"],l=s.provider?MC(e,s.provider,s.modelId):["off"],u=a.provider?MC(e,a.provider,a.modelId):["off"];return[{id:"solver-model",section:"Solver",label:"Solver model",value:s.provider?`${s.provider}/${s.modelId}`:"not set",kind:"model"},{id:"solver-thinking",section:"Solver",label:"Solver thinking",value:s.thinkingLevel,kind:"cycle",values:l},{id:"solvergap-timeout",section:"Solver",label:"Timeout",value:`${s.timeoutMinutes} min`,kind:"cycle",values:[10,20,30,40,50,jAe].filter((p,d,m)=>p>=HAe&&m.indexOf(p)===d).map(p=>`${p} min`)},{id:"solvergap-solver-count",section:"Solver",label:"Parallel agents",value:String(s.solverCount),kind:"cycle",values:Array.from({length:zAe-nte+1},(p,d)=>`${nte+d}`)},{id:"solvergap-save-artifacts",section:"Solver",label:"Save artifacts",value:s.saveArtifacts?"on":"off",kind:"cycle",values:["on","off"]},{id:"reviewer-model",section:"Solver",label:"Reviewer model",value:t?`${t.provider}/${t.modelId}`:"not set",kind:"model"},{id:"reviewer-thinking",section:"Solver",label:"Reviewer thinking",value:t?.thinkingLevel??"off",kind:"cycle",values:c},{id:"analyze-enabled",section:"Analysis Tools",label:"Enabled here",value:AL(e.cwd)?"on":"off",kind:"cycle",values:["on","off"]},{id:"analyze-timeout",section:"Analysis Tools",label:"Timeout",value:`${a.timeoutMinutes} min`,kind:"cycle",values:[10,15,20,25,hL].filter((p,d,m)=>p>=_L&&m.indexOf(p)===d).map(p=>`${p} min`)},{id:"analyze-gap-model",section:"Analysis Tools",label:"Analysis model",value:a.provider?`${a.provider}/${a.modelId}`:"not set",kind:"model"},{id:"analyze-gap-thinking",section:"Analysis Tools",label:"Analysis thinking",value:a.thinkingLevel,kind:"cycle",values:u}]}var GAe=class{constructor(t,s,a,c,l,u){this.ctx=t;this.settingsListTheme=s;this.theme=a;this.onCycleSaved=c;this.onActivateModel=l;this.onExit=u;this.rows=MEn(t,Ea(),Ea()?.solverGap??Jee,ete(Ea()))}selectedIndex=0;rows;refresh(){let t=Ea();this.rows=MEn(this.ctx,t,t?.solverGap??Jee,ete(t)),this.selectedIndex=Math.min(this.selectedIndex,this.rows.length-1)}invalidate(){}render(t){let s=[],a=Math.min(30,Math.max(...this.rows.map(l=>l.label.length))),c;for(let l=0;l<this.rows.length;l++){let u=this.rows[l];if(!u)continue;u.section!==c&&(c!==void 0&&s.push(""),s.push(this.theme.bold(this.theme.fg("accent",u.section))),c=u.section);let p=l===this.selectedIndex,d=p?this.settingsListTheme.cursor:" ",m=u.label+" ".repeat(Math.max(0,a-u.label.length)),S=this.settingsListTheme.label(m,p),v=this.settingsListTheme.value(u.value,p);s.push(`${d}${S} ${v}`)}return s}handleInput(t){if(eI(t,Jy.ctrl("s"))||eI(t,Jy.escape)||eI(t,Jy.ctrl("c"))){this.onExit();return}if(eI(t,Jy.up)){this.selectedIndex=this.selectedIndex===0?this.rows.length-1:this.selectedIndex-1;return}if(eI(t,Jy.down)){this.selectedIndex=this.selectedIndex===this.rows.length-1?0:this.selectedIndex+1;return}if(eI(t,Jy.enter)||eI(t,Jy.return)||t===" "){let s=this.rows[this.selectedIndex];if(!s)return;if(s.kind==="model"){this.onActivateModel(s.id);return}if(!s.values||s.values.length===0)return;let a=Ea();if(!a){this.ctx.ui.notify("Set the reviewer model first \u2014 it's required before solver settings","warning");return}let c=a.solverGap??Jee,l=ete(a),u=s.values.indexOf(s.value),p=s.values[(u+1)%s.values.length];if(p===void 0)return;s.id==="reviewer-thinking"?Pi({...a,thinkingLevel:p,solverGap:c,analyzeGap:l}):s.id==="solver-thinking"?Pi({...a,solverGap:{...c,thinkingLevel:p}}):s.id==="solvergap-timeout"?Pi({...a,solverGap:{...c,timeoutMinutes:Number.parseInt(p,10)}}):s.id==="solvergap-solver-count"?Pi({...a,solverGap:{...c,solverCount:Number.parseInt(p,10)}}):s.id==="solvergap-save-artifacts"?Pi({...a,solverGap:{...c,saveArtifacts:p==="on"}}):s.id==="analyze-enabled"?TL(this.ctx.cwd,p==="on"):s.id==="analyze-timeout"?Pi({...a,analyzeGap:{...l,timeoutMinutes:Number.parseInt(p,10)}}):s.id==="analyze-gap-thinking"&&Pi({...a,analyzeGap:{...l,thinkingLevel:p}}),this.refresh(),this.onCycleSaved()}}};async function WPo(e,t){if(t.mode!=="tui"){t.ui.notify("/checks --config requires interactive mode","error");return}let s=0;for(;;){let a=await t.ui.custom((p,d,m,S)=>{let v=new GAe(t,FPo(),d,()=>{wEn(e,t.cwd),p.requestRender()},x=>S(x),()=>S(void 0));v.selectedIndex=s;let T=new BPo;return T.addChild(new LEn(d.bold(d.fg("accent","Checks settings")))),T.addChild({render:x=>v.render(x),invalidate:()=>v.invalidate()}),T.addChild(new UPo(1)),T.addChild(new LEn(d.fg("dim","\u2191\u2193 navigate enter select ctrl+s save & exit esc cancel"))),{render(x){return T.render(x)},invalidate(){T.invalidate()},handleInput(x){v.handleInput(x),s=v.selectedIndex,p.requestRender()}}});if(!a)return;let c=Ea(),l=c?.solverGap??Jee,u=ete(c);if(a==="reviewer-model"){let p=await MAe(t,c,"Select reviewer model");if(!p)continue;let d=MC(t,p.provider,p.modelId),m=d.includes(c?.thinkingLevel??"off")?c?.thinkingLevel??"off":d[0]??"off";Pi({...p,thinkingLevel:m,solverGap:l,analyzeGap:u,enabledProjects:c?.enabledProjects??ate(),enableAnalyzeTool:c?.enableAnalyzeTool}),t.ui.notify(`Reviewer model saved: ${p.provider}/${p.modelId}`,"info");continue}if(a==="solver-model"){if(!c){t.ui.notify("Set the reviewer model first \u2014 it's required before solver settings.","warning");continue}let p=await MAe(t,l.provider?l:null,"Select solver model");if(!p)continue;let d=MC(t,p.provider,p.modelId),m=d.includes(l.thinkingLevel)?l.thinkingLevel:d[0]??"off";Pi({...c,solverGap:{...l,...p,thinkingLevel:m}}),t.ui.notify(`Solver model saved: ${p.provider}/${p.modelId}`,"info");continue}if(a==="analyze-gap-model"){if(!c){t.ui.notify("Set the reviewer model first \u2014 it's required before analyze-tool settings.","warning");continue}let p=await MAe(t,u.provider?u:null,"Select gap-finder model");if(!p)continue;let d=MC(t,p.provider,p.modelId),m=d.includes(u.thinkingLevel)?u.thinkingLevel:d[0]??"off";Pi({...c,analyzeGap:{...u,...p,thinkingLevel:m}}),t.ui.notify(`Gap-finder model saved: ${p.provider}/${p.modelId}`,"info")}}}function OEn(e){e.registerCommand("checks",{description:"Configure checks or run the solver-based behavioral gap finder.",getArgumentCompletions:zPo,handler:async(t,s)=>{let a=t.trim();if(!a){let Ie=await s.ui.select("Choose a checks action",[...jPo]);if(!Ie)return;a=`--${Ie}`}let c=[...new Set(a.split(/\s+/).filter(Boolean))],l=new Set(GEn.map(Ie=>Ie.value)),u=c.filter(Ie=>!l.has(Ie));if(u.length){s.ui.notify(`Unknown option(s): ${u.join(", ")}. Run /checks with no arguments to see options.`,"warning");return}if(c.includes("--config")){c.length>1?s.ui.notify("--config cannot be combined with other options.","warning"):await WPo(e,s);return}let p=c.includes("--solver-gap-finder");if(!p)return;if(Kee()){s.ui.notify("A checks run is already in progress.","warning");return}let d=Ea();if(!d){s.ui.notify("No gap-finder model configured. Run /checks --config first.","error");return}let m=s.modelRegistry.find(d.provider,d.modelId);if(!m||!s.modelRegistry.hasConfiguredAuth(m)){s.ui.notify("Configured behavioral gap-finder model is unavailable or unauthenticated. Run /checks --config.","error");return}let S=hAe.filter(Ie=>!kEn(Xy(s.cwd,Ie)));if(S.length){s.ui.notify(`Missing required file(s): ${S.join(", ")}`,"error");return}let v=null,T;if(p){if(v=YAe(),!v||!kEn(Xy(s.cwd,"test.sh"))){s.ui.notify("Configure solver gap finder and provide test.sh before running it.","error");return}if(T=s.modelRegistry.find(v.provider,v.modelId),!T||!s.modelRegistry.hasConfiguredAuth(T)){s.ui.notify("Configured solver gap-finder model is unavailable or unauthenticated.","error");return}}let x=cEn(),w=v?.solverCount??0,L=w+1,O=Date.now(),G=p?"preparing image":"preparing clean snapshot",z=0,ce=!p,re=(Ie=G,he=z,Be=ce)=>{G=Ie,z=he,ce=Be,s.ui.setWidget(pL,sEn(Ie,he,L,{showBar:Be,startedAt:O}))},Le=setInterval(()=>re(),1e3);re(),s.ui.notify(`checks (${c.join(" ")}) started. Press ${HPo} to cancel.`,"info");let xe;try{let Ie=Xy(qPo(),`checks-${MPo()}`);OPo(Ie,{recursive:!0}),xe=Ie;let he=await ute(e,s.cwd,Ie,x.signal);if(he.status==="error"){s.ui.notify(`checks: ${he.error}`,"error");return}if(x.signal.aborted)return;for(let Xo of hAe)GPo(Xy(s.cwd,Xo),Xy(Ie,Xo));let Be=Uee(),be=jee(),Ce=Hee(),Je=0,ie=[],M,J={status:"ok",gaps:[]};if(p&&v){if(x.signal.aborted){s.ui.notify("checks: cancelled.","warning");return}let Xo=new Map,jg=!1,tI=()=>Array.from({length:w},(_o,km)=>{let gL=Xo.get(km+1);return gL?`${km+1}${gL.passed?"\u2713":"\u2717"}`:`${km+1}\u2026`}).join(" "),nI=_o=>{_o==="running agents"&&(jg=!0);let km=jg&&(_o==="running agents"||_o==="finalizing"),gL=_o==="running agents"?`${_o} (${tI()})`:_o,BEn=[...Xo.values()];re(gL,Je+BEn.length,km)},VEn=_o=>{for(let km of _o)Xo.set(km.index,km);nI("running agents")},qEn=_o=>{Xo.set(_o.index,_o),nI("finalizing")};nI("preparing image");let FEn=vAe().replace(/[:.]/g,"-");if(ie=await nEn({pi:e,repoDir:s.cwd,snapshotDir:Ie,config:d,solverConfig:v,cancelSignal:x.signal,runId:FEn,onSolverCompleted:qEn,onSolverProgress:VEn,onResourceUsage:_o=>{M=_o},onPhase:nI}),x.signal.aborted){s.ui.notify("checks: cancelled.","warning");return}if(ie.some(_o=>_o.status!=="patchFailed"&&_o.status!=="error")&&(nI("finalizing"),aDe(Ie,ie),J=await Gke({tempDir:Ie,model:m,thinkingLevel:d.thinkingLevel,solverResults:ie,testRubric:Be,gapRules:be,fairnessRules:Ce,cancelSignal:x.signal})),Je+=1,nI("finalizing"),M&&Pi(ZAe(d,s.cwd,M)),x.signal.aborted){s.ui.notify("checks: cancelled.","warning");return}}let ur=oEn({existingReport:rEn(Xy(s.cwd,"shipd_report.json")),config:d,runGapFinder:!1,testGaps:[],gapAnalysisIncomplete:!1,gapFinderStatus:"ok",gapFilterStatus:"ok",runSolverGapFinder:p,solverResults:ie,fargateResourceUsage:M,solverGaps:J.gaps,solverGapAnalysisIncomplete:p&&(ie.some(Xo=>Xo.status==="error")||J.status!=="ok"),solverComparisonStatus:J.status}),GC=aEn({merged:ur,runGapFinder:!1,runSolverGapFinder:p});e.sendMessage({customType:"shipd_checks_report",content:GC.content,display:!0,details:GC.details});let fL=Xy(s.cwd,"shipd_report.json");VPo(fL,JSON.stringify(ur,null,2),"utf-8"),s.ui.notify(`checks: wrote details to ${fL}`,"info")}catch(Ie){s.ui.notify(`checks failed: ${Ie instanceof Error?Ie.message:String(Ie)}`,"error")}finally{clearInterval(Le),s.ui.setWidget(pL,void 0),xe&&NPo(xe,{recursive:!0,force:!0}),lEn()}}})}var NEn=ZPo.ctrlShift("x");function XPo(e){e.registerCommand("analyze:on",{description:"Enable the gap-finder and solution-precheck tools for this project folder",handler:async(t,s)=>{LAe(e,s.cwd,!0),s.ui.notify("Gap-finder and solution-precheck tools ON in this project","info")}}),e.registerCommand("analyze:off",{description:"Disable the gap-finder and solution-precheck tools for this project folder",handler:async(t,s)=>{LAe(e,s.cwd,!1),s.ui.notify("Gap-finder and solution-precheck tools OFF in this project","info")}}),e.registerMessageRenderer("shipd_checks_report",(t,s,a)=>{let c=t.details??{},l=new QPo;l.addChild(new OAe(a.bold(a.fg("accent","Checks")),0,0));let u=!1,p=d=>{l.addChild(new OAe(d,0,0)),u=!0};if(c.showGaps){let d=c.gapsCount??0;p(d>0?a.fg("warning",` \u26A0 ${d} test gap(s) found`):a.fg("success"," \u2713 no test gaps found"))}if(c.showSolverGaps){let d=c.solverDetails??[],m=d.filter(x=>x.passed).length,S=c.solverGapsCount??0;l.addChild(new YPo(1)),p(a.bold(a.fg("accent","Solver gap finder")));for(let x of d){let w=x.passed?a.fg("success","\u2713"):a.fg("error","\u2717"),L=x.passed?"passed":x.status==="ok"?"failed tests":x.status;p(` ${w} Solver ${x.index}: ${L} (${Hg(x.durationMs)})`)}let v=m===d.length&&d.length>0?a.fg("success",`${m}/${d.length} solvers passed`):a.fg("warning",`${m}/${d.length} solvers passed`),T=S>0?a.fg("warning",`${S} behavioral gap(s) found`):a.fg("success","no behavioral gaps found");p(` ${v}, ${T}`)}return u||l.addChild(new OAe(a.fg("dim"," nothing to report"),0,0)),l}),e.registerShortcut(NEn,{description:"Cancel an in-progress /checks run",handler:t=>{Kee()&&uEn()&&(t.ui.setWidget(pL,[`checks: cancelling (${NEn})...`]),t.ui.notify("checks: cancelling...","warning"))}}),JPo(e),DEn(e),OEn(e)}function JPo(e){e.on("session_start",(t,s)=>{kAe(e,s.cwd),(t.reason==="resume"||t.reason==="reload"||t.reason==="startup")&&PEn(e)}),e.on("session_tree",async(t,s)=>{kAe(e,s.cwd)}),e.on("before_agent_start",()=>{!$En()&&!Xee(e)||LC(e)})}export{XPo as default};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sreetej510/pi-shipd-checks",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.2",
|
|
4
4
|
"description": "Pi extension that runs a strict, multi-agent fairness review of a benchmark task's agent_prompt.md, test.patch, and solution.patch, plus behavioral test-gap analysis, via /checks.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|