@sreetej510/pi-shipd-checks 0.11.6 → 0.11.7

package/README.md

@@ -37,8 +37,9 @@ or correctness evidence. The adapter is experimental and was trained on all 519
 assigned to EVAL and HOLDOUT; treat all outputs as unapproved drafts.
 
 The agent-callable `quality-check` tool runs `create_patches.sh` in the current working directory and reads
-`agent_prompt.md`, `test.patch`, and `solution.patch`. It fills the authenticated Shipd draft fields and computes a SHA-256
-of the exact prompt content. The last passing prompt hash is stored as a custom entry in the current chat session. If the
+`agent_prompt.md`, `test.patch`, and `solution.patch`. Before checking or running Prompt Quality, it uploads all three
+contents to the authenticated Shipd draft fields and waits for the editor update to settle. It computes a SHA-256 of the
+exact prompt content. The last passing prompt hash is stored as a custom entry in the current chat session. If the
 hash matches, it skips the Shipd Prompt Quality check entirely and reports a cached `PASS`; if it differs or is absent, it
 checks the refreshed status after filling the fields and runs only when the check is stale or has a `Run` button. Otherwise it
 uses the current report. A new hash is saved only after Prompt Quality passes. On failure, only `promptQuality` is returned; there are no Test/Solution results or `stoppedReason`.

package/dist/index.js

@@ -107,7 +107,7 @@ ${T}`,0,0)},async execute(t,r,i,c,l){if(!gy(l.cwd))throw new Error(`${dL} is dis
    Required behavior: ${r.requiredBehavior}
    Recommendation: ${r.recommendation}`)].join(`
 `)}import{spawn as PEa}from"node:child_process";import{createHash as $Ea,randomUUID as wEa}from"node:crypto";import{existsSync as uAe,mkdirSync as kEa}from"node:fs";import{mkdtemp as Cbn,readFile as tAe,rm as nAe,writeFile as Tbn}from"node:fs/promises";import{createServer as LEa}from"node:net";import{homedir as MEa,tmpdir as Abn}from"node:os";import{dirname as GEa,join as nl}from"node:path";import{Text as mL}from"@earendil-works/pi-tui";import{chromium as kbn}from"playwright-core";var Nee="quality-check",sAe="shipd:link",cAe="shipd_job_link",Lbn="shipd_prompt_quality_sha",Dbn=nl(MEa(),".pi","agent","shipd-auth","shipd.ai.json"),OEa=15,bbn="https://shipd.ai/",NEa=12e4,VEa=3e5,qEa=9e4,FEa=12e4,BEa=3e4,Nm=5e3,Vm=3,pAe=250,UEa=2e3,Mbn=["Task Prompt Quality","Test Quality","Solution Quality"],HEa="Keep agent_prompt.md unchanged if possible\u2014small edits can make it fail. Prefer aligning tests and solution to the prompt; only if the prompt is demonstrably wrong, make the smallest correction while preserving its human style.",jEa=pt.Object({});function HC(e){return(e??"").trim().replace(/\s+/g," ")}function zEa(e){return e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}function tr(e){return e!==null&&typeof e=="object"&&!Array.isArray(e)}function eR(e,t){return new Promise((r,i)=>{if(t?.aborted){i(new Error("Cancelled by user."));return}let c=()=>{clearTimeout(l),t?.removeEventListener("abort",c),i(new Error("Cancelled by user."))},l=setTimeout(()=>{t?.removeEventListener("abort",c),r()},e);t?.addEventListener("abort",c,{once:!0})})}function Ao(e){if(e?.aborted)throw new Error("Cancelled by user.")}function Ebn(){return[process.env.SHIPD_CHROME_PATH,process.env.PROGRAMFILES&&nl(process.env.PROGRAMFILES,"Google","Chrome","Application","chrome.exe"),process.env["PROGRAMFILES(X86)"]&&nl(process.env["PROGRAMFILES(X86)"],"Google","Chrome","Application","chrome.exe"),process.env.LOCALAPPDATA&&nl(process.env.LOCALAPPDATA,"Google","Chrome","Application","chrome.exe"),"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome","/usr/bin/google-chrome","/usr/bin/chromium","/usr/bin/chromium-browser"].filter(t=>!!t).find(t=>uAe(t))}function KEa(){let e=LEa();return new Promise((t,r)=>{e.once("error",r),e.listen(0,"127.0.0.1",()=>{let i=e.address();if(!i||typeof i=="string"){e.close(),r(new Error("Could not allocate a local Chrome debugging port."));return}e.close(c=>c?r(c):t(i.port))})})}async function WEa(e,t,r=15e3){let i=Date.now()+r;for(;Date.now()<i;){if(t.exitCode!==null)throw new Error("Chrome exited before its debugging connection became available.");try{if((await fetch(`http://127.0.0.1:${e}/json/version`)).ok)return}catch{}await eR(250)}throw new Error("Timed out waiting for Chrome to start.")}async function QEa(e,t=3e4){let r=Date.now()+t;for(;Date.now()<r;){let i=e.pages();for(let c of i){if(!jC(c.url()))continue;if(await c.locator('button[aria-label="Open user menu"]').isVisible({timeout:1e3}).catch(()=>!1))return c}for(let c of i)jC(c.url())&&await c.reload({waitUntil:"domcontentloaded",timeout:15e3}).catch(()=>{});await eR(1e3)}}function jC(e){try{let t=new URL(e),r=t.hostname.toLowerCase();return t.protocol==="https:"&&(r==="shipd.ai"||r.endsWith(".shipd.ai"))}catch{return!1}}function ZEa(e){try{return/\/sign-in(?:\/|$)/i.test(new URL(e).pathname)}catch{return!1}}function YEa(e){let t;for(let r of e){if(!tr(r)||r.type!=="custom"||r.customType!==cAe)continue;let i=r.data;tr(i)&&(typeof i.url=="string"&&jC(i.url)?t=i.url:i.url===null&&(t=void 0))}return t}function XEa(e){let t;for(let r of e){if(!tr(r)||r.type!=="custom"||r.customType!==Lbn)continue;let i=r.data;tr(i)&&(typeof i.sha256=="string"&&/^[a-f0-9]{64}$/i.test(i.sha256)?t=i.sha256:i.sha256===null&&(t=void 0))}return t}function Jg(e){return e==="Task Prompt Quality"?"Prompt Quality":e}function Gbn(e,t){return e.locator("span").filter({hasText:new RegExp(`^${zEa(t)}$`)}).first()}function fL(e,t){return Gbn(e,t).locator("..")}function JEa(e,t){return e.locator("div.fixed").filter({hasText:t}).last()}function dAe(e){return/(?:starting|queued|pending|running|processing|reviewing|finalizing|building|evaluating|checking|\d+%)/i.test(e)}function mAe(e){return/\bstale\b/i.test(e)}function exa(e,t){let r=e.trim().replace(/^```(?:json)?\s*/i,"").replace(/\s*```$/i,"").trim(),i;try{i=JSON.parse(r)}catch(c){throw new Error(`${t} raw output was not valid JSON: ${c instanceof Error?c.message:String(c)}`)}if(!tr(i))throw new Error(`${t} raw output was not a JSON object.`);return i}async function txa(e,t,r){let i=nl(t,"create_patches.sh"),c=nl(t,"agent_prompt.md"),l=nl(t,"test.patch"),u=nl(t,"solution.patch"),p=[["create_patches.sh",i],["agent_prompt.md",c],["test.patch",l],["solution.patch",u]].filter(([,T])=>!uAe(T)).map(([T])=>T);if(p.length)throw new Error(`Missing required file(s) in the working directory: ${p.join(", ")}`);Ao(r);let d=await e.exec(QC(),["create_patches.sh"],{cwd:t,timeout:NEa,signal:r});if(d.killed||d.code!==0){let T=(d.stderr||d.stdout||"").trim().slice(-2e3);throw new Error(`create_patches.sh failed${d.killed?" or timed out":` (exit ${d.code})`}.${T?`
-${T}`:""}`)}Ao(r);let[m,I,v]=await Promise.all([tAe(c,"utf8"),tAe(l,"utf8"),tAe(u,"utf8")]);return{taskPrompt:m,testPatch:I,solutionPatch:v}}async function nxa(e,t,r){Ao(r);let i=await e.goto(t,{waitUntil:"domcontentloaded",timeout:6e4});if(Ao(r),!jC(e.url()))throw new Error("Shipd authentication or job-link navigation failed. Check the saved job link and auth state.");if(ZEa(e.url()))throw new Error("Shipd authentication failed: the saved session is expired or signed out. Run /shipd:auth again.");if(i&&i.status()>=400)throw new Error(`Shipd job-link navigation returned HTTP ${i.status()}.`);await e.waitForTimeout(4e3),Ao(r)}async function rAe(e,t,r,i){Ao(i);let c=await kbn.launch({executablePath:e,headless:!0});try{let l=await c.newContext({storageState:t});await l.route("**/*",async p=>{let d=p.request().resourceType();d==="image"||d==="font"||d==="media"?await p.abort():await p.continue()});let u=await l.newPage();return await nxa(u,r,i),{browser:c,page:u}}catch(l){throw await c.close().catch(()=>{}),l}}async function xbn(e,t,r){await t.click({timeout:Nm}),await e.keyboard.press(`${process.platform==="darwin"?"Meta":"Control"}+A`),await e.keyboard.insertText(r)}async function Pbn(e,t,r){Ao(r);let i=e.locator("#problem-description"),c=e.locator('#problem-test-patch [contenteditable="true"][role="textbox"]'),l=e.locator('#problem-solution-patch [contenteditable="true"][role="textbox"]');for(let u of[i,c,l])if(await u.count()!==1)throw new Error("Shipd challenge form selectors were not found exactly once.");await i.fill(t.taskPrompt),await xbn(e,c,t.testPatch),await xbn(e,l,t.solutionPatch),await fL(e,"Task Prompt Quality").hover(),await eR(1e3,r)}async function Obn(e,t,r,i){let c=fL(e,t),l=HC(await c.innerText()),u=await c.locator('button[title^="Run"]').count();if(dAe(l)||l&&r&&!mAe(l)||l&&i&&u===0)return l}async function sxa(e,t,r,i,c){let l=Date.now()+UEa;for(;Date.now()<l;){Ao(c);let u=await Obn(e,t,r,i);if(u!==void 0)return u;await eR(200,c)}}async function $bn(e,t,r){let i="The Run action did not start the quality check.";for(let c=1;c<=Vm;c+=1){Ao(r);let l=!1,u=!1,p=null,d="";try{let I=fL(e,t),v=HC(await I.innerText());if(dAe(v))return{quality:Jg(t),status:"already-running",rowTextAfterConfirm:v};l=mAe(v);let T=I.locator('button[title^="Run"]'),x=I.locator('button[title^="Re-run"]');if(u=await T.count()>0,!l&&!u)return{quality:Jg(t),status:"no-longer-stale",rowTextAfterConfirm:v};let k=await x.count(),L=await T.count(),M=k===1?x:T,N=k===1?k:L;if(N!==1)throw new Error(`${t}: expected one Run button, found ${N}`);if(!await M.isEnabled())throw new Error(`${t}: Run button is disabled.`);p=await M.getAttribute("title"),await M.click({timeout:Nm});let H=e.getByRole("dialog").last();await H.waitFor({state:"visible",timeout:Nm}),d=HC(await H.innerText());let re=H.getByRole("button",{name:/confirm/i});if(await re.count()!==1)throw new Error(`${t}: rerun confirmation button was not found.`);await re.click({timeout:Nm});let me=await sxa(e,t,l,u,r);if(me!==void 0)return{quality:Jg(t),status:"started",rerunTitle:p,confirmation:d,rowTextAfterConfirm:me};i=`${t}: the confirmation was accepted, but the stale status did not clear and no running state appeared.`}catch(I){if(r?.aborted)throw new Error("Cancelled by user.");i=I instanceof Error?I.message:String(I)}let m=await Obn(e,t,l,u).catch(()=>{});if(m!==void 0)return{quality:Jg(t),status:"started",rerunTitle:p,confirmation:d,rowTextAfterConfirm:m};await e.keyboard.press("Escape").catch(()=>{}),c<Vm&&await eR(pAe*c,r)}throw new Error(`${t}: could not start after ${Vm} attempts. ${i}`)}async function rxa(e,t,r=Mbn){let i=[];for(let c of r){Ao(t);let l=Gbn(e,c);if(await l.count()!==1)throw new Error(`${c}: quality status row was not found.`);await l.waitFor({state:"visible",timeout:Nm});let u=l.locator(".."),p=HC(await u.innerText()),d=await u.locator('button[title^="Run"]').count()>0;i.push({quality:c,rowText:p,busy:dAe(p),shouldRun:d||mAe(p)})}return i}async function aAe(e,t,r=Mbn){let i="Quality status rows were unavailable.";for(let c=1;c<=Vm;c+=1){Ao(t);try{return await rxa(e,t,r)}catch(l){if(t?.aborted)throw new Error("Cancelled by user.");i=l instanceof Error?l.message:String(l),c<Vm&&await eR(pAe*c,t)}}throw new Error(`Could not read quality status after ${Vm} attempts. ${i}`)}async function oAe(e,t){return await fL(e,t).locator('button[title^="Re-run"]').count()>0}async function axa(e,t){await fL(e,t).hover();let r=JEa(e,t);await r.waitFor({state:"visible",timeout:Nm});let i=r.locator('button[title="Expand to full view"]'),c=await i.count();if(c!==1)throw new Error(`${t}: expected one quality expander, found ${c}`);return i.first()}async function oxa(e,t,r){Ao(r),await(await axa(e,t)).click();let c=e.getByRole("dialog").last();await c.waitFor({state:"visible",timeout:Nm});let l=HC(await c.locator("h2").first().innerText().catch(()=>""));if(l!==t)throw new Error(`${t}: expanded the wrong report dialog (${l||"unknown"}).`);let u=c.locator("pre code").last();if(!await u.isVisible().catch(()=>!1)){let m=c.locator("button").filter({hasText:/raw output/i}).first();try{await m.waitFor({state:"visible",timeout:Nm})}catch{let I=HC(await c.innerText());throw new Error(`${t}: raw output control did not appear; dialog text: ${I.slice(0,1e3)}`)}await m.click()}await u.waitFor({state:"visible",timeout:Nm});let p=await u.innerText(),d=exa(p,t);return await e.keyboard.press("Escape").catch(()=>{}),{quality:t,parsed:d}}async function iAe(e,t,r){let i="The report dialog could not be opened.";for(let c=1;c<=Vm;c+=1){Ao(r);try{return await oxa(e,t,r)}catch(l){if(r?.aborted)throw new Error("Cancelled by user.");i=l instanceof Error?l.message:String(l),await e.keyboard.press("Escape").catch(()=>{}),c<Vm&&await eR(pAe*c,r)}}throw new Error(`${t}: report dialog failed after ${Vm} attempts. ${i}`)}function Nbn(e){return Array.isArray(e.comments)?e.comments.filter(t=>tr(t)&&t.severity==="major").map(t=>({category:t.category,quote:t.quote,suggestion:lAe(t.suggestion,500)??t.suggestion})):[]}function Vbn(e){if(typeof e!="string")return!1;let t=e.toLowerCase().replace(/[^a-z0-9]/g,"");return t==="ai"||t.startsWith("ai")||t.includes("aigenerated")||/(^|[^a-z0-9])ai([^a-z0-9]|$)/i.test(e)}function ixa(e){return Array.isArray(e.comments)&&e.comments.some(t=>tr(t)&&Vbn(t.category))}function cxa(e){let t={verdict:"FAIL",reason:"Prompt Quality flagged this prompt as AI. Use humanize-agent-prompt for a broader rewrite, or make a small targeted manual change if the flag is due to a recent edit."},r=Nbn(e).filter(i=>!Vbn(i.category));return r.length>0&&(t.comments=r),t}function lxa(e){if(e?.completed!==!0||!tr(e.evaluation))return{verdict:"FAIL",reason:"No completed Prompt Quality report was available."};let t=e.evaluation,r=tr(t.requirementCount)?t.requirementCount:{};if(r.blocking===!0&&r.mode==="blocking")return{verdict:"FAIL",reason:"There are too many requirements in the prompt. Decrease the very easy requirements that any agent can easily implement."};if(ixa(t))return cxa(t);if(e.verdict==="PASS"&&t.verdict==="PASS")return{verdict:"PASS"};let c=Nbn(t),l={verdict:"FAIL"};return c.length>0?l.comments=c:typeof t.summary=="string"&&(l.reason=t.summary),l}function uxa(e,t,r){let i=Array.isArray(e?.coverageSuggestions)?e.coverageSuggestions:[],c=Array.isArray(e?.tests)?e.tests.filter(v=>tr(v)&&(v.fairness==="Not fair"||Array.isArray(v.concerns)&&v.concerns.length>0)).map(v=>v.fairness==="Not fair"&&Array.isArray(v.concerns)&&v.concerns.length===0?{...v,concerns:["Not fully fair"]}:v):[],l=typeof e?.unfairTestCount=="number"&&Number.isFinite(e.unfairTestCount)?Math.max(0,Math.floor(e.unfairTestCount)):0,u=Math.max(c.length,l),p=!!e&&u===0&&i.length===0,d=e?p?{verdict:"PASS"}:{verdict:"FAIL",...i.length>0?{coverageSuggestions:i}:{},unfairnessCount:u,...c.length>0?{tests:c}:{}}:{verdict:"FAIL",reason:"No completed Test Quality report was available."},m=!!r&&dxa(r?.evaluation),I=r?m?{verdict:"PASS"}:{verdict:"FAIL",failures:pxa(r.evaluation)}:{verdict:"FAIL",reason:"No completed Solution Quality report was available."};return{promptQuality:t,...t.verdict==="PASS"?{note:HEa}:{},testQuality:d,solutionQuality:I}}function lAe(e,t=400){if(typeof e!="string"||!e.trim())return;let r=e.trim().replace(/\s+/g," ");if(r.length<=t)return r;let i=r.slice(0,t),c=i.lastIndexOf(" ");return`${i.slice(0,c>0?c:t).trim()}\u2026`}function pxa(e){if(!tr(e))return[{feedback:"No scored Solution Quality evaluation was available."}];let t=Array.isArray(e.issues)?e.issues.filter(tr):[],r=Object.entries(e).filter(([,i])=>tr(i)&&typeof i.score=="number"&&typeof i.max_score=="number"&&i.score!==i.max_score);return r.length===0?[{feedback:lAe(e.summary)??"One or more required scores did not meet its maximum."}]:r.map(([i,c])=>{if(!tr(c))return{criterion:i};let l=i.toLowerCase().replace(/[^a-z0-9]/g,""),u=t.find(d=>{let m=typeof d.criterion=="string"?d.criterion.toLowerCase().replace(/[^a-z0-9]/g,""):"";return m&&(l.endsWith(m)||m.endsWith(l))}),p=lAe(u?.detail??c.reasoning??e.summary);return{criterion:i,score:c.score,max_score:c.max_score,...typeof u?.title=="string"?{title:u.title}:{},...p?{feedback:p}:{}}})}function dxa(e){return!tr(e)||!["code_quality","solution_comprehensiveness"].every(i=>{let c=e[i];return tr(c)&&typeof c.score=="number"&&typeof c.max_score=="number"})?!1:Object.values(e).filter(i=>tr(i)&&("score"in i||"max_score"in i)).every(i=>typeof i.score=="number"&&typeof i.max_score=="number"&&i.score===i.max_score)}function mxa(e,t){let r=tr(e)?e:{},i=tr(r.testQuality)?r.testQuality:void 0,c=i??{},l=tr(r.promptQuality)?r.promptQuality:{},u=tr(r.solutionQuality)?r.solutionQuality:void 0,p=u??{},d=typeof c.unfairnessCount=="number"?c.unfairnessCount:Array.isArray(c.tests)?c.tests.length:0,m=Array.isArray(c.coverageSuggestions)?c.coverageSuggestions.length:0,I=Array.isArray(l.comments)?l.comments:[],v=I.length,T=c.verdict===null||c.verdict===void 0?"skipped":c.verdict==="PASS"?"PASS":typeof c.reason=="string"?`FAIL \xB7 ${c.reason}`:`${d} unfairness \xB7 ${m} suggestions`,x=l.verdict===null||l.verdict===void 0?"skipped":l.cached===!0?`${l.verdict} \xB7 cached`:l.verdict==="PASS"?"PASS":typeof l.reason=="string"?`${l.verdict} \xB7 ${l.reason}`:`${l.verdict} \xB7 ${v} major comment${v===1?"":"s"}`,k=Array.isArray(p.failures)?p.failures:[],L=p.verdict===null||p.verdict===void 0?"skipped":p.verdict==="PASS"?"PASS":typeof p.reason=="string"?`FAIL \xB7 ${p.reason}`:`${k.length} solution-quality issue${k.length===1?"":"s"}`,M=I.flatMap(H=>tr(H)?[`    Category: ${String(H.category??"")}`,`    Quote: ${String(H.quote??"")}`,`    Suggestion: ${String(H.suggestion??"")}`]:[]),N=[`  ${t.fg("accent","Prompt Quality")}  ${t.fg("muted",x)}`,...M];return i&&N.push(`  ${t.fg("accent","Test Quality")}  ${t.fg("muted",T)}`),u&&N.push(`  ${t.fg("accent","Solution Quality")}  ${t.fg("muted",L)}`),N}function wbn(e,t){let i=e.find(c=>new RegExp(`^${t}:`,"i").test(c))?.match(/:\s*(\d+)/)?.[1];return i===void 0?void 0:Number.parseInt(i,10)}function fxa(e,t,r,i,c,l){return e==="new tests before solution"?t!==void 0&&t>0?"Some individual test cases or subtests pass before the solution; the precheck remains failed. Ensure every new case fails or errors before the solution.":l&&!/:\s*none$/i.test(l)?"New tests fail during collection; move solution-dependent imports or setup into each test so every testcase fails or errors individually.":t===0&&/:\s*none$/i.test(i)&&/:\s*none$/i.test(c)?"The test file did not produce individual failures or errors; use lazy imports so every test executes separately.":"Every new test must fail or error individually before the solution.":e==="new tests after solution"?r!==void 0&&r>0?"New tests were skipped after the solution; every new test must execute and pass.":"The solution does not make every new test pass.":e==="base tests after solution"?"Base-test regression: existing base tests fail after the solution.":e==="base tests before solution"?"Base tests fail before the solution; review the test patch and test setup.":"The patch precheck did not meet the required test outcomes."}function gxa(e){let t=(e.content??[]).map(c=>c.type==="text"?c.text??"":"").join(" "),r=t.split(/\r?\n/).map(c=>c.trim().replace(/\s+/g," ")).filter(Boolean),i=r.find(c=>/^phase:/i.test(c));if(/patch precheck/i.test(t)){if(i){let c=i.replace(/^phase:\s*/i,""),l=wbn(r,"passed tests"),u=wbn(r,"skipped tests"),p=r.find(I=>/^failed tests:/i.test(I))??"",d=r.find(I=>/^errored tests:/i.test(I))??"",m=r.find(I=>/^collection errors:/i.test(I))??"";return["Patch precheck failed.",`Phase: ${c}`,`Problem: ${fxa(c,l,u,p,d,m)}`].join(`
+${T}`:""}`)}Ao(r);let[m,I,v]=await Promise.all([tAe(c,"utf8"),tAe(l,"utf8"),tAe(u,"utf8")]);return{taskPrompt:m,testPatch:I,solutionPatch:v}}async function nxa(e,t,r){Ao(r);let i=await e.goto(t,{waitUntil:"domcontentloaded",timeout:6e4});if(Ao(r),!jC(e.url()))throw new Error("Shipd authentication or job-link navigation failed. Check the saved job link and auth state.");if(ZEa(e.url()))throw new Error("Shipd authentication failed: the saved session is expired or signed out. Run /shipd:auth again.");if(i&&i.status()>=400)throw new Error(`Shipd job-link navigation returned HTTP ${i.status()}.`);await e.waitForTimeout(4e3),Ao(r)}async function rAe(e,t,r,i){Ao(i);let c=await kbn.launch({executablePath:e,headless:!0});try{let l=await c.newContext({storageState:t});await l.route("**/*",async p=>{let d=p.request().resourceType();d==="image"||d==="font"||d==="media"?await p.abort():await p.continue()});let u=await l.newPage();return await nxa(u,r,i),{browser:c,page:u}}catch(l){throw await c.close().catch(()=>{}),l}}async function xbn(e,t,r){await t.click({timeout:Nm}),await e.keyboard.press(`${process.platform==="darwin"?"Meta":"Control"}+A`),await e.keyboard.insertText(r)}async function Pbn(e,t,r){Ao(r);let i=e.locator("#problem-description"),c=e.locator('#problem-test-patch [contenteditable="true"][role="textbox"]'),l=e.locator('#problem-solution-patch [contenteditable="true"][role="textbox"]');for(let u of[i,c,l])if(await u.count()!==1)throw new Error("Shipd challenge form selectors were not found exactly once.");await i.fill(t.taskPrompt),await xbn(e,c,t.testPatch),await xbn(e,l,t.solutionPatch),await l.evaluate(u=>u.blur()),await fL(e,"Task Prompt Quality").hover(),await eR(2e3,r)}async function Obn(e,t,r,i){let c=fL(e,t),l=HC(await c.innerText()),u=await c.locator('button[title^="Run"]').count();if(dAe(l)||l&&r&&!mAe(l)||l&&i&&u===0)return l}async function sxa(e,t,r,i,c){let l=Date.now()+UEa;for(;Date.now()<l;){Ao(c);let u=await Obn(e,t,r,i);if(u!==void 0)return u;await eR(200,c)}}async function $bn(e,t,r){let i="The Run action did not start the quality check.";for(let c=1;c<=Vm;c+=1){Ao(r);let l=!1,u=!1,p=null,d="";try{let I=fL(e,t),v=HC(await I.innerText());if(dAe(v))return{quality:Jg(t),status:"already-running",rowTextAfterConfirm:v};l=mAe(v);let T=I.locator('button[title^="Run"]'),x=I.locator('button[title^="Re-run"]');if(u=await T.count()>0,!l&&!u)return{quality:Jg(t),status:"no-longer-stale",rowTextAfterConfirm:v};let k=await x.count(),L=await T.count(),M=k===1?x:T,N=k===1?k:L;if(N!==1)throw new Error(`${t}: expected one Run button, found ${N}`);if(!await M.isEnabled())throw new Error(`${t}: Run button is disabled.`);p=await M.getAttribute("title"),await M.click({timeout:Nm});let H=e.getByRole("dialog").last();await H.waitFor({state:"visible",timeout:Nm}),d=HC(await H.innerText());let re=H.getByRole("button",{name:/confirm/i});if(await re.count()!==1)throw new Error(`${t}: rerun confirmation button was not found.`);await re.click({timeout:Nm});let me=await sxa(e,t,l,u,r);if(me!==void 0)return{quality:Jg(t),status:"started",rerunTitle:p,confirmation:d,rowTextAfterConfirm:me};i=`${t}: the confirmation was accepted, but the stale status did not clear and no running state appeared.`}catch(I){if(r?.aborted)throw new Error("Cancelled by user.");i=I instanceof Error?I.message:String(I)}let m=await Obn(e,t,l,u).catch(()=>{});if(m!==void 0)return{quality:Jg(t),status:"started",rerunTitle:p,confirmation:d,rowTextAfterConfirm:m};await e.keyboard.press("Escape").catch(()=>{}),c<Vm&&await eR(pAe*c,r)}throw new Error(`${t}: could not start after ${Vm} attempts. ${i}`)}async function rxa(e,t,r=Mbn){let i=[];for(let c of r){Ao(t);let l=Gbn(e,c);if(await l.count()!==1)throw new Error(`${c}: quality status row was not found.`);await l.waitFor({state:"visible",timeout:Nm});let u=l.locator(".."),p=HC(await u.innerText()),d=await u.locator('button[title^="Run"]').count()>0;i.push({quality:c,rowText:p,busy:dAe(p),shouldRun:d||mAe(p)})}return i}async function aAe(e,t,r=Mbn){let i="Quality status rows were unavailable.";for(let c=1;c<=Vm;c+=1){Ao(t);try{return await rxa(e,t,r)}catch(l){if(t?.aborted)throw new Error("Cancelled by user.");i=l instanceof Error?l.message:String(l),c<Vm&&await eR(pAe*c,t)}}throw new Error(`Could not read quality status after ${Vm} attempts. ${i}`)}async function oAe(e,t){return await fL(e,t).locator('button[title^="Re-run"]').count()>0}async function axa(e,t){await fL(e,t).hover();let r=JEa(e,t);await r.waitFor({state:"visible",timeout:Nm});let i=r.locator('button[title="Expand to full view"]'),c=await i.count();if(c!==1)throw new Error(`${t}: expected one quality expander, found ${c}`);return i.first()}async function oxa(e,t,r){Ao(r),await(await axa(e,t)).click();let c=e.getByRole("dialog").last();await c.waitFor({state:"visible",timeout:Nm});let l=HC(await c.locator("h2").first().innerText().catch(()=>""));if(l!==t)throw new Error(`${t}: expanded the wrong report dialog (${l||"unknown"}).`);let u=c.locator("pre code").last();if(!await u.isVisible().catch(()=>!1)){let m=c.locator("button").filter({hasText:/raw output/i}).first();try{await m.waitFor({state:"visible",timeout:Nm})}catch{let I=HC(await c.innerText());throw new Error(`${t}: raw output control did not appear; dialog text: ${I.slice(0,1e3)}`)}await m.click()}await u.waitFor({state:"visible",timeout:Nm});let p=await u.innerText(),d=exa(p,t);return await e.keyboard.press("Escape").catch(()=>{}),{quality:t,parsed:d}}async function iAe(e,t,r){let i="The report dialog could not be opened.";for(let c=1;c<=Vm;c+=1){Ao(r);try{return await oxa(e,t,r)}catch(l){if(r?.aborted)throw new Error("Cancelled by user.");i=l instanceof Error?l.message:String(l),await e.keyboard.press("Escape").catch(()=>{}),c<Vm&&await eR(pAe*c,r)}}throw new Error(`${t}: report dialog failed after ${Vm} attempts. ${i}`)}function Nbn(e){return Array.isArray(e.comments)?e.comments.filter(t=>tr(t)&&t.severity==="major").map(t=>({category:t.category,quote:t.quote,suggestion:lAe(t.suggestion,500)??t.suggestion})):[]}function Vbn(e){if(typeof e!="string")return!1;let t=e.toLowerCase().replace(/[^a-z0-9]/g,"");return t==="ai"||t.startsWith("ai")||t.includes("aigenerated")||/(^|[^a-z0-9])ai([^a-z0-9]|$)/i.test(e)}function ixa(e){return Array.isArray(e.comments)&&e.comments.some(t=>tr(t)&&Vbn(t.category))}function cxa(e){let t={verdict:"FAIL",reason:"Prompt Quality flagged this prompt as AI. Use humanize-agent-prompt for a broader rewrite, or make a small targeted manual change if the flag is due to a recent edit."},r=Nbn(e).filter(i=>!Vbn(i.category));return r.length>0&&(t.comments=r),t}function lxa(e){if(e?.completed!==!0||!tr(e.evaluation))return{verdict:"FAIL",reason:"No completed Prompt Quality report was available."};let t=e.evaluation,r=tr(t.requirementCount)?t.requirementCount:{};if(r.blocking===!0&&r.mode==="blocking")return{verdict:"FAIL",reason:"There are too many requirements in the prompt. Decrease the very easy requirements that any agent can easily implement."};if(ixa(t))return cxa(t);if(e.verdict==="PASS"&&t.verdict==="PASS")return{verdict:"PASS"};let c=Nbn(t),l={verdict:"FAIL"};return c.length>0?l.comments=c:typeof t.summary=="string"&&(l.reason=t.summary),l}function uxa(e,t,r){let i=Array.isArray(e?.coverageSuggestions)?e.coverageSuggestions:[],c=Array.isArray(e?.tests)?e.tests.filter(v=>tr(v)&&(v.fairness==="Not fair"||Array.isArray(v.concerns)&&v.concerns.length>0)).map(v=>v.fairness==="Not fair"&&Array.isArray(v.concerns)&&v.concerns.length===0?{...v,concerns:["Not fully fair"]}:v):[],l=typeof e?.unfairTestCount=="number"&&Number.isFinite(e.unfairTestCount)?Math.max(0,Math.floor(e.unfairTestCount)):0,u=Math.max(c.length,l),p=!!e&&u===0&&i.length===0,d=e?p?{verdict:"PASS"}:{verdict:"FAIL",...i.length>0?{coverageSuggestions:i}:{},unfairnessCount:u,...c.length>0?{tests:c}:{}}:{verdict:"FAIL",reason:"No completed Test Quality report was available."},m=!!r&&dxa(r?.evaluation),I=r?m?{verdict:"PASS"}:{verdict:"FAIL",failures:pxa(r.evaluation)}:{verdict:"FAIL",reason:"No completed Solution Quality report was available."};return{promptQuality:t,...t.verdict==="PASS"?{note:HEa}:{},testQuality:d,solutionQuality:I}}function lAe(e,t=400){if(typeof e!="string"||!e.trim())return;let r=e.trim().replace(/\s+/g," ");if(r.length<=t)return r;let i=r.slice(0,t),c=i.lastIndexOf(" ");return`${i.slice(0,c>0?c:t).trim()}\u2026`}function pxa(e){if(!tr(e))return[{feedback:"No scored Solution Quality evaluation was available."}];let t=Array.isArray(e.issues)?e.issues.filter(tr):[],r=Object.entries(e).filter(([,i])=>tr(i)&&typeof i.score=="number"&&typeof i.max_score=="number"&&i.score!==i.max_score);return r.length===0?[{feedback:lAe(e.summary)??"One or more required scores did not meet its maximum."}]:r.map(([i,c])=>{if(!tr(c))return{criterion:i};let l=i.toLowerCase().replace(/[^a-z0-9]/g,""),u=t.find(d=>{let m=typeof d.criterion=="string"?d.criterion.toLowerCase().replace(/[^a-z0-9]/g,""):"";return m&&(l.endsWith(m)||m.endsWith(l))}),p=lAe(u?.detail??c.reasoning??e.summary);return{criterion:i,score:c.score,max_score:c.max_score,...typeof u?.title=="string"?{title:u.title}:{},...p?{feedback:p}:{}}})}function dxa(e){return!tr(e)||!["code_quality","solution_comprehensiveness"].every(i=>{let c=e[i];return tr(c)&&typeof c.score=="number"&&typeof c.max_score=="number"})?!1:Object.values(e).filter(i=>tr(i)&&("score"in i||"max_score"in i)).every(i=>typeof i.score=="number"&&typeof i.max_score=="number"&&i.score===i.max_score)}function mxa(e,t){let r=tr(e)?e:{},i=tr(r.testQuality)?r.testQuality:void 0,c=i??{},l=tr(r.promptQuality)?r.promptQuality:{},u=tr(r.solutionQuality)?r.solutionQuality:void 0,p=u??{},d=typeof c.unfairnessCount=="number"?c.unfairnessCount:Array.isArray(c.tests)?c.tests.length:0,m=Array.isArray(c.coverageSuggestions)?c.coverageSuggestions.length:0,I=Array.isArray(l.comments)?l.comments:[],v=I.length,T=c.verdict===null||c.verdict===void 0?"skipped":c.verdict==="PASS"?"PASS":typeof c.reason=="string"?`FAIL \xB7 ${c.reason}`:`${d} unfairness \xB7 ${m} suggestions`,x=l.verdict===null||l.verdict===void 0?"skipped":l.cached===!0?`${l.verdict} \xB7 cached`:l.verdict==="PASS"?"PASS":typeof l.reason=="string"?`${l.verdict} \xB7 ${l.reason}`:`${l.verdict} \xB7 ${v} major comment${v===1?"":"s"}`,k=Array.isArray(p.failures)?p.failures:[],L=p.verdict===null||p.verdict===void 0?"skipped":p.verdict==="PASS"?"PASS":typeof p.reason=="string"?`FAIL \xB7 ${p.reason}`:`${k.length} solution-quality issue${k.length===1?"":"s"}`,M=I.flatMap(H=>tr(H)?[`    Category: ${String(H.category??"")}`,`    Quote: ${String(H.quote??"")}`,`    Suggestion: ${String(H.suggestion??"")}`]:[]),N=[`  ${t.fg("accent","Prompt Quality")}  ${t.fg("muted",x)}`,...M];return i&&N.push(`  ${t.fg("accent","Test Quality")}  ${t.fg("muted",T)}`),u&&N.push(`  ${t.fg("accent","Solution Quality")}  ${t.fg("muted",L)}`),N}function wbn(e,t){let i=e.find(c=>new RegExp(`^${t}:`,"i").test(c))?.match(/:\s*(\d+)/)?.[1];return i===void 0?void 0:Number.parseInt(i,10)}function fxa(e,t,r,i,c,l){return e==="new tests before solution"?t!==void 0&&t>0?"Some individual test cases or subtests pass before the solution; the precheck remains failed. Ensure every new case fails or errors before the solution.":l&&!/:\s*none$/i.test(l)?"New tests fail during collection; move solution-dependent imports or setup into each test so every testcase fails or errors individually.":t===0&&/:\s*none$/i.test(i)&&/:\s*none$/i.test(c)?"The test file did not produce individual failures or errors; use lazy imports so every test executes separately.":"Every new test must fail or error individually before the solution.":e==="new tests after solution"?r!==void 0&&r>0?"New tests were skipped after the solution; every new test must execute and pass.":"The solution does not make every new test pass.":e==="base tests after solution"?"Base-test regression: existing base tests fail after the solution.":e==="base tests before solution"?"Base tests fail before the solution; review the test patch and test setup.":"The patch precheck did not meet the required test outcomes."}function gxa(e){let t=(e.content??[]).map(c=>c.type==="text"?c.text??"":"").join(" "),r=t.split(/\r?\n/).map(c=>c.trim().replace(/\s+/g," ")).filter(Boolean),i=r.find(c=>/^phase:/i.test(c));if(/patch precheck/i.test(t)){if(i){let c=i.replace(/^phase:\s*/i,""),l=wbn(r,"passed tests"),u=wbn(r,"skipped tests"),p=r.find(I=>/^failed tests:/i.test(I))??"",d=r.find(I=>/^errored tests:/i.test(I))??"",m=r.find(I=>/^collection errors:/i.test(I))??"";return["Patch precheck failed.",`Phase: ${c}`,`Problem: ${fxa(c,l,u,p,d,m)}`].join(`
 `)}return`Patch precheck failed.
 Problem: The precheck task could not be completed.`}return r.map(c=>c.replace(/\bFargate\b\s*/gi,"").trim()).slice(0,4).join(`
 `)||"Quality checks failed"}function Rxa(e){let t=`Fargate patch precheck failed:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sreetej510/pi-shipd-checks",
-  "version": "0.11.6",
+  "version": "0.11.7",
   "description": "Pi extension for Shipd checks, behavioral test-gap analysis, and opt-in CoreWeave-backed agent_prompt.md humanization.",
   "type": "module",
   "license": "MIT",