@skyramp/testbot-core 0.13.0-v3.61 → 0.13.0-v3.62
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/VERSION.json +1 -1
- package/dist/main.js +2 -2
- package/dist/post.js +2 -2
- package/package.json +1 -1
package/dist/VERSION.json
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":"0.13.0-v3.
|
|
1
|
+
{"version":"0.13.0-v3.62","commit":"2519ef1efcb41022cd492f4b60b0a0015168c66d"}
|
package/dist/main.js
CHANGED
|
@@ -693,8 +693,8 @@ except Exception:
|
|
|
693
693
|
`);let u=Cb.join(t.outputsDir,"report.json");return Epe.writeFileSync(u,JSON.stringify(o,null,2)+`
|
|
694
694
|
`),{report:Cb.relative(t.runDir,u),"report-salvage":Cb.relative(t.runDir,Cb.join(t.outputsDir,"report-salvage.json"))}}var XP=$(require("fs")),nc=$(require("path"));var Nr=w.string().trim().min(1),Zor=t=>{if(!t||typeof t!="object")return t;let e=t;return typeof e.skip=="string"&&e.skip.trim()&&(e.expected===void 0||e.expected===null)?{...e,expected:"(not tested)"}:t},ear=w.preprocess(Zor,w.object({id:Nr.describe('requirement id + letter, e.g. "R1a"'),clause:Nr.describe("one testable part of the requirement"),expected:Nr.describe("the outcome the requirement calls for \u2014 taken from the requirement, never from what the code does"),skip:w.string().optional().describe("only when no test can check this clause here: why"),type:w.nativeEnum(sp).optional().describe("the test that checks this clause: contract / integration (API) or ui"),target:w.string().optional().describe('its "METHOD /path", or the page / flow'),steps:w.string().optional().describe("what that test does; its expected outcome is this clause's"),finding:w.string().optional().describe("the finding id, when this clause's test also catches that defect (the defect still gets a case of its own)"),code:w.string().optional().describe("path:line of the code that decides this clause, when there is one")})),tar=w.object({id:Nr.describe('e.g. "R1"'),statement:Nr,source:w.nativeEnum(tI).describe("spec, or pr-description (also for what the change is evidently meant to do when nothing is written down)"),evidence:Nr.describe("where it is stated: file:line, a doc section, or the quoted phrase"),clauses:w.array(ear)}),rar=w.object({id:Nr.describe('e.g. "F1"'),file:Nr,line:w.number().optional(),claim:Nr.describe("the defect, in one sentence"),severity:w.nativeEnum(eI),evidence:Nr.describe("the code line, or the probe request and response, that shows it"),conflictsWith:w.array(Nr).optional().describe("requirement or clause ids it violates")}),nar=w.object({id:Nr.describe('unique, a short kebab-case name for what the case checks, e.g. "ship-twice-409"'),type:w.nativeEnum(sp).describe("contract / integration: an API test; ui: drives the running frontend in a browser"),target:Nr.describe('"METHOD /path", or the page / flow'),clause:w.string().optional().describe("the clause id this case checks"),finding:w.string().optional().describe("the finding id this case catches"),steps:Nr.describe("what the test does"),expected:Nr.describe("the expected outcome, from the clause")}),sar=w.object({test:Nr.describe("existing test file"),action:w.nativeEnum(ob),reason:Nr}),nit=w.object({outcome:w.nativeEnum(tf).describe("clean | recipe-drift (you had to fix the recipe) | change-broke-bringup (this PR broke start-up) | bringup-failed (the environment, not the PR)"),baseUrls:w.record(w.string()).optional().describe("service \u2192 base URL of the running app"),health:w.object({url:w.string().optional(),command:w.string().optional(),output:w.string().optional()}).optional().describe("the check that shows the app serves real data, and what it returned"),teardown:w.string().optional().describe("command that stops the app"),dependencies:w.record(w.boolean()).optional().describe("every backing service: true = really running, false = mocked"),recipeFixes:w.array(w.string()).optional().describe("what you changed in the recipe"),error:w.string().optional().describe("when it did not come up: the failing command and what it printed"),suspectFiles:w.array(w.string()).optional().describe("change-broke-bringup: the changed files to blame")}),Hg=w.object({schemaVersion:w.literal(1),requirements:w.array(tar),findings:w.array(rar),cases:w.array(nar).describe("a second test for a clause, and one case per finding that fails on this branch"),maintenance:w.array(sar),triage:ub.optional().describe("the validation triage decision, made before the app is started; its behaviorIds are clause ids"),app:nit.optional().describe("omit when no app was needed or none was started"),summary:Nr.describe("one paragraph: what the change does and where the risk is")}),sit=(r=>(r.Backend="backend",r.Ui="ui",r))(sit||{}),iit=(s=>(s.Added="added",s.Updated="updated",s.Deleted="deleted",s.Unchanged="unchanged",s))(iit||{}),iar=w.object({file:Nr.describe("repo-relative path; in another repo of the change, that repo's directory name then its path"),kind:w.nativeEnum(sit),change:w.nativeEnum(iit).describe("added: a file you created; updated / deleted / unchanged: an existing test"),covers:w.array(w.string()).describe("the plan case ids this file tests"),defect:w.boolean().optional().describe("true when it fails because the app is wrong and the test is right \u2014 kept failing on purpose"),result:w.object({exit:w.number().nullable().describe("exit code of your last run of this file; null if it could not be run"),cases:w.array(w.object({case:Nr,status:w.nativeEnum(sf)})).describe("one entry per covered case, from the runner's own output"),details:w.string().optional().describe("what failed and why, or why it was not run")}),assertions:w.array(w.object({case:Nr,evidence:Nr})).optional().describe("per covered case, the assertion line that checks its expected outcome")}),oar=w.object({description:Nr.describe("what is wrong and how to reproduce it: endpoint or page, input, what you got vs what was expected"),severity:w.nativeEnum(eI).optional(),sourceFile:w.string().optional().describe("the app file at fault"),sourceLine:w.number().optional(),test:w.string().optional().describe("the test file that shows it")}),zg=w.object({schemaVersion:w.literal(1),tests:w.array(iar),notWritten:w.array(w.object({case:Nr,reason:Nr})).describe("every plan case no test covers, and why"),lint:w.object({command:w.string(),clean:w.boolean()}).optional().describe("the linter/formatter you ran on your test files, if the repo has one"),dependenciesAdded:w.array(w.object({package:Nr,manifest:Nr})).optional(),ciRunsTests:w.boolean().optional().describe("will the repo's CI pick these tests up as they are?"),commit:w.object({message:Nr.describe('"Skyramp Testbot: \u2026", subject line at most 72 characters'),prTitle:Nr.describe("at most 72 characters"),prBody:Nr.describe("what the tests cover and what they found")}),report:w.object({summary:Nr.describe("1-2 sentences for the PR author: does the change do what it says?"),issues:w.array(oar).describe("every defect the tests found"),nextSteps:w.array(w.string())})});function xpe(t,e,r=[]){let{value:n,error:s}=zr(e,r);return s?{errors:[s]}:Ts(t,n)}function Gg(t,e,r,n=[]){let s=xpe(t,e,n);if(s.value)return s;let i=ZP(r);return i===void 0?s:xpe(t,i,n)}function oit(t){let e=new Set(t.cases.map(n=>n.id)),r=t.requirements.flatMap(n=>n.clauses.filter(s=>s.steps?.trim()&&!s.skip&&!e.has(s.id)).map(s=>({id:s.id,type:s.type??"integration",target:s.target?.trim()||n.id,clause:s.id,...s.finding?{finding:s.finding}:{},steps:s.steps.trim(),expected:s.expected})));return r.length?{...t,cases:[...r,...t.cases]}:t}function ait(t){let e=t.triage;if(e?.decision!=="runtime_required")return t;let r=new Set(e.plan.filter(i=>i.mechanism==="runtime").map(i=>i.behaviorId)),n=new Set([...e.plan,...e.outOfScope??[],...e.gaps].flatMap(i=>i.behaviorId?[i.behaviorId]:[]).filter(i=>!r.has(i))),s=t.cases.filter(i=>!i.clause||!n.has(i.clause));return s.length===t.cases.length?t:{...t,cases:s}}function b2(t,e){for(let r of e){if(!r)continue;let{value:n}=xpe(t,r);if(n)return n}}function cit(t){if(t)try{let e=JSON.parse(t);if(!e||typeof e!="object"||!("app"in e)||nit.safeParse(e.app).success)return;let{app:r,...n}=e;return JSON.stringify(n)}catch{return}}function ZP(t){try{return XP.readFileSync(t,"utf-8")}catch{return}}function vI(t,e){let r=new Set;return t.filter(n=>{let s=e(n).trim();return r.has(s)?!1:(r.add(s),!0)})}function Cpe(t){return t?.outcome==="clean"||t?.outcome==="recipe-drift"}var lit=t=>t.code?.trim()||t.requirement.evidence;function Spe(t){return vI(t.requirements.flatMap(e=>e.clauses.map(r=>({id:r.id,evidence:lit({...r,requirement:e})}))),e=>e.id)}function uit(t,e,r=new Date){let n=vI(t.requirements,m=>m.id),s=vI(n.flatMap(m=>m.clauses.map(h=>({...h,requirement:m}))),m=>m.id),i=new Set(s.map(m=>m.id)),o=vI(t.findings,m=>m.id),a=new Set(o.map(m=>m.id)),c=vI(t.cases,m=>m.id),l=Cpe(t.app),u=s.map(m=>({id:m.id,statement:m.clause,surface:c.find(h=>h.clause===m.id)?.target??m.requirement.id,evidence:lit(m),satisfies:[m.requirement.id]})),d={schemaVersion:1,repository:e.repository,project:{type:"other",primaryLanguage:"unknown",languages:[],frameworks:[]},services:[],infrastructure:{containerized:!1,hasDockerCompose:!1,composeFiles:[],hasCiCd:!1},diff:{changedFiles:e.changedFiles,nonApplicationOnly:e.nonApplicationOnly,summary:t.summary.slice(0,400)},endpoints:{new:[],modified:[],removed:[]},existingTests:{locations:[]},entityMap:{entities:[],references:[],stateFields:[]},requirements:n.map(m=>({id:m.id,statement:m.statement,source:m.source,evidence:m.evidence})),behaviors:u,findings:o.map(m=>({id:m.id,file:m.file,line:m.line,claim:m.claim,severity:m.severity,evidence:m.evidence,status:"suspected",conflictsWith:m.conflictsWith})),unexamined:[],summary:t.summary},p={schemaVersion:1,mode:l?"full":"static",maintenance:vI(t.maintenance,m=>m.test).map(m=>({test:m.test,action:m.action,evidence:m.reason})),generate:c.map(m=>({id:m.id,type:m.type,target:m.target,from_behavior:m.clause&&i.has(m.clause)?m.clause:void 0,from_finding:m.finding&&a.has(m.finding)?m.finding:void 0,rationale:s.find(h=>h.id===m.clause)?.clause??o.find(h=>h.id===m.finding)?.claim??m.steps,expected_observable:m.expected,verifiable:l})),surfaces:[],budget:{total:c.length,rationale:"planned in one session from the requirements' clauses"},summary:t.summary};return{analyze:d,recommend:p,deploy:t.app?aar(t.app,e.defaultTeardown,r):void 0}}function aar(t,e,r){return{schemaVersion:1,baseUrls:t.baseUrls,health:t.health?{url:t.health.url,command:t.health.command,output:t.health.output??""}:void 0,teardown:t.teardown?.trim()||e||"true",dependencies:t.dependencies,outcome:t.outcome,recipeFixes:t.recipeFixes,bringupError:t.error?{command:"bring-up",output:t.error}:void 0,suspectFiles:t.suspectFiles,seeded:!1,startedAt:r.toISOString()}}var car={added:"place",updated:"update",deleted:"delete",unchanged:"verify"};function lar(t){return t.result.exit!==null&&t.result.exit!==0||t.result.cases.some(e=>e.status==="fail")}function rit(t){let[e,...r]=t.split(`
|
|
695
695
|
`);return[e.length<=$g?e:`${e.slice(0,$g-1).trimEnd()}\u2026`,...r].join(`
|
|
696
|
-
`)}function dit(t,e,r=[]){let n=[],s=[],i=new Set,o=new Map([...r].reverse().map(
|
|
697
|
-
`)[0],body:t.commit.prBody},ready:
|
|
696
|
+
`)}function dit(t,e,r=[]){let n=[],s=[],i=new Set,o=new Map([...r].reverse().map(h=>{let f=nc.basename(nc.resolve(h.path));return[f,{name:f,root:nc.resolve(h.path),slug:h.slug}]})),a=new Map,c=h=>`${h.repository??""}\0${h.file}`;for(let h of t.tests){let f=e,A=Mr(e,h.file),E,y,C=h.file.replace(/^\.\//,"").split("/")[0],I=nc.resolve(e,h.file),v=[...o.values()].find(N=>I.startsWith(N.root+nc.sep));if(o.has(C)&&!XP.existsSync(nc.join(e,C))?({name:y,root:f,slug:E}=o.get(C),A=Mr(f,h.file.replace(/^\.\//,"").slice(C.length+1))):v&&({name:y,root:f,slug:E}=v,A=Mr(f,I)),!A||A.startsWith("..")||nc.isAbsolute(A)){n.push(h.file);continue}if(h.change!=="deleted"&&!XP.existsSync(nc.join(f,A))){n.push(h.file);continue}let T={...h,file:A,shown:y?`${y}/${A}`:A,...E?{repository:E}:{}};i.has(c(T))||(i.add(c(T)),a.set(c(T),f),s.push(T))}let l=s.map(h=>{let f=car[h.change];return{...f==="place"?{from:h.shown}:{},file:h.file,...h.repository?{repository:h.repository}:{},action:f,covers:h.covers,...h.defect&&lar(h)?{finding:!0}:{},reran:{exit:h.result.exit,details:h.result.details,tests:h.result.cases},...f==="delete"?{}:{assertionCount:uf(a.get(c(h)),h.file,Lg)}}}),u=h=>({schemaVersion:1,tests:s.filter(f=>f.kind===h&&f.change==="added").map(f=>({file:f.shown,source:"llm",language:ib(f.file)??"unknown",covers:f.covers,assertions:f.assertions??[]})),deferredUi:[],summary:t.report.summary}),d=t.lint?.command?.trim()??"",p=t.lint?.clean??!0,m={schemaVersion:1,lint:[{tool:d.split(/\s+/)[0]??"",command:d,clean:p,uncovered:[]}],fileConventions:{ok:!0},ciReadiness:{covered:t.ciRunsTests??!0},dependencies:{added:t.dependenciesAdded??[]},commit:{message:rit(t.commit.message)},pr:{title:rit(t.commit.prTitle).split(`
|
|
697
|
+
`)[0],body:t.commit.prBody},ready:p,blockers:p?[]:[`the linter (${d||"the repo's linter"}) still reports problems in the test files`],summary:t.report.summary};return{adapt:{schemaVersion:1,items:l,ignored:[],summary:t.report.summary},generate:u("backend"),generateUi:u("ui"),finalize:m,dropped:n}}function uar(t,e,r,n=[]){let s=t.test;if(!s)return[];let i=e.tests.find(f=>f.file===s||s.endsWith(`/${f.file}`)||f.file.endsWith(`/${s}`));if(!i)return[];let o=new Map;for(let f of r.requirements){o.set(f.id,f);for(let A of f.clauses)o.set(A.id,f)}let a=new Set(i.result.cases.filter(f=>f.status==="fail").map(f=>f.case)),c=new Map;for(let f of i.covers.filter(A=>!a.size||a.has(A))){let A=r.cases.find(y=>y.id===f);if(!A)continue;let E=A.finding?r.findings.find(y=>y.id===A.finding):void 0;for(let y of[A.clause,...E?.conflictsWith??[]]){let C=y?o.get(y):void 0;C&&c.set(C.id,C)}}let l=n.filter(f=>f.trim()),u=f=>!l.length||l.some(A=>par(f,A)),d=f=>{if(f.source==="spec")return!0;let A=pit(f.evidence);return A!==void 0&&u(A)},p=[...c.values()].filter(d).slice(0,2),m=p.filter(f=>f.source!=="spec").map(f=>dar(f.evidence)),h=p.filter(f=>f.source==="spec").map(f=>f.evidence.trim());return[...m.length?[`the PR description, which says ${m.join(" and ")}`]:[],...h.length?[`the spec (${h.join("; ")})`]:[]]}function dar(t){let e=pit(t);return e!==void 0?`"${e}"`:t.trim()}function par(t,e){let r=BP(e),n=BP(t).split(/\.\.\.|\u2026/).map(s=>s.trim()).filter(s=>s.length>=3);return n.length>0&&n.every(s=>r.includes(s))}var mar=/["\u201c]([^"\u201d]{3,300})["\u201d]|(?:^|[\s:(\[])['\u2018]([^\n]{3,300}?)['\u2019](?=[\s.,;:)\]]|$)/;function pit(t){let e=t.match(mar);return e?(e[1]??e[2]).trim():void 0}function mit(t,e,r=[]){return{schemaVersion:1,businessCaseAnalysis:t.report.summary,issuesFound:t.report.issues.map(n=>({description:far(n.test&&!n.description.includes(n.test)?`${n.description} (shown by ${n.test})`:n.description,e?uar(n,t,e,r):[]),severity:n.severity,category:"bug",sourceFile:n.sourceFile,sourceLine:n.sourceLine})),nextSteps:t.report.nextSteps}}function far(t,e){let r=e.filter(n=>!t.includes(n.replace(/^the PR description, which says /,"")));return r.length?`${t}. This contradicts ${r.join(" and ")}.`.replace(/\.\. /,". "):t}function fit(t,e){let r=TI.join(t.outputsDir,e);if(NI.existsSync(r))try{return NI.readFileSync(r,"utf-8")}catch{return}}function har(t){let e=[],r=A=>{let E=fit(t,A);if(E)try{return JSON.parse(E)}catch{return}},n=r("deploy.json");n?.outcome==="recipe-drift"?e.push("that the repo's recorded way of starting the app did not work as recorded \u2014 say what was wrong with it and what had to change, and that the scripts are now part of this PR's diff"):n?.outcome&&n.outcome!=="clean"&&e.push(`the deploy outcome "${n.outcome}" (in user terms: what it meant for the app and the tests)`);let s=Object.entries(n?.dependencies??{}).filter(([,A])=>A===!1).map(([A])=>A);s.length&&e.push(`each dependency that was NOT really connected this run, by name: ${s.join(", ")} \u2014 say what was not exercised because of it`);let o=r("adapt.json")?.items??[],a=o.filter(A=>A.finding&&A.file).map(A=>A.file);a.length&&e.push(`an issuesFound entry NAMING each test that caught a real bug: ${a.join(", ")}`);let c=o.filter(A=>A.placementBroken&&A.file).map(A=>A.file);c.length&&e.push(`a Skipped row + nextSteps item for each test that could not be placed/run: ${c.join(", ")}`);let l=o.filter(A=>A.repro&&A.file).map(A=>A.file);l.length&&e.push(`these tests are written to FAIL until the bug is fixed (their failure is the intended catch \u2014 say so in their details): ${l.join(", ")}`);let u=o.filter(A=>A.action!=="delete"&&A.file&&typeof A.reran?.exit=="number").map(A=>`${A.file} \u2192 ${A.reran.exit===0?"Pass":"Fail"}`);u.length&&e.push(`each executed test's row carries its real result (never Skipped): ${u.join("; ")}`);let d=o.filter(A=>A.action!=="delete"&&A.action!=="verify"&&A.file&&(A.reran?.exit===null||A.reran?.exit===void 0)).map(A=>A.file);d.length&&e.push(`these tests were NOT executed \u2014 their rows are Skipped (never Pass) and the report says they were not run against a live app: ${d.join(", ")}`);let m=(r("recommend.json")?.generate??[]).map(A=>A.id).filter(Boolean);m.length&&e.push(`every recommended case id appears as a newTestsCreated testId or is named in nextSteps/issuesFound with what happened to it: ${m.join(", ")}`),t.state.stages.finalize?.status==="failed"&&e.push("the tests could not be finalized (lint / commit-readiness) \u2014 say so in user terms"),t.state.stages.deploy?.status==="failed"&&e.push("the app could not be brought up \u2014 say so in user terms");let f=uI(t);return f.length&&e.push(`parts of the change analysis could not be verified (${f.length} unresolved check${f.length>1?"s":""}) \u2014 say so in user terms, and which requirements or findings that touches`),e}function gar(t){let e=Object.entries(t.state.stages).map(([a,c])=>` - ${a}: ${c.status}${c.reason?` (${c.reason})`:""}`),n=["analyze.json","deploy.json","recommend.json","generate.json","generate-ui.json","adapt.json","finalize.json"].map(a=>({n:a,body:fit(t,a)})).filter(a=>a.body).map(a=>{let c=TI.join(t.outputsDir,a.n),l=a.body.length>6e3;return`### ${a.n}${l?` (truncated here \u2014 the complete file is ${c}; Read it before writing rows, the checks below read all of it)`:""}
|
|
698
698
|
\`\`\`json
|
|
699
699
|
${a.body.slice(0,6e3)}
|
|
700
700
|
\`\`\``}).join(`
|
package/dist/post.js
CHANGED
|
@@ -478,8 +478,8 @@ except Exception:
|
|
|
478
478
|
`);let u=th.join(t.outputsDir,"report.json");return WV.writeFileSync(u,JSON.stringify(o,null,2)+`
|
|
479
479
|
`),{report:th.relative(t.runDir,u),"report-salvage":th.relative(t.runDir,th.join(t.outputsDir,"report-salvage.json"))}}var MC=j(require("fs")),Rs=j(require("path"));var Tt=y.string().trim().min(1),Fpt=t=>{if(!t||typeof t!="object")return t;let e=t;return typeof e.skip=="string"&&e.skip.trim()&&(e.expected===void 0||e.expected===null)?{...e,expected:"(not tested)"}:t},Lpt=y.preprocess(Fpt,y.object({id:Tt.describe('requirement id + letter, e.g. "R1a"'),clause:Tt.describe("one testable part of the requirement"),expected:Tt.describe("the outcome the requirement calls for \u2014 taken from the requirement, never from what the code does"),skip:y.string().optional().describe("only when no test can check this clause here: why"),type:y.nativeEnum(yc).optional().describe("the test that checks this clause: contract / integration (API) or ui"),target:y.string().optional().describe('its "METHOD /path", or the page / flow'),steps:y.string().optional().describe("what that test does; its expected outcome is this clause's"),finding:y.string().optional().describe("the finding id, when this clause's test also catches that defect (the defect still gets a case of its own)"),code:y.string().optional().describe("path:line of the code that decides this clause, when there is one")})),Upt=y.object({id:Tt.describe('e.g. "R1"'),statement:Tt,source:y.nativeEnum(wA).describe("spec, or pr-description (also for what the change is evidently meant to do when nothing is written down)"),evidence:Tt.describe("where it is stated: file:line, a doc section, or the quoted phrase"),clauses:y.array(Lpt)}),Gpt=y.object({id:Tt.describe('e.g. "F1"'),file:Tt,line:y.number().optional(),claim:Tt.describe("the defect, in one sentence"),severity:y.nativeEnum(bA),evidence:Tt.describe("the code line, or the probe request and response, that shows it"),conflictsWith:y.array(Tt).optional().describe("requirement or clause ids it violates")}),qpt=y.object({id:Tt.describe('unique, a short kebab-case name for what the case checks, e.g. "ship-twice-409"'),type:y.nativeEnum(yc).describe("contract / integration: an API test; ui: drives the running frontend in a browser"),target:Tt.describe('"METHOD /path", or the page / flow'),clause:y.string().optional().describe("the clause id this case checks"),finding:y.string().optional().describe("the finding id this case catches"),steps:Tt.describe("what the test does"),expected:Tt.describe("the expected outcome, from the clause")}),$pt=y.object({test:Tt.describe("existing test file"),action:y.nativeEnum(Gf),reason:Tt}),rSe=y.object({outcome:y.nativeEnum(Nl).describe("clean | recipe-drift (you had to fix the recipe) | change-broke-bringup (this PR broke start-up) | bringup-failed (the environment, not the PR)"),baseUrls:y.record(y.string()).optional().describe("service \u2192 base URL of the running app"),health:y.object({url:y.string().optional(),command:y.string().optional(),output:y.string().optional()}).optional().describe("the check that shows the app serves real data, and what it returned"),teardown:y.string().optional().describe("command that stops the app"),dependencies:y.record(y.boolean()).optional().describe("every backing service: true = really running, false = mocked"),recipeFixes:y.array(y.string()).optional().describe("what you changed in the recipe"),error:y.string().optional().describe("when it did not come up: the failing command and what it printed"),suspectFiles:y.array(y.string()).optional().describe("change-broke-bringup: the changed files to blame")}),Od=y.object({schemaVersion:y.literal(1),requirements:y.array(Upt),findings:y.array(Gpt),cases:y.array(qpt).describe("a second test for a clause, and one case per finding that fails on this branch"),maintenance:y.array($pt),triage:Uf.optional().describe("the validation triage decision, made before the app is started; its behaviorIds are clause ids"),app:rSe.optional().describe("omit when no app was needed or none was started"),summary:Tt.describe("one paragraph: what the change does and where the risk is")}),nSe=(r=>(r.Backend="backend",r.Ui="ui",r))(nSe||{}),iSe=(i=>(i.Added="added",i.Updated="updated",i.Deleted="deleted",i.Unchanged="unchanged",i))(iSe||{}),Hpt=y.object({file:Tt.describe("repo-relative path; in another repo of the change, that repo's directory name then its path"),kind:y.nativeEnum(nSe),change:y.nativeEnum(iSe).describe("added: a file you created; updated / deleted / unchanged: an existing test"),covers:y.array(y.string()).describe("the plan case ids this file tests"),defect:y.boolean().optional().describe("true when it fails because the app is wrong and the test is right \u2014 kept failing on purpose"),result:y.object({exit:y.number().nullable().describe("exit code of your last run of this file; null if it could not be run"),cases:y.array(y.object({case:Tt,status:y.nativeEnum(zl)})).describe("one entry per covered case, from the runner's own output"),details:y.string().optional().describe("what failed and why, or why it was not run")}),assertions:y.array(y.object({case:Tt,evidence:Tt})).optional().describe("per covered case, the assertion line that checks its expected outcome")}),zpt=y.object({description:Tt.describe("what is wrong and how to reproduce it: endpoint or page, input, what you got vs what was expected"),severity:y.nativeEnum(bA).optional(),sourceFile:y.string().optional().describe("the app file at fault"),sourceLine:y.number().optional(),test:y.string().optional().describe("the test file that shows it")}),Md=y.object({schemaVersion:y.literal(1),tests:y.array(Hpt),notWritten:y.array(y.object({case:Tt,reason:Tt})).describe("every plan case no test covers, and why"),lint:y.object({command:y.string(),clean:y.boolean()}).optional().describe("the linter/formatter you ran on your test files, if the repo has one"),dependenciesAdded:y.array(y.object({package:Tt,manifest:Tt})).optional(),ciRunsTests:y.boolean().optional().describe("will the repo's CI pick these tests up as they are?"),commit:y.object({message:Tt.describe('"Skyramp Testbot: \u2026", subject line at most 72 characters'),prTitle:Tt.describe("at most 72 characters"),prBody:Tt.describe("what the tests cover and what they found")}),report:y.object({summary:Tt.describe("1-2 sentences for the PR author: does the change do what it says?"),issues:y.array(zpt).describe("every defect the tests found"),nextSteps:y.array(y.string())})});function KV(t,e,r=[]){let{value:n,error:i}=Vt(e,r);return i?{errors:[i]}:Wr(t,n)}function Fd(t,e,r,n=[]){let i=KV(t,e,n);if(i.value)return i;let s=FC(r);return s===void 0?i:KV(t,s,n)}function sSe(t){let e=new Set(t.cases.map(n=>n.id)),r=t.requirements.flatMap(n=>n.clauses.filter(i=>i.steps?.trim()&&!i.skip&&!e.has(i.id)).map(i=>({id:i.id,type:i.type??"integration",target:i.target?.trim()||n.id,clause:i.id,...i.finding?{finding:i.finding}:{},steps:i.steps.trim(),expected:i.expected})));return r.length?{...t,cases:[...r,...t.cases]}:t}function oSe(t){let e=t.triage;if(e?.decision!=="runtime_required")return t;let r=new Set(e.plan.filter(s=>s.mechanism==="runtime").map(s=>s.behaviorId)),n=new Set([...e.plan,...e.outOfScope??[],...e.gaps].flatMap(s=>s.behaviorId?[s.behaviorId]:[]).filter(s=>!r.has(s))),i=t.cases.filter(s=>!s.clause||!n.has(s.clause));return i.length===t.cases.length?t:{...t,cases:i}}function SN(t,e){for(let r of e){if(!r)continue;let{value:n}=KV(t,r);if(n)return n}}function aSe(t){if(t)try{let e=JSON.parse(t);if(!e||typeof e!="object"||!("app"in e)||rSe.safeParse(e.app).success)return;let{app:r,...n}=e;return JSON.stringify(n)}catch{return}}function FC(t){try{return MC.readFileSync(t,"utf-8")}catch{return}}function LA(t,e){let r=new Set;return t.filter(n=>{let i=e(n).trim();return r.has(i)?!1:(r.add(i),!0)})}function ZV(t){return t?.outcome==="clean"||t?.outcome==="recipe-drift"}var cSe=t=>t.code?.trim()||t.requirement.evidence;function XV(t){return LA(t.requirements.flatMap(e=>e.clauses.map(r=>({id:r.id,evidence:cSe({...r,requirement:e})}))),e=>e.id)}function lSe(t,e,r=new Date){let n=LA(t.requirements,f=>f.id),i=LA(n.flatMap(f=>f.clauses.map(A=>({...A,requirement:f}))),f=>f.id),s=new Set(i.map(f=>f.id)),o=LA(t.findings,f=>f.id),a=new Set(o.map(f=>f.id)),c=LA(t.cases,f=>f.id),l=ZV(t.app),u=i.map(f=>({id:f.id,statement:f.clause,surface:c.find(A=>A.clause===f.id)?.target??f.requirement.id,evidence:cSe(f),satisfies:[f.requirement.id]})),d={schemaVersion:1,repository:e.repository,project:{type:"other",primaryLanguage:"unknown",languages:[],frameworks:[]},services:[],infrastructure:{containerized:!1,hasDockerCompose:!1,composeFiles:[],hasCiCd:!1},diff:{changedFiles:e.changedFiles,nonApplicationOnly:e.nonApplicationOnly,summary:t.summary.slice(0,400)},endpoints:{new:[],modified:[],removed:[]},existingTests:{locations:[]},entityMap:{entities:[],references:[],stateFields:[]},requirements:n.map(f=>({id:f.id,statement:f.statement,source:f.source,evidence:f.evidence})),behaviors:u,findings:o.map(f=>({id:f.id,file:f.file,line:f.line,claim:f.claim,severity:f.severity,evidence:f.evidence,status:"suspected",conflictsWith:f.conflictsWith})),unexamined:[],summary:t.summary},p={schemaVersion:1,mode:l?"full":"static",maintenance:LA(t.maintenance,f=>f.test).map(f=>({test:f.test,action:f.action,evidence:f.reason})),generate:c.map(f=>({id:f.id,type:f.type,target:f.target,from_behavior:f.clause&&s.has(f.clause)?f.clause:void 0,from_finding:f.finding&&a.has(f.finding)?f.finding:void 0,rationale:i.find(A=>A.id===f.clause)?.clause??o.find(A=>A.id===f.finding)?.claim??f.steps,expected_observable:f.expected,verifiable:l})),surfaces:[],budget:{total:c.length,rationale:"planned in one session from the requirements' clauses"},summary:t.summary};return{analyze:d,recommend:p,deploy:t.app?jpt(t.app,e.defaultTeardown,r):void 0}}function jpt(t,e,r){return{schemaVersion:1,baseUrls:t.baseUrls,health:t.health?{url:t.health.url,command:t.health.command,output:t.health.output??""}:void 0,teardown:t.teardown?.trim()||e||"true",dependencies:t.dependencies,outcome:t.outcome,recipeFixes:t.recipeFixes,bringupError:t.error?{command:"bring-up",output:t.error}:void 0,suspectFiles:t.suspectFiles,seeded:!1,startedAt:r.toISOString()}}var Vpt={added:"place",updated:"update",deleted:"delete",unchanged:"verify"};function Wpt(t){return t.result.exit!==null&&t.result.exit!==0||t.result.cases.some(e=>e.status==="fail")}function tSe(t){let[e,...r]=t.split(`
|
|
480
480
|
`);return[e.length<=Pd?e:`${e.slice(0,Pd-1).trimEnd()}\u2026`,...r].join(`
|
|
481
|
-
`)}function uSe(t,e,r=[]){let n=[],i=[],s=new Set,o=new Map([...r].reverse().map(
|
|
482
|
-
`)[0],body:t.commit.prBody},ready:
|
|
481
|
+
`)}function uSe(t,e,r=[]){let n=[],i=[],s=new Set,o=new Map([...r].reverse().map(A=>{let m=Rs.basename(Rs.resolve(A.path));return[m,{name:m,root:Rs.resolve(A.path),slug:A.slug}]})),a=new Map,c=A=>`${A.repository??""}\0${A.file}`;for(let A of t.tests){let m=e,w=Ft(e,A.file),C,b,S=A.file.replace(/^\.\//,"").split("/")[0],x=Rs.resolve(e,A.file),Q=[...o.values()].find(k=>x.startsWith(k.root+Rs.sep));if(o.has(S)&&!MC.existsSync(Rs.join(e,S))?({name:b,root:m,slug:C}=o.get(S),w=Ft(m,A.file.replace(/^\.\//,"").slice(S.length+1))):Q&&({name:b,root:m,slug:C}=Q,w=Ft(m,x)),!w||w.startsWith("..")||Rs.isAbsolute(w)){n.push(A.file);continue}if(A.change!=="deleted"&&!MC.existsSync(Rs.join(m,w))){n.push(A.file);continue}let P={...A,file:w,shown:b?`${b}/${w}`:w,...C?{repository:C}:{}};s.has(c(P))||(s.add(c(P)),a.set(c(P),m),i.push(P))}let l=i.map(A=>{let m=Vpt[A.change];return{...m==="place"?{from:A.shown}:{},file:A.file,...A.repository?{repository:A.repository}:{},action:m,covers:A.covers,...A.defect&&Wpt(A)?{finding:!0}:{},reran:{exit:A.result.exit,details:A.result.details,tests:A.result.cases},...m==="delete"?{}:{assertionCount:Gl(a.get(c(A)),A.file,Nd)}}}),u=A=>({schemaVersion:1,tests:i.filter(m=>m.kind===A&&m.change==="added").map(m=>({file:m.shown,source:"llm",language:Hf(m.file)??"unknown",covers:m.covers,assertions:m.assertions??[]})),deferredUi:[],summary:t.report.summary}),d=t.lint?.command?.trim()??"",p=t.lint?.clean??!0,f={schemaVersion:1,lint:[{tool:d.split(/\s+/)[0]??"",command:d,clean:p,uncovered:[]}],fileConventions:{ok:!0},ciReadiness:{covered:t.ciRunsTests??!0},dependencies:{added:t.dependenciesAdded??[]},commit:{message:tSe(t.commit.message)},pr:{title:tSe(t.commit.prTitle).split(`
|
|
482
|
+
`)[0],body:t.commit.prBody},ready:p,blockers:p?[]:[`the linter (${d||"the repo's linter"}) still reports problems in the test files`],summary:t.report.summary};return{adapt:{schemaVersion:1,items:l,ignored:[],summary:t.report.summary},generate:u("backend"),generateUi:u("ui"),finalize:f,dropped:n}}function Ypt(t,e,r,n=[]){let i=t.test;if(!i)return[];let s=e.tests.find(m=>m.file===i||i.endsWith(`/${m.file}`)||m.file.endsWith(`/${i}`));if(!s)return[];let o=new Map;for(let m of r.requirements){o.set(m.id,m);for(let w of m.clauses)o.set(w.id,m)}let a=new Set(s.result.cases.filter(m=>m.status==="fail").map(m=>m.case)),c=new Map;for(let m of s.covers.filter(w=>!a.size||a.has(w))){let w=r.cases.find(b=>b.id===m);if(!w)continue;let C=w.finding?r.findings.find(b=>b.id===w.finding):void 0;for(let b of[w.clause,...C?.conflictsWith??[]]){let S=b?o.get(b):void 0;S&&c.set(S.id,S)}}let l=n.filter(m=>m.trim()),u=m=>!l.length||l.some(w=>Kpt(m,w)),d=m=>{if(m.source==="spec")return!0;let w=dSe(m.evidence);return w!==void 0&&u(w)},p=[...c.values()].filter(d).slice(0,2),f=p.filter(m=>m.source!=="spec").map(m=>Jpt(m.evidence)),A=p.filter(m=>m.source==="spec").map(m=>m.evidence.trim());return[...f.length?[`the PR description, which says ${f.join(" and ")}`]:[],...A.length?[`the spec (${A.join("; ")})`]:[]]}function Jpt(t){let e=dSe(t);return e!==void 0?`"${e}"`:t.trim()}function Kpt(t,e){let r=hC(e),n=hC(t).split(/\.\.\.|\u2026/).map(i=>i.trim()).filter(i=>i.length>=3);return n.length>0&&n.every(i=>r.includes(i))}var Zpt=/["\u201c]([^"\u201d]{3,300})["\u201d]|(?:^|[\s:(\[])['\u2018]([^\n]{3,300}?)['\u2019](?=[\s.,;:)\]]|$)/;function dSe(t){let e=t.match(Zpt);return e?(e[1]??e[2]).trim():void 0}function pSe(t,e,r=[]){return{schemaVersion:1,businessCaseAnalysis:t.report.summary,issuesFound:t.report.issues.map(n=>({description:Xpt(n.test&&!n.description.includes(n.test)?`${n.description} (shown by ${n.test})`:n.description,e?Ypt(n,t,e,r):[]),severity:n.severity,category:"bug",sourceFile:n.sourceFile,sourceLine:n.sourceLine})),nextSteps:t.report.nextSteps}}function Xpt(t,e){let r=e.filter(n=>!t.includes(n.replace(/^the PR description, which says /,"")));return r.length?`${t}. This contradicts ${r.join(" and ")}.`.replace(/\.\. /,". "):t}function fSe(t,e){let r=GA.join(t.outputsDir,e);if(UA.existsSync(r))try{return UA.readFileSync(r,"utf-8")}catch{return}}function eft(t){let e=[],r=w=>{let C=fSe(t,w);if(C)try{return JSON.parse(C)}catch{return}},n=r("deploy.json");n?.outcome==="recipe-drift"?e.push("that the repo's recorded way of starting the app did not work as recorded \u2014 say what was wrong with it and what had to change, and that the scripts are now part of this PR's diff"):n?.outcome&&n.outcome!=="clean"&&e.push(`the deploy outcome "${n.outcome}" (in user terms: what it meant for the app and the tests)`);let i=Object.entries(n?.dependencies??{}).filter(([,w])=>w===!1).map(([w])=>w);i.length&&e.push(`each dependency that was NOT really connected this run, by name: ${i.join(", ")} \u2014 say what was not exercised because of it`);let o=r("adapt.json")?.items??[],a=o.filter(w=>w.finding&&w.file).map(w=>w.file);a.length&&e.push(`an issuesFound entry NAMING each test that caught a real bug: ${a.join(", ")}`);let c=o.filter(w=>w.placementBroken&&w.file).map(w=>w.file);c.length&&e.push(`a Skipped row + nextSteps item for each test that could not be placed/run: ${c.join(", ")}`);let l=o.filter(w=>w.repro&&w.file).map(w=>w.file);l.length&&e.push(`these tests are written to FAIL until the bug is fixed (their failure is the intended catch \u2014 say so in their details): ${l.join(", ")}`);let u=o.filter(w=>w.action!=="delete"&&w.file&&typeof w.reran?.exit=="number").map(w=>`${w.file} \u2192 ${w.reran.exit===0?"Pass":"Fail"}`);u.length&&e.push(`each executed test's row carries its real result (never Skipped): ${u.join("; ")}`);let d=o.filter(w=>w.action!=="delete"&&w.action!=="verify"&&w.file&&(w.reran?.exit===null||w.reran?.exit===void 0)).map(w=>w.file);d.length&&e.push(`these tests were NOT executed \u2014 their rows are Skipped (never Pass) and the report says they were not run against a live app: ${d.join(", ")}`);let f=(r("recommend.json")?.generate??[]).map(w=>w.id).filter(Boolean);f.length&&e.push(`every recommended case id appears as a newTestsCreated testId or is named in nextSteps/issuesFound with what happened to it: ${f.join(", ")}`),t.state.stages.finalize?.status==="failed"&&e.push("the tests could not be finalized (lint / commit-readiness) \u2014 say so in user terms"),t.state.stages.deploy?.status==="failed"&&e.push("the app could not be brought up \u2014 say so in user terms");let m=SA(t);return m.length&&e.push(`parts of the change analysis could not be verified (${m.length} unresolved check${m.length>1?"s":""}) \u2014 say so in user terms, and which requirements or findings that touches`),e}function tft(t){let e=Object.entries(t.state.stages).map(([a,c])=>` - ${a}: ${c.status}${c.reason?` (${c.reason})`:""}`),n=["analyze.json","deploy.json","recommend.json","generate.json","generate-ui.json","adapt.json","finalize.json"].map(a=>({n:a,body:fSe(t,a)})).filter(a=>a.body).map(a=>{let c=GA.join(t.outputsDir,a.n),l=a.body.length>6e3;return`### ${a.n}${l?` (truncated here \u2014 the complete file is ${c}; Read it before writing rows, the checks below read all of it)`:""}
|
|
483
483
|
\`\`\`json
|
|
484
484
|
${a.body.slice(0,6e3)}
|
|
485
485
|
\`\`\``}).join(`
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@skyramp/testbot-core",
|
|
3
|
-
"version": "0.13.0-v3.
|
|
3
|
+
"version": "0.13.0-v3.62",
|
|
4
4
|
"description": "Skyramp Testbot engine: AI-powered deterministic test generation, execution and maintenance",
|
|
5
5
|
"main": "dist/main.js",
|
|
6
6
|
"//files": "An explicit allowlist, never a glob. `dist/` is gitignored, so without this key npm's .gitignore fallback would publish a package with no engine in it; with a glob it would publish the sourcemaps, whose sourcesContent embeds every src/*.ts verbatim. eval-entry.js is a development entry and must not ship. dist/VERSION.json exists only in a release-engine.yml build. assets/gen-reference/ is the staged codegen reference corpus, which generate.ts finds at ../assets from dist/.",
|