omniqa 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/.tsbuildinfo +1 -1
- package/dist/cli.js +1 -1
- package/dist/commands/init.d.ts +8 -1
- package/dist/commands/init.d.ts.map +1 -1
- package/dist/commands/init.js +180 -22
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/knowledge.d.ts.map +1 -1
- package/dist/commands/knowledge.js +3 -2
- package/dist/commands/knowledge.js.map +1 -1
- package/dist/commands/plan.d.ts.map +1 -1
- package/dist/commands/plan.js +3 -2
- package/dist/commands/plan.js.map +1 -1
- package/dist/commands/status.js +2 -2
- package/dist/commands/status.js.map +1 -1
- package/dist/commands/sync.d.ts.map +1 -1
- package/dist/commands/sync.js +26 -14
- package/dist/commands/sync.js.map +1 -1
- package/dist/commands/test.d.ts +7 -1
- package/dist/commands/test.d.ts.map +1 -1
- package/dist/commands/test.js +60 -15
- package/dist/commands/test.js.map +1 -1
- package/dist/fixer.d.ts +1 -1
- package/dist/fixer.d.ts.map +1 -1
- package/dist/qa.js +3 -3
- package/dist/qa.js.map +1 -1
- package/dist/sync.d.ts +36 -9
- package/dist/sync.d.ts.map +1 -1
- package/dist/sync.js +185 -75
- package/dist/sync.js.map +1 -1
- package/package.json +6 -6
package/dist/sync.js
CHANGED
|
@@ -2,7 +2,7 @@ import { existsSync } from "node:fs";
|
|
|
2
2
|
import { copyFile, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
3
3
|
import { tmpdir } from "node:os";
|
|
4
4
|
import { join, relative, resolve } from "node:path";
|
|
5
|
-
import { budgeted, BudgetExceeded, changeSetLines, createModels, describeChangeSet, git, ingest, KNOWLEDGE_DIR, knowledgeCommit, localChangeSet, loadConfig, loadProduct, mapImpact, mergeEndpoints, openTranscript, parseEndpoints, parseStaticSections, planPath, readCrawl, readPlan, readState, README_FILE, readmeHasBlock, recordKnowledge, recordLastSeen, recordRun, recordSync, redactor, updateSync, removePendingChange, renderImpact, renderReview, repoDirsOf, repoNameOf, resolveEnvironmentUrl, RUNS_DIR, schemeCounts, secretValues, stampKnowledge, stampMaps, STATE_FILE, tierEnvironment, transcriptName, withReview, writeCoverageFile, writePlan } from "@omniqa/core";
|
|
5
|
+
import { budgeted, BudgetExceeded, changelogGitEnv, changeSetLines, createModels, describeChangeSet, git, ingest, KNOWLEDGE_DIR, knowledgeCommit, localChangeSet, loadConfig, loadProduct, mapImpact, mergeEndpoints, openTranscript, parseEndpoints, parseStaticSections, planPath, postText, readCrawl, readPlan, readState, README_FILE, readmeHasBlock, recordKnowledge, recordLastSeen, recordRun, recordSync, redactor, updateSync, removePendingChange, renderImpact, renderReview, repoDirsOf, repoNameOf, resolveEnvironmentUrl, RUNS_DIR, selectEnvironment, schemeCounts, secretValues, stampKnowledge, stampMaps, STATE_FILE, tierEnvironment, transcriptName, withReview, writeCoverageFile, writePlan } from "@omniqa/core";
|
|
6
6
|
import { crawlSite, createNetMap, launchBrowser, writeNetMap } from "@omniqa/web";
|
|
7
7
|
import { createGitHub } from "@omniqa/github";
|
|
8
8
|
import { createGitLab } from "@omniqa/gitlab";
|
|
@@ -33,12 +33,19 @@ export async function syncContext(cwd, env, o) {
|
|
|
33
33
|
const product = await loadProduct(cwd, env);
|
|
34
34
|
return { cwd, env, config, product, repoDirs: repoDirsOf(cwd, product), now: o.now ?? (() => new Date()), out: o.out, log: o.log };
|
|
35
35
|
}
|
|
36
|
-
/** The git host that answers for the product repository: the configured changelog host, else the one gitHosts names. */
|
|
36
|
+
/** The git host that answers for the product repository: the configured changelog host, else the one gitHosts names; it reads with changelog.tokenEnv when that is set (a token for a private product repository), else the host's own token. */
|
|
37
37
|
export function changeSource(ctx, host) {
|
|
38
38
|
const which = host ?? ctx.config.changelog?.host ?? (ctx.config.gitHosts.gitlab !== undefined && ctx.config.gitHosts.github === undefined ? "gitlab" : "github");
|
|
39
|
-
|
|
39
|
+
// the product token belongs to the changelog host only: a webhook from the other host keeps that host's own token
|
|
40
|
+
const tokenEnv = which === ctx.config.changelog?.host ? ctx.config.changelog.tokenEnv : undefined;
|
|
41
|
+
const withToken = (raw) => (tokenEnv === undefined ? raw : { ...raw, tokenEnv });
|
|
42
|
+
return which === "gitlab" ? createGitLab(withToken(ctx.config.gitHosts.gitlab ?? {}), ctx.env) : createGitHub(withToken(ctx.config.gitHosts.github ?? {}), ctx.env);
|
|
40
43
|
}
|
|
41
44
|
const knows = async (repoDir, sha) => git(repoDir, ["cat-file", "-e", `${sha}^{commit}`]).then(() => true, () => false);
|
|
45
|
+
/** The commit of `name` the knowledge base describes: `state.knowledge.commit` is `name sha[, name sha]` (knowledgeCommit). */
|
|
46
|
+
const knowledgeCommitOf = (state, name) => state.knowledge?.commit?.split(", ").map((part) => part.split(" ")).find(([n]) => n === name)?.[1];
|
|
47
|
+
/** Where a sync without --from starts: the last synced commit of the repository, else the commit its knowledge was built from. */
|
|
48
|
+
export const lastSynced = (state, name) => state.changelog?.lastSeen[name] ?? knowledgeCommitOf(state, name);
|
|
42
49
|
/** The directory of the repository named by a host path (`o/r`) or a name, when product.yaml has it. */
|
|
43
50
|
const localDir = (ctx, repo) => ctx.repoDirs[repoNameOf(repo)];
|
|
44
51
|
/** A change set for a webhook event: from the repository on disk when it has both commits, else from the host's compare. */
|
|
@@ -70,10 +77,15 @@ export async function resolveChangeSets(ctx, request, dryRun = false) {
|
|
|
70
77
|
let repo = request.repo ?? (names.length === 1 ? names[0] : configured);
|
|
71
78
|
if (repo === undefined)
|
|
72
79
|
throw new Error(names.length === 0 ? "no repository to compare: add repos: to product.yaml or changelog.repo to omniqa.config.ts" : `name the repository with --repo (${names.join(", ")})`);
|
|
80
|
+
const name = repoNameOf(repo);
|
|
81
|
+
// ponytail: a lastSeen ahead of --to (a poll recorded the head after a deploy) compares backwards; check ancestry with merge-base when it bites.
|
|
82
|
+
const from = request.from ?? lastSynced(await readState(ctx.cwd), name);
|
|
83
|
+
if (from === undefined)
|
|
84
|
+
throw new Error(`no commit to compare from for ${name}: nothing synced yet and no knowledge built from it; pass --from <sha>`);
|
|
73
85
|
const dir = localDir(ctx, repo);
|
|
74
86
|
if (dir !== undefined) {
|
|
75
87
|
const to = request.to ?? (await git(dir, ["rev-parse", "HEAD"]));
|
|
76
|
-
return [{ changeSet: await localChangeSet(dir,
|
|
88
|
+
return [{ changeSet: await localChangeSet(dir, name, from, to), repoDir: dir }];
|
|
77
89
|
}
|
|
78
90
|
if (configured !== undefined && (repo === configured || repoNameOf(configured) === repo))
|
|
79
91
|
repo = configured;
|
|
@@ -81,8 +93,8 @@ export async function resolveChangeSets(ctx, request, dryRun = false) {
|
|
|
81
93
|
throw new Error(`unknown repository ${JSON.stringify(repo)}: not a repos: path of product.yaml (${names.join(", ") || "none"}) nor changelog.repo (${configured ?? "none"})`);
|
|
82
94
|
const source = changeSource(ctx);
|
|
83
95
|
const to = request.to ?? (await source.branchHead(repo, ctx.config.changelog?.branch ?? "main")).sha;
|
|
84
|
-
const c = await source.compare(repo,
|
|
85
|
-
return [{ changeSet: { host: ctx.config.changelog?.host ?? source.name, repo, kind: "manual", before:
|
|
96
|
+
const c = await source.compare(repo, from, to);
|
|
97
|
+
return [{ changeSet: { host: ctx.config.changelog?.host ?? source.name, repo, kind: "manual", before: from, after: to, files: c.files, commits: c.commits, ...(c.url === undefined ? {} : { url: c.url }), ...(c.commits.at(-1) === undefined ? {} : { title: c.commits.at(-1).title }) } }];
|
|
86
98
|
}
|
|
87
99
|
if (request.kind === "poll") {
|
|
88
100
|
const changelog = ctx.config.changelog;
|
|
@@ -159,8 +171,8 @@ export function impactLine(impact, file, cwd) {
|
|
|
159
171
|
...(impact.cases.length === 0 ? [] : [` cases: ${impact.cases.map((c) => `${c.id} (${c.confidence})`).join(", ")}; exact ${count("exact")}, pattern ${count("pattern")}, guess ${count("guess")}`]),
|
|
160
172
|
];
|
|
161
173
|
}
|
|
162
|
-
/** The impacted knowledge rewritten: the changed repository's map (its overview kept), a crawl of the impacted pages as every actor with their endpoint rows merged, the cross-reference, the scheme's impacted statements revised by the analyst behind the reviewer, the index and the knowledge commit. */
|
|
163
|
-
export async function updateKnowledge(ctx, r, impact, run) {
|
|
174
|
+
/** The impacted knowledge rewritten: the changed repository's map (its overview kept), a crawl of the impacted pages as every actor with their endpoint rows merged, on `preferred` (the --env environment, M6) when it is given and explores, the cross-reference, the scheme's impacted statements revised by the analyst behind the reviewer, the index and the knowledge commit. */
|
|
175
|
+
export async function updateKnowledge(ctx, r, impact, run, preferred) {
|
|
164
176
|
const knowledgeDir = join(ctx.cwd, KNOWLEDGE_DIR);
|
|
165
177
|
const name = repoNameOf(r.changeSet.repo);
|
|
166
178
|
const mapFile = join(knowledgeDir, "repos", name, "map.md");
|
|
@@ -175,7 +187,8 @@ export async function updateKnowledge(ctx, r, impact, run) {
|
|
|
175
187
|
ctx.out(`static: ${r.changeSet.repo} is not a repos: source of product.yaml; its map was not rewritten`);
|
|
176
188
|
}
|
|
177
189
|
else {
|
|
178
|
-
const
|
|
190
|
+
const gitEnv = changelogGitEnv(ctx.config, ctx.env);
|
|
191
|
+
const result = await ingest({ source: isRemote(source) ? source : resolve(ctx.cwd, source), knowledgeDir, keepOverview: true, ...(gitEnv === undefined ? {} : { gitEnv }), log: (l) => ctx.log(`ingest: ${l}`) });
|
|
179
192
|
const parsed = result.static === undefined ? parseStaticSections(await readFile(result.file, "utf8")) : undefined;
|
|
180
193
|
ctx.out(staticLine(result.name, result.files, result.status, result.static, parsed, 0));
|
|
181
194
|
if (result.commit !== undefined && !r.changeSet.after.startsWith(result.commit) && !result.commit.startsWith(r.changeSet.after))
|
|
@@ -201,8 +214,11 @@ export async function updateKnowledge(ctx, r, impact, run) {
|
|
|
201
214
|
// a route with a placeholder is opened at the path the crawl once used for it
|
|
202
215
|
const targets = impact.pages.toSorted((a, b) => RANK[b.confidence] - RANK[a.confidence]).map((p) => known.find((k) => k.route === p.name.route)?.path ?? p.name.route).filter((p) => !/[{}]/.test(p)).slice(0, MAX_TARGET_PAGES);
|
|
203
216
|
const pages = [];
|
|
204
|
-
// The knowledge build's rule
|
|
205
|
-
const
|
|
217
|
+
// The crawl follows --env (M6) unless that is a production environment with explore off: the knowledge build's rule, a replay-only production is never crawled; then the dev tier, else a production environment that explores.
|
|
218
|
+
const fallback = run.dev ?? (run.production?.environment.explore === true ? run.production : undefined);
|
|
219
|
+
const crawlOn = preferred !== undefined && explores(preferred, ctx.env) ? preferred : fallback;
|
|
220
|
+
if (preferred !== undefined && !explores(preferred, ctx.env))
|
|
221
|
+
ctx.out(`crawl: ${preferred.name} is a production environment with explore off; ${fallback === undefined ? "nothing is crawled" : `crawling on ${fallback.name} instead`}`);
|
|
206
222
|
if (crawlOn === undefined)
|
|
207
223
|
ctx.out("crawl: skipped (no dev-tier environment, no production one with explore on)");
|
|
208
224
|
else if (targets.length === 0)
|
|
@@ -352,110 +368,155 @@ export function classifyRun(report) {
|
|
|
352
368
|
visit(report);
|
|
353
369
|
return out;
|
|
354
370
|
}
|
|
355
|
-
/**
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
371
|
+
/** A dev-tier environment, a production one with explore on, or one OMNIQA_ALLOW_EXPLORE names: the crawl and the fixer may act there. */
|
|
372
|
+
const explores = (w, env) => w.environment.tier !== "production" || w.environment.explore || env.OMNIQA_ALLOW_EXPLORE === w.name;
|
|
373
|
+
const FAILED = new Set(["failed", "behaviour-changed", "unsure"]);
|
|
374
|
+
/** The written cases a sync runs, grouped by environment: with `suite`, every written case of that environment's tier, on it; else the impacted written cases, the dev-tier ones on the dev environment, the production-only ones on the production one. Cases that cannot run land in `notRun`. */
|
|
375
|
+
function selectCases(plan, impact, run, suite, notRun) {
|
|
376
|
+
const groups = new Map();
|
|
377
|
+
const add = (where, s) => {
|
|
378
|
+
const g = groups.get(where.name) ?? { where, heal: where.environment.tier !== "production", selected: [] };
|
|
379
|
+
g.selected.push(s);
|
|
380
|
+
groups.set(where.name, g);
|
|
381
|
+
};
|
|
382
|
+
const all = plan.suites.flatMap((s) => s.cases.map((c) => ({ suite: s, case: c })));
|
|
383
|
+
if (suite !== undefined) {
|
|
384
|
+
for (const s of all)
|
|
385
|
+
if (s.case.status === "written" && s.case.file !== undefined && s.case.tiers.includes(suite.environment.tier))
|
|
386
|
+
add(suite, s);
|
|
387
|
+
return [...groups.values()];
|
|
363
388
|
}
|
|
364
|
-
const byId = new Map(
|
|
365
|
-
|
|
366
|
-
// filled as the stage goes, so a budget stop keeps what ran
|
|
367
|
-
const cases = into.cases;
|
|
368
|
-
const toRun = [];
|
|
369
|
-
const onProduction = [];
|
|
370
|
-
const vanished = (c) => c.routes.find((route) => k.removedRoutes.includes(route));
|
|
389
|
+
const byId = new Map(all.map((x) => [x.case.id, x]));
|
|
390
|
+
// ponytail: a case on a route the change removed runs too (removed routes are known only after the map is rewritten) and is marked skipped by the plan pass; the plan pass runs last, so a budget stop leaves it failed.
|
|
371
391
|
for (const i of impact.cases) {
|
|
372
392
|
const found = byId.get(i.id);
|
|
373
393
|
if (found === undefined)
|
|
374
394
|
continue;
|
|
375
395
|
const c = found.case;
|
|
376
396
|
const base = { id: c.id, title: c.title, ...(c.file === undefined ? {} : { file: c.file }) };
|
|
377
|
-
const gone = vanished(c);
|
|
378
397
|
if (c.status !== "written" || c.file === undefined)
|
|
379
|
-
|
|
380
|
-
else if (gone !== undefined)
|
|
381
|
-
cases.push({ ...base, outcome: "not-run", detail: `route ${gone} is no longer declared` });
|
|
398
|
+
notRun.push({ ...base, outcome: "not-run", detail: c.status });
|
|
382
399
|
else if (!c.tiers.includes("dev")) {
|
|
383
400
|
if (run.production === undefined)
|
|
384
|
-
|
|
401
|
+
notRun.push({ ...base, outcome: "not-run", detail: "production-only and no production-tier environment" });
|
|
385
402
|
else
|
|
386
|
-
|
|
403
|
+
add(run.production, found);
|
|
387
404
|
}
|
|
388
405
|
else if (run.dev === undefined)
|
|
389
|
-
|
|
406
|
+
notRun.push({ ...base, outcome: "not-run", detail: "no dev-tier environment" });
|
|
390
407
|
else
|
|
391
|
-
|
|
408
|
+
add(run.dev, found);
|
|
409
|
+
}
|
|
410
|
+
return [...groups.values()];
|
|
411
|
+
}
|
|
412
|
+
/** Pass 1: the selected cases run, one omniqa test run per environment; each ends pass, healed, behaviour changed, unsure, failed or not run. Resolves to the failures for the fixer. The runs' model calls are counted, not charged (M5). */
|
|
413
|
+
export async function runCases(ctx, impact, run, suite, into) {
|
|
414
|
+
const plan = await readPlan(planPath(join(ctx.cwd, ctx.config.testDir)));
|
|
415
|
+
if (plan === undefined) {
|
|
416
|
+
ctx.out("tests: skipped (no plan at tests/plan.md; run omniqa plan)");
|
|
417
|
+
return [];
|
|
392
418
|
}
|
|
419
|
+
const project = ctx.config.projects[0]?.name;
|
|
420
|
+
const cases = into.cases;
|
|
421
|
+
const groups = selectCases(plan, impact, run, suite, cases);
|
|
393
422
|
const notRun = cases.filter((c) => c.outcome === "not-run");
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
423
|
+
if (suite !== undefined) {
|
|
424
|
+
const g = groups[0];
|
|
425
|
+
ctx.out(`tests: suite on ${suite.name} (${project}): ${plural(g?.selected.length ?? 0, "written case")} of the ${suite.environment.tier} tier${g === undefined ? "" : `: ${ids(g.selected)}`}`);
|
|
426
|
+
}
|
|
427
|
+
else {
|
|
428
|
+
const dev = groups.find((g) => g.where.environment.tier !== "production");
|
|
429
|
+
const prod = groups.find((g) => g.where.environment.tier === "production");
|
|
430
|
+
ctx.out(`tests: ${plural(impact.cases.length, "impacted case")}; running ${dev?.selected.length ?? 0} written${run.dev === undefined ? "" : ` on ${run.dev.name} (${project})`}: ${dev === undefined ? "none" : ids(dev.selected)}${prod === undefined ? "" : `; ${prod.selected.length} production-only on ${prod.where.name}: ${ids(prod.selected)}`}${notRun.length === 0 ? "" : `; not run: ${notRun.map((c) => `${c.id} (${c.detail})`).join(", ")}`}`);
|
|
431
|
+
}
|
|
432
|
+
const failed = [];
|
|
433
|
+
for (const g of groups) {
|
|
434
|
+
const onProduction = g.where.environment.tier === "production";
|
|
435
|
+
if (onProduction)
|
|
436
|
+
run.ranOnProduction = true;
|
|
437
|
+
const reportName = onProduction ? "omniqa-sync-production" : "omniqa-sync";
|
|
438
|
+
const result = await runCase(ctx.cwd, ctx.env, { ids: g.selected.map((s) => s.case.id), envName: g.where.name, project, heal: g.heal, reportName, log: ctx.log });
|
|
439
|
+
// ponytail: a run that leaves no report (no browser, a runner crash) fails the sync loudly instead of passing as not run; per-group not-run outcomes are the upgrade when a second environment's results matter.
|
|
398
440
|
if (result.report === undefined)
|
|
399
441
|
throw new Error(result.error ?? "the run left no report");
|
|
400
|
-
run.
|
|
442
|
+
run.runCalls += result.summary?.modelCalls ?? 0;
|
|
401
443
|
if (result.summary !== undefined)
|
|
402
444
|
await recordRun(ctx.cwd, { ...result.summary, report: join("test-results", `${reportName}.json`) });
|
|
403
445
|
const classified = classifyRun(result.report);
|
|
404
|
-
const
|
|
405
|
-
for (const s of selected) {
|
|
446
|
+
for (const s of g.selected) {
|
|
406
447
|
const title = `${s.case.id}: ${s.case.title}`;
|
|
407
448
|
const got = classified.get(title) ?? { outcome: "failed", detail: `no test titled ${JSON.stringify(title)} ran`, calls: 0 };
|
|
408
449
|
cases.push({ id: s.case.id, title: s.case.title, file: s.case.file, outcome: got.outcome, ...(got.detail === undefined ? {} : { detail: got.detail }) });
|
|
409
450
|
ctx.out(` ${s.case.id}: ${OUTCOME_WORDS[got.outcome]}${got.detail === undefined ? "" : ` — ${got.detail}`}`);
|
|
410
|
-
if (got.outcome
|
|
411
|
-
|
|
451
|
+
if (FAILED.has(got.outcome))
|
|
452
|
+
failed.push({ selected: s, detail: got.detail ?? OUTCOME_WORDS[got.outcome], where: g.where });
|
|
412
453
|
}
|
|
413
454
|
for (const line of result.summary?.productionSteps ?? [])
|
|
414
455
|
ctx.out(` ${line}`);
|
|
415
|
-
return changed;
|
|
416
|
-
};
|
|
417
|
-
const changed = toRun.length > 0 && run.dev !== undefined ? await runOn(run.dev, toRun, true, "omniqa-sync") : [];
|
|
418
|
-
if (onProduction.length > 0 && run.production !== undefined) {
|
|
419
|
-
run.ranOnProduction = true;
|
|
420
|
-
// No --heal on production: a miss heals there only where its explore is on, and explore: false keeps it a replay.
|
|
421
|
-
// ponytail: a behaviour change found on production is reported, not handed to the fixer, which rewrites on the dev tier; a fixer on production is the upgrade.
|
|
422
|
-
await runOn(run.production, onProduction, false, "omniqa-sync-production");
|
|
423
456
|
}
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
457
|
+
return failed;
|
|
458
|
+
}
|
|
459
|
+
/** Pass 5: every failed case goes to the fixer on the environment it failed on, with the change set and the rewritten knowledge (M4); a production environment that does not explore gets no fixer (D5), a case on a route the change removed waits for the plan pass. */
|
|
460
|
+
export async function fixFailed(ctx, r, impact, k, run, failed, cases) {
|
|
461
|
+
const fixable = failed.filter((f) => !f.selected.case.routes.some((route) => k.removedRoutes.includes(route)));
|
|
462
|
+
if (fixable.length === 0)
|
|
463
|
+
return;
|
|
464
|
+
ctx.out(`fix: ${plural(fixable.length, "failed case")}: ${fixable.map((f) => f.selected.case.id).join(", ")}`);
|
|
465
|
+
const project = ctx.config.projects[0]?.name;
|
|
466
|
+
const knowledge = [`sections rewritten: ${impact.sections.map((x) => x.name).join(", ") || "none"}`, `pages re-crawled: ${k.pages.join(", ") || "none"}`, `scheme: ${k.scheme}`].join("\n");
|
|
467
|
+
const byEnv = new Map();
|
|
468
|
+
for (const f of fixable)
|
|
469
|
+
byEnv.set(f.where.name, [...(byEnv.get(f.where.name) ?? []), f]);
|
|
470
|
+
for (const group of byEnv.values()) {
|
|
471
|
+
const where = group[0].where;
|
|
472
|
+
const entry = (f) => cases.find((c) => c.id === f.selected.case.id);
|
|
473
|
+
if (!explores(where, ctx.env)) {
|
|
474
|
+
for (const f of group) {
|
|
475
|
+
entry(f).outcome = "failed";
|
|
476
|
+
entry(f).detail = `${f.detail}; no fixer: ${where.name} does not explore`;
|
|
477
|
+
ctx.out(` ${f.selected.case.id}: not fixed — ${where.name} is a production environment with explore off`);
|
|
478
|
+
}
|
|
479
|
+
continue;
|
|
480
|
+
}
|
|
481
|
+
const fixer = await createFixer({ cwd: ctx.cwd, env: ctx.env, config: ctx.config, product: ctx.product, envName: where.name, environment: where.environment, project, budget: run.budget, transcript: run.transcript, changeSet: r.changeSet, mappedFiles: impact.files.filter((f) => f.confidence !== undefined).map((f) => f.path), knowledge, out: ctx.out, log: ctx.log });
|
|
428
482
|
try {
|
|
429
|
-
for (const
|
|
430
|
-
const fixed = await fixer.fix(selected, detail);
|
|
431
|
-
const
|
|
432
|
-
|
|
483
|
+
for (const f of group) {
|
|
484
|
+
const fixed = await fixer.fix(f.selected, f.detail);
|
|
485
|
+
const e = entry(f);
|
|
486
|
+
e.outcome = fixed.status;
|
|
433
487
|
if (fixed.feedback !== undefined)
|
|
434
|
-
|
|
488
|
+
e.detail = fixed.feedback;
|
|
435
489
|
}
|
|
436
490
|
}
|
|
437
491
|
finally {
|
|
438
492
|
fixer.close();
|
|
439
493
|
}
|
|
440
494
|
}
|
|
441
|
-
|
|
495
|
+
}
|
|
496
|
+
/** Pass 6: cases on routes the change removed are proposed as skipped; added routes and handlers become planned cases through one planner call. */
|
|
497
|
+
export async function updatePlan(ctx, k, run, into) {
|
|
498
|
+
const planFile = planPath(join(ctx.cwd, ctx.config.testDir));
|
|
499
|
+
const current = await readPlan(planFile);
|
|
500
|
+
if (current === undefined)
|
|
501
|
+
return;
|
|
502
|
+
const cases = into.cases;
|
|
442
503
|
const planLines = [];
|
|
443
|
-
const current = (await readPlan(planFile)) ?? plan;
|
|
444
504
|
const skipped = [];
|
|
445
505
|
for (const c of current.suites.flatMap((s) => s.cases)) {
|
|
446
|
-
const gone =
|
|
506
|
+
const gone = c.routes.find((route) => k.removedRoutes.includes(route));
|
|
447
507
|
if (gone === undefined || c.status === "skipped")
|
|
448
508
|
continue;
|
|
449
509
|
c.status = "skipped";
|
|
450
510
|
skipped.push(c.id);
|
|
451
|
-
|
|
511
|
+
const detail = `route ${gone} is no longer declared`;
|
|
512
|
+
planLines.push(`${c.id} skipped (${detail})`);
|
|
452
513
|
const entry = cases.find((x) => x.id === c.id);
|
|
453
514
|
if (entry !== undefined) {
|
|
454
515
|
entry.outcome = "skipped";
|
|
455
|
-
entry.detail =
|
|
516
|
+
entry.detail = detail;
|
|
456
517
|
}
|
|
457
518
|
else
|
|
458
|
-
cases.push({ id: c.id, title: c.title, ...(c.file === undefined ? {} : { file: c.file }), outcome: "skipped", detail
|
|
519
|
+
cases.push({ id: c.id, title: c.title, ...(c.file === undefined ? {} : { file: c.file }), outcome: "skipped", detail });
|
|
459
520
|
}
|
|
460
521
|
if (skipped.length > 0)
|
|
461
522
|
await writePlan(planFile, current);
|
|
@@ -473,8 +534,6 @@ export async function runImpacted(ctx, r, impact, k, run, into) {
|
|
|
473
534
|
}
|
|
474
535
|
}
|
|
475
536
|
ctx.out(`plan: ${planLines.join("; ") || "unchanged"}`);
|
|
476
|
-
const order = new Map(impact.cases.map((c, i) => [c.id, i]));
|
|
477
|
-
cases.sort((a, b) => (order.get(a.id) ?? Infinity) - (order.get(b.id) ?? Infinity));
|
|
478
537
|
}
|
|
479
538
|
const count = (cases, outcome) => cases.filter((c) => c.outcome === outcome).length;
|
|
480
539
|
const cell = (text) => text.replace(/\|/g, "\\|").replace(/\s*\n\s*/g, " ").trim() || "none";
|
|
@@ -508,13 +567,34 @@ export function testsLine(o) {
|
|
|
508
567
|
return "tests: none";
|
|
509
568
|
return `tests: ran ${t.ran}: passed ${t.passed}, healed ${t.healed}, fixed ${t.fixed}, regressions ${t.regressions}, needs a human ${t.needsHuman}, failed ${t.failed}; skipped ${t.skipped}, planned ${o.summary?.planned ?? 0}; ${plural(o.modelCalls, "model call")}`;
|
|
510
569
|
}
|
|
511
|
-
|
|
570
|
+
const MAX_NOTIFY_LINES = 10;
|
|
571
|
+
/** The first notification: which tests failed where, after which deploy, and whether the healer runs. */
|
|
572
|
+
export function failuresText(changeSet, failed, ran, healer) {
|
|
573
|
+
const where = [...new Set(failed.map((f) => f.where.name))].join(", ");
|
|
574
|
+
const head = `omniqa: ${failed.length} of ${ran} tests failed on ${where} after ${changeSet.repo} ${short(changeSet.after)}${changeSet.title === undefined ? "" : ` (${changeSet.title})`}; ${healer}`;
|
|
575
|
+
const lines = failed.slice(0, MAX_NOTIFY_LINES).map((f) => ` ${f.selected.case.id}: ${f.selected.case.title} — ${f.detail}`);
|
|
576
|
+
const more = failed.length - lines.length;
|
|
577
|
+
return [head, ...lines, ...(more > 0 ? [` … and ${more} more (see the sync report)`] : [])].join("\n");
|
|
578
|
+
}
|
|
579
|
+
/** The second notification: what the healer made of the failures, what the knowledge update did, where it all went. */
|
|
580
|
+
export function outcomeText(o, failed) {
|
|
581
|
+
const s = o.summary;
|
|
582
|
+
const where = [...new Set(failed.map((f) => f.where.name))].join(", ");
|
|
583
|
+
const list = (outcomes) => {
|
|
584
|
+
const ids = o.cases.filter((c) => outcomes.includes(c.outcome)).map((c) => c.id);
|
|
585
|
+
return ids.length === 0 ? "" : ` (${ids.slice(0, MAX_NOTIFY_LINES).join(", ")}${ids.length > MAX_NOTIFY_LINES ? ", …" : ""})`;
|
|
586
|
+
};
|
|
587
|
+
const t = s.tests;
|
|
588
|
+
return `omniqa: healer on ${where} after ${s.repo} ${short(s.after)}: fixed ${t.fixed}${list(["fixed"])}, regression ${t.regressions}${list(["regression"])}, needs a human ${t.needsHuman}${list(["needs-human", "unsure"])}, still failing ${t.failed}${list(["failed", "behaviour-changed"])}; knowledge: ${plural(s.knowledge.maps, "map")}, ${plural(s.knowledge.pages, "page")}, scheme ${s.knowledge.scheme}; ${persistedText(s.persisted)}${s.stopped === undefined ? "" : `; stopped: ${s.stopped}`}`;
|
|
589
|
+
}
|
|
590
|
+
function summarize(o, at, budget, runCalls, suite) {
|
|
512
591
|
const ran = o.cases.filter((c) => !["not-run", "skipped"].includes(c.outcome));
|
|
513
592
|
return {
|
|
514
593
|
at,
|
|
515
594
|
host: o.changeSet.host,
|
|
516
595
|
repo: o.changeSet.repo,
|
|
517
596
|
kind: o.changeSet.kind,
|
|
597
|
+
...(suite === undefined ? {} : { suite: true, environment: suite.name }),
|
|
518
598
|
before: o.changeSet.before,
|
|
519
599
|
after: o.changeSet.after,
|
|
520
600
|
...(o.changeSet.title === undefined ? {} : { title: o.changeSet.title }),
|
|
@@ -526,12 +606,13 @@ function summarize(o, at, budget) {
|
|
|
526
606
|
tests: { ran: ran.length, passed: count(o.cases, "pass"), healed: count(o.cases, "healed"), fixed: count(o.cases, "fixed"), regressions: count(o.cases, "regression"), needsHuman: count(o.cases, "needs-human") + count(o.cases, "unsure"), skipped: count(o.cases, "skipped"), failed: count(o.cases, "failed") + count(o.cases, "behaviour-changed") },
|
|
527
607
|
planned: o.planned.length,
|
|
528
608
|
modelCalls: o.modelCalls,
|
|
609
|
+
...(runCalls === 0 ? {} : { runCalls }),
|
|
529
610
|
budget,
|
|
530
611
|
...(o.stopped === undefined ? {} : { stopped: o.stopped }),
|
|
531
612
|
persisted: { kind: "off" },
|
|
532
613
|
};
|
|
533
614
|
}
|
|
534
|
-
/** The pipeline for one request: resolve the change sets
|
|
615
|
+
/** The pipeline for one request: resolve the change sets; for each: the run, the impact, the knowledge update, the fixer, the plan and the persistence, in that order. */
|
|
535
616
|
export async function runSync(cwd, env, request, o) {
|
|
536
617
|
const ctx = await syncContext(cwd, env, o);
|
|
537
618
|
const dev = devEnvironment(ctx.product, o.env);
|
|
@@ -542,10 +623,27 @@ export async function runSync(cwd, env, request, o) {
|
|
|
542
623
|
const outcomes = [];
|
|
543
624
|
const lines = [];
|
|
544
625
|
const out = ctx.out;
|
|
626
|
+
/** One { text } post to notify.webhookEnv, redacted; a missing variable or a channel that is down is one line, never a failure. */
|
|
627
|
+
const notify = async (text) => {
|
|
628
|
+
const n = ctx.config.notify;
|
|
629
|
+
if (n === undefined)
|
|
630
|
+
return;
|
|
631
|
+
const url = env[n.webhookEnv];
|
|
632
|
+
if (!url) {
|
|
633
|
+
ctx.log(`notify: ${n.webhookEnv} is not set; nothing sent`);
|
|
634
|
+
return;
|
|
635
|
+
}
|
|
636
|
+
const error = await postText(url, redact(text));
|
|
637
|
+
ctx.out(error === undefined ? "notify: sent" : redact(`notify: not sent: ${error}`));
|
|
638
|
+
};
|
|
545
639
|
ctx.out = (line) => {
|
|
546
640
|
lines.push(line);
|
|
547
641
|
out(line);
|
|
548
642
|
};
|
|
643
|
+
const named = o.env === undefined ? undefined : selectEnvironment(ctx.product, o.env);
|
|
644
|
+
if (o.suite === true && named === undefined)
|
|
645
|
+
throw new Error("--suite needs --env <name>, the environment the deploy landed on");
|
|
646
|
+
const suite = o.suite === true ? named : undefined;
|
|
549
647
|
for (const r of await resolveChangeSets(ctx, request, o.dryRun === true)) {
|
|
550
648
|
lines.length = 0;
|
|
551
649
|
for (const line of describeLines(r.changeSet))
|
|
@@ -562,10 +660,17 @@ export async function runSync(cwd, env, request, o) {
|
|
|
562
660
|
const transcript = await openTranscript(join(cwd, RUNS_DIR), transcriptName("sync", ctx.now()), { redact });
|
|
563
661
|
await transcript.append({ kind: "sync", changeSet: { ...r.changeSet, files: r.changeSet.files.map((f) => ({ path: f.path, status: f.status })) }, impact: relative(cwd, file) });
|
|
564
662
|
const outcome = { changeSet: r.changeSet, impact, impactFile: file, cases: [], planned: [], modelCalls: 0 };
|
|
565
|
-
const run = { budget, transcript, redact, ...(dev === undefined ? {} : { dev }), ...(production === undefined ? {} : { production }) };
|
|
663
|
+
const run = { budget, transcript, redact, runCalls: 0, ...(dev === undefined ? {} : { dev }), ...(production === undefined ? {} : { production }) };
|
|
664
|
+
let failed = [];
|
|
566
665
|
try {
|
|
567
|
-
|
|
568
|
-
|
|
666
|
+
failed = await runCases(ctx, impact, run, suite, outcome);
|
|
667
|
+
if (failed.length > 0) {
|
|
668
|
+
const healer = failed.some((f) => explores(f.where, env)) ? "healer launched" : `no healer: ${[...new Set(failed.filter((f) => !explores(f.where, env)).map((f) => f.where.name))].join(", ")} does not explore`;
|
|
669
|
+
await notify(failuresText(r.changeSet, failed, outcome.cases.filter((c) => !["not-run", "skipped"].includes(c.outcome)).length, healer));
|
|
670
|
+
}
|
|
671
|
+
outcome.knowledge = await updateKnowledge(ctx, r, impact, run, named);
|
|
672
|
+
await fixFailed(ctx, r, impact, outcome.knowledge, run, failed, outcome.cases);
|
|
673
|
+
await updatePlan(ctx, outcome.knowledge, run, outcome);
|
|
569
674
|
if (r.eventId !== undefined)
|
|
570
675
|
await removePendingChange(cwd, r.eventId);
|
|
571
676
|
}
|
|
@@ -574,8 +679,10 @@ export async function runSync(cwd, env, request, o) {
|
|
|
574
679
|
throw err;
|
|
575
680
|
outcome.stopped = err.message;
|
|
576
681
|
}
|
|
577
|
-
|
|
578
|
-
outcome.
|
|
682
|
+
const order = new Map(impact.cases.map((c, i) => [c.id, i]));
|
|
683
|
+
outcome.cases.sort((a, b) => (order.get(a.id) ?? Infinity) - (order.get(b.id) ?? Infinity));
|
|
684
|
+
outcome.modelCalls = budget.used + run.runCalls;
|
|
685
|
+
outcome.summary = summarize(outcome, ctx.now().toISOString(), limit, run.runCalls, suite);
|
|
579
686
|
ctx.out(testsLine(outcome));
|
|
580
687
|
try {
|
|
581
688
|
ctx.out(`coverage: ${(await writeCoverageFile(cwd)).line}`);
|
|
@@ -600,6 +707,9 @@ export async function runSync(cwd, env, request, o) {
|
|
|
600
707
|
if (persisted.error !== undefined)
|
|
601
708
|
throw new Error(`persist failed: ${persisted.error}`);
|
|
602
709
|
}
|
|
710
|
+
// a persist error threw above: the CI job fails and says why, so the outcome message is skipped
|
|
711
|
+
if (failed.length > 0)
|
|
712
|
+
await notify(outcomeText(outcome, failed));
|
|
603
713
|
outcomes.push(outcome);
|
|
604
714
|
}
|
|
605
715
|
return outcomes;
|