omniqa 0.2.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/cli.js +1 -1
- package/dist/commands/doctor.d.ts +10 -0
- package/dist/commands/doctor.d.ts.map +1 -1
- package/dist/commands/doctor.js +83 -4
- package/dist/commands/doctor.js.map +1 -1
- package/dist/commands/init.d.ts +11 -1
- package/dist/commands/init.d.ts.map +1 -1
- package/dist/commands/init.js +189 -21
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/knowledge.d.ts.map +1 -1
- package/dist/commands/knowledge.js +3 -2
- package/dist/commands/knowledge.js.map +1 -1
- package/dist/commands/plan.d.ts.map +1 -1
- package/dist/commands/plan.js +3 -2
- package/dist/commands/plan.js.map +1 -1
- package/dist/commands/status.js +2 -2
- package/dist/commands/status.js.map +1 -1
- package/dist/commands/sync.d.ts.map +1 -1
- package/dist/commands/sync.js +32 -16
- package/dist/commands/sync.js.map +1 -1
- package/dist/deploy-setup.d.ts +36 -0
- package/dist/deploy-setup.d.ts.map +1 -0
- package/dist/deploy-setup.js +132 -0
- package/dist/deploy-setup.js.map +1 -0
- package/dist/fixer.d.ts +1 -1
- package/dist/fixer.d.ts.map +1 -1
- package/dist/qa.js +3 -3
- package/dist/qa.js.map +1 -1
- package/dist/sync.d.ts +44 -11
- package/dist/sync.d.ts.map +1 -1
- package/dist/sync.js +200 -82
- package/dist/sync.js.map +1 -1
- package/package.json +6 -6
package/dist/sync.js
CHANGED
|
@@ -2,7 +2,7 @@ import { existsSync } from "node:fs";
|
|
|
2
2
|
import { copyFile, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
3
3
|
import { tmpdir } from "node:os";
|
|
4
4
|
import { join, relative, resolve } from "node:path";
|
|
5
|
-
import { budgeted, BudgetExceeded, changeSetLines, createModels, describeChangeSet, git, ingest, KNOWLEDGE_DIR, knowledgeCommit, localChangeSet, loadConfig, loadProduct, mapImpact, mergeEndpoints, openTranscript, parseEndpoints, parseStaticSections, planPath, readCrawl, readPlan, readState, README_FILE, readmeHasBlock, recordKnowledge, recordLastSeen, recordRun, recordSync, redactor, updateSync, removePendingChange, renderImpact, renderReview, repoDirsOf, repoNameOf, resolveEnvironmentUrl, RUNS_DIR, schemeCounts, secretValues, stampKnowledge, stampMaps, STATE_FILE, tierEnvironment, transcriptName, withReview, writeCoverageFile, writePlan } from "@omniqa/core";
|
|
5
|
+
import { budgeted, BudgetExceeded, changelogGitEnv, changeSetLines, createModels, describeChangeSet, git, ingest, KNOWLEDGE_DIR, knowledgeCommit, localChangeSet, loadConfig, loadProduct, mapImpact, mergeEndpoints, openTranscript, parseEndpoints, parseRemote, parseStaticSections, planPath, postText, readCrawl, readPlan, readState, README_FILE, readmeHasBlock, recordKnowledge, recordLastSeen, recordRun, recordSync, redactor, updateSync, removePendingChange, renderImpact, renderReview, repoDirsOf, repoNameOf, resolveEnvironmentUrl, RUNS_DIR, selectEnvironment, schemeCounts, secretValues, stampKnowledge, stampMaps, STATE_FILE, tierEnvironment, transcriptName, withReview, writeCoverageFile, writePlan } from "@omniqa/core";
|
|
6
6
|
import { crawlSite, createNetMap, launchBrowser, writeNetMap } from "@omniqa/web";
|
|
7
7
|
import { createGitHub } from "@omniqa/github";
|
|
8
8
|
import { createGitLab } from "@omniqa/gitlab";
|
|
@@ -33,12 +33,19 @@ export async function syncContext(cwd, env, o) {
|
|
|
33
33
|
const product = await loadProduct(cwd, env);
|
|
34
34
|
return { cwd, env, config, product, repoDirs: repoDirsOf(cwd, product), now: o.now ?? (() => new Date()), out: o.out, log: o.log };
|
|
35
35
|
}
|
|
36
|
-
/** The git host that answers for the product repository: the configured changelog host, else the one gitHosts names. */
|
|
36
|
+
/** The git host that answers for the product repository: the configured changelog host, else the one gitHosts names; it reads with the token changelog.tokenEnv names when that variable is set (a private product repository), else the host's own token. */
|
|
37
37
|
export function changeSource(ctx, host) {
|
|
38
38
|
const which = host ?? ctx.config.changelog?.host ?? (ctx.config.gitHosts.gitlab !== undefined && ctx.config.gitHosts.github === undefined ? "gitlab" : "github");
|
|
39
|
-
|
|
39
|
+
// the product token belongs to the changelog host only: a webhook from the other host keeps that host's own token, and so does the changelog host while the product token is unset
|
|
40
|
+
const tokenEnv = which === ctx.config.changelog?.host && ctx.env[ctx.config.changelog.tokenEnv] ? ctx.config.changelog.tokenEnv : undefined;
|
|
41
|
+
const withToken = (raw) => (tokenEnv === undefined ? raw : { ...raw, tokenEnv });
|
|
42
|
+
return which === "gitlab" ? createGitLab(withToken(ctx.config.gitHosts.gitlab ?? {}), ctx.env) : createGitHub(withToken(ctx.config.gitHosts.github ?? {}), ctx.env);
|
|
40
43
|
}
|
|
41
44
|
const knows = async (repoDir, sha) => git(repoDir, ["cat-file", "-e", `${sha}^{commit}`]).then(() => true, () => false);
|
|
45
|
+
/** The commit of `name` the knowledge base describes: `state.knowledge.commit` is `name sha[, name sha]` (knowledgeCommit). */
|
|
46
|
+
const knowledgeCommitOf = (state, name) => state.knowledge?.commit?.split(", ").map((part) => part.split(" ")).find(([n]) => n === name)?.[1];
|
|
47
|
+
/** Where a sync without --from starts: the last synced commit of the repository, else the commit its knowledge was built from. */
|
|
48
|
+
export const lastSynced = (state, name) => state.changelog?.lastSeen[name] ?? knowledgeCommitOf(state, name);
|
|
42
49
|
/** The directory of the repository named by a host path (`o/r`) or a name, when product.yaml has it. */
|
|
43
50
|
const localDir = (ctx, repo) => ctx.repoDirs[repoNameOf(repo)];
|
|
44
51
|
/** A change set for a webhook event: from the repository on disk when it has both commits, else from the host's compare. */
|
|
@@ -62,27 +69,43 @@ async function fromEvent(ctx, event) {
|
|
|
62
69
|
return { changeSet: { host: event.host, repo: event.repo, before, after, ...compact(base), files: c.files, commits: c.commits, ...(c.url === undefined ? {} : { url: c.url }) }, ...(dir === undefined ? {} : { repoDir: dir }) };
|
|
63
70
|
}
|
|
64
71
|
const compact = (o) => Object.fromEntries(Object.entries(o).filter(([, v]) => v !== undefined));
|
|
65
|
-
/**
|
|
72
|
+
/**
|
|
73
|
+
* The change sets a request asks for; `--poll` and the first poll are explained on `out`. A manual request names a repository of
|
|
74
|
+
* product.yaml (the only one, or `--repo`) or `changelog.repo`: a `repos:` path is read with git on disk (no token; `--to` is its
|
|
75
|
+
* HEAD); a `repos:` git url or `changelog.repo` goes through the host as a webhook does (`--to` is the head of `changelog.branch`),
|
|
76
|
+
* since the url's clone under .omniqa/repos is gitignored, so stale by hand and absent in a fresh CI checkout: it answers only when
|
|
77
|
+
* it has both commits, and the knowledge pass clones or pulls it afterwards.
|
|
78
|
+
*/
|
|
66
79
|
export async function resolveChangeSets(ctx, request, dryRun = false) {
|
|
67
80
|
if (request.kind === "manual") {
|
|
68
81
|
const names = Object.keys(ctx.repoDirs);
|
|
69
82
|
const configured = ctx.config.changelog?.repo;
|
|
70
|
-
|
|
83
|
+
const repo = request.repo ?? (names.length === 1 ? names[0] : configured);
|
|
71
84
|
if (repo === undefined)
|
|
72
85
|
throw new Error(names.length === 0 ? "no repository to compare: add repos: to product.yaml or changelog.repo to omniqa.config.ts" : `name the repository with --repo (${names.join(", ")})`);
|
|
86
|
+
const name = repoNameOf(repo);
|
|
87
|
+
// ponytail: a lastSeen ahead of --to (a poll recorded the head after a deploy) compares backwards; check ancestry with merge-base when it bites.
|
|
88
|
+
const from = request.from ?? lastSynced(await readState(ctx.cwd), name);
|
|
89
|
+
if (from === undefined)
|
|
90
|
+
throw new Error(`no commit to compare from for ${name}: nothing synced yet and no knowledge built from it; pass --from <sha>`);
|
|
73
91
|
const dir = localDir(ctx, repo);
|
|
74
|
-
|
|
92
|
+
const entry = (ctx.product.repos ?? []).find((x) => repoNameOf(x) === name);
|
|
93
|
+
if (dir !== undefined && entry !== undefined && !isRemote(entry)) {
|
|
75
94
|
const to = request.to ?? (await git(dir, ["rev-parse", "HEAD"]));
|
|
76
|
-
return [{ changeSet: await localChangeSet(dir,
|
|
95
|
+
return [{ changeSet: await localChangeSet(dir, name, from, to), repoDir: dir }];
|
|
77
96
|
}
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
else
|
|
97
|
+
const hostRepo = configured !== undefined && (repo === configured || repoNameOf(configured) === repo) ? configured : entry === undefined ? undefined : parseRemote(entry)?.path;
|
|
98
|
+
if (hostRepo === undefined)
|
|
81
99
|
throw new Error(`unknown repository ${JSON.stringify(repo)}: not a repos: path of product.yaml (${names.join(", ") || "none"}) nor changelog.repo (${configured ?? "none"})`);
|
|
82
100
|
const source = changeSource(ctx);
|
|
83
|
-
const
|
|
84
|
-
const
|
|
85
|
-
|
|
101
|
+
const host = ctx.config.changelog?.host ?? source.name;
|
|
102
|
+
const to = request.to ?? (await source.branchHead(hostRepo, ctx.config.changelog?.branch ?? "main")).sha;
|
|
103
|
+
// the change set is named as product.yaml names the repository (changelog.repo alone keeps its host path); the clone answers when it has both commits, else the host's compare (the token)
|
|
104
|
+
const named = entry === undefined ? hostRepo : name;
|
|
105
|
+
if (dir !== undefined && (await knows(dir, from)) && (await knows(dir, to)))
|
|
106
|
+
return [{ changeSet: { ...(await localChangeSet(dir, named, from, to)), host }, repoDir: dir }];
|
|
107
|
+
const c = await source.compare(hostRepo, from, to);
|
|
108
|
+
return [{ changeSet: { host, repo: named, kind: "manual", before: from, after: to, files: c.files, commits: c.commits, ...(c.url === undefined ? {} : { url: c.url }), ...(c.commits.at(-1) === undefined ? {} : { title: c.commits.at(-1).title }) }, ...(dir === undefined ? {} : { repoDir: dir }) }];
|
|
86
109
|
}
|
|
87
110
|
if (request.kind === "poll") {
|
|
88
111
|
const changelog = ctx.config.changelog;
|
|
@@ -159,8 +182,8 @@ export function impactLine(impact, file, cwd) {
|
|
|
159
182
|
...(impact.cases.length === 0 ? [] : [` cases: ${impact.cases.map((c) => `${c.id} (${c.confidence})`).join(", ")}; exact ${count("exact")}, pattern ${count("pattern")}, guess ${count("guess")}`]),
|
|
160
183
|
];
|
|
161
184
|
}
|
|
162
|
-
/** The impacted knowledge rewritten: the changed repository's map (its overview kept), a crawl of the impacted pages as every actor with their endpoint rows merged, the cross-reference, the scheme's impacted statements revised by the analyst behind the reviewer, the index and the knowledge commit. */
|
|
163
|
-
export async function updateKnowledge(ctx, r, impact, run) {
|
|
185
|
+
/** The impacted knowledge rewritten: the changed repository's map (its overview kept), a crawl of the impacted pages as every actor with their endpoint rows merged, on `preferred` (the --env environment, M6) when it is given and explores, the cross-reference, the scheme's impacted statements revised by the analyst behind the reviewer, the index and the knowledge commit. */
|
|
186
|
+
export async function updateKnowledge(ctx, r, impact, run, preferred) {
|
|
164
187
|
const knowledgeDir = join(ctx.cwd, KNOWLEDGE_DIR);
|
|
165
188
|
const name = repoNameOf(r.changeSet.repo);
|
|
166
189
|
const mapFile = join(knowledgeDir, "repos", name, "map.md");
|
|
@@ -175,7 +198,8 @@ export async function updateKnowledge(ctx, r, impact, run) {
|
|
|
175
198
|
ctx.out(`static: ${r.changeSet.repo} is not a repos: source of product.yaml; its map was not rewritten`);
|
|
176
199
|
}
|
|
177
200
|
else {
|
|
178
|
-
const
|
|
201
|
+
const gitEnv = changelogGitEnv(ctx.config, ctx.env);
|
|
202
|
+
const result = await ingest({ source: isRemote(source) ? source : resolve(ctx.cwd, source), knowledgeDir, keepOverview: true, ...(gitEnv === undefined ? {} : { gitEnv }), log: (l) => ctx.log(`ingest: ${l}`) });
|
|
179
203
|
const parsed = result.static === undefined ? parseStaticSections(await readFile(result.file, "utf8")) : undefined;
|
|
180
204
|
ctx.out(staticLine(result.name, result.files, result.status, result.static, parsed, 0));
|
|
181
205
|
if (result.commit !== undefined && !r.changeSet.after.startsWith(result.commit) && !result.commit.startsWith(r.changeSet.after))
|
|
@@ -201,8 +225,11 @@ export async function updateKnowledge(ctx, r, impact, run) {
|
|
|
201
225
|
// a route with a placeholder is opened at the path the crawl once used for it
|
|
202
226
|
const targets = impact.pages.toSorted((a, b) => RANK[b.confidence] - RANK[a.confidence]).map((p) => known.find((k) => k.route === p.name.route)?.path ?? p.name.route).filter((p) => !/[{}]/.test(p)).slice(0, MAX_TARGET_PAGES);
|
|
203
227
|
const pages = [];
|
|
204
|
-
// The knowledge build's rule
|
|
205
|
-
const
|
|
228
|
+
// The crawl follows --env (M6) unless that is a production environment with explore off: the knowledge build's rule, a replay-only production is never crawled; then the dev tier, else a production environment that explores.
|
|
229
|
+
const fallback = run.dev ?? (run.production?.environment.explore === true ? run.production : undefined);
|
|
230
|
+
const crawlOn = preferred !== undefined && explores(preferred, ctx.env) ? preferred : fallback;
|
|
231
|
+
if (preferred !== undefined && !explores(preferred, ctx.env))
|
|
232
|
+
ctx.out(`crawl: ${preferred.name} is a production environment with explore off; ${fallback === undefined ? "nothing is crawled" : `crawling on ${fallback.name} instead`}`);
|
|
206
233
|
if (crawlOn === undefined)
|
|
207
234
|
ctx.out("crawl: skipped (no dev-tier environment, no production one with explore on)");
|
|
208
235
|
else if (targets.length === 0)
|
|
@@ -352,110 +379,155 @@ export function classifyRun(report) {
|
|
|
352
379
|
visit(report);
|
|
353
380
|
return out;
|
|
354
381
|
}
|
|
355
|
-
/**
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
382
|
+
/** A dev-tier environment, a production one with explore on, or one OMNIQA_ALLOW_EXPLORE names: the crawl and the fixer may act there. */
|
|
383
|
+
const explores = (w, env) => w.environment.tier !== "production" || w.environment.explore || env.OMNIQA_ALLOW_EXPLORE === w.name;
|
|
384
|
+
const FAILED = new Set(["failed", "behaviour-changed", "unsure"]);
|
|
385
|
+
/** The written cases a sync runs, grouped by environment: with `suite`, every written case of that environment's tier, on it; else the impacted written cases, the dev-tier ones on the dev environment, the production-only ones on the production one. Cases that cannot run land in `notRun`. */
|
|
386
|
+
function selectCases(plan, impact, run, suite, notRun) {
|
|
387
|
+
const groups = new Map();
|
|
388
|
+
const add = (where, s) => {
|
|
389
|
+
const g = groups.get(where.name) ?? { where, heal: where.environment.tier !== "production", selected: [] };
|
|
390
|
+
g.selected.push(s);
|
|
391
|
+
groups.set(where.name, g);
|
|
392
|
+
};
|
|
393
|
+
const all = plan.suites.flatMap((s) => s.cases.map((c) => ({ suite: s, case: c })));
|
|
394
|
+
if (suite !== undefined) {
|
|
395
|
+
for (const s of all)
|
|
396
|
+
if (s.case.status === "written" && s.case.file !== undefined && s.case.tiers.includes(suite.environment.tier))
|
|
397
|
+
add(suite, s);
|
|
398
|
+
return [...groups.values()];
|
|
363
399
|
}
|
|
364
|
-
const byId = new Map(
|
|
365
|
-
|
|
366
|
-
// filled as the stage goes, so a budget stop keeps what ran
|
|
367
|
-
const cases = into.cases;
|
|
368
|
-
const toRun = [];
|
|
369
|
-
const onProduction = [];
|
|
370
|
-
const vanished = (c) => c.routes.find((route) => k.removedRoutes.includes(route));
|
|
400
|
+
const byId = new Map(all.map((x) => [x.case.id, x]));
|
|
401
|
+
// ponytail: a case on a route the change removed runs too (removed routes are known only after the map is rewritten) and is marked skipped by the plan pass; the plan pass runs last, so a budget stop leaves it failed.
|
|
371
402
|
for (const i of impact.cases) {
|
|
372
403
|
const found = byId.get(i.id);
|
|
373
404
|
if (found === undefined)
|
|
374
405
|
continue;
|
|
375
406
|
const c = found.case;
|
|
376
407
|
const base = { id: c.id, title: c.title, ...(c.file === undefined ? {} : { file: c.file }) };
|
|
377
|
-
const gone = vanished(c);
|
|
378
408
|
if (c.status !== "written" || c.file === undefined)
|
|
379
|
-
|
|
380
|
-
else if (gone !== undefined)
|
|
381
|
-
cases.push({ ...base, outcome: "not-run", detail: `route ${gone} is no longer declared` });
|
|
409
|
+
notRun.push({ ...base, outcome: "not-run", detail: c.status });
|
|
382
410
|
else if (!c.tiers.includes("dev")) {
|
|
383
411
|
if (run.production === undefined)
|
|
384
|
-
|
|
412
|
+
notRun.push({ ...base, outcome: "not-run", detail: "production-only and no production-tier environment" });
|
|
385
413
|
else
|
|
386
|
-
|
|
414
|
+
add(run.production, found);
|
|
387
415
|
}
|
|
388
416
|
else if (run.dev === undefined)
|
|
389
|
-
|
|
417
|
+
notRun.push({ ...base, outcome: "not-run", detail: "no dev-tier environment" });
|
|
390
418
|
else
|
|
391
|
-
|
|
419
|
+
add(run.dev, found);
|
|
392
420
|
}
|
|
421
|
+
return [...groups.values()];
|
|
422
|
+
}
|
|
423
|
+
/** Pass 1: the selected cases run, one omniqa test run per environment; each ends pass, healed, behaviour changed, unsure, failed or not run. Resolves to the failures for the fixer. The runs' model calls are counted, not charged (M5). */
|
|
424
|
+
export async function runCases(ctx, impact, run, suite, into) {
|
|
425
|
+
const plan = await readPlan(planPath(join(ctx.cwd, ctx.config.testDir)));
|
|
426
|
+
if (plan === undefined) {
|
|
427
|
+
ctx.out("tests: skipped (no plan at tests/plan.md; run omniqa plan)");
|
|
428
|
+
return [];
|
|
429
|
+
}
|
|
430
|
+
const project = ctx.config.projects[0]?.name;
|
|
431
|
+
const cases = into.cases;
|
|
432
|
+
const groups = selectCases(plan, impact, run, suite, cases);
|
|
393
433
|
const notRun = cases.filter((c) => c.outcome === "not-run");
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
434
|
+
if (suite !== undefined) {
|
|
435
|
+
const g = groups[0];
|
|
436
|
+
ctx.out(`tests: suite on ${suite.name} (${project}): ${plural(g?.selected.length ?? 0, "written case")} of the ${suite.environment.tier} tier${g === undefined ? "" : `: ${ids(g.selected)}`}`);
|
|
437
|
+
}
|
|
438
|
+
else {
|
|
439
|
+
const dev = groups.find((g) => g.where.environment.tier !== "production");
|
|
440
|
+
const prod = groups.find((g) => g.where.environment.tier === "production");
|
|
441
|
+
ctx.out(`tests: ${plural(impact.cases.length, "impacted case")}; running ${dev?.selected.length ?? 0} written${run.dev === undefined ? "" : ` on ${run.dev.name} (${project})`}: ${dev === undefined ? "none" : ids(dev.selected)}${prod === undefined ? "" : `; ${prod.selected.length} production-only on ${prod.where.name}: ${ids(prod.selected)}`}${notRun.length === 0 ? "" : `; not run: ${notRun.map((c) => `${c.id} (${c.detail})`).join(", ")}`}`);
|
|
442
|
+
}
|
|
443
|
+
const failed = [];
|
|
444
|
+
for (const g of groups) {
|
|
445
|
+
const onProduction = g.where.environment.tier === "production";
|
|
446
|
+
if (onProduction)
|
|
447
|
+
run.ranOnProduction = true;
|
|
448
|
+
const reportName = onProduction ? "omniqa-sync-production" : "omniqa-sync";
|
|
449
|
+
const result = await runCase(ctx.cwd, ctx.env, { ids: g.selected.map((s) => s.case.id), envName: g.where.name, project, heal: g.heal, reportName, log: ctx.log });
|
|
450
|
+
// ponytail: a run that leaves no report (no browser, a runner crash) fails the sync loudly instead of passing as not run; per-group not-run outcomes are the upgrade when a second environment's results matter.
|
|
398
451
|
if (result.report === undefined)
|
|
399
452
|
throw new Error(result.error ?? "the run left no report");
|
|
400
|
-
run.
|
|
453
|
+
run.runCalls += result.summary?.modelCalls ?? 0;
|
|
401
454
|
if (result.summary !== undefined)
|
|
402
455
|
await recordRun(ctx.cwd, { ...result.summary, report: join("test-results", `${reportName}.json`) });
|
|
403
456
|
const classified = classifyRun(result.report);
|
|
404
|
-
const
|
|
405
|
-
for (const s of selected) {
|
|
457
|
+
for (const s of g.selected) {
|
|
406
458
|
const title = `${s.case.id}: ${s.case.title}`;
|
|
407
459
|
const got = classified.get(title) ?? { outcome: "failed", detail: `no test titled ${JSON.stringify(title)} ran`, calls: 0 };
|
|
408
460
|
cases.push({ id: s.case.id, title: s.case.title, file: s.case.file, outcome: got.outcome, ...(got.detail === undefined ? {} : { detail: got.detail }) });
|
|
409
461
|
ctx.out(` ${s.case.id}: ${OUTCOME_WORDS[got.outcome]}${got.detail === undefined ? "" : ` — ${got.detail}`}`);
|
|
410
|
-
if (got.outcome
|
|
411
|
-
|
|
462
|
+
if (FAILED.has(got.outcome))
|
|
463
|
+
failed.push({ selected: s, detail: got.detail ?? OUTCOME_WORDS[got.outcome], where: g.where });
|
|
412
464
|
}
|
|
413
465
|
for (const line of result.summary?.productionSteps ?? [])
|
|
414
466
|
ctx.out(` ${line}`);
|
|
415
|
-
return changed;
|
|
416
|
-
};
|
|
417
|
-
const changed = toRun.length > 0 && run.dev !== undefined ? await runOn(run.dev, toRun, true, "omniqa-sync") : [];
|
|
418
|
-
if (onProduction.length > 0 && run.production !== undefined) {
|
|
419
|
-
run.ranOnProduction = true;
|
|
420
|
-
// No --heal on production: a miss heals there only where its explore is on, and explore: false keeps it a replay.
|
|
421
|
-
// ponytail: a behaviour change found on production is reported, not handed to the fixer, which rewrites on the dev tier; a fixer on production is the upgrade.
|
|
422
|
-
await runOn(run.production, onProduction, false, "omniqa-sync-production");
|
|
423
467
|
}
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
468
|
+
return failed;
|
|
469
|
+
}
|
|
470
|
+
/** Pass 5: every failed case goes to the fixer on the environment it failed on, with the change set and the rewritten knowledge (M4); a production environment that does not explore gets no fixer (D5), a case on a route the change removed waits for the plan pass. */
|
|
471
|
+
export async function fixFailed(ctx, r, impact, k, run, failed, cases) {
|
|
472
|
+
const fixable = failed.filter((f) => !f.selected.case.routes.some((route) => k.removedRoutes.includes(route)));
|
|
473
|
+
if (fixable.length === 0)
|
|
474
|
+
return;
|
|
475
|
+
ctx.out(`fix: ${plural(fixable.length, "failed case")}: ${fixable.map((f) => f.selected.case.id).join(", ")}`);
|
|
476
|
+
const project = ctx.config.projects[0]?.name;
|
|
477
|
+
const knowledge = [`sections rewritten: ${impact.sections.map((x) => x.name).join(", ") || "none"}`, `pages re-crawled: ${k.pages.join(", ") || "none"}`, `scheme: ${k.scheme}`].join("\n");
|
|
478
|
+
const byEnv = new Map();
|
|
479
|
+
for (const f of fixable)
|
|
480
|
+
byEnv.set(f.where.name, [...(byEnv.get(f.where.name) ?? []), f]);
|
|
481
|
+
for (const group of byEnv.values()) {
|
|
482
|
+
const where = group[0].where;
|
|
483
|
+
const entry = (f) => cases.find((c) => c.id === f.selected.case.id);
|
|
484
|
+
if (!explores(where, ctx.env)) {
|
|
485
|
+
for (const f of group) {
|
|
486
|
+
entry(f).outcome = "failed";
|
|
487
|
+
entry(f).detail = `${f.detail}; no fixer: ${where.name} does not explore`;
|
|
488
|
+
ctx.out(` ${f.selected.case.id}: not fixed — ${where.name} is a production environment with explore off`);
|
|
489
|
+
}
|
|
490
|
+
continue;
|
|
491
|
+
}
|
|
492
|
+
const fixer = await createFixer({ cwd: ctx.cwd, env: ctx.env, config: ctx.config, product: ctx.product, envName: where.name, environment: where.environment, project, budget: run.budget, transcript: run.transcript, changeSet: r.changeSet, mappedFiles: impact.files.filter((f) => f.confidence !== undefined).map((f) => f.path), knowledge, out: ctx.out, log: ctx.log });
|
|
428
493
|
try {
|
|
429
|
-
for (const
|
|
430
|
-
const fixed = await fixer.fix(selected, detail);
|
|
431
|
-
const
|
|
432
|
-
|
|
494
|
+
for (const f of group) {
|
|
495
|
+
const fixed = await fixer.fix(f.selected, f.detail);
|
|
496
|
+
const e = entry(f);
|
|
497
|
+
e.outcome = fixed.status;
|
|
433
498
|
if (fixed.feedback !== undefined)
|
|
434
|
-
|
|
499
|
+
e.detail = fixed.feedback;
|
|
435
500
|
}
|
|
436
501
|
}
|
|
437
502
|
finally {
|
|
438
503
|
fixer.close();
|
|
439
504
|
}
|
|
440
505
|
}
|
|
441
|
-
|
|
506
|
+
}
|
|
507
|
+
/** Pass 6: cases on routes the change removed are proposed as skipped; added routes and handlers become planned cases through one planner call. */
|
|
508
|
+
export async function updatePlan(ctx, k, run, into) {
|
|
509
|
+
const planFile = planPath(join(ctx.cwd, ctx.config.testDir));
|
|
510
|
+
const current = await readPlan(planFile);
|
|
511
|
+
if (current === undefined)
|
|
512
|
+
return;
|
|
513
|
+
const cases = into.cases;
|
|
442
514
|
const planLines = [];
|
|
443
|
-
const current = (await readPlan(planFile)) ?? plan;
|
|
444
515
|
const skipped = [];
|
|
445
516
|
for (const c of current.suites.flatMap((s) => s.cases)) {
|
|
446
|
-
const gone =
|
|
517
|
+
const gone = c.routes.find((route) => k.removedRoutes.includes(route));
|
|
447
518
|
if (gone === undefined || c.status === "skipped")
|
|
448
519
|
continue;
|
|
449
520
|
c.status = "skipped";
|
|
450
521
|
skipped.push(c.id);
|
|
451
|
-
|
|
522
|
+
const detail = `route ${gone} is no longer declared`;
|
|
523
|
+
planLines.push(`${c.id} skipped (${detail})`);
|
|
452
524
|
const entry = cases.find((x) => x.id === c.id);
|
|
453
525
|
if (entry !== undefined) {
|
|
454
526
|
entry.outcome = "skipped";
|
|
455
|
-
entry.detail =
|
|
527
|
+
entry.detail = detail;
|
|
456
528
|
}
|
|
457
529
|
else
|
|
458
|
-
cases.push({ id: c.id, title: c.title, ...(c.file === undefined ? {} : { file: c.file }), outcome: "skipped", detail
|
|
530
|
+
cases.push({ id: c.id, title: c.title, ...(c.file === undefined ? {} : { file: c.file }), outcome: "skipped", detail });
|
|
459
531
|
}
|
|
460
532
|
if (skipped.length > 0)
|
|
461
533
|
await writePlan(planFile, current);
|
|
@@ -473,8 +545,6 @@ export async function runImpacted(ctx, r, impact, k, run, into) {
|
|
|
473
545
|
}
|
|
474
546
|
}
|
|
475
547
|
ctx.out(`plan: ${planLines.join("; ") || "unchanged"}`);
|
|
476
|
-
const order = new Map(impact.cases.map((c, i) => [c.id, i]));
|
|
477
|
-
cases.sort((a, b) => (order.get(a.id) ?? Infinity) - (order.get(b.id) ?? Infinity));
|
|
478
548
|
}
|
|
479
549
|
const count = (cases, outcome) => cases.filter((c) => c.outcome === outcome).length;
|
|
480
550
|
const cell = (text) => text.replace(/\|/g, "\\|").replace(/\s*\n\s*/g, " ").trim() || "none";
|
|
@@ -508,13 +578,34 @@ export function testsLine(o) {
|
|
|
508
578
|
return "tests: none";
|
|
509
579
|
return `tests: ran ${t.ran}: passed ${t.passed}, healed ${t.healed}, fixed ${t.fixed}, regressions ${t.regressions}, needs a human ${t.needsHuman}, failed ${t.failed}; skipped ${t.skipped}, planned ${o.summary?.planned ?? 0}; ${plural(o.modelCalls, "model call")}`;
|
|
510
580
|
}
|
|
511
|
-
|
|
581
|
+
const MAX_NOTIFY_LINES = 10;
|
|
582
|
+
/** The first notification: which tests failed where, after which deploy, and whether the healer runs. */
|
|
583
|
+
export function failuresText(changeSet, failed, ran, healer) {
|
|
584
|
+
const where = [...new Set(failed.map((f) => f.where.name))].join(", ");
|
|
585
|
+
const head = `omniqa: ${failed.length} of ${ran} tests failed on ${where} after ${changeSet.repo} ${short(changeSet.after)}${changeSet.title === undefined ? "" : ` (${changeSet.title})`}; ${healer}`;
|
|
586
|
+
const lines = failed.slice(0, MAX_NOTIFY_LINES).map((f) => ` ${f.selected.case.id}: ${f.selected.case.title} — ${f.detail}`);
|
|
587
|
+
const more = failed.length - lines.length;
|
|
588
|
+
return [head, ...lines, ...(more > 0 ? [` … and ${more} more (see the sync report)`] : [])].join("\n");
|
|
589
|
+
}
|
|
590
|
+
/** The second notification: what the healer made of the failures, what the knowledge update did, where it all went. */
|
|
591
|
+
export function outcomeText(o, failed) {
|
|
592
|
+
const s = o.summary;
|
|
593
|
+
const where = [...new Set(failed.map((f) => f.where.name))].join(", ");
|
|
594
|
+
const list = (outcomes) => {
|
|
595
|
+
const ids = o.cases.filter((c) => outcomes.includes(c.outcome)).map((c) => c.id);
|
|
596
|
+
return ids.length === 0 ? "" : ` (${ids.slice(0, MAX_NOTIFY_LINES).join(", ")}${ids.length > MAX_NOTIFY_LINES ? ", …" : ""})`;
|
|
597
|
+
};
|
|
598
|
+
const t = s.tests;
|
|
599
|
+
return `omniqa: healer on ${where} after ${s.repo} ${short(s.after)}: fixed ${t.fixed}${list(["fixed"])}, regression ${t.regressions}${list(["regression"])}, needs a human ${t.needsHuman}${list(["needs-human", "unsure"])}, still failing ${t.failed}${list(["failed", "behaviour-changed"])}; knowledge: ${plural(s.knowledge.maps, "map")}, ${plural(s.knowledge.pages, "page")}, scheme ${s.knowledge.scheme}; ${persistedText(s.persisted)}${s.stopped === undefined ? "" : `; stopped: ${s.stopped}`}`;
|
|
600
|
+
}
|
|
601
|
+
function summarize(o, at, budget, runCalls, suite) {
|
|
512
602
|
const ran = o.cases.filter((c) => !["not-run", "skipped"].includes(c.outcome));
|
|
513
603
|
return {
|
|
514
604
|
at,
|
|
515
605
|
host: o.changeSet.host,
|
|
516
606
|
repo: o.changeSet.repo,
|
|
517
607
|
kind: o.changeSet.kind,
|
|
608
|
+
...(suite === undefined ? {} : { suite: true, environment: suite.name }),
|
|
518
609
|
before: o.changeSet.before,
|
|
519
610
|
after: o.changeSet.after,
|
|
520
611
|
...(o.changeSet.title === undefined ? {} : { title: o.changeSet.title }),
|
|
@@ -526,12 +617,13 @@ function summarize(o, at, budget) {
|
|
|
526
617
|
tests: { ran: ran.length, passed: count(o.cases, "pass"), healed: count(o.cases, "healed"), fixed: count(o.cases, "fixed"), regressions: count(o.cases, "regression"), needsHuman: count(o.cases, "needs-human") + count(o.cases, "unsure"), skipped: count(o.cases, "skipped"), failed: count(o.cases, "failed") + count(o.cases, "behaviour-changed") },
|
|
527
618
|
planned: o.planned.length,
|
|
528
619
|
modelCalls: o.modelCalls,
|
|
620
|
+
...(runCalls === 0 ? {} : { runCalls }),
|
|
529
621
|
budget,
|
|
530
622
|
...(o.stopped === undefined ? {} : { stopped: o.stopped }),
|
|
531
623
|
persisted: { kind: "off" },
|
|
532
624
|
};
|
|
533
625
|
}
|
|
534
|
-
/** The pipeline for one request: resolve the change sets
|
|
626
|
+
/** The pipeline for one request: resolve the change sets; for each: the run, the impact, the knowledge update, the fixer, the plan and the persistence, in that order. */
|
|
535
627
|
export async function runSync(cwd, env, request, o) {
|
|
536
628
|
const ctx = await syncContext(cwd, env, o);
|
|
537
629
|
const dev = devEnvironment(ctx.product, o.env);
|
|
@@ -542,10 +634,24 @@ export async function runSync(cwd, env, request, o) {
|
|
|
542
634
|
const outcomes = [];
|
|
543
635
|
const lines = [];
|
|
544
636
|
const out = ctx.out;
|
|
637
|
+
/** One { text } post to notify.webhookEnv, redacted; a missing variable or a channel that is down is one line, never a failure. */
|
|
638
|
+
const notify = async (text) => {
|
|
639
|
+
const n = ctx.config.notify;
|
|
640
|
+
const url = env[n.webhookEnv];
|
|
641
|
+
if (!url) {
|
|
642
|
+
ctx.log(`notify: ${n.webhookEnv} is not set; nothing sent`);
|
|
643
|
+
return;
|
|
644
|
+
}
|
|
645
|
+
const error = await postText(url, redact(text));
|
|
646
|
+
ctx.out(error === undefined ? "notify: sent" : redact(`notify: not sent: ${error}`));
|
|
647
|
+
};
|
|
545
648
|
ctx.out = (line) => {
|
|
546
649
|
lines.push(line);
|
|
547
650
|
out(line);
|
|
548
651
|
};
|
|
652
|
+
// --suite without --env: the deploy landed on the production environment, the one the production-only cases run on; a product without one (init's default: a dev environment only) runs its suite on its default environment
|
|
653
|
+
const named = o.env !== undefined ? selectEnvironment(ctx.product, o.env) : o.suite === true ? (production ?? selectEnvironment(ctx.product)) : undefined;
|
|
654
|
+
const suite = o.suite === true ? named : undefined;
|
|
549
655
|
for (const r of await resolveChangeSets(ctx, request, o.dryRun === true)) {
|
|
550
656
|
lines.length = 0;
|
|
551
657
|
for (const line of describeLines(r.changeSet))
|
|
@@ -562,10 +668,17 @@ export async function runSync(cwd, env, request, o) {
|
|
|
562
668
|
const transcript = await openTranscript(join(cwd, RUNS_DIR), transcriptName("sync", ctx.now()), { redact });
|
|
563
669
|
await transcript.append({ kind: "sync", changeSet: { ...r.changeSet, files: r.changeSet.files.map((f) => ({ path: f.path, status: f.status })) }, impact: relative(cwd, file) });
|
|
564
670
|
const outcome = { changeSet: r.changeSet, impact, impactFile: file, cases: [], planned: [], modelCalls: 0 };
|
|
565
|
-
const run = { budget, transcript, redact, ...(dev === undefined ? {} : { dev }), ...(production === undefined ? {} : { production }) };
|
|
671
|
+
const run = { budget, transcript, redact, runCalls: 0, ...(dev === undefined ? {} : { dev }), ...(production === undefined ? {} : { production }) };
|
|
672
|
+
let failed = [];
|
|
566
673
|
try {
|
|
567
|
-
|
|
568
|
-
|
|
674
|
+
failed = await runCases(ctx, impact, run, suite, outcome);
|
|
675
|
+
if (failed.length > 0) {
|
|
676
|
+
const healer = failed.some((f) => explores(f.where, env)) ? "healer launched" : `no healer: ${[...new Set(failed.filter((f) => !explores(f.where, env)).map((f) => f.where.name))].join(", ")} does not explore`;
|
|
677
|
+
await notify(failuresText(r.changeSet, failed, outcome.cases.filter((c) => !["not-run", "skipped"].includes(c.outcome)).length, healer));
|
|
678
|
+
}
|
|
679
|
+
outcome.knowledge = await updateKnowledge(ctx, r, impact, run, named);
|
|
680
|
+
await fixFailed(ctx, r, impact, outcome.knowledge, run, failed, outcome.cases);
|
|
681
|
+
await updatePlan(ctx, outcome.knowledge, run, outcome);
|
|
569
682
|
if (r.eventId !== undefined)
|
|
570
683
|
await removePendingChange(cwd, r.eventId);
|
|
571
684
|
}
|
|
@@ -574,8 +687,10 @@ export async function runSync(cwd, env, request, o) {
|
|
|
574
687
|
throw err;
|
|
575
688
|
outcome.stopped = err.message;
|
|
576
689
|
}
|
|
577
|
-
|
|
578
|
-
outcome.
|
|
690
|
+
const order = new Map(impact.cases.map((c, i) => [c.id, i]));
|
|
691
|
+
outcome.cases.sort((a, b) => (order.get(a.id) ?? Infinity) - (order.get(b.id) ?? Infinity));
|
|
692
|
+
outcome.modelCalls = budget.used + run.runCalls;
|
|
693
|
+
outcome.summary = summarize(outcome, ctx.now().toISOString(), limit, run.runCalls, suite);
|
|
579
694
|
ctx.out(testsLine(outcome));
|
|
580
695
|
try {
|
|
581
696
|
ctx.out(`coverage: ${(await writeCoverageFile(cwd)).line}`);
|
|
@@ -600,6 +715,9 @@ export async function runSync(cwd, env, request, o) {
|
|
|
600
715
|
if (persisted.error !== undefined)
|
|
601
716
|
throw new Error(`persist failed: ${persisted.error}`);
|
|
602
717
|
}
|
|
718
|
+
// a persist error threw above: the CI job fails and says why, so the outcome message is skipped
|
|
719
|
+
if (failed.length > 0)
|
|
720
|
+
await notify(outcomeText(outcome, failed));
|
|
603
721
|
outcomes.push(outcome);
|
|
604
722
|
}
|
|
605
723
|
return outcomes;
|