omniqa 0.2.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/sync.js CHANGED
@@ -2,7 +2,7 @@ import { existsSync } from "node:fs";
2
2
  import { copyFile, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
3
3
  import { tmpdir } from "node:os";
4
4
  import { join, relative, resolve } from "node:path";
5
- import { budgeted, BudgetExceeded, changeSetLines, createModels, describeChangeSet, git, ingest, KNOWLEDGE_DIR, knowledgeCommit, localChangeSet, loadConfig, loadProduct, mapImpact, mergeEndpoints, openTranscript, parseEndpoints, parseStaticSections, planPath, readCrawl, readPlan, readState, README_FILE, readmeHasBlock, recordKnowledge, recordLastSeen, recordRun, recordSync, redactor, updateSync, removePendingChange, renderImpact, renderReview, repoDirsOf, repoNameOf, resolveEnvironmentUrl, RUNS_DIR, schemeCounts, secretValues, stampKnowledge, stampMaps, STATE_FILE, tierEnvironment, transcriptName, withReview, writeCoverageFile, writePlan } from "@omniqa/core";
5
+ import { budgeted, BudgetExceeded, changelogGitEnv, changeSetLines, createModels, describeChangeSet, git, ingest, KNOWLEDGE_DIR, knowledgeCommit, localChangeSet, loadConfig, loadProduct, mapImpact, mergeEndpoints, openTranscript, parseEndpoints, parseRemote, parseStaticSections, planPath, postText, readCrawl, readPlan, readState, README_FILE, readmeHasBlock, recordKnowledge, recordLastSeen, recordRun, recordSync, redactor, updateSync, removePendingChange, renderImpact, renderReview, repoDirsOf, repoNameOf, resolveEnvironmentUrl, RUNS_DIR, selectEnvironment, schemeCounts, secretValues, stampKnowledge, stampMaps, STATE_FILE, tierEnvironment, transcriptName, withReview, writeCoverageFile, writePlan } from "@omniqa/core";
6
6
  import { crawlSite, createNetMap, launchBrowser, writeNetMap } from "@omniqa/web";
7
7
  import { createGitHub } from "@omniqa/github";
8
8
  import { createGitLab } from "@omniqa/gitlab";
@@ -33,12 +33,19 @@ export async function syncContext(cwd, env, o) {
33
33
  const product = await loadProduct(cwd, env);
34
34
  return { cwd, env, config, product, repoDirs: repoDirsOf(cwd, product), now: o.now ?? (() => new Date()), out: o.out, log: o.log };
35
35
  }
36
- /** The git host that answers for the product repository: the configured changelog host, else the one gitHosts names. */
36
+ /** The git host that answers for the product repository: the configured changelog host, else the one gitHosts names; it reads with the token changelog.tokenEnv names when that variable is set (a private product repository), else the host's own token. */
37
37
  export function changeSource(ctx, host) {
38
38
  const which = host ?? ctx.config.changelog?.host ?? (ctx.config.gitHosts.gitlab !== undefined && ctx.config.gitHosts.github === undefined ? "gitlab" : "github");
39
- return which === "gitlab" ? createGitLab(ctx.config.gitHosts.gitlab ?? {}, ctx.env) : createGitHub(ctx.config.gitHosts.github ?? {}, ctx.env);
39
+ // the product token belongs to the changelog host only: a webhook from the other host keeps that host's own token, and so does the changelog host while the product token is unset
40
+ const tokenEnv = which === ctx.config.changelog?.host && ctx.env[ctx.config.changelog.tokenEnv] ? ctx.config.changelog.tokenEnv : undefined;
41
+ const withToken = (raw) => (tokenEnv === undefined ? raw : { ...raw, tokenEnv });
42
+ return which === "gitlab" ? createGitLab(withToken(ctx.config.gitHosts.gitlab ?? {}), ctx.env) : createGitHub(withToken(ctx.config.gitHosts.github ?? {}), ctx.env);
40
43
  }
41
44
  const knows = async (repoDir, sha) => git(repoDir, ["cat-file", "-e", `${sha}^{commit}`]).then(() => true, () => false);
45
+ /** The commit of `name` the knowledge base describes: `state.knowledge.commit` is `name sha[, name sha]` (knowledgeCommit). */
46
+ const knowledgeCommitOf = (state, name) => state.knowledge?.commit?.split(", ").map((part) => part.split(" ")).find(([n]) => n === name)?.[1];
47
+ /** Where a sync without --from starts: the last synced commit of the repository, else the commit its knowledge was built from. */
48
+ export const lastSynced = (state, name) => state.changelog?.lastSeen[name] ?? knowledgeCommitOf(state, name);
42
49
  /** The directory of the repository named by a host path (`o/r`) or a name, when product.yaml has it. */
43
50
  const localDir = (ctx, repo) => ctx.repoDirs[repoNameOf(repo)];
44
51
  /** A change set for a webhook event: from the repository on disk when it has both commits, else from the host's compare. */
@@ -62,27 +69,43 @@ async function fromEvent(ctx, event) {
62
69
  return { changeSet: { host: event.host, repo: event.repo, before, after, ...compact(base), files: c.files, commits: c.commits, ...(c.url === undefined ? {} : { url: c.url }) }, ...(dir === undefined ? {} : { repoDir: dir }) };
63
70
  }
64
71
  const compact = (o) => Object.fromEntries(Object.entries(o).filter(([, v]) => v !== undefined));
65
- /** The change sets a request asks for; `--poll` and the first poll are explained on `out`. */
72
+ /**
73
+ * The change sets a request asks for; `--poll` and the first poll are explained on `out`. A manual request names a repository of
74
+ * product.yaml (the only one, or `--repo`) or `changelog.repo`: a `repos:` path is read with git on disk (no token; `--to` is its
75
+ * HEAD); a `repos:` git url or `changelog.repo` goes through the host as a webhook does (`--to` is the head of `changelog.branch`),
76
+ * since the url's clone under .omniqa/repos is gitignored, so stale by hand and absent in a fresh CI checkout: it answers only when
77
+ * it has both commits, and the knowledge pass clones or pulls it afterwards.
78
+ */
66
79
  export async function resolveChangeSets(ctx, request, dryRun = false) {
67
80
  if (request.kind === "manual") {
68
81
  const names = Object.keys(ctx.repoDirs);
69
82
  const configured = ctx.config.changelog?.repo;
70
- let repo = request.repo ?? (names.length === 1 ? names[0] : configured);
83
+ const repo = request.repo ?? (names.length === 1 ? names[0] : configured);
71
84
  if (repo === undefined)
72
85
  throw new Error(names.length === 0 ? "no repository to compare: add repos: to product.yaml or changelog.repo to omniqa.config.ts" : `name the repository with --repo (${names.join(", ")})`);
86
+ const name = repoNameOf(repo);
87
+ // ponytail: a lastSeen ahead of --to (a poll recorded the head after a deploy) compares backwards; check ancestry with merge-base when it bites.
88
+ const from = request.from ?? lastSynced(await readState(ctx.cwd), name);
89
+ if (from === undefined)
90
+ throw new Error(`no commit to compare from for ${name}: nothing synced yet and no knowledge built from it; pass --from <sha>`);
73
91
  const dir = localDir(ctx, repo);
74
- if (dir !== undefined) {
92
+ const entry = (ctx.product.repos ?? []).find((x) => repoNameOf(x) === name);
93
+ if (dir !== undefined && entry !== undefined && !isRemote(entry)) {
75
94
  const to = request.to ?? (await git(dir, ["rev-parse", "HEAD"]));
76
- return [{ changeSet: await localChangeSet(dir, repoNameOf(repo), request.from, to), repoDir: dir }];
95
+ return [{ changeSet: await localChangeSet(dir, name, from, to), repoDir: dir }];
77
96
  }
78
- if (configured !== undefined && (repo === configured || repoNameOf(configured) === repo))
79
- repo = configured;
80
- else
97
+ const hostRepo = configured !== undefined && (repo === configured || repoNameOf(configured) === repo) ? configured : entry === undefined ? undefined : parseRemote(entry)?.path;
98
+ if (hostRepo === undefined)
81
99
  throw new Error(`unknown repository ${JSON.stringify(repo)}: not a repos: path of product.yaml (${names.join(", ") || "none"}) nor changelog.repo (${configured ?? "none"})`);
82
100
  const source = changeSource(ctx);
83
- const to = request.to ?? (await source.branchHead(repo, ctx.config.changelog?.branch ?? "main")).sha;
84
- const c = await source.compare(repo, request.from, to);
85
- return [{ changeSet: { host: ctx.config.changelog?.host ?? source.name, repo, kind: "manual", before: request.from, after: to, files: c.files, commits: c.commits, ...(c.url === undefined ? {} : { url: c.url }), ...(c.commits.at(-1) === undefined ? {} : { title: c.commits.at(-1).title }) } }];
101
+ const host = ctx.config.changelog?.host ?? source.name;
102
+ const to = request.to ?? (await source.branchHead(hostRepo, ctx.config.changelog?.branch ?? "main")).sha;
103
+ // the change set is named as product.yaml names the repository (changelog.repo alone keeps its host path); the clone answers when it has both commits, else the host's compare (the token)
104
+ const named = entry === undefined ? hostRepo : name;
105
+ if (dir !== undefined && (await knows(dir, from)) && (await knows(dir, to)))
106
+ return [{ changeSet: { ...(await localChangeSet(dir, named, from, to)), host }, repoDir: dir }];
107
+ const c = await source.compare(hostRepo, from, to);
108
+ return [{ changeSet: { host, repo: named, kind: "manual", before: from, after: to, files: c.files, commits: c.commits, ...(c.url === undefined ? {} : { url: c.url }), ...(c.commits.at(-1) === undefined ? {} : { title: c.commits.at(-1).title }) }, ...(dir === undefined ? {} : { repoDir: dir }) }];
86
109
  }
87
110
  if (request.kind === "poll") {
88
111
  const changelog = ctx.config.changelog;
@@ -159,8 +182,8 @@ export function impactLine(impact, file, cwd) {
159
182
  ...(impact.cases.length === 0 ? [] : [` cases: ${impact.cases.map((c) => `${c.id} (${c.confidence})`).join(", ")}; exact ${count("exact")}, pattern ${count("pattern")}, guess ${count("guess")}`]),
160
183
  ];
161
184
  }
162
- /** The impacted knowledge rewritten: the changed repository's map (its overview kept), a crawl of the impacted pages as every actor with their endpoint rows merged, the cross-reference, the scheme's impacted statements revised by the analyst behind the reviewer, the index and the knowledge commit. */
163
- export async function updateKnowledge(ctx, r, impact, run) {
185
+ /** The impacted knowledge rewritten: the changed repository's map (its overview kept), a crawl of the impacted pages as every actor with their endpoint rows merged, on `preferred` (the --env environment, M6) when it is given and explores, the cross-reference, the scheme's impacted statements revised by the analyst behind the reviewer, the index and the knowledge commit. */
186
+ export async function updateKnowledge(ctx, r, impact, run, preferred) {
164
187
  const knowledgeDir = join(ctx.cwd, KNOWLEDGE_DIR);
165
188
  const name = repoNameOf(r.changeSet.repo);
166
189
  const mapFile = join(knowledgeDir, "repos", name, "map.md");
@@ -175,7 +198,8 @@ export async function updateKnowledge(ctx, r, impact, run) {
175
198
  ctx.out(`static: ${r.changeSet.repo} is not a repos: source of product.yaml; its map was not rewritten`);
176
199
  }
177
200
  else {
178
- const result = await ingest({ source: isRemote(source) ? source : resolve(ctx.cwd, source), knowledgeDir, keepOverview: true, log: (l) => ctx.log(`ingest: ${l}`) });
201
+ const gitEnv = changelogGitEnv(ctx.config, ctx.env);
202
+ const result = await ingest({ source: isRemote(source) ? source : resolve(ctx.cwd, source), knowledgeDir, keepOverview: true, ...(gitEnv === undefined ? {} : { gitEnv }), log: (l) => ctx.log(`ingest: ${l}`) });
179
203
  const parsed = result.static === undefined ? parseStaticSections(await readFile(result.file, "utf8")) : undefined;
180
204
  ctx.out(staticLine(result.name, result.files, result.status, result.static, parsed, 0));
181
205
  if (result.commit !== undefined && !r.changeSet.after.startsWith(result.commit) && !result.commit.startsWith(r.changeSet.after))
@@ -201,8 +225,11 @@ export async function updateKnowledge(ctx, r, impact, run) {
201
225
  // a route with a placeholder is opened at the path the crawl once used for it
202
226
  const targets = impact.pages.toSorted((a, b) => RANK[b.confidence] - RANK[a.confidence]).map((p) => known.find((k) => k.route === p.name.route)?.path ?? p.name.route).filter((p) => !/[{}]/.test(p)).slice(0, MAX_TARGET_PAGES);
203
227
  const pages = [];
204
- // The knowledge build's rule: the dev tier, else a production environment that explores.
205
- const crawlOn = run.dev ?? (run.production?.environment.explore === true ? run.production : undefined);
228
+ // The crawl follows --env (M6) unless that is a production environment with explore off: the knowledge build's rule, a replay-only production is never crawled; then the dev tier, else a production environment that explores.
229
+ const fallback = run.dev ?? (run.production?.environment.explore === true ? run.production : undefined);
230
+ const crawlOn = preferred !== undefined && explores(preferred, ctx.env) ? preferred : fallback;
231
+ if (preferred !== undefined && !explores(preferred, ctx.env))
232
+ ctx.out(`crawl: ${preferred.name} is a production environment with explore off; ${fallback === undefined ? "nothing is crawled" : `crawling on ${fallback.name} instead`}`);
206
233
  if (crawlOn === undefined)
207
234
  ctx.out("crawl: skipped (no dev-tier environment, no production one with explore on)");
208
235
  else if (targets.length === 0)
@@ -352,110 +379,155 @@ export function classifyRun(report) {
352
379
  visit(report);
353
380
  return out;
354
381
  }
355
- /** The impacted cases run on the dev tier (replay; a miss heals behind the verdict), the production-only ones on production (healing where it explores), the fixer takes those whose behaviour changed on dev, vanished routes are proposed as skipped, added routes and handlers become planned cases. */
356
- export async function runImpacted(ctx, r, impact, k, run, into) {
357
- const testDir = join(ctx.cwd, ctx.config.testDir);
358
- const planFile = planPath(testDir);
359
- const plan = await readPlan(planFile);
360
- if (plan === undefined) {
361
- ctx.out("tests: skipped (no plan at tests/plan.md; run omniqa plan)");
362
- return;
382
+ /** A dev-tier environment, a production one with explore on, or one OMNIQA_ALLOW_EXPLORE names: the crawl and the fixer may act there. */
383
+ const explores = (w, env) => w.environment.tier !== "production" || w.environment.explore || env.OMNIQA_ALLOW_EXPLORE === w.name;
384
+ const FAILED = new Set(["failed", "behaviour-changed", "unsure"]);
385
+ /** The written cases a sync runs, grouped by environment: with `suite`, every written case of that environment's tier, on it; else the impacted written cases, the dev-tier ones on the dev environment, the production-only ones on the production one. Cases that cannot run land in `notRun`. */
386
+ function selectCases(plan, impact, run, suite, notRun) {
387
+ const groups = new Map();
388
+ const add = (where, s) => {
389
+ const g = groups.get(where.name) ?? { where, heal: where.environment.tier !== "production", selected: [] };
390
+ g.selected.push(s);
391
+ groups.set(where.name, g);
392
+ };
393
+ const all = plan.suites.flatMap((s) => s.cases.map((c) => ({ suite: s, case: c })));
394
+ if (suite !== undefined) {
395
+ for (const s of all)
396
+ if (s.case.status === "written" && s.case.file !== undefined && s.case.tiers.includes(suite.environment.tier))
397
+ add(suite, s);
398
+ return [...groups.values()];
363
399
  }
364
- const byId = new Map(plan.suites.flatMap((s) => s.cases.map((c) => ({ suite: s, case: c }))).map((x) => [x.case.id, x]));
365
- const project = ctx.config.projects[0]?.name;
366
- // filled as the stage goes, so a budget stop keeps what ran
367
- const cases = into.cases;
368
- const toRun = [];
369
- const onProduction = [];
370
- const vanished = (c) => c.routes.find((route) => k.removedRoutes.includes(route));
400
+ const byId = new Map(all.map((x) => [x.case.id, x]));
401
+ // ponytail: a case on a route the change removed runs too (removed routes are known only after the map is rewritten) and is marked skipped by the plan pass; the plan pass runs last, so a budget stop leaves it failed.
371
402
  for (const i of impact.cases) {
372
403
  const found = byId.get(i.id);
373
404
  if (found === undefined)
374
405
  continue;
375
406
  const c = found.case;
376
407
  const base = { id: c.id, title: c.title, ...(c.file === undefined ? {} : { file: c.file }) };
377
- const gone = vanished(c);
378
408
  if (c.status !== "written" || c.file === undefined)
379
- cases.push({ ...base, outcome: "not-run", detail: c.status });
380
- else if (gone !== undefined)
381
- cases.push({ ...base, outcome: "not-run", detail: `route ${gone} is no longer declared` });
409
+ notRun.push({ ...base, outcome: "not-run", detail: c.status });
382
410
  else if (!c.tiers.includes("dev")) {
383
411
  if (run.production === undefined)
384
- cases.push({ ...base, outcome: "not-run", detail: "production-only and no production-tier environment" });
412
+ notRun.push({ ...base, outcome: "not-run", detail: "production-only and no production-tier environment" });
385
413
  else
386
- onProduction.push(found);
414
+ add(run.production, found);
387
415
  }
388
416
  else if (run.dev === undefined)
389
- cases.push({ ...base, outcome: "not-run", detail: "no dev-tier environment" });
417
+ notRun.push({ ...base, outcome: "not-run", detail: "no dev-tier environment" });
390
418
  else
391
- toRun.push(found);
419
+ add(run.dev, found);
392
420
  }
421
+ return [...groups.values()];
422
+ }
423
+ /** Pass 1: the selected cases run, one omniqa test run per environment; each ends pass, healed, behaviour changed, unsure, failed or not run. Resolves to the failures for the fixer. The runs' model calls are counted, not charged (M5). */
424
+ export async function runCases(ctx, impact, run, suite, into) {
425
+ const plan = await readPlan(planPath(join(ctx.cwd, ctx.config.testDir)));
426
+ if (plan === undefined) {
427
+ ctx.out("tests: skipped (no plan at tests/plan.md; run omniqa plan)");
428
+ return [];
429
+ }
430
+ const project = ctx.config.projects[0]?.name;
431
+ const cases = into.cases;
432
+ const groups = selectCases(plan, impact, run, suite, cases);
393
433
  const notRun = cases.filter((c) => c.outcome === "not-run");
394
- ctx.out(`tests: ${plural(impact.cases.length, "impacted case")}; running ${toRun.length} written${run.dev === undefined ? "" : ` on ${run.dev.name} (${project})`}: ${ids(toRun) || "none"}${onProduction.length === 0 ? "" : `; ${onProduction.length} production-only on ${run.production.name}: ${ids(onProduction)}`}${notRun.length === 0 ? "" : `; not run: ${notRun.map((c) => `${c.id} (${c.detail})`).join(", ")}`}`);
395
- /** One run of the selected cases on an environment, recorded and classified; resolves to those whose behaviour changed. */
396
- const runOn = async (where, selected, heal, reportName) => {
397
- const result = await runCase(ctx.cwd, ctx.env, { ids: selected.map((s) => s.case.id), envName: where.name, project, heal, reportName, log: ctx.log });
434
+ if (suite !== undefined) {
435
+ const g = groups[0];
436
+ ctx.out(`tests: suite on ${suite.name} (${project}): ${plural(g?.selected.length ?? 0, "written case")} of the ${suite.environment.tier} tier${g === undefined ? "" : `: ${ids(g.selected)}`}`);
437
+ }
438
+ else {
439
+ const dev = groups.find((g) => g.where.environment.tier !== "production");
440
+ const prod = groups.find((g) => g.where.environment.tier === "production");
441
+ ctx.out(`tests: ${plural(impact.cases.length, "impacted case")}; running ${dev?.selected.length ?? 0} written${run.dev === undefined ? "" : ` on ${run.dev.name} (${project})`}: ${dev === undefined ? "none" : ids(dev.selected)}${prod === undefined ? "" : `; ${prod.selected.length} production-only on ${prod.where.name}: ${ids(prod.selected)}`}${notRun.length === 0 ? "" : `; not run: ${notRun.map((c) => `${c.id} (${c.detail})`).join(", ")}`}`);
442
+ }
443
+ const failed = [];
444
+ for (const g of groups) {
445
+ const onProduction = g.where.environment.tier === "production";
446
+ if (onProduction)
447
+ run.ranOnProduction = true;
448
+ const reportName = onProduction ? "omniqa-sync-production" : "omniqa-sync";
449
+ const result = await runCase(ctx.cwd, ctx.env, { ids: g.selected.map((s) => s.case.id), envName: g.where.name, project, heal: g.heal, reportName, log: ctx.log });
450
+ // ponytail: a run that leaves no report (no browser, a runner crash) fails the sync loudly instead of passing as not run; per-group not-run outcomes are the upgrade when a second environment's results matter.
398
451
  if (result.report === undefined)
399
452
  throw new Error(result.error ?? "the run left no report");
400
- run.budget.charge(result.summary?.modelCalls ?? 0);
453
+ run.runCalls += result.summary?.modelCalls ?? 0;
401
454
  if (result.summary !== undefined)
402
455
  await recordRun(ctx.cwd, { ...result.summary, report: join("test-results", `${reportName}.json`) });
403
456
  const classified = classifyRun(result.report);
404
- const changed = [];
405
- for (const s of selected) {
457
+ for (const s of g.selected) {
406
458
  const title = `${s.case.id}: ${s.case.title}`;
407
459
  const got = classified.get(title) ?? { outcome: "failed", detail: `no test titled ${JSON.stringify(title)} ran`, calls: 0 };
408
460
  cases.push({ id: s.case.id, title: s.case.title, file: s.case.file, outcome: got.outcome, ...(got.detail === undefined ? {} : { detail: got.detail }) });
409
461
  ctx.out(` ${s.case.id}: ${OUTCOME_WORDS[got.outcome]}${got.detail === undefined ? "" : ` — ${got.detail}`}`);
410
- if (got.outcome === "behaviour-changed")
411
- changed.push({ selected: s, detail: got.detail ?? "" });
462
+ if (FAILED.has(got.outcome))
463
+ failed.push({ selected: s, detail: got.detail ?? OUTCOME_WORDS[got.outcome], where: g.where });
412
464
  }
413
465
  for (const line of result.summary?.productionSteps ?? [])
414
466
  ctx.out(` ${line}`);
415
- return changed;
416
- };
417
- const changed = toRun.length > 0 && run.dev !== undefined ? await runOn(run.dev, toRun, true, "omniqa-sync") : [];
418
- if (onProduction.length > 0 && run.production !== undefined) {
419
- run.ranOnProduction = true;
420
- // No --heal on production: a miss heals there only where its explore is on, and explore: false keeps it a replay.
421
- // ponytail: a behaviour change found on production is reported, not handed to the fixer, which rewrites on the dev tier; a fixer on production is the upgrade.
422
- await runOn(run.production, onProduction, false, "omniqa-sync-production");
423
467
  }
424
- if (changed.length > 0 && run.dev !== undefined) {
425
- ctx.out(`fix: ${plural(changed.length, "case")} whose behaviour changed: ${changed.map((c) => c.selected.case.id).join(", ")}`);
426
- const knowledge = [`sections rewritten: ${impact.sections.map((x) => x.name).join(", ") || "none"}`, `pages re-crawled: ${k.pages.join(", ") || "none"}`, `scheme: ${k.scheme}`].join("\n");
427
- const fixer = await createFixer({ cwd: ctx.cwd, env: ctx.env, config: ctx.config, product: ctx.product, envName: run.dev.name, environment: run.dev.environment, project, budget: run.budget, transcript: run.transcript, changeSet: r.changeSet, mappedFiles: impact.files.filter((f) => f.confidence !== undefined).map((f) => f.path), knowledge, out: ctx.out, log: ctx.log });
468
+ return failed;
469
+ }
470
+ /** Pass 5: every failed case goes to the fixer on the environment it failed on, with the change set and the rewritten knowledge (M4); a production environment that does not explore gets no fixer (D5), a case on a route the change removed waits for the plan pass. */
471
+ export async function fixFailed(ctx, r, impact, k, run, failed, cases) {
472
+ const fixable = failed.filter((f) => !f.selected.case.routes.some((route) => k.removedRoutes.includes(route)));
473
+ if (fixable.length === 0)
474
+ return;
475
+ ctx.out(`fix: ${plural(fixable.length, "failed case")}: ${fixable.map((f) => f.selected.case.id).join(", ")}`);
476
+ const project = ctx.config.projects[0]?.name;
477
+ const knowledge = [`sections rewritten: ${impact.sections.map((x) => x.name).join(", ") || "none"}`, `pages re-crawled: ${k.pages.join(", ") || "none"}`, `scheme: ${k.scheme}`].join("\n");
478
+ const byEnv = new Map();
479
+ for (const f of fixable)
480
+ byEnv.set(f.where.name, [...(byEnv.get(f.where.name) ?? []), f]);
481
+ for (const group of byEnv.values()) {
482
+ const where = group[0].where;
483
+ const entry = (f) => cases.find((c) => c.id === f.selected.case.id);
484
+ if (!explores(where, ctx.env)) {
485
+ for (const f of group) {
486
+ entry(f).outcome = "failed";
487
+ entry(f).detail = `${f.detail}; no fixer: ${where.name} does not explore`;
488
+ ctx.out(` ${f.selected.case.id}: not fixed — ${where.name} is a production environment with explore off`);
489
+ }
490
+ continue;
491
+ }
492
+ const fixer = await createFixer({ cwd: ctx.cwd, env: ctx.env, config: ctx.config, product: ctx.product, envName: where.name, environment: where.environment, project, budget: run.budget, transcript: run.transcript, changeSet: r.changeSet, mappedFiles: impact.files.filter((f) => f.confidence !== undefined).map((f) => f.path), knowledge, out: ctx.out, log: ctx.log });
428
493
  try {
429
- for (const { selected, detail } of changed) {
430
- const fixed = await fixer.fix(selected, detail);
431
- const entry = cases.find((c) => c.id === selected.case.id);
432
- entry.outcome = fixed.status;
494
+ for (const f of group) {
495
+ const fixed = await fixer.fix(f.selected, f.detail);
496
+ const e = entry(f);
497
+ e.outcome = fixed.status;
433
498
  if (fixed.feedback !== undefined)
434
- entry.detail = fixed.feedback;
499
+ e.detail = fixed.feedback;
435
500
  }
436
501
  }
437
502
  finally {
438
503
  fixer.close();
439
504
  }
440
505
  }
441
- // the plan: cases on routes the change removed are proposed as skipped; added routes and handlers get planned cases through one writer call
506
+ }
507
+ /** Pass 6: cases on routes the change removed are proposed as skipped; added routes and handlers become planned cases through one planner call. */
508
+ export async function updatePlan(ctx, k, run, into) {
509
+ const planFile = planPath(join(ctx.cwd, ctx.config.testDir));
510
+ const current = await readPlan(planFile);
511
+ if (current === undefined)
512
+ return;
513
+ const cases = into.cases;
442
514
  const planLines = [];
443
- const current = (await readPlan(planFile)) ?? plan;
444
515
  const skipped = [];
445
516
  for (const c of current.suites.flatMap((s) => s.cases)) {
446
- const gone = vanished(c);
517
+ const gone = c.routes.find((route) => k.removedRoutes.includes(route));
447
518
  if (gone === undefined || c.status === "skipped")
448
519
  continue;
449
520
  c.status = "skipped";
450
521
  skipped.push(c.id);
451
- planLines.push(`${c.id} skipped (route ${gone} is no longer declared)`);
522
+ const detail = `route ${gone} is no longer declared`;
523
+ planLines.push(`${c.id} skipped (${detail})`);
452
524
  const entry = cases.find((x) => x.id === c.id);
453
525
  if (entry !== undefined) {
454
526
  entry.outcome = "skipped";
455
- entry.detail = `route ${gone} is no longer declared`;
527
+ entry.detail = detail;
456
528
  }
457
529
  else
458
- cases.push({ id: c.id, title: c.title, ...(c.file === undefined ? {} : { file: c.file }), outcome: "skipped", detail: `route ${gone} is no longer declared` });
530
+ cases.push({ id: c.id, title: c.title, ...(c.file === undefined ? {} : { file: c.file }), outcome: "skipped", detail });
459
531
  }
460
532
  if (skipped.length > 0)
461
533
  await writePlan(planFile, current);
@@ -473,8 +545,6 @@ export async function runImpacted(ctx, r, impact, k, run, into) {
473
545
  }
474
546
  }
475
547
  ctx.out(`plan: ${planLines.join("; ") || "unchanged"}`);
476
- const order = new Map(impact.cases.map((c, i) => [c.id, i]));
477
- cases.sort((a, b) => (order.get(a.id) ?? Infinity) - (order.get(b.id) ?? Infinity));
478
548
  }
479
549
  const count = (cases, outcome) => cases.filter((c) => c.outcome === outcome).length;
480
550
  const cell = (text) => text.replace(/\|/g, "\\|").replace(/\s*\n\s*/g, " ").trim() || "none";
@@ -508,13 +578,34 @@ export function testsLine(o) {
508
578
  return "tests: none";
509
579
  return `tests: ran ${t.ran}: passed ${t.passed}, healed ${t.healed}, fixed ${t.fixed}, regressions ${t.regressions}, needs a human ${t.needsHuman}, failed ${t.failed}; skipped ${t.skipped}, planned ${o.summary?.planned ?? 0}; ${plural(o.modelCalls, "model call")}`;
510
580
  }
511
- function summarize(o, at, budget) {
581
+ const MAX_NOTIFY_LINES = 10;
582
+ /** The first notification: which tests failed where, after which deploy, and whether the healer runs. */
583
+ export function failuresText(changeSet, failed, ran, healer) {
584
+ const where = [...new Set(failed.map((f) => f.where.name))].join(", ");
585
+ const head = `omniqa: ${failed.length} of ${ran} tests failed on ${where} after ${changeSet.repo} ${short(changeSet.after)}${changeSet.title === undefined ? "" : ` (${changeSet.title})`}; ${healer}`;
586
+ const lines = failed.slice(0, MAX_NOTIFY_LINES).map((f) => ` ${f.selected.case.id}: ${f.selected.case.title} — ${f.detail}`);
587
+ const more = failed.length - lines.length;
588
+ return [head, ...lines, ...(more > 0 ? [` … and ${more} more (see the sync report)`] : [])].join("\n");
589
+ }
590
+ /** The second notification: what the healer made of the failures, what the knowledge update did, where it all went. */
591
+ export function outcomeText(o, failed) {
592
+ const s = o.summary;
593
+ const where = [...new Set(failed.map((f) => f.where.name))].join(", ");
594
+ const list = (outcomes) => {
595
+ const ids = o.cases.filter((c) => outcomes.includes(c.outcome)).map((c) => c.id);
596
+ return ids.length === 0 ? "" : ` (${ids.slice(0, MAX_NOTIFY_LINES).join(", ")}${ids.length > MAX_NOTIFY_LINES ? ", …" : ""})`;
597
+ };
598
+ const t = s.tests;
599
+ return `omniqa: healer on ${where} after ${s.repo} ${short(s.after)}: fixed ${t.fixed}${list(["fixed"])}, regression ${t.regressions}${list(["regression"])}, needs a human ${t.needsHuman}${list(["needs-human", "unsure"])}, still failing ${t.failed}${list(["failed", "behaviour-changed"])}; knowledge: ${plural(s.knowledge.maps, "map")}, ${plural(s.knowledge.pages, "page")}, scheme ${s.knowledge.scheme}; ${persistedText(s.persisted)}${s.stopped === undefined ? "" : `; stopped: ${s.stopped}`}`;
600
+ }
601
+ function summarize(o, at, budget, runCalls, suite) {
512
602
  const ran = o.cases.filter((c) => !["not-run", "skipped"].includes(c.outcome));
513
603
  return {
514
604
  at,
515
605
  host: o.changeSet.host,
516
606
  repo: o.changeSet.repo,
517
607
  kind: o.changeSet.kind,
608
+ ...(suite === undefined ? {} : { suite: true, environment: suite.name }),
518
609
  before: o.changeSet.before,
519
610
  after: o.changeSet.after,
520
611
  ...(o.changeSet.title === undefined ? {} : { title: o.changeSet.title }),
@@ -526,12 +617,13 @@ function summarize(o, at, budget) {
526
617
  tests: { ran: ran.length, passed: count(o.cases, "pass"), healed: count(o.cases, "healed"), fixed: count(o.cases, "fixed"), regressions: count(o.cases, "regression"), needsHuman: count(o.cases, "needs-human") + count(o.cases, "unsure"), skipped: count(o.cases, "skipped"), failed: count(o.cases, "failed") + count(o.cases, "behaviour-changed") },
527
618
  planned: o.planned.length,
528
619
  modelCalls: o.modelCalls,
620
+ ...(runCalls === 0 ? {} : { runCalls }),
529
621
  budget,
530
622
  ...(o.stopped === undefined ? {} : { stopped: o.stopped }),
531
623
  persisted: { kind: "off" },
532
624
  };
533
625
  }
534
- /** The pipeline for one request: resolve the change sets, map each one's impact; the knowledge update, the runs, the fixer and the persistence follow. */
626
+ /** The pipeline for one request: resolve the change sets; for each: the run, the impact, the knowledge update, the fixer, the plan and the persistence, in that order. */
535
627
  export async function runSync(cwd, env, request, o) {
536
628
  const ctx = await syncContext(cwd, env, o);
537
629
  const dev = devEnvironment(ctx.product, o.env);
@@ -542,10 +634,24 @@ export async function runSync(cwd, env, request, o) {
542
634
  const outcomes = [];
543
635
  const lines = [];
544
636
  const out = ctx.out;
637
+ /** One { text } post to notify.webhookEnv, redacted; a missing variable or a channel that is down is one line, never a failure. */
638
+ const notify = async (text) => {
639
+ const n = ctx.config.notify;
640
+ const url = env[n.webhookEnv];
641
+ if (!url) {
642
+ ctx.log(`notify: ${n.webhookEnv} is not set; nothing sent`);
643
+ return;
644
+ }
645
+ const error = await postText(url, redact(text));
646
+ ctx.out(error === undefined ? "notify: sent" : redact(`notify: not sent: ${error}`));
647
+ };
545
648
  ctx.out = (line) => {
546
649
  lines.push(line);
547
650
  out(line);
548
651
  };
652
+ // --suite without --env: the deploy landed on the production environment, the one the production-only cases run on; a product without one (init's default: a dev environment only) runs its suite on its default environment
653
+ const named = o.env !== undefined ? selectEnvironment(ctx.product, o.env) : o.suite === true ? (production ?? selectEnvironment(ctx.product)) : undefined;
654
+ const suite = o.suite === true ? named : undefined;
549
655
  for (const r of await resolveChangeSets(ctx, request, o.dryRun === true)) {
550
656
  lines.length = 0;
551
657
  for (const line of describeLines(r.changeSet))
@@ -562,10 +668,17 @@ export async function runSync(cwd, env, request, o) {
562
668
  const transcript = await openTranscript(join(cwd, RUNS_DIR), transcriptName("sync", ctx.now()), { redact });
563
669
  await transcript.append({ kind: "sync", changeSet: { ...r.changeSet, files: r.changeSet.files.map((f) => ({ path: f.path, status: f.status })) }, impact: relative(cwd, file) });
564
670
  const outcome = { changeSet: r.changeSet, impact, impactFile: file, cases: [], planned: [], modelCalls: 0 };
565
- const run = { budget, transcript, redact, ...(dev === undefined ? {} : { dev }), ...(production === undefined ? {} : { production }) };
671
+ const run = { budget, transcript, redact, runCalls: 0, ...(dev === undefined ? {} : { dev }), ...(production === undefined ? {} : { production }) };
672
+ let failed = [];
566
673
  try {
567
- outcome.knowledge = await updateKnowledge(ctx, r, impact, run);
568
- await runImpacted(ctx, r, impact, outcome.knowledge, run, outcome);
674
+ failed = await runCases(ctx, impact, run, suite, outcome);
675
+ if (failed.length > 0) {
676
+ const healer = failed.some((f) => explores(f.where, env)) ? "healer launched" : `no healer: ${[...new Set(failed.filter((f) => !explores(f.where, env)).map((f) => f.where.name))].join(", ")} does not explore`;
677
+ await notify(failuresText(r.changeSet, failed, outcome.cases.filter((c) => !["not-run", "skipped"].includes(c.outcome)).length, healer));
678
+ }
679
+ outcome.knowledge = await updateKnowledge(ctx, r, impact, run, named);
680
+ await fixFailed(ctx, r, impact, outcome.knowledge, run, failed, outcome.cases);
681
+ await updatePlan(ctx, outcome.knowledge, run, outcome);
569
682
  if (r.eventId !== undefined)
570
683
  await removePendingChange(cwd, r.eventId);
571
684
  }
@@ -574,8 +687,10 @@ export async function runSync(cwd, env, request, o) {
574
687
  throw err;
575
688
  outcome.stopped = err.message;
576
689
  }
577
- outcome.modelCalls = budget.used;
578
- outcome.summary = summarize(outcome, ctx.now().toISOString(), limit);
690
+ const order = new Map(impact.cases.map((c, i) => [c.id, i]));
691
+ outcome.cases.sort((a, b) => (order.get(a.id) ?? Infinity) - (order.get(b.id) ?? Infinity));
692
+ outcome.modelCalls = budget.used + run.runCalls;
693
+ outcome.summary = summarize(outcome, ctx.now().toISOString(), limit, run.runCalls, suite);
579
694
  ctx.out(testsLine(outcome));
580
695
  try {
581
696
  ctx.out(`coverage: ${(await writeCoverageFile(cwd)).line}`);
@@ -600,6 +715,9 @@ export async function runSync(cwd, env, request, o) {
600
715
  if (persisted.error !== undefined)
601
716
  throw new Error(`persist failed: ${persisted.error}`);
602
717
  }
718
+ // a persist error threw above: the CI job fails and says why, so the outcome message is skipped
719
+ if (failed.length > 0)
720
+ await notify(outcomeText(outcome, failed));
603
721
  outcomes.push(outcome);
604
722
  }
605
723
  return outcomes;