omniqa 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/sync.js CHANGED
@@ -2,7 +2,7 @@ import { existsSync } from "node:fs";
2
2
  import { copyFile, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
3
3
  import { tmpdir } from "node:os";
4
4
  import { join, relative, resolve } from "node:path";
5
- import { budgeted, BudgetExceeded, changeSetLines, createModels, describeChangeSet, git, ingest, KNOWLEDGE_DIR, knowledgeCommit, localChangeSet, loadConfig, loadProduct, mapImpact, mergeEndpoints, openTranscript, parseEndpoints, parseStaticSections, planPath, readCrawl, readPlan, readState, README_FILE, readmeHasBlock, recordKnowledge, recordLastSeen, recordRun, recordSync, redactor, updateSync, removePendingChange, renderImpact, renderReview, repoDirsOf, repoNameOf, resolveEnvironmentUrl, RUNS_DIR, schemeCounts, secretValues, stampKnowledge, stampMaps, STATE_FILE, tierEnvironment, transcriptName, withReview, writeCoverageFile, writePlan } from "@omniqa/core";
5
+ import { budgeted, BudgetExceeded, changelogGitEnv, changeSetLines, createModels, describeChangeSet, git, ingest, KNOWLEDGE_DIR, knowledgeCommit, localChangeSet, loadConfig, loadProduct, mapImpact, mergeEndpoints, openTranscript, parseEndpoints, parseStaticSections, planPath, postText, readCrawl, readPlan, readState, README_FILE, readmeHasBlock, recordKnowledge, recordLastSeen, recordRun, recordSync, redactor, updateSync, removePendingChange, renderImpact, renderReview, repoDirsOf, repoNameOf, resolveEnvironmentUrl, RUNS_DIR, selectEnvironment, schemeCounts, secretValues, stampKnowledge, stampMaps, STATE_FILE, tierEnvironment, transcriptName, withReview, writeCoverageFile, writePlan } from "@omniqa/core";
6
6
  import { crawlSite, createNetMap, launchBrowser, writeNetMap } from "@omniqa/web";
7
7
  import { createGitHub } from "@omniqa/github";
8
8
  import { createGitLab } from "@omniqa/gitlab";
@@ -33,12 +33,19 @@ export async function syncContext(cwd, env, o) {
33
33
  const product = await loadProduct(cwd, env);
34
34
  return { cwd, env, config, product, repoDirs: repoDirsOf(cwd, product), now: o.now ?? (() => new Date()), out: o.out, log: o.log };
35
35
  }
36
- /** The git host that answers for the product repository: the configured changelog host, else the one gitHosts names. */
36
+ /** The git host that answers for the product repository: the configured changelog host, else the one gitHosts names; it reads with changelog.tokenEnv when that is set (a token for a private product repository), else the host's own token. */
37
37
  export function changeSource(ctx, host) {
38
38
  const which = host ?? ctx.config.changelog?.host ?? (ctx.config.gitHosts.gitlab !== undefined && ctx.config.gitHosts.github === undefined ? "gitlab" : "github");
39
- return which === "gitlab" ? createGitLab(ctx.config.gitHosts.gitlab ?? {}, ctx.env) : createGitHub(ctx.config.gitHosts.github ?? {}, ctx.env);
39
+ // the product token belongs to the changelog host only: a webhook from the other host keeps that host's own token
40
+ const tokenEnv = which === ctx.config.changelog?.host ? ctx.config.changelog.tokenEnv : undefined;
41
+ const withToken = (raw) => (tokenEnv === undefined ? raw : { ...raw, tokenEnv });
42
+ return which === "gitlab" ? createGitLab(withToken(ctx.config.gitHosts.gitlab ?? {}), ctx.env) : createGitHub(withToken(ctx.config.gitHosts.github ?? {}), ctx.env);
40
43
  }
41
44
  const knows = async (repoDir, sha) => git(repoDir, ["cat-file", "-e", `${sha}^{commit}`]).then(() => true, () => false);
45
+ /** The commit of `name` the knowledge base describes: `state.knowledge.commit` is `name sha[, name sha]` (knowledgeCommit). */
46
+ const knowledgeCommitOf = (state, name) => state.knowledge?.commit?.split(", ").map((part) => part.split(" ")).find(([n]) => n === name)?.[1];
47
+ /** Where a sync without --from starts: the last synced commit of the repository, else the commit its knowledge was built from. */
48
+ export const lastSynced = (state, name) => state.changelog?.lastSeen[name] ?? knowledgeCommitOf(state, name);
42
49
  /** The directory of the repository named by a host path (`o/r`) or a name, when product.yaml has it. */
43
50
  const localDir = (ctx, repo) => ctx.repoDirs[repoNameOf(repo)];
44
51
  /** A change set for a webhook event: from the repository on disk when it has both commits, else from the host's compare. */
@@ -70,10 +77,15 @@ export async function resolveChangeSets(ctx, request, dryRun = false) {
70
77
  let repo = request.repo ?? (names.length === 1 ? names[0] : configured);
71
78
  if (repo === undefined)
72
79
  throw new Error(names.length === 0 ? "no repository to compare: add repos: to product.yaml or changelog.repo to omniqa.config.ts" : `name the repository with --repo (${names.join(", ")})`);
80
+ const name = repoNameOf(repo);
81
+ // ponytail: a lastSeen ahead of --to (a poll recorded the head after a deploy) compares backwards; check ancestry with merge-base when it bites.
82
+ const from = request.from ?? lastSynced(await readState(ctx.cwd), name);
83
+ if (from === undefined)
84
+ throw new Error(`no commit to compare from for ${name}: nothing synced yet and no knowledge built from it; pass --from <sha>`);
73
85
  const dir = localDir(ctx, repo);
74
86
  if (dir !== undefined) {
75
87
  const to = request.to ?? (await git(dir, ["rev-parse", "HEAD"]));
76
- return [{ changeSet: await localChangeSet(dir, repoNameOf(repo), request.from, to), repoDir: dir }];
88
+ return [{ changeSet: await localChangeSet(dir, name, from, to), repoDir: dir }];
77
89
  }
78
90
  if (configured !== undefined && (repo === configured || repoNameOf(configured) === repo))
79
91
  repo = configured;
@@ -81,8 +93,8 @@ export async function resolveChangeSets(ctx, request, dryRun = false) {
81
93
  throw new Error(`unknown repository ${JSON.stringify(repo)}: not a repos: path of product.yaml (${names.join(", ") || "none"}) nor changelog.repo (${configured ?? "none"})`);
82
94
  const source = changeSource(ctx);
83
95
  const to = request.to ?? (await source.branchHead(repo, ctx.config.changelog?.branch ?? "main")).sha;
84
- const c = await source.compare(repo, request.from, to);
85
- return [{ changeSet: { host: ctx.config.changelog?.host ?? source.name, repo, kind: "manual", before: request.from, after: to, files: c.files, commits: c.commits, ...(c.url === undefined ? {} : { url: c.url }), ...(c.commits.at(-1) === undefined ? {} : { title: c.commits.at(-1).title }) } }];
96
+ const c = await source.compare(repo, from, to);
97
+ return [{ changeSet: { host: ctx.config.changelog?.host ?? source.name, repo, kind: "manual", before: from, after: to, files: c.files, commits: c.commits, ...(c.url === undefined ? {} : { url: c.url }), ...(c.commits.at(-1) === undefined ? {} : { title: c.commits.at(-1).title }) } }];
86
98
  }
87
99
  if (request.kind === "poll") {
88
100
  const changelog = ctx.config.changelog;
@@ -159,8 +171,8 @@ export function impactLine(impact, file, cwd) {
159
171
  ...(impact.cases.length === 0 ? [] : [` cases: ${impact.cases.map((c) => `${c.id} (${c.confidence})`).join(", ")}; exact ${count("exact")}, pattern ${count("pattern")}, guess ${count("guess")}`]),
160
172
  ];
161
173
  }
162
- /** The impacted knowledge rewritten: the changed repository's map (its overview kept), a crawl of the impacted pages as every actor with their endpoint rows merged, the cross-reference, the scheme's impacted statements revised by the analyst behind the reviewer, the index and the knowledge commit. */
163
- export async function updateKnowledge(ctx, r, impact, run) {
174
+ /** The impacted knowledge rewritten: the changed repository's map (its overview kept), a crawl of the impacted pages as every actor with their endpoint rows merged, on `preferred` (the --env environment, M6) when it is given and explores, the cross-reference, the scheme's impacted statements revised by the analyst behind the reviewer, the index and the knowledge commit. */
175
+ export async function updateKnowledge(ctx, r, impact, run, preferred) {
164
176
  const knowledgeDir = join(ctx.cwd, KNOWLEDGE_DIR);
165
177
  const name = repoNameOf(r.changeSet.repo);
166
178
  const mapFile = join(knowledgeDir, "repos", name, "map.md");
@@ -175,7 +187,8 @@ export async function updateKnowledge(ctx, r, impact, run) {
175
187
  ctx.out(`static: ${r.changeSet.repo} is not a repos: source of product.yaml; its map was not rewritten`);
176
188
  }
177
189
  else {
178
- const result = await ingest({ source: isRemote(source) ? source : resolve(ctx.cwd, source), knowledgeDir, keepOverview: true, log: (l) => ctx.log(`ingest: ${l}`) });
190
+ const gitEnv = changelogGitEnv(ctx.config, ctx.env);
191
+ const result = await ingest({ source: isRemote(source) ? source : resolve(ctx.cwd, source), knowledgeDir, keepOverview: true, ...(gitEnv === undefined ? {} : { gitEnv }), log: (l) => ctx.log(`ingest: ${l}`) });
179
192
  const parsed = result.static === undefined ? parseStaticSections(await readFile(result.file, "utf8")) : undefined;
180
193
  ctx.out(staticLine(result.name, result.files, result.status, result.static, parsed, 0));
181
194
  if (result.commit !== undefined && !r.changeSet.after.startsWith(result.commit) && !result.commit.startsWith(r.changeSet.after))
@@ -201,8 +214,11 @@ export async function updateKnowledge(ctx, r, impact, run) {
201
214
  // a route with a placeholder is opened at the path the crawl once used for it
202
215
  const targets = impact.pages.toSorted((a, b) => RANK[b.confidence] - RANK[a.confidence]).map((p) => known.find((k) => k.route === p.name.route)?.path ?? p.name.route).filter((p) => !/[{}]/.test(p)).slice(0, MAX_TARGET_PAGES);
203
216
  const pages = [];
204
- // The knowledge build's rule: the dev tier, else a production environment that explores.
205
- const crawlOn = run.dev ?? (run.production?.environment.explore === true ? run.production : undefined);
217
+ // The crawl follows --env (M6) unless that is a production environment with explore off: the knowledge build's rule, a replay-only production is never crawled; then the dev tier, else a production environment that explores.
218
+ const fallback = run.dev ?? (run.production?.environment.explore === true ? run.production : undefined);
219
+ const crawlOn = preferred !== undefined && explores(preferred, ctx.env) ? preferred : fallback;
220
+ if (preferred !== undefined && !explores(preferred, ctx.env))
221
+ ctx.out(`crawl: ${preferred.name} is a production environment with explore off; ${fallback === undefined ? "nothing is crawled" : `crawling on ${fallback.name} instead`}`);
206
222
  if (crawlOn === undefined)
207
223
  ctx.out("crawl: skipped (no dev-tier environment, no production one with explore on)");
208
224
  else if (targets.length === 0)
@@ -352,110 +368,155 @@ export function classifyRun(report) {
352
368
  visit(report);
353
369
  return out;
354
370
  }
355
- /** The impacted cases run on the dev tier (replay; a miss heals behind the verdict), the production-only ones on production (healing where it explores), the fixer takes those whose behaviour changed on dev, vanished routes are proposed as skipped, added routes and handlers become planned cases. */
356
- export async function runImpacted(ctx, r, impact, k, run, into) {
357
- const testDir = join(ctx.cwd, ctx.config.testDir);
358
- const planFile = planPath(testDir);
359
- const plan = await readPlan(planFile);
360
- if (plan === undefined) {
361
- ctx.out("tests: skipped (no plan at tests/plan.md; run omniqa plan)");
362
- return;
371
+ /** A dev-tier environment, a production one with explore on, or one OMNIQA_ALLOW_EXPLORE names: the crawl and the fixer may act there. */
372
+ const explores = (w, env) => w.environment.tier !== "production" || w.environment.explore || env.OMNIQA_ALLOW_EXPLORE === w.name;
373
+ const FAILED = new Set(["failed", "behaviour-changed", "unsure"]);
374
+ /** The written cases a sync runs, grouped by environment: with `suite`, every written case of that environment's tier, on it; else the impacted written cases, the dev-tier ones on the dev environment, the production-only ones on the production one. Cases that cannot run land in `notRun`. */
375
+ function selectCases(plan, impact, run, suite, notRun) {
376
+ const groups = new Map();
377
+ const add = (where, s) => {
378
+ const g = groups.get(where.name) ?? { where, heal: where.environment.tier !== "production", selected: [] };
379
+ g.selected.push(s);
380
+ groups.set(where.name, g);
381
+ };
382
+ const all = plan.suites.flatMap((s) => s.cases.map((c) => ({ suite: s, case: c })));
383
+ if (suite !== undefined) {
384
+ for (const s of all)
385
+ if (s.case.status === "written" && s.case.file !== undefined && s.case.tiers.includes(suite.environment.tier))
386
+ add(suite, s);
387
+ return [...groups.values()];
363
388
  }
364
- const byId = new Map(plan.suites.flatMap((s) => s.cases.map((c) => ({ suite: s, case: c }))).map((x) => [x.case.id, x]));
365
- const project = ctx.config.projects[0]?.name;
366
- // filled as the stage goes, so a budget stop keeps what ran
367
- const cases = into.cases;
368
- const toRun = [];
369
- const onProduction = [];
370
- const vanished = (c) => c.routes.find((route) => k.removedRoutes.includes(route));
389
+ const byId = new Map(all.map((x) => [x.case.id, x]));
390
+ // ponytail: a case on a route the change removed runs too (removed routes are known only after the map is rewritten) and is marked skipped by the plan pass; the plan pass runs last, so a budget stop leaves it failed.
371
391
  for (const i of impact.cases) {
372
392
  const found = byId.get(i.id);
373
393
  if (found === undefined)
374
394
  continue;
375
395
  const c = found.case;
376
396
  const base = { id: c.id, title: c.title, ...(c.file === undefined ? {} : { file: c.file }) };
377
- const gone = vanished(c);
378
397
  if (c.status !== "written" || c.file === undefined)
379
- cases.push({ ...base, outcome: "not-run", detail: c.status });
380
- else if (gone !== undefined)
381
- cases.push({ ...base, outcome: "not-run", detail: `route ${gone} is no longer declared` });
398
+ notRun.push({ ...base, outcome: "not-run", detail: c.status });
382
399
  else if (!c.tiers.includes("dev")) {
383
400
  if (run.production === undefined)
384
- cases.push({ ...base, outcome: "not-run", detail: "production-only and no production-tier environment" });
401
+ notRun.push({ ...base, outcome: "not-run", detail: "production-only and no production-tier environment" });
385
402
  else
386
- onProduction.push(found);
403
+ add(run.production, found);
387
404
  }
388
405
  else if (run.dev === undefined)
389
- cases.push({ ...base, outcome: "not-run", detail: "no dev-tier environment" });
406
+ notRun.push({ ...base, outcome: "not-run", detail: "no dev-tier environment" });
390
407
  else
391
- toRun.push(found);
408
+ add(run.dev, found);
409
+ }
410
+ return [...groups.values()];
411
+ }
412
+ /** Pass 1: the selected cases run, one omniqa test run per environment; each ends pass, healed, behaviour changed, unsure, failed or not run. Resolves to the failures for the fixer. The runs' model calls are counted, not charged (M5). */
413
+ export async function runCases(ctx, impact, run, suite, into) {
414
+ const plan = await readPlan(planPath(join(ctx.cwd, ctx.config.testDir)));
415
+ if (plan === undefined) {
416
+ ctx.out("tests: skipped (no plan at tests/plan.md; run omniqa plan)");
417
+ return [];
392
418
  }
419
+ const project = ctx.config.projects[0]?.name;
420
+ const cases = into.cases;
421
+ const groups = selectCases(plan, impact, run, suite, cases);
393
422
  const notRun = cases.filter((c) => c.outcome === "not-run");
394
- ctx.out(`tests: ${plural(impact.cases.length, "impacted case")}; running ${toRun.length} written${run.dev === undefined ? "" : ` on ${run.dev.name} (${project})`}: ${ids(toRun) || "none"}${onProduction.length === 0 ? "" : `; ${onProduction.length} production-only on ${run.production.name}: ${ids(onProduction)}`}${notRun.length === 0 ? "" : `; not run: ${notRun.map((c) => `${c.id} (${c.detail})`).join(", ")}`}`);
395
- /** One run of the selected cases on an environment, recorded and classified; resolves to those whose behaviour changed. */
396
- const runOn = async (where, selected, heal, reportName) => {
397
- const result = await runCase(ctx.cwd, ctx.env, { ids: selected.map((s) => s.case.id), envName: where.name, project, heal, reportName, log: ctx.log });
423
+ if (suite !== undefined) {
424
+ const g = groups[0];
425
+ ctx.out(`tests: suite on ${suite.name} (${project}): ${plural(g?.selected.length ?? 0, "written case")} of the ${suite.environment.tier} tier${g === undefined ? "" : `: ${ids(g.selected)}`}`);
426
+ }
427
+ else {
428
+ const dev = groups.find((g) => g.where.environment.tier !== "production");
429
+ const prod = groups.find((g) => g.where.environment.tier === "production");
430
+ ctx.out(`tests: ${plural(impact.cases.length, "impacted case")}; running ${dev?.selected.length ?? 0} written${run.dev === undefined ? "" : ` on ${run.dev.name} (${project})`}: ${dev === undefined ? "none" : ids(dev.selected)}${prod === undefined ? "" : `; ${prod.selected.length} production-only on ${prod.where.name}: ${ids(prod.selected)}`}${notRun.length === 0 ? "" : `; not run: ${notRun.map((c) => `${c.id} (${c.detail})`).join(", ")}`}`);
431
+ }
432
+ const failed = [];
433
+ for (const g of groups) {
434
+ const onProduction = g.where.environment.tier === "production";
435
+ if (onProduction)
436
+ run.ranOnProduction = true;
437
+ const reportName = onProduction ? "omniqa-sync-production" : "omniqa-sync";
438
+ const result = await runCase(ctx.cwd, ctx.env, { ids: g.selected.map((s) => s.case.id), envName: g.where.name, project, heal: g.heal, reportName, log: ctx.log });
439
+ // ponytail: a run that leaves no report (no browser, a runner crash) fails the sync loudly instead of passing as not run; per-group not-run outcomes are the upgrade when a second environment's results matter.
398
440
  if (result.report === undefined)
399
441
  throw new Error(result.error ?? "the run left no report");
400
- run.budget.charge(result.summary?.modelCalls ?? 0);
442
+ run.runCalls += result.summary?.modelCalls ?? 0;
401
443
  if (result.summary !== undefined)
402
444
  await recordRun(ctx.cwd, { ...result.summary, report: join("test-results", `${reportName}.json`) });
403
445
  const classified = classifyRun(result.report);
404
- const changed = [];
405
- for (const s of selected) {
446
+ for (const s of g.selected) {
406
447
  const title = `${s.case.id}: ${s.case.title}`;
407
448
  const got = classified.get(title) ?? { outcome: "failed", detail: `no test titled ${JSON.stringify(title)} ran`, calls: 0 };
408
449
  cases.push({ id: s.case.id, title: s.case.title, file: s.case.file, outcome: got.outcome, ...(got.detail === undefined ? {} : { detail: got.detail }) });
409
450
  ctx.out(` ${s.case.id}: ${OUTCOME_WORDS[got.outcome]}${got.detail === undefined ? "" : ` — ${got.detail}`}`);
410
- if (got.outcome === "behaviour-changed")
411
- changed.push({ selected: s, detail: got.detail ?? "" });
451
+ if (FAILED.has(got.outcome))
452
+ failed.push({ selected: s, detail: got.detail ?? OUTCOME_WORDS[got.outcome], where: g.where });
412
453
  }
413
454
  for (const line of result.summary?.productionSteps ?? [])
414
455
  ctx.out(` ${line}`);
415
- return changed;
416
- };
417
- const changed = toRun.length > 0 && run.dev !== undefined ? await runOn(run.dev, toRun, true, "omniqa-sync") : [];
418
- if (onProduction.length > 0 && run.production !== undefined) {
419
- run.ranOnProduction = true;
420
- // No --heal on production: a miss heals there only where its explore is on, and explore: false keeps it a replay.
421
- // ponytail: a behaviour change found on production is reported, not handed to the fixer, which rewrites on the dev tier; a fixer on production is the upgrade.
422
- await runOn(run.production, onProduction, false, "omniqa-sync-production");
423
456
  }
424
- if (changed.length > 0 && run.dev !== undefined) {
425
- ctx.out(`fix: ${plural(changed.length, "case")} whose behaviour changed: ${changed.map((c) => c.selected.case.id).join(", ")}`);
426
- const knowledge = [`sections rewritten: ${impact.sections.map((x) => x.name).join(", ") || "none"}`, `pages re-crawled: ${k.pages.join(", ") || "none"}`, `scheme: ${k.scheme}`].join("\n");
427
- const fixer = await createFixer({ cwd: ctx.cwd, env: ctx.env, config: ctx.config, product: ctx.product, envName: run.dev.name, environment: run.dev.environment, project, budget: run.budget, transcript: run.transcript, changeSet: r.changeSet, mappedFiles: impact.files.filter((f) => f.confidence !== undefined).map((f) => f.path), knowledge, out: ctx.out, log: ctx.log });
457
+ return failed;
458
+ }
459
+ /** Pass 5: every failed case goes to the fixer on the environment it failed on, with the change set and the rewritten knowledge (M4); a production environment that does not explore gets no fixer (D5), a case on a route the change removed waits for the plan pass. */
460
+ export async function fixFailed(ctx, r, impact, k, run, failed, cases) {
461
+ const fixable = failed.filter((f) => !f.selected.case.routes.some((route) => k.removedRoutes.includes(route)));
462
+ if (fixable.length === 0)
463
+ return;
464
+ ctx.out(`fix: ${plural(fixable.length, "failed case")}: ${fixable.map((f) => f.selected.case.id).join(", ")}`);
465
+ const project = ctx.config.projects[0]?.name;
466
+ const knowledge = [`sections rewritten: ${impact.sections.map((x) => x.name).join(", ") || "none"}`, `pages re-crawled: ${k.pages.join(", ") || "none"}`, `scheme: ${k.scheme}`].join("\n");
467
+ const byEnv = new Map();
468
+ for (const f of fixable)
469
+ byEnv.set(f.where.name, [...(byEnv.get(f.where.name) ?? []), f]);
470
+ for (const group of byEnv.values()) {
471
+ const where = group[0].where;
472
+ const entry = (f) => cases.find((c) => c.id === f.selected.case.id);
473
+ if (!explores(where, ctx.env)) {
474
+ for (const f of group) {
475
+ entry(f).outcome = "failed";
476
+ entry(f).detail = `${f.detail}; no fixer: ${where.name} does not explore`;
477
+ ctx.out(` ${f.selected.case.id}: not fixed — ${where.name} is a production environment with explore off`);
478
+ }
479
+ continue;
480
+ }
481
+ const fixer = await createFixer({ cwd: ctx.cwd, env: ctx.env, config: ctx.config, product: ctx.product, envName: where.name, environment: where.environment, project, budget: run.budget, transcript: run.transcript, changeSet: r.changeSet, mappedFiles: impact.files.filter((f) => f.confidence !== undefined).map((f) => f.path), knowledge, out: ctx.out, log: ctx.log });
428
482
  try {
429
- for (const { selected, detail } of changed) {
430
- const fixed = await fixer.fix(selected, detail);
431
- const entry = cases.find((c) => c.id === selected.case.id);
432
- entry.outcome = fixed.status;
483
+ for (const f of group) {
484
+ const fixed = await fixer.fix(f.selected, f.detail);
485
+ const e = entry(f);
486
+ e.outcome = fixed.status;
433
487
  if (fixed.feedback !== undefined)
434
- entry.detail = fixed.feedback;
488
+ e.detail = fixed.feedback;
435
489
  }
436
490
  }
437
491
  finally {
438
492
  fixer.close();
439
493
  }
440
494
  }
441
- // the plan: cases on routes the change removed are proposed as skipped; added routes and handlers get planned cases through one writer call
495
+ }
496
+ /** Pass 6: cases on routes the change removed are proposed as skipped; added routes and handlers become planned cases through one planner call. */
497
+ export async function updatePlan(ctx, k, run, into) {
498
+ const planFile = planPath(join(ctx.cwd, ctx.config.testDir));
499
+ const current = await readPlan(planFile);
500
+ if (current === undefined)
501
+ return;
502
+ const cases = into.cases;
442
503
  const planLines = [];
443
- const current = (await readPlan(planFile)) ?? plan;
444
504
  const skipped = [];
445
505
  for (const c of current.suites.flatMap((s) => s.cases)) {
446
- const gone = vanished(c);
506
+ const gone = c.routes.find((route) => k.removedRoutes.includes(route));
447
507
  if (gone === undefined || c.status === "skipped")
448
508
  continue;
449
509
  c.status = "skipped";
450
510
  skipped.push(c.id);
451
- planLines.push(`${c.id} skipped (route ${gone} is no longer declared)`);
511
+ const detail = `route ${gone} is no longer declared`;
512
+ planLines.push(`${c.id} skipped (${detail})`);
452
513
  const entry = cases.find((x) => x.id === c.id);
453
514
  if (entry !== undefined) {
454
515
  entry.outcome = "skipped";
455
- entry.detail = `route ${gone} is no longer declared`;
516
+ entry.detail = detail;
456
517
  }
457
518
  else
458
- cases.push({ id: c.id, title: c.title, ...(c.file === undefined ? {} : { file: c.file }), outcome: "skipped", detail: `route ${gone} is no longer declared` });
519
+ cases.push({ id: c.id, title: c.title, ...(c.file === undefined ? {} : { file: c.file }), outcome: "skipped", detail });
459
520
  }
460
521
  if (skipped.length > 0)
461
522
  await writePlan(planFile, current);
@@ -473,8 +534,6 @@ export async function runImpacted(ctx, r, impact, k, run, into) {
473
534
  }
474
535
  }
475
536
  ctx.out(`plan: ${planLines.join("; ") || "unchanged"}`);
476
- const order = new Map(impact.cases.map((c, i) => [c.id, i]));
477
- cases.sort((a, b) => (order.get(a.id) ?? Infinity) - (order.get(b.id) ?? Infinity));
478
537
  }
479
538
  const count = (cases, outcome) => cases.filter((c) => c.outcome === outcome).length;
480
539
  const cell = (text) => text.replace(/\|/g, "\\|").replace(/\s*\n\s*/g, " ").trim() || "none";
@@ -508,13 +567,34 @@ export function testsLine(o) {
508
567
  return "tests: none";
509
568
  return `tests: ran ${t.ran}: passed ${t.passed}, healed ${t.healed}, fixed ${t.fixed}, regressions ${t.regressions}, needs a human ${t.needsHuman}, failed ${t.failed}; skipped ${t.skipped}, planned ${o.summary?.planned ?? 0}; ${plural(o.modelCalls, "model call")}`;
510
569
  }
511
- function summarize(o, at, budget) {
570
+ const MAX_NOTIFY_LINES = 10;
571
+ /** The first notification: which tests failed where, after which deploy, and whether the healer runs. */
572
+ export function failuresText(changeSet, failed, ran, healer) {
573
+ const where = [...new Set(failed.map((f) => f.where.name))].join(", ");
574
+ const head = `omniqa: ${failed.length} of ${ran} tests failed on ${where} after ${changeSet.repo} ${short(changeSet.after)}${changeSet.title === undefined ? "" : ` (${changeSet.title})`}; ${healer}`;
575
+ const lines = failed.slice(0, MAX_NOTIFY_LINES).map((f) => ` ${f.selected.case.id}: ${f.selected.case.title} — ${f.detail}`);
576
+ const more = failed.length - lines.length;
577
+ return [head, ...lines, ...(more > 0 ? [` … and ${more} more (see the sync report)`] : [])].join("\n");
578
+ }
579
+ /** The second notification: what the healer made of the failures, what the knowledge update did, where it all went. */
580
+ export function outcomeText(o, failed) {
581
+ const s = o.summary;
582
+ const where = [...new Set(failed.map((f) => f.where.name))].join(", ");
583
+ const list = (outcomes) => {
584
+ const ids = o.cases.filter((c) => outcomes.includes(c.outcome)).map((c) => c.id);
585
+ return ids.length === 0 ? "" : ` (${ids.slice(0, MAX_NOTIFY_LINES).join(", ")}${ids.length > MAX_NOTIFY_LINES ? ", …" : ""})`;
586
+ };
587
+ const t = s.tests;
588
+ return `omniqa: healer on ${where} after ${s.repo} ${short(s.after)}: fixed ${t.fixed}${list(["fixed"])}, regression ${t.regressions}${list(["regression"])}, needs a human ${t.needsHuman}${list(["needs-human", "unsure"])}, still failing ${t.failed}${list(["failed", "behaviour-changed"])}; knowledge: ${plural(s.knowledge.maps, "map")}, ${plural(s.knowledge.pages, "page")}, scheme ${s.knowledge.scheme}; ${persistedText(s.persisted)}${s.stopped === undefined ? "" : `; stopped: ${s.stopped}`}`;
589
+ }
590
+ function summarize(o, at, budget, runCalls, suite) {
512
591
  const ran = o.cases.filter((c) => !["not-run", "skipped"].includes(c.outcome));
513
592
  return {
514
593
  at,
515
594
  host: o.changeSet.host,
516
595
  repo: o.changeSet.repo,
517
596
  kind: o.changeSet.kind,
597
+ ...(suite === undefined ? {} : { suite: true, environment: suite.name }),
518
598
  before: o.changeSet.before,
519
599
  after: o.changeSet.after,
520
600
  ...(o.changeSet.title === undefined ? {} : { title: o.changeSet.title }),
@@ -526,12 +606,13 @@ function summarize(o, at, budget) {
526
606
  tests: { ran: ran.length, passed: count(o.cases, "pass"), healed: count(o.cases, "healed"), fixed: count(o.cases, "fixed"), regressions: count(o.cases, "regression"), needsHuman: count(o.cases, "needs-human") + count(o.cases, "unsure"), skipped: count(o.cases, "skipped"), failed: count(o.cases, "failed") + count(o.cases, "behaviour-changed") },
527
607
  planned: o.planned.length,
528
608
  modelCalls: o.modelCalls,
609
+ ...(runCalls === 0 ? {} : { runCalls }),
529
610
  budget,
530
611
  ...(o.stopped === undefined ? {} : { stopped: o.stopped }),
531
612
  persisted: { kind: "off" },
532
613
  };
533
614
  }
534
- /** The pipeline for one request: resolve the change sets, map each one's impact; the knowledge update, the runs, the fixer and the persistence follow. */
615
+ /** The pipeline for one request: resolve the change sets; for each: the run, the impact, the knowledge update, the fixer, the plan and the persistence, in that order. */
535
616
  export async function runSync(cwd, env, request, o) {
536
617
  const ctx = await syncContext(cwd, env, o);
537
618
  const dev = devEnvironment(ctx.product, o.env);
@@ -542,10 +623,27 @@ export async function runSync(cwd, env, request, o) {
542
623
  const outcomes = [];
543
624
  const lines = [];
544
625
  const out = ctx.out;
626
+ /** One { text } post to notify.webhookEnv, redacted; a missing variable or a channel that is down is one line, never a failure. */
627
+ const notify = async (text) => {
628
+ const n = ctx.config.notify;
629
+ if (n === undefined)
630
+ return;
631
+ const url = env[n.webhookEnv];
632
+ if (!url) {
633
+ ctx.log(`notify: ${n.webhookEnv} is not set; nothing sent`);
634
+ return;
635
+ }
636
+ const error = await postText(url, redact(text));
637
+ ctx.out(error === undefined ? "notify: sent" : redact(`notify: not sent: ${error}`));
638
+ };
545
639
  ctx.out = (line) => {
546
640
  lines.push(line);
547
641
  out(line);
548
642
  };
643
+ const named = o.env === undefined ? undefined : selectEnvironment(ctx.product, o.env);
644
+ if (o.suite === true && named === undefined)
645
+ throw new Error("--suite needs --env <name>, the environment the deploy landed on");
646
+ const suite = o.suite === true ? named : undefined;
549
647
  for (const r of await resolveChangeSets(ctx, request, o.dryRun === true)) {
550
648
  lines.length = 0;
551
649
  for (const line of describeLines(r.changeSet))
@@ -562,10 +660,17 @@ export async function runSync(cwd, env, request, o) {
562
660
  const transcript = await openTranscript(join(cwd, RUNS_DIR), transcriptName("sync", ctx.now()), { redact });
563
661
  await transcript.append({ kind: "sync", changeSet: { ...r.changeSet, files: r.changeSet.files.map((f) => ({ path: f.path, status: f.status })) }, impact: relative(cwd, file) });
564
662
  const outcome = { changeSet: r.changeSet, impact, impactFile: file, cases: [], planned: [], modelCalls: 0 };
565
- const run = { budget, transcript, redact, ...(dev === undefined ? {} : { dev }), ...(production === undefined ? {} : { production }) };
663
+ const run = { budget, transcript, redact, runCalls: 0, ...(dev === undefined ? {} : { dev }), ...(production === undefined ? {} : { production }) };
664
+ let failed = [];
566
665
  try {
567
- outcome.knowledge = await updateKnowledge(ctx, r, impact, run);
568
- await runImpacted(ctx, r, impact, outcome.knowledge, run, outcome);
666
+ failed = await runCases(ctx, impact, run, suite, outcome);
667
+ if (failed.length > 0) {
668
+ const healer = failed.some((f) => explores(f.where, env)) ? "healer launched" : `no healer: ${[...new Set(failed.filter((f) => !explores(f.where, env)).map((f) => f.where.name))].join(", ")} does not explore`;
669
+ await notify(failuresText(r.changeSet, failed, outcome.cases.filter((c) => !["not-run", "skipped"].includes(c.outcome)).length, healer));
670
+ }
671
+ outcome.knowledge = await updateKnowledge(ctx, r, impact, run, named);
672
+ await fixFailed(ctx, r, impact, outcome.knowledge, run, failed, outcome.cases);
673
+ await updatePlan(ctx, outcome.knowledge, run, outcome);
569
674
  if (r.eventId !== undefined)
570
675
  await removePendingChange(cwd, r.eventId);
571
676
  }
@@ -574,8 +679,10 @@ export async function runSync(cwd, env, request, o) {
574
679
  throw err;
575
680
  outcome.stopped = err.message;
576
681
  }
577
- outcome.modelCalls = budget.used;
578
- outcome.summary = summarize(outcome, ctx.now().toISOString(), limit);
682
+ const order = new Map(impact.cases.map((c, i) => [c.id, i]));
683
+ outcome.cases.sort((a, b) => (order.get(a.id) ?? Infinity) - (order.get(b.id) ?? Infinity));
684
+ outcome.modelCalls = budget.used + run.runCalls;
685
+ outcome.summary = summarize(outcome, ctx.now().toISOString(), limit, run.runCalls, suite);
579
686
  ctx.out(testsLine(outcome));
580
687
  try {
581
688
  ctx.out(`coverage: ${(await writeCoverageFile(cwd)).line}`);
@@ -600,6 +707,9 @@ export async function runSync(cwd, env, request, o) {
600
707
  if (persisted.error !== undefined)
601
708
  throw new Error(`persist failed: ${persisted.error}`);
602
709
  }
710
+ // a persist error threw above: the CI job fails and says why, so the outcome message is skipped
711
+ if (failed.length > 0)
712
+ await notify(outcomeText(outcome, failed));
603
713
  outcomes.push(outcome);
604
714
  }
605
715
  return outcomes;