omniqa 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -1
- package/dist/.tsbuildinfo +1 -1
- package/dist/auth.d.ts +17 -0
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +29 -1
- package/dist/auth.js.map +1 -1
- package/dist/cli.js +18 -5
- package/dist/cli.js.map +1 -1
- package/dist/commands/chat.js +1 -1
- package/dist/commands/chat.js.map +1 -1
- package/dist/commands/crawl.d.ts.map +1 -1
- package/dist/commands/crawl.js +17 -5
- package/dist/commands/crawl.js.map +1 -1
- package/dist/commands/doctor.d.ts +23 -0
- package/dist/commands/doctor.d.ts.map +1 -1
- package/dist/commands/doctor.js +152 -6
- package/dist/commands/doctor.js.map +1 -1
- package/dist/commands/generate.js +1 -1
- package/dist/commands/generate.js.map +1 -1
- package/dist/commands/init.d.ts +3 -1
- package/dist/commands/init.d.ts.map +1 -1
- package/dist/commands/init.js +217 -50
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/knowledge.d.ts +61 -1
- package/dist/commands/knowledge.d.ts.map +1 -1
- package/dist/commands/knowledge.js +232 -50
- package/dist/commands/knowledge.js.map +1 -1
- package/dist/commands/persist.d.ts +6 -0
- package/dist/commands/persist.d.ts.map +1 -1
- package/dist/commands/persist.js +9 -2
- package/dist/commands/persist.js.map +1 -1
- package/dist/commands/plan-add.d.ts.map +1 -1
- package/dist/commands/plan-add.js +13 -3
- package/dist/commands/plan-add.js.map +1 -1
- package/dist/commands/plan-review.d.ts +6 -2
- package/dist/commands/plan-review.d.ts.map +1 -1
- package/dist/commands/plan-review.js +16 -5
- package/dist/commands/plan-review.js.map +1 -1
- package/dist/commands/plan.d.ts +15 -0
- package/dist/commands/plan.d.ts.map +1 -1
- package/dist/commands/plan.js +103 -32
- package/dist/commands/plan.js.map +1 -1
- package/dist/commands/reindex.d.ts +1 -1
- package/dist/commands/reindex.d.ts.map +1 -1
- package/dist/commands/reindex.js +15 -1
- package/dist/commands/reindex.js.map +1 -1
- package/dist/commands/security.d.ts +4 -2
- package/dist/commands/security.d.ts.map +1 -1
- package/dist/commands/security.js +18 -7
- package/dist/commands/security.js.map +1 -1
- package/dist/commands/serve.d.ts +2 -0
- package/dist/commands/serve.d.ts.map +1 -1
- package/dist/commands/serve.js +126 -30
- package/dist/commands/serve.js.map +1 -1
- package/dist/commands/sync.d.ts.map +1 -1
- package/dist/commands/sync.js +19 -2
- package/dist/commands/sync.js.map +1 -1
- package/dist/commands/test.d.ts +5 -1
- package/dist/commands/test.d.ts.map +1 -1
- package/dist/commands/test.js +106 -6
- package/dist/commands/test.js.map +1 -1
- package/dist/commands/write.d.ts +8 -2
- package/dist/commands/write.d.ts.map +1 -1
- package/dist/commands/write.js +101 -18
- package/dist/commands/write.js.map +1 -1
- package/dist/devices.d.ts +3 -1
- package/dist/devices.d.ts.map +1 -1
- package/dist/devices.js +5 -3
- package/dist/devices.js.map +1 -1
- package/dist/fixer.d.ts.map +1 -1
- package/dist/fixer.js +19 -9
- package/dist/fixer.js.map +1 -1
- package/dist/jobs.d.ts +6 -0
- package/dist/jobs.d.ts.map +1 -1
- package/dist/jobs.js +10 -1
- package/dist/jobs.js.map +1 -1
- package/dist/mcp/server.d.ts +1 -1
- package/dist/mcp/server.d.ts.map +1 -1
- package/dist/mcp/server.js +1 -1
- package/dist/mcp/server.js.map +1 -1
- package/dist/mcp/session.d.ts.map +1 -1
- package/dist/mcp/session.js +7 -7
- package/dist/mcp/session.js.map +1 -1
- package/dist/mcp/tools.d.ts.map +1 -1
- package/dist/mcp/tools.js +9 -6
- package/dist/mcp/tools.js.map +1 -1
- package/dist/page.d.ts +1 -1
- package/dist/page.d.ts.map +1 -1
- package/dist/page.js +11 -0
- package/dist/page.js.map +1 -1
- package/dist/planner.d.ts.map +1 -1
- package/dist/planner.js +8 -4
- package/dist/planner.js.map +1 -1
- package/dist/prompt.d.ts +7 -1
- package/dist/prompt.d.ts.map +1 -1
- package/dist/prompt.js +23 -4
- package/dist/prompt.js.map +1 -1
- package/dist/qa.d.ts +7 -0
- package/dist/qa.d.ts.map +1 -1
- package/dist/qa.js +34 -9
- package/dist/qa.js.map +1 -1
- package/dist/runner.d.ts +7 -2
- package/dist/runner.d.ts.map +1 -1
- package/dist/runner.js +28 -8
- package/dist/runner.js.map +1 -1
- package/dist/security-checks.d.ts +46 -4
- package/dist/security-checks.d.ts.map +1 -1
- package/dist/security-checks.js +114 -39
- package/dist/security-checks.js.map +1 -1
- package/dist/security-gates.d.ts +64 -7
- package/dist/security-gates.d.ts.map +1 -1
- package/dist/security-gates.js +125 -16
- package/dist/security-gates.js.map +1 -1
- package/dist/security-mask.d.ts +35 -0
- package/dist/security-mask.d.ts.map +1 -0
- package/dist/security-mask.js +104 -0
- package/dist/security-mask.js.map +1 -0
- package/dist/security-report.d.ts +93 -2
- package/dist/security-report.d.ts.map +1 -1
- package/dist/security-report.js +153 -14
- package/dist/security-report.js.map +1 -1
- package/dist/security-request.d.ts +27 -4
- package/dist/security-request.d.ts.map +1 -1
- package/dist/security-request.js +56 -14
- package/dist/security-request.js.map +1 -1
- package/dist/security-research.d.ts +132 -0
- package/dist/security-research.d.ts.map +1 -0
- package/dist/security-research.js +599 -0
- package/dist/security-research.js.map +1 -0
- package/dist/security-scan.d.ts +98 -0
- package/dist/security-scan.d.ts.map +1 -0
- package/dist/security-scan.js +463 -0
- package/dist/security-scan.js.map +1 -0
- package/dist/security.d.ts +18 -4
- package/dist/security.d.ts.map +1 -1
- package/dist/security.js +181 -37
- package/dist/security.js.map +1 -1
- package/dist/sync.d.ts +55 -3
- package/dist/sync.d.ts.map +1 -1
- package/dist/sync.js +403 -54
- package/dist/sync.js.map +1 -1
- package/dist/test.d.ts +26 -2
- package/dist/test.d.ts.map +1 -1
- package/dist/test.js +152 -29
- package/dist/test.js.map +1 -1
- package/dist/writer.d.ts +29 -10
- package/dist/writer.d.ts.map +1 -1
- package/dist/writer.js +116 -36
- package/dist/writer.js.map +1 -1
- package/package.json +7 -7
- package/skills/knowledge.md +1 -1
- package/skills/security-research.md +57 -0
- package/skills/security-review.md +27 -0
- package/skills/onboarding.md +0 -5
package/dist/sync.js
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
1
2
|
import { existsSync } from "node:fs";
|
|
2
3
|
import { copyFile, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
3
4
|
import { tmpdir } from "node:os";
|
|
4
5
|
import { join, relative, resolve } from "node:path";
|
|
5
|
-
import { budgeted, BudgetExceeded, changelogGitEnv, changeSetLines, createModels, describeChangeSet, git, ingest, KNOWLEDGE_DIR, knowledgeCommit, localChangeSet, loadConfig, loadProduct, mapImpact, mergeEndpoints, openTranscript, parseEndpoints, parseRemote, parseStaticSections, planPath, postText, readCrawl, readPlan, readState, README_FILE, readmeHasBlock, recordKnowledge, recordLastSeen, recordRun, recordSync, redactor, updateSync, removePendingChange, renderImpact, renderProposals, renderReview, repoDirsOf, repoNameOf, resolveEnvironmentUrl, RUNS_DIR, selectEnvironment, schemeCounts, secretValues, stampKnowledge, stampMaps, STATE_FILE, tierEnvironment, transcriptName, withReview, writeCoverageFile, writePlan } from "@omniqa/core";
|
|
6
|
-
import { crawlSite, createNetMap, launchBrowser, writeNetMap } from "@omniqa/web";
|
|
6
|
+
import { budgeted, carriedProposals, BudgetExceeded, changedPaths, changelogGitEnv, changeSetLines, createModels, currentBranch, defaultBranch, describeChangeSet, git, ingest, HOSTS_FILE, KNOWLEDGE_DIR, KNOWLEDGE_STAMP, knowledgeCommit, localChangeSet, loadConfig, loadProduct, mapImpact, mergeEndpoints, openTranscript, parseEndpoints, parseRemote, parseStaticSections, planPath, postText, pushSynced, readCrawl, readPlan, readState, readSynced, README_FILE, RECORDINGS_DIR, readmeHasBlock, recordKnowledge, recordLastSeen, recordOwedCrawl, recordRun, recordSync, redactor, updateSync, removePendingChange, renderImpact, renderProposals, remoteBranch, renderReview, repoDirsOf, repoNameOf, resolveEnvironmentUrl, RUNS_DIR, selectEnvironment, schemeCounts, secretValues, SPEC_FILE, stampKnowledge, stampMaps, STATE_FILE, syncedRef, takeBranch, tierEnvironment, transcriptName, withReview, writeCoverageFile, writePlan, productSecrets } from "@omniqa/core";
|
|
7
|
+
import { contextOptionsFor, crawlSite, createNetMap, launchBrowser, writeNetMap } from "@omniqa/web";
|
|
7
8
|
import { createGitHub } from "@omniqa/github";
|
|
8
9
|
import { createGitLab } from "@omniqa/gitlab";
|
|
9
|
-
import { staticLine } from "./commands/knowledge.js";
|
|
10
|
+
import { gatherHosts, staticLine } from "./commands/knowledge.js";
|
|
10
11
|
import { persistOutcome, selectGitHost } from "./commands/persist.js";
|
|
11
12
|
import { indexProject } from "./commands/reindex.js";
|
|
12
13
|
import { persistedText, syncLine } from "./commands/status.js";
|
|
@@ -46,8 +47,10 @@ export function changeSource(ctx, host) {
|
|
|
46
47
|
const knows = async (repoDir, sha) => git(repoDir, ["cat-file", "-e", `${sha}^{commit}`]).then(() => true, () => false);
|
|
47
48
|
/** The commit of `name` the knowledge base describes: `state.knowledge.commit` is `name sha[, name sha]` (knowledgeCommit). */
|
|
48
49
|
const knowledgeCommitOf = (state, name) => state.knowledge?.commit?.split(", ").map((part) => part.split(" ")).find(([n]) => n === name)?.[1];
|
|
50
|
+
/** The last synced commit of a repository: what the remote's marker says where it is the later of the two (runSync decides), else state.json. */
|
|
51
|
+
const seenCommit = (synced, state, name) => (synced !== undefined && Object.hasOwn(synced, name) ? synced[name] : undefined) ?? state.changelog?.lastSeen[name];
|
|
49
52
|
/** Where a sync without --from starts: the last synced commit of the repository, else the commit its knowledge was built from. */
|
|
50
|
-
export const lastSynced = (state, name) => state
|
|
53
|
+
export const lastSynced = (state, name, synced) => seenCommit(synced, state, name) ?? knowledgeCommitOf(state, name);
|
|
51
54
|
/** The directory of the repository named by a host path (`o/r`) or a name, when product.yaml has it. */
|
|
52
55
|
const localDir = (ctx, repo) => ctx.repoDirs[repoNameOf(repo)];
|
|
53
56
|
/** A change set for a webhook event: from the repository on disk when it has both commits, else from the host's compare. */
|
|
@@ -57,7 +60,7 @@ async function fromEvent(ctx, event) {
|
|
|
57
60
|
const after = event.after ?? event.ref?.replace(/^refs\/tags\//, "");
|
|
58
61
|
if (after === undefined)
|
|
59
62
|
throw new Error(`the ${event.kind} on ${event.repo} names no commit or tag to sync to`);
|
|
60
|
-
const before = event.before ?? state
|
|
63
|
+
const before = event.before ?? seenCommit(ctx.synced, state, name);
|
|
61
64
|
if (before === undefined)
|
|
62
65
|
throw new Error(`no earlier commit known for ${event.repo}: the ${event.kind} names none and no sync recorded one; run omniqa sync --from <sha> --to ${after} once`);
|
|
63
66
|
const dir = localDir(ctx, event.repo);
|
|
@@ -87,7 +90,7 @@ export async function resolveChangeSets(ctx, request, dryRun = false) {
|
|
|
87
90
|
throw new Error(names.length === 0 ? "no repository to compare: add repos: to product.yaml or changelog.repo to omniqa.config.ts" : `name the repository with --repo (${names.join(", ")})`);
|
|
88
91
|
const name = repoNameOf(repo);
|
|
89
92
|
// ponytail: a lastSeen ahead of --to (a poll recorded the head after a deploy) compares backwards; check ancestry with merge-base when it bites.
|
|
90
|
-
const from = request.from ?? lastSynced(await readState(ctx.cwd), name);
|
|
93
|
+
const from = request.from ?? lastSynced(await readState(ctx.cwd), name, ctx.synced);
|
|
91
94
|
if (from === undefined)
|
|
92
95
|
throw new Error(`no commit to compare from for ${name}: nothing synced yet and no knowledge built from it; pass --from <sha>`);
|
|
93
96
|
const dir = localDir(ctx, repo);
|
|
@@ -117,7 +120,7 @@ export async function resolveChangeSets(ctx, request, dryRun = false) {
|
|
|
117
120
|
const head = await source.branchHead(changelog.repo, changelog.branch);
|
|
118
121
|
const name = repoNameOf(changelog.repo);
|
|
119
122
|
const state = await readState(ctx.cwd);
|
|
120
|
-
const seen = state
|
|
123
|
+
const seen = seenCommit(ctx.synced, state, name);
|
|
121
124
|
if (seen === undefined) {
|
|
122
125
|
await recordLastSeen(ctx.cwd, name, head.sha, ctx.now());
|
|
123
126
|
ctx.out(`poll: ${changelog.repo} ${changelog.branch} at ${short(head.sha)}: first poll, recorded the head; nothing to compare`);
|
|
@@ -184,6 +187,17 @@ export function impactLine(impact, file, cwd) {
|
|
|
184
187
|
...(impact.cases.length === 0 ? [] : [` cases: ${impact.cases.map((c) => `${c.id} (${c.confidence})`).join(", ")}; exact ${count("exact")}, pattern ${count("pattern")}, guess ${count("guess")}`]),
|
|
185
188
|
];
|
|
186
189
|
}
|
|
190
|
+
/**
|
|
191
|
+
* What a change itself added to the rows of a repository's map, or took out of them: a row that is new, or gone, in a file
|
|
192
|
+
* the change touches. The map in a checkout is as old as the last sync that committed (one that changes no test commits
|
|
193
|
+
* none), so its difference with the product also holds what earlier changes did, and what an earlier sync already acted on:
|
|
194
|
+
* the same case skipped again and the same route proposed again, at every deploy until a pull request is merged.
|
|
195
|
+
*/
|
|
196
|
+
export function ownChanges(old, now, key, touched) {
|
|
197
|
+
const [before, after] = [new Set(old.map(key)), new Set(now.map(key))];
|
|
198
|
+
const own = (row) => touched === undefined || touched.has(row.file);
|
|
199
|
+
return { added: [...new Set(now.filter((row) => !before.has(key(row)) && own(row)).map(key))], removed: [...new Set(old.filter((row) => !after.has(key(row)) && own(row)).map(key))] };
|
|
200
|
+
}
|
|
187
201
|
/** The impacted knowledge rewritten: the changed repository's map (its overview kept), a crawl of the impacted pages as every actor with their endpoint rows merged, on `preferred` (the --env environment, M6) when it is given and explores, the cross-reference, the scheme's impacted statements revised by the analyst behind the reviewer, the index and the knowledge commit. */
|
|
188
202
|
export async function updateKnowledge(ctx, r, impact, run, preferred) {
|
|
189
203
|
const knowledgeDir = join(ctx.cwd, KNOWLEDGE_DIR);
|
|
@@ -210,12 +224,13 @@ export async function updateKnowledge(ctx, r, impact, run, preferred) {
|
|
|
210
224
|
maps = 1;
|
|
211
225
|
// both sides read through the map's rows, so a row the parser cannot read (a path with parentheses) counts on neither
|
|
212
226
|
const facts = parsed ?? parseStaticSections(await readFile(result.file, "utf8"));
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
227
|
+
// The files this change touched, a renamed one under both names; undefined where the list is not whole (GitHub's
|
|
228
|
+
// compare stops at 300 files), and every difference with the old map counts, as it did.
|
|
229
|
+
const touched = r.changeSet.host === "local" || r.changeSet.files.length < 300 ? new Set(r.changeSet.files.flatMap((f) => (f.from === undefined ? [f.path] : [f.path, f.from]))) : undefined;
|
|
230
|
+
const routes = ownChanges(oldFacts.routes, facts.routes, (x) => x.value, touched);
|
|
231
|
+
additions.routes = routes.added;
|
|
232
|
+
removedRoutes = routes.removed;
|
|
233
|
+
additions.endpoints = ownChanges(oldFacts.handlers, facts.handlers, (h) => `${h.method} ${h.value}`, touched).added;
|
|
219
234
|
counts.routes = facts.routes.length;
|
|
220
235
|
counts.handlers = facts.handlers.length;
|
|
221
236
|
counts.models = facts.models.length;
|
|
@@ -225,7 +240,12 @@ export async function updateKnowledge(ctx, r, impact, run, preferred) {
|
|
|
225
240
|
// 2. the impacted pages, re-crawled as a guest and as each account into their own directories; their endpoint rows replace the old ones
|
|
226
241
|
const known = await readCrawl(join(knowledgeDir, "crawl")).catch(() => []);
|
|
227
242
|
// a route with a placeholder is opened at the path the crawl once used for it
|
|
228
|
-
const
|
|
243
|
+
const impacted = impact.pages.toSorted((a, b) => RANK[b.confidence] - RANK[a.confidence]).map((p) => known.find((k) => k.route === p.name.route)?.path ?? p.name.route).filter((p) => !/[{}]/.test(p));
|
|
244
|
+
// The pages an earlier sync could not crawl (its site gave no answer, an account did not sign in) are crawled with this
|
|
245
|
+
// one's: a sync moves the marker whatever its crawl did, so nothing else would crawl them before a change touches them again.
|
|
246
|
+
const owed = ctx.owed ?? [];
|
|
247
|
+
const targets = [...new Set([...impacted, ...owed])].slice(0, MAX_TARGET_PAGES);
|
|
248
|
+
let owes = owed;
|
|
229
249
|
const pages = [];
|
|
230
250
|
// The crawl follows --env (M6) unless that is a production environment with explore off: the knowledge build's rule, a replay-only production is never crawled; then the dev tier, else a production environment that explores.
|
|
231
251
|
const fallback = run.dev ?? (run.production?.environment.explore === true ? run.production : undefined);
|
|
@@ -238,44 +258,71 @@ export async function updateKnowledge(ctx, r, impact, run, preferred) {
|
|
|
238
258
|
ctx.out("crawl: skipped (no impacted page)");
|
|
239
259
|
else {
|
|
240
260
|
const { name: envName, environment } = crawlOn;
|
|
261
|
+
const earlier = targets.filter((page) => !impacted.includes(page));
|
|
262
|
+
if (earlier.length > 0)
|
|
263
|
+
ctx.out(`crawl: ${plural(earlier.length, "page")} an earlier sync could not crawl: ${earlier.join(", ")}`);
|
|
241
264
|
const baseUrl = resolveEnvironmentUrl(environment, ctx.env.OMNIQA_BRANCH ? { branch: ctx.env.OMNIQA_BRANCH } : {});
|
|
242
265
|
const origin = new URL(baseUrl).origin;
|
|
243
266
|
const capture = createNetMap(baseUrl);
|
|
244
267
|
const accounts = Object.entries(ctx.product.accounts);
|
|
245
268
|
const passes = [
|
|
246
269
|
{ as: "guest", outDir: accounts.length === 0 ? join(knowledgeDir, "crawl") : join(knowledgeDir, "crawl", "guest") },
|
|
247
|
-
...accounts.map(([n, fields], i) => ({ as: n, outDir: i === 0 ? join(knowledgeDir, "crawl") : join(knowledgeDir, "crawl", n), account: { name: n, fields, loginPath: ctx.product.crawl.loginPath } })),
|
|
270
|
+
...accounts.map(([n, fields], i) => ({ as: n, outDir: i === 0 ? join(knowledgeDir, "crawl") : join(knowledgeDir, "crawl", n), account: { name: n, fields, loginPath: ctx.product.crawl.loginPath, ...(ctx.product.crawl.signedIn === undefined ? {} : { signedIn: ctx.product.crawl.signedIn }) } })),
|
|
248
271
|
];
|
|
249
272
|
const tmp = await mkdtemp(join(tmpdir(), "omniqa-sync-crawl-"));
|
|
250
273
|
const crawled = new Set();
|
|
274
|
+
/** The actors whose walk was not made (the site did not answer, an account did not sign in), each with why. */
|
|
275
|
+
const notWalked = new Map();
|
|
251
276
|
let browser;
|
|
252
277
|
try {
|
|
253
278
|
browser = await launchBrowser();
|
|
279
|
+
const walks = [];
|
|
254
280
|
for (const pass of passes) {
|
|
255
|
-
const page = await browser.newPage();
|
|
281
|
+
const page = await browser.newPage(contextOptionsFor(environment, baseUrl));
|
|
256
282
|
const outDir = join(tmp, pass.as);
|
|
257
283
|
try {
|
|
258
284
|
// ponytail: the first target is the base url and the rest are seeds at depth 1 with the budget set to their count, so nothing else is followed; more than five targets under one parent path hit the crawler's sibling cap and are dropped.
|
|
259
|
-
|
|
285
|
+
// The targets are pages the change touches, not the site's start page: the first one's error (a page the deploy removed, a member's page asked for as a guest) is its own.
|
|
286
|
+
const walk = await crawlSite(page, { baseUrl: `${origin}${targets[0]}`, outDir, maxPages: targets.length, maxDepth: 1, exclude: ctx.product.crawl.exclude, deny: environment.deny, seeds: targets.slice(1), listed: true, seed: environment, capture, stamp, redact: run.redact, ...(pass.account === undefined ? {} : { account: pass.account }), log: (l) => ctx.log(`crawl ${envName} as ${pass.as}: ${l}`) });
|
|
287
|
+
if (walk.unreachable !== undefined)
|
|
288
|
+
notWalked.set(pass.as, walk.unreachable);
|
|
289
|
+
walks.push(walk);
|
|
260
290
|
}
|
|
261
291
|
finally {
|
|
262
292
|
await page.close();
|
|
263
293
|
}
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
294
|
+
}
|
|
295
|
+
// An actor whose walk was not made (the site gave no answer, an account did not sign in): what the others saw of these
|
|
296
|
+
// pages would replace the endpoint rows only that actor's walk gave, as a guest's walk in an account's place would its
|
|
297
|
+
// pages. Nothing of the crawl is rewritten, as the knowledge build rewrites nothing then, and the sync goes on.
|
|
298
|
+
if (notWalked.size > 0) {
|
|
299
|
+
owes = [...new Set([...owed, ...targets])];
|
|
300
|
+
ctx.out(`crawl ${envName}: not made: ${[...notWalked].map(([as, why]) => `${as} (${why})`).join(", ")}; the pages and the endpoints of the last crawl stay, and the next sync crawls ${targets.join(", ")}`);
|
|
301
|
+
}
|
|
302
|
+
else {
|
|
303
|
+
owes = owed.filter((page) => !targets.includes(page));
|
|
304
|
+
for (const pass of passes) {
|
|
305
|
+
const outDir = join(tmp, pass.as);
|
|
306
|
+
for (const p of await readCrawl(outDir).catch(() => [])) {
|
|
307
|
+
await mkdir(pass.outDir, { recursive: true });
|
|
308
|
+
await copyFile(join(outDir, p.file), join(pass.outDir, p.file));
|
|
309
|
+
crawled.add(p.route);
|
|
310
|
+
}
|
|
268
311
|
}
|
|
312
|
+
const tmpKnowledge = join(tmp, "knowledge");
|
|
313
|
+
const net = await writeNetMap(capture, tmpKnowledge, stamp, { log: ctx.log, redact: run.redact });
|
|
314
|
+
const fresh = parseEndpoints(await readFile(join(tmpKnowledge, "api", "endpoints.md"), "utf8").catch(() => ""));
|
|
315
|
+
const previous = await readFile(join(knowledgeDir, "api", "endpoints.md"), "utf8").catch(() => "");
|
|
316
|
+
await mkdir(join(knowledgeDir, "api"), { recursive: true });
|
|
317
|
+
await writeFile(join(knowledgeDir, "api", "endpoints.md"), run.redact(stampKnowledge(mergeEndpoints(previous, fresh, [...crawled]), stamp)));
|
|
318
|
+
pages.push(...crawled);
|
|
319
|
+
const rows = fresh.filter((e) => e.pages.length === 0 || e.pages.some((p) => crawled.has(p))).length;
|
|
320
|
+
ctx.out(`crawl ${envName} as ${passes.map((p) => p.as).join(", ")}: ${plural(pages.length, "page")} (${pages.join(", ")}): ${net.requests} requests, ${plural(rows, "endpoint row")} merged`);
|
|
269
321
|
}
|
|
270
|
-
|
|
271
|
-
const
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
await mkdir(join(knowledgeDir, "api"), { recursive: true });
|
|
275
|
-
await writeFile(join(knowledgeDir, "api", "endpoints.md"), run.redact(stampKnowledge(mergeEndpoints(previous, fresh, [...crawled]), stamp)));
|
|
276
|
-
pages.push(...crawled);
|
|
277
|
-
const rows = fresh.filter((e) => e.pages.length === 0 || e.pages.some((p) => crawled.has(p))).length;
|
|
278
|
-
ctx.out(`crawl ${envName} as ${passes.map((p) => p.as).join(", ")}: ${plural(pages.length, "page")} (${pages.join(", ")}): ${net.requests} requests, ${plural(rows, "endpoint row")} merged`);
|
|
322
|
+
// A host these pages link to or ask that the file does not name yet: added switched off, and said.
|
|
323
|
+
const hosts = await gatherHosts(ctx.cwd, baseUrl, walks);
|
|
324
|
+
if (hosts !== undefined)
|
|
325
|
+
ctx.out(hosts);
|
|
279
326
|
}
|
|
280
327
|
finally {
|
|
281
328
|
await browser?.close();
|
|
@@ -352,7 +399,7 @@ export async function updateKnowledge(ctx, r, impact, run, preferred) {
|
|
|
352
399
|
const builtAt = ctx.now().toISOString();
|
|
353
400
|
await recordKnowledge(ctx.cwd, { commit, builtAt, ...(schemeState === undefined ? {} : { scheme: schemeState }), modelCalls: run.budget.used, counts });
|
|
354
401
|
ctx.out(`knowledge commit: ${commit} built ${builtAt}; ${plural(run.budget.used, "model call")} of ${run.budget.limit}`);
|
|
355
|
-
return { maps, pages, scheme, additions, removedRoutes, commit, stamp };
|
|
402
|
+
return { maps, pages, scheme, additions, removedRoutes, commit, stamp, owed: owes };
|
|
356
403
|
}
|
|
357
404
|
const OUTCOME_WORDS = { pass: "pass", healed: "healed", "behaviour-changed": "behaviour changed", unsure: "unsure", failed: "failed", fixed: "fixed", regression: "regression", "needs-human": "needs a human", skipped: "skipped", "not-run": "not run" };
|
|
358
405
|
/** Each test of a JSON report by its title: pass, healed (a heal or re-verify line), behaviour changed, unsure or failed, with the first line that says why and its model calls. */
|
|
@@ -473,7 +520,7 @@ export async function runCases(ctx, impact, run, suite, into) {
|
|
|
473
520
|
return failed;
|
|
474
521
|
}
|
|
475
522
|
/** Pass 5: every failed case goes to the fixer on the environment it failed on, with the change set and the rewritten knowledge (M4); a production environment that does not explore gets no fixer (D5), a case on a route the change removed waits for the plan pass. */
|
|
476
|
-
export async function fixFailed(ctx, r, impact, k, run, failed, cases) {
|
|
523
|
+
export async function fixFailed(ctx, r, impact, k, run, failed, cases, /** The change set is wider than the one the impact was mapped for: the fixer is shown every file's hunks. */ wider = false) {
|
|
477
524
|
const fixable = failed.filter((f) => !f.selected.case.routes.some((route) => k.removedRoutes.includes(route)));
|
|
478
525
|
if (fixable.length === 0)
|
|
479
526
|
return;
|
|
@@ -494,7 +541,7 @@ export async function fixFailed(ctx, r, impact, k, run, failed, cases) {
|
|
|
494
541
|
}
|
|
495
542
|
continue;
|
|
496
543
|
}
|
|
497
|
-
const fixer = await createFixer({ cwd: ctx.cwd, env: ctx.env, config: ctx.config, product: ctx.product, envName: where.name, environment: where.environment, project, budget: run.budget, transcript: run.transcript, changeSet: r.changeSet, mappedFiles: impact.files.filter((f) => f.confidence !== undefined).map((f) => f.path), knowledge, out: ctx.out, log: ctx.log });
|
|
544
|
+
const fixer = await createFixer({ cwd: ctx.cwd, env: ctx.env, config: ctx.config, product: ctx.product, envName: where.name, environment: where.environment, project, budget: run.budget, transcript: run.transcript, changeSet: r.changeSet, ...(wider ? {} : { mappedFiles: impact.files.filter((f) => f.confidence !== undefined).map((f) => f.path) }), knowledge, out: ctx.out, log: ctx.log });
|
|
498
545
|
try {
|
|
499
546
|
for (const f of group) {
|
|
500
547
|
const fixed = await fixer.fix(f.selected, f.detail);
|
|
@@ -641,11 +688,174 @@ async function proposalsSection(ctx, o, named) {
|
|
|
641
688
|
request: ci === "gitlab" ? "merge request" : "pull request",
|
|
642
689
|
};
|
|
643
690
|
if (live !== undefined && (await hasWriteJob(ctx.cwd, ci)))
|
|
644
|
-
return
|
|
691
|
+
return { ...section, submit: ci === "gitlab" ? "job" : "box", env: live.name };
|
|
645
692
|
const instead = live !== undefined ? `This repository has no job that writes the chosen tests yet: run \`omniqa init --jobs\` once and push. Until then \`omniqa write --case ${wanted} --env ${live.name}\` writes them.`
|
|
646
693
|
: named !== undefined ? `No job can walk these flows on ${named.name}, which does not explore: \`omniqa write --case ${wanted}\` writes them where the dev environment is reachable.`
|
|
647
694
|
: `\`omniqa write --case ${wanted}\` writes them.`;
|
|
648
|
-
return
|
|
695
|
+
return { ...section, instead };
|
|
696
|
+
}
|
|
697
|
+
/** The branch the one pull request of a branch's syncs comes from: one per branch under test and per project folder, as the marker is. */
|
|
698
|
+
const collectBranch = (marker) => marker.replace(/^refs\/omniqa\/synced\//, "omniqa/sync/");
|
|
699
|
+
/**
|
|
700
|
+
* An earlier sync's pull request that nobody merged yet holds what it healed, fixed and wrote. A CI job's checkout has none
|
|
701
|
+
* of it and would do that work again, with the model calls it took: the sync starts from the request's files (`takeBranch`).
|
|
702
|
+
* The open request, and whether the checkout now holds all it changed; undefined where none is open. What cannot be asked or
|
|
703
|
+
* taken stops nothing: the sync then starts from the checkout's own tests, as it did before, and says so.
|
|
704
|
+
*/
|
|
705
|
+
async function startFromRequest(ctx, branch, paths) {
|
|
706
|
+
try {
|
|
707
|
+
const host = await selectGitHost(ctx.config, ctx.cwd, ctx.env);
|
|
708
|
+
const head = host === undefined ? undefined : await remoteBranch(ctx.cwd, branch, ctx.env);
|
|
709
|
+
const open = host === undefined || head === undefined ? undefined : await host.findPullRequest({ repoDir: ctx.cwd, branch });
|
|
710
|
+
if (head === undefined || open === undefined)
|
|
711
|
+
return undefined;
|
|
712
|
+
const took = await takeBranch({ repoDir: ctx.cwd, head, paths, env: ctx.env });
|
|
713
|
+
// What stays as it is here and matters to a run: a test, a recording, the plan. A work tree that stays holds its own state
|
|
714
|
+
// file and knowledge, later than the request's, and those are said of no sync.
|
|
715
|
+
const left = (took?.left ?? []).filter(reviewable);
|
|
716
|
+
if (took === undefined)
|
|
717
|
+
ctx.out(`sync: ${open.url} holds what earlier syncs changed, and this checkout does not reach the commit its branch left from: the sync starts from the checkout's own tests`);
|
|
718
|
+
else
|
|
719
|
+
ctx.out(`sync: starting from ${open.url}, which holds what earlier syncs changed: ${plural(took.taken.length, "file")} taken from ${branch}${left.length === 0 ? "" : `; ${plural(left.length, "file")} left as here (changed here, or on this branch since): ${left.join(", ")}`}`);
|
|
720
|
+
return { url: open.url, whole: took !== undefined && left.length === 0 };
|
|
721
|
+
}
|
|
722
|
+
catch (err) {
|
|
723
|
+
ctx.out(`sync: an open pull request of earlier syncs could not be looked for (${firstLine(err.message)}): the sync starts from the checkout's own tests`);
|
|
724
|
+
return undefined;
|
|
725
|
+
}
|
|
726
|
+
}
|
|
727
|
+
/**
|
|
728
|
+
* What a sync changes that someone reviews: a test, its recording, the sign-in helper, the plan. The state file, the date and
|
|
729
|
+
* the commit in the knowledge, the coverage and the README change with every sync: they go along with a commit of those, and
|
|
730
|
+
* are no commit and no pull request by themselves.
|
|
731
|
+
*/
|
|
732
|
+
const reviewable = (path) => !path.includes(`${KNOWLEDGE_DIR}/`) && (SPEC_FILE.test(path) || path.includes(`${RECORDINGS_DIR}/`) || /(^|\/)(plan\.md|_auth\.ts)$/.test(path));
|
|
733
|
+
const digest = (data) => createHash("sha256").update(data).digest("hex");
|
|
734
|
+
/**
|
|
735
|
+
* A knowledge file as it says, without what moves with every sync whatever the file says: its stamp, and in a repository's
|
|
736
|
+
* map the commit and the date it was made from and its last section, the files that commit changed.
|
|
737
|
+
*/
|
|
738
|
+
const unstamped = (text) => text
|
|
739
|
+
.replace(new RegExp(`${KNOWLEDGE_STAMP.source}\n?`, "m"), "")
|
|
740
|
+
.replace(/\n## Changes since last ingest\n[\s\S]*$/, "")
|
|
741
|
+
.replace(/^(Generated by `omniqa ingest` from .*|- commit: `[0-9a-f]+`|- date: \d{4}-\d\d-\d\d)$/gm, "")
|
|
742
|
+
.trimEnd();
|
|
743
|
+
/**
|
|
744
|
+
* The files under `paths` that differ from HEAD, each with what it holds now: as it is, and as it says (a knowledge file
|
|
745
|
+
* without its stamp). Taken when a sync starts and again when it ends: what differs between the two is what this sync
|
|
746
|
+
* changed. A work tree that stays (omniqa serve, a developer's machine) keeps the files of an earlier sync whose pull request
|
|
747
|
+
* nobody merged, and so does a checkout that started from that request: those are no reason for another commit.
|
|
748
|
+
*/
|
|
749
|
+
async function differingNow(cwd, paths) {
|
|
750
|
+
const root = await git(cwd, ["rev-parse", "--show-toplevel"]).catch(() => undefined);
|
|
751
|
+
const held = new Map();
|
|
752
|
+
if (root === undefined)
|
|
753
|
+
return held;
|
|
754
|
+
for (const path of await changedPaths(cwd, paths)) {
|
|
755
|
+
const bytes = await readFile(join(root, path)).catch(() => undefined);
|
|
756
|
+
held.set(join(root, path), bytes === undefined ? { bytes: "gone", says: "gone" } : { bytes: digest(bytes), says: path.includes(`${KNOWLEDGE_DIR}/`) ? digest(unstamped(bytes.toString("utf8"))) : digest(bytes) });
|
|
757
|
+
}
|
|
758
|
+
return held;
|
|
759
|
+
}
|
|
760
|
+
/**
|
|
761
|
+
* The change set a fixer judges a failure by, when this checkout's tests were last synced at `from`, an earlier commit than
|
|
762
|
+
* the sync starts from: an earlier sync's commit or pull request is not merged here, so a test may fail for a change that
|
|
763
|
+
* sync already followed. Shown only the newer commits, the fixer would find nothing that accounts for the failure and call
|
|
764
|
+
* it a regression.
|
|
765
|
+
*/
|
|
766
|
+
async function sinceCheckout(ctx, r, from) {
|
|
767
|
+
const { repo, after, host, kind } = r.changeSet;
|
|
768
|
+
try {
|
|
769
|
+
if (r.repoDir !== undefined && (await knows(r.repoDir, from)) && (await knows(r.repoDir, after)))
|
|
770
|
+
return { ...r, changeSet: { ...(await localChangeSet(r.repoDir, repo, from, after, kind)), host } };
|
|
771
|
+
const configured = ctx.config.changelog?.repo;
|
|
772
|
+
const entry = (ctx.product.repos ?? []).find((x) => repoNameOf(x) === repoNameOf(repo));
|
|
773
|
+
const hostRepo = configured !== undefined && repoNameOf(configured) === repoNameOf(repo) ? configured : entry === undefined ? undefined : parseRemote(entry)?.path;
|
|
774
|
+
if (hostRepo === undefined)
|
|
775
|
+
return r;
|
|
776
|
+
const c = await changeSource(ctx, host === "local" ? undefined : host).compare(hostRepo, from, after);
|
|
777
|
+
return { ...r, changeSet: { ...r.changeSet, before: from, files: c.files, commits: c.commits } };
|
|
778
|
+
}
|
|
779
|
+
catch (err) {
|
|
780
|
+
ctx.log(`fix: the changes since ${short(from)} could not be read (${firstLine(err.message)}); the fixer reads this sync's alone`);
|
|
781
|
+
return r;
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
const FULL_COMMIT = /^([0-9a-f]{40}|[0-9a-f]{64})$/i;
|
|
785
|
+
/** Two ids of one commit: equal, or one the other's short form (seven characters at least). */
|
|
786
|
+
export const sameCommit = (a, b) => Math.min(a.length, b.length) >= 7 && (a.toLowerCase().startsWith(b.toLowerCase()) || b.toLowerCase().startsWith(a.toLowerCase()));
|
|
787
|
+
/**
|
|
788
|
+
* The commit this checkout's tests were last synced at, where the marker says a sync's pull request waits; undefined where
|
|
789
|
+
* nothing waits, and the checkout is as synced as the marker (its own state file may name an older commit: a sync that changes
|
|
790
|
+
* no test commits none). `seen` is what that state file names now. The same as when the request was opened: it still waits,
|
|
791
|
+
* and the tests stand where the marker says. Another: a sync's commit was merged since, and the checkout says where it stands.
|
|
792
|
+
*/
|
|
793
|
+
export function testsStandAt(waiting, seen) {
|
|
794
|
+
if (waiting === undefined)
|
|
795
|
+
return undefined;
|
|
796
|
+
return waiting.seen === (seen ?? "") ? waiting.from : seen;
|
|
797
|
+
}
|
|
798
|
+
/**
|
|
799
|
+
* What the marker says of a waiting pull request once a sync is done, for tests that stood at `stand` when it began (from
|
|
800
|
+
* `before`, where nothing waited). A commit on the branch brings the branch everything, and so does a whole suite that passed
|
|
801
|
+
* with nothing to commit: nothing waits. A pull request leaves the tests where they stood. Nothing to commit otherwise leaves
|
|
802
|
+
* things as they were: still waiting, or not.
|
|
803
|
+
*/
|
|
804
|
+
export function waitingAfter(o) {
|
|
805
|
+
if (o.persisted === "commit")
|
|
806
|
+
return null;
|
|
807
|
+
if (o.persisted === "nothing" && (o.cleanSuite || sameCommit(o.stand, o.before)))
|
|
808
|
+
return null;
|
|
809
|
+
return { from: o.stand, seen: o.seen ?? "" };
|
|
810
|
+
}
|
|
811
|
+
/**
|
|
812
|
+
* The repositories the remote's marker answers for before this checkout's state.json: for each, the later of the two. A sync
|
|
813
|
+
* made here without --persist moved the state file's commit of its repository and no marker, so the file answers for that
|
|
814
|
+
* one; it says nothing later of the others, and one time for the whole file had it answer for all of them. A repository
|
|
815
|
+
* whose commit the file notes without a time (an earlier version wrote it) has the file's last sync or poll for it, as
|
|
816
|
+
* before. (A marker's time is whole seconds: the second it was pushed in counts as its own.)
|
|
817
|
+
* ponytail: the marker has one time for all it names, so a marker moved since for another repository answers for this one
|
|
818
|
+
* too; a time per entry in the marker's commit is the upgrade.
|
|
819
|
+
*/
|
|
820
|
+
export function markerAnswersFor(marker, changelog) {
|
|
821
|
+
const whole = Math.max(Date.parse(changelog?.lastSyncAt ?? "") || 0, Date.parse(changelog?.lastPollAt ?? "") || 0);
|
|
822
|
+
const noted = (repo) => {
|
|
823
|
+
if (changelog === undefined || !Object.hasOwn(changelog.lastSeen, repo))
|
|
824
|
+
return 0;
|
|
825
|
+
const at = changelog.seenAt !== undefined && Object.hasOwn(changelog.seenAt, repo) ? Date.parse(changelog.seenAt[repo]) : Number.NaN;
|
|
826
|
+
return Number.isNaN(at) ? whole : at;
|
|
827
|
+
};
|
|
828
|
+
return Object.keys(marker.commits).filter((repo) => Date.parse(marker.at) + 999 >= noted(repo));
|
|
829
|
+
}
|
|
830
|
+
/** A marker that did not go, and was not refused: there was nowhere to push it, or nothing to push. */
|
|
831
|
+
const notPushed = (why) => ({ pushed: false, refused: false, why });
|
|
832
|
+
/** The commit a change set starts at, by its id, where it is one or the repository on disk can say; else undefined. */
|
|
833
|
+
async function startCommit(r) {
|
|
834
|
+
const { before } = r.changeSet;
|
|
835
|
+
if (FULL_COMMIT.test(before))
|
|
836
|
+
return before;
|
|
837
|
+
if (r.repoDir === undefined)
|
|
838
|
+
return /^[0-9a-f]{7,}$/i.test(before) ? before : undefined;
|
|
839
|
+
return git(r.repoDir, ["rev-parse", "--verify", "--quiet", `${before}^{commit}`]).catch(() => undefined);
|
|
840
|
+
}
|
|
841
|
+
/**
|
|
842
|
+
* The commit a change set ends at, by its id: `after` may be the name it was asked for by (a tag, a branch, a short id), and a
|
|
843
|
+
* marker holds commits. Undefined where nothing here can tell (a name compared through the host, with no commit in between).
|
|
844
|
+
*/
|
|
845
|
+
async function endCommit(r) {
|
|
846
|
+
const { after, commits } = r.changeSet;
|
|
847
|
+
if (FULL_COMMIT.test(after))
|
|
848
|
+
return after;
|
|
849
|
+
if (r.repoDir !== undefined) {
|
|
850
|
+
const sha = await git(r.repoDir, ["rev-parse", "--verify", "--quiet", `${after}^{commit}`]).catch(() => undefined);
|
|
851
|
+
if (sha !== undefined && FULL_COMMIT.test(sha))
|
|
852
|
+
return sha;
|
|
853
|
+
}
|
|
854
|
+
// ponytail: the last commit the host listed, where it lists them all (GitHub's compare stops at 250).
|
|
855
|
+
const last = commits.at(-1)?.sha;
|
|
856
|
+
if (last !== undefined && FULL_COMMIT.test(last) && commits.length < 250)
|
|
857
|
+
return last;
|
|
858
|
+
return /^[0-9a-f]{7,}$/i.test(after) ? after : undefined;
|
|
649
859
|
}
|
|
650
860
|
function summarize(o, at, budget, runCalls, suite) {
|
|
651
861
|
const ran = o.cases.filter((c) => !["not-run", "skipped"].includes(c.outcome));
|
|
@@ -678,7 +888,7 @@ export async function runSync(cwd, env, request, o) {
|
|
|
678
888
|
const dev = devEnvironment(ctx.product, o.env);
|
|
679
889
|
const production = tierEnvironment(ctx.product, "production", o.env);
|
|
680
890
|
const limit = o.budget ?? DEFAULT_SYNC_BUDGET;
|
|
681
|
-
const accountValues =
|
|
891
|
+
const accountValues = productSecrets(ctx.product);
|
|
682
892
|
const redact = redactor([...secretValues(ctx.config, env), ...accountValues]);
|
|
683
893
|
const outcomes = [];
|
|
684
894
|
const lines = [];
|
|
@@ -694,15 +904,95 @@ export async function runSync(cwd, env, request, o) {
|
|
|
694
904
|
const error = await postText(url, redact(text));
|
|
695
905
|
ctx.out(error === undefined ? "notify: sent" : redact(`notify: not sent: ${error}`));
|
|
696
906
|
};
|
|
907
|
+
// Every line is also the sync's report, and with it a commit's or a pull request's text: a test's error and a fixer's
|
|
908
|
+
// reasoning quote what a page showed.
|
|
697
909
|
ctx.out = (line) => {
|
|
698
|
-
|
|
699
|
-
|
|
910
|
+
const safe = redact(line);
|
|
911
|
+
lines.push(safe);
|
|
912
|
+
out(safe);
|
|
700
913
|
};
|
|
701
914
|
// --suite without --env: the deploy landed on the production environment, the one the production-only cases run on; a product without one (init's default: a dev environment only) runs its suite on its default environment
|
|
702
915
|
const named = o.env !== undefined ? selectEnvironment(ctx.product, o.env) : o.suite === true ? (production ?? selectEnvironment(ctx.product)) : undefined;
|
|
703
916
|
const suite = o.suite === true ? named : undefined;
|
|
704
|
-
|
|
917
|
+
// Where the last sync ended is asked of the remote first: a checkout only knows what was merged into its branch. The marker is
|
|
918
|
+
// this branch's and this folder's. A run that names its own --from needs none.
|
|
919
|
+
const branch = await currentBranch(cwd, env).catch(() => undefined);
|
|
920
|
+
const folder = await git(cwd, ["rev-parse", "--show-prefix"], env).catch(() => undefined);
|
|
921
|
+
const ref = branch === undefined || folder === undefined ? undefined : syncedRef(branch, folder);
|
|
922
|
+
const persistPaths = async () => [KNOWLEDGE_DIR, ctx.config.testDir, STATE_FILE, HOSTS_FILE, ...((await readmeHasBlock(cwd)) ? [README_FILE] : [])];
|
|
923
|
+
const began = await readState(cwd);
|
|
924
|
+
ctx.owed = began.changelog?.owedCrawl ?? [];
|
|
925
|
+
// What the branch itself says: the state file as committed, which a merge moves. A work tree that stays holds the file of the
|
|
926
|
+
// sync before, and with it the files of a pull request nobody merged: taken for the branch, it would say that request was merged.
|
|
927
|
+
const committed = folder === undefined ? began : await git(cwd, ["show", `HEAD:${folder}${STATE_FILE}`], env).then((text) => JSON.parse(text), () => began);
|
|
928
|
+
/** The default branch's marker, where this branch has none of its own yet: what the sync starts from, and what the branch's first marker starts from. */
|
|
929
|
+
let inherited;
|
|
930
|
+
if (ref !== undefined && !(request.kind === "manual" && request.from !== undefined)) {
|
|
931
|
+
// A marker that is there and cannot be read: a sync that persists stops, since it would do an earlier sync's work again and
|
|
932
|
+
// then write over what it could not read; one that only looks goes on from what its checkout says, and says so.
|
|
933
|
+
let unread = false;
|
|
934
|
+
const own = await readSynced(cwd, ref, env).catch((err) => {
|
|
935
|
+
if (o.persist === true && o.dryRun !== true)
|
|
936
|
+
throw err;
|
|
937
|
+
ctx.log(`${firstLine(err.message)}; starting from what ${STATE_FILE} says`);
|
|
938
|
+
unread = true;
|
|
939
|
+
return undefined;
|
|
940
|
+
});
|
|
941
|
+
// A branch that was never synced has no marker. It was cut from the default branch, whose marker says where that one's
|
|
942
|
+
// tests stand; the state file the branch inherited is as old as the last sync that committed, and a sync from there
|
|
943
|
+
// compared commits the default branch had synced already.
|
|
944
|
+
const main = own !== undefined || unread ? undefined : await defaultBranch(cwd, env);
|
|
945
|
+
if (main !== undefined && main !== branch)
|
|
946
|
+
inherited = syncedRef(main, folder);
|
|
947
|
+
const marker = own ?? (inherited === undefined ? undefined : await readSynced(cwd, inherited, env).catch(() => undefined));
|
|
948
|
+
if (own !== undefined || marker === undefined)
|
|
949
|
+
inherited = undefined;
|
|
950
|
+
const from = inherited === undefined ? `${ref} on the remote (` : `${inherited} on the remote, the default branch's (${branch} has no marker of its own yet; `;
|
|
951
|
+
// Repository by repository, the later of the two answers: a sync made here without --persist moved this checkout's
|
|
952
|
+
// state.json for its repository, and no marker.
|
|
953
|
+
if (marker !== undefined)
|
|
954
|
+
ctx.owed = [...new Set([...ctx.owed, ...marker.owed])];
|
|
955
|
+
const answers = marker === undefined ? [] : markerAnswersFor(marker, began.changelog);
|
|
956
|
+
if (marker !== undefined && answers.length > 0) {
|
|
957
|
+
ctx.synced = Object.fromEntries(answers.map((repo) => [repo, marker.commits[repo]]));
|
|
958
|
+
ctx.waiting = Object.fromEntries(answers.filter((repo) => Object.hasOwn(marker.waiting, repo)).map((repo) => [repo, marker.waiting[repo]]));
|
|
959
|
+
const known = began.changelog?.lastSeen ?? {};
|
|
960
|
+
for (const [repo, sha] of Object.entries(ctx.synced))
|
|
961
|
+
if (known[repo] !== sha)
|
|
962
|
+
ctx.log(`last synced: ${repo} at ${short(sha)}, from ${from}${STATE_FILE} says ${known[repo] === undefined ? "nothing" : short(known[repo])})`);
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
const resolved = await resolveChangeSets(ctx, request, o.dryRun === true).catch((err) => {
|
|
966
|
+
// A marker that names a commit the product repository does not have stops every sync: the way out is said with the error.
|
|
967
|
+
// Only there: said under any other error (no changelog configured, a --to that does not exist), it would have a sound marker removed.
|
|
968
|
+
const markers = Object.values(ctx.synced ?? {});
|
|
969
|
+
const ofMarker = markers.some((sha) => err.message.includes(sha));
|
|
970
|
+
const named = inherited ?? ref;
|
|
971
|
+
throw !ofMarker || named === undefined ? err : new Error(`${firstLine(err.message)} (the sync starts from the commit ${named} names on the remote; git push origin :${named} removes that marker, and the next sync starts from ${STATE_FILE})`, { cause: err });
|
|
972
|
+
});
|
|
973
|
+
// A poll that found nothing to sync still learned where the branch stands. A CI job's checkout is thrown away, and the
|
|
974
|
+
// state file's note of the head with it: every scheduled poll was then the first one. The marker on the remote holds it.
|
|
975
|
+
if (request.kind === "poll" && resolved.length === 0 && o.persist === true && o.dryRun !== true && ref !== undefined && ctx.config.changelog !== undefined) {
|
|
976
|
+
const polled = repoNameOf(ctx.config.changelog.repo);
|
|
977
|
+
const head = (await readState(cwd)).changelog?.lastSeen[polled];
|
|
978
|
+
if (head !== undefined && ctx.synced?.[polled] !== head) {
|
|
979
|
+
const marker = await pushSynced(cwd, ref, { [polled]: head }, env, {}, inherited === undefined ? {} : { from: inherited });
|
|
980
|
+
ctx.log(marker.pushed ? `last synced: ${polled} at ${short(head)}, pushed to ${ref}` : `last synced: the marker was not pushed (${redact(marker.why)}); the next poll here starts from what ${STATE_FILE} says`);
|
|
981
|
+
}
|
|
982
|
+
}
|
|
983
|
+
// What earlier syncs changed and nobody merged yet: a sync that persists starts from it, once there is something to sync.
|
|
984
|
+
// ponytail: a poll has written the head it found into the state file by now, so that file stays the checkout's and the
|
|
985
|
+
// request's own (its list of the syncs before) is not taken; take it before the poll writes if that list is missed.
|
|
986
|
+
const open = resolved.length > 0 && o.persist === true && o.dryRun !== true && ref !== undefined ? await startFromRequest(ctx, collectBranch(ref), await persistPaths()) : undefined;
|
|
987
|
+
for (const r of resolved) {
|
|
705
988
|
lines.length = 0;
|
|
989
|
+
// Where this checkout's tests stand, when the marker says a sync's pull request waits (undefined: as synced as the marker).
|
|
990
|
+
const repoName = repoNameOf(r.changeSet.repo);
|
|
991
|
+
const seen = committed.changelog?.lastSeen[repoName];
|
|
992
|
+
const stand = testsStandAt(ctx.waiting !== undefined && Object.hasOwn(ctx.waiting, repoName) ? ctx.waiting[repoName] : undefined, seen);
|
|
993
|
+
// What differs from the branch before this sync touches anything, and of it what a reviewer would look at.
|
|
994
|
+
const before = o.persist === true && o.dryRun !== true ? await differingNow(cwd, await persistPaths()) : new Map();
|
|
995
|
+
const held = new Map([...before].filter(([path]) => reviewable(path)).map(([path, file]) => [path, file.bytes]));
|
|
706
996
|
for (const line of describeLines(r.changeSet))
|
|
707
997
|
ctx.out(line);
|
|
708
998
|
const { impact, file } = await computeImpact(ctx, r.changeSet);
|
|
@@ -726,7 +1016,17 @@ export async function runSync(cwd, env, request, o) {
|
|
|
726
1016
|
await notify(failuresText(r.changeSet, failed, outcome.cases.filter((c) => !["not-run", "skipped"].includes(c.outcome)).length, healer));
|
|
727
1017
|
}
|
|
728
1018
|
outcome.knowledge = await updateKnowledge(ctx, r, impact, run, named);
|
|
729
|
-
|
|
1019
|
+
// What the crawl could not make is owed: noted here for a work tree that stays, and beside the marker for a CI job's.
|
|
1020
|
+
ctx.owed = outcome.knowledge.owed;
|
|
1021
|
+
await recordOwedCrawl(cwd, ctx.owed);
|
|
1022
|
+
// A sync's pull request waits, so this checkout's tests stand at an earlier commit than this sync starts from: the fixer
|
|
1023
|
+
// reads the changes since that commit, or it would call a test an earlier sync already followed a regression.
|
|
1024
|
+
// Not where the sync started from that request's files, all of them: the tests then stand where the last sync left them.
|
|
1025
|
+
const behind = failed.length > 0 && stand !== undefined && !sameCommit(stand, r.changeSet.before) && open?.whole !== true;
|
|
1026
|
+
const judged = behind ? await sinceCheckout(ctx, r, stand) : r;
|
|
1027
|
+
if (judged !== r)
|
|
1028
|
+
ctx.out(`fix: this checkout's tests were last synced at ${short(stand)} (an earlier sync's pull request is not merged here): the fixer reads the changes since then, ${plural(judged.changeSet.commits.length, "commit")}`);
|
|
1029
|
+
await fixFailed(ctx, judged, impact, outcome.knowledge, run, failed, outcome.cases, judged !== r);
|
|
730
1030
|
await updatePlan(ctx, r, impact, outcome.knowledge, run, named, outcome);
|
|
731
1031
|
if (r.eventId !== undefined)
|
|
732
1032
|
await removePendingChange(cwd, r.eventId);
|
|
@@ -754,25 +1054,74 @@ export async function runSync(cwd, env, request, o) {
|
|
|
754
1054
|
const reportFile = join(cwd, RUNS_DIR, `${ctx.now().toISOString().replace(/[:.]/g, "-")}-sync.md`);
|
|
755
1055
|
outcome.report = relative(cwd, reportFile);
|
|
756
1056
|
outcome.summary.report = outcome.report;
|
|
757
|
-
await writeFile(reportFile, renderSyncReport(outcome, lines));
|
|
1057
|
+
await writeFile(reportFile, redact(renderSyncReport(outcome, lines)));
|
|
758
1058
|
await recordSync(cwd, outcome.summary, ctx.now());
|
|
759
1059
|
let requestUrl;
|
|
760
1060
|
if (o.persist === true && outcome.stopped === undefined) {
|
|
761
1061
|
const body = await readFile(reportFile, "utf8");
|
|
762
1062
|
// the proposed tests head the pull request's description only: the commit keeps the report
|
|
763
|
-
const
|
|
764
|
-
|
|
765
|
-
const
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
1063
|
+
const section = outcome.planned.length === 0 ? undefined : await proposalsSection(ctx, outcome, named);
|
|
1064
|
+
const prBody = section === undefined ? undefined : `${redact(renderProposals(section))}\n\n${body}`;
|
|
1065
|
+
const paths = await persistPaths();
|
|
1066
|
+
// What this sync changed. Of it, what a reviewer looks at is a reason for a commit or a pull request.
|
|
1067
|
+
const after = await differingNow(cwd, paths);
|
|
1068
|
+
const changed = new Set([...after.keys(), ...before.keys()].filter((path) => before.get(path)?.bytes !== after.get(path)?.bytes));
|
|
1069
|
+
const touched = new Set([...changed].filter(reviewable));
|
|
1070
|
+
// While earlier syncs wait in a pull request, more is a reason to add to it: a page crawled again, a statement of the
|
|
1071
|
+
// scheme revised (the knowledge, beyond the stamp every sync moves), a host the product links to since. None of it
|
|
1072
|
+
// opens a request by itself.
|
|
1073
|
+
const told = new Set();
|
|
1074
|
+
for (const path of open === undefined ? [] : [...changed].filter((one) => one.includes(`${KNOWLEDGE_DIR}/`) || one.endsWith(HOSTS_FILE))) {
|
|
1075
|
+
const was = before.get(path)?.says ?? (await git(cwd, ["show", `HEAD:./${relative(cwd, path)}`]).then((text) => digest(unstamped(text)), () => undefined));
|
|
1076
|
+
if (was !== after.get(path)?.says)
|
|
1077
|
+
told.add(path);
|
|
1078
|
+
}
|
|
1079
|
+
const root = await git(cwd, ["rev-parse", "--show-toplevel"]).catch(() => cwd);
|
|
1080
|
+
const asked = (await selectGitHost(ctx.config, cwd, env))?.name === "gitlab" ? "merge request" : "pull request";
|
|
1081
|
+
// One request takes every sync of this branch until it is merged: a later sync adds its commit and its report, and its
|
|
1082
|
+
// proposals join the choice that is there.
|
|
1083
|
+
const collect = ref === undefined || branch === undefined ? undefined : {
|
|
1084
|
+
branch: collectBranch(ref),
|
|
1085
|
+
title: `sync: test updates for ${branch}`,
|
|
1086
|
+
body: (earlier) => redact([carriedProposals(earlier, section), `Every sync of \`${branch}\` adds to this ${asked} until it is merged: each commit is one sync, with its report as its message. The latest:`, body].filter((part) => part !== "").join("\n\n")),
|
|
1087
|
+
};
|
|
1088
|
+
/** One commit or pull request of what the sync changed, where `needs` finds a reason among the changed files (none given: any change is one). */
|
|
1089
|
+
const save = async (needs) => {
|
|
1090
|
+
// D5: a protected branch, a scheduled run or a case that ran on a production-tier environment persists through a pull request
|
|
1091
|
+
// Where the sync started from an open request, what goes to it is what this sync changed: the rest is there already, or is somebody's here.
|
|
1092
|
+
const persisted = await persistOutcome({ cwd, paths, ...(needs === undefined ? {} : { needs }), ...(open === undefined ? {} : { only: (path) => changed.has(join(root, path)) }), title: () => `sync: ${describeChangeSet(r.changeSet)}`, body, ...(prBody === undefined ? {} : { prBody }), scheduled: o.scheduled === true || run.ranOnProduction === true, env, prPrefix: "omniqa/sync", ...(collect === undefined ? {} : { collect }), out });
|
|
1093
|
+
const result = persisted.result;
|
|
1094
|
+
outcome.summary = { ...outcome.summary, persisted: persisted.error !== undefined || result === undefined ? { kind: "off" } : result.kind === "nothing" ? { kind: "nothing" } : result.kind === "commit" ? { kind: "commit", branch: result.branch, sha: result.sha } : { kind: "pull-request", branch: result.branch, sha: result.sha, url: result.url } };
|
|
1095
|
+
await writeFile(reportFile, redact(renderSyncReport(outcome, lines, persisted.error === undefined ? persistedText(outcome.summary.persisted) : `not persisted: ${persisted.error}`)));
|
|
1096
|
+
await updateSync(cwd, { persisted: outcome.summary.persisted });
|
|
1097
|
+
if (persisted.error !== undefined)
|
|
1098
|
+
throw new Error(`persist failed: ${persisted.error}`);
|
|
1099
|
+
if (result?.kind === "pull-request")
|
|
1100
|
+
requestUrl = result.url;
|
|
1101
|
+
};
|
|
1102
|
+
await save((path) => touched.has(path) || told.has(path));
|
|
1103
|
+
// The change is synced whether its commit is on the branch, waits in a pull request or was not needed: the marker moves now.
|
|
1104
|
+
const repo = repoNameOf(r.changeSet.repo);
|
|
1105
|
+
const at = await endCommit(r);
|
|
1106
|
+
// Whether a pull request waits once this sync is done: said only where the commit the tests stand at has an id.
|
|
1107
|
+
const from = stand ?? (await startCommit(r));
|
|
1108
|
+
const kind = outcome.summary.persisted.kind;
|
|
1109
|
+
const waits = from === undefined || kind === "off" ? {} : { [repo]: waitingAfter({ stand: from, before: r.changeSet.before, seen, persisted: kind, cleanSuite: o.suite === true && failed.length === 0 && held.size === 0 }) };
|
|
1110
|
+
const marker = ref === undefined ? notPushed("the checkout names no branch") : at === undefined ? notPushed(`${r.changeSet.after} could not be read as a commit`) : await pushSynced(cwd, ref, { [repo]: at }, env, waits, { ...(inherited === undefined ? {} : { from: inherited }), ...(outcome.knowledge === undefined ? {} : { owed: outcome.knowledge.owed }) });
|
|
1111
|
+
if (marker.pushed)
|
|
1112
|
+
ctx.log(`last synced: ${repo} at ${short(at)}, pushed to ${ref}`);
|
|
1113
|
+
else if (marker.refused) {
|
|
1114
|
+
// The remote answered and does not take such a ref: the last synced commit travels the old way, in the state file a
|
|
1115
|
+
// commit or a pull request holds. Where this sync made neither, it makes one now, of what it changed.
|
|
1116
|
+
out(`last synced: the marker was not pushed (${redact(marker.why)}): ${STATE_FILE} holds the commit, as before`);
|
|
1117
|
+
if (outcome.summary.persisted.kind === "nothing")
|
|
1118
|
+
await save();
|
|
1119
|
+
}
|
|
1120
|
+
else {
|
|
1121
|
+
// Nowhere to keep one (no branch, no remote), or the remote was not reached: a commit made to carry the state file would
|
|
1122
|
+
// fail the same way, or have no branch to go to. One line, and the sync ends as it would have.
|
|
1123
|
+
out(`last synced: the marker was not pushed (${redact(marker.why)}); the next sync here starts from what ${STATE_FILE} says`);
|
|
1124
|
+
}
|
|
776
1125
|
}
|
|
777
1126
|
// a persist error threw above: the CI job fails and says why, so the outcome message is skipped
|
|
778
1127
|
if (failed.length > 0)
|