omniqa 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -1
- package/dist/.tsbuildinfo +1 -1
- package/dist/auth.d.ts +17 -0
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +29 -1
- package/dist/auth.js.map +1 -1
- package/dist/cli.js +18 -5
- package/dist/cli.js.map +1 -1
- package/dist/commands/chat.js +1 -1
- package/dist/commands/chat.js.map +1 -1
- package/dist/commands/crawl.d.ts.map +1 -1
- package/dist/commands/crawl.js +17 -5
- package/dist/commands/crawl.js.map +1 -1
- package/dist/commands/doctor.d.ts +23 -0
- package/dist/commands/doctor.d.ts.map +1 -1
- package/dist/commands/doctor.js +152 -6
- package/dist/commands/doctor.js.map +1 -1
- package/dist/commands/generate.js +1 -1
- package/dist/commands/generate.js.map +1 -1
- package/dist/commands/init.d.ts +3 -1
- package/dist/commands/init.d.ts.map +1 -1
- package/dist/commands/init.js +217 -50
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/knowledge.d.ts +61 -1
- package/dist/commands/knowledge.d.ts.map +1 -1
- package/dist/commands/knowledge.js +232 -50
- package/dist/commands/knowledge.js.map +1 -1
- package/dist/commands/persist.d.ts +6 -0
- package/dist/commands/persist.d.ts.map +1 -1
- package/dist/commands/persist.js +9 -2
- package/dist/commands/persist.js.map +1 -1
- package/dist/commands/plan-add.d.ts.map +1 -1
- package/dist/commands/plan-add.js +13 -3
- package/dist/commands/plan-add.js.map +1 -1
- package/dist/commands/plan-review.d.ts +6 -2
- package/dist/commands/plan-review.d.ts.map +1 -1
- package/dist/commands/plan-review.js +16 -5
- package/dist/commands/plan-review.js.map +1 -1
- package/dist/commands/plan.d.ts +15 -0
- package/dist/commands/plan.d.ts.map +1 -1
- package/dist/commands/plan.js +103 -32
- package/dist/commands/plan.js.map +1 -1
- package/dist/commands/reindex.d.ts +1 -1
- package/dist/commands/reindex.d.ts.map +1 -1
- package/dist/commands/reindex.js +15 -1
- package/dist/commands/reindex.js.map +1 -1
- package/dist/commands/security.d.ts +4 -2
- package/dist/commands/security.d.ts.map +1 -1
- package/dist/commands/security.js +18 -7
- package/dist/commands/security.js.map +1 -1
- package/dist/commands/serve.d.ts +2 -0
- package/dist/commands/serve.d.ts.map +1 -1
- package/dist/commands/serve.js +126 -30
- package/dist/commands/serve.js.map +1 -1
- package/dist/commands/sync.d.ts.map +1 -1
- package/dist/commands/sync.js +19 -2
- package/dist/commands/sync.js.map +1 -1
- package/dist/commands/test.d.ts +5 -1
- package/dist/commands/test.d.ts.map +1 -1
- package/dist/commands/test.js +106 -6
- package/dist/commands/test.js.map +1 -1
- package/dist/commands/write.d.ts +8 -2
- package/dist/commands/write.d.ts.map +1 -1
- package/dist/commands/write.js +101 -18
- package/dist/commands/write.js.map +1 -1
- package/dist/devices.d.ts +3 -1
- package/dist/devices.d.ts.map +1 -1
- package/dist/devices.js +5 -3
- package/dist/devices.js.map +1 -1
- package/dist/fixer.d.ts.map +1 -1
- package/dist/fixer.js +19 -9
- package/dist/fixer.js.map +1 -1
- package/dist/jobs.d.ts +6 -0
- package/dist/jobs.d.ts.map +1 -1
- package/dist/jobs.js +10 -1
- package/dist/jobs.js.map +1 -1
- package/dist/mcp/server.d.ts +1 -1
- package/dist/mcp/server.d.ts.map +1 -1
- package/dist/mcp/server.js +1 -1
- package/dist/mcp/server.js.map +1 -1
- package/dist/mcp/session.d.ts.map +1 -1
- package/dist/mcp/session.js +7 -7
- package/dist/mcp/session.js.map +1 -1
- package/dist/mcp/tools.d.ts.map +1 -1
- package/dist/mcp/tools.js +9 -6
- package/dist/mcp/tools.js.map +1 -1
- package/dist/page.d.ts +1 -1
- package/dist/page.d.ts.map +1 -1
- package/dist/page.js +11 -0
- package/dist/page.js.map +1 -1
- package/dist/planner.d.ts.map +1 -1
- package/dist/planner.js +8 -4
- package/dist/planner.js.map +1 -1
- package/dist/prompt.d.ts +7 -1
- package/dist/prompt.d.ts.map +1 -1
- package/dist/prompt.js +23 -4
- package/dist/prompt.js.map +1 -1
- package/dist/qa.d.ts +7 -0
- package/dist/qa.d.ts.map +1 -1
- package/dist/qa.js +34 -9
- package/dist/qa.js.map +1 -1
- package/dist/runner.d.ts +7 -2
- package/dist/runner.d.ts.map +1 -1
- package/dist/runner.js +28 -8
- package/dist/runner.js.map +1 -1
- package/dist/security-checks.d.ts +46 -4
- package/dist/security-checks.d.ts.map +1 -1
- package/dist/security-checks.js +114 -39
- package/dist/security-checks.js.map +1 -1
- package/dist/security-gates.d.ts +64 -7
- package/dist/security-gates.d.ts.map +1 -1
- package/dist/security-gates.js +125 -16
- package/dist/security-gates.js.map +1 -1
- package/dist/security-mask.d.ts +35 -0
- package/dist/security-mask.d.ts.map +1 -0
- package/dist/security-mask.js +104 -0
- package/dist/security-mask.js.map +1 -0
- package/dist/security-report.d.ts +93 -2
- package/dist/security-report.d.ts.map +1 -1
- package/dist/security-report.js +153 -14
- package/dist/security-report.js.map +1 -1
- package/dist/security-request.d.ts +27 -4
- package/dist/security-request.d.ts.map +1 -1
- package/dist/security-request.js +56 -14
- package/dist/security-request.js.map +1 -1
- package/dist/security-research.d.ts +132 -0
- package/dist/security-research.d.ts.map +1 -0
- package/dist/security-research.js +599 -0
- package/dist/security-research.js.map +1 -0
- package/dist/security-scan.d.ts +98 -0
- package/dist/security-scan.d.ts.map +1 -0
- package/dist/security-scan.js +463 -0
- package/dist/security-scan.js.map +1 -0
- package/dist/security.d.ts +18 -4
- package/dist/security.d.ts.map +1 -1
- package/dist/security.js +181 -37
- package/dist/security.js.map +1 -1
- package/dist/sync.d.ts +55 -3
- package/dist/sync.d.ts.map +1 -1
- package/dist/sync.js +403 -54
- package/dist/sync.js.map +1 -1
- package/dist/test.d.ts +26 -2
- package/dist/test.d.ts.map +1 -1
- package/dist/test.js +152 -29
- package/dist/test.js.map +1 -1
- package/dist/writer.d.ts +29 -10
- package/dist/writer.d.ts.map +1 -1
- package/dist/writer.js +116 -36
- package/dist/writer.js.map +1 -1
- package/package.json +7 -7
- package/skills/knowledge.md +1 -1
- package/skills/security-research.md +57 -0
- package/skills/security-review.md +27 -0
- package/skills/onboarding.md +0 -5
|
@@ -4,8 +4,8 @@ import { join, relative, resolve } from "node:path";
|
|
|
4
4
|
import { stderr, stdout } from "node:process";
|
|
5
5
|
import { parseArgs } from "node:util";
|
|
6
6
|
import { interceptorInstalled } from "@omniqa/browser";
|
|
7
|
-
import { budgeted, BudgetExceeded, changelogGitEnv, createModels, docSources, ingest, ingestDocs, KNOWLEDGE_DIR, knowledgeCommit, knowledgeStamp, loadConfig, loadProduct, modelFor, openTranscript, parseSecurity, PRODUCT_FILE, parseStaticSections, readState, recordKnowledge, redactor, renderReview, renderSecurity, resolveEnvironmentUrl, RUNS_DIR, schemeCounts, secretValues, SECURITY_FILE, SECURITY_REQUESTS_READ, selectEnvironment, stampKnowledge, stampMaps, tierEnvironment, transcriptName, withReview, writeCoverageFile } from "@omniqa/core";
|
|
8
|
-
import { crawlSite, createNetMap, hostPatterns, launchBrowser, requestRoute, startInterceptor, writeNetMap } from "@omniqa/web";
|
|
7
|
+
import { budgeted, addHosts, BudgetExceeded, changelogGitEnv, createModels, docSources, HOSTS_FILE, ingest, ingestDocs, KNOWLEDGE_DIR, knowledgeCommit, knowledgeStamp, loadConfig, loadProduct, modelFor, openTranscript, parseSecurity, PRODUCT_FILE, parseStaticSections, readHosts, readState, recordKnowledge, redactor, renderReview, renderSecurity, resolveEnvironmentUrl, RUNS_DIR, schemeCounts, secretValues, SECURITY_FILE, SECURITY_REQUESTS_READ, selectEnvironment, stampKnowledge, switchedOn, stampMaps, tierEnvironment, transcriptName, withReview, writeCoverageFile, productSecrets } from "@omniqa/core";
|
|
8
|
+
import { contextOptionsFor, crawlSite, createNetMap, HELD_LANGUAGES, hostPatterns, launchBrowser, linkedHostsOf, linkedPatterns, requestRoute, startInterceptor, writeNetMap } from "@omniqa/web";
|
|
9
9
|
import { createKnowledgeCreator, rebuildXref, xrefLine } from "../knowledge.js";
|
|
10
10
|
import { plannedLine, planSecurity } from "../security.js";
|
|
11
11
|
import { SIGN_OUT } from "../security-checks.js";
|
|
@@ -13,7 +13,7 @@ import { readObserved, sessionsAndRequests } from "../security-request.js";
|
|
|
13
13
|
import { indexProject } from "./reindex.js";
|
|
14
14
|
import { runEnv } from "./test.js";
|
|
15
15
|
import { terminalSinks } from "../progress.js";
|
|
16
|
-
const USAGE = `omniqa knowledge build [--skip-crawl] [--skip-static] [--budget N] [--env <name>] | review
|
|
16
|
+
const USAGE = `omniqa knowledge build [--skip-crawl] [--skip-static] [--skip-security] [--budget N] [--env <name>] | review
|
|
17
17
|
|
|
18
18
|
build The knowledge base under .omniqa/knowledge: the repository maps with their dissection (routes, handlers, models,
|
|
19
19
|
auth, flags, integrations, config, pipelines, tests) and an analyst summary, the documentation in full, a crawl of
|
|
@@ -29,11 +29,17 @@ const USAGE = `omniqa knowledge build [--skip-crawl] [--skip-static] [--budget N
|
|
|
29
29
|
|
|
30
30
|
--skip-crawl Keep the crawl and the api files as they are (the security walk is its own and still runs)
|
|
31
31
|
--skip-static Keep the repository maps as they are
|
|
32
|
-
--
|
|
32
|
+
--skip-security Keep security.md as it is: no security walk, so no form is sent
|
|
33
|
+
--budget N Model calls at most for the build (default ${80}); the build stops with a message when it is reached,
|
|
34
|
+
and names the command that goes on from what it made (no page is opened and no form sent again)
|
|
33
35
|
--env <name> The environment to crawl, a production-tier one only where it explores (default: the first dev-tier
|
|
34
36
|
one, else the first production one with explore on)
|
|
35
37
|
`;
|
|
36
38
|
export const DEFAULT_BUDGET = 80;
|
|
39
|
+
/** What going on from a build stopped past its walks keeps, in the words of the line that names the command. */
|
|
40
|
+
export const KEPT = "what this build made stays, so no page is opened and no form is sent again";
|
|
41
|
+
/** The command that goes on from a build that failed past its walks, read off its error. */
|
|
42
|
+
export const failedPast = (err) => (typeof err?.goOn === "string" ? err.goOn : undefined);
|
|
37
43
|
const plural = (n, word) => `${n} ${word}${n === 1 ? "" : "s"}`;
|
|
38
44
|
/** The environment the build crawls: the named one (a production-tier one only where it explores), else the first dev-tier one, else the first production one that explores. */
|
|
39
45
|
export function crawlTarget(product, name) {
|
|
@@ -57,7 +63,7 @@ function passesOf(product, root) {
|
|
|
57
63
|
const accounts = Object.entries(product.accounts);
|
|
58
64
|
return [
|
|
59
65
|
{ as: "guest", outDir: accounts.length === 0 ? root : join(root, "guest") },
|
|
60
|
-
...accounts.map(([name, fields], i) => ({ as: name, outDir: i === 0 ? root : join(root, name), account: { name, fields, loginPath: product.crawl.loginPath } })),
|
|
66
|
+
...accounts.map(([name, fields], i) => ({ as: name, outDir: i === 0 ? root : join(root, name), account: { name, fields, loginPath: product.crawl.loginPath, ...(product.crawl.signedIn === undefined ? {} : { signedIn: product.crawl.signedIn }) } })),
|
|
61
67
|
];
|
|
62
68
|
}
|
|
63
69
|
/**
|
|
@@ -65,7 +71,7 @@ function passesOf(product, root) {
|
|
|
65
71
|
* each pass, and its findings and its requests are read after it, so both are known by the actor that made them.
|
|
66
72
|
* ponytail: the requests read are shared out evenly among the passes; a pass that needs less leaves its rest unused.
|
|
67
73
|
*/
|
|
68
|
-
async function crawlPasses(passes, crawl, sidecar) {
|
|
74
|
+
async function crawlPasses(passes, crawl, context, sidecar) {
|
|
69
75
|
const results = [];
|
|
70
76
|
const findings = [];
|
|
71
77
|
const observed = [];
|
|
@@ -74,7 +80,8 @@ async function crawlPasses(passes, crawl, sidecar) {
|
|
|
74
80
|
try {
|
|
75
81
|
for (const pass of passes) {
|
|
76
82
|
await sidecar?.clear();
|
|
77
|
-
|
|
83
|
+
// With what the environment asks of a context: its HTTP sign-in, its own certificate.
|
|
84
|
+
const page = await browser.newPage(context);
|
|
78
85
|
try {
|
|
79
86
|
results.push(await crawl(page, pass));
|
|
80
87
|
}
|
|
@@ -94,7 +101,108 @@ async function crawlPasses(passes, crawl, sidecar) {
|
|
|
94
101
|
return { results, findings, observed, capped };
|
|
95
102
|
}
|
|
96
103
|
/**
|
|
97
|
-
*
|
|
104
|
+
* Why the passes of a crawl are no crawl to build knowledge from, and what to do about it, each a part of one sentence for
|
|
105
|
+
* the user. A guest the product keeps out (a 401 for a product that signs its members in) is still a crawl, and so is an
|
|
106
|
+
* account with nothing to type, which walks as a guest and is kept out as one: those passes are not made, the members' are.
|
|
107
|
+
*/
|
|
108
|
+
export function noCrawl(envName, baseUrl, passes, results) {
|
|
109
|
+
const notMade = results.flatMap((result, i) => (result.unreachable === undefined ? [] : [{ result, why: result.unreachable, pass: passes[i] }]));
|
|
110
|
+
const again = "run omniqa knowledge build again";
|
|
111
|
+
const where = `environment ${envName} at ${baseUrl}`;
|
|
112
|
+
// The product fails: nothing answers, or a 5xx. First, whoever met it: an account's "could not sign in" behind a gateway's
|
|
113
|
+
// 502 names the wrong thing, and a file written without the one actor who met it reads as if that actor never asked.
|
|
114
|
+
const down = notMade.find(({ result }) => result.cause === "no-answer" || (result.status ?? 0) >= 500);
|
|
115
|
+
if (down !== undefined)
|
|
116
|
+
return { why: `${where} did not load${notMade.length === results.length ? "" : ` for ${down.pass.as}`} (is the product running there?): ${down.why}`, fix: `${again} once it answers` };
|
|
117
|
+
// An account that was to sign in and did not is what a user fixes in their own files.
|
|
118
|
+
const signIn = notMade.find(({ result }) => result.cause === "sign-in");
|
|
119
|
+
if (signIn !== undefined)
|
|
120
|
+
return { why: signIn.why.replace(/^(could not sign in as \S+)/, `$1 on ${envName}`), fix: `check the account's values in .env and crawl.login_path in ${PRODUCT_FILE}, then ${again}` };
|
|
121
|
+
// An account that signed in and was given no page, while others walk.
|
|
122
|
+
const member = notMade.find(({ result, pass }) => pass.account !== undefined && result.as === pass.as);
|
|
123
|
+
if (member !== undefined)
|
|
124
|
+
return { why: `${where} gave ${member.pass.as} no page to crawl: ${member.why}`, fix: `${again} once it does` };
|
|
125
|
+
// What is left not made was asked as a guest. Some pass was made: a crawl.
|
|
126
|
+
const first = notMade[0];
|
|
127
|
+
if (first === undefined || notMade.length < results.length)
|
|
128
|
+
return undefined;
|
|
129
|
+
// No pass at all, and the product answers: what would let the crawl in depends on what kept it out.
|
|
130
|
+
const fix = first.result.cause === "excluded"
|
|
131
|
+
? `take the entry that matches the start page out of crawl.exclude or the environment's deny list in ${PRODUCT_FILE}`
|
|
132
|
+
: first.result.status === 404
|
|
133
|
+
? `check the environment's url in ${PRODUCT_FILE}`
|
|
134
|
+
: `give the crawl a way in (an account with a password under accounts:, or the environment's cookies:, in ${PRODUCT_FILE})`;
|
|
135
|
+
return { why: `${where} gives no visitor a page to crawl: ${first.why}`, fix: `${fix}, then ${again}` };
|
|
136
|
+
}
|
|
137
|
+
/** `noCrawl` as the one sentence a build stops with: what was not rewritten, then what to do. */
|
|
138
|
+
const stopLine = (no, kept) => `${no.why}. ${kept}; ${no.fix} (--skip-crawl builds from the crawl on disk)`;
|
|
139
|
+
/**
|
|
140
|
+
* The hosts walks saw beside the product's own (a partner's site its pages link to, a service they ask) are added to
|
|
141
|
+
* `.omniqa/approved-hosts.yaml`, each switched off: a product links to hosts that are not its owner's, and nothing is opened or asked
|
|
142
|
+
* there before the owner says so. The line that names the new ones; none where the file already names them all.
|
|
143
|
+
*/
|
|
144
|
+
export async function gatherHosts(cwd, baseUrl, walks) {
|
|
145
|
+
const found = linkedHostsOf(baseUrl, walks.flatMap((walk) => walk.seen?.asked ?? []), walks.flatMap((walk) => walk.seen?.outbound ?? []));
|
|
146
|
+
let added;
|
|
147
|
+
try {
|
|
148
|
+
added = await addHosts(cwd, found);
|
|
149
|
+
}
|
|
150
|
+
catch (err) {
|
|
151
|
+
// A file its owner broke: the walk is made and paid for, so the build goes on and says it. A run that needs the switches stops on it.
|
|
152
|
+
return `hosts: ${HOSTS_FILE} was not read, so no host was added to it: ${firstLine(err.message)}`;
|
|
153
|
+
}
|
|
154
|
+
if (added.length === 0)
|
|
155
|
+
return undefined;
|
|
156
|
+
const shown = 10;
|
|
157
|
+
return `hosts: ${added.length} new in ${HOSTS_FILE}, switched off: ${added.slice(0, shown).join(", ")}${added.length > shown ? ` and ${added.length - shown} more` : ""} (there, tests: true lets a test open a host and security: true lets the security checks ask it; only for a host that is yours)`;
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* What a walk sent without being able to judge it: the words that keep a form from being sent (delete, cancel, pay and their
|
|
161
|
+
* like) are in thirteen languages, and the owner of a product in another one is told that every form of theirs with
|
|
162
|
+
* something to fill in is sent. One line names the buttons, the languages and the way to keep one unsent.
|
|
163
|
+
*/
|
|
164
|
+
export function foreignLine(buttons) {
|
|
165
|
+
const named = [...new Set(buttons)];
|
|
166
|
+
if (named.length === 0)
|
|
167
|
+
return undefined;
|
|
168
|
+
const one = named.length === 1;
|
|
169
|
+
const languages = Object.values(HELD_LANGUAGES);
|
|
170
|
+
return `security: ${one ? "this button" : "these buttons"} (${named.map((b) => JSON.stringify(b)).join(", ")}) ${one ? "is" : "are"} in a language the words that keep a form from being sent (delete, cancel, pay and their like) are not in: nothing could hold ${one ? "its form" : "their forms"} back. They are in ${languages.slice(0, -1).join(", ")} and ${languages.at(-1)}. Name in deny, in ${PRODUCT_FILE}, what a walk must not send`;
|
|
171
|
+
}
|
|
172
|
+
/** Each form the passes of a walk sent, with who sent it: said by the build, since what a form leaves stays in the product. */
|
|
173
|
+
function formsSentBy(walked, redact) {
|
|
174
|
+
const sent = new Map();
|
|
175
|
+
for (const w of walked) {
|
|
176
|
+
for (const form of w.result.formsSent) {
|
|
177
|
+
const one = sent.get(`${form.route}\n${form.submit}`) ?? { page: redact(form.route), button: redact(form.submit), actors: [] };
|
|
178
|
+
one.actors.push(w.pass.as);
|
|
179
|
+
if (form.denied !== undefined)
|
|
180
|
+
one.denied = form.denied;
|
|
181
|
+
if (form.foreign === true)
|
|
182
|
+
one.foreign = true;
|
|
183
|
+
sent.set(`${form.route}\n${form.submit}`, one);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
return [...sent.values()];
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* What the passes of a walk pressed with the proxy letting nothing through, a line a button and an actor, in the order they
|
|
190
|
+
* were pressed. `kept back`: the request the press left was recorded and never reached the product, and the line ends with
|
|
191
|
+
* the hosts the proxy does not read that the page sent something to meanwhile. `not kept back`: nothing was recorded, and
|
|
192
|
+
* something of that press was sent as the account itself, or may have been; the line says what. A page, a button and a
|
|
193
|
+
* route may echo an account's value: each is redacted by itself, and never the actor's name.
|
|
194
|
+
*/
|
|
195
|
+
export function pressedLines(walked, redact) {
|
|
196
|
+
return walked.flatMap((walk) => walk.pressed.flatMap((button) => {
|
|
197
|
+
const where = `${redact(button.route)} ${JSON.stringify(redact(button.submit))} as ${walk.as}`;
|
|
198
|
+
return [
|
|
199
|
+
...(button.kept === undefined ? [] : [`kept back: ${where}: ${redact(button.kept)}, recorded and not sent${button.elsewhere === undefined ? "" : `; the page also sent a request that changes something to ${button.elsewhere.map((host) => redact(host)).join(", ")}, which the proxy does not read: nothing kept that one back`}`]),
|
|
200
|
+
...(button.went === undefined ? [] : [`not kept back: ${where}: ${redact(button.went)}`]),
|
|
201
|
+
];
|
|
202
|
+
}));
|
|
203
|
+
}
|
|
204
|
+
/**
|
|
205
|
+
* security.md from a walk behind the sidecar: the actors that walked (an account with nothing to type walks as a guest, and is
|
|
98
206
|
* left out), their pages, the findings by the actors that raised them, each host's TLS. The checks planned from an earlier file stay.
|
|
99
207
|
* A route or a subject may echo an account's value (a user name in a path): those are redacted, one by one, and never the
|
|
100
208
|
* actors' names, which the plan and the tests read back.
|
|
@@ -121,25 +229,17 @@ async function writeSecurityFile(o) {
|
|
|
121
229
|
// No file yet, or one that no longer parses: the pass writes it anew, and planning gives the cases their rows again.
|
|
122
230
|
const previous = await readFile(file, "utf8").then(parseSecurity).catch(() => undefined);
|
|
123
231
|
// Names only reach the file, each redacted by itself: a cookie, a field or a route may echo an account's value.
|
|
124
|
-
const { sessions, requests } = sessionsAndRequests(walked.map((w) => ({ actor: w.pass.as, observed: w.observed })));
|
|
232
|
+
const { sessions, requests } = sessionsAndRequests(walked.map((w) => ({ actor: w.pass.as, observed: w.observed, pressed: w.result.pressed })));
|
|
125
233
|
// What the sidecar does not see and a written check needs: the pages that make each request (a test opens one), and the page
|
|
126
234
|
// an account signs out on (a link or a form's button called so; the crawl follows neither).
|
|
127
235
|
const made = (request) => o.capture.entries.filter((e) => actors.includes(e.as) && `${e.request.method} ${requestRoute(URL.parse(e.request.url)?.pathname ?? "/")}` === request);
|
|
128
236
|
const pagesOf = (request) => [...new Set(made(request).filter((e) => e.form === undefined).map((e) => e.pageref))].filter((route) => route !== "" && route !== "(login)");
|
|
129
237
|
// A form is named where no page makes the request by being opened: what a page asks for again once its form was sent is the page's.
|
|
130
238
|
// ponytail: a request one actor's page makes and another actor's form alone is listed with the page; list both if a product needs the form's test.
|
|
131
|
-
|
|
132
|
-
//
|
|
133
|
-
const
|
|
134
|
-
|
|
135
|
-
for (const form of w.result.formsSent) {
|
|
136
|
-
const one = sent.get(`${form.route}\n${form.submit}`) ?? { page: o.redact(form.route), button: o.redact(form.submit), actors: [] };
|
|
137
|
-
one.actors.push(w.pass.as);
|
|
138
|
-
if (form.denied !== undefined)
|
|
139
|
-
one.denied = form.denied;
|
|
140
|
-
sent.set(`${form.route}\n${form.submit}`, one);
|
|
141
|
-
}
|
|
142
|
-
}
|
|
239
|
+
// The form of a button that was pressed with nothing let through comes with its row (`pressed`): the capture keeps nothing of a press.
|
|
240
|
+
// A pressed button first: where a form that is sent posts to the same address, a test of that row presses, and takes the first form it can.
|
|
241
|
+
const formsOf = (request, pressed) => [...pressed.map((form) => ({ page: o.redact(form.page), button: o.redact(form.button) })), ...(pagesOf(request).length > 0 ? [] : [...new Map(made(request).flatMap((e) => (e.form === undefined ? [] : [[`${e.pageref}\n${e.form}`, { page: o.redact(e.pageref), button: o.redact(e.form) }]]))).values()])];
|
|
242
|
+
const sent = formsSentBy(walked, o.redact);
|
|
143
243
|
const signOutOf = (account) => o.capture.pages.find((p) => p.as === account && !p.route.includes("{") && (p.links.some((link) => SIGN_OUT.test(link.text)) || p.forms.some((form) => SIGN_OUT.test(form.submit))))?.route;
|
|
144
244
|
const text = renderSecurity({
|
|
145
245
|
environment: o.environment,
|
|
@@ -153,15 +253,16 @@ async function writeSecurityFile(o) {
|
|
|
153
253
|
const signOut = signOutOf(x.account);
|
|
154
254
|
return { ...x, cookies: x.cookies.map((c) => ({ ...c, name: o.redact(c.name) })), ...(x.antiForgery === undefined ? {} : { antiForgery: o.redact(x.antiForgery) }), ...(signOut === undefined ? {} : { signOut: o.redact(signOut) }) };
|
|
155
255
|
}),
|
|
156
|
-
requests: requests.map((r) => ({ ...r, request: o.redact(r.request), parameters: r.parameters.map((name) => o.redact(name)), pages: pagesOf(r.request).map((route) => o.redact(route)), forms: formsOf(r.request) })),
|
|
256
|
+
requests: requests.map((r) => ({ ...r, request: o.redact(r.request), parameters: r.parameters.map((name) => o.redact(name)), pages: pagesOf(r.request).map((route) => o.redact(route)), forms: formsOf(r.request, r.forms) })),
|
|
157
257
|
checks: previous?.checks ?? [],
|
|
158
258
|
flows: o.capture.entries.filter((e) => firstParty(e.request.url)).length,
|
|
159
259
|
thirdParty: new Set(o.capture.entries.filter((e) => !firstParty(e.request.url)).map((e) => URL.parse(e.request.url)?.host ?? "")).size,
|
|
160
260
|
capped: o.read.capped,
|
|
161
|
-
|
|
261
|
+
// A press of which something went on, or may have, is a form sent: the file does not call such a build passive.
|
|
262
|
+
formsSent: walked.reduce((n, w) => n + w.result.formsSent.length + w.result.pressed.filter((press) => press.went !== undefined).length, 0),
|
|
162
263
|
});
|
|
163
264
|
await writeFile(file, stampKnowledge(text, o.stamp));
|
|
164
|
-
return { actors, pages: walked.reduce((n, w) => n + w.result.pages, 0), findings: merged.size, requests: requests.length, forms:
|
|
265
|
+
return { actors, pages: walked.reduce((n, w) => n + w.result.pages, 0), findings: merged.size, requests: requests.length, forms: sent };
|
|
165
266
|
}
|
|
166
267
|
const isRemote = (s) => /^(https?:\/\/|git@|ssh:\/\/|git:\/\/)/.test(s);
|
|
167
268
|
/** The env var names the analyst and reviewer models need that are not set. */
|
|
@@ -186,7 +287,7 @@ export async function buildKnowledge(cwd, env, o) {
|
|
|
186
287
|
const knowledgeDir = join(cwd, KNOWLEDGE_DIR);
|
|
187
288
|
const limit = o.budget ?? DEFAULT_BUDGET;
|
|
188
289
|
const budget = budgeted(createModels(config.models, { env }), limit);
|
|
189
|
-
const accountValues =
|
|
290
|
+
const accountValues = productSecrets(product);
|
|
190
291
|
// A test phone number is typed into the product's pages by the security walk: what a page echoes of it stays out of what is written.
|
|
191
292
|
const phones = Object.values(product.environments).flatMap((environment) => (environment.phone === undefined ? [] : [environment.phone]));
|
|
192
293
|
const redact = redactor([...secretValues(config, env), ...accountValues, ...phones]);
|
|
@@ -198,6 +299,24 @@ export async function buildKnowledge(cwd, env, o) {
|
|
|
198
299
|
// only the scheme pass writes scheme.md; a build stopped after the analyst rewrote it leaves a draft
|
|
199
300
|
const readScheme = () => readFile(join(knowledgeDir, "scheme.md"), "utf8").catch(() => undefined);
|
|
200
301
|
const schemeBefore = await readScheme();
|
|
302
|
+
const branch = env.OMNIQA_BRANCH ? { branch: env.OMNIQA_BRANCH } : {};
|
|
303
|
+
const chosen = o.skipCrawl ? undefined : crawlTarget(product, o.env);
|
|
304
|
+
/**
|
|
305
|
+
* The look before a walk: every pass signs in and opens the first page the walk would, and writes and sends nothing. The
|
|
306
|
+
* walk's own frontier (its budget, its lists, the pages the product names for it), so the two agree on whether there is a
|
|
307
|
+
* page to open; it stops at the first one that does.
|
|
308
|
+
*/
|
|
309
|
+
const look = (envName, environment, baseUrl, passes) => crawlPasses(passes, (page, pass) => crawlSite(page, { baseUrl, outDir: pass.outDir, maxPages: product.crawl.maxPages, maxDepth: product.crawl.maxDepth, exclude: product.crawl.exclude, deny: environment.deny, seeds: product.crawl.seeds, seed: environment, redact, probe: true, ...(pass.account === undefined ? {} : { account: pass.account }), log: (l) => progress(`crawl ${envName} as ${pass.as}: ${l}`) }), contextOptionsFor(environment, baseUrl));
|
|
310
|
+
// 0. Is there a site to crawl? Asked before anything is rewritten or paid for: every pass signs in and opens the start page,
|
|
311
|
+
// and writes nothing. A build that went on over a site that was down wrote empty request files and paid for a scheme of them.
|
|
312
|
+
if (chosen !== undefined) {
|
|
313
|
+
const baseUrl = resolveEnvironmentUrl(chosen.environment, branch);
|
|
314
|
+
const passes = passesOf(product, join(knowledgeDir, "crawl"));
|
|
315
|
+
const { results } = await look(chosen.name, chosen.environment, baseUrl, passes);
|
|
316
|
+
const no = noCrawl(chosen.name, baseUrl, passes, results);
|
|
317
|
+
if (no !== undefined)
|
|
318
|
+
throw new Error(stopLine(no, "Nothing was rewritten"));
|
|
319
|
+
}
|
|
201
320
|
try {
|
|
202
321
|
// 1. the repository maps with their dissection and the analyst summary, then the stamp every later file carries
|
|
203
322
|
if (o.skipStatic)
|
|
@@ -241,17 +360,34 @@ export async function buildKnowledge(cwd, env, o) {
|
|
|
241
360
|
// 3. the crawl with network capture, as a guest and as each account, on a production tier only where it explores; then, with
|
|
242
361
|
// security tests on, the security pass: the same walk of a production-tier environment behind the Interceptor sidecar. Where
|
|
243
362
|
// that environment is the crawl's own, the one crawl serves both.
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
let secure = config.scope.security ? securityTarget(product) : undefined;
|
|
247
|
-
let securitySkipped = !config.scope.security || secure !== undefined ? undefined : "no production-tier environment with explore on";
|
|
363
|
+
let secure = config.scope.security && o.skipSecurity !== true ? securityTarget(product) : undefined;
|
|
364
|
+
let securitySkipped = !config.scope.security || secure !== undefined ? undefined : o.skipSecurity === true ? `--skip-security: ${relative(cwd, join(knowledgeDir, SECURITY_FILE))} stays as it was` : "no production-tier environment with explore on";
|
|
248
365
|
let sidecar;
|
|
366
|
+
/** The forms of a security walk whose file was not written (an account went missing on the way). */
|
|
367
|
+
let sentAnyway;
|
|
368
|
+
/** What the security walk pressed with nothing let through, a line each (`pressedLines`), whether or not its file was written. */
|
|
369
|
+
let pressed = [];
|
|
370
|
+
/** What a sent form leaves (a message, a sign-up) stays in the product: each is named, with what keeps the walk from it. Then what the walk pressed. */
|
|
371
|
+
const saySent = (sent) => {
|
|
372
|
+
for (const form of sent.forms)
|
|
373
|
+
o.out(`security: sent the form ${JSON.stringify(form.button)} on ${form.page} as ${form.actors.join(", ")}, filled in with placeholders (omniqa, omniqa@example.com${sent.phone ? `, the test phone number of ${sent.environment}` : ""})${form.denied === undefined ? "" : `; what it asked for matches ${JSON.stringify(form.denied)} in the deny list, which did not keep it (a script sends this form): deny it by its button or its page`}`);
|
|
374
|
+
if (sent.forms.length > 0)
|
|
375
|
+
o.out(`security: what a form left stays in ${sent.environment}; mutations: false or a deny entry in ${PRODUCT_FILE} keeps the walk from sending`);
|
|
376
|
+
const foreign = foreignLine(sent.forms.filter((form) => form.foreign === true).map((form) => form.button));
|
|
377
|
+
if (foreign !== undefined)
|
|
378
|
+
o.out(foreign);
|
|
379
|
+
// A press leaves nothing in the product where its request was kept back, and is said all the same. Where it was not,
|
|
380
|
+
// something left as the account itself, or may have: the line a user must not miss.
|
|
381
|
+
for (const line of pressed)
|
|
382
|
+
o.out(line);
|
|
383
|
+
};
|
|
249
384
|
if (secure !== undefined) {
|
|
250
385
|
// Not installed, or it does not start: one line says so and the build goes on.
|
|
251
386
|
try {
|
|
252
387
|
if (!interceptorInstalled({ env }))
|
|
253
388
|
throw new Error("Interceptor is not installed: omniqa install interceptor");
|
|
254
|
-
|
|
389
|
+
// The hosts the product links to that its owner switched on for the security tests are read with the product's own.
|
|
390
|
+
sidecar = await startInterceptor({ hosts: [...hostPatterns(resolveEnvironmentUrl(secure.environment, branch)), ...linkedPatterns(switchedOn(await readHosts(cwd)).security)], env, ownCertificate: secure.environment.ownCertificate === true });
|
|
255
391
|
}
|
|
256
392
|
catch (err) {
|
|
257
393
|
securitySkipped = firstLine(err.message);
|
|
@@ -261,13 +397,23 @@ export async function buildKnowledge(cwd, env, o) {
|
|
|
261
397
|
try {
|
|
262
398
|
// `inputs`: a walk the security pass reads also opens what its pages take as input (a link with its query, what a form that reads leads to),
|
|
263
399
|
// and where the environment allows a change it fills in and sends each form that may be sent, once (a phone field with the
|
|
264
|
-
// environment's test number, when it names one).
|
|
265
|
-
|
|
400
|
+
// environment's test number, when it names one). Such a walk runs behind the proxy, so there it also presses each button
|
|
401
|
+
// that deletes or ends something, once, with the proxy letting nothing through (`keepBack`).
|
|
402
|
+
const proxy = sidecar;
|
|
403
|
+
const walk = (envName, environment, baseUrl, capture, label, kept, inputs) => (page, pass) => crawlSite(page, { baseUrl, outDir: pass.outDir, maxPages: product.crawl.maxPages, maxDepth: product.crawl.maxDepth, exclude: product.crawl.exclude, deny: environment.deny, seeds: product.crawl.seeds, seed: environment, capture, redact, inputs, mutations: environment.mutations, ...(inputs && environment.mutations && proxy !== undefined ? { keepBack: proxy } : {}), ...(environment.phone === undefined ? {} : { phone: environment.phone }), ...(kept ? { stamp } : {}), ...(pass.account === undefined ? {} : { account: pass.account }), log: (l) => progress(`${label} ${envName} as ${pass.as}: ${l}`) });
|
|
404
|
+
const keptFile = (no) => `${no.why}; ${relative(cwd, join(knowledgeDir, SECURITY_FILE))} stays as it was: ${no.fix}`;
|
|
266
405
|
const secured = async (baseUrl, passes, results, read, capture) => {
|
|
267
406
|
if (secure === undefined || sidecar === undefined)
|
|
268
407
|
return;
|
|
269
|
-
|
|
270
|
-
|
|
408
|
+
// A pass that was made pressed its buttons whatever becomes of the file: each is said.
|
|
409
|
+
pressed = pressedLines(passes.map((pass, i) => ({ as: pass.as, pressed: results[i]?.pressed ?? [] })), redact);
|
|
410
|
+
// As every actor, or not at all: a file written from a walk an account is missing from would lose that account's
|
|
411
|
+
// session, its pages and its statuses, and the checks planned from them. A guest alone kept out stops nothing here either.
|
|
412
|
+
const no = noCrawl(secure.name, baseUrl, passes, results);
|
|
413
|
+
if (no !== undefined) {
|
|
414
|
+
securitySkipped = keptFile(no);
|
|
415
|
+
// The passes that were made sent their forms all the same: each is named, as after a walk that is kept (and what they pressed with them).
|
|
416
|
+
sentAnyway = { environment: secure.name, phone: secure.environment.phone !== undefined, forms: formsSentBy(passes.map((pass, i) => ({ pass, result: results[i] })).filter((w) => w.result.unreachable === undefined), redact) };
|
|
271
417
|
return;
|
|
272
418
|
}
|
|
273
419
|
const written = await writeSecurityFile({ knowledgeDir, environment: secure.name, baseUrl, passes, results, read, capture, sidecar, stamp, redact });
|
|
@@ -283,12 +429,22 @@ export async function buildKnowledge(cwd, env, o) {
|
|
|
283
429
|
const capture = createNetMap(baseUrl);
|
|
284
430
|
const passes = passesOf(product, join(knowledgeDir, "crawl"));
|
|
285
431
|
const shared = secure?.name === envName;
|
|
286
|
-
const { results, ...read } = await crawlPasses(passes, walk(envName, environment, baseUrl, capture, "crawl", true, shared), shared ? sidecar : undefined);
|
|
432
|
+
const { results, ...read } = await crawlPasses(passes, walk(envName, environment, baseUrl, capture, "crawl", true, shared), contextOptionsFor(environment, baseUrl), shared ? sidecar : undefined);
|
|
433
|
+
// The site was there a moment ago (pass 0) and is not now, or an account no longer signs in: what is on disk of the requests and the scheme stays.
|
|
434
|
+
const gone = noCrawl(envName, baseUrl, passes, results);
|
|
435
|
+
if (gone !== undefined)
|
|
436
|
+
throw new Error(stopLine(gone, "The requests, the cross-reference and the scheme were not rewritten"));
|
|
287
437
|
const pages = results.reduce((n, r) => n + r.pages, 0);
|
|
288
438
|
const net = await writeNetMap(capture, knowledgeDir, stamp, { log: o.log, redact });
|
|
289
|
-
|
|
439
|
+
// Who the crawl was made as: a guest the product keeps out is named with why, not among the actors.
|
|
440
|
+
const walkedAs = passes.filter((_, i) => results[i]?.unreachable === undefined).map((p) => p.as);
|
|
441
|
+
const notAs = passes.flatMap((p, i) => (results[i]?.unreachable === undefined ? [] : [`${p.as} (${results[i]?.unreachable})`]));
|
|
442
|
+
report.crawl = { environment: envName, actors: walkedAs, pages, requests: net.requests, endpoints: net.endpoints, scripts: net.scripts, sourceMaps: net.sourceMaps, forms: net.forms, edges: net.edges };
|
|
290
443
|
await recordKnowledge(cwd, { lastCrawl: new Date().toISOString(), pages: results[passes.findIndex((p) => p.outDir === join(knowledgeDir, "crawl"))]?.pages ?? 0 });
|
|
291
|
-
o.out(`crawl ${envName} as ${
|
|
444
|
+
o.out(`crawl ${envName} as ${walkedAs.join(", ")}: ${pages} pages, ${net.requests} requests, ${net.endpoints} endpoints, ${plural(net.scripts, "script")} (${plural(net.sourceMaps, "source map")}), ${net.forms} forms, ${net.edges} links${notAs.length === 0 ? "" : `; not walked as ${notAs.join(", ")}`}`);
|
|
445
|
+
const hosts = await gatherHosts(cwd, baseUrl, results);
|
|
446
|
+
if (hosts !== undefined)
|
|
447
|
+
o.out(hosts);
|
|
292
448
|
if (shared)
|
|
293
449
|
await secured(baseUrl, passes, results, read, capture);
|
|
294
450
|
}
|
|
@@ -299,8 +455,17 @@ export async function buildKnowledge(cwd, env, o) {
|
|
|
299
455
|
const scratch = await mkdtemp(join(tmpdir(), "omniqa-security-"));
|
|
300
456
|
try {
|
|
301
457
|
const passes = passesOf(product, scratch);
|
|
302
|
-
|
|
303
|
-
|
|
458
|
+
// The look before, as the crawl has one: an account that does not sign in here is found before a form is sent.
|
|
459
|
+
const no = noCrawl(secure.name, baseUrl, passes, (await look(secure.name, secure.environment, baseUrl, passes)).results);
|
|
460
|
+
if (no !== undefined)
|
|
461
|
+
securitySkipped = keptFile(no);
|
|
462
|
+
else {
|
|
463
|
+
const { results, ...read } = await crawlPasses(passes, walk(secure.name, secure.environment, baseUrl, capture, "security", false, true), contextOptionsFor(secure.environment, baseUrl), sidecar);
|
|
464
|
+
const hosts = await gatherHosts(cwd, baseUrl, results);
|
|
465
|
+
if (hosts !== undefined)
|
|
466
|
+
o.out(hosts);
|
|
467
|
+
await secured(baseUrl, passes, results, read, capture);
|
|
468
|
+
}
|
|
304
469
|
}
|
|
305
470
|
finally {
|
|
306
471
|
await rm(scratch, { recursive: true, force: true });
|
|
@@ -312,11 +477,7 @@ export async function buildKnowledge(cwd, env, o) {
|
|
|
312
477
|
}
|
|
313
478
|
if (report.security !== undefined) {
|
|
314
479
|
o.out(`security ${report.security.environment} as ${report.security.actors.join(", ")}: ${report.security.pages} pages through Interceptor, ${plural(report.security.findings, "finding")}, ${plural(report.security.requests, "request")}${report.security.forms.length === 0 ? "" : `, ${plural(report.security.forms.length, "form")} sent`} → ${relative(cwd, join(knowledgeDir, SECURITY_FILE))}`);
|
|
315
|
-
|
|
316
|
-
for (const form of report.security.forms)
|
|
317
|
-
o.out(`security: sent the form ${JSON.stringify(form.button)} on ${form.page} as ${form.actors.join(", ")}, filled in with placeholders (omniqa, omniqa@example.com${report.security.phone ? `, the test phone number of ${report.security.environment}` : ""})${form.denied === undefined ? "" : `; what it asked for matches ${JSON.stringify(form.denied)} in the deny list, which did not keep it (a script sends this form): deny it by its button or its page`}`);
|
|
318
|
-
if (report.security.forms.length > 0)
|
|
319
|
-
o.out(`security: what a form left stays in ${report.security.environment}; mutations: false or a deny entry in ${PRODUCT_FILE} keeps the walk from sending`);
|
|
480
|
+
saySent(report.security);
|
|
320
481
|
// The security cases are planned from what the pass saw, by rule; the plan the planner wrote before this knowledge existed has none.
|
|
321
482
|
try {
|
|
322
483
|
const planned = await planSecurity(cwd, env, { firstOnly: true });
|
|
@@ -329,8 +490,11 @@ export async function buildKnowledge(cwd, env, o) {
|
|
|
329
490
|
o.out(`security: no case planned: ${firstLine(err.message)}`);
|
|
330
491
|
}
|
|
331
492
|
}
|
|
332
|
-
else if (securitySkipped !== undefined)
|
|
493
|
+
else if (securitySkipped !== undefined) {
|
|
333
494
|
o.out(`security: skipped (${securitySkipped})`);
|
|
495
|
+
if (sentAnyway !== undefined)
|
|
496
|
+
saySent(sentAnyway);
|
|
497
|
+
}
|
|
334
498
|
// 4. the cross-reference from the files on disk, so skipped passes still count
|
|
335
499
|
const x = await rebuildXref(knowledgeDir, stamp);
|
|
336
500
|
report.xref = { exact: x.exact, pattern: x.pattern, guess: x.guess, unmatchedEndpoints: x.unmatchedEndpoints, unmatchedHandlers: x.unmatchedHandlers, unmatchedPages: x.unmatchedPages };
|
|
@@ -381,13 +545,19 @@ export async function buildKnowledge(cwd, env, o) {
|
|
|
381
545
|
report.index = await indexProject(cwd);
|
|
382
546
|
o.out(`index: ${plural(report.index.chunks, "chunk")} from ${plural(report.index.files, "file")}`);
|
|
383
547
|
const builtAt = new Date().toISOString();
|
|
384
|
-
|
|
548
|
+
// A pass this build skipped left its files, so its counts stay too.
|
|
549
|
+
await recordKnowledge(cwd, { commit: report.commit, builtAt, scheme: report.scheme?.state ?? "draft", modelCalls: budget.used, counts: { ...(await readState(cwd)).knowledge?.counts, ...counts } });
|
|
385
550
|
o.out(`knowledge commit: ${report.commit} built ${builtAt}; ${plural(budget.used, "model call")} of ${limit}`);
|
|
386
551
|
}
|
|
387
552
|
catch (err) {
|
|
553
|
+
// The cross-reference is written after the walks: a build that ends past it has made them, and going on needs none again.
|
|
554
|
+
const walked = report.xref !== undefined;
|
|
555
|
+
const keep = walked ? `--skip-static --skip-crawl ${config.scope.security ? "--skip-security " : ""}` : "";
|
|
556
|
+
// A failure past the walks (the analyst ran out of steps, a model's key) carries the command too: `goOn` on the error.
|
|
388
557
|
if (!(err instanceof BudgetExceeded))
|
|
389
|
-
throw err;
|
|
558
|
+
throw walked ? Object.assign(err, { goOn: `omniqa knowledge build ${keep}`.trim() }) : err;
|
|
390
559
|
report.stopped = err.message;
|
|
560
|
+
report.goOn = { command: `omniqa knowledge build ${keep}--budget ${limit * 2}`, walked };
|
|
391
561
|
await recordKnowledge(cwd, { modelCalls: budget.used, ...((await readScheme()) === schemeBefore ? {} : { scheme: "draft" }) });
|
|
392
562
|
}
|
|
393
563
|
await writeCoverageFile(cwd).catch((err) => o.log(`coverage: ${err.message.split("\n")[0]}`));
|
|
@@ -407,7 +577,7 @@ export async function reviewKnowledge(cwd, env, o) {
|
|
|
407
577
|
if (text === undefined)
|
|
408
578
|
throw new Error("no scheme at .omniqa/knowledge/scheme.md yet: run omniqa knowledge build");
|
|
409
579
|
const budget = budgeted(createModels(config.models, { env }), 3);
|
|
410
|
-
const accountValues =
|
|
580
|
+
const accountValues = productSecrets(product);
|
|
411
581
|
const transcript = await openTranscript(join(cwd, RUNS_DIR), transcriptName("knowledge-review"), { redact: redactor([...secretValues(config, env), ...accountValues]) });
|
|
412
582
|
const creator = await createKnowledgeCreator({ cwd, env, config, product, models: budget.models, transcript, log: o.progress ?? o.log });
|
|
413
583
|
try {
|
|
@@ -438,7 +608,7 @@ export async function knowledge(args, cwd) {
|
|
|
438
608
|
}
|
|
439
609
|
let values;
|
|
440
610
|
try {
|
|
441
|
-
values = parseArgs({ args: rest, strict: true, options: { "skip-crawl": { type: "boolean" }, "skip-static": { type: "boolean" }, budget: { type: "string" }, env: { type: "string" }, help: { type: "boolean", short: "h" } } }).values;
|
|
611
|
+
values = parseArgs({ args: rest, strict: true, options: { "skip-crawl": { type: "boolean" }, "skip-static": { type: "boolean" }, "skip-security": { type: "boolean" }, budget: { type: "string" }, env: { type: "string" }, help: { type: "boolean", short: "h" } } }).values;
|
|
442
612
|
}
|
|
443
613
|
catch (err) {
|
|
444
614
|
stderr.write(`${err.message}\n\n${USAGE}`);
|
|
@@ -448,6 +618,12 @@ export async function knowledge(args, cwd) {
|
|
|
448
618
|
stdout.write(USAGE);
|
|
449
619
|
return 0;
|
|
450
620
|
}
|
|
621
|
+
// The review is the reviewer once more on the scheme as it is: a flag of the build would be read and do nothing.
|
|
622
|
+
const idle = sub === "review" ? ["budget", "skip-crawl", "skip-static", "skip-security", "env"].find((k) => values[k] !== undefined) : undefined;
|
|
623
|
+
if (idle !== undefined) {
|
|
624
|
+
stderr.write(`omniqa knowledge review: --${idle} is a flag of the build; the review is the reviewer once more on the scheme as it is, and takes none\n`);
|
|
625
|
+
return 1;
|
|
626
|
+
}
|
|
451
627
|
const budget = values.budget === undefined ? undefined : Number(values.budget);
|
|
452
628
|
if (budget !== undefined && (!Number.isInteger(budget) || budget < 1)) {
|
|
453
629
|
stderr.write("omniqa knowledge: --budget must be a positive integer\n");
|
|
@@ -460,6 +636,8 @@ export async function knowledge(args, cwd) {
|
|
|
460
636
|
o.skipCrawl = true;
|
|
461
637
|
if (values["skip-static"] === true)
|
|
462
638
|
o.skipStatic = true;
|
|
639
|
+
if (values["skip-security"] === true)
|
|
640
|
+
o.skipSecurity = true;
|
|
463
641
|
if (budget !== undefined)
|
|
464
642
|
o.budget = budget;
|
|
465
643
|
if (values.env !== undefined)
|
|
@@ -472,17 +650,21 @@ export async function knowledge(args, cwd) {
|
|
|
472
650
|
const report = await buildKnowledge(cwd, env, o);
|
|
473
651
|
if (report.stopped !== undefined) {
|
|
474
652
|
stdout.write(`omniqa knowledge build: stopped: ${report.stopped}\n`);
|
|
653
|
+
if (report.goOn?.walked === true)
|
|
654
|
+
stdout.write(`omniqa knowledge build: go on with ${report.goOn.command}: ${KEPT}\n`);
|
|
475
655
|
return 1;
|
|
476
656
|
}
|
|
477
657
|
return 0;
|
|
478
658
|
}
|
|
479
659
|
catch (err) {
|
|
480
660
|
stderr.write(`omniqa knowledge ${sub}: ${firstLine(err.message)}\n`);
|
|
661
|
+
const goOn = failedPast(err);
|
|
662
|
+
if (goOn !== undefined)
|
|
663
|
+
stderr.write(`omniqa knowledge build: once that is fixed, go on with ${goOn}: ${KEPT}\n`);
|
|
481
664
|
return 1;
|
|
482
665
|
}
|
|
483
666
|
finally {
|
|
484
667
|
sinks.done();
|
|
485
668
|
}
|
|
486
669
|
}
|
|
487
|
-
export const relativeKnowledge = (cwd, file) => relative(cwd, file);
|
|
488
670
|
//# sourceMappingURL=knowledge.js.map
|