omniqa 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -1
- package/dist/.tsbuildinfo +1 -1
- package/dist/auth.d.ts +17 -0
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +29 -1
- package/dist/auth.js.map +1 -1
- package/dist/cli.js +18 -5
- package/dist/cli.js.map +1 -1
- package/dist/commands/chat.js +1 -1
- package/dist/commands/chat.js.map +1 -1
- package/dist/commands/crawl.d.ts.map +1 -1
- package/dist/commands/crawl.js +17 -5
- package/dist/commands/crawl.js.map +1 -1
- package/dist/commands/doctor.d.ts +23 -0
- package/dist/commands/doctor.d.ts.map +1 -1
- package/dist/commands/doctor.js +152 -6
- package/dist/commands/doctor.js.map +1 -1
- package/dist/commands/generate.js +1 -1
- package/dist/commands/generate.js.map +1 -1
- package/dist/commands/init.d.ts +3 -1
- package/dist/commands/init.d.ts.map +1 -1
- package/dist/commands/init.js +217 -50
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/knowledge.d.ts +61 -1
- package/dist/commands/knowledge.d.ts.map +1 -1
- package/dist/commands/knowledge.js +232 -50
- package/dist/commands/knowledge.js.map +1 -1
- package/dist/commands/persist.d.ts +6 -0
- package/dist/commands/persist.d.ts.map +1 -1
- package/dist/commands/persist.js +9 -2
- package/dist/commands/persist.js.map +1 -1
- package/dist/commands/plan-add.d.ts.map +1 -1
- package/dist/commands/plan-add.js +13 -3
- package/dist/commands/plan-add.js.map +1 -1
- package/dist/commands/plan-review.d.ts +6 -2
- package/dist/commands/plan-review.d.ts.map +1 -1
- package/dist/commands/plan-review.js +16 -5
- package/dist/commands/plan-review.js.map +1 -1
- package/dist/commands/plan.d.ts +15 -0
- package/dist/commands/plan.d.ts.map +1 -1
- package/dist/commands/plan.js +103 -32
- package/dist/commands/plan.js.map +1 -1
- package/dist/commands/reindex.d.ts +1 -1
- package/dist/commands/reindex.d.ts.map +1 -1
- package/dist/commands/reindex.js +15 -1
- package/dist/commands/reindex.js.map +1 -1
- package/dist/commands/security.d.ts +4 -2
- package/dist/commands/security.d.ts.map +1 -1
- package/dist/commands/security.js +18 -7
- package/dist/commands/security.js.map +1 -1
- package/dist/commands/serve.d.ts +2 -0
- package/dist/commands/serve.d.ts.map +1 -1
- package/dist/commands/serve.js +126 -30
- package/dist/commands/serve.js.map +1 -1
- package/dist/commands/sync.d.ts.map +1 -1
- package/dist/commands/sync.js +19 -2
- package/dist/commands/sync.js.map +1 -1
- package/dist/commands/test.d.ts +5 -1
- package/dist/commands/test.d.ts.map +1 -1
- package/dist/commands/test.js +106 -6
- package/dist/commands/test.js.map +1 -1
- package/dist/commands/write.d.ts +8 -2
- package/dist/commands/write.d.ts.map +1 -1
- package/dist/commands/write.js +101 -18
- package/dist/commands/write.js.map +1 -1
- package/dist/devices.d.ts +3 -1
- package/dist/devices.d.ts.map +1 -1
- package/dist/devices.js +5 -3
- package/dist/devices.js.map +1 -1
- package/dist/fixer.d.ts.map +1 -1
- package/dist/fixer.js +19 -9
- package/dist/fixer.js.map +1 -1
- package/dist/jobs.d.ts +6 -0
- package/dist/jobs.d.ts.map +1 -1
- package/dist/jobs.js +10 -1
- package/dist/jobs.js.map +1 -1
- package/dist/mcp/server.d.ts +1 -1
- package/dist/mcp/server.d.ts.map +1 -1
- package/dist/mcp/server.js +1 -1
- package/dist/mcp/server.js.map +1 -1
- package/dist/mcp/session.d.ts.map +1 -1
- package/dist/mcp/session.js +7 -7
- package/dist/mcp/session.js.map +1 -1
- package/dist/mcp/tools.d.ts.map +1 -1
- package/dist/mcp/tools.js +9 -6
- package/dist/mcp/tools.js.map +1 -1
- package/dist/page.d.ts +1 -1
- package/dist/page.d.ts.map +1 -1
- package/dist/page.js +11 -0
- package/dist/page.js.map +1 -1
- package/dist/planner.d.ts.map +1 -1
- package/dist/planner.js +8 -4
- package/dist/planner.js.map +1 -1
- package/dist/prompt.d.ts +7 -1
- package/dist/prompt.d.ts.map +1 -1
- package/dist/prompt.js +23 -4
- package/dist/prompt.js.map +1 -1
- package/dist/qa.d.ts +7 -0
- package/dist/qa.d.ts.map +1 -1
- package/dist/qa.js +34 -9
- package/dist/qa.js.map +1 -1
- package/dist/runner.d.ts +7 -2
- package/dist/runner.d.ts.map +1 -1
- package/dist/runner.js +28 -8
- package/dist/runner.js.map +1 -1
- package/dist/security-checks.d.ts +46 -4
- package/dist/security-checks.d.ts.map +1 -1
- package/dist/security-checks.js +114 -39
- package/dist/security-checks.js.map +1 -1
- package/dist/security-gates.d.ts +64 -7
- package/dist/security-gates.d.ts.map +1 -1
- package/dist/security-gates.js +125 -16
- package/dist/security-gates.js.map +1 -1
- package/dist/security-mask.d.ts +35 -0
- package/dist/security-mask.d.ts.map +1 -0
- package/dist/security-mask.js +104 -0
- package/dist/security-mask.js.map +1 -0
- package/dist/security-report.d.ts +93 -2
- package/dist/security-report.d.ts.map +1 -1
- package/dist/security-report.js +153 -14
- package/dist/security-report.js.map +1 -1
- package/dist/security-request.d.ts +27 -4
- package/dist/security-request.d.ts.map +1 -1
- package/dist/security-request.js +56 -14
- package/dist/security-request.js.map +1 -1
- package/dist/security-research.d.ts +132 -0
- package/dist/security-research.d.ts.map +1 -0
- package/dist/security-research.js +599 -0
- package/dist/security-research.js.map +1 -0
- package/dist/security-scan.d.ts +98 -0
- package/dist/security-scan.d.ts.map +1 -0
- package/dist/security-scan.js +463 -0
- package/dist/security-scan.js.map +1 -0
- package/dist/security.d.ts +18 -4
- package/dist/security.d.ts.map +1 -1
- package/dist/security.js +181 -37
- package/dist/security.js.map +1 -1
- package/dist/sync.d.ts +55 -3
- package/dist/sync.d.ts.map +1 -1
- package/dist/sync.js +403 -54
- package/dist/sync.js.map +1 -1
- package/dist/test.d.ts +26 -2
- package/dist/test.d.ts.map +1 -1
- package/dist/test.js +152 -29
- package/dist/test.js.map +1 -1
- package/dist/writer.d.ts +29 -10
- package/dist/writer.d.ts.map +1 -1
- package/dist/writer.js +116 -36
- package/dist/writer.js.map +1 -1
- package/package.json +7 -7
- package/skills/knowledge.md +1 -1
- package/skills/security-research.md +57 -0
- package/skills/security-review.md +27 -0
- package/skills/onboarding.md +0 -5
package/dist/commands/init.js
CHANGED
|
@@ -3,9 +3,9 @@ import { basename, dirname, join, relative, resolve, sep as pathSep } from "node
|
|
|
3
3
|
import { stderr, stdout } from "node:process";
|
|
4
4
|
import { parseArgs, styleText } from "node:util";
|
|
5
5
|
import { installInterceptor, INTERCEPTOR_VERSION } from "@omniqa/browser";
|
|
6
|
-
import { DEFAULT_KEY_ENV, DEVICE_PLATFORMS, git, loadConfig, parseRemote, PRODUCT_FILE, readmeHasBlock, readState, README_FILE, recordOnboarding, remoteUrl, writeReadme } from "@omniqa/core";
|
|
6
|
+
import { DEFAULT_KEY_ENV, DEVICE_PLATFORMS, git, loadConfig, parseRemote, PRODUCT_FILE, readmeHasBlock, readState, README_FILE, recordOnboarding, remoteUrl, RUNS_DIR, transcriptName, writeReadme } from "@omniqa/core";
|
|
7
7
|
import { deploySetup } from "../deploy-setup.js";
|
|
8
|
-
import { DEPLOY_WORKFLOW, GITLAB_CI, hasWriteJob, WRITE_WORKFLOW, writeJobGitlab, writeWorkflow } from "../jobs.js";
|
|
8
|
+
import { DEPLOY_WORKFLOW, GITLAB_CI, GITLAB_PUSH, hasWriteJob, WRITE_WORKFLOW, writeJobGitlab, writeWorkflow } from "../jobs.js";
|
|
9
9
|
import { linkOmniqa, PACKAGE_DIR } from "../link.js";
|
|
10
10
|
import { lineReader } from "../prompt.js";
|
|
11
11
|
import { projectCi } from "./doctor.js";
|
|
@@ -82,7 +82,8 @@ function ciTest(a) {
|
|
|
82
82
|
*/
|
|
83
83
|
const secEnvironment = (a) => (a.security === true ? a.environments.find((e) => e.tier === "production" && e.explore && !onThisMachine(e.url))?.name : undefined);
|
|
84
84
|
/** What the CI test job runs with security tests on: they are the `sec` job's, so the other job leaves them out. The tag as a whole word: a tag of the user's that starts like it (@security) stays in. */
|
|
85
|
-
|
|
85
|
+
// --pass-with-no-tests: a tier, or a project, that has no test yet is no red job (a pattern that matches nothing fails a run otherwise).
|
|
86
|
+
const ciTestCommand = (a) => `${ciTest(a).command}${a.security === true ? ' --grep-invert "@sec\\b"' : ""} --pass-with-no-tests`;
|
|
86
87
|
/** Under a CI file, with the kind on: the job that runs the security tests, or why there is none yet. */
|
|
87
88
|
const secNote = (a) => {
|
|
88
89
|
if (a.security !== true)
|
|
@@ -90,15 +91,20 @@ const secNote = (a) => {
|
|
|
90
91
|
const at = secEnvironment(a);
|
|
91
92
|
return at === undefined
|
|
92
93
|
? `sec: no job yet: every production-tier environment that explores runs on this machine, not on a CI runner; once one is deployed, omniqa doctor names the job to add`
|
|
93
|
-
: `sec: the security tests on ${at}, apart from the others`;
|
|
94
|
+
: `sec: the security tests on ${at}, apart from the others${a.securityActive === true ? ", then the agent scan there (model calls on every run of the job; its report is the job's artifact)" : ""}`;
|
|
94
95
|
};
|
|
96
|
+
/** The agent scan's two roles: asked for when an environment's scan is on, and read by no stage of init. */
|
|
97
|
+
const SCAN_ROLES = ["security", "securityReviewer"];
|
|
95
98
|
// Enter at the model questions, and --yes, write the provider's suggestion: Sonnet 5.5 at effort high, the executor on the same model at
|
|
96
99
|
// medium (one action or one verdict a call, where more effort buys nothing measurable) and the reviewer Opus 5.5 at high (the setup the
|
|
97
100
|
// install guide measured), through OpenRouter by its ids (the executor and the verifier read screenshots, and Claude does); on OpenAI,
|
|
98
101
|
// GPT-6.1 Sol and the flagship GPT-6 Astra for the reviewer.
|
|
99
|
-
|
|
102
|
+
// The agent scan's two roles are the maintainer's choice: through OpenRouter the researcher on Kimi K3.1 Pro and its reviewer on Opus 5.5, which
|
|
103
|
+
// Opus 4.8 answers for when it declines, each at effort high; on Anthropic's API, which does not serve the first, Opus 5.5 for both with that fallback.
|
|
104
|
+
const OPUS_SCAN = { model: "claude-opus-5-5", effort: "high", fallback: "claude-opus-4-8" };
|
|
105
|
+
const ANTHROPIC = { name: "anthropic", label: "Anthropic", provider: "anthropic", keyEnv: "ANTHROPIC_API_KEY", models: { default: { model: "claude-sonnet-5-5", effort: "high" }, executor: { model: "claude-sonnet-5-5", effort: "medium" }, reviewer: { model: "claude-opus-5-5", effort: "high" } }, scan: { security: OPUS_SCAN, securityReviewer: OPUS_SCAN } };
|
|
100
106
|
const OPENAI = { name: "openai", label: "OpenAI", provider: "openai-compatible", baseUrl: "https://api.openai.com/v1", keyEnv: "OPENAI_API_KEY", models: { default: { model: "gpt-6.1-sol", effort: "high" }, executor: { model: "gpt-6.1-sol", effort: "medium" }, reviewer: { model: "gpt-6-astra", effort: "high" } } };
|
|
101
|
-
const OPENROUTER = { name: "openrouter", label: "OpenRouter", provider: "openai-compatible", baseUrl: "https://openrouter.ai/api/v1", keyEnv: "OPENROUTER_API_KEY", models: { default: { model: "anthropic/claude-sonnet-5.5", effort: "high" }, executor: { model: "anthropic/claude-sonnet-5.5", effort: "medium" }, reviewer: { model: "anthropic/claude-opus-5.5", effort: "high" } } };
|
|
107
|
+
const OPENROUTER = { name: "openrouter", label: "OpenRouter", provider: "openai-compatible", baseUrl: "https://openrouter.ai/api/v1", keyEnv: "OPENROUTER_API_KEY", models: { default: { model: "anthropic/claude-sonnet-5.5", effort: "high" }, executor: { model: "anthropic/claude-sonnet-5.5", effort: "medium" }, reviewer: { model: "anthropic/claude-opus-5.5", effort: "high" } }, scan: { security: { model: "moonshotai/kimi-k3.1-pro", effort: "high" }, securityReviewer: { model: "anthropic/claude-opus-5.5", effort: "high", fallback: "anthropic/claude-opus-4.8" } } };
|
|
102
108
|
const OTHER = { name: "other", label: "other OpenAI-compatible API", provider: "openai-compatible", keyEnv: "OPENAI_API_KEY" };
|
|
103
109
|
const CHOICES = [ANTHROPIC, OPENAI, OPENROUTER, OTHER];
|
|
104
110
|
/** The choice a provider and its base url make: OpenAI's or OpenRouter's url, else another API. */
|
|
@@ -117,9 +123,16 @@ const exploreComment = (env) => `true: the model plans a step without a recordin
|
|
|
117
123
|
const REPLAY = "true: replay the recordings, model calls only for a step without one or one that misses; false: every run plans each step and verifies each assert through the models, the recordings stay the reference (omniqa test --replay or --no-replay switches one run)";
|
|
118
124
|
const DENY = "url substrings and control texts the agent never acts on here";
|
|
119
125
|
const PHONE = "a test number of your product's own: the security tests type it into a field that takes a phone number; without it such a field stays empty (any other number may be somebody's)";
|
|
120
|
-
const SECURITY = "
|
|
126
|
+
const SECURITY = "off: the security tests alone, which send the product's forms and requests again with probe values and a few probes of their own (well-known files, wrong passwords), changing data only where mutations is true; on: an agent scan follows them here, sending requests of its own (model calls on every such run)";
|
|
121
127
|
const KINDS_QUESTION = "Testing kinds (1. e2e, 2. API, 3. security, 4. all; several as 2,3; e2e is always included)";
|
|
122
|
-
const SECURITY_QUESTION = "
|
|
128
|
+
const SECURITY_QUESTION = "Agent scan on production (securityScan): 1. off (the security tests alone: the guard and the basic checks, which send the product's forms and requests again with probe values and a few probes of their own, such as wrong passwords; they change data only where mutations is true), 2. on (omniqa test and omniqa security go on to an agent scan there: a researcher that sends requests of its own and a reviewer that checks its claims; model calls on every such run)";
|
|
129
|
+
/** What each of the scan's roles is, in its model question. */
|
|
130
|
+
const SCAN_ROLE = {
|
|
131
|
+
security: "the researcher of the agent scan: it reads the product's traffic and sends requests of its own",
|
|
132
|
+
securityReviewer: "the reviewer of the agent scan: it confirms or discards each claim of the researcher",
|
|
133
|
+
};
|
|
134
|
+
/** Said before the two questions, where the user picks who reads it: what an agent is shown of the traffic leaves the machine. */
|
|
135
|
+
const SCAN_TRAFFIC = "The agent scan reads the product's traffic: excerpts of its requests and answers go to the providers of the next two models, with every credential masked (session cookies, Authorization values, the accounts' secrets).";
|
|
123
136
|
/**
|
|
124
137
|
* The answer to the kinds question: numbers or names, several at once (`2,3`, `api security`, `all`); e2e is always on. The answers
|
|
125
138
|
* of the question this one replaced (`e2e`, `api`, `both`, `e2e and api`) mean what they meant, so an answers file still works.
|
|
@@ -141,7 +154,8 @@ const USAGE = `omniqa init [folder] [options]
|
|
|
141
154
|
folder Create it (and its parents) and set the project up inside; the product name defaults to its name, and the
|
|
142
155
|
output ends by saying cd <folder>
|
|
143
156
|
--yes, -y Use defaults for every prompt and stop after the files
|
|
144
|
-
--force
|
|
157
|
+
--force Ask the interview again, the earlier answers as its defaults, and write the files again (one changed by
|
|
158
|
+
hand since is kept under .omniqa/runs/)
|
|
145
159
|
--jobs For a project set up by an older OmniQA: add the CI job it is missing and change nothing else. Today that is the
|
|
146
160
|
job that writes the tests you tick on a pull request OmniQA opened after a deploy (.github/workflows/omniqa-write.yml,
|
|
147
161
|
or the manual job omniqa write selected in .omniqa/ci/gitlab-ci.yml); then commit and push it
|
|
@@ -173,6 +187,20 @@ const USAGE = `omniqa init [folder] [options]
|
|
|
173
187
|
async function exists(path) {
|
|
174
188
|
return access(path).then(() => true, () => false);
|
|
175
189
|
}
|
|
190
|
+
/** The flags that answer the interview: read where it is asked or taken as answered (--yes), never by an init that resumes at a later stage. */
|
|
191
|
+
const ANSWER_FLAGS = ["name", "url", "prod-url", "provider", "model", "base-url", "key-env", "test-id", "ci", "pages", "repo", "deploy-from"];
|
|
192
|
+
/** The answers state.json keeps, when they are ones the interview can offer again: what it reads of them is there and of its kind. */
|
|
193
|
+
function storedAnswers(v) {
|
|
194
|
+
const a = v;
|
|
195
|
+
const strings = (x) => Array.isArray(x) && x.every((item) => typeof item === "string");
|
|
196
|
+
const record = (x) => typeof x === "object" && x !== null && !Array.isArray(x);
|
|
197
|
+
const ok = record(a) && typeof a.name === "string" && typeof a.provider === "string" && typeof a.keyEnv === "string" && typeof a.testId === "string" && typeof a.ci === "string" &&
|
|
198
|
+
Array.isArray(a.environments) && a.environments.length > 0 && a.environments.every((e) => record(e) && typeof e.name === "string" && typeof e.url === "string") &&
|
|
199
|
+
strings(a.repos) && strings(a.docs) && strings(a.notes) && record(a.apps) &&
|
|
200
|
+
Array.isArray(a.accounts) && a.accounts.every((acc) => record(acc) && typeof acc.name === "string" && typeof acc.role === "string" && record(acc.fields)) &&
|
|
201
|
+
record(a.models) && Object.values(a.models).every((m) => record(m) && typeof m.model === "string");
|
|
202
|
+
return ok ? a : undefined;
|
|
203
|
+
}
|
|
176
204
|
/** Today's `--yes` defaults, from the flags. */
|
|
177
205
|
export function defaultsFrom(cwd, flags, remote, repos = []) {
|
|
178
206
|
const ci = flags.ci ?? ciDefault(repos, remote);
|
|
@@ -194,7 +222,7 @@ export function defaultsFrom(cwd, flags, remote, repos = []) {
|
|
|
194
222
|
throw new Error(`--key-env takes ${keyEnvHint(choice.keyEnv)}`);
|
|
195
223
|
for (const flag of ["url", "prod-url"])
|
|
196
224
|
if (flags[flag] !== undefined && !isEnvironmentUrl(flags[flag]))
|
|
197
|
-
throw new Error(`--${flag} takes ${
|
|
225
|
+
throw new Error(`--${flag} takes ${environmentUrlHint(flags[flag])}`);
|
|
198
226
|
// ponytail: --yes without --url scaffolds the placeholder, which the interview never offers and the plan stage cannot crawl; make --url required there if that bites.
|
|
199
227
|
const environments = [{ name: "staging", url: flags.url ?? "https://staging.example.com", tier: "dev", explore: true, mutations: true }];
|
|
200
228
|
if (flags["prod-url"] !== undefined)
|
|
@@ -213,15 +241,30 @@ export function defaultsFrom(cwd, flags, remote, repos = []) {
|
|
|
213
241
|
const envName = (s) => s.toUpperCase().replace(/[^A-Z0-9]+/g, "_").replace(/^_+|_+$/g, "");
|
|
214
242
|
/** What an environment's url is: where the product is running now, so the groundwork crawl reaches it. example.com is the documentation's illustration; nothing runs there. */
|
|
215
243
|
const URL_HINT = "an http:// or https:// url where the product is running (a local run too, e.g. http://127.0.0.1:9000); not example.com, the documentation's placeholder";
|
|
216
|
-
|
|
244
|
+
/** The url is written to product.yaml and state.json, printed by `omniqa status` and put into the prompts: it holds no secret. */
|
|
245
|
+
const URL_KEPT = `${PRODUCT_FILE} and .omniqa/state.json, which git keeps`;
|
|
246
|
+
/** Why a url is not an environment's, as the hint that says what one is; undefined when it is one. The hint never repeats the answer. */
|
|
247
|
+
const environmentUrlHint = (url) => {
|
|
248
|
+
let u;
|
|
217
249
|
try {
|
|
218
|
-
|
|
219
|
-
return /^https?:$/.test(u.protocol) && !/(^|\.)example\.com$/i.test(u.hostname);
|
|
250
|
+
u = new URL(url);
|
|
220
251
|
}
|
|
221
252
|
catch {
|
|
222
|
-
return
|
|
253
|
+
return URL_HINT;
|
|
223
254
|
}
|
|
255
|
+
if (!/^https?:$/.test(u.protocol) || /(^|\.)example\.com$/i.test(u.hostname))
|
|
256
|
+
return URL_HINT;
|
|
257
|
+
if (u.username !== "" || u.password !== "")
|
|
258
|
+
return `a url without a name and a password before its host (user:password@): the url is written to ${URL_KEPT}, printed by omniqa status and shown to the models. A site behind an HTTP sign-in gets its url as it is here, and httpAuth: { username, password: $VAR } on its environment in ${PRODUCT_FILE} afterwards`;
|
|
259
|
+
if (u.search !== "" || u.hash !== "")
|
|
260
|
+
return `a url without a query or a fragment: pages are opened under the url and neither is carried to them, and a token there would be written to ${URL_KEPT}, and shown to the models`;
|
|
261
|
+
return undefined;
|
|
224
262
|
};
|
|
263
|
+
const isEnvironmentUrl = (url) => environmentUrlHint(url) === undefined;
|
|
264
|
+
/** An environment's name is a key of product.yaml, what --env takes and a word of the CI jobs' commands. */
|
|
265
|
+
const ENVIRONMENT_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
|
266
|
+
/** An account's name is a key of product.yaml and a folder of the crawl, its role an item of the roles list; in any script. */
|
|
267
|
+
const ACCOUNT_NAME = /^[\p{L}\p{N}][\p{L}\p{N} ._-]*$/u;
|
|
225
268
|
const ENV_NAME = /^[A-Z_][A-Z0-9_]*$/;
|
|
226
269
|
/** What a key name is: the variable's name, never the key. A key pasted in its place would be written to the config, .env.example, the CI workflow and state.json, so nothing built from this repeats the value. */
|
|
227
270
|
const keyEnvHint = (example) => `the name of the env var that holds the model key (A-Z, 0-9 and _, e.g. ${example}), not the key itself`;
|
|
@@ -238,12 +281,13 @@ export async function interview(reader, d, o = {}) {
|
|
|
238
281
|
/** Asks until `parse` takes the answer; input that has ended would give the same answer forever, so it stops the interview there. */
|
|
239
282
|
const askUntil = async (label, def, parse, hint) => {
|
|
240
283
|
for (;;) {
|
|
241
|
-
const
|
|
284
|
+
const answer = await ask(label, def);
|
|
285
|
+
const value = parse(answer);
|
|
242
286
|
if (value !== undefined)
|
|
243
287
|
return value;
|
|
244
288
|
if (ended)
|
|
245
289
|
throw new Error(`omniqa init: the input ended before "${label.trim()}" was answered`);
|
|
246
|
-
stdout.write(` ${hint}\n`);
|
|
290
|
+
stdout.write(` ${typeof hint === "string" ? hint : hint(answer)}\n`);
|
|
247
291
|
}
|
|
248
292
|
};
|
|
249
293
|
const yes = async (label, def) => /^y(es)?$/i.test(await ask(label, def ? "y" : "n"));
|
|
@@ -270,12 +314,13 @@ export async function interview(reader, d, o = {}) {
|
|
|
270
314
|
const environments = [];
|
|
271
315
|
for (let i = 1;; i++) {
|
|
272
316
|
const given = d.environments[i - 1];
|
|
273
|
-
|
|
317
|
+
// Two keys of one name, or a name that ends at its colon, are a product.yaml no command loads.
|
|
318
|
+
const envNameAnswer = await askUntil(`Environment ${i} name${i === 1 ? "" : ", empty when done"}`, given?.name ?? "", (n) => (n === "" || (ENVIRONMENT_NAME.test(n) && !environments.some((e) => e.name === n)) ? n : undefined), `a name of letters, digits, ., _ and - that no other environment has (a key of ${PRODUCT_FILE}, and what --env takes)`);
|
|
274
319
|
if (envNameAnswer === "")
|
|
275
320
|
break;
|
|
276
321
|
const defTier = given?.tier ?? (envNameAnswer === "production" ? "production" : "dev");
|
|
277
322
|
// No placeholder to take with Enter: a url from the flags is the default, the --yes scaffold's example.com is not.
|
|
278
|
-
const url = await askUntil(" url where the product is running (a local run too, e.g. http://127.0.0.1:9000)", given !== undefined && isEnvironmentUrl(given.url) ? given.url : "", (u) => (isEnvironmentUrl(u) ? u : undefined), URL_HINT);
|
|
323
|
+
const url = await askUntil(" url where the product is running (a local run too, e.g. http://127.0.0.1:9000)", given !== undefined && isEnvironmentUrl(given.url) ? given.url : "", (u) => (isEnvironmentUrl(u) ? u : undefined), (u) => environmentUrlHint(u) ?? URL_HINT);
|
|
279
324
|
const tier = await choice(" tier, dev or production", ["dev", "production"], defTier);
|
|
280
325
|
const explore = await yes(" allow exploration (y: the model plans a step without a recording and heals a stale one, on production too; n: replay only)", given?.explore ?? true);
|
|
281
326
|
const mutations = await yes(` allow mutations: n makes the environment ${READ_ONLY}`, given?.mutations ?? true);
|
|
@@ -303,7 +348,7 @@ export async function interview(reader, d, o = {}) {
|
|
|
303
348
|
security = false;
|
|
304
349
|
}
|
|
305
350
|
else if (security) {
|
|
306
|
-
securityActive = await askUntil(SECURITY_QUESTION, d.securityActive === true ? "2" : "1", (answer) => ({ "1": false, basic: false, "2": true, active: true })[answer.toLowerCase()], "please answer 1 or 2");
|
|
351
|
+
securityActive = await askUntil(SECURITY_QUESTION, d.securityActive === true ? "2" : "1", (answer) => ({ "1": false, off: false, basic: false, "2": true, on: true, active: true })[answer.toLowerCase()], "please answer 1 or 2");
|
|
307
352
|
}
|
|
308
353
|
const apps = {};
|
|
309
354
|
const platforms = (await ask(`Native apps: none, or platforms among ${DEVICE_PLATFORMS.join(", ")}`, Object.keys(d.apps).join(", ") || "none")).toLowerCase();
|
|
@@ -322,11 +367,13 @@ export async function interview(reader, d, o = {}) {
|
|
|
322
367
|
const accounts = [];
|
|
323
368
|
for (;;) {
|
|
324
369
|
const given = d.accounts[accounts.length];
|
|
325
|
-
const accountName = await
|
|
370
|
+
const accountName = await askUntil("Account name, empty when done", given?.name ?? "", (n) => (n === "" || (ACCOUNT_NAME.test(n) && !accounts.some((acc) => acc.name === n)) ? n : undefined), `a name of letters, digits, spaces, ., _ and - that no other account has (a key of ${PRODUCT_FILE} and a folder of the crawl)`);
|
|
326
371
|
if (accountName === "")
|
|
327
372
|
break;
|
|
328
|
-
const role = await
|
|
329
|
-
|
|
373
|
+
const role = await askUntil(" role", given?.role ?? "member", (n) => (ACCOUNT_NAME.test(n) ? n : undefined), `a name of letters, digits, spaces, ., _ and - (an item of roles: in ${PRODUCT_FILE})`);
|
|
374
|
+
// A name that gives no variable name (another script), or the one an earlier account's fields have, is suggested by its place.
|
|
375
|
+
const stem = `OMNIQA_${envName(accountName)}`;
|
|
376
|
+
const prefix = stem === "OMNIQA_" || accounts.some((acc) => Object.values(acc.fields).some((v) => v.startsWith(`${stem}_`))) ? `OMNIQA_ACCOUNT_${accounts.length + 1}` : stem;
|
|
330
377
|
const defFields = given === undefined ? `email=${prefix}_EMAIL password=${prefix}_PASSWORD` : Object.entries(given.fields).map(([k, v]) => `${k}=${v}`).join(" ");
|
|
331
378
|
let fields;
|
|
332
379
|
while (fields === undefined) {
|
|
@@ -373,6 +420,20 @@ export async function interview(reader, d, o = {}) {
|
|
|
373
420
|
if (answer.model !== defaultModel.model || answer.effort !== defaultModel.effort)
|
|
374
421
|
models[role] = answer;
|
|
375
422
|
}
|
|
423
|
+
// The scan's two roles are asked for only where a scan will run. The provider's suggestion is the maintainer's choice, whatever the default model is.
|
|
424
|
+
if (securityActive) {
|
|
425
|
+
stdout.write(` ${SCAN_TRAFFIC}\n`);
|
|
426
|
+
for (const role of SCAN_ROLES) {
|
|
427
|
+
const def = chosen.scan?.[role] ?? defaultModel;
|
|
428
|
+
const answer = await askModel(`${role} model and effort (${SCAN_ROLE[role]}${def.fallback === undefined ? "" : `; ${def.fallback} answers when ${def.model} declines or its provider fails`})`, def);
|
|
429
|
+
// The suggested fallback stays with the suggested model; another model's is the user's to name in the config.
|
|
430
|
+
const fallback = answer.model === def.model ? def.fallback : undefined;
|
|
431
|
+
if (fallback !== undefined)
|
|
432
|
+
models[role] = { ...answer, fallback };
|
|
433
|
+
else if (answer.model !== defaultModel.model || answer.effort !== defaultModel.effort)
|
|
434
|
+
models[role] = answer;
|
|
435
|
+
}
|
|
436
|
+
}
|
|
376
437
|
const testId = await ask("Test id attribute", d.testId);
|
|
377
438
|
const notes = await list("Anything else the agents should know, one note a line (e.g. production signs in through Google only; written to .omniqa/instructions.md, which every model prompt carries), empty when done", d.notes);
|
|
378
439
|
const a = { name, environments, repos, docs, api, apps, accounts, provider, keyEnv, models, testId, ci, pages, notes };
|
|
@@ -397,6 +458,8 @@ const modelLine = (a, m) => {
|
|
|
397
458
|
parts.push(`apiKeyEnv: ${JSON.stringify(a.keyEnv)}`);
|
|
398
459
|
if (m.effort !== undefined)
|
|
399
460
|
parts.push(`effort: ${JSON.stringify(m.effort)}`);
|
|
461
|
+
if (m.fallback !== undefined)
|
|
462
|
+
parts.push(`fallback: ${JSON.stringify(m.fallback)}`);
|
|
400
463
|
return `{ ${parts.join(", ")} }`;
|
|
401
464
|
};
|
|
402
465
|
function environmentBlock(e, first, a) {
|
|
@@ -414,10 +477,11 @@ function environmentBlock(e, first, a) {
|
|
|
414
477
|
if (e.tier === "production")
|
|
415
478
|
lines.push(" # api: { mode: page } # API tests go through a page that opened the url first (the production default; a host behind bot management serves browsers only); request is the dev default");
|
|
416
479
|
if (e.tier === "production")
|
|
417
|
-
lines.push(a.security === true ? `
|
|
480
|
+
lines.push(a.security === true ? ` securityScan: ${(a.securityActive === true ? "on" : "off").padEnd(8)}# ${SECURITY}` : ` # securityScan: off # ${SECURITY}`);
|
|
418
481
|
if (e.tier === "production" && a.security === true)
|
|
419
482
|
lines.push(` # phone: "+15550100" # ${PHONE}`);
|
|
420
483
|
if (first) {
|
|
484
|
+
lines.push(" # httpAuth: { username: preview, password: $STAGING_PASSWORD } # the site sits behind an HTTP sign-in (a staging server's prompt): answered on this environment's host alone, the password by a variable's name", " # ownCertificate: true # the site's certificate is its own (self-signed, an authority this machine does not know): taken for this environment, refused for every other");
|
|
421
485
|
lines.push(' # cookies: [{ name: cookiesAccepted, value: "1" }] # seeded on every browser context before the first page: what the product stores when its cookie banner is answered', ' # localStorage: [{ key: hideNewsModal, value: "1" }] # likewise for a news or promo modal; values may be $VAR references');
|
|
422
486
|
lines.push(" # apps: # the build each device project opens here, by platform (omniqa.config.ts projects with a platform)", " # ios: com.example.app", " # android: com.example.app");
|
|
423
487
|
}
|
|
@@ -446,7 +510,7 @@ const described = (rel, ci, names, pages, changelog, ciNote, sec) => {
|
|
|
446
510
|
if (rel.endsWith(WRITE_WORKFLOW))
|
|
447
511
|
return [`${rel} (the tests you tick on a pull request OmniQA opened are written into it)`];
|
|
448
512
|
if (rel.endsWith(".github/workflows/omniqa.yml"))
|
|
449
|
-
return [`${rel} (the tests on every push
|
|
513
|
+
return [`${rel} (the tests on every push that touches a test file, and by hand from the Actions tab; heals and the refreshed coverage come back as a commit, or a pull request on a protected branch)`, ...note, ...secrets, ...(changelog === undefined ? [` ${bold("allow GitHub Actions to create and approve pull requests under Settings → Actions → General → Workflow permissions (heals come as pull requests on a protected branch or from a production environment)")}`] : [])];
|
|
450
514
|
if (rel.endsWith(".github/workflows/omniqa-pages.yml"))
|
|
451
515
|
return [`${rel} (the scheme page on GitHub Pages after every push to the default branch that changes it)`, ` ${bold("enable Pages under Settings → Pages with GitHub Actions as the source; the site is public unless the organization is on GitHub Enterprise")}`];
|
|
452
516
|
if (rel === ".omniqa/ci/gitlab-ci.yml")
|
|
@@ -454,6 +518,8 @@ const described = (rel, ci, names, pages, changelog, ciNote, sec) => {
|
|
|
454
518
|
return [rel];
|
|
455
519
|
};
|
|
456
520
|
const NPM_COMMENT = "The newest OmniQA from npm (pin omniqa@<version> here to hold CI on one).";
|
|
521
|
+
/** What the sec job says on a public repository, where the scan does not run (omniqa security sees to that itself, and says so in its own line). A plain YAML scalar inside an echo: no colon followed by a space, no dollar sign. */
|
|
522
|
+
const PUBLIC_NOTICE = "this repository is public, so the agent scan does not run here (its report could be kept neither as an artifact nor in the cache, where anyone could read it). omniqa security on your machine runs the scan and writes the report to .omniqa/security/report.md.";
|
|
457
523
|
const CI_COMMENT = `${NPM_COMMENT} Heals and the refreshed coverage are committed on the branch under test, or opened as a`;
|
|
458
524
|
// ponytail: a push or merge request runs the tests only when it touches a test file; everything else (recordings, the config, the knowledge, a README) waits for a run by hand. Widen the list here if a project wants more.
|
|
459
525
|
const RUN_ON = (dir) => [`${dir}tests/**/*.ts`];
|
|
@@ -462,7 +528,17 @@ function ciFiles(a, place, secretNames) {
|
|
|
462
528
|
const at = place.root === "" ? "" : `${place.root}/`;
|
|
463
529
|
const sec = secEnvironment(a);
|
|
464
530
|
const dir = place.dir === "" ? "" : `${place.dir}/`;
|
|
465
|
-
/**
|
|
531
|
+
/** An environment says securityScan: on: the sec job's run ends with the agent scan, whose report it keeps from one run to the next. */
|
|
532
|
+
const scans = a.securityActive === true;
|
|
533
|
+
/**
|
|
534
|
+
* The GitHub job of the security tests: the steps of the other job and Interceptor, read-only, with no `needs`. Where the
|
|
535
|
+
* scan runs, the last report is restored from the cache before the run (what makes a confirmed claim new or known), saved
|
|
536
|
+
* after it whatever its outcome, and kept as an artifact on every run: what the scan found is in no log. The three steps run
|
|
537
|
+
* on a private repository alone: an artifact of a public one is anyone's to download, and its cache a fork's pull request's
|
|
538
|
+
* to restore, so there the job keeps no report, omniqa security makes no scan, and a notice says so. No artifact step
|
|
539
|
+
* here needs `include-hidden-files`: upload-artifact leaves out the dot-named entries inside a path it is given, and a path
|
|
540
|
+
* that leads through `.omniqa` is searched like any other (its v7 code, read on 2026-11-17; OmniQA writes no dot-named file there).
|
|
541
|
+
*/
|
|
466
542
|
const secJob = (command, env) => ` sec:
|
|
467
543
|
runs-on: ubuntu-latest
|
|
468
544
|
permissions:
|
|
@@ -477,9 +553,29 @@ ${place.dir === "" ? "" : ` defaults:\n run:\n working-directory:
|
|
|
477
553
|
npm install -g omniqa@latest
|
|
478
554
|
omniqa install --with-deps
|
|
479
555
|
omniqa install interceptor
|
|
480
|
-
-
|
|
556
|
+
${scans ? ` - if: github.event.repository.private
|
|
557
|
+
uses: actions/cache/restore@v6
|
|
558
|
+
with:
|
|
559
|
+
path: ${dir}.omniqa/security
|
|
560
|
+
key: omniqa-security-\${{ github.run_id }}
|
|
561
|
+
restore-keys: omniqa-security-
|
|
562
|
+
` : ""} - run: ${command}
|
|
481
563
|
env:
|
|
482
|
-
${env}${secretNames.map((n) => ` ${n}: \${{ secrets.${n} }}\n`).join("")}
|
|
564
|
+
${env}${secretNames.map((n) => ` ${n}: \${{ secrets.${n} }}\n`).join("")}${scans ? ` SLACK_WEBHOOK_URL: \${{ secrets.SLACK_WEBHOOK_URL }}
|
|
565
|
+
- if: always() && github.event.repository.private
|
|
566
|
+
uses: actions/cache/save@v6
|
|
567
|
+
with:
|
|
568
|
+
path: ${dir}.omniqa/security
|
|
569
|
+
key: omniqa-security-\${{ github.run_id }}
|
|
570
|
+
- if: always() && github.event.repository.private
|
|
571
|
+
uses: actions/upload-artifact@v7
|
|
572
|
+
with:
|
|
573
|
+
name: omniqa-security-report
|
|
574
|
+
path: ${dir}.omniqa/security
|
|
575
|
+
if-no-files-found: ignore
|
|
576
|
+
- if: always() && !github.event.repository.private
|
|
577
|
+
run: echo "::notice::${PUBLIC_NOTICE}"
|
|
578
|
+
` : ""} - if: failure()
|
|
483
579
|
uses: actions/upload-artifact@v7
|
|
484
580
|
with:
|
|
485
581
|
name: omniqa-security-results
|
|
@@ -490,20 +586,26 @@ ${env}${secretNames.map((n) => ` ${n}: \${{ secrets.${n} }}\n`).join(""
|
|
|
490
586
|
`;
|
|
491
587
|
if (a.ci === "github") {
|
|
492
588
|
return {
|
|
493
|
-
[`${at}.github/workflows/omniqa.yml`]: `# OmniQA: the tests on every push
|
|
494
|
-
# workflow); anything else, a README, the recordings, the config or the knowledge, runs nothing on its own.
|
|
589
|
+
[`${at}.github/workflows/omniqa.yml`]: `# OmniQA: the tests on every push that touches a test file (tests/**/*.ts), and by hand from the Actions tab (Run
|
|
590
|
+
# workflow); anything else, a README, the recordings, the config or the knowledge, runs nothing on its own. A pull request
|
|
591
|
+
# is checked by the run of its branch's push: a pull_request trigger beside it would run the branch twice, the second
|
|
592
|
+
# time on GitHub's merge commit, which a heal must not be committed on. ${CI_COMMENT}
|
|
495
593
|
# pull request when the branch is protected or the environment is production: allow GitHub Actions to create and approve pull
|
|
496
594
|
# requests under Settings → Actions → General → Workflow permissions. A failed run keeps its screenshots, report and transcripts
|
|
497
595
|
# as the omniqa-test-results artifact. Every secret below has the name and the value it has in .env (Settings, Secrets and
|
|
498
596
|
# variables, Actions).${sec === undefined ? "" : `
|
|
499
597
|
# The security tests run apart, in the sec job: on ${sec}, in a browser behind the Interceptor proxy. Neither job waits for the
|
|
500
|
-
# other or fails with it; sec only reads the repository, and a failed run keeps its results as omniqa-security-results
|
|
598
|
+
# other or fails with it; sec only reads the repository, and a failed run keeps its results as omniqa-security-results.${scans ? `
|
|
599
|
+
# ${sec} says securityScan: on, so the sec job's run ends with the agent scan (model calls on every run; --no-scan on its
|
|
600
|
+
# line keeps a run to the tests). The scan's report says where the product is open and is in no log: the job restores the
|
|
601
|
+
# last one from the cache (a finding it already listed does not fail the run again), saves the new one, and keeps it as
|
|
602
|
+
# the omniqa-security-report artifact. On a public repository anyone could download either, so there the scan does not run
|
|
603
|
+
# (omniqa security says so in one line) and the report is the one omniqa security writes on your machine.
|
|
604
|
+
# SLACK_WEBHOOK_URL, when it is set, gets one line with the counts when a scan confirms something new.` : ""}`}
|
|
501
605
|
name: omniqa
|
|
502
606
|
on:
|
|
503
607
|
push:
|
|
504
608
|
paths:
|
|
505
|
-
${RUN_ON(place.dir === "" ? "" : `${place.dir}/`).map((p) => ` - ${p}\n`).join("")} pull_request:
|
|
506
|
-
paths:
|
|
507
609
|
${RUN_ON(place.dir === "" ? "" : `${place.dir}/`).map((p) => ` - ${p}\n`).join("")} workflow_dispatch:
|
|
508
610
|
permissions:
|
|
509
611
|
contents: write # heals and the refreshed coverage are committed on the branch under test
|
|
@@ -534,8 +636,9 @@ ${secretNames.map((n) => ` ${n}: \${{ secrets.${n} }}\n`).join("")}
|
|
|
534
636
|
${sec === undefined ? "" : secJob(`omniqa security --env ${sec}`, "")}`,
|
|
535
637
|
...(a.changelog === undefined ? {} : { [`${at}.github/workflows/omniqa-deploy.yml`]: `# OmniQA: after a product deploy. The product's deploy pipeline starts this workflow (repository_dispatch omniqa-deploy, a
|
|
536
638
|
# curl with no payload); it runs the suite on the deployed environment, posts the failures to the team channel and fixes
|
|
537
|
-
# them behind the reviewers, rewrites the knowledge where the change lands and commits
|
|
538
|
-
# production environment or a protected branch
|
|
639
|
+
# them behind the reviewers, rewrites the knowledge where the change lands and commits what it changed (a pull request on a
|
|
640
|
+
# production environment or a protected branch; nothing when no test changed). The run is red when a test ended as a
|
|
641
|
+
# regression or still fails. Run workflow in the Actions tab runs it by hand. The three setup steps
|
|
539
642
|
# (secrets, the product's hook, a first run): omniqa doctor. A failed run keeps its screenshots, report and transcripts as
|
|
540
643
|
# the omniqa-deploy-results artifact. ${NPM_COMMENT}
|
|
541
644
|
name: omniqa deploy
|
|
@@ -635,7 +738,7 @@ ${RUN_ON(place.dir === "" ? "" : `${place.dir}/`).map((p) => ` - ${p}\n`)
|
|
|
635
738
|
- if: $CI_COMMIT_BRANCH
|
|
636
739
|
changes:
|
|
637
740
|
${RUN_ON(place.dir === "" ? "" : `${place.dir}/`).map((p) => ` - ${p}\n`).join("")} before_script:
|
|
638
|
-
|
|
741
|
+
${GITLAB_PUSH}
|
|
639
742
|
- npm install -g omniqa@latest
|
|
640
743
|
- omniqa install --with-deps
|
|
641
744
|
script:
|
|
@@ -648,7 +751,13 @@ ${place.dir === "" ? "" : ` - cd ${place.dir}\n`} - ${ciTestCommand(a)}
|
|
|
648
751
|
${sec === undefined ? "" : `
|
|
649
752
|
# The security tests, apart from the others: on ${sec}, or on the environment a deploy names (OMNIQA_ENV), in a browser
|
|
650
753
|
# behind the Interceptor proxy. It runs when the omniqa job runs${a.changelog === undefined ? "" : " and when the omniqa deploy job does"}, waits for neither and fails with neither;
|
|
651
|
-
# it pushes nothing, so it needs no GITLAB_TOKEN. A failed run keeps its results as the job's artifacts
|
|
754
|
+
# it pushes nothing, so it needs no GITLAB_TOKEN. A failed run keeps its results as the job's artifacts.${scans ? `
|
|
755
|
+
# ${sec} says securityScan: on, so the job's run ends with the agent scan (model calls on every run; --no-scan on its line
|
|
756
|
+
# keeps a run to the tests). The scan's report says where the product is open and is in no log: the job's cache carries the
|
|
757
|
+
# last one to the next run (a finding it already listed does not fail the run again), and every run keeps it among its
|
|
758
|
+
# artifacts, which the project's members alone can download (public: false; a self-managed GitLab before 16.7 takes the
|
|
759
|
+
# keyword and does nothing with it unless its non_public_artifacts feature flag is on). SLACK_WEBHOOK_URL, when it is set,
|
|
760
|
+
# gets one line with the counts when a scan confirms something new.` : ""}
|
|
652
761
|
omniqa sec:
|
|
653
762
|
image: node:26
|
|
654
763
|
rules:
|
|
@@ -665,25 +774,33 @@ ${RUN_ON(dir).map((p) => ` - ${p}\n`).join("")} before_script:
|
|
|
665
774
|
- omniqa install interceptor
|
|
666
775
|
script:
|
|
667
776
|
${place.dir === "" ? "" : ` - cd ${place.dir}\n`} - omniqa security --env "\${OMNIQA_ENV:-${sec}}"
|
|
668
|
-
|
|
669
|
-
|
|
777
|
+
${scans ? ` cache:
|
|
778
|
+
key: omniqa-security
|
|
670
779
|
paths:
|
|
780
|
+
- ${dir}.omniqa/security
|
|
781
|
+
when: always
|
|
782
|
+
` : ""} artifacts:
|
|
783
|
+
when: ${scans ? "always" : "on_failure"}
|
|
784
|
+
${scans ? " public: false\n" : ""} paths:
|
|
671
785
|
- ${dir}test-results
|
|
672
786
|
- ${dir}.omniqa/runs
|
|
673
|
-
`}${a.changelog === undefined ? "" : `
|
|
787
|
+
${scans ? ` - ${dir}.omniqa/security\n` : ""}`}${a.changelog === undefined ? "" : `
|
|
674
788
|
# After a product deploy: the product's deploy pipeline starts this one (a pipeline trigger token, a curl with no payload);
|
|
675
789
|
# the job runs the suite on the deployed environment, posts the failures to the team channel and fixes them behind the
|
|
676
|
-
# reviewers, rewrites the knowledge where the change lands and commits
|
|
677
|
-
# on a production environment or a protected branch).
|
|
790
|
+
# reviewers, rewrites the knowledge where the change lands and commits what it changed through GITLAB_TOKEN (a merge request
|
|
791
|
+
# on a production environment or a protected branch; nothing when no test changed). The job is red when a test ended as a
|
|
792
|
+
# regression or still fails. New pipeline (Build → Pipelines) with a variable OMNIQA_ENV (the
|
|
678
793
|
# environment, e.g. ${deployEnvironment(a)}; OMNIQA_SHA names a commit other than the branch head) runs it by hand. The
|
|
679
|
-
# three setup steps (variables, the product's hook, a first run): omniqa doctor.
|
|
794
|
+
# three setup steps (variables, the product's hook, a first run): omniqa doctor. One deploy's job at a time
|
|
795
|
+
# (resource_group): two that ran side by side each moved the last synced commit, and the one that ended last won.
|
|
680
796
|
omniqa deploy:
|
|
681
797
|
image: node:26
|
|
798
|
+
resource_group: omniqa-deploy
|
|
682
799
|
rules:
|
|
683
800
|
- if: $CI_PIPELINE_SOURCE == "trigger" || $CI_PIPELINE_SOURCE == "pipeline"
|
|
684
801
|
- if: $CI_PIPELINE_SOURCE == "web" && ($OMNIQA_ENV || $OMNIQA_SHA)
|
|
685
802
|
before_script:
|
|
686
|
-
|
|
803
|
+
${GITLAB_PUSH}
|
|
687
804
|
- npm install -g omniqa@latest
|
|
688
805
|
- omniqa install --with-deps
|
|
689
806
|
script:
|
|
@@ -722,6 +839,13 @@ export function render(a, place = { root: "", dir: "" }) {
|
|
|
722
839
|
const m = a.models[r] ?? examples[r];
|
|
723
840
|
return m === undefined ? "" : ` ${a.models[r] === undefined ? "// " : ""}${r}: ${modelLine(a, m)},\n`;
|
|
724
841
|
}).join("");
|
|
842
|
+
// The scan's two roles, with the security kind alone: as the interview wrote them, else the provider's suggestion as a commented example, there for the day an environment says securityScan: on.
|
|
843
|
+
const scan = choiceOf(a.provider, a.baseUrl).scan;
|
|
844
|
+
const scanRoles = a.security !== true ? "" : SCAN_ROLES.map((r) => {
|
|
845
|
+
const m = a.models[r] ?? scan?.[r];
|
|
846
|
+
return m === undefined ? "" : ` ${a.models[r] === undefined ? "// " : ""}${r}: ${modelLine(a, m)},\n`;
|
|
847
|
+
}).join("");
|
|
848
|
+
const scanLines = scanRoles === "" ? "" : ` // The agent scan of an environment that says securityScan: on (.omniqa/product.yaml) runs on these two roles: security is the researcher, which reads the\n // product's traffic and sends requests of its own; securityReviewer confirms or discards each of its claims. fallback names the model that answers\n // when the first declines or its provider fails, and the report says which model answered. A role left out uses default.\n${scanRoles}`;
|
|
725
849
|
const platforms = DEVICE_PLATFORMS.filter((p) => a.apps[p] !== undefined);
|
|
726
850
|
const projectLines = platforms.length === 0
|
|
727
851
|
? ' // { name: "ios", platform: "ios", app: "com.example.app" },\n // { name: "android", platform: "android", app: "com.example.app", device: "Pixel_9" },\n'
|
|
@@ -755,7 +879,7 @@ export default defineConfig({
|
|
|
755
879
|
models: {
|
|
756
880
|
default: ${modelLine(a, a.models.default)},
|
|
757
881
|
// Roles left out use default. A cheaper executor saves the most on first runs; the reviewer is the quality gate:
|
|
758
|
-
${roleLines} },
|
|
882
|
+
${roleLines}${scanLines} },
|
|
759
883
|
testIdAttribute: ${JSON.stringify(a.testId)},
|
|
760
884
|
projects: [
|
|
761
885
|
{ name: "desktop" },
|
|
@@ -779,7 +903,7 @@ ${scopeLine}});
|
|
|
779
903
|
`,
|
|
780
904
|
".omniqa/product.yaml": `name: ${JSON.stringify(a.name)}
|
|
781
905
|
environments:
|
|
782
|
-
${a.environments.map((e, i) => environmentBlock(e, i === 0, a)).join("")}${productionExample}${accountLines}${sources}# crawl: # omniqa crawl
|
|
906
|
+
${a.environments.map((e, i) => environmentBlock(e, i === 0, a)).join("")}${productionExample}${accountLines}${sources}# crawl: # omniqa crawl and the knowledge build's walks; omniqa crawl logs in as \`account\` (default: the first account)
|
|
783
907
|
# max_pages: 50
|
|
784
908
|
# max_depth: 3
|
|
785
909
|
# exclude: [/admin] # path substrings, on top of logout, delete and payment routes
|
|
@@ -1018,11 +1142,22 @@ export async function init(args, cwd) {
|
|
|
1018
1142
|
return code;
|
|
1019
1143
|
};
|
|
1020
1144
|
try {
|
|
1021
|
-
const
|
|
1145
|
+
const onboarding = (await readState(cwd)).onboarding;
|
|
1146
|
+
const stage = onboarding?.stage;
|
|
1147
|
+
const stored = storedAnswers(onboarding?.answers);
|
|
1148
|
+
const answeredOn = (onboarding?.updatedAt ?? "").slice(0, 10);
|
|
1149
|
+
const flagged = ANSWER_FLAGS.filter((flag) => v[flag] !== undefined);
|
|
1022
1150
|
if (reader !== undefined && v.force !== true && stage !== undefined && stage !== "interview") {
|
|
1151
|
+
// A flag answers the interview, which is not asked here: read and dropped, the stages ran on the earlier answers.
|
|
1152
|
+
if (flagged.length > 0) {
|
|
1153
|
+
stderr.write(`omniqa init: --${flagged[0]} answers the interview, and this project is past it (stage ${stage}): omniqa init goes on from that stage, omniqa init --force asks the interview again\n`);
|
|
1154
|
+
return 1;
|
|
1155
|
+
}
|
|
1023
1156
|
out(`omniqa init: resuming at stage ${stage}`);
|
|
1024
1157
|
return await stages(stage, reader);
|
|
1025
1158
|
}
|
|
1159
|
+
// Stage interview is an init that took its answers and was stopped while it set the project up: it goes on from them.
|
|
1160
|
+
const goOn = reader !== undefined && v.force !== true && stage === "interview" && stored !== undefined && flagged.length === 0;
|
|
1026
1161
|
// The CI file goes to the repository's root (a qa/ folder's job runs in the folder); a folder in no repository becomes one below.
|
|
1027
1162
|
const top = await git(cwd, ["rev-parse", "--show-toplevel"]).catch(() => undefined);
|
|
1028
1163
|
const here = await realpath(cwd);
|
|
@@ -1034,14 +1169,34 @@ export async function init(args, cwd) {
|
|
|
1034
1169
|
defaults.pages = v.pages === true && defaults.ci !== "none";
|
|
1035
1170
|
if (reader === undefined && defaults.models.default.model === "")
|
|
1036
1171
|
throw new Error(`--model is required with --base-url ${defaults.baseUrl}: only OpenAI's and OpenRouter's urls come with suggested models`);
|
|
1037
|
-
|
|
1172
|
+
if (goOn) {
|
|
1173
|
+
out(`omniqa init: going on from the interview answered on ${answeredOn} (omniqa init --force asks it again)`);
|
|
1174
|
+
answers = stored;
|
|
1175
|
+
}
|
|
1176
|
+
else if (reader === undefined)
|
|
1177
|
+
answers = defaults;
|
|
1178
|
+
else {
|
|
1179
|
+
// A project that is there is said before the first question, by the files every project has whatever the answers.
|
|
1180
|
+
const there = [];
|
|
1181
|
+
if (v.force !== true)
|
|
1182
|
+
for (const rel of Object.keys(render({ ...defaults, ci: "none", pages: false }, place)))
|
|
1183
|
+
if (await exists(join(cwd, rel)))
|
|
1184
|
+
there.push(rel);
|
|
1185
|
+
if (there.length > 0)
|
|
1186
|
+
throw new Error(`omniqa init: ${there.join(", ")} already exists. Re-run with --force to overwrite.`);
|
|
1187
|
+
// Asked again, the earlier answers are the defaults; flags name others, as on a first init.
|
|
1188
|
+
const again = v.force === true && stored !== undefined && flagged.length === 0;
|
|
1189
|
+
if (again)
|
|
1190
|
+
out(`omniqa init --force: your answers of ${answeredOn} are the defaults (Enter keeps each)`);
|
|
1191
|
+
answers = await interview(reader, again ? { ...defaults, ...stored } : defaults, { ...(remote === undefined ? {} : { remote }), ...(strings.ci === undefined ? {} : { ciFlag: strings.ci }) });
|
|
1192
|
+
}
|
|
1038
1193
|
}
|
|
1039
1194
|
catch (err) {
|
|
1040
1195
|
stderr.write(`${err.message}\n`);
|
|
1041
1196
|
return 1;
|
|
1042
1197
|
}
|
|
1043
1198
|
const files = render(answers, place);
|
|
1044
|
-
if (v.force !== true) {
|
|
1199
|
+
if (v.force !== true && !goOn) {
|
|
1045
1200
|
const clashes = [];
|
|
1046
1201
|
for (const rel of Object.keys(files))
|
|
1047
1202
|
if (await exists(join(cwd, rel)))
|
|
@@ -1052,9 +1207,21 @@ export async function init(args, cwd) {
|
|
|
1052
1207
|
}
|
|
1053
1208
|
}
|
|
1054
1209
|
// init appends the block to a README of the user's only when it creates the project: one without the markers stays the user's on a later init --force
|
|
1055
|
-
const fresh = !(await exists(join(cwd, PRODUCT_FILE)));
|
|
1210
|
+
const fresh = goOn || !(await exists(join(cwd, PRODUCT_FILE)));
|
|
1211
|
+
// The answers first: an init stopped from here on (the proxy's install takes a minute) goes on from them, asking nothing.
|
|
1212
|
+
await recordOnboarding(cwd, "interview", answers);
|
|
1213
|
+
// A file that is neither what the earlier answers wrote nor what these write was changed by hand: it is kept beside the runs.
|
|
1214
|
+
const own = stored === undefined ? {} : render(stored, place);
|
|
1215
|
+
const keptIn = join(RUNS_DIR, transcriptName("init").replace(/\.jsonl$/, ""));
|
|
1216
|
+
const kept = [];
|
|
1056
1217
|
for (const [rel, content] of Object.entries(files)) {
|
|
1057
1218
|
const path = join(cwd, rel);
|
|
1219
|
+
const before = await readFile(path, "utf8").catch(() => undefined);
|
|
1220
|
+
if (before !== undefined && before !== content && before !== own[rel]) {
|
|
1221
|
+
await mkdir(dirname(join(cwd, keptIn, rel)), { recursive: true });
|
|
1222
|
+
await writeFile(join(cwd, keptIn, rel), before);
|
|
1223
|
+
kept.push(`${rel} as it was is kept in ${keptIn}/ (what was changed in it by hand is there, not in the new file)`);
|
|
1224
|
+
}
|
|
1058
1225
|
await mkdir(dirname(path), { recursive: true });
|
|
1059
1226
|
await writeFile(path, content);
|
|
1060
1227
|
}
|
|
@@ -1086,7 +1253,7 @@ export async function init(args, cwd) {
|
|
|
1086
1253
|
gitlabLine = current.includes(job) ? `${shown} (includes it already)` : `${shown} left as is: add \`include: [{ local: ${job} }]\` to it`;
|
|
1087
1254
|
}
|
|
1088
1255
|
const names = secretNamesOf(answers);
|
|
1089
|
-
const listed = [...Object.keys(files).flatMap((rel) => [...described(rel, answers.ci, names, answers.pages, answers.changelog, ciTest(answers).note, secNote(answers)), ...(rel === ".omniqa/ci/gitlab-ci.yml" && gitlabLine !== undefined ? [gitlabLine] : [])]), ".mcp.json (the omniqa MCP server for Claude Code; Cursor reads the same shape from .cursor/mcp.json)"];
|
|
1256
|
+
const listed = [...kept, ...Object.keys(files).flatMap((rel) => [...described(rel, answers.ci, names, answers.pages, answers.changelog, ciTest(answers).note, secNote(answers)), ...(rel === ".omniqa/ci/gitlab-ci.yml" && gitlabLine !== undefined ? [gitlabLine] : [])]), ".mcp.json (the omniqa MCP server for Claude Code; Cursor reads the same shape from .cursor/mcp.json)"];
|
|
1090
1257
|
const pkg = await modulePackage(cwd);
|
|
1091
1258
|
if (pkg !== undefined)
|
|
1092
1259
|
listed.push(pkg);
|