omniqa 0.6.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -1
- package/dist/.tsbuildinfo +1 -1
- package/dist/cli.js +9 -1
- package/dist/cli.js.map +1 -1
- package/dist/commands/doctor.d.ts +7 -0
- package/dist/commands/doctor.d.ts.map +1 -1
- package/dist/commands/doctor.js +77 -5
- package/dist/commands/doctor.js.map +1 -1
- package/dist/commands/init.d.ts +15 -0
- package/dist/commands/init.d.ts.map +1 -1
- package/dist/commands/init.js +219 -26
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/install.d.ts.map +1 -1
- package/dist/commands/install.js +20 -3
- package/dist/commands/install.js.map +1 -1
- package/dist/commands/knowledge.d.ts +23 -0
- package/dist/commands/knowledge.d.ts.map +1 -1
- package/dist/commands/knowledge.js +215 -39
- package/dist/commands/knowledge.js.map +1 -1
- package/dist/commands/plan-add.d.ts.map +1 -1
- package/dist/commands/plan-add.js +23 -1
- package/dist/commands/plan-add.js.map +1 -1
- package/dist/commands/plan.d.ts.map +1 -1
- package/dist/commands/plan.js +10 -6
- package/dist/commands/plan.js.map +1 -1
- package/dist/commands/security.d.ts +7 -0
- package/dist/commands/security.d.ts.map +1 -0
- package/dist/commands/security.js +76 -0
- package/dist/commands/security.js.map +1 -0
- package/dist/commands/status.d.ts.map +1 -1
- package/dist/commands/status.js +2 -2
- package/dist/commands/status.js.map +1 -1
- package/dist/commands/test.d.ts +10 -0
- package/dist/commands/test.d.ts.map +1 -1
- package/dist/commands/test.js +61 -7
- package/dist/commands/test.js.map +1 -1
- package/dist/commands/write.d.ts.map +1 -1
- package/dist/commands/write.js +52 -5
- package/dist/commands/write.js.map +1 -1
- package/dist/coverage-page.d.ts.map +1 -1
- package/dist/coverage-page.js +2 -2
- package/dist/coverage-page.js.map +1 -1
- package/dist/devices.d.ts +4 -2
- package/dist/devices.d.ts.map +1 -1
- package/dist/devices.js +3 -3
- package/dist/devices.js.map +1 -1
- package/dist/mcp/tools.d.ts.map +1 -1
- package/dist/mcp/tools.js +3 -2
- package/dist/mcp/tools.js.map +1 -1
- package/dist/planner.d.ts.map +1 -1
- package/dist/planner.js +13 -9
- package/dist/planner.js.map +1 -1
- package/dist/qa.d.ts.map +1 -1
- package/dist/qa.js +2 -1
- package/dist/qa.js.map +1 -1
- package/dist/runner.d.ts +1 -1
- package/dist/runner.d.ts.map +1 -1
- package/dist/runner.js +64 -56
- package/dist/runner.js.map +1 -1
- package/dist/security-checks.d.ts +146 -0
- package/dist/security-checks.d.ts.map +1 -0
- package/dist/security-checks.js +661 -0
- package/dist/security-checks.js.map +1 -0
- package/dist/security-gates.d.ts +37 -0
- package/dist/security-gates.d.ts.map +1 -0
- package/dist/security-gates.js +46 -0
- package/dist/security-gates.js.map +1 -0
- package/dist/security-mask.d.ts +35 -0
- package/dist/security-mask.d.ts.map +1 -0
- package/dist/security-mask.js +104 -0
- package/dist/security-mask.js.map +1 -0
- package/dist/security-report.d.ts +93 -0
- package/dist/security-report.d.ts.map +1 -0
- package/dist/security-report.js +170 -0
- package/dist/security-report.js.map +1 -0
- package/dist/security-request.d.ts +109 -0
- package/dist/security-request.d.ts.map +1 -0
- package/dist/security-request.js +254 -0
- package/dist/security-request.js.map +1 -0
- package/dist/security-research.d.ts +131 -0
- package/dist/security-research.d.ts.map +1 -0
- package/dist/security-research.js +581 -0
- package/dist/security-research.js.map +1 -0
- package/dist/security-scan.d.ts +98 -0
- package/dist/security-scan.d.ts.map +1 -0
- package/dist/security-scan.js +415 -0
- package/dist/security-scan.js.map +1 -0
- package/dist/security-sources.d.ts +55 -0
- package/dist/security-sources.d.ts.map +1 -0
- package/dist/security-sources.js +75 -0
- package/dist/security-sources.js.map +1 -0
- package/dist/security.d.ts +65 -0
- package/dist/security.d.ts.map +1 -0
- package/dist/security.js +624 -0
- package/dist/security.js.map +1 -0
- package/dist/sync.d.ts +12 -1
- package/dist/sync.d.ts.map +1 -1
- package/dist/sync.js +11 -3
- package/dist/sync.js.map +1 -1
- package/dist/test.d.ts +85 -0
- package/dist/test.d.ts.map +1 -1
- package/dist/test.js +337 -19
- package/dist/test.js.map +1 -1
- package/dist/writer.d.ts +5 -1
- package/dist/writer.d.ts.map +1 -1
- package/dist/writer.js +15 -12
- package/dist/writer.js.map +1 -1
- package/package.json +7 -7
- package/skills/security-research.md +57 -0
- package/skills/security-review.md +27 -0
package/dist/commands/init.js
CHANGED
|
@@ -2,6 +2,7 @@ import { access, mkdir, readFile, realpath, writeFile } from "node:fs/promises";
|
|
|
2
2
|
import { basename, dirname, join, relative, resolve, sep as pathSep } from "node:path";
|
|
3
3
|
import { stderr, stdout } from "node:process";
|
|
4
4
|
import { parseArgs, styleText } from "node:util";
|
|
5
|
+
import { installInterceptor, INTERCEPTOR_VERSION } from "@omniqa/browser";
|
|
5
6
|
import { DEFAULT_KEY_ENV, DEVICE_PLATFORMS, git, loadConfig, parseRemote, PRODUCT_FILE, readmeHasBlock, readState, README_FILE, recordOnboarding, remoteUrl, writeReadme } from "@omniqa/core";
|
|
6
7
|
import { deploySetup } from "../deploy-setup.js";
|
|
7
8
|
import { DEPLOY_WORKFLOW, GITLAB_CI, hasWriteJob, WRITE_WORKFLOW, writeJobGitlab, writeWorkflow } from "../jobs.js";
|
|
@@ -75,13 +76,34 @@ function ciTest(a) {
|
|
|
75
76
|
? { command: `omniqa test --tier production --env ${reach.name}`, note: `in CI the tests run on ${reach.name}, the production ones only: ${here}` }
|
|
76
77
|
: { command: `omniqa test --env ${reach.name}`, note: `in CI the tests run on ${reach.name}: ${here}` };
|
|
77
78
|
}
|
|
79
|
+
/**
|
|
80
|
+
* The production-tier environment the `sec` CI job runs the security tests on: the first one a runner can reach among those that
|
|
81
|
+
* explore, since that is where the pass walks and the guards are written; undefined without the kind or without such an environment.
|
|
82
|
+
*/
|
|
83
|
+
const secEnvironment = (a) => (a.security === true ? a.environments.find((e) => e.tier === "production" && e.explore && !onThisMachine(e.url))?.name : undefined);
|
|
84
|
+
/** What the CI test job runs with security tests on: they are the `sec` job's, so the other job leaves them out. The tag as a whole word: a tag of the user's that starts like it (@security) stays in. */
|
|
85
|
+
const ciTestCommand = (a) => `${ciTest(a).command}${a.security === true ? ' --grep-invert "@sec\\b"' : ""}`;
|
|
86
|
+
/** Under a CI file, with the kind on: the job that runs the security tests, or why there is none yet. */
|
|
87
|
+
const secNote = (a) => {
|
|
88
|
+
if (a.security !== true)
|
|
89
|
+
return undefined;
|
|
90
|
+
const at = secEnvironment(a);
|
|
91
|
+
return at === undefined
|
|
92
|
+
? `sec: no job yet: every production-tier environment that explores runs on this machine, not on a CI runner; once one is deployed, omniqa doctor names the job to add`
|
|
93
|
+
: `sec: the security tests on ${at}, apart from the others${a.securityActive === true ? ", then the agent scan there (model calls on every run of the job; its report is the job's artifact)" : ""}`;
|
|
94
|
+
};
|
|
95
|
+
/** The agent scan's two roles: asked for when an environment's scan is on, and read by no stage of init. */
|
|
96
|
+
const SCAN_ROLES = ["security", "securityReviewer"];
|
|
78
97
|
// Enter at the model questions, and --yes, write the provider's suggestion: Sonnet 5.5 at effort high, the executor on the same model at
|
|
79
98
|
// medium (one action or one verdict a call, where more effort buys nothing measurable) and the reviewer Opus 5.5 at high (the setup the
|
|
80
99
|
// install guide measured), through OpenRouter by its ids (the executor and the verifier read screenshots, and Claude does); on OpenAI,
|
|
81
100
|
// GPT-6.1 Sol and the flagship GPT-6 Astra for the reviewer.
|
|
82
|
-
|
|
101
|
+
// The agent scan's two roles are the maintainer's choice: through OpenRouter the researcher on Kimi K3.1 Pro and its reviewer on Opus 5.5, which
|
|
102
|
+
// Opus 4.8 answers for when it declines, each at effort high; on Anthropic's API, which does not serve the first, Opus 5.5 for both with that fallback.
|
|
103
|
+
const OPUS_SCAN = { model: "claude-opus-5-5", effort: "high", fallback: "claude-opus-4-8" };
|
|
104
|
+
const ANTHROPIC = { name: "anthropic", label: "Anthropic", provider: "anthropic", keyEnv: "ANTHROPIC_API_KEY", models: { default: { model: "claude-sonnet-5-5", effort: "high" }, executor: { model: "claude-sonnet-5-5", effort: "medium" }, reviewer: { model: "claude-opus-5-5", effort: "high" } }, scan: { security: OPUS_SCAN, securityReviewer: OPUS_SCAN } };
|
|
83
105
|
const OPENAI = { name: "openai", label: "OpenAI", provider: "openai-compatible", baseUrl: "https://api.openai.com/v1", keyEnv: "OPENAI_API_KEY", models: { default: { model: "gpt-6.1-sol", effort: "high" }, executor: { model: "gpt-6.1-sol", effort: "medium" }, reviewer: { model: "gpt-6-astra", effort: "high" } } };
|
|
84
|
-
const OPENROUTER = { name: "openrouter", label: "OpenRouter", provider: "openai-compatible", baseUrl: "https://openrouter.ai/api/v1", keyEnv: "OPENROUTER_API_KEY", models: { default: { model: "anthropic/claude-sonnet-5.5", effort: "high" }, executor: { model: "anthropic/claude-sonnet-5.5", effort: "medium" }, reviewer: { model: "anthropic/claude-opus-5.5", effort: "high" } } };
|
|
106
|
+
const OPENROUTER = { name: "openrouter", label: "OpenRouter", provider: "openai-compatible", baseUrl: "https://openrouter.ai/api/v1", keyEnv: "OPENROUTER_API_KEY", models: { default: { model: "anthropic/claude-sonnet-5.5", effort: "high" }, executor: { model: "anthropic/claude-sonnet-5.5", effort: "medium" }, reviewer: { model: "anthropic/claude-opus-5.5", effort: "high" } }, scan: { security: { model: "moonshotai/kimi-k3.1-pro", effort: "high" }, securityReviewer: { model: "anthropic/claude-opus-5.5", effort: "high", fallback: "anthropic/claude-opus-4.8" } } };
|
|
85
107
|
const OTHER = { name: "other", label: "other OpenAI-compatible API", provider: "openai-compatible", keyEnv: "OPENAI_API_KEY" };
|
|
86
108
|
const CHOICES = [ANTHROPIC, OPENAI, OPENROUTER, OTHER];
|
|
87
109
|
/** The choice a provider and its base url make: OpenAI's or OpenRouter's url, else another API. */
|
|
@@ -90,7 +112,8 @@ const PROVIDERS = ["anthropic", "openai-compatible"];
|
|
|
90
112
|
const EFFORTS = ["low", "medium", "high", "xhigh", "max"];
|
|
91
113
|
const MODEL_ROLES = ["executor", "reviewer", "writer", "analyst", "chat"];
|
|
92
114
|
const RUNNERS = ["local", "remote", "cloud"];
|
|
93
|
-
|
|
115
|
+
// .omniqa/security/ is where a security report lives: it says where the product is open, so it never reaches git.
|
|
116
|
+
const GITIGNORE_LINES = [".env", "node_modules/", ".omniqa/index.sqlite", ".omniqa/jobs.sqlite*", ".omniqa/repos/", ".omniqa/runs/", ".omniqa/security/", "test-results/"];
|
|
94
117
|
const READ_ONLY = "read-only: the agent may navigate, scroll and follow links, never fill, select, press or click buttons";
|
|
95
118
|
// The comments on tier and the switches say what each value means whatever the value, so a value flipped by hand never contradicts them.
|
|
96
119
|
const TIER = "dev: first runs, heals and generated tests land here; production: the same powers behind its deny list, heals and new tests reach git through pull requests only";
|
|
@@ -98,6 +121,29 @@ const MUTATIONS = "true: the agent may also fill, select, press and click; false
|
|
|
98
121
|
const exploreComment = (env) => `true: the model plans a step without a recording and heals a stale one; false: replay only (--allow-explore ${env} explores for one run)`;
|
|
99
122
|
const REPLAY = "true: replay the recordings, model calls only for a step without one or one that misses; false: every run plans each step and verifies each assert through the models, the recordings stay the reference (omniqa test --replay or --no-replay switches one run)";
|
|
100
123
|
const DENY = "url substrings and control texts the agent never acts on here";
|
|
124
|
+
const PHONE = "a test number of your product's own: the security tests type it into a field that takes a phone number; without it such a field stays empty (any other number may be somebody's)";
|
|
125
|
+
const SECURITY = "off: the security tests alone, fixed probes that change nothing; on: an agent scan follows them here, sending requests of its own (model calls on every such run)";
|
|
126
|
+
const KINDS_QUESTION = "Testing kinds (1. e2e, 2. API, 3. security, 4. all; several as 2,3; e2e is always included)";
|
|
127
|
+
const SECURITY_QUESTION = "Agent scan on production (securityScan): 1. off (the security tests alone: the guard and the basic checks, fixed probes that change nothing), 2. on (omniqa test and omniqa security go on to an agent scan there: a researcher that sends requests of its own and a reviewer that checks its claims; model calls on every such run)";
|
|
128
|
+
/** What each of the scan's roles is, in its model question. */
|
|
129
|
+
const SCAN_ROLE = {
|
|
130
|
+
security: "the researcher of the agent scan: it reads the product's traffic and sends requests of its own",
|
|
131
|
+
securityReviewer: "the reviewer of the agent scan: it confirms or discards each claim of the researcher",
|
|
132
|
+
};
|
|
133
|
+
/** Said before the two questions, where the user picks who reads it: what an agent is shown of the traffic leaves the machine. */
|
|
134
|
+
const SCAN_TRAFFIC = "The agent scan reads the product's traffic: excerpts of its requests and answers go to the providers of the next two models, with every credential masked (session cookies, Authorization values, the accounts' secrets).";
|
|
135
|
+
/**
|
|
136
|
+
* The answer to the kinds question: numbers or names, several at once (`2,3`, `api security`, `all`); e2e is always on. The answers
|
|
137
|
+
* of the question this one replaced (`e2e`, `api`, `both`, `e2e and api`) mean what they meant, so an answers file still works.
|
|
138
|
+
* Undefined for anything else.
|
|
139
|
+
*/
|
|
140
|
+
export function kindsOf(answer) {
|
|
141
|
+
const words = answer.toLowerCase().split(/[^a-z0-9]+/).filter((w) => w !== "" && w !== "and");
|
|
142
|
+
if (words.length === 0 || !words.every((w) => ["1", "e2e", "2", "api", "both", "3", "security", "4", "all"].includes(w)))
|
|
143
|
+
return undefined;
|
|
144
|
+
const has = (...names) => words.some((w) => names.includes(w));
|
|
145
|
+
return { api: has("2", "api", "both", "4", "all"), security: has("3", "security", "4", "all") };
|
|
146
|
+
}
|
|
101
147
|
const USAGE = `omniqa init [folder] [options]
|
|
102
148
|
|
|
103
149
|
Without --yes: an interview (every answer has a default in brackets) that writes omniqa.config.ts, .omniqa/product.yaml,
|
|
@@ -254,14 +300,22 @@ export async function interview(reader, d, o = {}) {
|
|
|
254
300
|
const docs = await list("Documentation path or url besides each repository's README.md and docs/ (markdown, HTML, text; PDF through pdftotext), empty when done", d.docs);
|
|
255
301
|
const ci = await choice("CI for this project's repository (github, gitlab, none)", CI_CHOICES, o.ciFlag ?? ciDefault(repos, o.remote));
|
|
256
302
|
const pages = ci === "none" ? false : await yes(PAGES_QUESTION[ci], d.pages);
|
|
257
|
-
// e2e is always on:
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
303
|
+
// e2e is always on: the one question says which kinds come with it.
|
|
304
|
+
const kinds = await askUntil(KINDS_QUESTION, d.api && d.security === true ? "4" : d.security === true ? "3" : d.api ? "2" : "1", kindsOf, "please answer with numbers from 1 to 4 or with the names e2e, api, security and all, e.g. 2,3");
|
|
305
|
+
const api = kinds.api;
|
|
306
|
+
let security = kinds.security;
|
|
307
|
+
let securityActive = false;
|
|
308
|
+
if (security && !environments.some((e) => e.tier === "production")) {
|
|
309
|
+
stdout.write(` security tests run on a production-tier environment only: none here, so the kind is left off (add one to ${PRODUCT_FILE}, then scope: { security: true })\n`);
|
|
310
|
+
security = false;
|
|
311
|
+
}
|
|
312
|
+
else if (security && !environments.some((e) => e.tier === "production" && e.explore)) {
|
|
313
|
+
// The security pass walks production as the crawl does, and the writer walks a guard there: both need explore.
|
|
314
|
+
stdout.write(` security tests are written from a walk of a production-tier environment, which needs exploration allowed there: none here, so the kind is left off (explore: true on one in ${PRODUCT_FILE}, then scope: { security: true })\n`);
|
|
315
|
+
security = false;
|
|
316
|
+
}
|
|
317
|
+
else if (security) {
|
|
318
|
+
securityActive = await askUntil(SECURITY_QUESTION, d.securityActive === true ? "2" : "1", (answer) => ({ "1": false, off: false, basic: false, "2": true, on: true, active: true })[answer.toLowerCase()], "please answer 1 or 2");
|
|
265
319
|
}
|
|
266
320
|
const apps = {};
|
|
267
321
|
const platforms = (await ask(`Native apps: none, or platforms among ${DEVICE_PLATFORMS.join(", ")}`, Object.keys(d.apps).join(", ") || "none")).toLowerCase();
|
|
@@ -331,9 +385,27 @@ export async function interview(reader, d, o = {}) {
|
|
|
331
385
|
if (answer.model !== defaultModel.model || answer.effort !== defaultModel.effort)
|
|
332
386
|
models[role] = answer;
|
|
333
387
|
}
|
|
388
|
+
// The scan's two roles are asked for only where a scan will run. The provider's suggestion is the maintainer's choice, whatever the default model is.
|
|
389
|
+
if (securityActive) {
|
|
390
|
+
stdout.write(` ${SCAN_TRAFFIC}\n`);
|
|
391
|
+
for (const role of SCAN_ROLES) {
|
|
392
|
+
const def = chosen.scan?.[role] ?? defaultModel;
|
|
393
|
+
const answer = await askModel(`${role} model and effort (${SCAN_ROLE[role]}${def.fallback === undefined ? "" : `; ${def.fallback} answers when ${def.model} declines or its provider fails`})`, def);
|
|
394
|
+
// The suggested fallback stays with the suggested model; another model's is the user's to name in the config.
|
|
395
|
+
const fallback = answer.model === def.model ? def.fallback : undefined;
|
|
396
|
+
if (fallback !== undefined)
|
|
397
|
+
models[role] = { ...answer, fallback };
|
|
398
|
+
else if (answer.model !== defaultModel.model || answer.effort !== defaultModel.effort)
|
|
399
|
+
models[role] = answer;
|
|
400
|
+
}
|
|
401
|
+
}
|
|
334
402
|
const testId = await ask("Test id attribute", d.testId);
|
|
335
403
|
const notes = await list("Anything else the agents should know, one note a line (e.g. production signs in through Google only; written to .omniqa/instructions.md, which every model prompt carries), empty when done", d.notes);
|
|
336
404
|
const a = { name, environments, repos, docs, api, apps, accounts, provider, keyEnv, models, testId, ci, pages, notes };
|
|
405
|
+
if (security) {
|
|
406
|
+
a.security = true;
|
|
407
|
+
a.securityActive = securityActive;
|
|
408
|
+
}
|
|
337
409
|
if (changelog !== undefined)
|
|
338
410
|
a.changelog = changelog;
|
|
339
411
|
if (deployFrom !== undefined)
|
|
@@ -351,9 +423,11 @@ const modelLine = (a, m) => {
|
|
|
351
423
|
parts.push(`apiKeyEnv: ${JSON.stringify(a.keyEnv)}`);
|
|
352
424
|
if (m.effort !== undefined)
|
|
353
425
|
parts.push(`effort: ${JSON.stringify(m.effort)}`);
|
|
426
|
+
if (m.fallback !== undefined)
|
|
427
|
+
parts.push(`fallback: ${JSON.stringify(m.fallback)}`);
|
|
354
428
|
return `{ ${parts.join(", ")} }`;
|
|
355
429
|
};
|
|
356
|
-
function environmentBlock(e, first) {
|
|
430
|
+
function environmentBlock(e, first, a) {
|
|
357
431
|
const lines = [` ${e.name}:`, ` url: ${e.url}`];
|
|
358
432
|
lines.push(` tier: ${e.tier.padEnd(16)}# ${TIER}`);
|
|
359
433
|
lines.push(` explore: ${String(e.explore).padEnd(13)}# ${exploreComment(e.name)}`);
|
|
@@ -367,6 +441,10 @@ function environmentBlock(e, first) {
|
|
|
367
441
|
lines.push(` # deny: [/checkout, "Delete account"] # ${DENY}`);
|
|
368
442
|
if (e.tier === "production")
|
|
369
443
|
lines.push(" # api: { mode: page } # API tests go through a page that opened the url first (the production default; a host behind bot management serves browsers only); request is the dev default");
|
|
444
|
+
if (e.tier === "production")
|
|
445
|
+
lines.push(a.security === true ? ` securityScan: ${(a.securityActive === true ? "on" : "off").padEnd(8)}# ${SECURITY}` : ` # securityScan: off # ${SECURITY}`);
|
|
446
|
+
if (e.tier === "production" && a.security === true)
|
|
447
|
+
lines.push(` # phone: "+15550100" # ${PHONE}`);
|
|
370
448
|
if (first) {
|
|
371
449
|
lines.push(' # cookies: [{ name: cookiesAccepted, value: "1" }] # seeded on every browser context before the first page: what the product stores when its cookie banner is answered', ' # localStorage: [{ key: hideNewsModal, value: "1" }] # likewise for a news or promo modal; values may be $VAR references');
|
|
372
450
|
lines.push(" # apps: # the build each device project opens here, by platform (omniqa.config.ts projects with a platform)", " # ios: com.example.app", " # android: com.example.app");
|
|
@@ -385,14 +463,14 @@ const secretsWhere = (ci, names) => ci === "github" ? { under: "Settings → Sec
|
|
|
385
463
|
* The listing lines of a file init wrote: a CI file says what it does, then in bold which secrets to add where and which settings to
|
|
386
464
|
* flip. With a changelog the three setup steps under the listing say the secrets, so the bold lines are left out: nothing twice on one screen.
|
|
387
465
|
*/
|
|
388
|
-
const described = (rel, ci, names, pages, changelog, ciNote) => {
|
|
466
|
+
const described = (rel, ci, names, pages, changelog, ciNote, sec) => {
|
|
389
467
|
const where = secretsWhere(ci, names);
|
|
390
468
|
if (where === undefined)
|
|
391
469
|
return [rel];
|
|
392
|
-
const note = ciNote === undefined ? [] : [` ${ciNote}`];
|
|
470
|
+
const note = [...(ciNote === undefined ? [] : [` ${ciNote}`]), ...(sec === undefined ? [] : [` ${sec}`])];
|
|
393
471
|
const secrets = changelog === undefined ? [` ${bold(`add under ${where.under}: ${where.names} (${where.note})`)}`] : [];
|
|
394
472
|
if (rel.endsWith(".github/workflows/omniqa-deploy.yml"))
|
|
395
|
-
return [`${rel} (after a product deploy: the suite on the deployed environment, the failures posted and fixed, the knowledge where the change lands; set up below)`];
|
|
473
|
+
return [`${rel} (after a product deploy: the suite on the deployed environment, the failures posted and fixed, the knowledge where the change lands; set up below)`, ...(sec === undefined ? [] : [` ${sec}`])];
|
|
396
474
|
if (rel.endsWith(WRITE_WORKFLOW))
|
|
397
475
|
return [`${rel} (the tests you tick on a pull request OmniQA opened are written into it)`];
|
|
398
476
|
if (rel.endsWith(".github/workflows/omniqa.yml"))
|
|
@@ -410,6 +488,59 @@ const RUN_ON = (dir) => [`${dir}tests/**/*.ts`];
|
|
|
410
488
|
/** The CI files of `a.ci`, keyed by their path relative to the project: the GitHub workflow at the repository root, the GitLab job under .omniqa/ci (its including .gitlab-ci.yml is init's, since one of the user's may exist). */
|
|
411
489
|
function ciFiles(a, place, secretNames) {
|
|
412
490
|
const at = place.root === "" ? "" : `${place.root}/`;
|
|
491
|
+
const sec = secEnvironment(a);
|
|
492
|
+
const dir = place.dir === "" ? "" : `${place.dir}/`;
|
|
493
|
+
/** An environment says securityScan: on: the sec job's run ends with the agent scan, whose report it keeps from one run to the next. */
|
|
494
|
+
const scans = a.securityActive === true;
|
|
495
|
+
/**
|
|
496
|
+
* The GitHub job of the security tests: the steps of the other job and Interceptor, read-only, with no `needs`. Where the
|
|
497
|
+
* scan runs, the last report is restored from the cache before the run (what makes a confirmed claim new or known), saved
|
|
498
|
+
* after it whatever its outcome, and kept as an artifact on every run: what the scan found is in no log. No artifact step
|
|
499
|
+
* here needs `include-hidden-files`: upload-artifact leaves out the dot-named entries inside a path it is given, and a path
|
|
500
|
+
* that leads through `.omniqa` is searched like any other (its v7 code, read on 2026-11-17; OmniQA writes no dot-named file there).
|
|
501
|
+
*/
|
|
502
|
+
const secJob = (command, env) => ` sec:
|
|
503
|
+
runs-on: ubuntu-latest
|
|
504
|
+
permissions:
|
|
505
|
+
contents: read # a security run heals and records nothing: there is nothing to commit
|
|
506
|
+
${place.dir === "" ? "" : ` defaults:\n run:\n working-directory: ${place.dir}\n`} steps:
|
|
507
|
+
- uses: actions/checkout@v7
|
|
508
|
+
- uses: actions/setup-node@v7
|
|
509
|
+
with:
|
|
510
|
+
node-version: 26
|
|
511
|
+
check-latest: true
|
|
512
|
+
- run: |
|
|
513
|
+
npm install -g omniqa@latest
|
|
514
|
+
omniqa install --with-deps
|
|
515
|
+
omniqa install interceptor
|
|
516
|
+
${scans ? ` - uses: actions/cache/restore@v6
|
|
517
|
+
with:
|
|
518
|
+
path: ${dir}.omniqa/security
|
|
519
|
+
key: omniqa-security-\${{ github.run_id }}
|
|
520
|
+
restore-keys: omniqa-security-
|
|
521
|
+
` : ""} - run: ${command}
|
|
522
|
+
env:
|
|
523
|
+
${env}${secretNames.map((n) => ` ${n}: \${{ secrets.${n} }}\n`).join("")}${scans ? ` SLACK_WEBHOOK_URL: \${{ secrets.SLACK_WEBHOOK_URL }}
|
|
524
|
+
- if: always()
|
|
525
|
+
uses: actions/cache/save@v6
|
|
526
|
+
with:
|
|
527
|
+
path: ${dir}.omniqa/security
|
|
528
|
+
key: omniqa-security-\${{ github.run_id }}
|
|
529
|
+
- if: always()
|
|
530
|
+
uses: actions/upload-artifact@v7
|
|
531
|
+
with:
|
|
532
|
+
name: omniqa-security-report
|
|
533
|
+
path: ${dir}.omniqa/security
|
|
534
|
+
if-no-files-found: ignore
|
|
535
|
+
` : ""} - if: failure()
|
|
536
|
+
uses: actions/upload-artifact@v7
|
|
537
|
+
with:
|
|
538
|
+
name: omniqa-security-results
|
|
539
|
+
path: |
|
|
540
|
+
${dir}test-results
|
|
541
|
+
${dir}.omniqa/runs
|
|
542
|
+
if-no-files-found: ignore
|
|
543
|
+
`;
|
|
413
544
|
if (a.ci === "github") {
|
|
414
545
|
return {
|
|
415
546
|
[`${at}.github/workflows/omniqa.yml`]: `# OmniQA: the tests on every push and pull request that touches a test file (tests/**/*.ts), and by hand from the Actions tab (Run
|
|
@@ -417,7 +548,14 @@ function ciFiles(a, place, secretNames) {
|
|
|
417
548
|
# pull request when the branch is protected or the environment is production: allow GitHub Actions to create and approve pull
|
|
418
549
|
# requests under Settings → Actions → General → Workflow permissions. A failed run keeps its screenshots, report and transcripts
|
|
419
550
|
# as the omniqa-test-results artifact. Every secret below has the name and the value it has in .env (Settings, Secrets and
|
|
420
|
-
# variables, Actions)
|
|
551
|
+
# variables, Actions).${sec === undefined ? "" : `
|
|
552
|
+
# The security tests run apart, in the sec job: on ${sec}, in a browser behind the Interceptor proxy. Neither job waits for the
|
|
553
|
+
# other or fails with it; sec only reads the repository, and a failed run keeps its results as omniqa-security-results.${scans ? `
|
|
554
|
+
# ${sec} says securityScan: on, so the sec job's run ends with the agent scan (model calls on every run; --no-scan on its
|
|
555
|
+
# line keeps a run to the tests). The scan's report says where the product is open and is in no log: the job restores the
|
|
556
|
+
# last one from the cache (a finding it already listed does not fail the run again), saves the new one, and keeps it as
|
|
557
|
+
# the omniqa-security-report artifact, so keep this repository's artifacts private. SLACK_WEBHOOK_URL, when it is set,
|
|
558
|
+
# gets one line with the counts when a scan confirms something new.` : ""}`}
|
|
421
559
|
name: omniqa
|
|
422
560
|
on:
|
|
423
561
|
push:
|
|
@@ -440,7 +578,7 @@ ${place.dir === "" ? "" : ` defaults:\n run:\n working-directory:
|
|
|
440
578
|
- run: |
|
|
441
579
|
npm install -g omniqa@latest
|
|
442
580
|
omniqa install --with-deps
|
|
443
|
-
- run: ${
|
|
581
|
+
- run: ${ciTestCommand(a)}
|
|
444
582
|
env:
|
|
445
583
|
${secretNames.map((n) => ` ${n}: \${{ secrets.${n} }}\n`).join("")} GITHUB_TOKEN: \${{ secrets.GITHUB_TOKEN }}
|
|
446
584
|
- if: failure()
|
|
@@ -451,7 +589,7 @@ ${secretNames.map((n) => ` ${n}: \${{ secrets.${n} }}\n`).join("")}
|
|
|
451
589
|
${place.dir === "" ? "" : `${place.dir}/`}test-results
|
|
452
590
|
${place.dir === "" ? "" : `${place.dir}/`}.omniqa/runs
|
|
453
591
|
if-no-files-found: ignore
|
|
454
|
-
`,
|
|
592
|
+
${sec === undefined ? "" : secJob(`omniqa security --env ${sec}`, "")}`,
|
|
455
593
|
...(a.changelog === undefined ? {} : { [`${at}.github/workflows/omniqa-deploy.yml`]: `# OmniQA: after a product deploy. The product's deploy pipeline starts this workflow (repository_dispatch omniqa-deploy, a
|
|
456
594
|
# curl with no payload); it runs the suite on the deployed environment, posts the failures to the team channel and fixes
|
|
457
595
|
# them behind the reviewers, rewrites the knowledge where the change lands and commits the result (a pull request on a
|
|
@@ -503,7 +641,7 @@ ${secretNames.map((n) => ` ${n}: \${{ secrets.${n} }}\n`).join("")}
|
|
|
503
641
|
${place.dir === "" ? "" : `${place.dir}/`}test-results
|
|
504
642
|
${place.dir === "" ? "" : `${place.dir}/`}.omniqa/runs
|
|
505
643
|
if-no-files-found: ignore
|
|
506
|
-
`, [`${at}${WRITE_WORKFLOW}`]: writeWorkflow(place, secretNames) }),
|
|
644
|
+
${sec === undefined ? "" : secJob(`omniqa security --env "\${OMNIQA_ENV:-${deployEnvironment(a)}}"`, " OMNIQA_ENV: ${{ github.event.client_payload.environment || inputs.environment }}\n")}`, [`${at}${WRITE_WORKFLOW}`]: writeWorkflow(place, secretNames) }),
|
|
507
645
|
...(a.pages ? { [`${at}.github/workflows/omniqa-pages.yml`]: `# OmniQA: the product scheme page (${place.dir === "" ? "" : `${place.dir}/`}.omniqa/knowledge/scheme.html, written with the README) on GitHub Pages after every push to the default
|
|
508
646
|
# branch that changes it. Enable Pages under Settings → Pages with GitHub Actions as the source. The site is public unless the
|
|
509
647
|
# organization is on GitHub Enterprise, and a private repository needs a paid plan; delete this file to stop publishing.
|
|
@@ -559,13 +697,48 @@ ${RUN_ON(place.dir === "" ? "" : `${place.dir}/`).map((p) => ` - ${p}\n`)
|
|
|
559
697
|
- npm install -g omniqa@latest
|
|
560
698
|
- omniqa install --with-deps
|
|
561
699
|
script:
|
|
562
|
-
${place.dir === "" ? "" : ` - cd ${place.dir}\n`} - ${
|
|
700
|
+
${place.dir === "" ? "" : ` - cd ${place.dir}\n`} - ${ciTestCommand(a)}
|
|
563
701
|
artifacts:
|
|
564
702
|
when: on_failure
|
|
565
703
|
paths:
|
|
566
704
|
- ${place.dir === "" ? "" : `${place.dir}/`}test-results
|
|
567
705
|
- ${place.dir === "" ? "" : `${place.dir}/`}.omniqa/runs
|
|
568
|
-
${
|
|
706
|
+
${sec === undefined ? "" : `
|
|
707
|
+
# The security tests, apart from the others: on ${sec}, or on the environment a deploy names (OMNIQA_ENV), in a browser
|
|
708
|
+
# behind the Interceptor proxy. It runs when the omniqa job runs${a.changelog === undefined ? "" : " and when the omniqa deploy job does"}, waits for neither and fails with neither;
|
|
709
|
+
# it pushes nothing, so it needs no GITLAB_TOKEN. A failed run keeps its results as the job's artifacts.${scans ? `
|
|
710
|
+
# ${sec} says securityScan: on, so the job's run ends with the agent scan (model calls on every run; --no-scan on its line
|
|
711
|
+
# keeps a run to the tests). The scan's report says where the product is open and is in no log: the job's cache carries the
|
|
712
|
+
# last one to the next run (a finding it already listed does not fail the run again), and every run keeps it among its
|
|
713
|
+
# artifacts, so keep them private. SLACK_WEBHOOK_URL, when it is set, gets one line with the counts when a scan confirms
|
|
714
|
+
# something new.` : ""}
|
|
715
|
+
omniqa sec:
|
|
716
|
+
image: node:26
|
|
717
|
+
rules:
|
|
718
|
+
${a.changelog === undefined ? "" : ` - if: $CI_PIPELINE_SOURCE == "trigger" || $CI_PIPELINE_SOURCE == "pipeline"\n`} - if: $CI_PIPELINE_SOURCE == "web"
|
|
719
|
+
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
|
720
|
+
changes:
|
|
721
|
+
${RUN_ON(dir).map((p) => ` - ${p}\n`).join("")} - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
|
|
722
|
+
when: never
|
|
723
|
+
- if: $CI_COMMIT_BRANCH
|
|
724
|
+
changes:
|
|
725
|
+
${RUN_ON(dir).map((p) => ` - ${p}\n`).join("")} before_script:
|
|
726
|
+
- npm install -g omniqa@latest
|
|
727
|
+
- omniqa install --with-deps
|
|
728
|
+
- omniqa install interceptor
|
|
729
|
+
script:
|
|
730
|
+
${place.dir === "" ? "" : ` - cd ${place.dir}\n`} - omniqa security --env "\${OMNIQA_ENV:-${sec}}"
|
|
731
|
+
${scans ? ` cache:
|
|
732
|
+
key: omniqa-security
|
|
733
|
+
paths:
|
|
734
|
+
- ${dir}.omniqa/security
|
|
735
|
+
when: always
|
|
736
|
+
` : ""} artifacts:
|
|
737
|
+
when: ${scans ? "always" : "on_failure"}
|
|
738
|
+
paths:
|
|
739
|
+
- ${dir}test-results
|
|
740
|
+
- ${dir}.omniqa/runs
|
|
741
|
+
${scans ? ` - ${dir}.omniqa/security\n` : ""}`}${a.changelog === undefined ? "" : `
|
|
569
742
|
# After a product deploy: the product's deploy pipeline starts this one (a pipeline trigger token, a curl with no payload);
|
|
570
743
|
# the job runs the suite on the deployed environment, posts the failures to the team channel and fixes them behind the
|
|
571
744
|
# reviewers, rewrites the knowledge where the change lands and commits the result through GITLAB_TOKEN (a merge request
|
|
@@ -617,6 +790,13 @@ export function render(a, place = { root: "", dir: "" }) {
|
|
|
617
790
|
const m = a.models[r] ?? examples[r];
|
|
618
791
|
return m === undefined ? "" : ` ${a.models[r] === undefined ? "// " : ""}${r}: ${modelLine(a, m)},\n`;
|
|
619
792
|
}).join("");
|
|
793
|
+
// The scan's two roles, with the security kind alone: as the interview wrote them, else the provider's suggestion as a commented example, there for the day an environment says securityScan: on.
|
|
794
|
+
const scan = choiceOf(a.provider, a.baseUrl).scan;
|
|
795
|
+
const scanRoles = a.security !== true ? "" : SCAN_ROLES.map((r) => {
|
|
796
|
+
const m = a.models[r] ?? scan?.[r];
|
|
797
|
+
return m === undefined ? "" : ` ${a.models[r] === undefined ? "// " : ""}${r}: ${modelLine(a, m)},\n`;
|
|
798
|
+
}).join("");
|
|
799
|
+
const scanLines = scanRoles === "" ? "" : ` // The agent scan of an environment that says securityScan: on (.omniqa/product.yaml) runs on these two roles: security is the researcher, which reads the\n // product's traffic and sends requests of its own; securityReviewer confirms or discards each of its claims. fallback names the model that answers\n // when the first declines or its provider fails, and the report says which model answered. A role left out uses default.\n${scanRoles}`;
|
|
620
800
|
const platforms = DEVICE_PLATFORMS.filter((p) => a.apps[p] !== undefined);
|
|
621
801
|
const projectLines = platforms.length === 0
|
|
622
802
|
? ' // { name: "ios", platform: "ios", app: "com.example.app" },\n // { name: "android", platform: "android", app: "com.example.app", device: "Pixel_9" },\n'
|
|
@@ -628,7 +808,7 @@ export function render(a, place = { root: "", dir: "" }) {
|
|
|
628
808
|
const r = a.apps[p];
|
|
629
809
|
return ` ${p}: { runner: ${JSON.stringify(r.runner)}${r.url === undefined ? "" : `, url: ${JSON.stringify(r.url)}`}${r.provider === undefined ? "" : `, provider: ${JSON.stringify(r.provider)}`}, tokenEnv: "AGENT_DEVICE_DAEMON_AUTH_TOKEN" },\n`;
|
|
630
810
|
}).join("")} },\n`;
|
|
631
|
-
const scopeLine = a.api ? " scope: { e2e: true, api: true },\n" : " // scope: { e2e: true, api: false },\n";
|
|
811
|
+
const scopeLine = a.security === true ? ` scope: { e2e: true, api: ${a.api}, security: true },\n` : a.api ? " scope: { e2e: true, api: true },\n" : " // scope: { e2e: true, api: false },\n";
|
|
632
812
|
const productionExample = a.environments.length === 1
|
|
633
813
|
? ` # production:\n # url: https://example.com\n # tier: production # ${TIER}\n # explore: true # ${exploreComment("production")}\n # replay: false # ${REPLAY}\n # # mutations: false # ${MUTATIONS}\n # # deny: [/checkout, "Delete account"] # ${DENY}\n`
|
|
634
814
|
: "";
|
|
@@ -650,7 +830,7 @@ export default defineConfig({
|
|
|
650
830
|
models: {
|
|
651
831
|
default: ${modelLine(a, a.models.default)},
|
|
652
832
|
// Roles left out use default. A cheaper executor saves the most on first runs; the reviewer is the quality gate:
|
|
653
|
-
${roleLines} },
|
|
833
|
+
${roleLines}${scanLines} },
|
|
654
834
|
testIdAttribute: ${JSON.stringify(a.testId)},
|
|
655
835
|
projects: [
|
|
656
836
|
{ name: "desktop" },
|
|
@@ -668,12 +848,13 @@ ${devicesLines} // Heals and generated tests are committed on the branch under
|
|
|
668
848
|
// (and always as a pull request from a production-tier environment):
|
|
669
849
|
// persist: { protectedBranches: ["main", "master"] },
|
|
670
850
|
// gitHosts: { github: { tokenEnv: "GITHUB_TOKEN" } }, // or gitlab: { url: "https://gitlab.example.com", tokenEnv: "GITLAB_TOKEN" }
|
|
671
|
-
${changelogLines} // What the suite covers: agentic end-to-end tests
|
|
851
|
+
${changelogLines} // What the suite covers: agentic end-to-end tests always; api: true adds deterministic API tests from the endpoint map, security: true
|
|
852
|
+
// security tests on the production tier, run in a browser behind the Interceptor proxy (omniqa install interceptor).
|
|
672
853
|
${scopeLine}});
|
|
673
854
|
`,
|
|
674
855
|
".omniqa/product.yaml": `name: ${JSON.stringify(a.name)}
|
|
675
856
|
environments:
|
|
676
|
-
${a.environments.map((e, i) => environmentBlock(e, i === 0)).join("")}${productionExample}${accountLines}${sources}# crawl: # omniqa crawl: read-only, logs in as \`account\` (default: the first account)
|
|
857
|
+
${a.environments.map((e, i) => environmentBlock(e, i === 0, a)).join("")}${productionExample}${accountLines}${sources}# crawl: # omniqa crawl: read-only, logs in as \`account\` (default: the first account)
|
|
677
858
|
# max_pages: 50
|
|
678
859
|
# max_depth: 3
|
|
679
860
|
# exclude: [/admin] # path substrings, on top of logout, delete and payment routes
|
|
@@ -980,10 +1161,22 @@ export async function init(args, cwd) {
|
|
|
980
1161
|
gitlabLine = current.includes(job) ? `${shown} (includes it already)` : `${shown} left as is: add \`include: [{ local: ${job} }]\` to it`;
|
|
981
1162
|
}
|
|
982
1163
|
const names = secretNamesOf(answers);
|
|
983
|
-
const listed = [...Object.keys(files).flatMap((rel) => [...described(rel, answers.ci, names, answers.pages, answers.changelog, ciTest(answers).note), ...(rel === ".omniqa/ci/gitlab-ci.yml" && gitlabLine !== undefined ? [gitlabLine] : [])]), ".mcp.json (the omniqa MCP server for Claude Code; Cursor reads the same shape from .cursor/mcp.json)"];
|
|
1164
|
+
const listed = [...Object.keys(files).flatMap((rel) => [...described(rel, answers.ci, names, answers.pages, answers.changelog, ciTest(answers).note, secNote(answers)), ...(rel === ".omniqa/ci/gitlab-ci.yml" && gitlabLine !== undefined ? [gitlabLine] : [])]), ".mcp.json (the omniqa MCP server for Claude Code; Cursor reads the same shape from .cursor/mcp.json)"];
|
|
984
1165
|
const pkg = await modulePackage(cwd);
|
|
985
1166
|
if (pkg !== undefined)
|
|
986
1167
|
listed.push(pkg);
|
|
1168
|
+
// The user chose the kind, so the proxy it runs through is installed now; a failed install is said and stops nothing.
|
|
1169
|
+
if (answers.security === true) {
|
|
1170
|
+
const repeats = "omniqa install interceptor repeats it, e.g. in CI";
|
|
1171
|
+
try {
|
|
1172
|
+
progress(`installing Interceptor ${INTERCEPTOR_VERSION} (the proxy security tests run through; a minute the first time)`);
|
|
1173
|
+
const { dir } = await installInterceptor({ log: progress });
|
|
1174
|
+
listed.push(process.env.OMNIQA_INTERCEPTOR_PATH ? `interceptor at ${dir} (the proxy the security pass and the security tests run through; ${repeats})` : `interceptor ${INTERCEPTOR_VERSION} → ${dir} (the proxy the security pass and the security tests run through; its Python 3.13 and packages stay in that folder; ${repeats})`);
|
|
1175
|
+
}
|
|
1176
|
+
catch (err) {
|
|
1177
|
+
listed.push(`interceptor is not installed: ${err.message.split("\n")[0]}; omniqa install interceptor tries again (until then the security pass and the security tests do not run)`);
|
|
1178
|
+
}
|
|
1179
|
+
}
|
|
987
1180
|
try {
|
|
988
1181
|
if ((await linkOmniqa(cwd)) !== undefined)
|
|
989
1182
|
listed.push(`node_modules/omniqa → ${PACKAGE_DIR} (linked: the config and the tests import omniqa from it)`);
|