omniqa 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/commands/doctor.d.ts +10 -0
- package/dist/commands/doctor.d.ts.map +1 -1
- package/dist/commands/doctor.js +85 -5
- package/dist/commands/doctor.js.map +1 -1
- package/dist/commands/init.d.ts +5 -2
- package/dist/commands/init.d.ts.map +1 -1
- package/dist/commands/init.js +114 -76
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/sync.d.ts.map +1 -1
- package/dist/commands/sync.js +18 -14
- package/dist/commands/sync.js.map +1 -1
- package/dist/deploy-setup.d.ts +36 -0
- package/dist/deploy-setup.d.ts.map +1 -0
- package/dist/deploy-setup.js +132 -0
- package/dist/deploy-setup.js.map +1 -0
- package/dist/sync.d.ts +11 -5
- package/dist/sync.d.ts.map +1 -1
- package/dist/sync.js +26 -18
- package/dist/sync.js.map +1 -1
- package/package.json +6 -6
package/dist/commands/init.js
CHANGED
|
@@ -3,6 +3,7 @@ import { basename, dirname, join, relative, resolve, sep as pathSep } from "node
|
|
|
3
3
|
import { stderr, stdout } from "node:process";
|
|
4
4
|
import { parseArgs, styleText } from "node:util";
|
|
5
5
|
import { DEFAULT_KEY_ENV, DEVICE_PLATFORMS, git, parseRemote, PRODUCT_FILE, readmeHasBlock, readState, README_FILE, recordOnboarding, remoteUrl, writeReadme } from "@omniqa/core";
|
|
6
|
+
import { deploySetup } from "../deploy-setup.js";
|
|
6
7
|
import { linkOmniqa, PACKAGE_DIR } from "../link.js";
|
|
7
8
|
import { lineReader } from "../prompt.js";
|
|
8
9
|
import { pageByDefault, runStages } from "./plan.js";
|
|
@@ -38,8 +39,17 @@ export function changelogOf(repos) {
|
|
|
38
39
|
}
|
|
39
40
|
return undefined;
|
|
40
41
|
}
|
|
41
|
-
const
|
|
42
|
-
|
|
42
|
+
const DEPLOY_FROM = ["github", "gitlab", "other"];
|
|
43
|
+
const DEPLOY_FROM_HINT = "--deploy-from must be github, gitlab or other";
|
|
44
|
+
const DEPLOY_FROM_QUESTION = "Where is the product deployed from? (1. GitHub Actions, 2. GitLab CI, 3. elsewhere: a script, a server, by hand)";
|
|
45
|
+
/** The answer to the deploy-from question: its number, or a name (elsewhere is other). */
|
|
46
|
+
const deployFromOf = (answer) => {
|
|
47
|
+
const a = answer.toLowerCase();
|
|
48
|
+
if (/^[1-3]$/.test(a))
|
|
49
|
+
return DEPLOY_FROM[Number(a) - 1];
|
|
50
|
+
return a === "elsewhere" ? "other" : DEPLOY_FROM.find((f) => f === a);
|
|
51
|
+
};
|
|
52
|
+
/** The environment the deploy job runs on without a payload, named by the setup steps and the run-by-hand hints: the first production-tier one, else the first. */
|
|
43
53
|
const deployEnvironment = (a) => (a.environments.find((e) => e.tier === "production") ?? a.environments[0])?.name ?? "production";
|
|
44
54
|
// Enter at the model questions, and --yes, write the provider's suggestion: Sonnet 5.5 at effort high, the executor on the same model at
|
|
45
55
|
// medium (one action or one verdict a call, where more effort buys nothing measurable) and the reviewer Opus 5.5 at high (the setup the
|
|
@@ -78,6 +88,8 @@ const USAGE = `omniqa init [folder] [options]
|
|
|
78
88
|
--url <url> Where the product is running, the dev-tier environment staging: a local run too, e.g. http://127.0.0.1:9000;
|
|
79
89
|
never example.com (--yes without --url writes https://staging.example.com, to edit before the plan stage)
|
|
80
90
|
--prod-url <url> The same for the production-tier environment production
|
|
91
|
+
--repo <path|url> A repository of the product (repeatable; the interview's default list): a local path, or a git url cloned shallowly
|
|
92
|
+
into .omniqa/repos/. A GitHub or GitLab url also writes the changelog block and the deploy job (see --ci)
|
|
81
93
|
--provider <p> anthropic | openai-compatible
|
|
82
94
|
--model <id> Default model, alone: no effort, no suggested executor or reviewer (default: the provider's suggestion at effort high,
|
|
83
95
|
the executor on the same model at medium;
|
|
@@ -90,6 +102,9 @@ const USAGE = `omniqa init [folder] [options]
|
|
|
90
102
|
(default: the host of this folder's git remote, else of the first repository url, else github); at the repository root,
|
|
91
103
|
the job running in the project's folder
|
|
92
104
|
a repository url on GitHub or GitLab also writes the changelog block and the deploy job (.github/workflows/omniqa-deploy.yml, or an omniqa deploy job in .omniqa/ci/gitlab-ci.yml) the product's deploy pipeline starts
|
|
105
|
+
--deploy-from <f> Where the product is deployed from, for the three setup steps printed after the files: github (GitHub Actions),
|
|
106
|
+
gitlab (GitLab CI) or other (a script, a server, by hand); default: the product repository's host; needs a --repo
|
|
107
|
+
url on GitHub or GitLab
|
|
93
108
|
--pages Publish the scheme page (.omniqa/knowledge/scheme.html) on the CI host's Pages: .github/workflows/omniqa-pages.yml
|
|
94
109
|
(public unless the organization is on GitHub Enterprise; a private repository needs a paid plan), or a pages job
|
|
95
110
|
in .omniqa/ci/gitlab-ci.yml (visibility per project); the README links the site (default: no)
|
|
@@ -98,10 +113,16 @@ async function exists(path) {
|
|
|
98
113
|
return access(path).then(() => true, () => false);
|
|
99
114
|
}
|
|
100
115
|
/** Today's `--yes` defaults, from the flags. */
|
|
101
|
-
export function defaultsFrom(cwd, flags, remote) {
|
|
102
|
-
const ci = flags.ci ?? ciDefault(
|
|
116
|
+
export function defaultsFrom(cwd, flags, remote, repos = []) {
|
|
117
|
+
const ci = flags.ci ?? ciDefault(repos, remote);
|
|
103
118
|
if (!CI_CHOICES.includes(ci))
|
|
104
119
|
throw new Error(`${CI_HINT}, got ${JSON.stringify(ci)}`);
|
|
120
|
+
const changelog = changelogOf(repos);
|
|
121
|
+
const deployFrom = flags["deploy-from"];
|
|
122
|
+
if (deployFrom !== undefined && !DEPLOY_FROM.includes(deployFrom))
|
|
123
|
+
throw new Error(`${DEPLOY_FROM_HINT}, got ${JSON.stringify(deployFrom)}`);
|
|
124
|
+
if (deployFrom !== undefined && changelog === undefined)
|
|
125
|
+
throw new Error("--deploy-from needs --repo <url> on GitHub or GitLab");
|
|
105
126
|
const provider = flags.provider ?? "anthropic";
|
|
106
127
|
if (!PROVIDERS.includes(provider))
|
|
107
128
|
throw new Error(`--provider must be anthropic or openai-compatible, got "${provider}"`);
|
|
@@ -119,9 +140,13 @@ export function defaultsFrom(cwd, flags, remote) {
|
|
|
119
140
|
environments.push({ name: "production", url: flags["prod-url"], tier: "production", explore: true, mutations: true });
|
|
120
141
|
// the suggested executor, reviewer and efforts go with the suggested model only; another API suggests none, so --yes needs --model there
|
|
121
142
|
const models = choice.models !== undefined && (!flags.model || flags.model === choice.models.default.model) ? { ...choice.models } : { default: { model: flags.model ?? "" } };
|
|
122
|
-
const a = { name: flags.name ?? basename(cwd), environments, repos
|
|
143
|
+
const a = { name: flags.name ?? basename(cwd), environments, repos, docs: [], api: false, apps: {}, accounts: [], provider, keyEnv, models, testId: flags["test-id"] ?? "data-testid", ci, pages: false, notes: [] };
|
|
123
144
|
if (baseUrl !== undefined)
|
|
124
145
|
a.baseUrl = baseUrl;
|
|
146
|
+
if (changelog !== undefined) {
|
|
147
|
+
a.changelog = changelog;
|
|
148
|
+
a.deployFrom = deployFrom ?? changelog.host;
|
|
149
|
+
}
|
|
125
150
|
return a;
|
|
126
151
|
}
|
|
127
152
|
const envName = (s) => s.toUpperCase().replace(/[^A-Z0-9]+/g, "_").replace(/^_+|_+$/g, "");
|
|
@@ -196,6 +221,9 @@ export async function interview(reader, d, o = {}) {
|
|
|
196
221
|
environments.push({ name: envNameAnswer, url, tier, explore, mutations });
|
|
197
222
|
}
|
|
198
223
|
const repos = await list("Repository path or git url (urls are cloned shallowly into .omniqa/repos/), empty when done", d.repos);
|
|
224
|
+
// Where the product deploys from picks the hook of the setup steps: asked once a repository url names a host, whose CI is the default.
|
|
225
|
+
const changelog = changelogOf(repos);
|
|
226
|
+
const deployFrom = changelog === undefined ? undefined : await askUntil(DEPLOY_FROM_QUESTION, String(DEPLOY_FROM.indexOf(d.deployFrom ?? changelog.host) + 1), deployFromOf, "please answer 1, 2 or 3");
|
|
199
227
|
const docs = await list("Documentation path or url besides each repository's README.md and docs/ (markdown, HTML, text; PDF through pdftotext), empty when done", d.docs);
|
|
200
228
|
const ci = await choice("CI for this project's repository (github, gitlab, none)", CI_CHOICES, o.ciFlag ?? ciDefault(repos, o.remote));
|
|
201
229
|
const pages = ci === "none" ? false : await yes(PAGES_QUESTION[ci], d.pages);
|
|
@@ -279,9 +307,10 @@ export async function interview(reader, d, o = {}) {
|
|
|
279
307
|
const testId = await ask("Test id attribute", d.testId);
|
|
280
308
|
const notes = await list("Anything else the agents should know, one note a line (e.g. production signs in through Google only; written to .omniqa/instructions.md, which every model prompt carries), empty when done", d.notes);
|
|
281
309
|
const a = { name, environments, repos, docs, api, apps, accounts, provider, keyEnv, models, testId, ci, pages, notes };
|
|
282
|
-
const changelog = changelogOf(a.repos);
|
|
283
310
|
if (changelog !== undefined)
|
|
284
311
|
a.changelog = changelog;
|
|
312
|
+
if (deployFrom !== undefined)
|
|
313
|
+
a.deployFrom = deployFrom;
|
|
285
314
|
if (baseUrl !== undefined)
|
|
286
315
|
a.baseUrl = baseUrl;
|
|
287
316
|
return a;
|
|
@@ -325,27 +354,31 @@ const bold = (line) => styleText("bold", line);
|
|
|
325
354
|
const secretsWhere = (ci, names) => ci === "github" ? { under: "Settings → Secrets and variables → Actions", names: names.join(", "), note: "GITHUB_TOKEN comes from Actions" }
|
|
326
355
|
: ci === "gitlab" ? { under: "Settings → CI/CD → Variables", names: [...names, "GITLAB_TOKEN"].join(", "), note: "a project access token with the api and write_repository scopes" }
|
|
327
356
|
: undefined;
|
|
328
|
-
/**
|
|
357
|
+
/**
|
|
358
|
+
* The listing lines of a file init wrote: a CI file says what it does, then in bold which secrets to add where and which settings to
|
|
359
|
+
* flip. With a changelog the three setup steps under the listing say the secrets, so the bold lines are left out: nothing twice on one screen.
|
|
360
|
+
*/
|
|
329
361
|
const described = (rel, ci, names, pages, changelog) => {
|
|
330
362
|
const where = secretsWhere(ci, names);
|
|
331
363
|
if (where === undefined)
|
|
332
364
|
return [rel];
|
|
333
|
-
const
|
|
365
|
+
const secrets = changelog === undefined ? [` ${bold(`add under ${where.under}: ${where.names} (${where.note})`)}`] : [];
|
|
334
366
|
if (rel.endsWith(".github/workflows/omniqa-deploy.yml"))
|
|
335
|
-
return [`${rel} (after a product deploy: the suite on the deployed environment, the failures posted and fixed, the knowledge where the change lands;
|
|
367
|
+
return [`${rel} (after a product deploy: the suite on the deployed environment, the failures posted and fixed, the knowledge where the change lands; set up below)`];
|
|
336
368
|
if (rel.endsWith(".github/workflows/omniqa.yml"))
|
|
337
|
-
return [`${rel} (the tests on every push and pull request that touches a test file, and by hand from the Actions tab; heals and the refreshed coverage come back as a commit, or a pull request on a protected branch)`,
|
|
369
|
+
return [`${rel} (the tests on every push and pull request that touches a test file, and by hand from the Actions tab; heals and the refreshed coverage come back as a commit, or a pull request on a protected branch)`, ...secrets, ...(changelog === undefined ? [` ${bold("allow GitHub Actions to create and approve pull requests under Settings → Actions → General → Workflow permissions (heals come as pull requests on a protected branch or from a production environment)")}`] : [])];
|
|
338
370
|
if (rel.endsWith(".github/workflows/omniqa-pages.yml"))
|
|
339
371
|
return [`${rel} (the scheme page on GitHub Pages after every push to the default branch that changes it)`, ` ${bold("enable Pages under Settings → Pages with GitHub Actions as the source; the site is public unless the organization is on GitHub Enterprise")}`];
|
|
340
372
|
if (rel === ".omniqa/ci/gitlab-ci.yml")
|
|
341
|
-
return [`${rel} (the tests on every push and merge request that touches a test file, and by hand with
|
|
373
|
+
return [`${rel} (the tests on every push and merge request that touches a test file, and by hand with New pipeline (Build → Pipelines); heals and the refreshed coverage come back through GITLAB_TOKEN${changelog !== undefined ? "; after a product deploy, the suite, the healer and the knowledge through the omniqa deploy job; set up below" : ""}${pages ? "; the scheme page on GitLab Pages after every push to the default branch that changes it" : ""})`, ...secrets, ...(pages ? [` ${bold("who sees the Pages site is set under Settings → General → Visibility, project features, permissions: project members, or everyone")}`] : [])];
|
|
342
374
|
return [rel];
|
|
343
375
|
};
|
|
344
|
-
const
|
|
376
|
+
const NPM_COMMENT = "The newest OmniQA from npm (pin omniqa@<version> here to hold CI on one).";
|
|
377
|
+
const CI_COMMENT = `${NPM_COMMENT} Heals and the refreshed coverage are committed on the branch under test, or opened as a`;
|
|
345
378
|
// ponytail: a push or merge request runs the tests only when it touches a test file; everything else (recordings, the config, the knowledge, a README) waits for a run by hand. Widen the list here if a project wants more.
|
|
346
379
|
const RUN_ON = (dir) => [`${dir}tests/**/*.ts`];
|
|
347
380
|
/** The CI files of `a.ci`, keyed by their path relative to the project: the GitHub workflow at the repository root, the GitLab job under .omniqa/ci (its including .gitlab-ci.yml is init's, since one of the user's may exist). */
|
|
348
|
-
function ciFiles(a, place, secretNames
|
|
381
|
+
function ciFiles(a, place, secretNames) {
|
|
349
382
|
const at = place.root === "" ? "" : `${place.root}/`;
|
|
350
383
|
if (a.ci === "github") {
|
|
351
384
|
return {
|
|
@@ -389,25 +422,12 @@ ${secretNames.map((n) => ` ${n}: \${{ secrets.${n} }}\n`).join("")}
|
|
|
389
422
|
${place.dir === "" ? "" : `${place.dir}/`}.omniqa/runs
|
|
390
423
|
if-no-files-found: ignore
|
|
391
424
|
`,
|
|
392
|
-
...(a.changelog === undefined ? {} : { [`${at}.github/workflows/omniqa-deploy.yml`]: `# OmniQA: after a product deploy. The product's deploy pipeline
|
|
393
|
-
#
|
|
394
|
-
#
|
|
395
|
-
# production environment or a protected branch). Run workflow in the Actions tab
|
|
396
|
-
#
|
|
397
|
-
#
|
|
398
|
-
#
|
|
399
|
-
${a.changelog.host === "github" ? `# In the product's deploy workflow, after the deploy step (OMNIQA_DISPATCH_TOKEN: a fine-grained token on ${qaRepo} with Contents: read and write):
|
|
400
|
-
# - run: gh api repos/${qaRepo}/dispatches -f event_type=omniqa-deploy -f "client_payload[sha]=$GITHUB_SHA" -f "client_payload[environment]=${deployEnvironment(a)}"
|
|
401
|
-
# env:
|
|
402
|
-
# GH_TOKEN: \${{ secrets.OMNIQA_DISPATCH_TOKEN }}` : `# In the product's deploy pipeline, after the deploy step (OMNIQA_DISPATCH_TOKEN: a fine-grained token on ${qaRepo} with Contents: read and
|
|
403
|
-
# write; $CI_COMMIT_SHA is GitLab's deployed commit):
|
|
404
|
-
# curl -sS -X POST -H "Authorization: Bearer $OMNIQA_DISPATCH_TOKEN" -H "Accept: application/vnd.github+json" \\
|
|
405
|
-
# https://api.github.com/repos/${qaRepo}/dispatches \\
|
|
406
|
-
# -d '{"event_type":"omniqa-deploy","client_payload":{"sha":"'"$CI_COMMIT_SHA"'","environment":"${deployEnvironment(a)}"}}'`}
|
|
407
|
-
#
|
|
408
|
-
# Secrets (Settings, Secrets and variables, Actions), each with the value it has in .env: ${secretNames.join(", ")}; ${productTokenEnv(a.changelog.host)}
|
|
409
|
-
# when the product repository is private (and tokenEnv: "${productTokenEnv(a.changelog.host)}" in the changelog block of omniqa.config.ts);
|
|
410
|
-
# SLACK_WEBHOOK_URL for the notifications (notify.webhookEnv).
|
|
425
|
+
...(a.changelog === undefined ? {} : { [`${at}.github/workflows/omniqa-deploy.yml`]: `# OmniQA: after a product deploy. The product's deploy pipeline starts this workflow (repository_dispatch omniqa-deploy, a
|
|
426
|
+
# curl with no payload); it runs the suite on the deployed environment, posts the failures to the team channel and fixes
|
|
427
|
+
# them behind the reviewers, rewrites the knowledge where the change lands and commits the result (a pull request on a
|
|
428
|
+
# production environment or a protected branch). Run workflow in the Actions tab runs it by hand. The three setup steps
|
|
429
|
+
# (secrets, the product's hook, a first run): omniqa doctor. A failed run keeps its screenshots, report and transcripts as
|
|
430
|
+
# the omniqa-deploy-results artifact. ${NPM_COMMENT}
|
|
411
431
|
name: omniqa deploy
|
|
412
432
|
on:
|
|
413
433
|
repository_dispatch:
|
|
@@ -415,12 +435,11 @@ on:
|
|
|
415
435
|
workflow_dispatch:
|
|
416
436
|
inputs:
|
|
417
437
|
sha:
|
|
418
|
-
description: The product commit that was deployed
|
|
419
|
-
required:
|
|
438
|
+
description: "The product commit that was deployed (default: the head of main)"
|
|
439
|
+
required: false
|
|
420
440
|
environment:
|
|
421
|
-
description: Its environment in .omniqa/product.yaml
|
|
422
|
-
required:
|
|
423
|
-
default: ${deployEnvironment(a)}
|
|
441
|
+
description: "Its environment in .omniqa/product.yaml (default: ${deployEnvironment(a)})"
|
|
442
|
+
required: false
|
|
424
443
|
permissions:
|
|
425
444
|
contents: write
|
|
426
445
|
pull-requests: write
|
|
@@ -439,12 +458,12 @@ ${place.dir === "" ? "" : ` defaults:\n run:\n working-directory:
|
|
|
439
458
|
- run: |
|
|
440
459
|
npm install -g omniqa@latest
|
|
441
460
|
omniqa install --with-deps
|
|
442
|
-
- run: omniqa sync --to "$OMNIQA_SHA" --env "$OMNIQA_ENV"
|
|
461
|
+
- run: omniqa sync --suite --to "$OMNIQA_SHA" --env "$OMNIQA_ENV"
|
|
443
462
|
env:
|
|
444
463
|
OMNIQA_SHA: \${{ github.event.client_payload.sha || inputs.sha }}
|
|
445
464
|
OMNIQA_ENV: \${{ github.event.client_payload.environment || inputs.environment }}
|
|
446
465
|
${secretNames.map((n) => ` ${n}: \${{ secrets.${n} }}\n`).join("")} GITHUB_TOKEN: \${{ secrets.GITHUB_TOKEN }}
|
|
447
|
-
|
|
466
|
+
PRODUCT_REPO_TOKEN: \${{ secrets.PRODUCT_REPO_TOKEN }}
|
|
448
467
|
SLACK_WEBHOOK_URL: \${{ secrets.SLACK_WEBHOOK_URL }}
|
|
449
468
|
- if: failure()
|
|
450
469
|
uses: actions/upload-artifact@v7
|
|
@@ -488,9 +507,9 @@ jobs:
|
|
|
488
507
|
}
|
|
489
508
|
if (a.ci === "gitlab") {
|
|
490
509
|
return {
|
|
491
|
-
".omniqa/ci/gitlab-ci.yml": `# OmniQA: the tests on every push and merge request that touches a test file (tests/**/*.ts), and by hand with
|
|
492
|
-
# anything else, a README, the recordings, the config or the knowledge, runs nothing
|
|
493
|
-
# the repository root. ${CI_COMMENT}
|
|
510
|
+
".omniqa/ci/gitlab-ci.yml": `# OmniQA: the tests on every push and merge request that touches a test file (tests/**/*.ts), and by hand with
|
|
511
|
+
# New pipeline (Build → Pipelines); anything else, a README, the recordings, the config or the knowledge, runs nothing
|
|
512
|
+
# on its own. Included from .gitlab-ci.yml at the repository root. ${CI_COMMENT}
|
|
494
513
|
# merge request when the branch is protected: the job token cannot push, so GITLAB_TOKEN is a project access token with the api
|
|
495
514
|
# and write_repository scopes. A failed run keeps its screenshots, report and transcripts as the job's artifacts. Every variable
|
|
496
515
|
# below has the name and the value it has in .env (Settings, CI/CD, Variables):
|
|
@@ -498,7 +517,7 @@ jobs:
|
|
|
498
517
|
omniqa:
|
|
499
518
|
image: node:26
|
|
500
519
|
rules:
|
|
501
|
-
${a.changelog === undefined ? "" : ` - if: $CI_PIPELINE_SOURCE == "
|
|
520
|
+
${a.changelog === undefined ? "" : ` - if: $CI_PIPELINE_SOURCE == "trigger" || $CI_PIPELINE_SOURCE == "pipeline"\n when: never\n - if: $OMNIQA_ENV || $OMNIQA_SHA\n when: never\n`} - if: $CI_PIPELINE_SOURCE == "web"
|
|
502
521
|
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
|
503
522
|
changes:
|
|
504
523
|
${RUN_ON(place.dir === "" ? "" : `${place.dir}/`).map((p) => ` - ${p}\n`).join("")} - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
|
|
@@ -517,34 +536,23 @@ ${place.dir === "" ? "" : ` - cd ${place.dir}\n`} - omniqa test
|
|
|
517
536
|
- ${place.dir === "" ? "" : `${place.dir}/`}test-results
|
|
518
537
|
- ${place.dir === "" ? "" : `${place.dir}/`}.omniqa/runs
|
|
519
538
|
${a.changelog === undefined ? "" : `
|
|
520
|
-
# After a product deploy: the product's pipeline starts this one
|
|
521
|
-
# the job runs the suite
|
|
522
|
-
# rewrites the knowledge where the change lands and commits the result through GITLAB_TOKEN (a merge request
|
|
523
|
-
# environment or a protected branch).
|
|
524
|
-
# .
|
|
525
|
-
#
|
|
526
|
-
# stage: .post
|
|
527
|
-
# needs: [deploy]
|
|
528
|
-
# rules: [{ if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH }]
|
|
529
|
-
# trigger: { project: ${qaRepo}, branch: main }
|
|
530
|
-
# variables: { OMNIQA_SHA: $CI_COMMIT_SHA, OMNIQA_ENV: ${deployEnvironment(a)} }` : `In the product's
|
|
531
|
-
# deploy pipeline, after the deploy step (OMNIQA_TRIGGER_TOKEN: a pipeline trigger token of this project, under Settings → CI/CD →
|
|
532
|
-
# Pipeline trigger tokens; $GITHUB_SHA is GitHub's deployed commit):
|
|
533
|
-
# curl -sS -X POST -F "token=$OMNIQA_TRIGGER_TOKEN" -F ref=main -F "variables[OMNIQA_SHA]=$GITHUB_SHA" -F "variables[OMNIQA_ENV]=${deployEnvironment(a)}" \\
|
|
534
|
-
# "https://gitlab.com/api/v4/projects/${qaRepo.replaceAll("/", "%2F")}/trigger/pipeline"`}
|
|
535
|
-
# Variables (Settings, CI/CD, Variables): ${productTokenEnv(a.changelog.host)} when the product repository is private (and
|
|
536
|
-
# tokenEnv: "${productTokenEnv(a.changelog.host)}" in the changelog block of omniqa.config.ts), SLACK_WEBHOOK_URL for the notifications (notify.webhookEnv).
|
|
539
|
+
# After a product deploy: the product's deploy pipeline starts this one (a pipeline trigger token, a curl with no payload);
|
|
540
|
+
# the job runs the suite on the deployed environment, posts the failures to the team channel and fixes them behind the
|
|
541
|
+
# reviewers, rewrites the knowledge where the change lands and commits the result through GITLAB_TOKEN (a merge request
|
|
542
|
+
# on a production environment or a protected branch). New pipeline (Build → Pipelines) with a variable OMNIQA_ENV (the
|
|
543
|
+
# environment, e.g. ${deployEnvironment(a)}; OMNIQA_SHA names a commit other than the branch head) runs it by hand. The
|
|
544
|
+
# three setup steps (variables, the product's hook, a first run): omniqa doctor.
|
|
537
545
|
omniqa deploy:
|
|
538
546
|
image: node:26
|
|
539
547
|
rules:
|
|
540
|
-
- if:
|
|
541
|
-
- if: $CI_PIPELINE_SOURCE == "web" && $OMNIQA_SHA
|
|
548
|
+
- if: $CI_PIPELINE_SOURCE == "trigger" || $CI_PIPELINE_SOURCE == "pipeline"
|
|
549
|
+
- if: $CI_PIPELINE_SOURCE == "web" && ($OMNIQA_ENV || $OMNIQA_SHA)
|
|
542
550
|
before_script:
|
|
543
551
|
- git remote set-url --push origin "https://oauth2:\${GITLAB_TOKEN}@\${CI_SERVER_HOST}/\${CI_PROJECT_PATH}.git"
|
|
544
552
|
- npm install -g omniqa@latest
|
|
545
553
|
- omniqa install --with-deps
|
|
546
554
|
script:
|
|
547
|
-
${place.dir === "" ? "" : ` - cd ${place.dir}\n`} - omniqa sync --to "$OMNIQA_SHA" --env "$OMNIQA_ENV"
|
|
555
|
+
${place.dir === "" ? "" : ` - cd ${place.dir}\n`} - omniqa sync --suite --to "$OMNIQA_SHA" --env "$OMNIQA_ENV"
|
|
548
556
|
artifacts:
|
|
549
557
|
when: on_failure
|
|
550
558
|
paths:
|
|
@@ -557,7 +565,7 @@ ${place.dir === "" ? "" : ` - cd ${place.dir}\n`} - omniqa sync --to "$OMN
|
|
|
557
565
|
pages:
|
|
558
566
|
image: alpine
|
|
559
567
|
rules:
|
|
560
|
-
${a.changelog === undefined ? "" : ` - if: $CI_PIPELINE_SOURCE == "
|
|
568
|
+
${a.changelog === undefined ? "" : ` - if: $CI_PIPELINE_SOURCE == "trigger" || $CI_PIPELINE_SOURCE == "pipeline"\n when: never\n - if: $OMNIQA_ENV || $OMNIQA_SHA\n when: never\n`} - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
|
561
569
|
changes:
|
|
562
570
|
- ${place.dir === "" ? "" : `${place.dir}/`}.omniqa/knowledge/scheme.html
|
|
563
571
|
script:
|
|
@@ -570,7 +578,7 @@ ${a.changelog === undefined ? "" : ` - if: $CI_PIPELINE_SOURCE == "pipeline"
|
|
|
570
578
|
}
|
|
571
579
|
return {};
|
|
572
580
|
}
|
|
573
|
-
export function render(a, place = { root: "", dir: "" }
|
|
581
|
+
export function render(a, place = { root: "", dir: "" }) {
|
|
574
582
|
// An executor or a reviewer left on default gets a commented example: a cheaper executor (the suggested model at effort medium), the suggested reviewer.
|
|
575
583
|
const suggestion = choiceOf(a.provider, a.baseUrl).models;
|
|
576
584
|
const examples = { executor: { model: (suggestion?.default ?? a.models.default).model, effort: "medium" }, ...(suggestion === undefined ? {} : { reviewer: suggestion.reviewer }) };
|
|
@@ -602,9 +610,8 @@ export function render(a, place = { root: "", dir: "" }, qaRepo = "OWNER/REPO")
|
|
|
602
610
|
const deviceToken = runners.length === 0 ? "# AGENT_DEVICE_DAEMON_AUTH_TOKEN=\n" : "AGENT_DEVICE_DAEMON_AUTH_TOKEN=\n";
|
|
603
611
|
const secretNames = secretNamesOf(a);
|
|
604
612
|
const changelogLines = a.changelog === undefined
|
|
605
|
-
? ` // Where the product's changes come from, for omniqa sync (--to <sha> after a deploy, --poll on a schedule, the webhooks of omniqa serve):\n // the repository and branch to follow; init writes it when a repos: entry is a GitHub or GitLab url.\n // changelog: { host: "github", repo: "org/web", branch: "main" }, //
|
|
606
|
-
: ` // Where the product's changes come from, for omniqa sync (--to <sha> after a deploy, --poll on a schedule, the webhooks of omniqa serve).\n changelog: { host: ${JSON.stringify(a.changelog.host)}, repo: ${JSON.stringify(a.changelog.repo)}, branch: "main" }, // or master;
|
|
607
|
-
const notifyLine = ` // notify: { webhookEnv: "SLACK_WEBHOOK_URL" }, // a Slack incoming webhook (Mattermost, Rocket.Chat and Google Chat take the same body): which tests failed after a deploy and what the healer did\n`;
|
|
613
|
+
? ` // Where the product's changes come from, for omniqa sync (--to <sha> after a deploy, --poll on a schedule, the webhooks of omniqa serve):\n // the repository and branch to follow; init writes it when a repos: entry is a GitHub or GitLab url.\n // changelog: { host: "github", repo: "org/web", branch: "main" }, // PRODUCT_REPO_TOKEN reads a private product repository; webhookSecretEnv: "GITHUB_WEBHOOK_SECRET" for omniqa serve\n`
|
|
614
|
+
: ` // Where the product's changes come from, for omniqa sync (--to <sha> after a deploy, --poll on a schedule, the webhooks of omniqa serve).\n changelog: { host: ${JSON.stringify(a.changelog.host)}, repo: ${JSON.stringify(a.changelog.repo)}, branch: "main" }, // or master; PRODUCT_REPO_TOKEN reads a private product repository and SLACK_WEBHOOK_URL turns the notifications on; tokenEnv: and notify: { webhookEnv } rename them\n`;
|
|
608
615
|
return {
|
|
609
616
|
"omniqa.config.ts": `import { defineConfig } from "omniqa";
|
|
610
617
|
|
|
@@ -630,7 +637,7 @@ ${devicesLines} // Heals and generated tests are committed on the branch under
|
|
|
630
637
|
// (and always as a pull request from a production-tier environment):
|
|
631
638
|
// persist: { protectedBranches: ["main", "master"] },
|
|
632
639
|
// gitHosts: { github: { tokenEnv: "GITHUB_TOKEN" } }, // or gitlab: { url: "https://gitlab.example.com", tokenEnv: "GITLAB_TOKEN" }
|
|
633
|
-
${changelogLines}
|
|
640
|
+
${changelogLines} // What the suite covers: agentic end-to-end tests now; deterministic API tests from the endpoint map arrive in V6.
|
|
634
641
|
${scopeLine}});
|
|
635
642
|
`,
|
|
636
643
|
".omniqa/product.yaml": `name: ${JSON.stringify(a.name)}
|
|
@@ -662,9 +669,8 @@ ${accountEnv}# Persistence (gitHosts): a token that can push and open pull reque
|
|
|
662
669
|
# GITHUB_WEBHOOK_SECRET=
|
|
663
670
|
# GITLAB_WEBHOOK_SECRET=
|
|
664
671
|
# A private product repository (changelog.tokenEnv): a token that reads it
|
|
665
|
-
#
|
|
666
|
-
#
|
|
667
|
-
# Notifications (notify.webhookEnv): a Slack incoming webhook url; which tests failed after a deploy and what the healer did
|
|
672
|
+
# PRODUCT_REPO_TOKEN=
|
|
673
|
+
# Notifications after a deploy (notify.webhookEnv): a Slack incoming webhook url
|
|
668
674
|
# SLACK_WEBHOOK_URL=
|
|
669
675
|
# Device runners (devices in omniqa.config.ts): the token an agent-device proxy printed, or a device cloud's credential
|
|
670
676
|
${deviceToken}`,
|
|
@@ -687,7 +693,7 @@ CMD ["omniqa", "serve"]
|
|
|
687
693
|
.env
|
|
688
694
|
node_modules
|
|
689
695
|
`,
|
|
690
|
-
...ciFiles(a, place, secretNames
|
|
696
|
+
...ciFiles(a, place, secretNames),
|
|
691
697
|
};
|
|
692
698
|
}
|
|
693
699
|
async function mergeGitignore(cwd) {
|
|
@@ -737,6 +743,30 @@ async function mergeMcpJson(cwd) {
|
|
|
737
743
|
doc.mcpServers = { ...doc.mcpServers, omniqa: MCP_SERVER };
|
|
738
744
|
await writeFile(file, `${JSON.stringify(doc, null, 2)}\n`);
|
|
739
745
|
}
|
|
746
|
+
/**
|
|
747
|
+
* The three setup steps of the deploy flow, printed under the listing (omniqa doctor prints them again): the QA repository and,
|
|
748
|
+
* on GitLab, its instance come from the remote (OWNER/REPO without one), the branch the GitLab trigger names from git.
|
|
749
|
+
*/
|
|
750
|
+
async function setupSteps(cwd, a, changelog, ci, remote) {
|
|
751
|
+
const qa = remote === undefined ? undefined : parseRemote(remote);
|
|
752
|
+
const s = {
|
|
753
|
+
ci,
|
|
754
|
+
qaBranch: (await git(cwd, ["branch", "--show-current"]).catch(() => "")) || "main",
|
|
755
|
+
product: { host: changelog.host, repo: changelog.repo, branch: "main" },
|
|
756
|
+
deployFrom: a.deployFrom ?? changelog.host,
|
|
757
|
+
environment: deployEnvironment(a),
|
|
758
|
+
keyEnvs: [a.keyEnv],
|
|
759
|
+
otherEnvs: secretNamesOf(a).filter((n) => n !== a.keyEnv),
|
|
760
|
+
tokenEnv: "PRODUCT_REPO_TOKEN",
|
|
761
|
+
webhookEnv: "SLACK_WEBHOOK_URL",
|
|
762
|
+
};
|
|
763
|
+
if (qa !== undefined) {
|
|
764
|
+
s.qaRepo = qa.path;
|
|
765
|
+
if (ci === "gitlab")
|
|
766
|
+
s.qaUrl = `https://${qa.host}`;
|
|
767
|
+
}
|
|
768
|
+
return deploySetup(s);
|
|
769
|
+
}
|
|
740
770
|
const sinks = terminalSinks("omniqa init: ");
|
|
741
771
|
const { out, log, progress } = sinks;
|
|
742
772
|
export async function init(args, cwd) {
|
|
@@ -760,6 +790,8 @@ export async function init(args, cwd) {
|
|
|
760
790
|
"test-id": { type: "string" },
|
|
761
791
|
ci: { type: "string" },
|
|
762
792
|
pages: { type: "boolean" },
|
|
793
|
+
repo: { type: "string", multiple: true },
|
|
794
|
+
"deploy-from": { type: "string" },
|
|
763
795
|
},
|
|
764
796
|
});
|
|
765
797
|
}
|
|
@@ -784,6 +816,8 @@ export async function init(args, cwd) {
|
|
|
784
816
|
await mkdir(cwd, { recursive: true });
|
|
785
817
|
}
|
|
786
818
|
const strings = Object.fromEntries(Object.entries(v).filter(([, x]) => typeof x === "string"));
|
|
819
|
+
// --repo repeats: the one flag that is a list
|
|
820
|
+
const repos = Array.isArray(v.repo) ? v.repo : [];
|
|
787
821
|
const yes = v.yes === true;
|
|
788
822
|
const reader = yes ? undefined : askingReader(lineReader(), sinks);
|
|
789
823
|
const stages = async (from, lines) => {
|
|
@@ -805,7 +839,7 @@ export async function init(args, cwd) {
|
|
|
805
839
|
const remote = top === undefined ? undefined : await remoteUrl(cwd);
|
|
806
840
|
let answers;
|
|
807
841
|
try {
|
|
808
|
-
const defaults = defaultsFrom(cwd, strings, remote);
|
|
842
|
+
const defaults = defaultsFrom(cwd, strings, remote, repos);
|
|
809
843
|
defaults.pages = v.pages === true && defaults.ci !== "none";
|
|
810
844
|
if (reader === undefined && defaults.models.default.model === "")
|
|
811
845
|
throw new Error(`--model is required with --base-url ${defaults.baseUrl}: only OpenAI's and OpenRouter's urls come with suggested models`);
|
|
@@ -815,7 +849,7 @@ export async function init(args, cwd) {
|
|
|
815
849
|
stderr.write(`${err.message}\n`);
|
|
816
850
|
return 1;
|
|
817
851
|
}
|
|
818
|
-
const files = render(answers, place
|
|
852
|
+
const files = render(answers, place);
|
|
819
853
|
if (v.force !== true) {
|
|
820
854
|
const clashes = [];
|
|
821
855
|
for (const rel of Object.keys(files))
|
|
@@ -899,12 +933,16 @@ export async function init(args, cwd) {
|
|
|
899
933
|
}
|
|
900
934
|
await recordOnboarding(cwd, "plan", answers);
|
|
901
935
|
out(`OmniQA project ready for ${answers.name}.\n\n ${listed.join("\n ")}\n`);
|
|
936
|
+
// The three setup steps of the deploy flow, on both paths; --ci none has no deploy job to set up.
|
|
937
|
+
if (answers.changelog !== undefined && answers.ci !== "none")
|
|
938
|
+
out(await setupSteps(cwd, answers, answers.changelog, answers.ci, remote));
|
|
902
939
|
if (reader === undefined) {
|
|
903
940
|
const where = secretsWhere(answers.ci, names);
|
|
904
941
|
const steps = [
|
|
905
942
|
"omniqa doctor # Node, Chrome and the model key",
|
|
906
943
|
`cp .env.example .env and set ${answers.keyEnv}`,
|
|
907
|
-
|
|
944
|
+
// with a changelog the three setup steps above already say where the CI secrets go
|
|
945
|
+
...(where === undefined ? [] : answers.changelog !== undefined ? ["the three setup steps above # the CI secrets, the product's hook, a first run"] : [bold(`the same names${answers.ci === "gitlab" ? " and GITLAB_TOKEN" : ""} under ${where.under} # CI (${where.note})`)]),
|
|
908
946
|
"omniqa test",
|
|
909
947
|
"agent-device doctor # device projects only: native apps on simulators, emulators and devices",
|
|
910
948
|
"omniqa init # the test plan: groundwork, planner and reviewer, your approval, the keys gate",
|