@memberjunction/cli 5.37.0 → 5.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -0
- package/dist/baseline/cli-helpers.d.ts +12 -0
- package/dist/baseline/cli-helpers.d.ts.map +1 -0
- package/dist/baseline/cli-helpers.js +40 -0
- package/dist/baseline/cli-helpers.js.map +1 -0
- package/dist/baseline/comparator.d.ts +26 -0
- package/dist/baseline/comparator.d.ts.map +1 -0
- package/dist/baseline/comparator.js +527 -0
- package/dist/baseline/comparator.js.map +1 -0
- package/dist/baseline/connection.d.ts +39 -0
- package/dist/baseline/connection.d.ts.map +1 -0
- package/dist/baseline/connection.js +104 -0
- package/dist/baseline/connection.js.map +1 -0
- package/dist/baseline/data-dumper.d.ts +23 -0
- package/dist/baseline/data-dumper.d.ts.map +1 -0
- package/dist/baseline/data-dumper.js +94 -0
- package/dist/baseline/data-dumper.js.map +1 -0
- package/dist/baseline/emitter.d.ts +36 -0
- package/dist/baseline/emitter.d.ts.map +1 -0
- package/dist/baseline/emitter.js +512 -0
- package/dist/baseline/emitter.js.map +1 -0
- package/dist/baseline/index.d.ts +18 -0
- package/dist/baseline/index.d.ts.map +1 -0
- package/dist/baseline/index.js +14 -0
- package/dist/baseline/index.js.map +1 -0
- package/dist/baseline/introspector-mssql.d.ts +15 -0
- package/dist/baseline/introspector-mssql.d.ts.map +1 -0
- package/dist/baseline/introspector-mssql.js +673 -0
- package/dist/baseline/introspector-mssql.js.map +1 -0
- package/dist/baseline/introspector-postgres.d.ts +14 -0
- package/dist/baseline/introspector-postgres.d.ts.map +1 -0
- package/dist/baseline/introspector-postgres.js +277 -0
- package/dist/baseline/introspector-postgres.js.map +1 -0
- package/dist/baseline/report.d.ts +7 -0
- package/dist/baseline/report.d.ts.map +1 -0
- package/dist/baseline/report.js +105 -0
- package/dist/baseline/report.js.map +1 -0
- package/dist/baseline/types.d.ts +286 -0
- package/dist/baseline/types.d.ts.map +1 -0
- package/dist/baseline/types.js +10 -0
- package/dist/baseline/types.js.map +1 -0
- package/dist/baseline/util.d.ts +112 -0
- package/dist/baseline/util.d.ts.map +1 -0
- package/dist/baseline/util.js +325 -0
- package/dist/baseline/util.js.map +1 -0
- package/dist/commands/baseline/build.d.ts +29 -0
- package/dist/commands/baseline/build.d.ts.map +1 -0
- package/dist/commands/baseline/build.js +188 -0
- package/dist/commands/baseline/build.js.map +1 -0
- package/dist/commands/baseline/compare.d.ts +19 -0
- package/dist/commands/baseline/compare.d.ts.map +1 -0
- package/dist/commands/baseline/compare.js +133 -0
- package/dist/commands/baseline/compare.js.map +1 -0
- package/dist/commands/baseline/roundtrip.d.ts +23 -0
- package/dist/commands/baseline/roundtrip.d.ts.map +1 -0
- package/dist/commands/baseline/roundtrip.js +245 -0
- package/dist/commands/baseline/roundtrip.js.map +1 -0
- package/dist/commands/test/compare.d.ts +3 -0
- package/dist/commands/test/compare.d.ts.map +1 -1
- package/dist/commands/test/compare.js +23 -3
- package/dist/commands/test/compare.js.map +1 -1
- package/dist/commands/test/regression/build.d.ts +14 -0
- package/dist/commands/test/regression/build.d.ts.map +1 -0
- package/dist/commands/test/regression/build.js +48 -0
- package/dist/commands/test/regression/build.js.map +1 -0
- package/dist/commands/test/regression/compare.d.ts +15 -0
- package/dist/commands/test/regression/compare.d.ts.map +1 -0
- package/dist/commands/test/regression/compare.js +66 -0
- package/dist/commands/test/regression/compare.js.map +1 -0
- package/dist/commands/test/regression/down.d.ts +10 -0
- package/dist/commands/test/regression/down.d.ts.map +1 -0
- package/dist/commands/test/regression/down.js +29 -0
- package/dist/commands/test/regression/down.js.map +1 -0
- package/dist/commands/test/regression/export.d.ts +11 -0
- package/dist/commands/test/regression/export.d.ts.map +1 -0
- package/dist/commands/test/regression/export.js +56 -0
- package/dist/commands/test/regression/export.js.map +1 -0
- package/dist/commands/test/regression/gen-forms.d.ts +7 -0
- package/dist/commands/test/regression/gen-forms.d.ts.map +1 -0
- package/dist/commands/test/regression/gen-forms.js +18 -0
- package/dist/commands/test/regression/gen-forms.js.map +1 -0
- package/dist/commands/test/regression/index.d.ts +7 -0
- package/dist/commands/test/regression/index.d.ts.map +1 -0
- package/dist/commands/test/regression/index.js +33 -0
- package/dist/commands/test/regression/index.js.map +1 -0
- package/dist/commands/test/regression/init.d.ts +21 -0
- package/dist/commands/test/regression/init.d.ts.map +1 -0
- package/dist/commands/test/regression/init.js +111 -0
- package/dist/commands/test/regression/init.js.map +1 -0
- package/dist/commands/test/regression/remote.d.ts +23 -0
- package/dist/commands/test/regression/remote.d.ts.map +1 -0
- package/dist/commands/test/regression/remote.js +140 -0
- package/dist/commands/test/regression/remote.js.map +1 -0
- package/dist/commands/test/regression/up.d.ts +21 -0
- package/dist/commands/test/regression/up.d.ts.map +1 -0
- package/dist/commands/test/regression/up.js +111 -0
- package/dist/commands/test/regression/up.js.map +1 -0
- package/dist/commands/test/run.d.ts +1 -0
- package/dist/commands/test/run.d.ts.map +1 -1
- package/dist/commands/test/run.js +6 -0
- package/dist/commands/test/run.js.map +1 -1
- package/dist/commands/test/suite.d.ts +5 -0
- package/dist/commands/test/suite.d.ts.map +1 -1
- package/dist/commands/test/suite.js +27 -0
- package/dist/commands/test/suite.js.map +1 -1
- package/dist/commands/translate-sql/index.d.ts.map +1 -1
- package/dist/commands/translate-sql/index.js +3 -2
- package/dist/commands/translate-sql/index.js.map +1 -1
- package/dist/init-templates/bring-your-own-app/README.md +74 -0
- package/dist/init-templates/bring-your-own-app/byo-metadata/.mj-sync.json +10 -0
- package/dist/init-templates/bring-your-own-app/byo-metadata/test-suites/.byo-suite.json +51 -0
- package/dist/init-templates/bring-your-own-app/byo-metadata/test-suites/.mj-sync.json +12 -0
- package/dist/init-templates/bring-your-own-app/byo-metadata/tests/.T01-counter-increment.json +63 -0
- package/dist/init-templates/bring-your-own-app/byo-metadata/tests/.T02-contact-form-submit.json +63 -0
- package/dist/init-templates/bring-your-own-app/byo-metadata/tests/.mj-sync.json +12 -0
- package/dist/init-templates/bring-your-own-app/docker-compose.app.yml +76 -0
- package/dist/init-templates/bring-your-own-app/target.json +23 -0
- package/dist/init-templates/generic-web/README.md +115 -0
- package/dist/init-templates/generic-web/metadata/.mj-sync.json +10 -0
- package/dist/init-templates/generic-web/metadata/test-suites/.generic-web-suite.json +48 -0
- package/dist/init-templates/generic-web/metadata/test-suites/.mj-sync.json +12 -0
- package/dist/init-templates/generic-web/metadata/tests/.GW01-home-loads.json +74 -0
- package/dist/init-templates/generic-web/metadata/tests/.GW02-search-array-map.json +82 -0
- package/dist/init-templates/generic-web/metadata/tests/.mj-sync.json +12 -0
- package/dist/init-templates/generic-web/oracles/mdn-oracles.cjs +121 -0
- package/dist/init-templates/generic-web/target.json +25 -0
- package/dist/init-templates/github-actions/README.md +26 -0
- package/dist/init-templates/github-actions/regression.yml +52 -0
- package/dist/init-templates/remote-mj/README.md +95 -0
- package/dist/init-templates/remote-mj/target.json +22 -0
- package/dist/init-templates/static-file-server/Dockerfile +23 -0
- package/dist/init-templates/static-file-server/README.md +88 -0
- package/dist/init-templates/static-file-server/docker-compose.app.yml +44 -0
- package/dist/init-templates/static-file-server/metadata/.mj-sync.json +10 -0
- package/dist/init-templates/static-file-server/metadata/test-suites/.mj-sync.json +12 -0
- package/dist/init-templates/static-file-server/metadata/test-suites/.static-server-suite.json +27 -0
- package/dist/init-templates/static-file-server/metadata/tests/.SS01-home-loads.json +45 -0
- package/dist/init-templates/static-file-server/metadata/tests/.SS02-navigate-about.json +45 -0
- package/dist/init-templates/static-file-server/metadata/tests/.mj-sync.json +12 -0
- package/dist/init-templates/static-file-server/public/about.html +12 -0
- package/dist/init-templates/static-file-server/public/index.html +27 -0
- package/dist/init-templates/static-file-server/target.json +16 -0
- package/dist/lib/regression/docker-helpers.d.ts +98 -0
- package/dist/lib/regression/docker-helpers.d.ts.map +1 -0
- package/dist/lib/regression/docker-helpers.js +184 -0
- package/dist/lib/regression/docker-helpers.js.map +1 -0
- package/oclif.manifest.json +1356 -509
- package/package.json +22 -17
- package/regression-compose/docker-compose.bacpac-standalone.yml +109 -0
- package/regression-compose/docker-compose.standalone.yml +50 -0
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# Mode D overlay template — boots YOUR app on the regression network so the
|
|
2
|
+
# test-runner can reach it at http://byo-app:3000, and points the runner at
|
|
3
|
+
# the BYO suite via env vars.
|
|
4
|
+
#
|
|
5
|
+
# Run from the MJ repo root with the base compose + this overlay:
|
|
6
|
+
#
|
|
7
|
+
# docker compose \
|
|
8
|
+
# -f docker/regression/docker-compose.test.yml \
|
|
9
|
+
# -f <path/to/this/overlay>.yml \
|
|
10
|
+
# --env-file docker/regression/.env.test \
|
|
11
|
+
# --profile full \
|
|
12
|
+
# up --abort-on-container-exit --exit-code-from test-runner
|
|
13
|
+
#
|
|
14
|
+
# Or via the CLI:
|
|
15
|
+
#
|
|
16
|
+
# mj test regression remote \
|
|
17
|
+
# --target=./<your-init-folder>/target.json \
|
|
18
|
+
# --overlay=./<your-init-folder>/docker-compose.app.yml
|
|
19
|
+
#
|
|
20
|
+
# What this overlay does vs. the base compose:
|
|
21
|
+
# - Adds a `byo-app` service (YOUR app under test)
|
|
22
|
+
# - Mounts your BYO tests + suite into the test-runner
|
|
23
|
+
# - Sets TEST_SUITE_NAME so the runner executes your BYO suite (not the MJ suite)
|
|
24
|
+
# - Sets EXTRA_METADATA_DIRS so the runner pushes your tests + suite to the DB
|
|
25
|
+
# - Sets MJ_TEST_VAR_* env vars that flow into {{baseUrl}} / {{allowedDomains}}
|
|
26
|
+
# placeholders in your BYO test JSONs
|
|
27
|
+
|
|
28
|
+
services:
|
|
29
|
+
byo-app:
|
|
30
|
+
# The base compose is profile-gated; the BYO app must join the `full` profile.
|
|
31
|
+
profiles: ["full"]
|
|
32
|
+
# TODO: Replace EITHER `build:` or `image:` with how YOUR app gets into a
|
|
33
|
+
# container. The test-runner reaches this service at http://byo-app:3000.
|
|
34
|
+
#
|
|
35
|
+
# Option A — build from source (compose builds your image at `up` time):
|
|
36
|
+
# build:
|
|
37
|
+
# context: ../path/to/your/app # relative to the FIRST `-f` file (the base compose)
|
|
38
|
+
# dockerfile: Dockerfile
|
|
39
|
+
#
|
|
40
|
+
# Option B — pull an already-built image:
|
|
41
|
+
image: your-app-image:latest
|
|
42
|
+
container_name: mj-regression-byo-app
|
|
43
|
+
environment:
|
|
44
|
+
PORT: "3000"
|
|
45
|
+
NODE_ENV: "production"
|
|
46
|
+
healthcheck:
|
|
47
|
+
# TODO: Replace with a healthcheck that confirms YOUR app is ready.
|
|
48
|
+
# Example for an HTTP service with a /health endpoint:
|
|
49
|
+
test: ["CMD-SHELL", "wget -q --spider http://localhost:3000/health || exit 1"]
|
|
50
|
+
interval: 10s
|
|
51
|
+
timeout: 5s
|
|
52
|
+
retries: 6
|
|
53
|
+
start_period: 15s
|
|
54
|
+
|
|
55
|
+
test-runner:
|
|
56
|
+
depends_on:
|
|
57
|
+
byo-app:
|
|
58
|
+
condition: service_healthy
|
|
59
|
+
environment:
|
|
60
|
+
# Run YOUR suite, not the MJ Explorer suite.
|
|
61
|
+
TEST_SUITE_NAME: "BYO Regression Suite"
|
|
62
|
+
# Push your BYO metadata (tests + suite) to the runner's DB before tests run.
|
|
63
|
+
EXTRA_METADATA_DIRS: "/app/byo-metadata"
|
|
64
|
+
# Drive {{baseUrl}} and {{allowedDomains}} substitution in your test JSONs.
|
|
65
|
+
MJ_TEST_VAR_baseUrl: "http://byo-app:3000"
|
|
66
|
+
MJ_TEST_VAR_allowedDomains: "[\"byo-app\"]"
|
|
67
|
+
# Archive flow (optional): inherit ARCHIVE_DB_* from .env.test and tag your
|
|
68
|
+
# run distinctly so BYO rows are recognizable in the archive DB.
|
|
69
|
+
ARCHIVE_TAG: "byo-app"
|
|
70
|
+
ARCHIVE_SOURCE: "mode-d-byo"
|
|
71
|
+
volumes:
|
|
72
|
+
# NOT read-only: mj-sync writes auto-generated primaryKey + sync state back
|
|
73
|
+
# to your JSON files after first push so subsequent runs are idempotent.
|
|
74
|
+
# Path is relative to the FIRST `-f` file (the base compose); adjust if
|
|
75
|
+
# your scaffold lives somewhere else than `docker/regression/`.
|
|
76
|
+
- ./<your-init-folder>/byo-metadata:/app/byo-metadata
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": "1.0",
|
|
3
|
+
"name": "bring-your-own-app",
|
|
4
|
+
"kind": "generic-web",
|
|
5
|
+
"description": "Mode D scaffold (realistic): targets the BYO Angular+Express demo brought up by docker-compose.app.yml. Used by `mj test regression init bring-your-own-app` to seed a project; see the README in this directory for the full adoption walkthrough.",
|
|
6
|
+
|
|
7
|
+
"baseUrl": "http://byo-app:3000",
|
|
8
|
+
"allowedDomains": ["byo-app"],
|
|
9
|
+
|
|
10
|
+
"auth": { "bindings": [] },
|
|
11
|
+
|
|
12
|
+
"suite": "BYO Regression Suite",
|
|
13
|
+
"extraMetadataDirs": [
|
|
14
|
+
"/app/byo-metadata"
|
|
15
|
+
],
|
|
16
|
+
|
|
17
|
+
"archive": {
|
|
18
|
+
"tag": "byo-loopback",
|
|
19
|
+
"source": "byo-app-demo"
|
|
20
|
+
},
|
|
21
|
+
|
|
22
|
+
"notes": "This app ships with intentional bugs to demonstrate the suite catching real defects (see README). Replace the Dockerfile + Angular source with your own app, edit byo-metadata/tests/ to match its UI flows, and re-run."
|
|
23
|
+
}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
# Generic Web Example — Mode C Reference
|
|
2
|
+
|
|
3
|
+
A working **Mode C** (non-MJ web app) scaffold that drives the MJ regression
|
|
4
|
+
runner against [MDN Web Docs](https://developer.mozilla.org). Three Computer
|
|
5
|
+
Use tests plus one **custom IOracle** plugged in via `--oracles-module`.
|
|
6
|
+
|
|
7
|
+
Use this directory as the starting point when authoring tests for your own web
|
|
8
|
+
app: copy it, edit `target.json` to point at your URL, replace the three test
|
|
9
|
+
JSONs, and tweak (or remove) the custom oracle.
|
|
10
|
+
|
|
11
|
+
## What's in here
|
|
12
|
+
|
|
13
|
+
```
|
|
14
|
+
generic-web/
|
|
15
|
+
├── README.md # this file
|
|
16
|
+
├── target.json # Mode C target profile (kind: generic-web)
|
|
17
|
+
├── metadata/
|
|
18
|
+
│ ├── .mj-sync.json # directoryOrder = [tests, test-suites]
|
|
19
|
+
│ ├── tests/
|
|
20
|
+
│ │ ├── .mj-sync.json # entity = "MJ: Tests"
|
|
21
|
+
│ │ ├── .GW01-home-loads.json # MDN home page smoke
|
|
22
|
+
│ │ ├── .GW02-search-array-map.json # search-flow test
|
|
23
|
+
│ │ └── .GW03-navigate-references.json # top-nav navigation test
|
|
24
|
+
│ └── test-suites/
|
|
25
|
+
│ ├── .mj-sync.json # entity = "MJ: Test Suites"
|
|
26
|
+
│ └── .generic-web-suite.json # binds the 3 tests
|
|
27
|
+
└── oracles/
|
|
28
|
+
└── mdn-oracles.cjs # custom IOracle exports
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## The custom oracle module
|
|
32
|
+
|
|
33
|
+
[`oracles/mdn-oracles.cjs`](oracles/mdn-oracles.cjs) exports two oracles in two
|
|
34
|
+
different styles to show both supported patterns:
|
|
35
|
+
|
|
36
|
+
| Type | Export style | Purpose |
|
|
37
|
+
|---|---|---|
|
|
38
|
+
| `final-url-host` | `class FinalUrlHostOracle implements IOracle` | Verifies the test ended on a specific hostname (catches off-domain wanders). Used by GW02 + GW03. |
|
|
39
|
+
| `min-step-count` | object instance `{ type, evaluate }` | Fails when the agent finished too fast (likely a silent navigation error). Available to any test but not currently referenced — feel free to wire it in. |
|
|
40
|
+
|
|
41
|
+
The loader (`packages/TestingFramework/CLI/src/utils/oracle-module-loader.ts`)
|
|
42
|
+
duck-types each export: if it has `type: string` + async `evaluate()`, it
|
|
43
|
+
gets registered. Anything else is silently skipped.
|
|
44
|
+
|
|
45
|
+
## Running it
|
|
46
|
+
|
|
47
|
+
### Inside Docker (Mode C via `mj test regression remote`)
|
|
48
|
+
|
|
49
|
+
From the MJ monorepo root:
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
mj test regression remote --target=docker/regression/examples/generic-web/target.json
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
What happens:
|
|
56
|
+
|
|
57
|
+
1. The CLI loads [`target.json`](target.json), resolves any `env:` refs (none
|
|
58
|
+
in this example), and exports `MJ_TEST_VAR_baseUrl=https://developer.mozilla.org`,
|
|
59
|
+
`MJ_TEST_VAR_allowedDomains=["developer.mozilla.org","*.mozilla.org"]`,
|
|
60
|
+
`TEST_SUITE_NAME="Generic Web Example Suite"`,
|
|
61
|
+
`EXTRA_METADATA_DIRS=/app/docker/regression/examples/generic-web/metadata`,
|
|
62
|
+
`ORACLES_MODULE=/app/docker/regression/examples/generic-web/oracles/mdn-oracles.cjs`,
|
|
63
|
+
plus the entrypoint dispatch (`TEST_RUNNER_ENTRYPOINT=test-runner-remote-entrypoint.sh`).
|
|
64
|
+
2. Compose brings up the full profile so the test-runner has a local
|
|
65
|
+
ephemeral DB for `TestRun` / `TestRunOutput` rows (the remote URL itself
|
|
66
|
+
isn't MJ — the DB is just for results recording).
|
|
67
|
+
3. The remote entrypoint pushes the example's metadata to the DB, then
|
|
68
|
+
invokes `mj test suite --name "Generic Web Example Suite"
|
|
69
|
+
--oracles-module=/app/.../mdn-oracles.cjs --parallel --max-parallel 4`.
|
|
70
|
+
4. Reports land in `docker/regression/test-results/run-{TIMESTAMP}/`.
|
|
71
|
+
|
|
72
|
+
### Locally (against your dev DB) for fast iteration
|
|
73
|
+
|
|
74
|
+
If you're authoring tests and don't want the Docker round-trip every time,
|
|
75
|
+
push the metadata to your local MJ DB and run the suite directly:
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
# 1. Push the example's test + suite metadata
|
|
79
|
+
npx mj sync push --dir=docker/regression/examples/generic-web/metadata
|
|
80
|
+
|
|
81
|
+
# 2. Run the suite with the custom oracle module + variable substitutions
|
|
82
|
+
mj test suite \
|
|
83
|
+
--name "Generic Web Example Suite" \
|
|
84
|
+
--oracles-module=docker/regression/examples/generic-web/oracles/mdn-oracles.cjs \
|
|
85
|
+
--var baseUrl=https://developer.mozilla.org \
|
|
86
|
+
--var 'allowedDomains=["developer.mozilla.org","*.mozilla.org"]' \
|
|
87
|
+
--parallel --max-parallel 3
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
The `--var` flags drive the `{{baseUrl}}` / `{{allowedDomains}}` substitution
|
|
91
|
+
inside the test JSONs. JSON-shaped values (the array) work because the
|
|
92
|
+
substitution layer auto-parses them.
|
|
93
|
+
|
|
94
|
+
## Copying this example for your own app
|
|
95
|
+
|
|
96
|
+
1. **Copy the directory** to `docker/regression/examples/my-app/` (or anywhere
|
|
97
|
+
outside the monorepo, then mount it).
|
|
98
|
+
2. **Edit `target.json`**:
|
|
99
|
+
- Set `baseUrl` to your app's URL.
|
|
100
|
+
- Set `allowedDomains` to the hosts the agent is allowed to touch.
|
|
101
|
+
- Add an `auth` block if your app requires login (see
|
|
102
|
+
[`docker/regression/targets/staging-mj.example.target.json`](../../../../../docker/regression/targets/staging-mj.example.target.json)
|
|
103
|
+
for the auth shape).
|
|
104
|
+
- Rename `suite` to match your suite, update `extraMetadataDirs` to your
|
|
105
|
+
metadata path, point `oraclesModule` at your oracles file.
|
|
106
|
+
3. **Replace the tests** in `metadata/tests/` with goals specific to your app.
|
|
107
|
+
Reference `{{baseUrl}}` (and any custom vars you add) inside `startUrl`,
|
|
108
|
+
oracle `pattern` configs, etc.
|
|
109
|
+
4. **Edit the suite** in `metadata/test-suites/.generic-web-suite.json` to
|
|
110
|
+
list your tests by name.
|
|
111
|
+
5. **Write your oracles** in `oracles/*.cjs`. The duck-type contract is:
|
|
112
|
+
`{ type: string, async evaluate(input, config) → OracleResult }`.
|
|
113
|
+
|
|
114
|
+
See [`../bring-your-own-app/`](../bring-your-own-app/) for an example that
|
|
115
|
+
also boots its own app inside the regression network (Mode D).
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
{
|
|
2
|
+
"fields": {
|
|
3
|
+
"Name": "Generic Web Example Suite",
|
|
4
|
+
"Description": "Mode C reference suite — runs 3 tests against MDN Web Docs to demonstrate variable templating ({{baseUrl}}, {{allowedDomains}}) and a custom IOracle plugged in via `mj test suite --oracles-module=…`. Used by the docker/regression/examples/generic-web/ scaffold.",
|
|
5
|
+
"Status": "Active"
|
|
6
|
+
},
|
|
7
|
+
"relatedEntities": {
|
|
8
|
+
"MJ: Test Suite Tests": [
|
|
9
|
+
{
|
|
10
|
+
"fields": {
|
|
11
|
+
"SuiteID": "@parent:ID",
|
|
12
|
+
"TestID": "@lookup:MJ: Tests.Name=GW01 - MDN Home Loads",
|
|
13
|
+
"Sequence": 1,
|
|
14
|
+
"Status": "Active"
|
|
15
|
+
},
|
|
16
|
+
"primaryKey": {
|
|
17
|
+
"ID": "EE53002B-DCD5-49AC-AE16-211254BCE6FE"
|
|
18
|
+
},
|
|
19
|
+
"sync": {
|
|
20
|
+
"lastModified": "2026-05-17T23:05:53.394Z",
|
|
21
|
+
"checksum": "09664d1aa1dba6101156b1ad5cb56893aa025fee4b4569ecd1f3e86325617d6f"
|
|
22
|
+
}
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"fields": {
|
|
26
|
+
"SuiteID": "@parent:ID",
|
|
27
|
+
"TestID": "@lookup:MJ: Tests.Name=GW02 - Search Array.prototype.map",
|
|
28
|
+
"Sequence": 2,
|
|
29
|
+
"Status": "Active"
|
|
30
|
+
},
|
|
31
|
+
"primaryKey": {
|
|
32
|
+
"ID": "DB858A9C-BA30-40E1-9FAA-1452A48611F2"
|
|
33
|
+
},
|
|
34
|
+
"sync": {
|
|
35
|
+
"lastModified": "2026-05-17T23:05:53.408Z",
|
|
36
|
+
"checksum": "db7b7b0e7106adcae9bcbf9d1c06c2d4175b70fad2dd6d2282ea44d9d93e3cd4"
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
]
|
|
40
|
+
},
|
|
41
|
+
"primaryKey": {
|
|
42
|
+
"ID": "3C13E3D6-F022-409A-A6FE-B5857BCE7FA4"
|
|
43
|
+
},
|
|
44
|
+
"sync": {
|
|
45
|
+
"lastModified": "2026-05-17T23:05:53.359Z",
|
|
46
|
+
"checksum": "b6ac99fec8087fb090387e0869228705c4ed7e36dcae2ef667e729ede0101bba"
|
|
47
|
+
}
|
|
48
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
{
|
|
2
|
+
"fields": {
|
|
3
|
+
"TypeID": "@lookup:MJ: Test Types.Name=Computer Use",
|
|
4
|
+
"Name": "GW01 - MDN Home Loads",
|
|
5
|
+
"Description": "Navigates to the MDN Web Docs home page and verifies the landing page renders with its main navigation. Demonstrates Mode C variable templating ({{baseUrl}}) — no MJ assumptions, no auth.",
|
|
6
|
+
"InputDefinition": {
|
|
7
|
+
"goal": "Verify the home page has loaded successfully. Look for the main navigation header (References, Guides, Plus) and the prominent 'MDN Web Docs' branding. Do NOT click anything — just confirm the page rendered without errors and the nav elements are visible.",
|
|
8
|
+
"startUrl": "{{baseUrl}}",
|
|
9
|
+
"allowedDomains": "{{allowedDomains}}",
|
|
10
|
+
"auth": {
|
|
11
|
+
"bindings": []
|
|
12
|
+
}
|
|
13
|
+
},
|
|
14
|
+
"ExpectedOutcomes": {
|
|
15
|
+
"goalCompleted": true,
|
|
16
|
+
"finalUrlPattern": "^https?://(www\\.)?developer\\.mozilla\\.org/?",
|
|
17
|
+
"minConfidence": 0.7,
|
|
18
|
+
"maxSteps": 10,
|
|
19
|
+
"judgeValidationCriteria": [
|
|
20
|
+
"Page rendered without errors",
|
|
21
|
+
"Main navigation visible (HTML, CSS, Learn, Tools, etc)",
|
|
22
|
+
"MDN branding is present"
|
|
23
|
+
]
|
|
24
|
+
},
|
|
25
|
+
"Configuration": {
|
|
26
|
+
"headless": true,
|
|
27
|
+
"maxSteps": 10,
|
|
28
|
+
"maxExecutionTime": 120000,
|
|
29
|
+
"screenshotHistoryDepth": 3,
|
|
30
|
+
"viewportWidth": 1280,
|
|
31
|
+
"viewportHeight": 720,
|
|
32
|
+
"controllerPromptName": "Computer Use - Controller",
|
|
33
|
+
"judgePromptName": "Computer Use - Judge",
|
|
34
|
+
"judgeFrequency": "EveryNSteps:3",
|
|
35
|
+
"oracles": [
|
|
36
|
+
{
|
|
37
|
+
"type": "goal-completion",
|
|
38
|
+
"weight": 0.5,
|
|
39
|
+
"config": {
|
|
40
|
+
"minConfidence": 0.7
|
|
41
|
+
}
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
"type": "url-match",
|
|
45
|
+
"weight": 0.3,
|
|
46
|
+
"config": {
|
|
47
|
+
"pattern": "^https?://(www\\.)?developer\\.mozilla\\.org/?"
|
|
48
|
+
}
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
"type": "step-count",
|
|
52
|
+
"weight": 0.2,
|
|
53
|
+
"config": {
|
|
54
|
+
"maxSteps": 10
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
],
|
|
58
|
+
"scoringWeights": {
|
|
59
|
+
"goal-completion": 0.5,
|
|
60
|
+
"url-match": 0.3,
|
|
61
|
+
"step-count": 0.2
|
|
62
|
+
}
|
|
63
|
+
},
|
|
64
|
+
"Status": "Active",
|
|
65
|
+
"Tags": "P0,generic-web,smoke"
|
|
66
|
+
},
|
|
67
|
+
"primaryKey": {
|
|
68
|
+
"ID": "C21C91DF-8EC8-4741-9251-EB14187A2532"
|
|
69
|
+
},
|
|
70
|
+
"sync": {
|
|
71
|
+
"lastModified": "2026-05-17T23:05:53.299Z",
|
|
72
|
+
"checksum": "cd52d0df6d51b2d14f5559cd9c2bb8894b29f7343733673fd618687db8506876"
|
|
73
|
+
}
|
|
74
|
+
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
{
|
|
2
|
+
"fields": {
|
|
3
|
+
"TypeID": "@lookup:MJ: Test Types.Name=Computer Use",
|
|
4
|
+
"Name": " GW02 - Search Array.prototype.map",
|
|
5
|
+
"Description": "Uses the search box to find documentation for Array.prototype.map and verifies a relevant result page loads. Exercises text input + click + page navigation against a public site.",
|
|
6
|
+
"InputDefinition": {
|
|
7
|
+
"goal": "Open the search box (top-right magnifying-glass icon or 'Search MDN' input). Type 'Array.prototype.map' and submit. Follow the most relevant result. Verify a page about Array.prototype.map (the JavaScript method) loads",
|
|
8
|
+
"startUrl": "{{baseUrl}}",
|
|
9
|
+
"allowedDomains": "{{allowedDomains}}",
|
|
10
|
+
"auth": {
|
|
11
|
+
"bindings": []
|
|
12
|
+
}
|
|
13
|
+
},
|
|
14
|
+
"ExpectedOutcomes": {
|
|
15
|
+
"goalCompleted": true,
|
|
16
|
+
"finalUrlPattern": "(?i)/array(/prototype)?/map",
|
|
17
|
+
"minConfidence": 0.7,
|
|
18
|
+
"maxSteps": 20,
|
|
19
|
+
"judgeValidationCriteria": [
|
|
20
|
+
"Final page is about Array.prototype.map",
|
|
21
|
+
"Page contains a code example",
|
|
22
|
+
"Page is a reference doc (not an unrelated search result)"
|
|
23
|
+
]
|
|
24
|
+
},
|
|
25
|
+
"Configuration": {
|
|
26
|
+
"headless": true,
|
|
27
|
+
"maxSteps": 20,
|
|
28
|
+
"maxExecutionTime": 240000,
|
|
29
|
+
"screenshotHistoryDepth": 3,
|
|
30
|
+
"viewportWidth": 1280,
|
|
31
|
+
"viewportHeight": 720,
|
|
32
|
+
"controllerPromptName": "Computer Use - Controller",
|
|
33
|
+
"judgePromptName": "Computer Use - Judge",
|
|
34
|
+
"judgeFrequency": "EveryNSteps:3",
|
|
35
|
+
"oracles": [
|
|
36
|
+
{
|
|
37
|
+
"type": "goal-completion",
|
|
38
|
+
"weight": 0.4,
|
|
39
|
+
"config": {
|
|
40
|
+
"minConfidence": 0.7
|
|
41
|
+
}
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
"type": "url-match",
|
|
45
|
+
"weight": 0.3,
|
|
46
|
+
"config": {
|
|
47
|
+
"pattern": "(?i)/array(/prototype)?/map"
|
|
48
|
+
}
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
"type": "step-count",
|
|
52
|
+
"weight": 0.1,
|
|
53
|
+
"config": {
|
|
54
|
+
"maxSteps": 20
|
|
55
|
+
}
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
"type": "final-url-host",
|
|
59
|
+
"weight": 0.2,
|
|
60
|
+
"config": {
|
|
61
|
+
"host": "developer.mozilla.org"
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
],
|
|
65
|
+
"scoringWeights": {
|
|
66
|
+
"goal-completion": 0.4,
|
|
67
|
+
"url-match": 0.3,
|
|
68
|
+
"step-count": 0.1,
|
|
69
|
+
"final-url-host": 0.2
|
|
70
|
+
}
|
|
71
|
+
},
|
|
72
|
+
"Status": "Active",
|
|
73
|
+
"Tags": "P1,generic-web,search"
|
|
74
|
+
},
|
|
75
|
+
"primaryKey": {
|
|
76
|
+
"ID": "2E8DE64D-B817-4E3D-B471-FF1142FABFDC"
|
|
77
|
+
},
|
|
78
|
+
"sync": {
|
|
79
|
+
"lastModified": "2026-05-17T23:05:53.319Z",
|
|
80
|
+
"checksum": "eeec8bd96e8ff50437a3a20c2a821dc991fcece9277c23510b635e5b6501b88b"
|
|
81
|
+
}
|
|
82
|
+
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Example custom-oracle module for the generic-web example.
|
|
3
|
+
*
|
|
4
|
+
* Loaded at suite startup via:
|
|
5
|
+
* mj test suite --oracles-module=./oracles/mdn-oracles.cjs
|
|
6
|
+
*
|
|
7
|
+
* Each export that looks like an `IOracle` (has `type: string` + async
|
|
8
|
+
* `evaluate(input, config)`) gets registered on the TestEngine and becomes
|
|
9
|
+
* available to any test that references its `type` in `Configuration.oracles`.
|
|
10
|
+
*
|
|
11
|
+
* Supported export shapes (see packages/TestingFramework/CLI/src/utils/
|
|
12
|
+
* oracle-module-loader.ts for the duck-type check):
|
|
13
|
+
* - A class implementing IOracle (instantiated with no args)
|
|
14
|
+
* - An object instance with `{ type: '...', async evaluate() { } }`
|
|
15
|
+
*
|
|
16
|
+
* This module demonstrates both shapes so adopters can copy whichever style
|
|
17
|
+
* they prefer.
|
|
18
|
+
*/
|
|
19
|
+
'use strict';
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* FinalUrlHostOracle — class-export style.
|
|
23
|
+
*
|
|
24
|
+
* Validates that the test's final URL is on a specific host. Useful as a
|
|
25
|
+
* defensive check: if the agent wandered off-domain (e.g., chased an external
|
|
26
|
+
* link), the URL hostname won't match and this oracle fails the test.
|
|
27
|
+
*
|
|
28
|
+
* Config:
|
|
29
|
+
* - host: string — the expected hostname (e.g., "developer.mozilla.org").
|
|
30
|
+
* - allowSubdomains: boolean (default true) — accept "*.host" matches too.
|
|
31
|
+
*/
|
|
32
|
+
class FinalUrlHostOracle {
|
|
33
|
+
constructor() {
|
|
34
|
+
this.type = 'final-url-host';
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
async evaluate(input, config) {
|
|
38
|
+
const actual = (input && input.actualOutput) || {};
|
|
39
|
+
const finalUrl = actual.finalUrl || '';
|
|
40
|
+
const expectedHost = (config && config.host) || '';
|
|
41
|
+
const allowSubdomains = config && config.allowSubdomains !== false;
|
|
42
|
+
|
|
43
|
+
if (!expectedHost) {
|
|
44
|
+
return {
|
|
45
|
+
oracleType: this.type,
|
|
46
|
+
passed: false,
|
|
47
|
+
score: 0,
|
|
48
|
+
message: 'No host configured for final-url-host oracle',
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
if (!finalUrl) {
|
|
52
|
+
return {
|
|
53
|
+
oracleType: this.type,
|
|
54
|
+
passed: false,
|
|
55
|
+
score: 0,
|
|
56
|
+
message: 'No final URL recorded for the test',
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
let actualHost;
|
|
61
|
+
try {
|
|
62
|
+
actualHost = new URL(finalUrl).hostname.toLowerCase();
|
|
63
|
+
} catch (err) {
|
|
64
|
+
return {
|
|
65
|
+
oracleType: this.type,
|
|
66
|
+
passed: false,
|
|
67
|
+
score: 0,
|
|
68
|
+
message: `Final URL is not parseable: ${finalUrl}`,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const expected = expectedHost.toLowerCase();
|
|
73
|
+
const matches = allowSubdomains
|
|
74
|
+
? actualHost === expected || actualHost.endsWith('.' + expected)
|
|
75
|
+
: actualHost === expected;
|
|
76
|
+
|
|
77
|
+
return {
|
|
78
|
+
oracleType: this.type,
|
|
79
|
+
passed: matches,
|
|
80
|
+
score: matches ? 1.0 : 0,
|
|
81
|
+
message: matches
|
|
82
|
+
? `Final URL host matches: ${actualHost}`
|
|
83
|
+
: `Final URL host mismatch — expected ${expected}, got ${actualHost}`,
|
|
84
|
+
details: { actualHost, expectedHost: expected, finalUrl },
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* minStepCountOracle — instance-export style.
|
|
91
|
+
*
|
|
92
|
+
* Inverse of the built-in step-count oracle: fails when the agent finished too
|
|
93
|
+
* fast (e.g., immediately hit an error page and gave up). Useful for catching
|
|
94
|
+
* silent navigation failures.
|
|
95
|
+
*
|
|
96
|
+
* Config:
|
|
97
|
+
* - minSteps: number — minimum acceptable step count (default 2).
|
|
98
|
+
*/
|
|
99
|
+
const minStepCountOracle = {
|
|
100
|
+
type: 'min-step-count',
|
|
101
|
+
async evaluate(input, config) {
|
|
102
|
+
const totalSteps =
|
|
103
|
+
(input && input.actualOutput && input.actualOutput.totalSteps) || 0;
|
|
104
|
+
const minSteps = (config && typeof config.minSteps === 'number') ? config.minSteps : 2;
|
|
105
|
+
const passed = totalSteps >= minSteps;
|
|
106
|
+
return {
|
|
107
|
+
oracleType: this.type,
|
|
108
|
+
passed,
|
|
109
|
+
score: passed ? 1.0 : 0,
|
|
110
|
+
message: passed
|
|
111
|
+
? `Test took ${totalSteps} step(s) (≥ ${minSteps})`
|
|
112
|
+
: `Test finished too quickly: ${totalSteps} step(s) < min ${minSteps}`,
|
|
113
|
+
details: { totalSteps, minSteps },
|
|
114
|
+
};
|
|
115
|
+
},
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
module.exports = {
|
|
119
|
+
FinalUrlHostOracle,
|
|
120
|
+
minStepCountOracle,
|
|
121
|
+
};
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": "1.0",
|
|
3
|
+
"name": "generic-web-mdn",
|
|
4
|
+
"kind": "generic-web",
|
|
5
|
+
"description": "Mode C reference target: drives the MJ regression runner against MDN Web Docs (a stable public site) with 3 example tests and one custom IOracle. Used to verify the generic-web flow end-to-end.",
|
|
6
|
+
|
|
7
|
+
"baseUrl": "https://developer.mozilla.org",
|
|
8
|
+
"allowedDomains": ["developer.mozilla.org", "*.mozilla.org"],
|
|
9
|
+
|
|
10
|
+
"auth": {
|
|
11
|
+
"bindings": []
|
|
12
|
+
},
|
|
13
|
+
|
|
14
|
+
"suite": "Generic Web Example Suite",
|
|
15
|
+
"extraMetadataDirs": [
|
|
16
|
+
"/app/docker/regression/examples/generic-web/metadata"
|
|
17
|
+
],
|
|
18
|
+
|
|
19
|
+
"oraclesModule": "/app/docker/regression/examples/generic-web/oracles/mdn-oracles.cjs",
|
|
20
|
+
|
|
21
|
+
"archive": {
|
|
22
|
+
"tag": "generic-web-mdn",
|
|
23
|
+
"source": "mode-c-mdn"
|
|
24
|
+
}
|
|
25
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# GitHub Actions — run the regression suite from your own repo
|
|
2
|
+
|
|
3
|
+
[`regression.yml`](regression.yml) is a copy-paste workflow that runs the suite
|
|
4
|
+
against your MJ instance using the published `memberjunction/agentic-test-runner`
|
|
5
|
+
image — **no MJ monorepo checkout**.
|
|
6
|
+
|
|
7
|
+
## Use it
|
|
8
|
+
|
|
9
|
+
1. Copy `regression.yml` to `.github/workflows/regression.yml` in your repo.
|
|
10
|
+
2. Add your suite next to it: a `my-suite/target.json` (baseUrl + auth `env:`
|
|
11
|
+
refs + suite name) and `my-suite/metadata/` (tests + test-suites). Scaffold a
|
|
12
|
+
starting point with `mj test regression init remote-mj`.
|
|
13
|
+
3. Add repo **secrets** (names match the root `.env` convention):
|
|
14
|
+
- `AI_VENDOR_API_KEY__GeminiLLM` (primary model) — and/or `AI_VENDOR_API_KEY__AnthropicLLM`
|
|
15
|
+
- `DB_HOST` / `DB_DATABASE` / `DB_USERNAME` / `DB_PASSWORD` — a reachable MJ
|
|
16
|
+
provider DB (your instance under test, or a throwaway) where test
|
|
17
|
+
definitions/prompts are pushed and results recorded.
|
|
18
|
+
- any auth vars your `target.json` references (e.g. `STAGING_TEST_USER`).
|
|
19
|
+
4. Pin `IMAGE` to the MJ version your instance runs.
|
|
20
|
+
|
|
21
|
+
Results (incl. `report.html` + screenshots) upload as the `regression-results`
|
|
22
|
+
artifact. Diff locally with `mj test regression compare`.
|
|
23
|
+
|
|
24
|
+
> This is Mode B/C (drive a URL). For Mode D (boot your app in CI alongside the
|
|
25
|
+
> runner) add your app via `docker compose` and `mj test regression remote
|
|
26
|
+
> --overlay`, or a services: block. See [REGRESSION_TESTING.md](../../../../../docker/regression/REGRESSION_TESTING.md).
|