@memberjunction/cli 5.37.0 → 5.38.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/README.md +74 -0
  2. package/dist/baseline/cli-helpers.d.ts +12 -0
  3. package/dist/baseline/cli-helpers.d.ts.map +1 -0
  4. package/dist/baseline/cli-helpers.js +40 -0
  5. package/dist/baseline/cli-helpers.js.map +1 -0
  6. package/dist/baseline/comparator.d.ts +26 -0
  7. package/dist/baseline/comparator.d.ts.map +1 -0
  8. package/dist/baseline/comparator.js +527 -0
  9. package/dist/baseline/comparator.js.map +1 -0
  10. package/dist/baseline/connection.d.ts +39 -0
  11. package/dist/baseline/connection.d.ts.map +1 -0
  12. package/dist/baseline/connection.js +104 -0
  13. package/dist/baseline/connection.js.map +1 -0
  14. package/dist/baseline/data-dumper.d.ts +23 -0
  15. package/dist/baseline/data-dumper.d.ts.map +1 -0
  16. package/dist/baseline/data-dumper.js +94 -0
  17. package/dist/baseline/data-dumper.js.map +1 -0
  18. package/dist/baseline/emitter.d.ts +36 -0
  19. package/dist/baseline/emitter.d.ts.map +1 -0
  20. package/dist/baseline/emitter.js +512 -0
  21. package/dist/baseline/emitter.js.map +1 -0
  22. package/dist/baseline/index.d.ts +18 -0
  23. package/dist/baseline/index.d.ts.map +1 -0
  24. package/dist/baseline/index.js +14 -0
  25. package/dist/baseline/index.js.map +1 -0
  26. package/dist/baseline/introspector-mssql.d.ts +15 -0
  27. package/dist/baseline/introspector-mssql.d.ts.map +1 -0
  28. package/dist/baseline/introspector-mssql.js +673 -0
  29. package/dist/baseline/introspector-mssql.js.map +1 -0
  30. package/dist/baseline/introspector-postgres.d.ts +14 -0
  31. package/dist/baseline/introspector-postgres.d.ts.map +1 -0
  32. package/dist/baseline/introspector-postgres.js +277 -0
  33. package/dist/baseline/introspector-postgres.js.map +1 -0
  34. package/dist/baseline/report.d.ts +7 -0
  35. package/dist/baseline/report.d.ts.map +1 -0
  36. package/dist/baseline/report.js +105 -0
  37. package/dist/baseline/report.js.map +1 -0
  38. package/dist/baseline/types.d.ts +286 -0
  39. package/dist/baseline/types.d.ts.map +1 -0
  40. package/dist/baseline/types.js +10 -0
  41. package/dist/baseline/types.js.map +1 -0
  42. package/dist/baseline/util.d.ts +112 -0
  43. package/dist/baseline/util.d.ts.map +1 -0
  44. package/dist/baseline/util.js +325 -0
  45. package/dist/baseline/util.js.map +1 -0
  46. package/dist/commands/baseline/build.d.ts +29 -0
  47. package/dist/commands/baseline/build.d.ts.map +1 -0
  48. package/dist/commands/baseline/build.js +188 -0
  49. package/dist/commands/baseline/build.js.map +1 -0
  50. package/dist/commands/baseline/compare.d.ts +19 -0
  51. package/dist/commands/baseline/compare.d.ts.map +1 -0
  52. package/dist/commands/baseline/compare.js +133 -0
  53. package/dist/commands/baseline/compare.js.map +1 -0
  54. package/dist/commands/baseline/roundtrip.d.ts +23 -0
  55. package/dist/commands/baseline/roundtrip.d.ts.map +1 -0
  56. package/dist/commands/baseline/roundtrip.js +245 -0
  57. package/dist/commands/baseline/roundtrip.js.map +1 -0
  58. package/dist/commands/test/compare.d.ts +3 -0
  59. package/dist/commands/test/compare.d.ts.map +1 -1
  60. package/dist/commands/test/compare.js +23 -3
  61. package/dist/commands/test/compare.js.map +1 -1
  62. package/dist/commands/test/regression/build.d.ts +14 -0
  63. package/dist/commands/test/regression/build.d.ts.map +1 -0
  64. package/dist/commands/test/regression/build.js +48 -0
  65. package/dist/commands/test/regression/build.js.map +1 -0
  66. package/dist/commands/test/regression/compare.d.ts +15 -0
  67. package/dist/commands/test/regression/compare.d.ts.map +1 -0
  68. package/dist/commands/test/regression/compare.js +66 -0
  69. package/dist/commands/test/regression/compare.js.map +1 -0
  70. package/dist/commands/test/regression/down.d.ts +10 -0
  71. package/dist/commands/test/regression/down.d.ts.map +1 -0
  72. package/dist/commands/test/regression/down.js +29 -0
  73. package/dist/commands/test/regression/down.js.map +1 -0
  74. package/dist/commands/test/regression/export.d.ts +11 -0
  75. package/dist/commands/test/regression/export.d.ts.map +1 -0
  76. package/dist/commands/test/regression/export.js +56 -0
  77. package/dist/commands/test/regression/export.js.map +1 -0
  78. package/dist/commands/test/regression/gen-forms.d.ts +7 -0
  79. package/dist/commands/test/regression/gen-forms.d.ts.map +1 -0
  80. package/dist/commands/test/regression/gen-forms.js +18 -0
  81. package/dist/commands/test/regression/gen-forms.js.map +1 -0
  82. package/dist/commands/test/regression/index.d.ts +7 -0
  83. package/dist/commands/test/regression/index.d.ts.map +1 -0
  84. package/dist/commands/test/regression/index.js +33 -0
  85. package/dist/commands/test/regression/index.js.map +1 -0
  86. package/dist/commands/test/regression/init.d.ts +21 -0
  87. package/dist/commands/test/regression/init.d.ts.map +1 -0
  88. package/dist/commands/test/regression/init.js +111 -0
  89. package/dist/commands/test/regression/init.js.map +1 -0
  90. package/dist/commands/test/regression/remote.d.ts +23 -0
  91. package/dist/commands/test/regression/remote.d.ts.map +1 -0
  92. package/dist/commands/test/regression/remote.js +140 -0
  93. package/dist/commands/test/regression/remote.js.map +1 -0
  94. package/dist/commands/test/regression/up.d.ts +21 -0
  95. package/dist/commands/test/regression/up.d.ts.map +1 -0
  96. package/dist/commands/test/regression/up.js +111 -0
  97. package/dist/commands/test/regression/up.js.map +1 -0
  98. package/dist/commands/test/run.d.ts +1 -0
  99. package/dist/commands/test/run.d.ts.map +1 -1
  100. package/dist/commands/test/run.js +6 -0
  101. package/dist/commands/test/run.js.map +1 -1
  102. package/dist/commands/test/suite.d.ts +5 -0
  103. package/dist/commands/test/suite.d.ts.map +1 -1
  104. package/dist/commands/test/suite.js +27 -0
  105. package/dist/commands/test/suite.js.map +1 -1
  106. package/dist/commands/translate-sql/index.d.ts.map +1 -1
  107. package/dist/commands/translate-sql/index.js +3 -2
  108. package/dist/commands/translate-sql/index.js.map +1 -1
  109. package/dist/init-templates/bring-your-own-app/README.md +74 -0
  110. package/dist/init-templates/bring-your-own-app/byo-metadata/.mj-sync.json +10 -0
  111. package/dist/init-templates/bring-your-own-app/byo-metadata/test-suites/.byo-suite.json +51 -0
  112. package/dist/init-templates/bring-your-own-app/byo-metadata/test-suites/.mj-sync.json +12 -0
  113. package/dist/init-templates/bring-your-own-app/byo-metadata/tests/.T01-counter-increment.json +63 -0
  114. package/dist/init-templates/bring-your-own-app/byo-metadata/tests/.T02-contact-form-submit.json +63 -0
  115. package/dist/init-templates/bring-your-own-app/byo-metadata/tests/.mj-sync.json +12 -0
  116. package/dist/init-templates/bring-your-own-app/docker-compose.app.yml +76 -0
  117. package/dist/init-templates/bring-your-own-app/target.json +23 -0
  118. package/dist/init-templates/generic-web/README.md +115 -0
  119. package/dist/init-templates/generic-web/metadata/.mj-sync.json +10 -0
  120. package/dist/init-templates/generic-web/metadata/test-suites/.generic-web-suite.json +48 -0
  121. package/dist/init-templates/generic-web/metadata/test-suites/.mj-sync.json +12 -0
  122. package/dist/init-templates/generic-web/metadata/tests/.GW01-home-loads.json +74 -0
  123. package/dist/init-templates/generic-web/metadata/tests/.GW02-search-array-map.json +82 -0
  124. package/dist/init-templates/generic-web/metadata/tests/.mj-sync.json +12 -0
  125. package/dist/init-templates/generic-web/oracles/mdn-oracles.cjs +121 -0
  126. package/dist/init-templates/generic-web/target.json +25 -0
  127. package/dist/init-templates/github-actions/README.md +26 -0
  128. package/dist/init-templates/github-actions/regression.yml +52 -0
  129. package/dist/init-templates/remote-mj/README.md +95 -0
  130. package/dist/init-templates/remote-mj/target.json +22 -0
  131. package/dist/init-templates/static-file-server/Dockerfile +23 -0
  132. package/dist/init-templates/static-file-server/README.md +88 -0
  133. package/dist/init-templates/static-file-server/docker-compose.app.yml +44 -0
  134. package/dist/init-templates/static-file-server/metadata/.mj-sync.json +10 -0
  135. package/dist/init-templates/static-file-server/metadata/test-suites/.mj-sync.json +12 -0
  136. package/dist/init-templates/static-file-server/metadata/test-suites/.static-server-suite.json +27 -0
  137. package/dist/init-templates/static-file-server/metadata/tests/.SS01-home-loads.json +45 -0
  138. package/dist/init-templates/static-file-server/metadata/tests/.SS02-navigate-about.json +45 -0
  139. package/dist/init-templates/static-file-server/metadata/tests/.mj-sync.json +12 -0
  140. package/dist/init-templates/static-file-server/public/about.html +12 -0
  141. package/dist/init-templates/static-file-server/public/index.html +27 -0
  142. package/dist/init-templates/static-file-server/target.json +16 -0
  143. package/dist/lib/regression/docker-helpers.d.ts +98 -0
  144. package/dist/lib/regression/docker-helpers.d.ts.map +1 -0
  145. package/dist/lib/regression/docker-helpers.js +184 -0
  146. package/dist/lib/regression/docker-helpers.js.map +1 -0
  147. package/oclif.manifest.json +1356 -509
  148. package/package.json +22 -17
  149. package/regression-compose/docker-compose.bacpac-standalone.yml +109 -0
  150. package/regression-compose/docker-compose.standalone.yml +50 -0
@@ -0,0 +1,76 @@
1
+ # Mode D overlay template — boots YOUR app on the regression network so the
2
+ # test-runner can reach it at http://byo-app:3000, and points the runner at
3
+ # the BYO suite via env vars.
4
+ #
5
+ # Run from the MJ repo root with the base compose + this overlay:
6
+ #
7
+ # docker compose \
8
+ # -f docker/regression/docker-compose.test.yml \
9
+ # -f <path/to/this/overlay>.yml \
10
+ # --env-file docker/regression/.env.test \
11
+ # --profile full \
12
+ # up --abort-on-container-exit --exit-code-from test-runner
13
+ #
14
+ # Or via the CLI:
15
+ #
16
+ # mj test regression remote \
17
+ # --target=./<your-init-folder>/target.json \
18
+ # --overlay=./<your-init-folder>/docker-compose.app.yml
19
+ #
20
+ # What this overlay does vs. the base compose:
21
+ # - Adds a `byo-app` service (YOUR app under test)
22
+ # - Mounts your BYO tests + suite into the test-runner
23
+ # - Sets TEST_SUITE_NAME so the runner executes your BYO suite (not the MJ suite)
24
+ # - Sets EXTRA_METADATA_DIRS so the runner pushes your tests + suite to the DB
25
+ # - Sets MJ_TEST_VAR_* env vars that flow into {{baseUrl}} / {{allowedDomains}}
26
+ # placeholders in your BYO test JSONs
27
+
28
+ services:
29
+ byo-app:
30
+ # The base compose is profile-gated; the BYO app must join the `full` profile.
31
+ profiles: ["full"]
32
+ # TODO: Replace EITHER `build:` or `image:` with how YOUR app gets into a
33
+ # container. The test-runner reaches this service at http://byo-app:3000.
34
+ #
35
+ # Option A — build from source (compose builds your image at `up` time):
36
+ # build:
37
+ # context: ../path/to/your/app # relative to the FIRST `-f` file (the base compose)
38
+ # dockerfile: Dockerfile
39
+ #
40
+ # Option B — pull an already-built image:
41
+ image: your-app-image:latest
42
+ container_name: mj-regression-byo-app
43
+ environment:
44
+ PORT: "3000"
45
+ NODE_ENV: "production"
46
+ healthcheck:
47
+ # TODO: Replace with a healthcheck that confirms YOUR app is ready.
48
+ # Example for an HTTP service with a /health endpoint:
49
+ test: ["CMD-SHELL", "wget -q --spider http://localhost:3000/health || exit 1"]
50
+ interval: 10s
51
+ timeout: 5s
52
+ retries: 6
53
+ start_period: 15s
54
+
55
+ test-runner:
56
+ depends_on:
57
+ byo-app:
58
+ condition: service_healthy
59
+ environment:
60
+ # Run YOUR suite, not the MJ Explorer suite.
61
+ TEST_SUITE_NAME: "BYO Regression Suite"
62
+ # Push your BYO metadata (tests + suite) to the runner's DB before tests run.
63
+ EXTRA_METADATA_DIRS: "/app/byo-metadata"
64
+ # Drive {{baseUrl}} and {{allowedDomains}} substitution in your test JSONs.
65
+ MJ_TEST_VAR_baseUrl: "http://byo-app:3000"
66
+ MJ_TEST_VAR_allowedDomains: "[\"byo-app\"]"
67
+ # Archive flow (optional): inherit ARCHIVE_DB_* from .env.test and tag your
68
+ # run distinctly so BYO rows are recognizable in the archive DB.
69
+ ARCHIVE_TAG: "byo-app"
70
+ ARCHIVE_SOURCE: "mode-d-byo"
71
+ volumes:
72
+ # NOT read-only: mj-sync writes auto-generated primaryKey + sync state back
73
+ # to your JSON files after first push so subsequent runs are idempotent.
74
+ # Path is relative to the FIRST `-f` file (the base compose); adjust if
75
+ # your scaffold lives somewhere else than `docker/regression/`.
76
+ - ./<your-init-folder>/byo-metadata:/app/byo-metadata
@@ -0,0 +1,23 @@
1
+ {
2
+ "schemaVersion": "1.0",
3
+ "name": "bring-your-own-app",
4
+ "kind": "generic-web",
5
+ "description": "Mode D scaffold (realistic): targets the BYO Angular+Express demo brought up by docker-compose.app.yml. Used by `mj test regression init bring-your-own-app` to seed a project; see the README in this directory for the full adoption walkthrough.",
6
+
7
+ "baseUrl": "http://byo-app:3000",
8
+ "allowedDomains": ["byo-app"],
9
+
10
+ "auth": { "bindings": [] },
11
+
12
+ "suite": "BYO Regression Suite",
13
+ "extraMetadataDirs": [
14
+ "/app/byo-metadata"
15
+ ],
16
+
17
+ "archive": {
18
+ "tag": "byo-loopback",
19
+ "source": "byo-app-demo"
20
+ },
21
+
22
+ "notes": "This app ships with intentional bugs to demonstrate the suite catching real defects (see README). Replace the Dockerfile + Angular source with your own app, edit byo-metadata/tests/ to match its UI flows, and re-run."
23
+ }
@@ -0,0 +1,115 @@
1
+ # Generic Web Example — Mode C Reference
2
+
3
+ A working **Mode C** (non-MJ web app) scaffold that drives the MJ regression
4
+ runner against [MDN Web Docs](https://developer.mozilla.org). Three Computer
5
+ Use tests plus one **custom IOracle** plugged in via `--oracles-module`.
6
+
7
+ Use this directory as the starting point when authoring tests for your own web
8
+ app: copy it, edit `target.json` to point at your URL, replace the three test
9
+ JSONs, and tweak (or remove) the custom oracle.
10
+
11
+ ## What's in here
12
+
13
+ ```
14
+ generic-web/
15
+ ├── README.md # this file
16
+ ├── target.json # Mode C target profile (kind: generic-web)
17
+ ├── metadata/
18
+ │ ├── .mj-sync.json # directoryOrder = [tests, test-suites]
19
+ │ ├── tests/
20
+ │ │ ├── .mj-sync.json # entity = "MJ: Tests"
21
+ │ │ ├── .GW01-home-loads.json # MDN home page smoke
22
+ │ │ ├── .GW02-search-array-map.json # search-flow test
23
+ │ │ └── .GW03-navigate-references.json # top-nav navigation test
24
+ │ └── test-suites/
25
+ │ ├── .mj-sync.json # entity = "MJ: Test Suites"
26
+ │ └── .generic-web-suite.json # binds the 3 tests
27
+ └── oracles/
28
+ └── mdn-oracles.cjs # custom IOracle exports
29
+ ```
30
+
31
+ ## The custom oracle module
32
+
33
+ [`oracles/mdn-oracles.cjs`](oracles/mdn-oracles.cjs) exports two oracles in two
34
+ different styles to show both supported patterns:
35
+
36
+ | Type | Export style | Purpose |
37
+ |---|---|---|
38
+ | `final-url-host` | `class FinalUrlHostOracle implements IOracle` | Verifies the test ended on a specific hostname (catches off-domain wanders). Used by GW02 + GW03. |
39
+ | `min-step-count` | object instance `{ type, evaluate }` | Fails when the agent finished too fast (likely a silent navigation error). Available to any test but not currently referenced — feel free to wire it in. |
40
+
41
+ The loader (`packages/TestingFramework/CLI/src/utils/oracle-module-loader.ts`)
42
+ duck-types each export: if it has `type: string` + async `evaluate()`, it
43
+ gets registered. Anything else is silently skipped.
44
+
45
+ ## Running it
46
+
47
+ ### Inside Docker (Mode C via `mj test regression remote`)
48
+
49
+ From the MJ monorepo root:
50
+
51
+ ```bash
52
+ mj test regression remote --target=docker/regression/examples/generic-web/target.json
53
+ ```
54
+
55
+ What happens:
56
+
57
+ 1. The CLI loads [`target.json`](target.json), resolves any `env:` refs (none
58
+ in this example), and exports `MJ_TEST_VAR_baseUrl=https://developer.mozilla.org`,
59
+ `MJ_TEST_VAR_allowedDomains=["developer.mozilla.org","*.mozilla.org"]`,
60
+ `TEST_SUITE_NAME="Generic Web Example Suite"`,
61
+ `EXTRA_METADATA_DIRS=/app/docker/regression/examples/generic-web/metadata`,
62
+ `ORACLES_MODULE=/app/docker/regression/examples/generic-web/oracles/mdn-oracles.cjs`,
63
+ plus the entrypoint dispatch (`TEST_RUNNER_ENTRYPOINT=test-runner-remote-entrypoint.sh`).
64
+ 2. Compose brings up the full profile so the test-runner has a local
65
+ ephemeral DB for `TestRun` / `TestRunOutput` rows (the remote URL itself
66
+ isn't MJ — the DB is just for results recording).
67
+ 3. The remote entrypoint pushes the example's metadata to the DB, then
68
+ invokes `mj test suite --name "Generic Web Example Suite"
69
+ --oracles-module=/app/.../mdn-oracles.cjs --parallel --max-parallel 4`.
70
+ 4. Reports land in `docker/regression/test-results/run-{TIMESTAMP}/`.
71
+
72
+ ### Locally (against your dev DB) for fast iteration
73
+
74
+ If you're authoring tests and don't want the Docker round-trip every time,
75
+ push the metadata to your local MJ DB and run the suite directly:
76
+
77
+ ```bash
78
+ # 1. Push the example's test + suite metadata
79
+ npx mj sync push --dir=docker/regression/examples/generic-web/metadata
80
+
81
+ # 2. Run the suite with the custom oracle module + variable substitutions
82
+ mj test suite \
83
+ --name "Generic Web Example Suite" \
84
+ --oracles-module=docker/regression/examples/generic-web/oracles/mdn-oracles.cjs \
85
+ --var baseUrl=https://developer.mozilla.org \
86
+ --var 'allowedDomains=["developer.mozilla.org","*.mozilla.org"]' \
87
+ --parallel --max-parallel 3
88
+ ```
89
+
90
+ The `--var` flags drive the `{{baseUrl}}` / `{{allowedDomains}}` substitution
91
+ inside the test JSONs. JSON-shaped values (the array) work because the
92
+ substitution layer auto-parses them.
93
+
94
+ ## Copying this example for your own app
95
+
96
+ 1. **Copy the directory** to `docker/regression/examples/my-app/` (or anywhere
97
+ outside the monorepo, then mount it).
98
+ 2. **Edit `target.json`**:
99
+ - Set `baseUrl` to your app's URL.
100
+ - Set `allowedDomains` to the hosts the agent is allowed to touch.
101
+ - Add an `auth` block if your app requires login (see
102
+ [`docker/regression/targets/staging-mj.example.target.json`](../../../../../docker/regression/targets/staging-mj.example.target.json)
103
+ for the auth shape).
104
+ - Rename `suite` to match your suite, update `extraMetadataDirs` to your
105
+ metadata path, point `oraclesModule` at your oracles file.
106
+ 3. **Replace the tests** in `metadata/tests/` with goals specific to your app.
107
+ Reference `{{baseUrl}}` (and any custom vars you add) inside `startUrl`,
108
+ oracle `pattern` configs, etc.
109
+ 4. **Edit the suite** in `metadata/test-suites/.generic-web-suite.json` to
110
+ list your tests by name.
111
+ 5. **Write your oracles** in `oracles/*.cjs`. The duck-type contract is:
112
+ `{ type: string, async evaluate(input, config) → OracleResult }`.
113
+
114
+ See [`../bring-your-own-app/`](../bring-your-own-app/) for an example that
115
+ also boots its own app inside the regression network (Mode D).
@@ -0,0 +1,10 @@
1
+ {
2
+ "version": "1.0.0",
3
+ "push": {
4
+ "autoCreateMissingRecords": true
5
+ },
6
+ "directoryOrder": [
7
+ "tests",
8
+ "test-suites"
9
+ ]
10
+ }
@@ -0,0 +1,48 @@
1
+ {
2
+ "fields": {
3
+ "Name": "Generic Web Example Suite",
4
+ "Description": "Mode C reference suite — runs 3 tests against MDN Web Docs to demonstrate variable templating ({{baseUrl}}, {{allowedDomains}}) and a custom IOracle plugged in via `mj test suite --oracles-module=…`. Used by the docker/regression/examples/generic-web/ scaffold.",
5
+ "Status": "Active"
6
+ },
7
+ "relatedEntities": {
8
+ "MJ: Test Suite Tests": [
9
+ {
10
+ "fields": {
11
+ "SuiteID": "@parent:ID",
12
+ "TestID": "@lookup:MJ: Tests.Name=GW01 - MDN Home Loads",
13
+ "Sequence": 1,
14
+ "Status": "Active"
15
+ },
16
+ "primaryKey": {
17
+ "ID": "EE53002B-DCD5-49AC-AE16-211254BCE6FE"
18
+ },
19
+ "sync": {
20
+ "lastModified": "2026-05-17T23:05:53.394Z",
21
+ "checksum": "09664d1aa1dba6101156b1ad5cb56893aa025fee4b4569ecd1f3e86325617d6f"
22
+ }
23
+ },
24
+ {
25
+ "fields": {
26
+ "SuiteID": "@parent:ID",
27
+ "TestID": "@lookup:MJ: Tests.Name=GW02 - Search Array.prototype.map",
28
+ "Sequence": 2,
29
+ "Status": "Active"
30
+ },
31
+ "primaryKey": {
32
+ "ID": "DB858A9C-BA30-40E1-9FAA-1452A48611F2"
33
+ },
34
+ "sync": {
35
+ "lastModified": "2026-05-17T23:05:53.408Z",
36
+ "checksum": "db7b7b0e7106adcae9bcbf9d1c06c2d4175b70fad2dd6d2282ea44d9d93e3cd4"
37
+ }
38
+ }
39
+ ]
40
+ },
41
+ "primaryKey": {
42
+ "ID": "3C13E3D6-F022-409A-A6FE-B5857BCE7FA4"
43
+ },
44
+ "sync": {
45
+ "lastModified": "2026-05-17T23:05:53.359Z",
46
+ "checksum": "b6ac99fec8087fb090387e0869228705c4ed7e36dcae2ef667e729ede0101bba"
47
+ }
48
+ }
@@ -0,0 +1,12 @@
1
+ {
2
+ "entity": "MJ: Test Suites",
3
+ "filePattern": "**/.*.json",
4
+ "pull": {
5
+ "relatedEntities": {
6
+ "MJ: Test Suite Tests": {
7
+ "entity": "MJ: Test Suite Tests",
8
+ "foreignKey": "TestSuiteID"
9
+ }
10
+ }
11
+ }
12
+ }
@@ -0,0 +1,74 @@
1
+ {
2
+ "fields": {
3
+ "TypeID": "@lookup:MJ: Test Types.Name=Computer Use",
4
+ "Name": "GW01 - MDN Home Loads",
5
+ "Description": "Navigates to the MDN Web Docs home page and verifies the landing page renders with its main navigation. Demonstrates Mode C variable templating ({{baseUrl}}) — no MJ assumptions, no auth.",
6
+ "InputDefinition": {
7
+ "goal": "Verify the home page has loaded successfully. Look for the main navigation header (References, Guides, Plus) and the prominent 'MDN Web Docs' branding. Do NOT click anything — just confirm the page rendered without errors and the nav elements are visible.",
8
+ "startUrl": "{{baseUrl}}",
9
+ "allowedDomains": "{{allowedDomains}}",
10
+ "auth": {
11
+ "bindings": []
12
+ }
13
+ },
14
+ "ExpectedOutcomes": {
15
+ "goalCompleted": true,
16
+ "finalUrlPattern": "^https?://(www\\.)?developer\\.mozilla\\.org/?",
17
+ "minConfidence": 0.7,
18
+ "maxSteps": 10,
19
+ "judgeValidationCriteria": [
20
+ "Page rendered without errors",
21
+ "Main navigation visible (HTML, CSS, Learn, Tools, etc)",
22
+ "MDN branding is present"
23
+ ]
24
+ },
25
+ "Configuration": {
26
+ "headless": true,
27
+ "maxSteps": 10,
28
+ "maxExecutionTime": 120000,
29
+ "screenshotHistoryDepth": 3,
30
+ "viewportWidth": 1280,
31
+ "viewportHeight": 720,
32
+ "controllerPromptName": "Computer Use - Controller",
33
+ "judgePromptName": "Computer Use - Judge",
34
+ "judgeFrequency": "EveryNSteps:3",
35
+ "oracles": [
36
+ {
37
+ "type": "goal-completion",
38
+ "weight": 0.5,
39
+ "config": {
40
+ "minConfidence": 0.7
41
+ }
42
+ },
43
+ {
44
+ "type": "url-match",
45
+ "weight": 0.3,
46
+ "config": {
47
+ "pattern": "^https?://(www\\.)?developer\\.mozilla\\.org/?"
48
+ }
49
+ },
50
+ {
51
+ "type": "step-count",
52
+ "weight": 0.2,
53
+ "config": {
54
+ "maxSteps": 10
55
+ }
56
+ }
57
+ ],
58
+ "scoringWeights": {
59
+ "goal-completion": 0.5,
60
+ "url-match": 0.3,
61
+ "step-count": 0.2
62
+ }
63
+ },
64
+ "Status": "Active",
65
+ "Tags": "P0,generic-web,smoke"
66
+ },
67
+ "primaryKey": {
68
+ "ID": "C21C91DF-8EC8-4741-9251-EB14187A2532"
69
+ },
70
+ "sync": {
71
+ "lastModified": "2026-05-17T23:05:53.299Z",
72
+ "checksum": "cd52d0df6d51b2d14f5559cd9c2bb8894b29f7343733673fd618687db8506876"
73
+ }
74
+ }
@@ -0,0 +1,82 @@
1
+ {
2
+ "fields": {
3
+ "TypeID": "@lookup:MJ: Test Types.Name=Computer Use",
4
+ "Name": " GW02 - Search Array.prototype.map",
5
+ "Description": "Uses the search box to find documentation for Array.prototype.map and verifies a relevant result page loads. Exercises text input + click + page navigation against a public site.",
6
+ "InputDefinition": {
7
+ "goal": "Open the search box (top-right magnifying-glass icon or 'Search MDN' input). Type 'Array.prototype.map' and submit. Follow the most relevant result. Verify a page about Array.prototype.map (the JavaScript method) loads",
8
+ "startUrl": "{{baseUrl}}",
9
+ "allowedDomains": "{{allowedDomains}}",
10
+ "auth": {
11
+ "bindings": []
12
+ }
13
+ },
14
+ "ExpectedOutcomes": {
15
+ "goalCompleted": true,
16
+ "finalUrlPattern": "(?i)/array(/prototype)?/map",
17
+ "minConfidence": 0.7,
18
+ "maxSteps": 20,
19
+ "judgeValidationCriteria": [
20
+ "Final page is about Array.prototype.map",
21
+ "Page contains a code example",
22
+ "Page is a reference doc (not an unrelated search result)"
23
+ ]
24
+ },
25
+ "Configuration": {
26
+ "headless": true,
27
+ "maxSteps": 20,
28
+ "maxExecutionTime": 240000,
29
+ "screenshotHistoryDepth": 3,
30
+ "viewportWidth": 1280,
31
+ "viewportHeight": 720,
32
+ "controllerPromptName": "Computer Use - Controller",
33
+ "judgePromptName": "Computer Use - Judge",
34
+ "judgeFrequency": "EveryNSteps:3",
35
+ "oracles": [
36
+ {
37
+ "type": "goal-completion",
38
+ "weight": 0.4,
39
+ "config": {
40
+ "minConfidence": 0.7
41
+ }
42
+ },
43
+ {
44
+ "type": "url-match",
45
+ "weight": 0.3,
46
+ "config": {
47
+ "pattern": "(?i)/array(/prototype)?/map"
48
+ }
49
+ },
50
+ {
51
+ "type": "step-count",
52
+ "weight": 0.1,
53
+ "config": {
54
+ "maxSteps": 20
55
+ }
56
+ },
57
+ {
58
+ "type": "final-url-host",
59
+ "weight": 0.2,
60
+ "config": {
61
+ "host": "developer.mozilla.org"
62
+ }
63
+ }
64
+ ],
65
+ "scoringWeights": {
66
+ "goal-completion": 0.4,
67
+ "url-match": 0.3,
68
+ "step-count": 0.1,
69
+ "final-url-host": 0.2
70
+ }
71
+ },
72
+ "Status": "Active",
73
+ "Tags": "P1,generic-web,search"
74
+ },
75
+ "primaryKey": {
76
+ "ID": "2E8DE64D-B817-4E3D-B471-FF1142FABFDC"
77
+ },
78
+ "sync": {
79
+ "lastModified": "2026-05-17T23:05:53.319Z",
80
+ "checksum": "eeec8bd96e8ff50437a3a20c2a821dc991fcece9277c23510b635e5b6501b88b"
81
+ }
82
+ }
@@ -0,0 +1,12 @@
1
+ {
2
+ "entity": "MJ: Tests",
3
+ "filePattern": "**/.*.json",
4
+ "pull": {
5
+ "lookupFields": {
6
+ "TypeID": {
7
+ "entity": "MJ: Test Types",
8
+ "field": "Name"
9
+ }
10
+ }
11
+ }
12
+ }
@@ -0,0 +1,121 @@
1
+ /**
2
+ * Example custom-oracle module for the generic-web example.
3
+ *
4
+ * Loaded at suite startup via:
5
+ * mj test suite --oracles-module=./oracles/mdn-oracles.cjs
6
+ *
7
+ * Each export that looks like an `IOracle` (has `type: string` + async
8
+ * `evaluate(input, config)`) gets registered on the TestEngine and becomes
9
+ * available to any test that references its `type` in `Configuration.oracles`.
10
+ *
11
+ * Supported export shapes (see packages/TestingFramework/CLI/src/utils/
12
+ * oracle-module-loader.ts for the duck-type check):
13
+ * - A class implementing IOracle (instantiated with no args)
14
+ * - An object instance with `{ type: '...', async evaluate() { } }`
15
+ *
16
+ * This module demonstrates both shapes so adopters can copy whichever style
17
+ * they prefer.
18
+ */
19
+ 'use strict';
20
+
21
+ /**
22
+ * FinalUrlHostOracle — class-export style.
23
+ *
24
+ * Validates that the test's final URL is on a specific host. Useful as a
25
+ * defensive check: if the agent wandered off-domain (e.g., chased an external
26
+ * link), the URL hostname won't match and this oracle fails the test.
27
+ *
28
+ * Config:
29
+ * - host: string — the expected hostname (e.g., "developer.mozilla.org").
30
+ * - allowSubdomains: boolean (default true) — accept "*.host" matches too.
31
+ */
32
+ class FinalUrlHostOracle {
33
+ constructor() {
34
+ this.type = 'final-url-host';
35
+ }
36
+
37
+ async evaluate(input, config) {
38
+ const actual = (input && input.actualOutput) || {};
39
+ const finalUrl = actual.finalUrl || '';
40
+ const expectedHost = (config && config.host) || '';
41
+ const allowSubdomains = config && config.allowSubdomains !== false;
42
+
43
+ if (!expectedHost) {
44
+ return {
45
+ oracleType: this.type,
46
+ passed: false,
47
+ score: 0,
48
+ message: 'No host configured for final-url-host oracle',
49
+ };
50
+ }
51
+ if (!finalUrl) {
52
+ return {
53
+ oracleType: this.type,
54
+ passed: false,
55
+ score: 0,
56
+ message: 'No final URL recorded for the test',
57
+ };
58
+ }
59
+
60
+ let actualHost;
61
+ try {
62
+ actualHost = new URL(finalUrl).hostname.toLowerCase();
63
+ } catch (err) {
64
+ return {
65
+ oracleType: this.type,
66
+ passed: false,
67
+ score: 0,
68
+ message: `Final URL is not parseable: ${finalUrl}`,
69
+ };
70
+ }
71
+
72
+ const expected = expectedHost.toLowerCase();
73
+ const matches = allowSubdomains
74
+ ? actualHost === expected || actualHost.endsWith('.' + expected)
75
+ : actualHost === expected;
76
+
77
+ return {
78
+ oracleType: this.type,
79
+ passed: matches,
80
+ score: matches ? 1.0 : 0,
81
+ message: matches
82
+ ? `Final URL host matches: ${actualHost}`
83
+ : `Final URL host mismatch — expected ${expected}, got ${actualHost}`,
84
+ details: { actualHost, expectedHost: expected, finalUrl },
85
+ };
86
+ }
87
+ }
88
+
89
+ /**
90
+ * minStepCountOracle — instance-export style.
91
+ *
92
+ * Inverse of the built-in step-count oracle: fails when the agent finished too
93
+ * fast (e.g., immediately hit an error page and gave up). Useful for catching
94
+ * silent navigation failures.
95
+ *
96
+ * Config:
97
+ * - minSteps: number — minimum acceptable step count (default 2).
98
+ */
99
+ const minStepCountOracle = {
100
+ type: 'min-step-count',
101
+ async evaluate(input, config) {
102
+ const totalSteps =
103
+ (input && input.actualOutput && input.actualOutput.totalSteps) || 0;
104
+ const minSteps = (config && typeof config.minSteps === 'number') ? config.minSteps : 2;
105
+ const passed = totalSteps >= minSteps;
106
+ return {
107
+ oracleType: this.type,
108
+ passed,
109
+ score: passed ? 1.0 : 0,
110
+ message: passed
111
+ ? `Test took ${totalSteps} step(s) (≥ ${minSteps})`
112
+ : `Test finished too quickly: ${totalSteps} step(s) < min ${minSteps}`,
113
+ details: { totalSteps, minSteps },
114
+ };
115
+ },
116
+ };
117
+
118
+ module.exports = {
119
+ FinalUrlHostOracle,
120
+ minStepCountOracle,
121
+ };
@@ -0,0 +1,25 @@
1
+ {
2
+ "schemaVersion": "1.0",
3
+ "name": "generic-web-mdn",
4
+ "kind": "generic-web",
5
+ "description": "Mode C reference target: drives the MJ regression runner against MDN Web Docs (a stable public site) with 3 example tests and one custom IOracle. Used to verify the generic-web flow end-to-end.",
6
+
7
+ "baseUrl": "https://developer.mozilla.org",
8
+ "allowedDomains": ["developer.mozilla.org", "*.mozilla.org"],
9
+
10
+ "auth": {
11
+ "bindings": []
12
+ },
13
+
14
+ "suite": "Generic Web Example Suite",
15
+ "extraMetadataDirs": [
16
+ "/app/docker/regression/examples/generic-web/metadata"
17
+ ],
18
+
19
+ "oraclesModule": "/app/docker/regression/examples/generic-web/oracles/mdn-oracles.cjs",
20
+
21
+ "archive": {
22
+ "tag": "generic-web-mdn",
23
+ "source": "mode-c-mdn"
24
+ }
25
+ }
@@ -0,0 +1,26 @@
1
+ # GitHub Actions — run the regression suite from your own repo
2
+
3
+ [`regression.yml`](regression.yml) is a copy-paste workflow that runs the suite
4
+ against your MJ instance using the published `memberjunction/agentic-test-runner`
5
+ image — **no MJ monorepo checkout**.
6
+
7
+ ## Use it
8
+
9
+ 1. Copy `regression.yml` to `.github/workflows/regression.yml` in your repo.
10
+ 2. Add your suite next to it: a `my-suite/target.json` (baseUrl + auth `env:`
11
+ refs + suite name) and `my-suite/metadata/` (tests + test-suites). Scaffold a
12
+ starting point with `mj test regression init remote-mj`.
13
+ 3. Add repo **secrets** (names match the root `.env` convention):
14
+ - `AI_VENDOR_API_KEY__GeminiLLM` (primary model) — and/or `AI_VENDOR_API_KEY__AnthropicLLM`
15
+ - `DB_HOST` / `DB_DATABASE` / `DB_USERNAME` / `DB_PASSWORD` — a reachable MJ
16
+ provider DB (your instance under test, or a throwaway) where test
17
+ definitions/prompts are pushed and results recorded.
18
+ - any auth vars your `target.json` references (e.g. `STAGING_TEST_USER`).
19
+ 4. Pin `IMAGE` to the MJ version your instance runs.
20
+
21
+ Results (incl. `report.html` + screenshots) upload as the `regression-results`
22
+ artifact. Diff locally with `mj test regression compare`.
23
+
24
+ > This is Mode B/C (drive a URL). For Mode D (boot your app in CI alongside the
25
+ > runner) add your app via `docker compose` and `mj test regression remote
26
+ > --overlay`, or a services: block. See [REGRESSION_TESTING.md](../../../../../docker/regression/REGRESSION_TESTING.md).