@pzy560117/opentest 0.1.10 → 0.1.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/assets/manifest.json +15 -1
  2. package/assets/skills/opentest/references/android-app-testing.md +78 -0
  3. package/assets/skills/opentest/references/api-testing.md +77 -0
  4. package/assets/skills/opentest/references/codex-harness-coverage-heuristics.md +18 -1
  5. package/assets/skills/opentest/references/complete-testing-workflow.md +23 -0
  6. package/assets/skills/opentest/references/desktop-gui-testing.md +52 -0
  7. package/assets/skills/opentest/references/matrix-format.md +8 -2
  8. package/assets/skills/opentest/references/test-asset-layout.md +64 -0
  9. package/assets/skills/opentest/references/test-surfaces.md +101 -0
  10. package/assets/skills/opentest/references/web-browser-testing.md +40 -0
  11. package/assets/skills/opentest/templates/acceptance-template.md +3 -1
  12. package/assets/skills/opentest/templates/android-app-acceptance-template.md +46 -0
  13. package/assets/skills/opentest/templates/api-acceptance-template.md +44 -0
  14. package/assets/skills/opentest/templates/desktop-gui-acceptance-template.md +43 -0
  15. package/assets/skills/opentest/templates/matrix-template.md +12 -11
  16. package/assets/skills/opentest/templates/plan-template.md +2 -2
  17. package/assets/skills/opentest/templates/web-acceptance-template.md +27 -0
  18. package/assets/skills/opentest-accept/SKILL.md +18 -7
  19. package/assets/skills/opentest-android-app/SKILL.md +25 -0
  20. package/assets/skills/opentest-api/SKILL.md +25 -0
  21. package/assets/skills/opentest-author/SKILL.md +2 -1
  22. package/assets/skills/opentest-desktop-gui/SKILL.md +24 -0
  23. package/assets/skills/opentest-plan/SKILL.md +16 -9
  24. package/assets/skills/opentest-run/SKILL.md +15 -8
  25. package/assets/skills/opentest-web-browser/SKILL.md +26 -0
  26. package/assets/skills-zh/opentest/references/android-app-testing.md +78 -0
  27. package/assets/skills-zh/opentest/references/api-testing.md +77 -0
  28. package/assets/skills-zh/opentest/references/codex-harness-coverage-heuristics.md +18 -1
  29. package/assets/skills-zh/opentest/references/complete-testing-workflow.md +23 -0
  30. package/assets/skills-zh/opentest/references/desktop-gui-testing.md +52 -0
  31. package/assets/skills-zh/opentest/references/matrix-format.md +8 -2
  32. package/assets/skills-zh/opentest/references/test-asset-layout.md +64 -0
  33. package/assets/skills-zh/opentest/references/test-surfaces.md +101 -0
  34. package/assets/skills-zh/opentest/references/web-browser-testing.md +40 -0
  35. package/assets/skills-zh/opentest/templates/acceptance-template.md +3 -1
  36. package/assets/skills-zh/opentest/templates/android-app-acceptance-template.md +46 -0
  37. package/assets/skills-zh/opentest/templates/api-acceptance-template.md +44 -0
  38. package/assets/skills-zh/opentest/templates/desktop-gui-acceptance-template.md +43 -0
  39. package/assets/skills-zh/opentest/templates/matrix-template.md +12 -11
  40. package/assets/skills-zh/opentest/templates/plan-template.md +2 -2
  41. package/assets/skills-zh/opentest/templates/web-acceptance-template.md +27 -0
  42. package/assets/skills-zh/opentest-accept/SKILL.md +16 -5
  43. package/assets/skills-zh/opentest-android-app/SKILL.md +25 -0
  44. package/assets/skills-zh/opentest-api/SKILL.md +25 -0
  45. package/assets/skills-zh/opentest-author/SKILL.md +2 -1
  46. package/assets/skills-zh/opentest-desktop-gui/SKILL.md +24 -0
  47. package/assets/skills-zh/opentest-plan/SKILL.md +14 -7
  48. package/assets/skills-zh/opentest-run/SKILL.md +14 -7
  49. package/assets/skills-zh/opentest-web-browser/SKILL.md +26 -0
  50. package/package.json +1 -1
  51. package/scripts/smoke-test.js +321 -0
@@ -1,5 +1,5 @@
1
1
  {
2
- "version": "0.1.10",
2
+ "version": "0.1.12",
3
3
  "languages": [
4
4
  {
5
5
  "id": "en",
@@ -60,25 +60,39 @@
60
60
  "localized": [
61
61
  "opentest/SKILL.md",
62
62
  "opentest/references/acceptance-evidence.md",
63
+ "opentest/references/android-app-testing.md",
64
+ "opentest/references/api-testing.md",
63
65
  "opentest/references/codex-harness-coverage-heuristics.md",
64
66
  "opentest/references/command-routing.md",
65
67
  "opentest/references/complete-testing-workflow.md",
68
+ "opentest/references/desktop-gui-testing.md",
66
69
  "opentest/references/lifecycle.md",
67
70
  "opentest/references/matrix-format.md",
68
71
  "opentest/references/opentest-driven-development.md",
69
72
  "opentest/references/quality-gate.md",
73
+ "opentest/references/test-asset-layout.md",
74
+ "opentest/references/test-surfaces.md",
75
+ "opentest/references/web-browser-testing.md",
70
76
  "opentest/templates/acceptance-template.md",
77
+ "opentest/templates/android-app-acceptance-template.md",
78
+ "opentest/templates/api-acceptance-template.md",
71
79
  "opentest/templates/archive-layout.md",
80
+ "opentest/templates/desktop-gui-acceptance-template.md",
72
81
  "opentest/templates/fixtures-template.md",
73
82
  "opentest/templates/matrix-template.md",
74
83
  "opentest/templates/plan-template.md",
75
84
  "opentest/templates/report-template.md",
85
+ "opentest/templates/web-acceptance-template.md",
76
86
  "opentest-accept/SKILL.md",
87
+ "opentest-android-app/SKILL.md",
88
+ "opentest-api/SKILL.md",
77
89
  "opentest-archive/SKILL.md",
78
90
  "opentest-author/SKILL.md",
79
91
  "opentest-heal/SKILL.md",
80
92
  "opentest-plan/SKILL.md",
81
93
  "opentest-run/SKILL.md",
94
+ "opentest-desktop-gui/SKILL.md",
95
+ "opentest-web-browser/SKILL.md",
82
96
  "opentest-verify/SKILL.md"
83
97
  ],
84
98
  "shared": [
@@ -0,0 +1,78 @@
1
+ # Android App Testing
2
+
3
+ Use this reference for `android-app` execution-surface rows.
4
+
5
+ ## Adapter Boundary
6
+
7
+ `opentest-android-app` is the OpenTest execution-surface adapter. It keeps Android app tests visible in installed OpenTest skills and routes detailed execution to `android-midscene-pytest` when available.
8
+
9
+ Do not duplicate the full Android Midscene SOP here. Use this reference to decide the OpenTest matrix, layout, evidence, and blocked rules.
10
+
11
+ ## Default Layout
12
+
13
+ Durable Android assets follow `opentest/references/test-asset-layout.md`:
14
+
15
+ ```text
16
+ tests/android/
17
+ tests_py/
18
+ midscene/
19
+
20
+ scripts/
21
+ opentest-run-android.ps1
22
+ ```
23
+
24
+ If a repository already has an Android harness, use the closest existing paths but keep the same logical slots: pytest wrapper, Midscene scripts, device preparation, and run entry.
25
+
26
+ The canonical Midscene slot is `tests/android/midscene/`.
27
+
28
+ ## Default Route
29
+
30
+ ```text
31
+ opentest-android-app
32
+ -> android-midscene-pytest
33
+ -> pytest wrapper
34
+ -> npm/Vitest wrapper
35
+ -> @midscene/android
36
+ -> ADB + Android emulator/device
37
+ -> screenshots + logcat + Midscene HTML report
38
+ ```
39
+
40
+ User-facing entry is pytest. Prefer `python -m pytest tests_py -v` in an existing Android harness, or `scripts/opentest-run-android.ps1` when the repository owns the fixed layout.
41
+
42
+ Run `npm run test:android` only when Midscene model environment variables are complete or when debugging Midscene directly.
43
+
44
+ ## Required Evidence
45
+
46
+ `android-app` rows require:
47
+
48
+ - pytest report or command output
49
+ - ADB smoke result
50
+ - APK path, package name, emulator/device identity
51
+ - screenshot after key steps
52
+ - logcat or targeted Android logs
53
+ - Midscene HTML report when visual automation runs
54
+ - available `midscene_run/log/ai-call.log`, `agent.log`, `android-device.log`, and `midscene_run/report/*.html`
55
+ - read-after-write or persisted-state proof when the app writes data
56
+
57
+ ## Blocking Rules
58
+
59
+ Record `blocked` when any required prerequisite is missing:
60
+
61
+ - ADB
62
+ - emulator or real device
63
+ - APK path
64
+ - package name or launch activity
65
+ - Midscene model credentials when visual automation is required
66
+ - fixture data, reset path, or stable result surface
67
+
68
+ Do not mark Android app acceptance as PASS from a static screenshot alone.
69
+
70
+ ## Matrix Requirements
71
+
72
+ `android-app` rows must include:
73
+
74
+ - `Execution surface`: `android-app`
75
+ - `Acceptance mode`: `n/a`
76
+ - `Evidence layer`: `visual-acceptance`, `e2e`, `integration`, or `smoke`
77
+ - `Framework/command`: `opentest-android-app`, `android-midscene-pytest`, `python -m pytest tests_py -v`, or `scripts/opentest-run-android.ps1`
78
+ - `Required evidence`: pytest, ADB smoke, Midscene HTML report when used, screenshots, logcat, device/app metadata, `midscene_run` logs, and blocked prerequisites when unavailable
@@ -0,0 +1,77 @@
1
+ # API Testing
2
+
3
+ Use this reference for `api` execution-surface rows.
4
+
5
+ ## Default Architecture
6
+
7
+ Use the repository's existing API test framework first. If the project has no clear API test command, default to:
8
+
9
+ ```text
10
+ pytest
11
+ -> httpx or requests client
12
+ -> pytest fixtures for seed/teardown
13
+ -> jsonschema or existing Pydantic/DTO models for contract assertions
14
+ -> optional DB/storage/log read-back
15
+ -> pytest report or JUnit XML
16
+ ```
17
+
18
+ Use Docker Compose, testcontainers, or the repository's local service runner when dependencies need isolation. Mock or stub third-party APIs by default; live external services require an explicit requirement and recorded risk.
19
+
20
+ Place durable API assets under `tests/api/` from `opentest/references/test-asset-layout.md`: clients in `tests/api/clients/`, fixtures in `tests/api/fixtures/`, schemas in `tests/api/schemas/`, and repeatable entry through `scripts/opentest-run-api.ps1` or the repository's equivalent command.
21
+
22
+ ## Evidence Layers
23
+
24
+ | Layer | Proves | Typical command/tool |
25
+ | --- | --- | --- |
26
+ | contract | status code, headers, response fields, schema, error shape | project contract tests, `pytest`, OpenAPI-based checks |
27
+ | integration | API handler, service, database/storage, queue/log side effects | project integration command, `pytest`, local services |
28
+ | smoke | base URL and critical endpoints are alive | project smoke command, small `pytest`/curl script |
29
+ | security-review | auth, authorization, sensitive field exposure, injection risk | targeted tests plus review notes |
30
+
31
+ ## Required API Cases
32
+
33
+ For API changes, include applicable matrix rows for:
34
+
35
+ - happy path: expected status, payload, headers, and business state
36
+ - validation failure: invalid, empty, boundary, malformed, unsupported fields
37
+ - auth and permission: unauthenticated, expired token, wrong role, object-level authorization
38
+ - not found and stale state: missing resource, deleted resource, stale version
39
+ - conflict and idempotency: duplicate create, repeated submit, retry with idempotency key, concurrent update
40
+ - rate limit or throttling when applicable
41
+ - pagination, filtering, sorting, and empty result when list endpoints are changed
42
+ - data consistency: response, DB/storage, emitted event, queue message, file, or log
43
+ - teardown/cleanup: created resources removed or isolated fixture namespace reset
44
+
45
+ ## Contract Source
46
+
47
+ Prefer contract sources in this order:
48
+
49
+ 1. OpenAPI/protobuf/schema file committed in the repository.
50
+ 2. Existing request/response DTO, Pydantic model, serializer, or typed client.
51
+ 3. Requirement/design document with explicit fields and errors.
52
+ 4. Handwritten schema in the acceptance case.
53
+
54
+ Do not infer contract solely from current implementation behavior when it conflicts with requirements.
55
+
56
+ ## Blocking Rules
57
+
58
+ Record `blocked` when any required prerequisite is missing:
59
+
60
+ - base URL or service start command
61
+ - auth token, role, or test user
62
+ - fixture seed/teardown path
63
+ - dependency service, database, queue, or mock server
64
+ - stable contract source or expected schema
65
+ - deterministic read-after-write surface
66
+
67
+ Do not mark API acceptance as PASS from a 2xx response alone. Write operations require read-after-write evidence from a trustworthy surface such as API read endpoint, DB/storage record, queue/event/log, or another project-owned state surface.
68
+
69
+ ## Matrix Requirements
70
+
71
+ `api` rows must include:
72
+
73
+ - `Execution surface`: `api`
74
+ - `Acceptance mode`: `n/a`
75
+ - `Evidence layer`: `contract`, `integration`, `smoke`, or `security-review`
76
+ - `Framework/command`: project API command, `python -m pytest tests/api -v`, curl/httpie script, Postman/Newman when already used, or contract tool
77
+ - `Required evidence`: request/response record, status code, payload/schema assertion, auth/permission assertion when applicable, read-after-write/data consistency, and cleanup/teardown proof
@@ -13,6 +13,10 @@ This reference extracts short rules from the local Codex Harness knowledge base,
13
13
  - `tdd-workflow`
14
14
  - `e2e-runner`
15
15
  - `browser-e2e-testing`
16
+ - `opentest-android-app`
17
+ - `android-midscene-pytest`
18
+ - `opentest-desktop-gui`
19
+ - `opentest-api`
16
20
  - `verification-loop`
17
21
  - `code-reviewer`
18
22
  - `speckit-checklist`
@@ -28,6 +32,19 @@ This reference extracts short rules from the local Codex Harness knowledge base,
28
32
  | Permissions, payments, security, data writes, cross-page loops | high-risk acceptance or E2E evidence |
29
33
  | Copy, configuration, small non-behavioral changes | targeted review or light evidence |
30
34
 
35
+ ## Execution Surface Selection
36
+
37
+ Choose the execution surface separately from the evidence layer:
38
+
39
+ | Surface | Default route |
40
+ | --- | --- |
41
+ | `web-browser` | Chrome DevTools MCP, Playwright CLI, or browser acceptance |
42
+ | `android-app` | `opentest-android-app`; routes to `android-midscene-pytest` when available, with `python -m pytest tests_py -v` driving Midscene Android through ADB |
43
+ | `desktop-gui` | `opentest-desktop-gui`; project GUI automation first, `@midscene/computer` for visual/native/RDP GUI flows, or scripted manual GUI acceptance when automation is unavailable |
44
+ | `api` | `opentest-api`; project API/integration command first, otherwise `pytest` with `httpx`/`requests`, schema checks, fixtures, read-after-write, and cleanup/teardown |
45
+
46
+ Do not classify code checks such as unit, component, integration, contract, smoke, or security review as execution surfaces. Use them as evidence layers.
47
+
31
48
  ## Frontend Acceptance Dimensions
32
49
 
33
50
  Frontend or real workflow acceptance may choose applicable items from the following dimensions. Full coverage is not required every time:
@@ -74,7 +91,7 @@ The OpenTest plan phase checks these questions by default. If applicable, add th
74
91
  | ACC-001 | User sees success feedback after save | medium | UI acceptance | pending |
75
92
  ```
76
93
 
77
- Add coverage dimension, command, evidence path, and blocker reason columns only when risk or change type requires them.
94
+ Add execution surface, evidence layer, command/tool, evidence path, and blocker reason columns when the matrix drives acceptance execution or risk requires them.
78
95
 
79
96
  ## Quality Gate Heuristics
80
97
 
@@ -14,6 +14,29 @@ plan -> matrix -> fixtures -> tests -> run -> accept -> smoke -> pre-push -> ver
14
14
 
15
15
  Unit, component, integration, contract, E2E, smoke, and browser acceptance are evidence layers. They describe how to prove a requirement after or during implementation; they do not decide what the requirement is. If code does not exist yet, keep the acceptance case and mark code-dependent evidence as pending or blocked with a reason.
16
16
 
17
+ ## Execution Surfaces
18
+
19
+ Every matrix row must name both an execution surface and an evidence layer. The execution surface is where the requirement is exercised; the evidence layer is how the result is proven.
20
+
21
+ Before authoring tests, select the fixed asset layout from `opentest/references/test-asset-layout.md`. Option B, the standard framework skeleton, is the default; one-off scripts are allowed only for explicitly non-durable acceptance or blocked investigation.
22
+
23
+ Primary execution surfaces are:
24
+
25
+ - `web-browser`: browser-rendered pages and web apps
26
+ - `android-app`: Android APK/app GUI on emulator or device
27
+ - `desktop-gui`: native desktop GUI, Electron, Tauri, or similar app UI
28
+ - `api`: HTTP API, RPC, backend workflow, contract, or service endpoint
29
+
30
+ Do not use unit, component, integration, contract, smoke, or security review as the execution surface. Those are evidence layers or run gates. If an Android GUI requirement is present, route acceptance through `opentest-android-app`, which uses `android-midscene-pytest` when available, and require pytest/Midscene/screenshot/logcat evidence. If native desktop GUI behavior is present, route acceptance through `opentest-desktop-gui` and require project GUI automation or `@midscene/computer` evidence plus screenshots, GUI action logs, window/app metadata, and deterministic read-back. If API behavior is present, route acceptance through `opentest-api` and require contract, status code, payload/schema, auth/permission, read-after-write, and cleanup/teardown evidence when applicable.
31
+
32
+ For `web-browser`, choose an acceptance mode from `opentest/references/web-browser-testing.md`. MCP and Playwright CLI are immediate acceptance routes; durable regression requires a committed repeatable test such as `@playwright/test`.
33
+
34
+ For `android-app`, use `opentest/references/android-app-testing.md`. Durable Android assets stay in `tests/android/tests_py/`, `tests/android/midscene/`, and `scripts/opentest-run-android.ps1` or an existing Android harness.
35
+
36
+ For `desktop-gui`, use `opentest/references/desktop-gui-testing.md`. Electron/Tauri DOM-verifiable flows can stay in `web-browser`; native shell, tray, file picker, menu, OS dialog, installer, updater, RDP, and multi-window behavior stay in `desktop-gui`.
37
+
38
+ For `api`, use `opentest/references/api-testing.md`. Project API/integration commands are preferred; without them, use `pytest` with `httpx` or `requests`, schema checks, fixtures, and deterministic read-back.
39
+
17
40
  ## Test Data
18
41
 
19
42
  Create `docs/opentest/fixtures/` for changes that touch data, files, roles, permissions, APIs, or stateful workflows.
@@ -0,0 +1,52 @@
1
+ # Desktop GUI Testing
2
+
3
+ Use this reference for `desktop-gui` execution-surface rows.
4
+
5
+ Durable desktop GUI assets follow `opentest/references/test-asset-layout.md`: scripts under `tests/desktop/scripts/`, Midscene assets under `tests/desktop/midscene/`, metadata captures under `tests/desktop/metadata/`, and repeatable entry through `scripts/opentest-run-desktop.ps1` or the project GUI command.
6
+
7
+ ## Tool Routes
8
+
9
+ | Route | Use when | Required evidence |
10
+ | --- | --- | --- |
11
+ | project GUI automation | The repository already has a repeatable desktop automation command | command, report/log, screenshot or recording, post-action read-back |
12
+ | `@midscene/computer` | Native desktop controls, weak selectors, visual workflows, multi-window flows, or Windows RDP need AI visual assistance | Midscene/computer run log, screenshots, model env status, window/app metadata, deterministic read-back |
13
+ | accessibility/window metadata | Native controls expose stable accessibility tree, title, process, window handle, or menu state | metadata dump, action log, expected state assertion |
14
+ | scripted manual GUI acceptance | No reliable automation exists and the acceptance is one-off | exact steps, screenshots, window/app metadata, observed result, blocker/risk note |
15
+
16
+ ## Midscene Desktop Route
17
+
18
+ `@midscene/computer` is the Midscene desktop automation package. It can control local Windows, macOS, and Linux desktops, and can control remote Windows desktops through RDP when configured.
19
+
20
+ Use it as an OpenTest visual automation layer, not as the whole quality gate:
21
+
22
+ ```text
23
+ desktop-gui matrix row
24
+ -> project launch / environment check
25
+ -> @midscene/computer or project GUI automation
26
+ -> screenshots + GUI action log + window/app metadata
27
+ -> deterministic read/assert changed result
28
+ ```
29
+
30
+ For Electron or Tauri, first decide whether the requirement is DOM-verifiable. DOM-verifiable flows belong to `web-browser`; shell, tray, file picker, native menu, OS dialog, installer, updater, and multi-window behavior belong to `desktop-gui`.
31
+
32
+ ## Blocking Rules
33
+
34
+ Record `blocked` when any required prerequisite is missing:
35
+
36
+ - model credentials for Midscene visual automation
37
+ - desktop access, display, or RDP session
38
+ - app launch command or target process/window identity
39
+ - stable fixture data or reset/teardown path
40
+ - deterministic result surface after writes
41
+
42
+ Do not mark `desktop-gui` acceptance as PASS from an AI visual assertion alone. After create/update/delete/save actions, re-read a trustworthy result surface: reopened window state, file/config value, app storage/API, accessibility metadata, process/window metadata, or visible persisted value after restart.
43
+
44
+ ## Matrix Requirements
45
+
46
+ `desktop-gui` rows must include:
47
+
48
+ - `Execution surface`: `desktop-gui`
49
+ - `Acceptance mode`: `n/a`
50
+ - `Evidence layer`: `gui-acceptance`, `visual-acceptance`, `integration`, or `smoke`
51
+ - `Framework/command`: project GUI command, `@midscene/computer`, accessibility/window metadata script, or scripted manual GUI route
52
+ - `Required evidence`: screenshots or recording, GUI action log, window/app metadata, deterministic read-back, and blocked prerequisites when unavailable
@@ -2,11 +2,15 @@
2
2
 
3
3
  ## Minimal Columns
4
4
 
5
- | ID | Requirement source | Intent | Trigger/Input | Expected behavior | Risk | Evidence layer | Required evidence | Status |
6
- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
5
+ | ID | Requirement source | Intent | Execution surface | Acceptance mode | Trigger/Input | Expected behavior | Risk | Evidence layer | Required evidence | Status |
6
+ | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
7
7
 
8
8
  `Requirement source` is mandatory. Use a requirement ID, design section, user story, business rule, risk note, issue, or explicit user request. Do not use a function name, component path, or existing test file as the source of acceptance.
9
9
 
10
+ `Execution surface` is mandatory and should be one of `web-browser`, `android-app`, `desktop-gui`, or `api`.
11
+
12
+ `Acceptance mode` is mandatory for `web-browser`: `instant-acceptance`, `durable-regression`, or `visual-ai-assist`.
13
+
10
14
  ## Optional Columns
11
15
 
12
16
  - Coverage
@@ -28,4 +32,6 @@ Evidence layers describe how a requirement will be proven. They do not create or
28
32
  - `e2e`: cross-page flows, login, permissions, critical business loops.
29
33
  - `smoke`: key pages or happy paths do not crash.
30
34
  - `browser-acceptance`: real browser interaction, feedback location, responsive and visual state.
35
+ - `visual-acceptance`: visual GUI behavior on Android app or desktop GUI surfaces.
36
+ - `gui-acceptance`: desktop GUI behavior, window state, dialogs, menus, and native controls.
31
37
  - `security-review`: permissions, sensitive information, authorization bypass, duplicate submit, injection risk.
@@ -0,0 +1,64 @@
1
+ # Test Asset Layout
2
+
3
+ OpenTest default test assets use Option B: a standard framework skeleton. Do not decide directories ad hoc during `author`.
4
+
5
+ ## Default Layout
6
+
7
+ ```text
8
+ tests/
9
+ api/
10
+ conftest.py
11
+ clients/
12
+ fixtures/
13
+ schemas/
14
+ test_contract_*.py
15
+ test_permissions_*.py
16
+ test_crud_*.py
17
+ web/
18
+ playwright/
19
+ midscene/
20
+ android/
21
+ tests_py/
22
+ midscene/
23
+ desktop/
24
+ scripts/
25
+ midscene/
26
+ metadata/
27
+
28
+ docs/opentest/
29
+ matrix.md
30
+ fixtures/
31
+ acceptance/
32
+ runs/
33
+ reports/
34
+
35
+ scripts/
36
+ opentest-run-api.ps1
37
+ opentest-run-web.ps1
38
+ opentest-run-android.ps1
39
+ opentest-run-desktop.ps1
40
+ ```
41
+
42
+ Use the closest existing project directories when they already exist, but keep the same logical slots: surface tests under `tests/<surface>/`, evidence under `docs/opentest/`, and repeatable entry scripts under `scripts/opentest-run-*.ps1` or the repository's equivalent command.
43
+
44
+ ## Shape Rules
45
+
46
+ - The default is not a large QA platform. It is a stable skeleton for repeatable tests, fixtures, reports, and run entry points.
47
+ - Do not create one-off scripts as the durable path. One-off scripts may be used only for `instant-acceptance` or blocked investigation evidence.
48
+ - Do not create a separate top-level QA project unless the user explicitly chooses a team-scale template.
49
+ - `author` creates or updates assets only inside the chosen layout.
50
+ - `run` invokes fixed entry commands, not newly invented paths.
51
+ - `accept` records evidence under `docs/opentest/acceptance/` and run artifacts under `docs/opentest/runs/` or `docs/opentest/reports/`.
52
+
53
+ ## Surface Mapping
54
+
55
+ | Surface | Durable test location | Default run entry |
56
+ | --- | --- | --- |
57
+ | `api` | `tests/api/` | `scripts/opentest-run-api.ps1` or `python -m pytest tests/api -v` |
58
+ | `web-browser` | `tests/web/playwright/` and `tests/web/midscene/` | `scripts/opentest-run-web.ps1` or project E2E command |
59
+ | `android-app` | `tests/android/tests_py/` and `tests/android/midscene/` | `scripts/opentest-run-android.ps1` or `python -m pytest tests_py -v` in existing Android harness |
60
+ | `desktop-gui` | `tests/desktop/scripts/`, `tests/desktop/midscene/`, `tests/desktop/metadata/` | `scripts/opentest-run-desktop.ps1` or project GUI command |
61
+
62
+ ## When To Use A Lighter Shape
63
+
64
+ Use a lighter script-only shape only when the matrix row is explicitly one-off, exploratory, or blocked. Mark it as not durable regression and record the reason in `gap/blocker`.
@@ -0,0 +1,101 @@
1
+ # Test Surfaces
2
+
3
+ OpenTest classifies acceptance execution by surface and evidence layer:
4
+
5
+ - `Execution surface` is where the requirement is exercised from the user's or caller's point of view.
6
+ - `Evidence layer` is how the requirement is proven, for example unit, integration, contract, e2e, smoke, or security review.
7
+ - Test assets use the fixed layout in `opentest/references/test-asset-layout.md`; do not invent directories during authoring.
8
+
9
+ Keep the primary execution surface to one of these four values:
10
+
11
+ | Surface | Use when | Default acceptance path | Required artifacts |
12
+ | --- | --- | --- | --- |
13
+ | `web-browser` | Browser-rendered web pages, web apps, admin consoles, SaaS, dashboards | Use `opentest-web-browser`: Playwright MCP first, Playwright CLI fallback, `@playwright/test` for durable regression, Midscene only for visual assist | screenshots, snapshots, post-submit assertions, console/network notes, trace/report when durable |
14
+ | `android-app` | Android APK or Android app GUI on emulator/device | Use `opentest-android-app`, which routes to `android-midscene-pytest` when available: pytest orchestrates, `@midscene/android` executes visual automation, ADB/emulator controls device | pytest report, ADB smoke, Midscene HTML report, screenshots, logcat, device/app metadata, `midscene_run` logs |
15
+ | `desktop-gui` | Native desktop GUI or Electron/Tauri/Windows/macOS/Linux app UI | Use `opentest-desktop-gui`: project GUI automation first; `@midscene/computer` for visual desktop automation, weak selectors, native controls, multi-window flows, or RDP; scripted manual GUI acceptance only when automation is unavailable | screenshots or recording, GUI action log, window/app metadata, deterministic read-back, failure capture |
16
+ | `api` | HTTP API, RPC, backend workflow, contract, service endpoint | Use `opentest-api`: project API/integration command first; otherwise `pytest` with `httpx` or `requests`, schema checks, fixtures, and read-after-write evidence | request/response records, status codes, payload/schema assertions, auth/permission results, data consistency, cleanup/teardown, logs |
17
+
18
+ Do not invent a fifth primary surface. Code checks such as `unit`, `component`, `integration`, `contract`, `smoke`, and `security-review` are evidence layers or run gates, not primary surfaces.
19
+
20
+ ## Surface vs Evidence Layer
21
+
22
+ Examples:
23
+
24
+ | Requirement | Execution surface | Evidence layer |
25
+ | --- | --- | --- |
26
+ | User submits a web form and sees the saved item | `web-browser` | `browser-acceptance` + `integration` |
27
+ | Android user creates a task in the app | `android-app` | `e2e` + `visual-acceptance` |
28
+ | Desktop app opens a settings dialog and saves a preference | `desktop-gui` | `gui-acceptance` + `integration` |
29
+ | Client creates an entity through REST API | `api` | `contract` + `integration` |
30
+
31
+ ## Android App Surface
32
+
33
+ For `android-app`, read `opentest/references/android-app-testing.md` or use `opentest-android-app`. It routes through `android-midscene-pytest` when installed:
34
+
35
+ ```text
36
+ python -m pytest tests_py -v
37
+ -> npm/Vitest wrapper
38
+ -> @midscene/android
39
+ -> ADB + Android emulator/device
40
+ -> screenshots + logcat + Midscene HTML report
41
+ ```
42
+
43
+ Route selection:
44
+
45
+ - Stable automation, demos, and repeatable reports: use `pytest -> npm/Vitest -> @midscene/android -> ADB`.
46
+ - One-off natural-language exploration: Midscene YAML runner is optional, but it must not replace the pytest entry.
47
+ - Agent-controlled Android: Midscene MCP is optional only when separately configured with `MIDSCENE_MCP_ANDROID_MODE=true`; do not write global MCP config automatically.
48
+ - Pure Python stack: evaluate `midscene-python` only when the user explicitly asks.
49
+
50
+ Layered run:
51
+
52
+ - User-facing entry is `python -m pytest tests_py -v`.
53
+ - pytest should check ADB, prepare emulator/device, install APK, run ADB smoke, and collect evidence before Midscene.
54
+ - Run `npm run test:android` only when model environment variables are complete.
55
+ - Run `npm run test:android` directly only to debug the Midscene layer.
56
+
57
+ If Midscene model credentials, ADB, emulator/device, APK path, or package name are missing, record `blocked` with the exact missing prerequisite. Do not mark Android GUI acceptance as pass from a static screenshot alone.
58
+
59
+ Failure evidence should include any available `midscene_run/log/ai-call.log`, `midscene_run/log/agent.log`, `midscene_run/log/android-device.log`, and `midscene_run/report/*.html`.
60
+
61
+ ## Web Browser Surface
62
+
63
+ For `web-browser`, read `opentest/references/web-browser-testing.md` or use `opentest-web-browser`.
64
+
65
+ Set `Acceptance mode`:
66
+
67
+ - `instant-acceptance`: Playwright MCP first, Playwright CLI fallback.
68
+ - `durable-regression`: `@playwright/test` or the repository's existing E2E framework.
69
+ - `visual-ai-assist`: Midscene for weak selectors, canvas, cross-frame UI, or visual matching.
70
+
71
+ Do not treat MCP or Playwright CLI evidence as durable regression by itself.
72
+
73
+ ## Desktop GUI Surface
74
+
75
+ For `desktop-gui`, read `opentest/references/desktop-gui-testing.md` or use `opentest-desktop-gui`.
76
+
77
+ Route selection:
78
+
79
+ - Prefer explicit project GUI automation when the repository already provides a repeatable command.
80
+ - Use `@midscene/computer` for native controls, visual workflows, weak selectors, multi-window flows, or Windows RDP that needs AI visual assistance.
81
+ - For Electron or Tauri, use `web-browser` when the requirement is DOM-verifiable; keep native shell, tray, file picker, menu, OS dialog, installer, updater, and multi-window behavior under `desktop-gui`.
82
+ - Scripted manual GUI acceptance is a fallback for one-off evidence, not durable regression.
83
+
84
+ Do not record `desktop-gui` as PASS from an AI visual assertion alone. Save/create/update/delete flows must include screenshots or recording, GUI action log, window/app metadata, and a deterministic read/assert changed result after reopening, restarting, or reading a trusted app/file/config state.
85
+
86
+ ## API Surface
87
+
88
+ For `api`, read `opentest/references/api-testing.md` or use `opentest-api`.
89
+
90
+ Route selection:
91
+
92
+ - Prefer explicit project API, integration, contract, or smoke commands.
93
+ - If no project command exists, default to `python -m pytest tests/api -v` with `httpx` or `requests`, `jsonschema` or existing Pydantic/DTO models, and pytest fixtures for seed/teardown.
94
+ - Use OpenAPI, protobuf, schema files, DTOs, serializers, typed clients, or requirement docs as contract sources.
95
+ - Mock or stub third-party APIs unless the requirement explicitly needs live external services.
96
+
97
+ Do not record `api` as PASS from a 2xx response alone. API writes must include request/response records, payload/schema assertions, auth/permission checks when applicable, read-after-write/data consistency, and cleanup or teardown proof.
98
+
99
+ ## Matrix Rule
100
+
101
+ Every matrix row must include both `Execution surface` and `Evidence layer`. `web-browser` rows must also include `Acceptance mode`. If a requirement needs more than one surface or mode, split it into separate rows or state the primary surface and add secondary evidence in `Required evidence`.
@@ -0,0 +1,40 @@
1
+ # Web Browser Testing
2
+
3
+ Use this reference for `web-browser` execution-surface rows.
4
+
5
+ Durable web assets follow `opentest/references/test-asset-layout.md`: Playwright tests under `tests/web/playwright/`, Midscene visual assists under `tests/web/midscene/`, and repeatable entry through `scripts/opentest-run-web.ps1` or the repository's existing E2E command.
6
+
7
+ ## Acceptance Modes
8
+
9
+ | Mode | Use when | Default tool | Required evidence |
10
+ | --- | --- | --- | --- |
11
+ | `instant-acceptance` | Prove the current change in a real browser now | Playwright MCP first, Playwright CLI fallback | snapshots, action steps, post-submit assertion, screenshot, console/network notes |
12
+ | `durable-regression` | The workflow must run repeatedly in CI or future releases | `@playwright/test` or existing E2E framework | committed test file, deterministic locators/assertions, command, report/trace path |
13
+ | `visual-ai-assist` | Selectors cannot reliably prove the UI state | Midscene plus Playwright or project browser driver | Midscene report, screenshot, and deterministic read/assert result |
14
+
15
+ ## Tool Rules
16
+
17
+ - Playwright MCP and Playwright CLI are immediate acceptance tools. They are useful for live exploration and proof, but they are not durable regression by themselves.
18
+ - `@playwright/test` or the project's existing E2E framework is the default durable regression path.
19
+ - Midscene is a supplemental AI visual UI automation layer for weak selectors, canvas, cross-frame UI, visual matching, or natural-language exploration.
20
+ - Do not record `visual-ai-assist` as PASS from an AI assertion alone. Re-read a trustworthy result surface after writes.
21
+
22
+ ## Required Web Write Chain
23
+
24
+ ```text
25
+ open -> snapshot -> fill/input -> click(submit/confirm) -> snapshot -> read/assert changed result -> screenshot -> PASS/FAIL
26
+ ```
27
+
28
+ PASS must name the changed value and where it was read back: page, list, detail view, API response, storage record, or logs.
29
+
30
+ ## Matrix Fields
31
+
32
+ For `web-browser`, include:
33
+
34
+ - `Execution surface`: `web-browser`
35
+ - `Acceptance mode`: `instant-acceptance`, `durable-regression`, or `visual-ai-assist`
36
+ - `Evidence layer`: `browser-acceptance`, `e2e`, `visual-acceptance`, `integration`, or `smoke`
37
+ - `Framework/command`: MCP, Playwright CLI, `npx playwright test`, project E2E command, or Midscene route
38
+ - `Required evidence`: snapshots, screenshot, post-submit assertion, report/trace, console/network notes, or Midscene report
39
+
40
+ If a feature needs both immediate acceptance and durable regression, split it into two rows.
@@ -5,7 +5,9 @@
5
5
  - intent:
6
6
  - context:
7
7
  - actor:
8
- - execution surface:
8
+ - execution surface: web-browser | android-app | desktop-gui | api
9
+ - acceptance mode:
10
+ - evidence layer:
9
11
  - trigger/input:
10
12
  - expected feedback location:
11
13
  - status: pending
@@ -0,0 +1,46 @@
1
+ # Android App Acceptance
2
+
3
+ ## ACC-Android-001
4
+
5
+ - execution surface: android-app
6
+ - acceptance mode: n/a
7
+ - tool route: opentest-android-app | android-midscene-pytest | pytest wrapper | @midscene/android
8
+ - evidence layer: visual-acceptance | e2e | integration | smoke
9
+ - APK path:
10
+ - package name:
11
+ - emulator/device:
12
+ - fixture/reset:
13
+ - status: pending
14
+
15
+ ### Environment
16
+
17
+ - ADB status:
18
+ - model env status:
19
+ - app version/build:
20
+ - device/app metadata:
21
+
22
+ ### Steps
23
+
24
+ 1.
25
+
26
+ ### Expected Outcome
27
+
28
+ -
29
+
30
+ ### Read-Back Contract
31
+
32
+ - app result surface:
33
+ - persisted state:
34
+ - refresh/reopen check:
35
+ - cleanup assertion:
36
+
37
+ ### Evidence
38
+
39
+ - status:
40
+ - pytest report:
41
+ - ADB smoke:
42
+ - screenshots:
43
+ - logcat:
44
+ - Midscene HTML report:
45
+ - midscene_run logs:
46
+ - blockers: