@brunosps00/dev-workflow 0.7.0 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/README.md +20 -4
  2. package/lib/constants.js +8 -0
  3. package/lib/install-deps.js +13 -0
  4. package/package.json +1 -1
  5. package/scaffold/en/commands/dw-deps-audit.md +326 -0
  6. package/scaffold/en/commands/dw-dockerize.md +321 -0
  7. package/scaffold/en/commands/dw-find-skills.md +158 -0
  8. package/scaffold/en/commands/dw-fix-qa.md +34 -13
  9. package/scaffold/en/commands/dw-help.md +4 -0
  10. package/scaffold/en/commands/dw-new-project.md +350 -0
  11. package/scaffold/en/commands/dw-run-qa.md +124 -23
  12. package/scaffold/en/templates/project-onepager.md +129 -0
  13. package/scaffold/pt-br/commands/dw-deps-audit.md +326 -0
  14. package/scaffold/pt-br/commands/dw-dockerize.md +321 -0
  15. package/scaffold/pt-br/commands/dw-find-skills.md +158 -0
  16. package/scaffold/pt-br/commands/dw-fix-qa.md +34 -13
  17. package/scaffold/pt-br/commands/dw-help.md +4 -0
  18. package/scaffold/pt-br/commands/dw-new-project.md +350 -0
  19. package/scaffold/pt-br/commands/dw-run-qa.md +124 -23
  20. package/scaffold/pt-br/templates/project-onepager.md +129 -0
  21. package/scaffold/skills/api-testing-recipes/SKILL.md +104 -0
  22. package/scaffold/skills/api-testing-recipes/recipes/dotnet-webapp-factory.md +168 -0
  23. package/scaffold/skills/api-testing-recipes/recipes/http-rest-client.md +130 -0
  24. package/scaffold/skills/api-testing-recipes/recipes/pytest-httpx.md +157 -0
  25. package/scaffold/skills/api-testing-recipes/recipes/rust-reqwest.md +173 -0
  26. package/scaffold/skills/api-testing-recipes/recipes/supertest-node.md +153 -0
  27. package/scaffold/skills/api-testing-recipes/references/auth-patterns.md +138 -0
  28. package/scaffold/skills/api-testing-recipes/references/log-conventions.md +117 -0
  29. package/scaffold/skills/api-testing-recipes/references/matrix-conventions.md +68 -0
  30. package/scaffold/skills/api-testing-recipes/references/openapi-driven.md +97 -0
  31. package/scaffold/skills/docker-compose-recipes/SKILL.md +84 -0
  32. package/scaffold/skills/docker-compose-recipes/references/compose-composition.md +91 -0
  33. package/scaffold/skills/docker-compose-recipes/references/env-conventions.md +51 -0
  34. package/scaffold/skills/docker-compose-recipes/references/healthcheck-patterns.md +54 -0
  35. package/scaffold/skills/docker-compose-recipes/references/prod-vs-dev.md +85 -0
  36. package/scaffold/skills/docker-compose-recipes/services/elasticsearch.yml +34 -0
  37. package/scaffold/skills/docker-compose-recipes/services/jaeger.yml +24 -0
  38. package/scaffold/skills/docker-compose-recipes/services/localstack.yml +30 -0
  39. package/scaffold/skills/docker-compose-recipes/services/mailhog.yml +23 -0
  40. package/scaffold/skills/docker-compose-recipes/services/mailpit.yml +27 -0
  41. package/scaffold/skills/docker-compose-recipes/services/meilisearch.yml +28 -0
  42. package/scaffold/skills/docker-compose-recipes/services/memcached.yml +19 -0
  43. package/scaffold/skills/docker-compose-recipes/services/minio.yml +30 -0
  44. package/scaffold/skills/docker-compose-recipes/services/mysql.yml +30 -0
  45. package/scaffold/skills/docker-compose-recipes/services/postgres.yml +30 -0
  46. package/scaffold/skills/docker-compose-recipes/services/rabbitmq.yml +29 -0
  47. package/scaffold/skills/docker-compose-recipes/services/redis.yml +25 -0
  48. package/scaffold/skills/docker-compose-recipes/services/smtp4dev.yml +27 -0
  49. package/scaffold/skills/docker-compose-recipes/services/traefik.yml +42 -0
  50. package/scaffold/skills/docker-compose-recipes/services/typesense.yml +25 -0
@@ -0,0 +1,173 @@
1
+ # Recipe: `reqwest + tokio::test` (Rust)
2
+
3
+ Use for Axum, Actix-web, Rocket. Async HTTP client that pairs naturally with each framework's tower / actix-rt runtime.
4
+
5
+ Two execution modes:
6
+
7
+ - **A: against a running server** — best when the project already exposes the API in dev.
8
+ - **B: in-process via `axum::Router::oneshot`** — fastest, no port, no flake.
9
+
10
+ ## File shape (mode A — running server, framework-agnostic)
11
+
12
+ `{{PRD_PATH}}/QA/scripts/api/rf_xx_[slug].rs` (typically a `tests/` integration target):
13
+
14
+ ```rust
15
+ // RF-XX [slug] — API QA suite
16
+ use reqwest::{Client, StatusCode};
17
+ use serde_json::{json, Value};
18
+
19
+ fn base() -> String { std::env::var("API_BASE_URL").unwrap_or_else(|_| "http://localhost:3000".into()) }
20
+ fn token_admin() -> String { std::env::var("QA_TOKEN_ADMIN").unwrap_or_default() }
21
+ fn token_other() -> String { std::env::var("QA_TOKEN_OTHER_ORG").unwrap_or_default() }
22
+
23
+ fn client() -> Client {
24
+ Client::builder().timeout(std::time::Duration::from_secs(10)).build().unwrap()
25
+ }
26
+
27
+ #[tokio::test]
28
+ async fn happy_path_returns_201() {
29
+ let r = client().post(format!("{}/users", base()))
30
+ .bearer_auth(token_admin())
31
+ .json(&json!({ "email": format!("qa-{}@example.com", uuid::Uuid::new_v4()), "name": "QA" }))
32
+ .send().await.unwrap();
33
+ assert_eq!(r.status(), StatusCode::CREATED);
34
+ let body: Value = r.json().await.unwrap();
35
+ assert!(body.get("id").is_some());
36
+ }
37
+
38
+ #[tokio::test]
39
+ async fn validation_missing_email_returns_422() {
40
+ let r = client().post(format!("{}/users", base()))
41
+ .bearer_auth(token_admin())
42
+ .json(&json!({ "name": "No email" }))
43
+ .send().await.unwrap();
44
+ assert_eq!(r.status(), StatusCode::UNPROCESSABLE_ENTITY);
45
+ let body: Value = r.json().await.unwrap();
46
+ let msg = body["error"]["message"].as_str().unwrap_or_default().to_lowercase();
47
+ assert!(msg.contains("email"));
48
+ }
49
+
50
+ #[tokio::test]
51
+ async fn no_token_returns_401() {
52
+ let r = client().post(format!("{}/users", base()))
53
+ .json(&json!({ "email": "x@y.com", "name": "x" }))
54
+ .send().await.unwrap();
55
+ assert_eq!(r.status(), StatusCode::UNAUTHORIZED);
56
+ }
57
+
58
+ #[tokio::test]
59
+ async fn cross_tenant_denied() {
60
+ if token_other().is_empty() { return; }
61
+ let r = client().get(format!("{}/users/{}", base(), "00000000-0000-0000-0000-000000000001"))
62
+ .bearer_auth(token_other())
63
+ .send().await.unwrap();
64
+ assert!(matches!(r.status(), StatusCode::FORBIDDEN | StatusCode::NOT_FOUND));
65
+ }
66
+
67
+ #[tokio::test]
68
+ async fn contract_required_fields_present_no_leaks() {
69
+ let create = client().post(format!("{}/users", base()))
70
+ .bearer_auth(token_admin())
71
+ .json(&json!({ "email": format!("c-{}@example.com", uuid::Uuid::new_v4()), "name": "C" }))
72
+ .send().await.unwrap();
73
+ let created: Value = create.json().await.unwrap();
74
+ let id = created["id"].as_str().unwrap();
75
+
76
+ let get = client().get(format!("{}/users/{}", base(), id))
77
+ .bearer_auth(token_admin())
78
+ .send().await.unwrap();
79
+ assert_eq!(get.status(), StatusCode::OK);
80
+ let body: Value = get.json().await.unwrap();
81
+
82
+ for f in ["id", "email", "name", "created_at"] { assert!(body.get(f).is_some(), "missing {f}"); }
83
+ for leak in ["password_hash", "internal_id", "_raw"] {
84
+ assert!(body.get(leak).is_none(), "leaked {leak}");
85
+ }
86
+ }
87
+ ```
88
+
89
+ ## File shape (mode B — Axum in-process via `Router::oneshot`)
90
+
91
+ ```rust
92
+ use axum::body::Body;
93
+ use axum::http::{Request, StatusCode};
94
+ use http_body_util::BodyExt;
95
+ use my_app::build_router;
96
+ use serde_json::{json, Value};
97
+ use tower::util::ServiceExt;
98
+
99
+ #[tokio::test]
100
+ async fn happy_path_oneshot() {
101
+ let app = build_router().await;
102
+ let req = Request::post("/users")
103
+ .header("authorization", "Bearer test-admin")
104
+ .header("content-type", "application/json")
105
+ .body(Body::from(json!({"email":"qa@example.com","name":"QA"}).to_string()))
106
+ .unwrap();
107
+ let res = app.oneshot(req).await.unwrap();
108
+ assert_eq!(res.status(), StatusCode::CREATED);
109
+ let bytes = res.into_body().collect().await.unwrap().to_bytes();
110
+ let body: Value = serde_json::from_slice(&bytes).unwrap();
111
+ assert!(body.get("id").is_some());
112
+ }
113
+ ```
114
+
115
+ `build_router()` is the project's exported async function that returns the `axum::Router` — same one used in `main.rs`.
116
+
117
+ ## Configuration
118
+
119
+ `Cargo.toml`:
120
+
121
+ ```toml
122
+ [dev-dependencies]
123
+ reqwest = { version = "0.12", features = ["json", "rustls-tls"] }
124
+ tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
125
+ serde_json = "1"
126
+ uuid = { version = "1", features = ["v4"] }
127
+ # mode B (Axum):
128
+ http-body-util = "0.1"
129
+ tower = { version = "0.5", features = ["util"] }
130
+ ```
131
+
132
+ ## Running
133
+
134
+ ```bash
135
+ # all
136
+ cargo test --test rf_xx_create_user -- --nocapture
137
+
138
+ # log to QA/logs/api/
139
+ cargo test --test rf_xx_create_user -- --nocapture 2>&1 \
140
+ | tee "QA/logs/api/run-$(date +%F).log"
141
+ ```
142
+
143
+ ## Logging request/response
144
+
145
+ `reqwest` doesn't ship a logging middleware out of the box. Two options:
146
+
147
+ - **Wrap at the test layer**: small helper that calls `client.execute(req)` and writes JSONL.
148
+ - **Tower middleware in mode B**: insert a `tower::Layer` that logs request/response. Reuse the project's tracing/logging layer if it has one.
149
+
150
+ ```rust
151
+ use std::fs::OpenOptions;
152
+ use std::io::Write;
153
+
154
+ async fn log_request(method: &str, url: &str, status: u16, ms: u128, body: &str) {
155
+ std::fs::create_dir_all("QA/logs/api").ok();
156
+ let mut f = OpenOptions::new().create(true).append(true)
157
+ .open("QA/logs/api/RF-XX-create-user.log").unwrap();
158
+ let entry = serde_json::json!({
159
+ "ts": chrono::Utc::now().timestamp_millis(),
160
+ "method": method, "url": url, "status": status, "ms": ms,
161
+ "response_body": body,
162
+ });
163
+ writeln!(f, "{entry}").ok();
164
+ }
165
+ ```
166
+
167
+ ## Pros / cons
168
+
169
+ - **Pro (mode B)**: in-process, no port, fastest, deterministic.
170
+ - **Pro**: `tokio::test` integrates with the project's existing `cargo test` flow.
171
+ - **Pro**: type-safe assertions on response bodies.
172
+ - **Con (mode A)**: depends on the server being up before `cargo test`.
173
+ - **Con (mode B)**: requires the project to expose a `build_router()` factory.
@@ -0,0 +1,153 @@
1
+ # Recipe: `supertest` (Node.js / TypeScript)
2
+
3
+ Use for Fastify, Express, NestJS projects that already run `vitest` or `jest`. supertest binds directly to the app instance and runs in-process — no port allocation, no flake.
4
+
5
+ ## File shape
6
+
7
+ `{{PRD_PATH}}/QA/scripts/api/RF-XX-[slug].test.ts`
8
+
9
+ ```ts
10
+ // RF-XX [slug] — API QA suite
11
+ import request from 'supertest';
12
+ import { describe, expect, it, beforeAll } from 'vitest';
13
+ import { buildApp } from '../../../src/app'; // or: import app from '../../../src/server';
14
+
15
+ const BASE = process.env.API_BASE_URL ?? 'http://localhost:3000';
16
+ const TOKEN_ADMIN = process.env.QA_TOKEN_ADMIN ?? '';
17
+ const TOKEN_USER = process.env.QA_TOKEN_USER ?? '';
18
+ const TOKEN_OTHER_ORG = process.env.QA_TOKEN_OTHER_ORG ?? '';
19
+
20
+ let app: Awaited<ReturnType<typeof buildApp>>;
21
+ let createdUserId: string;
22
+
23
+ beforeAll(async () => { app = await buildApp(); });
24
+
25
+ const auth = (t: string) => ({ Authorization: `Bearer ${t}` });
26
+
27
+ describe('RF-XX create user', () => {
28
+
29
+ it('happy path returns 201 and id', async () => {
30
+ const r = await request(app.server).post('/users')
31
+ .set(auth(TOKEN_ADMIN))
32
+ .send({ email: `qa-${Date.now()}@example.com`, name: 'QA' });
33
+ expect(r.status).toBe(201);
34
+ expect(r.body.id).toBeDefined();
35
+ createdUserId = r.body.id;
36
+ });
37
+
38
+ it.each([
39
+ [{ name: 'No email' }, 'email'],
40
+ [{ email: 'no-name@x.com' }, 'name'],
41
+ [{ email: 'not-an-email', name: 'X' }, 'email'],
42
+ ])('validation: %j → mentions %s', async (payload, missing) => {
43
+ const r = await request(app.server).post('/users')
44
+ .set(auth(TOKEN_ADMIN))
45
+ .send(payload);
46
+ expect(r.status).toBe(422);
47
+ expect(r.body.error.message.toLowerCase()).toContain(missing);
48
+ });
49
+
50
+ it('no token returns 401', async () => {
51
+ const r = await request(app.server).post('/users')
52
+ .send({ email: 'x@y.com', name: 'x' });
53
+ expect(r.status).toBe(401);
54
+ });
55
+
56
+ it('expired token returns 401', async () => {
57
+ const r = await request(app.server).post('/users')
58
+ .set({ Authorization: 'Bearer expired.token.here' })
59
+ .send({ email: 'x@y.com', name: 'x' });
60
+ expect(r.status).toBe(401);
61
+ });
62
+
63
+ it('cross-tenant access denied', async () => {
64
+ if (!TOKEN_OTHER_ORG) return;
65
+ const r = await request(app.server).get(`/users/${createdUserId}`)
66
+ .set(auth(TOKEN_OTHER_ORG));
67
+ expect([403, 404]).toContain(r.status);
68
+ });
69
+
70
+ it('contract: required fields present, leaked fields absent', async () => {
71
+ const r = await request(app.server).get(`/users/${createdUserId}`)
72
+ .set(auth(TOKEN_ADMIN));
73
+ expect(r.status).toBe(200);
74
+ for (const f of ['id', 'email', 'name', 'created_at']) {
75
+ expect(r.body[f]).toBeDefined();
76
+ }
77
+ for (const f of ['password_hash', 'internal_id', '_raw']) {
78
+ expect(r.body[f]).toBeUndefined();
79
+ }
80
+ });
81
+ });
82
+ ```
83
+
84
+ ## Configuration
85
+
86
+ `vitest.config.ts`:
87
+
88
+ ```ts
89
+ import { defineConfig } from 'vitest/config';
90
+ export default defineConfig({
91
+ test: {
92
+ include: ['QA/scripts/api/**/*.test.ts'],
93
+ testTimeout: 10_000,
94
+ hookTimeout: 30_000,
95
+ },
96
+ });
97
+ ```
98
+
99
+ ## Running
100
+
101
+ ```bash
102
+ # all
103
+ pnpm vitest run QA/scripts/api
104
+
105
+ # one RF
106
+ pnpm vitest run QA/scripts/api/RF-01-create-user.test.ts
107
+
108
+ # log to QA/logs/api/
109
+ pnpm vitest run QA/scripts/api 2>&1 | tee "QA/logs/api/run-$(date +%F).log"
110
+ ```
111
+
112
+ ## Logging request/response
113
+
114
+ Wrap the supertest agent in a small helper that emits to JSONL:
115
+
116
+ ```ts
117
+ import fs from 'node:fs';
118
+
119
+ const LOG = 'QA/logs/api/RF-XX-create-user.log';
120
+ fs.mkdirSync('QA/logs/api', { recursive: true });
121
+
122
+ function logRequest(method: string, url: string, status: number, ms: number, reqBody: unknown, resBody: unknown) {
123
+ fs.appendFileSync(LOG, JSON.stringify({
124
+ ts: Date.now(), method, url, status, ms, request_body: reqBody, response_body: resBody,
125
+ }) + '\n');
126
+ }
127
+ ```
128
+
129
+ ## NestJS variant
130
+
131
+ Use `@nestjs/testing`'s `Test.createTestingModule(...)` + `app.getHttpServer()` instead of `buildApp`:
132
+
133
+ ```ts
134
+ import { Test } from '@nestjs/testing';
135
+ import { AppModule } from '../../../src/app.module';
136
+
137
+ let app: INestApplication;
138
+ beforeAll(async () => {
139
+ const moduleRef = await Test.createTestingModule({ imports: [AppModule] }).compile();
140
+ app = moduleRef.createNestApplication();
141
+ await app.init();
142
+ });
143
+
144
+ // then: request(app.getHttpServer()) ...
145
+ ```
146
+
147
+ ## Pros / cons
148
+
149
+ - **Pro**: in-process, no port allocation, fastest possible.
150
+ - **Pro**: integrates with `vitest`/`jest` watch + coverage.
151
+ - **Pro**: `it.each` covers the 4xx matrix in one block.
152
+ - **Con**: only works against a `supertest`-compatible HTTP framework.
153
+ - **Con**: requires the `buildApp` factory pattern; one-off scripts/handlers may need refactor.
@@ -0,0 +1,138 @@
1
+ # Auth patterns — how to wire credentials into API tests
2
+
3
+ Tests need real credentials, but credentials must never live in the script files (which are committed). This file describes how each recipe handles the four common auth schemes and where credentials come from.
4
+
5
+ ## The four schemes
6
+
7
+ | Scheme | How it travels | Recipe handling |
8
+ |--------|----------------|-----------------|
9
+ | **Bearer JWT** | `Authorization: Bearer <token>` | Most common. Token comes from a login response or pre-issued for QA. |
10
+ | **Cookie session** | `Cookie: session=<sid>` (set by `Set-Cookie` on login) | Recipes capture the cookie from a login call and replay it. |
11
+ | **API key** | `X-API-Key: <key>` (header) or `?api_key=<key>` (query) | Header form is preferred; key comes from a per-environment env var. |
12
+ | **Basic auth** | `Authorization: Basic <base64(user:pass)>` | Rare in modern APIs; supported but discouraged. |
13
+
14
+ ## Where credentials come from (in priority order)
15
+
16
+ 1. **`.env` file** at the repo root, gitignored. Contains `QA_TOKEN_ADMIN`, `QA_ADMIN_EMAIL`, `QA_ADMIN_PASSWORD`, etc.
17
+ 2. **Pre-issued QA tokens** — long-lived JWTs minted by an admin tool (e.g., a `make qa-tokens` target) and stored in `.env`. Best for CI; avoids login-time flake.
18
+ 3. **Login at runtime** — a setup request hits `/auth/login` with `QA_ADMIN_EMAIL` + `QA_ADMIN_PASSWORD` and captures the token. Use when no pre-issued option exists.
19
+ 4. **`.dw/templates/qa-test-credentials.md`** — the project-level QA credentials registry that `dw-run-qa` already reads (UI mode). API mode reads the same file for env-var hints + role mapping.
20
+
21
+ ## Three roles every project should have
22
+
23
+ Even for single-tenant apps, define at minimum:
24
+
25
+ - **`token_admin`** — has every permission. Used for setup (create test data) and teardown.
26
+ - **`token_user`** — regular authenticated user. The role most happy-path tests run as.
27
+ - **`token_guest`** OR **`token_other_org_admin`** — for negative tests. In multi-tenant apps, this token belongs to a different org and powers the cross-tenant denial tests.
28
+
29
+ ## Per-recipe variable conventions
30
+
31
+ ### `.http` (REST Client)
32
+
33
+ Top of the file:
34
+
35
+ ```http
36
+ @base = {{$dotenv API_BASE_URL}}
37
+ @token_admin = {{$dotenv QA_TOKEN_ADMIN}}
38
+ @token_user = {{$dotenv QA_TOKEN_USER}}
39
+ @token_other_org = {{$dotenv QA_TOKEN_OTHER_ORG}}
40
+ ```
41
+
42
+ Or, if logging in at runtime, capture once and reuse:
43
+
44
+ ```http
45
+ ### Setup — login as admin
46
+ # @name login_admin
47
+ POST {{base}}/auth/login
48
+ Content-Type: application/json
49
+ { "email": "{{$dotenv QA_ADMIN_EMAIL}}", "password": "{{$dotenv QA_ADMIN_PASSWORD}}" }
50
+
51
+ > {%
52
+ client.global.set("token_admin", response.body.access_token);
53
+ client.test("login ok", () => client.assert(response.status === 200));
54
+ %}
55
+ ```
56
+
57
+ ### `pytest + httpx`
58
+
59
+ Read from environment in module scope; expose as fixtures if the test count grows:
60
+
61
+ ```python
62
+ TOKEN_ADMIN = os.environ["QA_TOKEN_ADMIN"]
63
+ TOKEN_USER = os.environ["QA_TOKEN_USER"]
64
+ TOKEN_OTHER_ORG = os.environ.get("QA_TOKEN_OTHER_ORG", "")
65
+
66
+ @pytest.fixture(scope="session")
67
+ async def admin_client():
68
+ async with httpx.AsyncClient(base_url=BASE,
69
+ headers={"Authorization": f"Bearer {TOKEN_ADMIN}"},
70
+ timeout=10.0) as c:
71
+ yield c
72
+ ```
73
+
74
+ ### `supertest` (Node)
75
+
76
+ Same `process.env` reads, optionally one helper per role:
77
+
78
+ ```ts
79
+ const auth = (token: string) => ({ Authorization: `Bearer ${token}` });
80
+ const TOKEN_ADMIN = process.env.QA_TOKEN_ADMIN!;
81
+ ```
82
+
83
+ ### `WebApplicationFactory` (.NET)
84
+
85
+ Subclass the factory once per role:
86
+
87
+ ```csharp
88
+ public class AdminAppFactory : WebApplicationFactory<Program>
89
+ {
90
+ protected override void ConfigureClient(HttpClient client)
91
+ {
92
+ client.DefaultRequestHeaders.Authorization =
93
+ new AuthenticationHeaderValue("Bearer",
94
+ Environment.GetEnvironmentVariable("QA_TOKEN_ADMIN") ?? "");
95
+ }
96
+ }
97
+ ```
98
+
99
+ ### `reqwest` (Rust)
100
+
101
+ Helper functions read env once:
102
+
103
+ ```rust
104
+ fn token_admin() -> String { std::env::var("QA_TOKEN_ADMIN").unwrap_or_default() }
105
+ fn admin_client() -> reqwest::Client {
106
+ reqwest::Client::builder().build().unwrap()
107
+ }
108
+ // then: admin_client().get(url).bearer_auth(token_admin()).send().await
109
+ ```
110
+
111
+ ## Refresh tokens
112
+
113
+ If the API uses refresh tokens, capture both `access_token` and `refresh_token` in the login setup. When a test needs a long-lived flow (e.g., wait for a webhook), refresh the access token before the wait.
114
+
115
+ For most QA suites, the access token's TTL (typically 15-60 min) is longer than the suite's runtime, so refresh is unnecessary.
116
+
117
+ ## Scoped credentials per role
118
+
119
+ For RBAC-heavy systems, define more roles:
120
+
121
+ - `token_admin` — global admin
122
+ - `token_org_admin` — admin within one org
123
+ - `token_member` — regular member of one org
124
+ - `token_billing` — read-only billing access
125
+ - `token_other_org_admin` — admin of a different org (for cross-tenant tests)
126
+
127
+ Add one env var per role; the recipe reads them as needed. Tests that don't need a particular role just don't reference it.
128
+
129
+ ## Anti-patterns
130
+
131
+ - **Don't hardcode `Bearer eyJ...` in any committed file.** Even "test" tokens leak.
132
+ - **Don't share one token across happy-path AND negative tests.** If a happy-path test mutates the token's user (e.g., suspends it), every later test fails.
133
+ - **Don't reuse production tokens for QA.** Mint QA-only tokens with a clearly distinct subject (`sub: qa-admin@example.com`).
134
+ - **Don't pass credentials via command-line args.** They land in shell history and process listings.
135
+
136
+ ## What `dw-run-qa` does
137
+
138
+ In API mode, `/dw-run-qa` reads `QA/test-credentials.md` (or `.env`) for the env var names, picks the recipe, and substitutes variables at test-generation time. The script files reference `@variable` references only — never raw tokens.
@@ -0,0 +1,117 @@
1
+ # Log conventions — request/response evidence as JSONL
2
+
3
+ In API mode, **logs replace screenshots** as the primary QA evidence. Every request/response pair the QA suite makes is captured as one JSONL line so the bug report links back to a reproducible event.
4
+
5
+ ## File location
6
+
7
+ `{{PRD_PATH}}/QA/logs/api/<scope>.log`
8
+
9
+ Where `<scope>` is one of:
10
+
11
+ - `RF-XX-[slug].log` — log for a single requirement run (1 file per RF).
12
+ - `BUG-NN-retest.log` — log for a fix retest (1 file per bug retest cycle).
13
+ - `run-<YYYY-MM-DD>.log` — global run log (full QA pass).
14
+
15
+ ## Line shape (JSONL — one JSON object per line)
16
+
17
+ ```json
18
+ {
19
+ "ts": 1715000000000,
20
+ "rf": "RF-03",
21
+ "case": "happy-path",
22
+ "method": "POST",
23
+ "url": "http://localhost:3000/users",
24
+ "request_headers": {
25
+ "authorization": "Bearer <redacted>",
26
+ "content-type": "application/json"
27
+ },
28
+ "request_body": {
29
+ "email": "qa-1@example.com",
30
+ "name": "QA"
31
+ },
32
+ "status": 201,
33
+ "response_headers": {
34
+ "content-type": "application/json",
35
+ "location": "/users/12345"
36
+ },
37
+ "response_body": {
38
+ "id": "12345",
39
+ "email": "qa-1@example.com",
40
+ "name": "QA",
41
+ "created_at": "2026-05-06T12:00:00Z"
42
+ },
43
+ "ms": 47,
44
+ "verdict": "PASS",
45
+ "assertion_failures": []
46
+ }
47
+ ```
48
+
49
+ ## Required fields
50
+
51
+ | Field | Type | Notes |
52
+ |-------|------|-------|
53
+ | `ts` | int (epoch ms, UTC) | When the request was sent |
54
+ | `rf` | string | Which `RF-XX` this request belongs to (or `"BUG-NN"` for retests) |
55
+ | `case` | string | One of `happy-path`, `validation`, `auth-missing`, `auth-expired`, `authz-cross-tenant`, `not-found`, `conflict`, `server-error`, `contract` |
56
+ | `method` | string | HTTP method |
57
+ | `url` | string | Full URL including query string |
58
+ | `status` | int | HTTP status code |
59
+ | `ms` | int | Elapsed milliseconds |
60
+ | `verdict` | string | `"PASS"` or `"FAIL"` |
61
+ | `assertion_failures` | array of strings | Each failed assertion as a one-line description (empty array on PASS) |
62
+
63
+ ## Optional fields
64
+
65
+ | Field | Type | Notes |
66
+ |-------|------|-------|
67
+ | `request_headers` | object | Map of header name → value |
68
+ | `request_body` | any | Parsed JSON if `Content-Type: application/json`; raw string otherwise |
69
+ | `response_headers` | object | Same shape as request_headers |
70
+ | `response_body` | any | Parsed JSON if `Content-Type: application/json`; raw string otherwise |
71
+ | `err` | string | Network/runtime error message (if no response was received at all) |
72
+
73
+ ## Redaction rules
74
+
75
+ The log goes to `QA/logs/api/` which **may end up in artifacts uploaded to CI** or attached to bug reports. Redact:
76
+
77
+ - **`Authorization` header** → `"Bearer <redacted>"` or `"Basic <redacted>"`. The token's presence is logged; the value never is.
78
+ - **`Cookie` header** → `"<redacted>"`. Same reasoning.
79
+ - **`X-API-Key` header** → `"<redacted>"`.
80
+ - **Response fields named `password*`, `secret*`, `*_hash`, `token*`, `apiKey*`** → `"<redacted>"`. These should never be in a response anyway; if they are, the log redacts AND the QA report flags the leak.
81
+ - **Free-form `request_body` fields named `password`** → `"<redacted>"`.
82
+
83
+ The redaction is applied at log-write time, never on read; even a leaked log file should not expose secrets.
84
+
85
+ ## Why JSONL (not pretty-printed JSON)
86
+
87
+ - **Append-friendly**: each request is one line; concurrent runs append safely without parsing the whole file.
88
+ - **Greppable**: `grep '"verdict":"FAIL"' QA/logs/api/RF-03.log` shows every failed case in one shot.
89
+ - **Queryable**: `jq -c 'select(.status >= 500)' QA/logs/api/run-*.log | jq -s 'group_by(.url) | map({url: .[0].url, count: length})'` finds the most-failing URLs.
90
+ - **Diffable across runs**: `diff <(jq -c 'del(.ts, .ms)' RF-03.log) <(jq -c 'del(.ts, .ms)' RF-03.log.prev)` shows behavior changes free of timing noise.
91
+
92
+ ## Per-recipe writers
93
+
94
+ Every recipe in `recipes/` includes a small writer helper in its example:
95
+
96
+ - `.http` — the agent writes via `Bash` after each `curl` invocation.
97
+ - `pytest+httpx` — `LoggingClient` subclass overriding `request`.
98
+ - `supertest` — small `logRequest` helper imported by tests.
99
+ - `.NET WebApplicationFactory` — `DelegatingHandler` registered on the test client.
100
+ - `reqwest` — wrapper function around `client.execute(req)`.
101
+
102
+ All of them produce the same JSONL shape so downstream tooling (the QA report renderer, the bug retest loop) doesn't care which recipe was used.
103
+
104
+ ## How `dw-run-qa` reads logs back
105
+
106
+ When generating the QA report (Step 8 in `dw-run-qa`), the agent reads each `RF-XX-[slug].log`, computes:
107
+
108
+ - **Total requests** per RF
109
+ - **Pass count vs fail count**
110
+ - **Failing cases** with the assertion message
111
+ - **Tail latency** (p99 if there are ≥10 requests, max otherwise)
112
+
113
+ These land in the report's "Verified Requirements" table and feed the bug entries (with `evidence_path: QA/logs/api/RF-03.log#L42` pointing to the failing line).
114
+
115
+ ## How `dw-fix-qa` consumes them
116
+
117
+ The retest loop reads `QA/bugs.md` for each open bug, finds the corresponding log line via `evidence_path`, replays the request via the same recipe + assertions, and writes a new line to `BUG-NN-retest.log` with `verdict: "PASS"` (closing the bug) or `verdict: "FAIL"` (cycling through the fix-retest loop again, max 5 cycles).
@@ -0,0 +1,68 @@
1
+ # Matrix conventions — deriving tests from a PRD requirement
2
+
3
+ Every API requirement (`RF-XX`) gets a structured matrix of test cases. The matrix is the bridge between "the PRD says this endpoint must exist" and "we have evidence it works under the cases that matter."
4
+
5
+ ## The five tiers
6
+
7
+ For each `RF-XX`, generate at least one test per tier that applies:
8
+
9
+ | Tier | Goal | When to skip |
10
+ |------|------|--------------|
11
+ | **200 happy path** | Prove the endpoint accepts the documented input and returns the documented output. | Never — every RF needs at least one happy path. |
12
+ | **4xx — validation** | Prove input validation rejects malformed payloads with a useful error. | Skip only for endpoints with no body (`GET` without query params). |
13
+ | **4xx — auth (401)** | Prove missing/expired/invalid credentials return 401. | Skip for endpoints documented as anonymous. |
14
+ | **4xx — authorization (403)** | Prove valid credentials without the required role/scope return 403. | Skip if the endpoint is open to any authenticated user. |
15
+ | **4xx — not found (404)** | Prove non-existent IDs return 404, not 500. | Skip for endpoints that don't take an ID. |
16
+ | **4xx — conflict (409)** | Prove duplicates / version mismatches return 409. | Skip if the endpoint is idempotent and conflict-free by design. |
17
+ | **5xx — server error** | Prove the system fails gracefully (no leaked stack trace, no half-write). | Skip if no synthetic failure is reproducible without invasive infrastructure changes. |
18
+ | **Contract drift** | Prove the response shape matches the documented spec (OpenAPI, TS types, README examples). | Never — this is the cheapest way to catch silent breakage. |
19
+ | **Authorization cross-tenant** | Prove tokens from org A cannot access data of org B. | Skip only for single-tenant systems (rare in practice). |
20
+
21
+ ## Why the cross-tenant test is mandatory
22
+
23
+ Cross-tenant data leakage is the most damaging API bug class — it's silent (no error), undetected by happy-path tests, and lethal in B2B SaaS. Every endpoint that returns or mutates tenant-scoped data must have a cross-tenant denial test. If the project is single-tenant, mark the test `pytest.skip` / `it.skip` / `[Fact(Skip="single-tenant")]` instead of omitting — the explicit skip is a record of the decision.
24
+
25
+ ## How to enumerate inputs per tier
26
+
27
+ For each tier, ask:
28
+
29
+ - **200**: what's the minimum valid payload? Build the test around that. Add 2-3 variations only if the endpoint has interesting branching (nullable fields, enum variants, optional sections).
30
+ - **4xx validation**: what fields are required? Drop each one. What types are constrained? Send the wrong type. What ranges? Test min-1 and max+1. Don't test all combinations — one per kind of constraint is enough.
31
+ - **4xx auth**: 3 variants — no token, expired token, malformed token. One test for each is enough.
32
+ - **4xx authorization**: identify role boundaries (admin vs user vs guest, owner vs member). One test per boundary.
33
+ - **4xx not found**: 1 test with a syntactically-valid-but-nonexistent ID (UUID, integer, etc.).
34
+ - **4xx conflict**: 1 test that triggers the documented conflict (duplicate email, race on version).
35
+ - **5xx**: skip if not reproducible. If the project has a way to inject failures (chaos hooks, dev-only error endpoints), use them.
36
+ - **Contract drift**: 1 test that asserts every documented field is present AND no leaked internal field is.
37
+ - **Cross-tenant**: 1 test per tenant-scoped endpoint with a token from a different tenant.
38
+
39
+ ## Example expansion: `POST /users`
40
+
41
+ PRD says: "RF-03 — admins can create users. Validation: email is required and must be unique. Returns 201 with the new user."
42
+
43
+ Matrix:
44
+
45
+ | # | Tier | Case | Expected |
46
+ |---|------|------|----------|
47
+ | 1 | 200 | admin creates user with valid payload | 201, body has id |
48
+ | 2 | 4xx validation | missing email | 422, error mentions email |
49
+ | 3 | 4xx validation | invalid email format | 422 |
50
+ | 4 | 4xx auth | no token | 401 |
51
+ | 5 | 4xx auth | expired token | 401 |
52
+ | 6 | 4xx authorization | regular user (not admin) | 403 |
53
+ | 7 | 4xx conflict | email already taken | 409 |
54
+ | 8 | Contract | all required fields present, no `password_hash` | matches spec |
55
+ | 9 | Cross-tenant | admin from another org tries to fetch this user | 403 or 404 |
56
+
57
+ That's 9 test cases for one RF — the floor for a real API surface, not the ceiling.
58
+
59
+ ## What NOT to do
60
+
61
+ - **Don't test every combination** of validation failures. The framework already enforces type + presence; one test per kind of constraint is the signal.
62
+ - **Don't test the framework**. `Content-Type: application/json` parsing, default routing, etc. — those belong to FastAPI / Fastify / ASP.NET, not to your QA suite.
63
+ - **Don't write tests for endpoints with no PRD reference**. If a route exists but no RF describes it, that's a documentation gap to flag, not a test to add.
64
+ - **Don't skip 5xx because "it shouldn't happen"**. If you have a way to reproduce, do it. If you genuinely can't, document the skip in the QA report so the gap is visible.
65
+
66
+ ## How `dw-run-qa` uses this
67
+
68
+ When in API mode, `/dw-run-qa` walks each `RF-XX` in the PRD, runs through this matrix, and emits PASS/FAIL per RF — not per test case. A single FAIL in any tier marks the RF as FAIL and lands a `BUG-NN` entry pointing to the failing log line.