comber 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +18 -0
- package/README.md +732 -0
- package/action.yml +279 -0
- package/comber.config.example.json +64 -0
- package/dist/api/classify.js +99 -0
- package/dist/api/classify.js.map +1 -0
- package/dist/api/driver.js +235 -0
- package/dist/api/driver.js.map +1 -0
- package/dist/api/graphql-driver.js +150 -0
- package/dist/api/graphql-driver.js.map +1 -0
- package/dist/api/graphql.js +115 -0
- package/dist/api/graphql.js.map +1 -0
- package/dist/api/har-driver.js +167 -0
- package/dist/api/har-driver.js.map +1 -0
- package/dist/api/har.js +100 -0
- package/dist/api/har.js.map +1 -0
- package/dist/api/load.js +83 -0
- package/dist/api/load.js.map +1 -0
- package/dist/api/openapi.js +159 -0
- package/dist/api/openapi.js.map +1 -0
- package/dist/api/request.js +141 -0
- package/dist/api/request.js.map +1 -0
- package/dist/api/schema.js +126 -0
- package/dist/api/schema.js.map +1 -0
- package/dist/cli.js +352 -0
- package/dist/cli.js.map +1 -0
- package/dist/comment.js +164 -0
- package/dist/comment.js.map +1 -0
- package/dist/config.js +664 -0
- package/dist/config.js.map +1 -0
- package/dist/connectors/builtin/discord.js +55 -0
- package/dist/connectors/builtin/github-checks.js +53 -0
- package/dist/connectors/builtin/junit.js +70 -0
- package/dist/connectors/builtin/linear.js +58 -0
- package/dist/connectors/builtin/slack.js +60 -0
- package/dist/connectors/builtin/webhook.js +39 -0
- package/dist/connectors/index.js +3 -0
- package/dist/connectors/loader.js +72 -0
- package/dist/connectors/registry.js +28 -0
- package/dist/connectors/types.js +23 -0
- package/dist/core/action-gate.js +48 -0
- package/dist/core/baseline.js +85 -0
- package/dist/core/baseline.js.map +1 -0
- package/dist/core/blind-spots.js +22 -0
- package/dist/core/blind-spots.js.map +1 -0
- package/dist/core/contract-extras.js +1 -0
- package/dist/core/contract-extras.js.map +1 -0
- package/dist/core/contract.js +1 -0
- package/dist/core/contract.js.map +1 -0
- package/dist/core/coverage.js +70 -0
- package/dist/core/coverage.js.map +1 -0
- package/dist/core/crawl.js +229 -0
- package/dist/core/crawl.js.map +1 -0
- package/dist/core/egress.js +128 -0
- package/dist/core/egress.js.map +1 -0
- package/dist/core/ledger.js +41 -0
- package/dist/core/ledger.js.map +1 -0
- package/dist/core/net.js +39 -0
- package/dist/core/pipeline.js +191 -0
- package/dist/core/pipeline.js.map +1 -0
- package/dist/core/plan.js +121 -0
- package/dist/core/plan.js.map +1 -0
- package/dist/core/redact.js +240 -0
- package/dist/core/redact.js.map +1 -0
- package/dist/core/replay.js +263 -0
- package/dist/core/replay.js.map +1 -0
- package/dist/core/report.js +253 -0
- package/dist/core/report.js.map +1 -0
- package/dist/core/safety-judge.js +116 -0
- package/dist/core/safety.js +92 -0
- package/dist/core/safety.js.map +1 -0
- package/dist/core/signal-normalize.js +190 -0
- package/dist/core/signal-normalize.js.map +1 -0
- package/dist/core/signals.js +5 -0
- package/dist/core/signals.js.map +1 -0
- package/dist/core/triage.js +104 -0
- package/dist/core/triage.js.map +1 -0
- package/dist/core/types.js +1 -0
- package/dist/core/types.js.map +1 -0
- package/dist/desktop/desktop-profile.js +6 -0
- package/dist/desktop/desktop-profile.js.map +1 -0
- package/dist/desktop/driver.js +433 -0
- package/dist/desktop/driver.js.map +1 -0
- package/dist/desktop/evidence.js +206 -0
- package/dist/desktop/evidence.js.map +1 -0
- package/dist/desktop/macos/accessibility-runner.js +118 -0
- package/dist/desktop/macos/accessibility-runner.js.map +1 -0
- package/dist/desktop/macos/types.js +1 -0
- package/dist/desktop/macos/types.js.map +1 -0
- package/dist/desktop/preflight.js +107 -0
- package/dist/desktop/preflight.js.map +1 -0
- package/dist/desktop/proxy-proof.js +185 -0
- package/dist/desktop/proxy-proof.js.map +1 -0
- package/dist/desktop/view.js +348 -0
- package/dist/desktop/view.js.map +1 -0
- package/dist/desktop/windows/types.js +1 -0
- package/dist/desktop/windows/types.js.map +1 -0
- package/dist/desktop/windows/uia-runner.js +127 -0
- package/dist/desktop/windows/uia-runner.js.map +1 -0
- package/dist/native/apple.js +14 -0
- package/dist/native/apple.js.map +1 -0
- package/dist/native/device-profile.js +6 -0
- package/dist/native/device-profile.js.map +1 -0
- package/dist/native/driver.js +416 -0
- package/dist/native/driver.js.map +1 -0
- package/dist/native/evidence.js +164 -0
- package/dist/native/evidence.js.map +1 -0
- package/dist/native/load.js +9 -0
- package/dist/native/load.js.map +1 -0
- package/dist/native/maestro.js +153 -0
- package/dist/native/maestro.js.map +1 -0
- package/dist/native/preflight.js +72 -0
- package/dist/native/preflight.js.map +1 -0
- package/dist/native/proxy-proof.js +182 -0
- package/dist/native/proxy-proof.js.map +1 -0
- package/dist/native/samsung.js +34 -0
- package/dist/native/samsung.js.map +1 -0
- package/dist/native/signals.js +50 -0
- package/dist/native/signals.js.map +1 -0
- package/dist/native/view.js +139 -0
- package/dist/native/view.js.map +1 -0
- package/dist/profile.js +237 -0
- package/dist/profile.js.map +1 -0
- package/dist/sink.js +134 -0
- package/dist/sink.js.map +1 -0
- package/dist/sinks/file.js +49 -0
- package/dist/sinks/file.js.map +1 -0
- package/dist/sinks/http.js +38 -0
- package/dist/sinks/http.js.map +1 -0
- package/dist/sinks/index.js +5 -0
- package/dist/sinks/index.js.map +1 -0
- package/dist/sinks/multi.js +40 -0
- package/dist/sinks/multi.js.map +1 -0
- package/dist/sinks/paths.js +32 -0
- package/dist/sinks/paths.js.map +1 -0
- package/dist/sinks/record.js +58 -0
- package/dist/sinks/record.js.map +1 -0
- package/dist/sinks/types.js +1 -0
- package/dist/sinks/types.js.map +1 -0
- package/dist/web/breakage.js +210 -0
- package/dist/web/breakage.js.map +1 -0
- package/dist/web/controls.js +102 -0
- package/dist/web/controls.js.map +1 -0
- package/dist/web/driver.js +803 -0
- package/dist/web/driver.js.map +1 -0
- package/dist/web/elements.js +151 -0
- package/dist/web/elements.js.map +1 -0
- package/dist/web/fingerprint.js +13 -0
- package/dist/web/fingerprint.js.map +1 -0
- package/dist/web/forms.js +74 -0
- package/dist/web/forms.js.map +1 -0
- package/dist/web/links.js +194 -0
- package/dist/web/links.js.map +1 -0
- package/dist/web/types.js +1 -0
- package/dist/web/types.js.map +1 -0
- package/package.json +80 -0
- package/runtime/macos-accessibility-runner.js +275 -0
- package/runtime/windows-uia-runner.ps1 +539 -0
package/README.md
ADDED
|
@@ -0,0 +1,732 @@
|
|
|
1
|
+
# 🦞 Comber
|
|
2
|
+
|
|
3
|
+
**Every-click AI QA agent.** Point it at a URL and it walks the app like a real user,
|
|
4
|
+
clicks what's safe, watches for breakage, and writes a self-contained HTML report —
|
|
5
|
+
leaving cheap deterministic replays behind so the expensive every-click run happens once.
|
|
6
|
+
|
|
7
|
+
**Comber** by RADLAB — product site **[codecomber.io](https://codecomber.io)**. CLI/package: `comber`.
|
|
8
|
+
|
|
9
|
+
> **Status: Phase 0 scaffold (2026-06-24).** Runnable DOM-first web crawler with breakage
|
|
10
|
+
> detection, optional Claude action-picker + judge, and an HTML report. Four drivers now sit
|
|
11
|
+
> behind the universal contract — **web** (Playwright), **api** (OpenAPI), **native**
|
|
12
|
+
> (iOS/Android over a pluggable Maestro runner), and **desktop** (Windows UI Automation
|
|
13
|
+
> or macOS Accessibility).
|
|
14
|
+
> The rest (Stagehand selector-cache replay polish, GitHub Action on Vercel preview deploys) is
|
|
15
|
+
> staged — see **Roadmap** below.
|
|
16
|
+
|
|
17
|
+
## Why DOM-first
|
|
18
|
+
|
|
19
|
+
The accessibility/DOM snapshot is ~200–400 tokens per step with stable element refs; a
|
|
20
|
+
screenshot is ~1,000–1,800 image tokens per step. So Comber drives the browser off the DOM
|
|
21
|
+
(Playwright today, Stagehand selector-cache next) and reserves Claude **vision/computer-use**
|
|
22
|
+
for the few surfaces the DOM can't describe (canvas, file-preview, drag-drop, visual diff).
|
|
23
|
+
That keeps a full crawl in the cents, deterministic, and CI-safe.
|
|
24
|
+
|
|
25
|
+
## Install
|
|
26
|
+
|
|
27
|
+
Comber ships as the **`comber`** package (CLI + reusable GitHub Action). The published
|
|
28
|
+
tarball carries a prebuilt `dist/` plus `action.yml`, so no build step is needed on a normal
|
|
29
|
+
npm/bun install.
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
# As a project dependency (bun or npm):
|
|
33
|
+
bun add comber
|
|
34
|
+
npm i -g comber # or install globally for a `comber` on your PATH
|
|
35
|
+
|
|
36
|
+
# One-off, no install (downloads + runs):
|
|
37
|
+
bunx comber https://example.com
|
|
38
|
+
|
|
39
|
+
# Web crawls need a Chromium for Playwright (once per machine / CI runner):
|
|
40
|
+
bunx playwright install chromium
|
|
41
|
+
|
|
42
|
+
# Then run:
|
|
43
|
+
comber https://example.com # heuristic crawl, no API key needed
|
|
44
|
+
comber https://example.com --max-states=30 --llm # Claude action-picker + judge
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
**Pre-publish / private install (no npm release yet).** Until `comber` is published to npm
|
|
48
|
+
(see the **Publish gate** under *CI integration*), install straight from the repo — the
|
|
49
|
+
package's `prepare` script builds `dist/` on install, so a git ref works the same as the
|
|
50
|
+
published package:
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
bun add github:RADLABtech/comber#<ref> # <ref> = a tag (v0.1.0), branch, or commit SHA
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
## Run from source
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
cd ~/repos/comber
|
|
60
|
+
bun install
|
|
61
|
+
bunx playwright install chromium
|
|
62
|
+
|
|
63
|
+
# Heuristic crawl (no API key needed):
|
|
64
|
+
bun run dev https://example.com
|
|
65
|
+
|
|
66
|
+
# LLM-driven (Haiku picks actions, Opus judges findings):
|
|
67
|
+
cp .env.example .env && echo "ANTHROPIC_API_KEY=sk-ant-..." >> .env
|
|
68
|
+
bun run dev https://example.com --max-states=30 --llm
|
|
69
|
+
|
|
70
|
+
# Authenticated crawl against a SYNTHETIC account:
|
|
71
|
+
# COMBER_STORAGE_STATE=./auth.json bun run dev https://example.com
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
### Generating a synthetic session
|
|
75
|
+
|
|
76
|
+
An authenticated crawl needs a Playwright `storageState` (cookies + localStorage)
|
|
77
|
+
for a **synthetic / throwaway** account — never a real user's. Two helpers produce it:
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
# Interactive: opens a browser, you log in, press ENTER to capture.
|
|
81
|
+
bun run save-session https://example.com/login auth.json
|
|
82
|
+
|
|
83
|
+
# Headless (CI-friendly): credentials from the environment, never the CLI.
|
|
84
|
+
SYNTH_EMAIL=you@synthetic.example SYNTH_PW=… bun run login https://example.com/login auth.json
|
|
85
|
+
# …or read the password from a file instead of $SYNTH_PW:
|
|
86
|
+
SYNTH_EMAIL=you@synthetic.example bun run login https://example.com/login auth.json ./pw.txt
|
|
87
|
+
|
|
88
|
+
# Then crawl with it:
|
|
89
|
+
COMBER_STORAGE_STATE=auth.json bun run dev https://example.com
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
The saved file holds live session tokens — `.gitignore` already covers
|
|
93
|
+
`auth*.json` / `*-auth.json` / `auth/`. Keep it there.
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
Flags: `--max-states=N` `--max-actions=N` `--max-ms=N` `--max-tokens=N` `--headed`
|
|
97
|
+
`--llm` `--cross-origin` `--external-links` `--allow-writes` `--baseline=PATH`
|
|
98
|
+
`--pr-comment=PATH` `--config=PATH` `--profile=NAME` `--strict` (exit 1 on any
|
|
99
|
+
fail-severity finding; default is soft).
|
|
100
|
+
|
|
101
|
+
API target (`comber api …`): `--openapi=PATH|URL` (JSON or YAML) `--api-base=URL`
|
|
102
|
+
`--graphql=URL` `--har=PATH` `--seed=PATH` `--allow-writes`. Credentials are env-only:
|
|
103
|
+
`COMBER_API_TOKEN` (single Bearer) and `COMBER_API_AUTH_PROFILES` (a JSON `{role:token}`
|
|
104
|
+
map for the multi-role IDOR/broken-auth harness). See **API driver** below.
|
|
105
|
+
|
|
106
|
+
## Profiles, baselines, and PR comments
|
|
107
|
+
|
|
108
|
+
Profiles keep per-app crawl settings out of CI YAML:
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
cp comber.config.example.json comber.config.json
|
|
112
|
+
bun run dev --profile local --config comber.config.json
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
`comber.config.json` can be either one direct config object or a named `profiles` object
|
|
116
|
+
with `defaultProfile`. Supported aliases include `url`/`startUrl`, `maxMs`/`maxWallClockMs`,
|
|
117
|
+
`maxTokens`/`maxTokenBudget`, `maxPerState`/`maxActionsPerState`, `maxRegotos`/
|
|
118
|
+
`maxRegotosPerState`, `nearDup`/`nearDupThreshold`, `storageState`/`storageStatePath`,
|
|
119
|
+
`baseline`/`baselinePath`, and `prComment`/`prCommentPath`.
|
|
120
|
+
|
|
121
|
+
Baseline diffing compares current findings by stable `signature` against a prior
|
|
122
|
+
`result.json`:
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
bun run dev https://preview.example.com --baseline baselines/preview-result.json
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
When no baseline exists, Comber treats all current findings as new and keeps running. To
|
|
129
|
+
produce a Markdown body for a non-blocking pull-request comment:
|
|
130
|
+
|
|
131
|
+
```bash
|
|
132
|
+
bun run dev https://preview.example.com --baseline baselines/preview-result.json --pr-comment runs/pr-comment.md
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
### Informational findings
|
|
136
|
+
|
|
137
|
+
Comber reports `fail`, `warn`, and `info` severities. Strict gates and baseline headline
|
|
138
|
+
counts use only `fail` and `warn`; `info` findings are visible context that should not block
|
|
139
|
+
a run.
|
|
140
|
+
|
|
141
|
+
The web crawler normalizes crawler-caused noise before dedupe:
|
|
142
|
+
|
|
143
|
+
- Comber-blocked egress fan-out (`egress-blocked` plus matching request/console/fetch errors)
|
|
144
|
+
becomes one `crawler-suppressed` info finding.
|
|
145
|
+
- Known telemetry infrastructure such as Cloudflare Insights, Statsig, Sentry ingest, and
|
|
146
|
+
Microsoft browser events becomes `third-party-telemetry` info.
|
|
147
|
+
- Turnstile, hCaptcha, and Cloudflare challenge endpoints or widget errors become
|
|
148
|
+
`bot-challenge` info findings.
|
|
149
|
+
- `.ico` images are not marked `broken-image` solely because browser decode reports
|
|
150
|
+
`naturalWidth === 0`.
|
|
151
|
+
|
|
152
|
+
The bundled manual workflow (`.github/workflows/comber.yml`) runs Comber from source. It
|
|
153
|
+
uploads `runs/` as an artifact and can optionally post `runs/pr-comment.md` to a PR when
|
|
154
|
+
`pr_number` is supplied. For dropping Comber into *another* repo, use the reusable Action
|
|
155
|
+
below.
|
|
156
|
+
|
|
157
|
+
## CI integration (reusable Action)
|
|
158
|
+
|
|
159
|
+
`action.yml` at the repo root is a **composite GitHub Action** so any repo can run Comber on
|
|
160
|
+
a preview deploy. It sets up bun → installs comber → runs it → uploads the `runs/` report
|
|
161
|
+
artifact → posts a non-blocking PR comment → and gates the job only when `strict` and
|
|
162
|
+
fail-severity findings are present.
|
|
163
|
+
|
|
164
|
+
```yaml
|
|
165
|
+
# .github/workflows/comber-pr.yml in the consumer repo
|
|
166
|
+
name: comber
|
|
167
|
+
on:
|
|
168
|
+
pull_request:
|
|
169
|
+
types: [opened, synchronize, reopened]
|
|
170
|
+
permissions:
|
|
171
|
+
contents: read
|
|
172
|
+
pull-requests: write
|
|
173
|
+
jobs:
|
|
174
|
+
comber:
|
|
175
|
+
runs-on: ubuntu-latest
|
|
176
|
+
steps:
|
|
177
|
+
- uses: actions/checkout@v4
|
|
178
|
+
- uses: RADLABtech/comber@v1 # see the publish gate below
|
|
179
|
+
with:
|
|
180
|
+
url: https://your-preview-deploy.example.com
|
|
181
|
+
max-states: "20"
|
|
182
|
+
strict: "false" # non-blocking PR comment first; flip to "true" to fail on defects
|
|
183
|
+
pr-comment: "true"
|
|
184
|
+
env: {}
|
|
185
|
+
# secrets:
|
|
186
|
+
# anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} # only when llm: "true"
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
For an **API target**, swap `url` for `openapi` + `api-base`:
|
|
190
|
+
|
|
191
|
+
```yaml
|
|
192
|
+
- uses: RADLABtech/comber@v1
|
|
193
|
+
with:
|
|
194
|
+
openapi: ./openapi.json # file path or http(s) URL
|
|
195
|
+
api-base: https://api-preview.example.com
|
|
196
|
+
strict: "true"
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
### Inputs
|
|
200
|
+
|
|
201
|
+
| Input | Default | Purpose |
|
|
202
|
+
|---|---|---|
|
|
203
|
+
| `url` | `""` | Web target URL to crawl (use this **or** `openapi`). |
|
|
204
|
+
| `openapi` | `""` | OpenAPI 3.x source (path/URL); selects the API driver. |
|
|
205
|
+
| `api-base` | `""` | API base URL (required with `openapi`). |
|
|
206
|
+
| `seed` | `""` | API seed-data JSON path (per-operation overrides). |
|
|
207
|
+
| `max-states` | `""` | Max states to exercise (blank = Comber default). |
|
|
208
|
+
| `llm` | `"false"` | Use the Claude action-picker + judge (needs `anthropic-api-key`). |
|
|
209
|
+
| `external-links` | `"false"` | Check external links (egress to private/metadata hosts stays blocked). |
|
|
210
|
+
| `allow-writes` | `"false"` | Allow mutating HTTP methods — **synthetic environments only**. |
|
|
211
|
+
| `strict` | `"false"` | Exit non-zero (fail the job) when fail-severity findings exist. |
|
|
212
|
+
| `pr-comment` | `"true"` | Post a non-blocking comment on the PR. |
|
|
213
|
+
| `baseline` | `""` | Prior `result.json` for baseline diffing. |
|
|
214
|
+
| `config` / `profile` | `""` | `comber.config.json` path / profile name. |
|
|
215
|
+
| `install-spec` | `"comber"` | Spec passed to `bun add` (see the publish gate). |
|
|
216
|
+
| `comber-version` | `""` | Version pin when `install-spec` is the default (→ `comber@<version>`). |
|
|
217
|
+
| `bun-version` | `"latest"` | Bun version for `oven-sh/setup-bun`. |
|
|
218
|
+
| `anthropic-api-key` | `""` | `ANTHROPIC_API_KEY` pass-through (LLM mode). |
|
|
219
|
+
| `storage-state` | `""` | Absolute path to a synthetic Playwright `storageState` (`COMBER_STORAGE_STATE`). |
|
|
220
|
+
| `api-token` | `""` | Bearer for the API target (`COMBER_API_TOKEN`). |
|
|
221
|
+
| `sink-file` / `sink-dir` / `sink-url` / `sink-token` | `""` | `COMBER_SINK_*` persistence pass-through. |
|
|
222
|
+
| `github-token` | `${{ github.token }}` | Token for `gh pr comment` (needs `pull-requests: write`). |
|
|
223
|
+
| `pr-number` | `""` | PR to comment on (defaults to the triggering `pull_request`). |
|
|
224
|
+
|
|
225
|
+
The job's only output, `exit-code`, is Comber's process code (`0` clean/soft · `1` strict + fails · `2` crash).
|
|
226
|
+
|
|
227
|
+
### Script-injection safety
|
|
228
|
+
|
|
229
|
+
Every consumer-supplied value (`url`, `openapi`, `api-base`, …) is forwarded to the run step
|
|
230
|
+
through **`env:`** and read inside a **quoted bash array** as `"$VAR"` — it is *never*
|
|
231
|
+
interpolated into a `run:` script via `${{ … }}`. That is the GitHub Actions untrusted-input
|
|
232
|
+
rule: a `url` like `"; rm -rf / #` cannot break out of an env var into the shell. The action
|
|
233
|
+
follows the same pattern as `.github/workflows/comber.yml`, and pins every third-party action
|
|
234
|
+
to a commit SHA.
|
|
235
|
+
|
|
236
|
+
### Synthetic session + secrets
|
|
237
|
+
|
|
238
|
+
An authenticated crawl needs a Playwright `storageState` for a **synthetic / throwaway**
|
|
239
|
+
account (never a real user). Generate it once (see *Generating a synthetic session* above),
|
|
240
|
+
store it as a CI secret, write it to a file in a prior step, and pass its **absolute** path:
|
|
241
|
+
|
|
242
|
+
```yaml
|
|
243
|
+
- name: Materialize synthetic session
|
|
244
|
+
run: printf '%s' "$STATE" > "$GITHUB_WORKSPACE/auth.json"
|
|
245
|
+
env: { STATE: ${{ secrets.COMBER_STORAGE_STATE }} }
|
|
246
|
+
- uses: RADLABtech/comber@v1
|
|
247
|
+
with:
|
|
248
|
+
url: https://your-preview-deploy.example.com
|
|
249
|
+
storage-state: ${{ github.workspace }}/auth.json
|
|
250
|
+
anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
251
|
+
llm: "true"
|
|
252
|
+
```
|
|
253
|
+
|
|
254
|
+
Secrets are pass-through only — they reach Comber as env (`ANTHROPIC_API_KEY`,
|
|
255
|
+
`COMBER_STORAGE_STATE`, `COMBER_API_TOKEN`, `COMBER_SINK_*`) and never appear in a `run:`
|
|
256
|
+
script, the report, the PR comment, or logs. Every **text** artifact (report, `result.json`,
|
|
257
|
+
PR comment, sink records, logs) is redacted before it leaves the process. The one exception is
|
|
258
|
+
**per-state screenshots** (`state-*.png`): those are raw pixels of the target and can carry
|
|
259
|
+
on-screen secrets/PII that text redaction can't reach — pass `--no-screenshots` for sensitive
|
|
260
|
+
targets or restricted CI artifact retention. See [`docs/DATA_HANDLING.md`](docs/DATA_HANDLING.md).
|
|
261
|
+
|
|
262
|
+
### Publish gate
|
|
263
|
+
|
|
264
|
+
**Packaging is ready.** The package is public (`"private": false`), ships a prebuilt `dist/`
|
|
265
|
+
+ `action.yml` in its tarball, and builds `dist/` on a git install via its `prepare` script.
|
|
266
|
+
The only thing left before `bun add comber` resolves from npm is the actual publish:
|
|
267
|
+
|
|
268
|
+
1. Set the **`NPM_TOKEN`** repo/org secret (an npm automation token with publish rights).
|
|
269
|
+
Search the workflows for `# TODO: set NPM_TOKEN secret`.
|
|
270
|
+
2. Push a version tag: `git tag v0.1.0 && git push origin v0.1.0`. The
|
|
271
|
+
[`release.yml`](.github/workflows/release.yml) workflow runs build + tests and then
|
|
272
|
+
`npm publish --access public` — but **only** when `NPM_TOKEN` is set (otherwise the
|
|
273
|
+
publish step is skipped and the job is a green no-op). No tag + no secret = nothing publishes.
|
|
274
|
+
|
|
275
|
+
Meanwhile, the action's `install-spec` input bridges the gap for either state:
|
|
276
|
+
|
|
277
|
+
- **Once published** — `uses: RADLABtech/comber@v1`, default `install-spec: comber`
|
|
278
|
+
(optionally pin with `comber-version: <ver>` → `bun add comber@<ver>`).
|
|
279
|
+
- **Same-org / private today** — point `install-spec` at a git ref:
|
|
280
|
+
`install-spec: github:RADLABtech/comber#<ref>` (`<ref>` = tag, branch, or commit SHA). The
|
|
281
|
+
`prepare` script builds `dist/` on install, so this works with no published tarball.
|
|
282
|
+
|
|
283
|
+
## API driver — OpenAPI / GraphQL / HAR + multi-role
|
|
284
|
+
|
|
285
|
+
The API target discovers an API's operation set, exercises what's safe, and classifies breakage
|
|
286
|
+
into its own signal set — the same `discover → exercise → classify → dedupe → report` contract as
|
|
287
|
+
web/native. It read-only by default (only `GET/HEAD/OPTIONS` for REST, queries for GraphQL, the
|
|
288
|
+
recorded reads for HAR); writes/mutations are surfaced as a counted coverage skip and never fired
|
|
289
|
+
unless `--allow-writes`. Four spec sources feed the same pipeline:
|
|
290
|
+
|
|
291
|
+
```bash
|
|
292
|
+
comber api --openapi ./openapi.json --api-base https://api-preview.example.com # OpenAPI (JSON)
|
|
293
|
+
comber api --openapi ./openapi.yaml --api-base https://api-preview.example.com # OpenAPI (YAML; sniffed)
|
|
294
|
+
comber api --graphql https://api-preview.example.com/graphql # GraphQL introspection
|
|
295
|
+
comber api --har ./recorded-traffic.har # recorded HAR capture
|
|
296
|
+
```
|
|
297
|
+
|
|
298
|
+
| Source | Discover | Exercise | Breakage signal(s) |
|
|
299
|
+
|---|---|---|---|
|
|
300
|
+
| **OpenAPI** (JSON/YAML) | parse paths → operations (`$ref`s resolved) | build a concrete request per operation | `http-5xx`, `undocumented-status`, `content-type-mismatch`, **`response-schema-violation`** (full ajv JSON-Schema validation, carrying the offending paths), `response-shape-mismatch` (light fallback), `broken-auth`, `broken-object-authorization` |
|
|
301
|
+
| **GraphQL** | send the standard introspection query → query/mutation fields | POST a minimal valid operation document | `graphql-error` (non-empty `errors[]`), `graphql-http-status` (non-200), `request-failed` |
|
|
302
|
+
| **HAR** | recorded request entries, deduped by route template | re-send and diff against the recorded response | `contract-drift` (status or body-shape divergence), `request-failed` |
|
|
303
|
+
|
|
304
|
+
**Response-schema validation (ajv).** OpenAPI 2xx bodies are validated against the operation's
|
|
305
|
+
declared response schema with [ajv](https://ajv.js.org) + ajv-formats (`$ref`s resolved, cycles
|
|
306
|
+
broken, OpenAPI `nullable` honored). A violation emits `response-schema-violation` carrying only
|
|
307
|
+
the ajv error **paths** (`/items/0/id must be integer`) — never the body's values, so it stays
|
|
308
|
+
redaction-safe. When no compilable schema is declared, the light required-property check
|
|
309
|
+
(`response-shape-mismatch`) is the fallback.
|
|
310
|
+
|
|
311
|
+
**Multi-role auth → broken-auth + IDOR.** Supply named credential sets (role → token) via config
|
|
312
|
+
or `$COMBER_API_AUTH_PROFILES` (a JSON `{ "role": "token" }` map — **never a CLI flag**, env wins
|
|
313
|
+
over config). With ≥ 2 roles plus seed data tying a resource to its owning role, operations marked
|
|
314
|
+
`ownerScoped` get a cross-role object-access check: Comber confirms role B can read B's resource,
|
|
315
|
+
then re-issues the same request with role A's token — a 2xx for A is **`broken-object-authorization`**
|
|
316
|
+
(a classic IDOR). The no-credential `broken-auth` check (auth-required op returns 2xx without a
|
|
317
|
+
credential) is unchanged. Opt-in + sandbox-only.
|
|
318
|
+
|
|
319
|
+
```jsonc
|
|
320
|
+
// comber.config.json (api profile)
|
|
321
|
+
{
|
|
322
|
+
"target": "api",
|
|
323
|
+
"openapi": "./openapi.json",
|
|
324
|
+
"apiBase": "https://api-sandbox.example.com",
|
|
325
|
+
"apiAuthProfiles": { "alice": "", "bob": "" }, // tokens via $COMBER_API_AUTH_PROFILES
|
|
326
|
+
"ownerScoped": ["GET /users/{id}"],
|
|
327
|
+
"roleSeed": "./role-seed.json" // { "alice": { "GET /users/{id}": { "path": { "id": 1 } } }, "bob": { … } }
|
|
328
|
+
}
|
|
329
|
+
```
|
|
330
|
+
|
|
331
|
+
**Deferred (honest):** the IDOR harness checks operator-declared `ownerScoped` operations against
|
|
332
|
+
operator-supplied per-role seed data. Full automatic resource-ownership *discovery* — creating a
|
|
333
|
+
resource under role B at runtime and tracking its id across operations to test every read path — is
|
|
334
|
+
deferred; today the operator declares which operations are owner-scoped and which resource each role
|
|
335
|
+
owns. GraphQL argument synthesis covers scalars/enums/required-input-objects with minimal literals;
|
|
336
|
+
exotic custom scalars fall back to a string sample.
|
|
337
|
+
|
|
338
|
+
## Native (mobile) driver — Maestro
|
|
339
|
+
|
|
340
|
+
Comber's **third** target is a native iOS/Android app, driven over [Maestro](https://maestro.dev).
|
|
341
|
+
It proves the `Driver` abstraction holds for a non-page, non-API surface: the core's
|
|
342
|
+
frontier / dedupe / caps / report / baseline / redact / triage / coverage run **unchanged** against
|
|
343
|
+
the screens, controls, and breakage signals the native driver emits.
|
|
344
|
+
|
|
345
|
+
```bash
|
|
346
|
+
# Android app already installed on a booted emulator (default platform = android):
|
|
347
|
+
comber native --app com.example.app
|
|
348
|
+
|
|
349
|
+
# iOS bundle on a booted simulator, a specific device, and declared deeplinks:
|
|
350
|
+
comber native --app com.example.app --platform ios --device "iPhone 15" \
|
|
351
|
+
--deeplink "myapp://settings" --deeplink "myapp://profile"
|
|
352
|
+
```
|
|
353
|
+
|
|
354
|
+
Flags: `--app <appId|apk|path>` (required) · `--platform android|ios` (default `android`) ·
|
|
355
|
+
`--device <id>` · `--deeplink <uri>` (repeatable). The same `--profile` / `--config` / `--strict`
|
|
356
|
+
machinery applies; a profile can set `target: "native"`, `app`, `platform`, `device`, `deeplinks`.
|
|
357
|
+
|
|
358
|
+
Evidence flags for native/OEM runs: `--provider <name>`, `--device-class phone|tablet|foldable|emulator|simulator|unknown`,
|
|
359
|
+
`--device-model <model>`, `--os-version <version>`, `--android-version <version>`,
|
|
360
|
+
`--app-version <version>`, `--samsung`, `--one-ui <version>`, and
|
|
361
|
+
`--build-fingerprint <fingerprint>`. Evidence artifact flags are repeatable:
|
|
362
|
+
`--evidence-screenshot <path>`, `--evidence-video <path>`, `--evidence-log <path>`,
|
|
363
|
+
and `--evidence-crash-log <path>`. Provider audit fields include `--provider-run-id <id>`
|
|
364
|
+
and `--verification-command <command>`. These populate `result.json.extra.deviceEvidence`.
|
|
365
|
+
|
|
366
|
+
Comber validates native evidence at runtime. Blank artifact paths, unknown artifact kinds,
|
|
367
|
+
provider run ids without providers, and incomplete verified proxy evidence are surfaced in
|
|
368
|
+
`result.json.extra.deviceEvidence.validationErrors` and block exhaustive claims through the typed
|
|
369
|
+
`native-evidence-invalid` blind spot. Every native run also includes
|
|
370
|
+
`result.json.extra.nativeReadiness`, a provider-neutral preflight summary for device id, provider,
|
|
371
|
+
verification command, evidence artifacts, and Samsung/OEM claim readiness.
|
|
372
|
+
|
|
373
|
+
Native network egress remains a blocker unless the run declares complete verified proxy evidence:
|
|
374
|
+
`--egress-proxy-verified`, `--egress-proxy-provider <name>`, `--egress-proxy-command <command>`,
|
|
375
|
+
and `--egress-proxy-artifact <path>`. All three support fields are required; an incomplete proxy
|
|
376
|
+
claim is still reported as `native-egress-unenforced`.
|
|
377
|
+
|
|
378
|
+
Mac/iOS runs can produce that evidence through the bundled provider-neutral proof helper. It does
|
|
379
|
+
not install or configure a proxy service; it runs an operator-supplied local proof command with
|
|
380
|
+
`execFile` argv semantics, requires expected output markers, writes a JSON proof artifact, and
|
|
381
|
+
prints the `--egress-proxy-proof <path>` flag for the native run:
|
|
382
|
+
|
|
383
|
+
```bash
|
|
384
|
+
bunx tsx scripts/native-ios-proxy-verify.ts \
|
|
385
|
+
--device CAE50036-EF58-4880-B368-67CBD2EF0728 \
|
|
386
|
+
--provider local-mac-proxy \
|
|
387
|
+
--proxy-command "mitmproxy/Proxyman profile active for simulator CAE50036-EF58-4880-B368-67CBD2EF0728" \
|
|
388
|
+
--proof-command-json '["xcrun","simctl","spawn","CAE50036-EF58-4880-B368-67CBD2EF0728","log","show","--last","2m"]' \
|
|
389
|
+
--expect-output comber-proxy-marker \
|
|
390
|
+
--artifact runs/native/ios-proxy-proof.json
|
|
391
|
+
|
|
392
|
+
comber native --app com.example.ios --platform ios \
|
|
393
|
+
--device CAE50036-EF58-4880-B368-67CBD2EF0728 \
|
|
394
|
+
--egress-proxy-proof runs/native/ios-proxy-proof.json
|
|
395
|
+
```
|
|
396
|
+
|
|
397
|
+
The proof command must be the real local check that demonstrates the app/device traffic is flowing
|
|
398
|
+
through the device-level proxy. A host-only `curl --proxy ...` can show that a proxy listener works,
|
|
399
|
+
but by itself it is not proof that an iOS simulator or device is routed through that proxy. If the
|
|
400
|
+
proof command fails, the expected marker is absent, the proof file is missing, or the proof file is
|
|
401
|
+
malformed, Comber keeps `egressProxy.verified: false` and continues to report
|
|
402
|
+
`native-egress-unenforced`.
|
|
403
|
+
|
|
404
|
+
## Desktop drivers — Windows UIA and macOS Accessibility
|
|
405
|
+
|
|
406
|
+
Comber's desktop target ships adapter-owned native desktop runners behind the same `Driver`
|
|
407
|
+
contract as web, API, and mobile. The Windows path uses UI Automation and replays discovered
|
|
408
|
+
branch paths through `uiapath:` addresses. The macOS path uses Accessibility and replays discovered
|
|
409
|
+
branch paths through `axpath:` addresses. Neither path changes core traversal semantics.
|
|
410
|
+
|
|
411
|
+
```bash
|
|
412
|
+
# Windows: launch an app by executable path:
|
|
413
|
+
comber desktop --platform windows --app-path "C:\Apps\Example\Example.exe"
|
|
414
|
+
|
|
415
|
+
# Windows: attach to an already-running process:
|
|
416
|
+
comber desktop --platform windows --process-name Example.exe
|
|
417
|
+
|
|
418
|
+
# Prefer PID/window-title disambiguation before live action runs:
|
|
419
|
+
comber desktop --platform windows --process-id 4242 --window-title "Example"
|
|
420
|
+
|
|
421
|
+
# Optional local smoke, outside default CI, that performs real safe fixture actions:
|
|
422
|
+
bun run smoke:windows-uia -- --fixture --max-states 4 --max-actions 12
|
|
423
|
+
|
|
424
|
+
# Generic dry-run smoke against an app you provide:
|
|
425
|
+
bun run smoke:windows-uia -- --app-path "C:\Windows\System32\notepad.exe" --max-states 1 --dry-run
|
|
426
|
+
|
|
427
|
+
# macOS: launch by .app path or bundle id:
|
|
428
|
+
comber desktop --platform macos --app-path /Applications/Example.app --process-name Example \
|
|
429
|
+
--automation-backend accessibility \
|
|
430
|
+
--provider local-mac \
|
|
431
|
+
--verification-command "osascript -l JavaScript runtime/macos-accessibility-runner.js snapshot" \
|
|
432
|
+
--accessibility-permission-verified
|
|
433
|
+
|
|
434
|
+
comber desktop --platform macos --bundle-id com.example.Example --process-name Example \
|
|
435
|
+
--automation-backend accessibility
|
|
436
|
+
|
|
437
|
+
# macOS controlled dry-run smoke. Without --app-path it creates a temporary fixture app.
|
|
438
|
+
bun run smoke:macos-accessibility
|
|
439
|
+
|
|
440
|
+
# macOS desktop proxy proof from already-configured system proxy settings.
|
|
441
|
+
# This inspects networksetup state and does not modify macOS proxy configuration.
|
|
442
|
+
bun run verify:desktop-macos-proxy -- \
|
|
443
|
+
--app-identity Example \
|
|
444
|
+
--artifact runs/desktop/macos-proxy-proof.json
|
|
445
|
+
|
|
446
|
+
comber desktop --platform macos --app-path /Applications/Example.app --process-name Example \
|
|
447
|
+
--egress-proxy-proof runs/desktop/macos-proxy-proof.json
|
|
448
|
+
```
|
|
449
|
+
|
|
450
|
+
Windows controls are identified by AutomationId first, then Name plus tree path, then RuntimeId/path.
|
|
451
|
+
Buttons, menu items, hyperlinks, tabs, list/data rows, and tree items are treated as branch-capable
|
|
452
|
+
controls; edits, toggles, radio buttons, combo boxes, and sliders are exercised in place with
|
|
453
|
+
synthetic input only. Custom, owner-drawn, canvas, DirectX, or otherwise opaque UIA subtrees are
|
|
454
|
+
reported as `desktop-opaque-subtree` blind spots.
|
|
455
|
+
|
|
456
|
+
macOS controls are identified by Accessibility identifier first, then title plus tree path, then
|
|
457
|
+
role/path. AX buttons, links, tabs, rows, menu items, and tree items are branch-capable; text fields,
|
|
458
|
+
text areas, toggles, radio buttons, pop-up buttons, and sliders are exercised in place with synthetic
|
|
459
|
+
input only. Custom, canvas, Metal, WebView, and otherwise opaque Accessibility regions are reported
|
|
460
|
+
as `desktop-opaque-subtree` blind spots. If the terminal or Codex host process lacks macOS
|
|
461
|
+
Accessibility permission, the runner fails closed and Comber reports `desktop-accessibility-unavailable`.
|
|
462
|
+
|
|
463
|
+
Desktop evidence lands in `result.json.extra.desktopEvidence`; readiness lands in
|
|
464
|
+
`result.json.extra.desktopReadiness`. Windows readiness requires app identity, the `uia` backend,
|
|
465
|
+
provider/source metadata for complete proof, launch command, verification command, and UIA
|
|
466
|
+
automation-tree evidence. macOS readiness requires app identity, the `accessibility` backend,
|
|
467
|
+
provider/source metadata, launch command, verification command, verified Accessibility permission,
|
|
468
|
+
and an accessibility-tree artifact. Desktop network egress is still reported as
|
|
469
|
+
`desktop-egress-unenforced` unless verified OS-level proxy evidence includes provider, command,
|
|
470
|
+
and artifact path. `bun run verify:desktop-macos-proxy` can create that proof only for an already
|
|
471
|
+
configured macOS system proxy; if no active proxy is found, it writes an unverified artifact and the
|
|
472
|
+
desktop egress blocker remains. The macOS XCTest backend is reserved in the evidence contract but is
|
|
473
|
+
not a live default runner yet.
|
|
474
|
+
|
|
475
|
+
Process-name-only attachment is allowed for inspection, but it can be ambiguous when multiple
|
|
476
|
+
windows share the same executable. For non-dry-run desktop checks, prefer `--process-id` and
|
|
477
|
+
`--window-title`; the Windows helper refuses missing or ambiguous declared targets instead of
|
|
478
|
+
falling back to an arbitrary desktop window.
|
|
479
|
+
|
|
480
|
+
### Exhaustive action-tree traversal
|
|
481
|
+
|
|
482
|
+
Comber treats every discovered surface as an action tree:
|
|
483
|
+
|
|
484
|
+
- `exercise`: safe in-place controls are clicked, tapped, typed into, or nudged.
|
|
485
|
+
- `follow`: known destinations such as safe web links are enqueued without clicking through.
|
|
486
|
+
- `branch`: route-less controls such as tabs, accordions, dialogs, menus, SPA buttons,
|
|
487
|
+
Android/iOS tabs, native links, list cells, Windows UIA branch controls, and macOS AX branch
|
|
488
|
+
controls are activated; any
|
|
489
|
+
child state they reveal is added to the frontier.
|
|
490
|
+
|
|
491
|
+
A clean report is only an exhaustive claim when the frontier drains without caps and without
|
|
492
|
+
unreported blind spots. Capped runs, native/desktop egress limitations, desktop opaque UIA subtrees,
|
|
493
|
+
closed shadow roots, cross-origin iframes, unavailable devices, and skipped destructive controls are
|
|
494
|
+
reported as gaps.
|
|
495
|
+
|
|
496
|
+
Samsung support is Android-plus-OEM validation. Future Samsung profiles must record the exact
|
|
497
|
+
Galaxy model, Android version, One UI version, provider or device source, command,
|
|
498
|
+
screenshots/video, logcat crash buffer, and unavailable device classes. A generic Android
|
|
499
|
+
emulator is not Samsung coverage.
|
|
500
|
+
|
|
501
|
+
Every run also writes an `exhaustivenessLedger` into `result.json`. It records each discovered
|
|
502
|
+
unit as `exercised`, `followed`, `branched`, `skipped`, `blocked`, or `blind-spot`, with the reason
|
|
503
|
+
and child-state count when applicable. This is the machine-readable audit trail behind the
|
|
504
|
+
headline coverage percentage.
|
|
505
|
+
|
|
506
|
+
For machines with a working Playwright browser install, `bun run smoke:web-branch` runs a local
|
|
507
|
+
route-less web branch fixture and verifies from the exhaustiveness ledger that Comber creates a
|
|
508
|
+
replayable `webaction:` address and exercises the child control it reveals.
|
|
509
|
+
|
|
510
|
+
### How a phone maps onto the universal contract
|
|
511
|
+
|
|
512
|
+
| Contract | Native meaning |
|
|
513
|
+
|---|---|
|
|
514
|
+
| `Surface` | one **screen** (the current accessibility/view hierarchy) |
|
|
515
|
+
| `SurfaceNode` | one **control** - `kind` = control type (`button`/`text-input`/`toggle`/`tab`/`link`/`cell`), `id` = the stable test id (Android resource-id / iOS accessibilityIdentifier / testTag, else text+path), `label` = visible text, `handle` = the raw native node, `disabled`, `intent` (`exercise`/`follow`/`branch`) |
|
|
516
|
+
| `Address` | screen identity — **`id` is the goto handle** (`launch` · `deeplink:<uri>` · `tappath:<base>::<tap>…`), `scope` = screen name/route, `dedupeKey` = destination-screen fingerprint for a tap-path. Frontier de-dups by the goto handle; *structural* "seen this screen?" is the core's `stateFingerprint` — two distinct layers |
|
|
517
|
+
| `entryAddress()` | the launch screen (relaunch, no deeplink) |
|
|
518
|
+
| `goto(addr)` | relaunch + optional deeplink, **or** relaunch-base + replay-taps (tap-path) |
|
|
519
|
+
| `snapshot()` | `viewHierarchy()` → `Surface` |
|
|
520
|
+
| `exercise(node)` | **tap** (default) or **input** (text fields); a tap that reaches a new screen returns it as `ExerciseResult.discovered` (frontier expansion from acting) |
|
|
521
|
+
| probes | `session` (after-goto) · `dom` (after-snapshot) · `crash` (after-exercise) · `logs` (finalize) — each registered with its lifecycle slot |
|
|
522
|
+
|
|
523
|
+
**Native breakage signals:** `crash` (fail — app left the foreground / a `FATAL EXCEPTION` after an
|
|
524
|
+
action or launch), `anr` (warn — an "Application Not Responding" log line), `frozen` (warn — a tap or
|
|
525
|
+
`viewHierarchy()` that times out), `error-text` (warn — an on-screen error/canary string),
|
|
526
|
+
`assertion-fail` (warn — a declared Maestro flow assertion). All dedupe on a signature keyed by
|
|
527
|
+
**screen scope + kind**.
|
|
528
|
+
|
|
529
|
+
### The pluggable `MaestroRunner` transport
|
|
530
|
+
|
|
531
|
+
The driver depends **only** on a `MaestroRunner` interface — the exact same pattern as the API
|
|
532
|
+
driver's pluggable `ApiTransport` — so the whole driver is tested hermetically by injecting a stub
|
|
533
|
+
(no device, no maestro binary). The seam:
|
|
534
|
+
|
|
535
|
+
```ts
|
|
536
|
+
interface MaestroRunner {
|
|
537
|
+
launch(appId, deeplink?): Promise<void>; // the goto reset point
|
|
538
|
+
viewHierarchy(): Promise<NativeView>; // the current screen's view tree
|
|
539
|
+
tap(selector): Promise<void>;
|
|
540
|
+
input(selector, text): Promise<void>;
|
|
541
|
+
back(): Promise<void>; // softReset — dismiss a dialog / pop a screen
|
|
542
|
+
foregroundApp(): Promise<string>; // crash detection (≠ appId ⇒ died)
|
|
543
|
+
logsSince(sinceMs): Promise<string[]>; // crash / ANR log scan
|
|
544
|
+
runFlow?(yaml): Promise<void>; // optional — declared-flow assertions
|
|
545
|
+
screenshot?(): Promise<Buffer>; // optional — report evidence
|
|
546
|
+
}
|
|
547
|
+
```
|
|
548
|
+
|
|
549
|
+
The bundled `DefaultMaestroRunner` shells out to `maestro` (ephemeral one-command flows for
|
|
550
|
+
launch/tap/input/back, `maestro hierarchy` for the view tree) and to `adb logcat` / `dumpsys` for
|
|
551
|
+
crash/ANR detection on Android.
|
|
552
|
+
|
|
553
|
+
### What a LIVE run needs (and what was verified here)
|
|
554
|
+
|
|
555
|
+
The driver and its mapping/signal logic are covered by a **hermetic** test suite that stubs the
|
|
556
|
+
`MaestroRunner` — **no live device crawl was run** (none was available). A real run requires, none
|
|
557
|
+
of which ship with Comber:
|
|
558
|
+
|
|
559
|
+
- **Maestro CLI** on `PATH` (`maestro`), plus **`adb`** for Android crash/ANR logs.
|
|
560
|
+
- A **booted emulator / simulator** (or a **Maestro Cloud** session) with the app installed.
|
|
561
|
+
- The exact `DefaultMaestroRunner` command strings reconciled against your installed maestro
|
|
562
|
+
version — they are a best-effort starting point, marked in `src/native/maestro.ts`, **not** a
|
|
563
|
+
CI-verified contract.
|
|
564
|
+
- For iOS egress proof, a configured device-level proxy plus a proof command that produces a
|
|
565
|
+
durable artifact through `scripts/native-ios-proxy-verify.ts`.
|
|
566
|
+
|
|
567
|
+
### Known gaps (honest, not silent)
|
|
568
|
+
|
|
569
|
+
- **Network-egress safety is NOT enforced on native v1.** The web driver's SSRF / write-egress gate
|
|
570
|
+
works by intercepting browser requests; **Maestro cannot proxy traffic**, so there is no
|
|
571
|
+
equivalent wire-level guard here. The only write protection is the **name-denylist** (destructive
|
|
572
|
+
controls are observed, never tapped — the core's `deniedName` over the control label). The
|
|
573
|
+
egress-hardening path is a **device-level proxy** (e.g. mitmproxy) the runner routes through.
|
|
574
|
+
Comber clears the native egress blocker only when complete verified proxy evidence is provided
|
|
575
|
+
with provider, command, and artifact path, or when a verified `--egress-proxy-proof` artifact
|
|
576
|
+
provides those fields. The CLI banner and `nativeAudit.egressEnforced: false` still surface the
|
|
577
|
+
Maestro transport limitation on every run.
|
|
578
|
+
- **Network-egress safety is NOT enforced on desktop v1.** Windows UI Automation drives the UI tree;
|
|
579
|
+
it does not intercept process network traffic. Complete verified OS-level proxy evidence is still
|
|
580
|
+
required to clear `desktop-egress-unenforced`.
|
|
581
|
+
|
|
582
|
+
### Abstraction limits the native driver surfaced — RESOLVED in contract v2
|
|
583
|
+
|
|
584
|
+
Building the third driver stress-tested the fixed `Driver` contract. Three seams that web + API
|
|
585
|
+
never exercised showed their edges. Rather than bend the core per-target, the contract was evolved
|
|
586
|
+
(v2) so all three drivers conform and web + API stay **byte-identical** (same findings, coverage,
|
|
587
|
+
dedupe, ordering; the full suite stays green):
|
|
588
|
+
|
|
589
|
+
1. **`Address.id` conflated "dedupe key" with "goto handle"** — *resolved.* For web a URL is both;
|
|
590
|
+
for API an operation is both. A native screen's *structural* identity (for dedupe) and its
|
|
591
|
+
*re-reachability* (for `goto`) are different things. The contract now states the two **distinct
|
|
592
|
+
dedup layers** explicitly: **FRONTIER de-dup is by the goto handle** (`Address.dedupeKey ?? id`,
|
|
593
|
+
the `queued` set), while **"have I processed this screen?" is the core's `stateFingerprint`** over
|
|
594
|
+
the `Surface` (the `visited` set). `id` is documented as the goto handle; the optional
|
|
595
|
+
`dedupeKey` lets a driver whose handle ≠ structural identity (native tap-paths) collapse distinct
|
|
596
|
+
handles that reach the same screen. Web/API leave `dedupeKey` unset → unchanged.
|
|
597
|
+
2. **`DriverCapabilities.probes` was a *closed* phase set** the orchestrator hard-coded
|
|
598
|
+
(`session`/`dom`/`axe`/`links`/`forms`) — *resolved.* Probes are now **driver-registered with a
|
|
599
|
+
lifecycle slot** (`{ name, slot }`); the orchestrator iterates each driver's probes at the
|
|
600
|
+
matching slot. The slot set is fixed and reproduces web's exact order — `after-goto` (session),
|
|
601
|
+
`after-snapshot` (axe, dom, links), `after-exercise` (forms), `finalize` — but *which* probes run
|
|
602
|
+
is open, so native now registers its own **`crash` (after-exercise)** + **`logs` (finalize)**
|
|
603
|
+
phases the closed vocabulary forbade.
|
|
604
|
+
3. **The frontier grew only by FOLLOWING links** (`outboundAddresses`), stranding screens reachable
|
|
605
|
+
*only* by tapping — *resolved.* `exercise()` may now return **`ExerciseResult.discovered`**:
|
|
606
|
+
Addresses a tap/click navigated to. The orchestrator enqueues them (inScope + caps/dedup), so a
|
|
607
|
+
native tap-only screen is crawled — returned as a **replayable tap-path** `Address` whose `goto`
|
|
608
|
+
relaunches the base anchor and re-taps the recorded sequence. Web/API navigate via links
|
|
609
|
+
(followed, never clicked) and return none. Declaring `--deeplink` is now an *optional* seed for
|
|
610
|
+
depth, no longer the only way past the launch screen.
|
|
611
|
+
|
|
612
|
+
## Sinks / Persistence
|
|
613
|
+
|
|
614
|
+
By default a run writes only the local HTML report + `result.json` under `./runs/`. Opt-in
|
|
615
|
+
**sinks** persist or forward each run's results to any backend — the substrate a hosted tier
|
|
616
|
+
consumes. The baseline ships two generic, dependency-light adapters (no new dependencies; both
|
|
617
|
+
use only the Node stdlib). Each is configured purely by environment variables, fans out via a
|
|
618
|
+
`MultiSink`, **never breaks a run** (a sink failure is caught + logged, the run still passes),
|
|
619
|
+
and is a **no-op when unconfigured** (nothing set = unchanged local-only behavior).
|
|
620
|
+
|
|
621
|
+
Every record a sink emits is **redacted** (`redactResult`) before it leaves the process, so a
|
|
622
|
+
crawled target's secrets — finding URLs/messages, signed-URL params, OAuth fragments — never
|
|
623
|
+
reach a file or cross the wire.
|
|
624
|
+
|
|
625
|
+
| Env var | Sink | Effect |
|
|
626
|
+
|---|---|---|
|
|
627
|
+
| `COMBER_SINK_FILE` | **FileSink** | Append one redacted run-summary record (a JSON line) per run to this JSONL file. |
|
|
628
|
+
| `COMBER_SINK_DIR` | **FileSink** | Copy the run's screenshots into this directory, under a per-run `runId` subfolder. |
|
|
629
|
+
| `COMBER_SINK_URL` | **HttpSink** | POST the redacted run record (JSON) to this webhook URL. Covers backend ingest + Sentry-style error-tracking collectors. |
|
|
630
|
+
| `COMBER_SINK_TOKEN` | **HttpSink** | Optional Bearer token for the webhook — sent as an `Authorization` header only, never logged and never in the body. |
|
|
631
|
+
|
|
632
|
+
```bash
|
|
633
|
+
# Local persistence: a growing JSONL log + a screenshot archive.
|
|
634
|
+
COMBER_SINK_FILE=./comber-runs.jsonl COMBER_SINK_DIR=./comber-shots bun run dev https://example.com
|
|
635
|
+
|
|
636
|
+
# Forward each run to a webhook / collector with auth.
|
|
637
|
+
COMBER_SINK_URL=https://hooks.example.com/comber COMBER_SINK_TOKEN=… bun run dev https://example.com
|
|
638
|
+
```
|
|
639
|
+
|
|
640
|
+
Configured sink paths are path-traversal-guarded, and every copied screenshot filename is
|
|
641
|
+
constrained to stay within the target directory.
|
|
642
|
+
|
|
643
|
+
> **Vendor neutrality.** The baseline carries only these generic adapters. A vendor-specific
|
|
644
|
+
> backend (a database, object store, or hosted error tracker) belongs in an overlay as its own
|
|
645
|
+
> `Sink` implementation against the same interface — it is never added to `src/` here.
|
|
646
|
+
|
|
647
|
+
## Connectors & marketplace
|
|
648
|
+
|
|
649
|
+
Where a sink persists the raw record, a **connector** delivers a finished run to where developers
|
|
650
|
+
already work — Slack, Discord, a GitHub PR check, a Linear issue, a JUnit report, or any webhook.
|
|
651
|
+
Connectors are opt-in, declared in the `connectors: [...]` array of `comber.config.json`, and each
|
|
652
|
+
consumes the same **redacted** `RunRecord` a sink does — a third-party connector physically cannot
|
|
653
|
+
receive a crawled-app secret. Six connectors ship built-in; the marketplace is any npm package that
|
|
654
|
+
satisfies the contract, loaded by module specifier.
|
|
655
|
+
|
|
656
|
+
```jsonc
|
|
657
|
+
{
|
|
658
|
+
"url": "https://preview.example.com",
|
|
659
|
+
"connectors": [
|
|
660
|
+
{ "use": "slack", "on": "fail", "mentionOnFail": "<!here>" },
|
|
661
|
+
{ "use": "junit", "outFile": "runs/comber-junit.xml" },
|
|
662
|
+
{ "use": "@acme/comber-teams", "on": "warn" }
|
|
663
|
+
]
|
|
664
|
+
}
|
|
665
|
+
```
|
|
666
|
+
|
|
667
|
+
`use` names a built-in (`slack`, `discord`, `github`, `junit`, `linear`, `webhook`) or an installed
|
|
668
|
+
package; `on` gates delivery by severity (`always` | `warn` | `fail`); all other keys are non-secret
|
|
669
|
+
settings. **Secrets come from the environment, never from config** (e.g. `COMBER_SLACK_WEBHOOK_URL`).
|
|
670
|
+
List the built-ins with `comber connectors`. Full guide — the built-in table, the `RunRecord` shape,
|
|
671
|
+
authoring an external connector, and publishing to the catalog — in
|
|
672
|
+
**[docs/CONNECTORS.md](docs/CONNECTORS.md)**.
|
|
673
|
+
|
|
674
|
+
## Architecture
|
|
675
|
+
|
|
676
|
+
```
|
|
677
|
+
cli.ts → crawl.ts ─ drives ─▶ Driver (driver.ts: WebDriver over Playwright)
|
|
678
|
+
│ │
|
|
679
|
+
│ pickActions() └─ breakage.ts (CDP listeners, console canaries, axe, signatures)
|
|
680
|
+
│ (Haiku, DOM-first)
|
|
681
|
+
▼
|
|
682
|
+
triage.ts (Opus "genuine defect?" judge) ──▶ report.ts (self-contained HTML)
|
|
683
|
+
```
|
|
684
|
+
|
|
685
|
+
- **Driver adapter** (`core/contract.ts`) is the seam that makes a surface pluggable. `WebDriver`
|
|
686
|
+
(Playwright), `ApiDriver` (OpenAPI), and `NativeDriver` (native iOS/Android over a pluggable
|
|
687
|
+
`MaestroRunner`) all implement the *same shape* — so crawl/breakage/report never change.
|
|
688
|
+
- **Breakage** (`breakage.ts`): deterministic auto-fail (uncaught exceptions, 5xx, failed
|
|
689
|
+
assets, Next.js regression canaries) + judged tier (axe-core criticals, console noise).
|
|
690
|
+
Every signal carries a normalized `signature` so a broken shared footer across 50 pages
|
|
691
|
+
collapses to one finding.
|
|
692
|
+
- **Signal normalization** (`signal-normalize.ts`): removes crawler-caused egress fan-out
|
|
693
|
+
from warn/fail gates while keeping a transparent `info` finding in the report.
|
|
694
|
+
- **Frontier crawl** (`crawl.ts`): BFS over same-origin states with fingerprint dedupe and
|
|
695
|
+
hard caps (states / actions / wall-clock / token budget) — the runaway-cost backstops.
|
|
696
|
+
|
|
697
|
+
## Safety rails (non-negotiable)
|
|
698
|
+
|
|
699
|
+
- **Destructive-action denylist** (`config.ts`): elements named delete/remove/pay/checkout/
|
|
700
|
+
cancel/etc. are *observed, never clicked*.
|
|
701
|
+
- **Write blocking is on by default**: mutating HTTP verbs are aborted unless `--allow-writes`
|
|
702
|
+
is passed for a reviewed synthetic environment.
|
|
703
|
+
- **Private-network egress is always blocked**: browser requests and optional external link
|
|
704
|
+
checks cannot hit localhost, private IP ranges, link-local metadata, or hosts resolving there.
|
|
705
|
+
- **External link checking is opt-in** via `--external-links`.
|
|
706
|
+
- **Synthetic accounts only** via `COMBER_STORAGE_STATE` — never point an authed crawl at real
|
|
707
|
+
user data. Use a dedicated throwaway/test account (e.g. `synthetic+comber@yourdomain.test`).
|
|
708
|
+
- Hard caps abort runaway loops; same-origin confinement by default.
|
|
709
|
+
- (Planned) per-app convert-flow guard: upload a 1 KB dummy and STOP before the GPU worker.
|
|
710
|
+
|
|
711
|
+
## Roadmap
|
|
712
|
+
|
|
713
|
+
Comber's through-line is one contract — *discover a target's surface → exercise it → classify
|
|
714
|
+
breakage → dedupe → report / baseline / gate* — so new targets land as new **drivers** behind it,
|
|
715
|
+
not core rewrites. See [`PRODUCT.md`](PRODUCT.md) for the model.
|
|
716
|
+
|
|
717
|
+
- **Now** — web MVP: DOM-first crawler, breakage detection, optional Claude action-picker + judge,
|
|
718
|
+
self-contained HTML report, baseline diffing, PR comment. ✅
|
|
719
|
+
- **Next** — deterministic selector-cache replay (run the expensive every-click pass once, cheap
|
|
720
|
+
replays after) ✅; generic persistence sinks (JSONL + screenshot dir, webhook/collector POST)
|
|
721
|
+
✅; CI integration on preview deploys — reusable composite **GitHub Action** (`action.yml`),
|
|
722
|
+
non-blocking PR comment first ✅ (gated on npm publish — see **CI integration** above).
|
|
723
|
+
- **Native** — a `NativeDriver` driving native iOS/Android over a pluggable `MaestroRunner` behind the
|
|
724
|
+
same contract ✅ (hermetic; a live run needs `maestro` + a booted device — see **Native driver** above).
|
|
725
|
+
- **Desktop native** — Windows UI Automation and macOS Accessibility now ship as desktop adapters
|
|
726
|
+
behind the same contract ✅. macOS XCTest remains a reserved optional backend, not a default live
|
|
727
|
+
runner. Desktop evidence/readiness is in place for platform, app identity, automation backend, provider/source, launch command,
|
|
728
|
+
verification command, screenshots/video/logs/crash logs, accessibility-tree or automation-tree
|
|
729
|
+
artifacts, and verified OS-level proxy evidence when desktop network egress is claimed controlled.
|
|
730
|
+
- **API / bot** — request/response contract checks as their own breakage-signal set, same pipeline ✅.
|
|
731
|
+
- **Hosted** — multi-tenant dashboard, bring-your-own-key, and billing — the path toward
|
|
732
|
+
marketplace integrations with CI/deploy platforms.
|