comber 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +18 -0
- package/README.md +732 -0
- package/action.yml +279 -0
- package/comber.config.example.json +64 -0
- package/dist/api/classify.js +99 -0
- package/dist/api/classify.js.map +1 -0
- package/dist/api/driver.js +235 -0
- package/dist/api/driver.js.map +1 -0
- package/dist/api/graphql-driver.js +150 -0
- package/dist/api/graphql-driver.js.map +1 -0
- package/dist/api/graphql.js +115 -0
- package/dist/api/graphql.js.map +1 -0
- package/dist/api/har-driver.js +167 -0
- package/dist/api/har-driver.js.map +1 -0
- package/dist/api/har.js +100 -0
- package/dist/api/har.js.map +1 -0
- package/dist/api/load.js +83 -0
- package/dist/api/load.js.map +1 -0
- package/dist/api/openapi.js +159 -0
- package/dist/api/openapi.js.map +1 -0
- package/dist/api/request.js +141 -0
- package/dist/api/request.js.map +1 -0
- package/dist/api/schema.js +126 -0
- package/dist/api/schema.js.map +1 -0
- package/dist/cli.js +352 -0
- package/dist/cli.js.map +1 -0
- package/dist/comment.js +164 -0
- package/dist/comment.js.map +1 -0
- package/dist/config.js +664 -0
- package/dist/config.js.map +1 -0
- package/dist/connectors/builtin/discord.js +55 -0
- package/dist/connectors/builtin/github-checks.js +53 -0
- package/dist/connectors/builtin/junit.js +70 -0
- package/dist/connectors/builtin/linear.js +58 -0
- package/dist/connectors/builtin/slack.js +60 -0
- package/dist/connectors/builtin/webhook.js +39 -0
- package/dist/connectors/index.js +3 -0
- package/dist/connectors/loader.js +72 -0
- package/dist/connectors/registry.js +28 -0
- package/dist/connectors/types.js +23 -0
- package/dist/core/action-gate.js +48 -0
- package/dist/core/baseline.js +85 -0
- package/dist/core/baseline.js.map +1 -0
- package/dist/core/blind-spots.js +22 -0
- package/dist/core/blind-spots.js.map +1 -0
- package/dist/core/contract-extras.js +1 -0
- package/dist/core/contract-extras.js.map +1 -0
- package/dist/core/contract.js +1 -0
- package/dist/core/contract.js.map +1 -0
- package/dist/core/coverage.js +70 -0
- package/dist/core/coverage.js.map +1 -0
- package/dist/core/crawl.js +229 -0
- package/dist/core/crawl.js.map +1 -0
- package/dist/core/egress.js +128 -0
- package/dist/core/egress.js.map +1 -0
- package/dist/core/ledger.js +41 -0
- package/dist/core/ledger.js.map +1 -0
- package/dist/core/net.js +39 -0
- package/dist/core/pipeline.js +191 -0
- package/dist/core/pipeline.js.map +1 -0
- package/dist/core/plan.js +121 -0
- package/dist/core/plan.js.map +1 -0
- package/dist/core/redact.js +240 -0
- package/dist/core/redact.js.map +1 -0
- package/dist/core/replay.js +263 -0
- package/dist/core/replay.js.map +1 -0
- package/dist/core/report.js +253 -0
- package/dist/core/report.js.map +1 -0
- package/dist/core/safety-judge.js +116 -0
- package/dist/core/safety.js +92 -0
- package/dist/core/safety.js.map +1 -0
- package/dist/core/signal-normalize.js +190 -0
- package/dist/core/signal-normalize.js.map +1 -0
- package/dist/core/signals.js +5 -0
- package/dist/core/signals.js.map +1 -0
- package/dist/core/triage.js +104 -0
- package/dist/core/triage.js.map +1 -0
- package/dist/core/types.js +1 -0
- package/dist/core/types.js.map +1 -0
- package/dist/desktop/desktop-profile.js +6 -0
- package/dist/desktop/desktop-profile.js.map +1 -0
- package/dist/desktop/driver.js +433 -0
- package/dist/desktop/driver.js.map +1 -0
- package/dist/desktop/evidence.js +206 -0
- package/dist/desktop/evidence.js.map +1 -0
- package/dist/desktop/macos/accessibility-runner.js +118 -0
- package/dist/desktop/macos/accessibility-runner.js.map +1 -0
- package/dist/desktop/macos/types.js +1 -0
- package/dist/desktop/macos/types.js.map +1 -0
- package/dist/desktop/preflight.js +107 -0
- package/dist/desktop/preflight.js.map +1 -0
- package/dist/desktop/proxy-proof.js +185 -0
- package/dist/desktop/proxy-proof.js.map +1 -0
- package/dist/desktop/view.js +348 -0
- package/dist/desktop/view.js.map +1 -0
- package/dist/desktop/windows/types.js +1 -0
- package/dist/desktop/windows/types.js.map +1 -0
- package/dist/desktop/windows/uia-runner.js +127 -0
- package/dist/desktop/windows/uia-runner.js.map +1 -0
- package/dist/native/apple.js +14 -0
- package/dist/native/apple.js.map +1 -0
- package/dist/native/device-profile.js +6 -0
- package/dist/native/device-profile.js.map +1 -0
- package/dist/native/driver.js +416 -0
- package/dist/native/driver.js.map +1 -0
- package/dist/native/evidence.js +164 -0
- package/dist/native/evidence.js.map +1 -0
- package/dist/native/load.js +9 -0
- package/dist/native/load.js.map +1 -0
- package/dist/native/maestro.js +153 -0
- package/dist/native/maestro.js.map +1 -0
- package/dist/native/preflight.js +72 -0
- package/dist/native/preflight.js.map +1 -0
- package/dist/native/proxy-proof.js +182 -0
- package/dist/native/proxy-proof.js.map +1 -0
- package/dist/native/samsung.js +34 -0
- package/dist/native/samsung.js.map +1 -0
- package/dist/native/signals.js +50 -0
- package/dist/native/signals.js.map +1 -0
- package/dist/native/view.js +139 -0
- package/dist/native/view.js.map +1 -0
- package/dist/profile.js +237 -0
- package/dist/profile.js.map +1 -0
- package/dist/sink.js +134 -0
- package/dist/sink.js.map +1 -0
- package/dist/sinks/file.js +49 -0
- package/dist/sinks/file.js.map +1 -0
- package/dist/sinks/http.js +38 -0
- package/dist/sinks/http.js.map +1 -0
- package/dist/sinks/index.js +5 -0
- package/dist/sinks/index.js.map +1 -0
- package/dist/sinks/multi.js +40 -0
- package/dist/sinks/multi.js.map +1 -0
- package/dist/sinks/paths.js +32 -0
- package/dist/sinks/paths.js.map +1 -0
- package/dist/sinks/record.js +58 -0
- package/dist/sinks/record.js.map +1 -0
- package/dist/sinks/types.js +1 -0
- package/dist/sinks/types.js.map +1 -0
- package/dist/web/breakage.js +210 -0
- package/dist/web/breakage.js.map +1 -0
- package/dist/web/controls.js +102 -0
- package/dist/web/controls.js.map +1 -0
- package/dist/web/driver.js +803 -0
- package/dist/web/driver.js.map +1 -0
- package/dist/web/elements.js +151 -0
- package/dist/web/elements.js.map +1 -0
- package/dist/web/fingerprint.js +13 -0
- package/dist/web/fingerprint.js.map +1 -0
- package/dist/web/forms.js +74 -0
- package/dist/web/forms.js.map +1 -0
- package/dist/web/links.js +194 -0
- package/dist/web/links.js.map +1 -0
- package/dist/web/types.js +1 -0
- package/dist/web/types.js.map +1 -0
- package/package.json +80 -0
- package/runtime/macos-accessibility-runner.js +275 -0
- package/runtime/windows-uia-runner.ps1 +539 -0
package/action.yml
ADDED
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
# Comber — reusable GitHub Action.
|
|
2
|
+
#
|
|
3
|
+
# Drop Comber into any repo's CI to crawl a preview-deploy URL (web target) or check an
|
|
4
|
+
# OpenAPI surface (api target), upload the report, post a non-blocking PR comment, and
|
|
5
|
+
# optionally gate the job on fail-severity findings.
|
|
6
|
+
#
|
|
7
|
+
# Script-injection defense: every consumer-supplied value (url, openapi, api-base, …) is
|
|
8
|
+
# passed to the run step via `env:` and read inside a quoted bash array as "$VAR" — it is
|
|
9
|
+
# NEVER interpolated into a `run:` script with ${{ … }}. That is the GitHub Actions
|
|
10
|
+
# untrusted-input rule (a `url` of `"; rm -rf / #` can't break out of an env var). This
|
|
11
|
+
# mirrors the pattern already used in .github/workflows/comber.yml.
|
|
12
|
+
name: "Comber"
|
|
13
|
+
description: "Every-click AI QA checker — crawl a web preview deploy or check an OpenAPI surface in CI."
|
|
14
|
+
author: "RADLAB"
|
|
15
|
+
branding:
|
|
16
|
+
icon: "check-circle"
|
|
17
|
+
color: "red"
|
|
18
|
+
|
|
19
|
+
inputs:
|
|
20
|
+
# -- target selection: provide `url` (web) OR `openapi` + `api-base` (api) ---------------
|
|
21
|
+
url:
|
|
22
|
+
description: "Web target URL to crawl (the preview-deploy URL). Use this OR `openapi`."
|
|
23
|
+
required: false
|
|
24
|
+
default: ""
|
|
25
|
+
openapi:
|
|
26
|
+
description: "OpenAPI 3.x document source (file path or http(s) URL). Selects the API driver."
|
|
27
|
+
required: false
|
|
28
|
+
default: ""
|
|
29
|
+
api-base:
|
|
30
|
+
description: "API base URL (required when `openapi` is set; point Comber at a base you own)."
|
|
31
|
+
required: false
|
|
32
|
+
default: ""
|
|
33
|
+
seed:
|
|
34
|
+
description: "Optional API seed-data JSON path (per-operation request overrides)."
|
|
35
|
+
required: false
|
|
36
|
+
default: ""
|
|
37
|
+
|
|
38
|
+
# -- crawl knobs ------------------------------------------------------------------------
|
|
39
|
+
max-states:
|
|
40
|
+
description: "Max distinct states to exercise (the runaway-cost backstop). Blank = Comber default."
|
|
41
|
+
required: false
|
|
42
|
+
default: ""
|
|
43
|
+
llm:
|
|
44
|
+
description: "Use the Claude action-picker + judge (requires anthropic-api-key)."
|
|
45
|
+
required: false
|
|
46
|
+
default: "false"
|
|
47
|
+
external-links:
|
|
48
|
+
description: "Check external links (opt-in; egress to private/metadata hosts stays blocked)."
|
|
49
|
+
required: false
|
|
50
|
+
default: "false"
|
|
51
|
+
allow-writes:
|
|
52
|
+
description: "Allow mutating HTTP methods — SYNTHETIC environments only. Default blocks writes."
|
|
53
|
+
required: false
|
|
54
|
+
default: "false"
|
|
55
|
+
strict:
|
|
56
|
+
description: "Gate the job: exit non-zero when fail-severity findings are present."
|
|
57
|
+
required: false
|
|
58
|
+
default: "false"
|
|
59
|
+
pr-comment:
|
|
60
|
+
description: "Post a non-blocking Comber comment on the PR (when run on a pull_request)."
|
|
61
|
+
required: false
|
|
62
|
+
default: "true"
|
|
63
|
+
baseline:
|
|
64
|
+
description: "Optional prior result.json path for baseline diffing."
|
|
65
|
+
required: false
|
|
66
|
+
default: ""
|
|
67
|
+
config:
|
|
68
|
+
description: "Optional comber.config.json path."
|
|
69
|
+
required: false
|
|
70
|
+
default: ""
|
|
71
|
+
profile:
|
|
72
|
+
description: "Optional profile name from the config."
|
|
73
|
+
required: false
|
|
74
|
+
default: ""
|
|
75
|
+
|
|
76
|
+
# -- how to install comber (see the publish gate in the README) -------------------------
|
|
77
|
+
install-spec:
|
|
78
|
+
# Packaging is ready: the package ships a prebuilt dist/ in its tarball AND builds dist/ on
|
|
79
|
+
# a git install via its `prepare` script, so both forms below work.
|
|
80
|
+
# - Published (default): install-spec: comber (resolves from npm once a v* tag is pushed
|
|
81
|
+
# with the NPM_TOKEN secret set — see .github/workflows/release.yml).
|
|
82
|
+
# - Same-org / private today: install-spec: github:RADLABtech/comber#<ref> (tag/branch/SHA).
|
|
83
|
+
description: "Spec passed to `bun add` to install comber (default: the published package name)."
|
|
84
|
+
required: false
|
|
85
|
+
default: "comber"
|
|
86
|
+
comber-version:
|
|
87
|
+
description: "Convenience version pin used only when install-spec is the default (resolves to comber@<version>)."
|
|
88
|
+
required: false
|
|
89
|
+
default: ""
|
|
90
|
+
bun-version:
|
|
91
|
+
description: "Bun version for oven-sh/setup-bun."
|
|
92
|
+
required: false
|
|
93
|
+
default: "latest"
|
|
94
|
+
|
|
95
|
+
# -- secrets / pass-through env ---------------------------------------------------------
|
|
96
|
+
anthropic-api-key:
|
|
97
|
+
description: "ANTHROPIC_API_KEY — enables the LLM action-picker + judge when `llm` is true."
|
|
98
|
+
required: false
|
|
99
|
+
default: ""
|
|
100
|
+
storage-state:
|
|
101
|
+
description: "Absolute path to a Playwright storageState JSON for a SYNTHETIC session (COMBER_STORAGE_STATE). Materialize it from a secret in a prior step."
|
|
102
|
+
required: false
|
|
103
|
+
default: ""
|
|
104
|
+
api-token:
|
|
105
|
+
description: "Bearer credential for the API target (COMBER_API_TOKEN). Never logged; redacted in artifacts."
|
|
106
|
+
required: false
|
|
107
|
+
default: ""
|
|
108
|
+
sink-file:
|
|
109
|
+
description: "COMBER_SINK_FILE — append one redacted run-summary JSON line per run to this JSONL file."
|
|
110
|
+
required: false
|
|
111
|
+
default: ""
|
|
112
|
+
sink-dir:
|
|
113
|
+
description: "COMBER_SINK_DIR — copy each run's screenshots into this directory."
|
|
114
|
+
required: false
|
|
115
|
+
default: ""
|
|
116
|
+
sink-url:
|
|
117
|
+
description: "COMBER_SINK_URL — POST the redacted run record to this webhook/collector URL."
|
|
118
|
+
required: false
|
|
119
|
+
default: ""
|
|
120
|
+
sink-token:
|
|
121
|
+
description: "COMBER_SINK_TOKEN — optional Bearer token for the webhook (Authorization header only)."
|
|
122
|
+
required: false
|
|
123
|
+
default: ""
|
|
124
|
+
|
|
125
|
+
# -- PR comment wiring ------------------------------------------------------------------
|
|
126
|
+
github-token:
|
|
127
|
+
description: "Token used by `gh pr comment`. Needs pull-requests: write."
|
|
128
|
+
required: false
|
|
129
|
+
default: ${{ github.token }}
|
|
130
|
+
pr-number:
|
|
131
|
+
description: "PR number to comment on. Defaults to the triggering pull_request when blank."
|
|
132
|
+
required: false
|
|
133
|
+
default: ""
|
|
134
|
+
|
|
135
|
+
outputs:
|
|
136
|
+
exit-code:
|
|
137
|
+
description: "Comber's process exit code (0 = clean / soft, 1 = strict + fails, 2 = crashed)."
|
|
138
|
+
value: ${{ steps.run.outputs.exit-code }}
|
|
139
|
+
|
|
140
|
+
runs:
|
|
141
|
+
using: "composite"
|
|
142
|
+
steps:
|
|
143
|
+
- name: Setup bun
|
|
144
|
+
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
|
145
|
+
with:
|
|
146
|
+
bun-version: ${{ inputs.bun-version }}
|
|
147
|
+
|
|
148
|
+
- name: Install comber
|
|
149
|
+
id: install
|
|
150
|
+
shell: bash
|
|
151
|
+
env:
|
|
152
|
+
# Passed via env (never interpolated into the script) — consumer-controlled, still
|
|
153
|
+
# treated as the script-injection rule demands.
|
|
154
|
+
INSTALL_SPEC: ${{ inputs.install-spec }}
|
|
155
|
+
COMBER_VERSION: ${{ inputs.comber-version }}
|
|
156
|
+
run: |
|
|
157
|
+
# Install into an isolated dir under the runner temp so we never touch the caller's
|
|
158
|
+
# package.json / lockfile. Comber's bin is invoked by absolute path from the run step.
|
|
159
|
+
APP_DIR="$RUNNER_TEMP/comber-app"
|
|
160
|
+
mkdir -p "$APP_DIR"
|
|
161
|
+
if [ ! -f "$APP_DIR/package.json" ]; then
|
|
162
|
+
echo '{"name":"comber-ci-host","private":true}' > "$APP_DIR/package.json"
|
|
163
|
+
fi
|
|
164
|
+
spec="$INSTALL_SPEC"
|
|
165
|
+
# `comber-version` is a convenience pin that only applies to the default spec.
|
|
166
|
+
if [ "$spec" = "comber" ] && [ -n "$COMBER_VERSION" ]; then
|
|
167
|
+
spec="comber@$COMBER_VERSION"
|
|
168
|
+
fi
|
|
169
|
+
echo "Installing comber from: $spec"
|
|
170
|
+
# The default `comber` resolves from npm once a v* tag is published (release.yml +
|
|
171
|
+
# NPM_TOKEN). Pre-publish / private: install-spec: github:RADLABtech/comber#<ref>
|
|
172
|
+
# (its `prepare` script builds dist/ on install).
|
|
173
|
+
( cd "$APP_DIR" && bun add "$spec" )
|
|
174
|
+
echo "app-dir=$APP_DIR" >> "$GITHUB_OUTPUT"
|
|
175
|
+
|
|
176
|
+
- name: Install Playwright Chromium
|
|
177
|
+
# Web target only — the API driver needs no browser.
|
|
178
|
+
if: ${{ inputs.openapi == '' }}
|
|
179
|
+
shell: bash
|
|
180
|
+
env:
|
|
181
|
+
APP_DIR: ${{ steps.install.outputs.app-dir }}
|
|
182
|
+
run: |
|
|
183
|
+
( cd "$APP_DIR" && bunx playwright install --with-deps chromium )
|
|
184
|
+
|
|
185
|
+
- name: Run comber
|
|
186
|
+
id: run
|
|
187
|
+
shell: bash
|
|
188
|
+
env:
|
|
189
|
+
APP_DIR: ${{ steps.install.outputs.app-dir }}
|
|
190
|
+
OUT_DIR: ${{ github.workspace }}/runs
|
|
191
|
+
# ---- consumer-supplied values: env only, NEVER inlined into run: via ${{ }} -------
|
|
192
|
+
TARGET_URL: ${{ inputs.url }}
|
|
193
|
+
OPENAPI: ${{ inputs.openapi }}
|
|
194
|
+
API_BASE: ${{ inputs.api-base }}
|
|
195
|
+
SEED: ${{ inputs.seed }}
|
|
196
|
+
MAX_STATES: ${{ inputs.max-states }}
|
|
197
|
+
USE_LLM: ${{ inputs.llm }}
|
|
198
|
+
CHECK_EXTERNAL_LINKS: ${{ inputs.external-links }}
|
|
199
|
+
ALLOW_WRITES: ${{ inputs.allow-writes }}
|
|
200
|
+
STRICT: ${{ inputs.strict }}
|
|
201
|
+
PR_COMMENT: ${{ inputs.pr-comment }}
|
|
202
|
+
BASELINE: ${{ inputs.baseline }}
|
|
203
|
+
CONFIG_PATH: ${{ inputs.config }}
|
|
204
|
+
PROFILE_NAME: ${{ inputs.profile }}
|
|
205
|
+
# ---- pass-through env consumed directly by comber ---------------------------------
|
|
206
|
+
ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }}
|
|
207
|
+
COMBER_STORAGE_STATE: ${{ inputs.storage-state }}
|
|
208
|
+
COMBER_API_TOKEN: ${{ inputs.api-token }}
|
|
209
|
+
COMBER_SINK_FILE: ${{ inputs.sink-file }}
|
|
210
|
+
COMBER_SINK_DIR: ${{ inputs.sink-dir }}
|
|
211
|
+
COMBER_SINK_URL: ${{ inputs.sink-url }}
|
|
212
|
+
COMBER_SINK_TOKEN: ${{ inputs.sink-token }}
|
|
213
|
+
run: |
|
|
214
|
+
# Build the arg array from env vars only — quoted expansion, no interpolation of
|
|
215
|
+
# untrusted input into the script text (script-injection defense).
|
|
216
|
+
args=()
|
|
217
|
+
if [[ -n "$OPENAPI" ]]; then
|
|
218
|
+
# API target: comber api --openapi <src> --api-base <url>
|
|
219
|
+
args+=(api --openapi "$OPENAPI")
|
|
220
|
+
if [[ -n "$API_BASE" ]]; then args+=(--api-base "$API_BASE"); fi
|
|
221
|
+
if [[ -n "$SEED" ]]; then args+=(--seed "$SEED"); fi
|
|
222
|
+
else
|
|
223
|
+
# Web target: positional URL first.
|
|
224
|
+
if [[ -n "$TARGET_URL" ]]; then args+=("$TARGET_URL"); fi
|
|
225
|
+
fi
|
|
226
|
+
if [[ -n "$CONFIG_PATH" ]]; then args+=(--config "$CONFIG_PATH"); fi
|
|
227
|
+
if [[ -n "$PROFILE_NAME" ]]; then args+=(--profile "$PROFILE_NAME"); fi
|
|
228
|
+
if [[ -n "$MAX_STATES" ]]; then args+=(--max-states "$MAX_STATES"); fi
|
|
229
|
+
if [[ -n "$BASELINE" ]]; then args+=(--baseline "$BASELINE"); fi
|
|
230
|
+
if [[ "$USE_LLM" == "true" ]]; then args+=(--llm); else args+=(--no-llm); fi
|
|
231
|
+
if [[ "$CHECK_EXTERNAL_LINKS" == "true" ]]; then args+=(--external-links); else args+=(--no-external-links); fi
|
|
232
|
+
if [[ "$ALLOW_WRITES" == "true" ]]; then args+=(--allow-writes); else args+=(--block-writes); fi
|
|
233
|
+
if [[ "$STRICT" == "true" ]]; then args+=(--strict); fi
|
|
234
|
+
if [[ "$PR_COMMENT" == "true" ]]; then args+=(--pr-comment "$OUT_DIR/pr-comment.md"); fi
|
|
235
|
+
args+=(--out "$OUT_DIR")
|
|
236
|
+
|
|
237
|
+
# Don't fail the step here — capture the code so the artifact + PR comment still run.
|
|
238
|
+
# The Gate step re-exits with this code (0 clean/soft · 1 strict+fails · 2 crash).
|
|
239
|
+
set +e
|
|
240
|
+
"$APP_DIR/node_modules/.bin/comber" "${args[@]}"
|
|
241
|
+
code=$?
|
|
242
|
+
set -e
|
|
243
|
+
echo "exit-code=$code" >> "$GITHUB_OUTPUT"
|
|
244
|
+
echo "comber exited with $code"
|
|
245
|
+
|
|
246
|
+
- name: Upload report artifact
|
|
247
|
+
if: ${{ always() }}
|
|
248
|
+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
|
249
|
+
with:
|
|
250
|
+
name: comber-report
|
|
251
|
+
path: runs/
|
|
252
|
+
retention-days: 7
|
|
253
|
+
if-no-files-found: warn
|
|
254
|
+
|
|
255
|
+
- name: Comment on PR
|
|
256
|
+
# Non-blocking by design: a comment failure must never fail the run.
|
|
257
|
+
if: ${{ always() && inputs.pr-comment == 'true' && hashFiles('runs/pr-comment.md') != '' }}
|
|
258
|
+
continue-on-error: true
|
|
259
|
+
shell: bash
|
|
260
|
+
env:
|
|
261
|
+
GH_TOKEN: ${{ inputs.github-token }}
|
|
262
|
+
PR_NUMBER: ${{ inputs.pr-number != '' && inputs.pr-number || github.event.pull_request.number }}
|
|
263
|
+
COMMENT_FILE: ${{ github.workspace }}/runs/pr-comment.md
|
|
264
|
+
run: |
|
|
265
|
+
if [[ -z "$PR_NUMBER" ]]; then
|
|
266
|
+
echo "No PR number resolved (not a pull_request and no pr-number input) — skipping comment."
|
|
267
|
+
exit 0
|
|
268
|
+
fi
|
|
269
|
+
gh pr comment "$PR_NUMBER" --body-file "$COMMENT_FILE"
|
|
270
|
+
|
|
271
|
+
- name: Gate
|
|
272
|
+
# Final gate: re-exit with comber's code. Job fails only on strict+fails (1) or crash (2).
|
|
273
|
+
if: ${{ always() }}
|
|
274
|
+
shell: bash
|
|
275
|
+
env:
|
|
276
|
+
CODE: ${{ steps.run.outputs.exit-code }}
|
|
277
|
+
run: |
|
|
278
|
+
echo "Comber exit code: ${CODE:-<none>}"
|
|
279
|
+
exit "${CODE:-0}"
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
{
|
|
2
|
+
"defaultProfile": "local",
|
|
3
|
+
"profiles": {
|
|
4
|
+
"local": {
|
|
5
|
+
"url": "https://example.com",
|
|
6
|
+
"maxStates": 20,
|
|
7
|
+
"maxPerState": 20,
|
|
8
|
+
"maxMs": 180000,
|
|
9
|
+
"externalLinks": false,
|
|
10
|
+
"llm": false,
|
|
11
|
+
"allowWrites": false,
|
|
12
|
+
"baseline": "baselines/local-result.json",
|
|
13
|
+
"prComment": "runs/pr-comment.md"
|
|
14
|
+
},
|
|
15
|
+
"preview": {
|
|
16
|
+
"url": "https://preview.example.com",
|
|
17
|
+
"allowedOrigins": ["https://assets.example.com"],
|
|
18
|
+
"maxStates": 40,
|
|
19
|
+
"maxPerState": 40,
|
|
20
|
+
"maxVisitsPerRoute": 3,
|
|
21
|
+
"nearDup": 0.92,
|
|
22
|
+
"externalLinks": true,
|
|
23
|
+
"maxExternalLinks": 300,
|
|
24
|
+
"linkTimeoutMs": 10000,
|
|
25
|
+
"linkConcurrency": 8,
|
|
26
|
+
"llm": true,
|
|
27
|
+
"allowWrites": false,
|
|
28
|
+
"storageState": "./auth/synthetic-user.json",
|
|
29
|
+
"baseline": "baselines/preview-result.json",
|
|
30
|
+
"prComment": "runs/pr-comment.md",
|
|
31
|
+
"connectors": [
|
|
32
|
+
{ "use": "slack", "on": "fail", "mentionOnFail": "<!here>" },
|
|
33
|
+
{ "use": "junit", "outFile": "runs/comber-junit.xml" },
|
|
34
|
+
{ "use": "webhook", "on": "warn" }
|
|
35
|
+
]
|
|
36
|
+
},
|
|
37
|
+
"mobile": {
|
|
38
|
+
"target": "native",
|
|
39
|
+
"app": "com.example.app",
|
|
40
|
+
"platform": "android",
|
|
41
|
+
"device": "emulator-5554",
|
|
42
|
+
"deeplinks": ["myapp://settings", "myapp://profile"],
|
|
43
|
+
"maxStates": 10
|
|
44
|
+
},
|
|
45
|
+
"api": {
|
|
46
|
+
"target": "api",
|
|
47
|
+
"openapi": "./openapi.yaml",
|
|
48
|
+
"apiBase": "https://api-sandbox.example.com",
|
|
49
|
+
"seed": "./api-seed.json",
|
|
50
|
+
"allowWrites": false,
|
|
51
|
+
"apiAuthProfiles": { "alice": "", "bob": "" },
|
|
52
|
+
"ownerScoped": ["GET /users/{id}", "GET /orders/{id}"],
|
|
53
|
+
"roleSeed": "./role-seed.json"
|
|
54
|
+
},
|
|
55
|
+
"graphql": {
|
|
56
|
+
"target": "api",
|
|
57
|
+
"graphql": "https://api-sandbox.example.com/graphql"
|
|
58
|
+
},
|
|
59
|
+
"har": {
|
|
60
|
+
"target": "api",
|
|
61
|
+
"har": "./recorded-traffic.har"
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { documentedStatusList, isDocumented, operationLabel, responseFor } from "./openapi.js";
|
|
2
|
+
import { formatViolations } from "./schema.js";
|
|
3
|
+
/** Dedupe key — keyed on the route template + signal kind, per the API design. The same
|
|
4
|
+
* broken operation re-seen collapses to one finding (core dedupes on `signature`). */
|
|
5
|
+
export function signatureOf(op, kind) {
|
|
6
|
+
return `${kind}|${op.method} ${op.path}`.slice(0, 80);
|
|
7
|
+
}
|
|
8
|
+
/** Construct a BreakageSignal for an operation. `url` is the concrete request URL (a real
|
|
9
|
+
* URL so the core's redactor scrubs query/userinfo); the message carries METHOD + template. */
|
|
10
|
+
export function apiSignal(op, kind, severity, message, where) {
|
|
11
|
+
return {
|
|
12
|
+
kind,
|
|
13
|
+
severity,
|
|
14
|
+
message,
|
|
15
|
+
url: where,
|
|
16
|
+
signature: signatureOf(op, kind),
|
|
17
|
+
action: operationLabel(op),
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
/** A documented media type matches the actual base type, honoring `*` wildcards. */
|
|
21
|
+
function mediaMatches(declared, actual) {
|
|
22
|
+
if (declared === "*/*" || declared === actual)
|
|
23
|
+
return true;
|
|
24
|
+
const [dType, dSub] = declared.split("/");
|
|
25
|
+
const [aType] = actual.split("/");
|
|
26
|
+
return dSub === "*" && dType === aType;
|
|
27
|
+
}
|
|
28
|
+
function jsonSchemaOf(resp, resolver) {
|
|
29
|
+
const media = resp.content["application/json"] ?? Object.entries(resp.content).find(([mt]) => /\bjson\b/i.test(mt))?.[1];
|
|
30
|
+
return media ? resolver.deref(media.schema) : undefined;
|
|
31
|
+
}
|
|
32
|
+
/** Light structural check: which top-level required properties the body is missing. An
|
|
33
|
+
* object schema whose body isn't a JSON object reports ALL required keys as missing. */
|
|
34
|
+
function missingRequired(schema, bodyText, resolver) {
|
|
35
|
+
const s = resolver.deref(schema);
|
|
36
|
+
if (!s)
|
|
37
|
+
return [];
|
|
38
|
+
const required = Array.isArray(s.required) ? s.required.filter((k) => typeof k === "string") : [];
|
|
39
|
+
if (required.length === 0)
|
|
40
|
+
return [];
|
|
41
|
+
let body;
|
|
42
|
+
try {
|
|
43
|
+
body = JSON.parse(bodyText);
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
return []; // non-JSON body is a content-type concern, not a shape one
|
|
47
|
+
}
|
|
48
|
+
if (body === null || typeof body !== "object" || Array.isArray(body))
|
|
49
|
+
return required;
|
|
50
|
+
return required.filter((k) => !(k in body));
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Classify a response into zero or more breakage signals. A 5xx short-circuits (the body is
|
|
54
|
+
* an error page, not the documented shape). Otherwise: undocumented-status, then — against
|
|
55
|
+
* the best-matching documented response — content-type-mismatch and (for 2xx) full JSON-Schema
|
|
56
|
+
* validation (ajv) when a `validator` + schema are present, else the light required-prop check.
|
|
57
|
+
*/
|
|
58
|
+
export function classifyResponse(op, res, where, resolver, validator) {
|
|
59
|
+
const out = [];
|
|
60
|
+
const label = operationLabel(op);
|
|
61
|
+
if (res.status >= 500) {
|
|
62
|
+
out.push(apiSignal(op, "http-5xx", "fail", `${label} → HTTP ${res.status} (server error)`, where));
|
|
63
|
+
return out;
|
|
64
|
+
}
|
|
65
|
+
if (!isDocumented(op, res.status)) {
|
|
66
|
+
out.push(apiSignal(op, "undocumented-status", "warn", `${label} → HTTP ${res.status} is not among documented responses [${documentedStatusList(op)}]`, where));
|
|
67
|
+
}
|
|
68
|
+
const matched = responseFor(op, res.status);
|
|
69
|
+
if (!matched)
|
|
70
|
+
return out;
|
|
71
|
+
const declared = Object.keys(matched.content);
|
|
72
|
+
if (declared.length > 0 && res.contentType) {
|
|
73
|
+
const actual = res.contentType.split(";")[0].trim().toLowerCase();
|
|
74
|
+
if (actual && !declared.some((d) => mediaMatches(d.toLowerCase(), actual))) {
|
|
75
|
+
out.push(apiSignal(op, "content-type-mismatch", "warn", `${label} → Content-Type "${actual}" is not among documented [${declared.join(", ")}]`, where));
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
if (res.status >= 200 && res.status < 300) {
|
|
79
|
+
const schema = jsonSchemaOf(matched, resolver);
|
|
80
|
+
if (schema) {
|
|
81
|
+
// PRIMARY: full JSON-Schema validation via ajv (resolves $refs). Emits the offending
|
|
82
|
+
// PATHS only (redaction-safe — schema-derived messages, never body values).
|
|
83
|
+
const result = validator?.validate(schema, res.bodyText);
|
|
84
|
+
if (result?.validated) {
|
|
85
|
+
if (!result.ok) {
|
|
86
|
+
out.push(apiSignal(op, "response-schema-violation", "fail", `${label} → response body violates schema: ${formatViolations(result.violations)}`, where));
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
// FALLBACK (no validator, or schema didn't compile): the light required-prop check.
|
|
91
|
+
const missing = missingRequired(schema, res.bodyText, resolver);
|
|
92
|
+
if (missing.length > 0) {
|
|
93
|
+
out.push(apiSignal(op, "response-shape-mismatch", "fail", `${label} → response is missing required propert${missing.length > 1 ? "ies" : "y"}: ${missing.join(", ")}`, where));
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
return out;
|
|
99
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"classify.js","sourceRoot":"","sources":["../../src/api/classify.ts"],"names":[],"mappings":"AAkBA,OAAO,EAAE,oBAAoB,EAAE,YAAY,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC/F,OAAO,EAA0B,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAqBvE;uFACuF;AACvF,MAAM,UAAU,WAAW,CAAC,EAAgB,EAAE,IAAmB;IAC/D,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,MAAM,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACxD,CAAC;AAED;gGACgG;AAChG,MAAM,UAAU,SAAS,CACvB,EAAgB,EAChB,IAAmB,EACnB,QAAyB,EACzB,OAAe,EACf,KAAa;IAEb,OAAO;QACL,IAAI;QACJ,QAAQ;QACR,OAAO;QACP,GAAG,EAAE,KAAK;QACV,SAAS,EAAE,WAAW,CAAC,EAAE,EAAE,IAAI,CAAC;QAChC,MAAM,EAAE,cAAc,CAAC,EAAE,CAAC;KAC3B,CAAC;AACJ,CAAC;AAED,oFAAoF;AACpF,SAAS,YAAY,CAAC,QAAgB,EAAE,MAAc;IACpD,IAAI,QAAQ,KAAK,KAAK,IAAI,QAAQ,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IAC3D,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC1C,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAClC,OAAO,IAAI,KAAK,GAAG,IAAI,KAAK,KAAK,KAAK,CAAC;AACzC,CAAC;AAED,SAAS,YAAY,CAAC,IAAqB,EAAE,QAAsB;IACjE,MAAM,KAAK,GACT,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC7G,OAAO,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAyB,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAClF,CAAC;AAED;yFACyF;AACzF,SAAS,eAAe,CAAC,MAAkB,EAAE,QAAgB,EAAE,QAAsB;IACnF,MAAM,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAyB,MAAM,CAAC,CAAC;IACzD,IAAI,CAAC,CAAC;QAAE,OAAO,EAAE,CAAC;IAClB,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/G,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACrC,IAAI,IAAa,CAAC;IAClB,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC,CAAC,2DAA2D;IACxE,CAAC;IACD,IAAI,IAAI,KAAK,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,OAAO,QAAQ,CAAC;IACtF,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAK,IAAgC,CAAC,CAAC,CAAC;AAC3E,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAC9B,EAAgB,EAChB,GAAgB,EAChB,KAAa,EACb,QAAsB,EACtB,SAA6B;IAE7B,MAAM,GAAG,GAAqB,EAAE,CAAC;IACjC,MAAM,KAAK,GAAG,cAAc,CAAC,EAAE,CAAC,CAAC;IAEjC,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;QACtB,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,KAAK,WAAW,GAAG,CAAC,MAAM,iBAAiB,EAAE,KAAK,CAAC,CAAC,CAAC;QACnG,OAAO,GAAG,CAAC;IACb,CAAC;IAED,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;QAClC,GAAG,CAAC,IAAI,CACN,SAAS,CACP,EAAE,EACF,qBAAqB,EACrB,MAAM,EACN,GAAG,KAAK,WAAW,GAAG,CAAC,MAAM,uCAAuC,oBAAoB,CAAC,EAAE,CAAC,GAAG,EAC/F,KAAK,CACN,CACF,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,WAAW,CAAC,EAAE,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC5C,IAAI,CAAC,OAAO;QAAE,OAAO,GAAG,CAAC;IAEzB,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC9C,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,MAAM,GAAG,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACnE,IAAI,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC;YAC3E,GAAG,CAAC,IAAI,CACN,SAAS,CACP,EAAE,EACF,uBAAuB,EACvB,MAAM,EACN,GAAG,KAAK,oBAAoB,MAAM,8BAA8B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EACtF,KAAK,CACN,CACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAC/C,IAAI,MAAM,EAAE,CAAC;YACX,qFAAqF;YACrF,4EAA4E;YAC5E,MAAM,MAAM,GAAG,SAAS,EAAE,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;YACzD,IAAI,MAAM,EAAE,SAAS,EAAE,CAAC;gBACtB,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;oBACf,GAAG,CAAC,IAAI,CACN,SAAS,CACP,EAAE,EACF,2BAA2B,EAC3B,MAAM,EACN,GAAG,KAAK,qCAAqC,gBAAgB,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,EAClF,KAAK,CACN,CACF,CAAC;gBACJ,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,oFAAoF;gBACpF,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,EAAE,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;gBAChE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACvB,GAAG,CAAC,IAAI,CACN,SAAS,CACP,EAAE,EACF,yBAAyB,EACzB,MAAM,EACN,GAAG,KAAK,0CAA0C,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAC3G,KAAK,CACN,CACF,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC"}
|