@osfactory/har 0.50.1 → 0.51.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1 -1
- package/dist/templates/har-boilerplate/STAGES.md +3 -2
- package/dist/templates/har-boilerplate-cli/STAGES.md +3 -2
- package/dist/templates/har-boilerplate-ios/STAGES.md +3 -2
- package/dist/templates/plugins/kerno/.github/workflows/kerno.yml +43 -0
- package/dist/templates/plugins/kerno/.har/stages/KERNO.md +115 -0
- package/dist/templates/plugins/kerno/.har/stages/backend-validation.sh +307 -0
- package/dist/templates/plugins/kerno/README.md +67 -0
- package/dist/templates/plugins/kerno/template.manifest.json +46 -0
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -69193,7 +69193,7 @@ function harnessExists(repoPath) {
|
|
|
69193
69193
|
// src/harness/plugins.ts
|
|
69194
69194
|
init_schema2();
|
|
69195
69195
|
init_stages();
|
|
69196
|
-
var PLUGIN_IDS = ["playwright", "rocketsim"];
|
|
69196
|
+
var PLUGIN_IDS = ["playwright", "rocketsim", "kerno"];
|
|
69197
69197
|
var PluginManifestFileSchema = external_exports.object({
|
|
69198
69198
|
src: external_exports.string().min(1),
|
|
69199
69199
|
dest: external_exports.string().min(1),
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# HAR Stages — authoring guide
|
|
2
2
|
|
|
3
3
|
Stages are the harness's single vocabulary for runnable checks and lifecycle
|
|
4
|
-
actions. Everything — shipped plugins (`playwright`, `rocketsim`), your
|
|
4
|
+
actions. Everything — shipped plugins (`playwright`, `rocketsim`, `kerno`), your
|
|
5
5
|
project's test/lint commands, bespoke validation scripts — registers in
|
|
6
6
|
`.har/stages.json` with the same schema, and agents interact with stages only
|
|
7
7
|
through that registry (CLI `har env verify`, MCP `har_run_stage` /
|
|
@@ -90,6 +90,7 @@ committed.
|
|
|
90
90
|
## Shipped plugins
|
|
91
91
|
|
|
92
92
|
`har env add-plugin --list` shows available plugins; `har env add-plugin
|
|
93
|
-
playwright` (web)
|
|
93
|
+
playwright` (web), `har env add-plugin rocketsim` (iOS), or `har env
|
|
94
|
+
add-plugin kerno` (backend) installs one. A
|
|
94
95
|
plugin is just packaging: it copies files, merges `package.json` fragments,
|
|
95
96
|
and registers stages through the exact same registry as `add-stage --custom`.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# HAR Stages — authoring guide
|
|
2
2
|
|
|
3
3
|
Stages are the harness's single vocabulary for runnable checks and lifecycle
|
|
4
|
-
actions. Everything — shipped plugins (`playwright`, `rocketsim`), your
|
|
4
|
+
actions. Everything — shipped plugins (`playwright`, `rocketsim`, `kerno`), your
|
|
5
5
|
project's test/lint commands, bespoke validation scripts — registers in
|
|
6
6
|
`.har/stages.json` with the same schema, and agents interact with stages only
|
|
7
7
|
through that registry (CLI `har env verify`, MCP `har_run_stage` /
|
|
@@ -90,6 +90,7 @@ committed.
|
|
|
90
90
|
## Shipped plugins
|
|
91
91
|
|
|
92
92
|
`har env add-plugin --list` shows available plugins; `har env add-plugin
|
|
93
|
-
playwright` (web)
|
|
93
|
+
playwright` (web), `har env add-plugin rocketsim` (iOS), or `har env
|
|
94
|
+
add-plugin kerno` (backend) installs one. A
|
|
94
95
|
plugin is just packaging: it copies files, merges `package.json` fragments,
|
|
95
96
|
and registers stages through the exact same registry as `add-stage --custom`.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# HAR Stages — authoring guide
|
|
2
2
|
|
|
3
3
|
Stages are the harness's single vocabulary for runnable checks and lifecycle
|
|
4
|
-
actions. Everything — shipped plugins (`playwright`, `rocketsim`), your
|
|
4
|
+
actions. Everything — shipped plugins (`playwright`, `rocketsim`, `kerno`), your
|
|
5
5
|
project's test/lint commands, bespoke validation scripts — registers in
|
|
6
6
|
`.har/stages.json` with the same schema, and agents interact with stages only
|
|
7
7
|
through that registry (CLI `har env verify`, MCP `har_run_stage` /
|
|
@@ -90,6 +90,7 @@ committed.
|
|
|
90
90
|
## Shipped plugins
|
|
91
91
|
|
|
92
92
|
`har env add-plugin --list` shows available plugins; `har env add-plugin
|
|
93
|
-
playwright` (web)
|
|
93
|
+
playwright` (web), `har env add-plugin rocketsim` (iOS), or `har env
|
|
94
|
+
add-plugin kerno` (backend) installs one. A
|
|
94
95
|
plugin is just packaging: it copies files, merges `package.json` fragments,
|
|
95
96
|
and registers stages through the exact same registry as `add-stage --custom`.
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# Kerno backend validation (CI template).
|
|
2
|
+
#
|
|
3
|
+
# This is a starting scaffold, not a drop-in. The backend-validation stage needs three
|
|
4
|
+
# things that are repo-specific, so wire them up before enabling this workflow:
|
|
5
|
+
# 1. Docker available on the runner (Kerno runs scenarios in a sandbox container).
|
|
6
|
+
# 2. Your app running and reachable at the SUT URL, with its database migrated/seeded.
|
|
7
|
+
# 3. A Kerno agent bound to this checkout (`kerno init`) with a committed suite in .kerno/.
|
|
8
|
+
#
|
|
9
|
+
# The stage sources the HAR slot machinery (.har/harness.env, agent-slot.sh, .env.agent.<id>),
|
|
10
|
+
# so the supported path is to run it inside a launched HAR slot. Adapt the "start app" and
|
|
11
|
+
# "kerno init" steps for your stack. Remove `if: false` once wired up.
|
|
12
|
+
name: Kerno backend validation
|
|
13
|
+
|
|
14
|
+
on:
|
|
15
|
+
push:
|
|
16
|
+
pull_request:
|
|
17
|
+
|
|
18
|
+
jobs:
|
|
19
|
+
backend-validation:
|
|
20
|
+
if: false # ← remove after wiring up app startup + `kerno init` below
|
|
21
|
+
runs-on: ubuntu-latest
|
|
22
|
+
steps:
|
|
23
|
+
- uses: actions/checkout@v4
|
|
24
|
+
- name: Setup Node
|
|
25
|
+
uses: actions/setup-node@v4
|
|
26
|
+
with:
|
|
27
|
+
node-version: 20
|
|
28
|
+
- name: Install deps
|
|
29
|
+
run: npm ci
|
|
30
|
+
- name: Install Kerno CLI
|
|
31
|
+
run: npm install -g @kerno/cli
|
|
32
|
+
# - name: Start the app (adapt for your stack) and migrate/seed its database
|
|
33
|
+
# run: ./.har/launch.sh 1 && <start your app in the slot>
|
|
34
|
+
# - name: Bind a Kerno agent to this checkout
|
|
35
|
+
# run: kerno init
|
|
36
|
+
# - name: Run backend validation
|
|
37
|
+
# run: ./.har/stages/backend-validation.sh 1
|
|
38
|
+
- name: Upload Kerno artifacts
|
|
39
|
+
if: success() || failure()
|
|
40
|
+
uses: actions/upload-artifact@v4
|
|
41
|
+
with:
|
|
42
|
+
name: backend-validation
|
|
43
|
+
path: .har/artifacts/backend-validation
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
# Kerno Backend Validation
|
|
2
|
+
|
|
3
|
+
The `backend-validation` stage re-runs your committed [Kerno](https://kerno.io) scenario
|
|
4
|
+
suite against the app running in an agent slot, deterministically and with no LLM in the
|
|
5
|
+
loop. It turns the slot's isolated running app plus its per-slot database into a real
|
|
6
|
+
backend-validation target, then leaves a pass/fail exit code and an evidence trail.
|
|
7
|
+
|
|
8
|
+
Full verify (`verify --full`) runs this stage when `backend-validation` is listed in
|
|
9
|
+
`verificationStages` (the plugin adds it for you).
|
|
10
|
+
|
|
11
|
+
## How it works
|
|
12
|
+
|
|
13
|
+
For the slot you name, the stage:
|
|
14
|
+
|
|
15
|
+
1. Acquires a machine-wide lock so two slots never validate at once (see "One agent at a time").
|
|
16
|
+
2. Preflights that a Kerno agent is running and bound to this worktree, that Docker is up, and
|
|
17
|
+
that a committed suite exists under `.kerno/scenarios/`.
|
|
18
|
+
3. Points Kerno's target config at this slot over REST (the slot's SUT URL and, for greybox, its
|
|
19
|
+
per-slot database).
|
|
20
|
+
4. Runs every endpoint's scenarios via `POST /scenarios/run` and reads the verdicts straight from
|
|
21
|
+
the HTTP response.
|
|
22
|
+
5. Writes per-endpoint run responses and any fresh `report.json` files under
|
|
23
|
+
`.har/artifacts/backend-validation/`, and exits non-zero if any scenario is not `passed`.
|
|
24
|
+
|
|
25
|
+
## Prerequisites
|
|
26
|
+
|
|
27
|
+
- **Kerno CLI** installed: `npm install -g @kerno/cli`.
|
|
28
|
+
- **Docker** running. Kerno executes each scenario inside a sandbox container.
|
|
29
|
+
- **A Kerno agent bound to this worktree.** Run `kerno init` inside the agent slot's worktree so
|
|
30
|
+
Kerno's workspace path matches the slot. The stage never starts or rebinds the agent itself
|
|
31
|
+
(starting one would kill any other running Kerno agent, see below).
|
|
32
|
+
- **A committed suite** under `.kerno/scenarios/endpoints/`. Validate re-runs an existing suite,
|
|
33
|
+
it never generates one. Ask your Kerno agent to generate scenarios first, then commit them.
|
|
34
|
+
|
|
35
|
+
## One agent at a time
|
|
36
|
+
|
|
37
|
+
Kerno runs a single agent per machine. Starting a second agent (via `kerno init`) kills any other
|
|
38
|
+
running Kerno agent and can tear down its in-flight sandboxes. Because of that, this stage:
|
|
39
|
+
|
|
40
|
+
- **Never spawns or rebinds** the agent. If none is bound to this worktree, it fails with an
|
|
41
|
+
instruction to run `kerno init` here.
|
|
42
|
+
- **Serializes across slots** with a fail-fast lock at `~/.kerno/har-backend-validation.lock`. If a
|
|
43
|
+
second slot runs the stage while one is validating, the second fails fast rather than colliding.
|
|
44
|
+
|
|
45
|
+
So backend validation runs one slot at a time. Frontend stages (Playwright, and so on) still run
|
|
46
|
+
concurrently across slots.
|
|
47
|
+
|
|
48
|
+
## Configuration the stage reads
|
|
49
|
+
|
|
50
|
+
All values come from the slot's `.env.agent.<id>` (written by `launch.sh`) and `~/.kerno/agent.port`:
|
|
51
|
+
|
|
52
|
+
| Value | Source | Used for |
|
|
53
|
+
|-------|--------|----------|
|
|
54
|
+
| `API_PORT` / `SUT_URL` | slot env | the SUT base URL Kerno tests (`http://localhost:$API_PORT`) |
|
|
55
|
+
| `DATABASE_URL` / `DB_PORT` | slot env | greybox DB target (`agent_<id>`); omitted for black-box apps |
|
|
56
|
+
| `KERNO_AGENT_PORT` or `~/.kerno/agent.port` | Kerno state | which local Kerno agent to call (default `8085`) |
|
|
57
|
+
| `KERNO_APP_ID` (optional) | you | pin the application id when Kerno does not surface one (see below) |
|
|
58
|
+
| `KERNO_SANDBOX_ENV` (optional) | you | extra env injected into the Kerno sandbox for your scenarios (see below) |
|
|
59
|
+
|
|
60
|
+
Override the agent port with `KERNO_AGENT_PORT` if you run Kerno on a non-default port.
|
|
61
|
+
|
|
62
|
+
### Scenario env (`KERNO_SANDBOX_ENV`)
|
|
63
|
+
|
|
64
|
+
Some suites need runtime values, a test-user email, an API token, a feature flag, that
|
|
65
|
+
the scenarios read from the sandbox environment. Pass them as newline- or
|
|
66
|
+
semicolon-separated `KEY=VALUE` pairs; keys must be upper-snake-case. They are injected
|
|
67
|
+
into the Kerno sandbox verbatim alongside the slot database.
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
KERNO_SANDBOX_ENV="KERNO_TEST_USER_EMAIL=dev@example.com;FEATURE_X=on" \
|
|
71
|
+
./.har/stages/backend-validation.sh 1
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
For an authenticated suite, this is usually where the test user goes. The user must exist
|
|
75
|
+
in the slot's seeded database, and the app must accept it (for example a dev-login provider
|
|
76
|
+
enabled in the running app).
|
|
77
|
+
|
|
78
|
+
If Kerno analyzed your app but did not classify it as HTTP-serving, it is omitted from
|
|
79
|
+
the workspace's application list. The stage falls back to the analyzed module and logs a
|
|
80
|
+
note, so validation still runs. To pin the id explicitly, set `KERNO_APP_ID` (find it via
|
|
81
|
+
`kerno_get_applications`, under `unsupported_apps` when this happens).
|
|
82
|
+
|
|
83
|
+
## Verdicts
|
|
84
|
+
|
|
85
|
+
Overall pass requires every scenario across every endpoint to be `passed`. A `failed`, `blocked`,
|
|
86
|
+
or `not_implemented` scenario fails the stage. `blocked` usually means the target or its DB was not
|
|
87
|
+
reachable or a precondition could not be met, so check the run response in the artifacts.
|
|
88
|
+
|
|
89
|
+
## Artifacts
|
|
90
|
+
|
|
91
|
+
Under `.har/artifacts/backend-validation/`:
|
|
92
|
+
|
|
93
|
+
- `<METHOD>_<path>.run.json` for each endpoint: the raw `POST /scenarios/run` response, including
|
|
94
|
+
per-scenario `verdict`, response status/body, and any `potentialBug`.
|
|
95
|
+
- Copies of Kerno's own `report.json` files written during this run.
|
|
96
|
+
|
|
97
|
+
## Running
|
|
98
|
+
|
|
99
|
+
```bash
|
|
100
|
+
# One slot
|
|
101
|
+
./.har/stages/backend-validation.sh 1
|
|
102
|
+
|
|
103
|
+
# As part of full verify
|
|
104
|
+
./.har/verify.sh 1 --full
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
## Adapting per repo
|
|
108
|
+
|
|
109
|
+
- **Auth and runtime env vars** your scenarios or SUT need at runtime belong in the Kerno config
|
|
110
|
+
(`kerno save-config` via your agent, or the `.kerno/config.yaml` dependency blocks). The stage sets
|
|
111
|
+
only the SUT URL and the slot DB.
|
|
112
|
+
- **Black-box only** apps (no slot DB) are fine. Leave `DATABASE_URL`/`DB_PORT` unset in the slot and
|
|
113
|
+
the stage tests over HTTP only.
|
|
114
|
+
- **Regenerate on new endpoints.** When you add an endpoint, ask your Kerno agent to generate its
|
|
115
|
+
scenarios and commit them, so the next validation covers it.
|
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Kerno deterministic backend/API scenario validation for an agent slot.
|
|
3
|
+
# Re-runs the committed Kerno suite (.kerno/scenarios/) against the running slot
|
|
4
|
+
# via the local Kerno agent's REST API. Outputs JSON to stdout, progress to stderr.
|
|
5
|
+
#
|
|
6
|
+
# Usage: ./.har/stages/backend-validation.sh <agent-id>
|
|
7
|
+
# Prerequisite: ./.har/launch.sh <agent-id>, the app running in the slot, AND a Kerno
|
|
8
|
+
# agent started with `kerno init` in THIS worktree (Kerno allows ONE agent per machine).
|
|
9
|
+
# See: ./.har/stages/KERNO.md for the full setup + adaptation guide.
|
|
10
|
+
set -euo pipefail
|
|
11
|
+
|
|
12
|
+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
13
|
+
HARNESS_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
14
|
+
REPO_ROOT="$(cd "$HARNESS_DIR/.." && pwd)"
|
|
15
|
+
# agent-slot.sh expects SCRIPT_DIR to be .har/ (slot registry lives there)
|
|
16
|
+
SCRIPT_DIR="$HARNESS_DIR"
|
|
17
|
+
|
|
18
|
+
# shellcheck source=/dev/null
|
|
19
|
+
source "$HARNESS_DIR/harness.env"
|
|
20
|
+
# shellcheck source=/dev/null
|
|
21
|
+
source "$HARNESS_DIR/agent-slot.sh"
|
|
22
|
+
|
|
23
|
+
AGENT_ID="${1:?Usage: backend-validation.sh <agent-id>}"
|
|
24
|
+
validate_agent_id "$AGENT_ID"
|
|
25
|
+
|
|
26
|
+
log() { echo "==> [backend-validation agent-$AGENT_ID] $*" >&2; }
|
|
27
|
+
|
|
28
|
+
ARTIFACT_DIR="$REPO_ROOT/.har/artifacts/backend-validation"
|
|
29
|
+
TOTAL_MS=0
|
|
30
|
+
|
|
31
|
+
# Emit a result JSON envelope to stdout (used for both early failures and the final result).
|
|
32
|
+
emit_result() {
|
|
33
|
+
local status="$1"; local message="${2:-}"
|
|
34
|
+
STATUS="$status" MSG="$message" AID="$AGENT_ID" TMS="$TOTAL_MS" SUT="${API_URL:-}" node -e "
|
|
35
|
+
const out = {
|
|
36
|
+
status: process.env.STATUS,
|
|
37
|
+
stageId: 'backend-validation',
|
|
38
|
+
kind: 'test',
|
|
39
|
+
agent_id: Number(process.env.AID),
|
|
40
|
+
total_ms: Number(process.env.TMS || 0),
|
|
41
|
+
artifacts: [{ path: '.har/artifacts/backend-validation', kind: 'directory' }],
|
|
42
|
+
};
|
|
43
|
+
if (process.env.MSG) out.message = process.env.MSG;
|
|
44
|
+
if (process.env.SUT) out.urls = [{ label: 'sut', url: process.env.SUT }];
|
|
45
|
+
process.stdout.write(JSON.stringify(out, null, 2) + '\n');
|
|
46
|
+
"
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
# ── Rule 2: machine-level lock (Kerno allows one agent at a time) ──────────────
|
|
50
|
+
# Stock macOS bash 3.2 has no `flock` binary, so use an atomic mkdir lock with a
|
|
51
|
+
# stale-PID check. Fail fast (never block) so a second slot can't stomp a live run.
|
|
52
|
+
LOCK_DIR="$HOME/.kerno/har-backend-validation.lock"
|
|
53
|
+
MARKER=""
|
|
54
|
+
cleanup() {
|
|
55
|
+
[ -n "${LOCK_DIR:-}" ] && rm -rf "$LOCK_DIR" 2>/dev/null || true
|
|
56
|
+
[ -n "${MARKER:-}" ] && rm -f "$MARKER" 2>/dev/null || true
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
fail() {
|
|
60
|
+
log "✗ $*"
|
|
61
|
+
emit_result fail "$*"
|
|
62
|
+
exit 1
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
acquire_lock() {
|
|
66
|
+
mkdir -p "$HOME/.kerno" 2>/dev/null || true
|
|
67
|
+
if mkdir "$LOCK_DIR" 2>/dev/null; then echo "$$" > "$LOCK_DIR/pid"; return 0; fi
|
|
68
|
+
local other=""
|
|
69
|
+
[ -f "$LOCK_DIR/pid" ] && other="$(cat "$LOCK_DIR/pid" 2>/dev/null || true)"
|
|
70
|
+
if [ -n "$other" ] && kill -0 "$other" 2>/dev/null; then return 1; fi
|
|
71
|
+
# stale lock — reclaim once
|
|
72
|
+
rm -rf "$LOCK_DIR" 2>/dev/null || true
|
|
73
|
+
if mkdir "$LOCK_DIR" 2>/dev/null; then echo "$$" > "$LOCK_DIR/pid"; return 0; fi
|
|
74
|
+
return 1
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
if ! acquire_lock; then
|
|
78
|
+
# Do NOT trap-remove a lock we don't own.
|
|
79
|
+
fail "another Kerno validation is already in progress on this machine (lock: $LOCK_DIR). Kerno allows one agent at a time — retry after it finishes."
|
|
80
|
+
fi
|
|
81
|
+
trap cleanup EXIT
|
|
82
|
+
|
|
83
|
+
# ── Resolve agent env + slot target ───────────────────────────────────────────
|
|
84
|
+
ENV_FILE="$(resolve_agent_env_file "$AGENT_ID" "$REPO_ROOT")" || {
|
|
85
|
+
echo "No .env.agent.${AGENT_ID} found." >&2
|
|
86
|
+
har_suggest_launch "$AGENT_ID" >&2
|
|
87
|
+
exit 1
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
set -a
|
|
91
|
+
# shellcheck source=/dev/null
|
|
92
|
+
source "$ENV_FILE"
|
|
93
|
+
set +a
|
|
94
|
+
|
|
95
|
+
WORK_DIR="$(resolve_agent_work_dir "$ENV_FILE")"
|
|
96
|
+
|
|
97
|
+
# SUT URL: prefer an explicit override, then the app's own SITE_URL (both the default
|
|
98
|
+
# web profile and single-port apps like Next.js set SITE_URL to the running app), then a
|
|
99
|
+
# port fallback (PORT for single-port apps, else API_PORT).
|
|
100
|
+
APP_PORT="${PORT:-${API_PORT:-${FE_PORT:-}}}"
|
|
101
|
+
API_URL="${SUT_URL:-${SITE_URL:-}}"
|
|
102
|
+
if [ -z "$API_URL" ] && [ -n "$APP_PORT" ]; then
|
|
103
|
+
API_URL="http://localhost:${APP_PORT}"
|
|
104
|
+
fi
|
|
105
|
+
[ -n "$API_URL" ] || fail "Could not determine the slot SUT URL. Set SUT_URL, or ensure the slot env defines SITE_URL or PORT/API_PORT."
|
|
106
|
+
|
|
107
|
+
# Slot database (greybox). The launch env file already exports DATABASE_URL for the
|
|
108
|
+
# per-slot DB (agent_<id>); reconstruct it from DB_PORT only if absent. Omitted for
|
|
109
|
+
# black-box (HTTP-only) apps that expose no slot DB.
|
|
110
|
+
DB_URL="${DATABASE_URL:-}"
|
|
111
|
+
if [ -z "$DB_URL" ] && [ -n "${DB_PORT:-}" ]; then
|
|
112
|
+
DB_URL="postgresql://postgres:password@localhost:${DB_PORT}/agent_${AGENT_ID}"
|
|
113
|
+
fi
|
|
114
|
+
|
|
115
|
+
# ── Discover the Kerno agent (never spawn it — spawning kills other agents) ────
|
|
116
|
+
KERNO_PORT="${KERNO_AGENT_PORT:-}"
|
|
117
|
+
if [ -z "$KERNO_PORT" ] && [ -f "$HOME/.kerno/agent.port" ]; then
|
|
118
|
+
KERNO_PORT="$(cat "$HOME/.kerno/agent.port" 2>/dev/null || true)"
|
|
119
|
+
fi
|
|
120
|
+
KERNO_PORT="${KERNO_PORT:-8085}"
|
|
121
|
+
AGENT_URL="http://127.0.0.1:${KERNO_PORT}"
|
|
122
|
+
|
|
123
|
+
# ── Rule 1: preflight — assume-and-verify, never spawn/rebind ──────────────────
|
|
124
|
+
# The stage drives Kerno entirely over REST and never invokes the `kerno` CLI, so
|
|
125
|
+
# reachability of the agent (not the CLI being on PATH) is the real requirement.
|
|
126
|
+
command -v curl >/dev/null 2>&1 || fail "curl is required but was not found on PATH."
|
|
127
|
+
docker info >/dev/null 2>&1 || fail "Docker is not running. Kerno executes scenarios in a container — start Docker and retry."
|
|
128
|
+
curl -sf "$AGENT_URL/health" >/dev/null 2>&1 || fail "No Kerno agent reachable at $AGENT_URL. Install the Kerno CLI (npm install -g @kerno/cli) and run 'kerno init' in this worktree first (Kerno allows one agent per machine). Override the port with KERNO_AGENT_PORT."
|
|
129
|
+
|
|
130
|
+
WORKSPACES_JSON="$(curl -sf "$AGENT_URL/workspaces" 2>/dev/null || true)"
|
|
131
|
+
[ -n "$WORKSPACES_JSON" ] || fail "Could not read Kerno workspaces from $AGENT_URL/workspaces."
|
|
132
|
+
|
|
133
|
+
# Extract workspaceId, applicationId, and the bound workspace path from the first
|
|
134
|
+
# (only) workspace. Field-by-field extraction avoids delimiter-splitting pitfalls when
|
|
135
|
+
# a field (e.g. an absent applicationId) is empty.
|
|
136
|
+
ws_field() { printf '%s' "$WORKSPACES_JSON" | node -e "$1" 2>/dev/null || true; }
|
|
137
|
+
WID="$(ws_field 'const w=JSON.parse(require("fs").readFileSync(0,"utf8"))[0]||{};process.stdout.write(w.id||"")')"
|
|
138
|
+
AID="$(ws_field 'const w=JSON.parse(require("fs").readFileSync(0,"utf8"))[0]||{};const a=(w.applications||[])[0];process.stdout.write(a&&a.id?a.id:"")')"
|
|
139
|
+
WS_PATH="$(ws_field 'const w=JSON.parse(require("fs").readFileSync(0,"utf8"))[0]||{};process.stdout.write(w.path||"")')"
|
|
140
|
+
PATH_MATCH="$(WSP="$WS_PATH" WD="$WORK_DIR" node -e 'const fs=require("fs");const r=p=>{try{return fs.realpathSync(p)}catch(e){return p||""}};const wsp=process.env.WSP||"";process.stdout.write(wsp&&r(wsp)===r(process.env.WD||"")?"1":"0")' 2>/dev/null || echo 0)"
|
|
141
|
+
|
|
142
|
+
[ -n "${WID:-}" ] || fail "Kerno agent has no analyzed workspace yet. Run 'kerno init' in this worktree."
|
|
143
|
+
if [ "${PATH_MATCH:-0}" != "1" ]; then
|
|
144
|
+
fail "Kerno agent is bound to '${WS_PATH:-unknown}', not this slot worktree ('$WORK_DIR'). Run 'kerno init' in this worktree first (Kerno allows one agent per machine)."
|
|
145
|
+
fi
|
|
146
|
+
|
|
147
|
+
# Resolve the applicationId. Precedence: explicit KERNO_APP_ID override, then the
|
|
148
|
+
# workspace's HTTP/MCP-serving application (the normal case). If Kerno analyzed a
|
|
149
|
+
# module but did not classify it as HTTP-serving, /workspaces omits it — fall back to
|
|
150
|
+
# the module list Kerno leaks in a 404 body so validation still works, with a note.
|
|
151
|
+
if [ -n "${KERNO_APP_ID:-}" ]; then
|
|
152
|
+
AID="$KERNO_APP_ID"
|
|
153
|
+
elif [ -z "$AID" ]; then
|
|
154
|
+
PROBE="$(curl -s "$AGENT_URL/workspaces/$WID/applications/__har_kerno_probe__/config" 2>/dev/null || true)"
|
|
155
|
+
AID="$(printf '%s' "$PROBE" | node -e 'let d="";process.stdin.on("data",c=>d+=c);process.stdin.on("end",()=>{const m=d.match(/has modules: \[([^\]]*)\]/);process.stdout.write(m&&m[1].trim()?m[1].split(",")[0].trim():"")})' 2>/dev/null || true)"
|
|
156
|
+
[ -n "$AID" ] && log "note: no HTTP-classified application in /workspaces; falling back to analyzed module '$AID' (Kerno may not have classified this app as HTTP-serving). Set KERNO_APP_ID to pin it."
|
|
157
|
+
fi
|
|
158
|
+
[ -n "${AID:-}" ] || fail "Kerno detected no usable application in this workspace. Set KERNO_APP_ID (see 'kerno_get_applications' → unsupported_apps for the id) or confirm the app was analyzed and a suite generated."
|
|
159
|
+
|
|
160
|
+
# Suite must already exist — validate never generates.
|
|
161
|
+
SCEN_DIR="$WORK_DIR/.kerno/scenarios/endpoints"
|
|
162
|
+
SCEN_FILES="$(find "$SCEN_DIR" -name '*.scenario.ts' 2>/dev/null | sort)"
|
|
163
|
+
[ -n "$SCEN_FILES" ] || fail "No Kerno scenario suite under .kerno/scenarios/endpoints/. Ask your Kerno agent to generate scenarios first — validate only re-runs an existing suite."
|
|
164
|
+
|
|
165
|
+
# ── Point Kerno's config at this slot (REST PATCH — same writer as save_config) ──
|
|
166
|
+
# The env map is injected verbatim into the sandbox. It carries the slot DB plus any
|
|
167
|
+
# KERNO_SANDBOX_ENV the app's scenarios need at runtime (test-user email, tokens, flags),
|
|
168
|
+
# as newline- or semicolon-separated KEY=VALUE pairs. Kerno treats the block name as
|
|
169
|
+
# organisational only, so all values are injected regardless of the block.
|
|
170
|
+
CONFIG_BODY="$(API_URL="$API_URL" DB_URL="$DB_URL" EXTRA="${KERNO_SANDBOX_ENV:-}" node -e '
|
|
171
|
+
const body = { sut: { inputUrl: process.env.API_URL }, targetEnvironment: "local" };
|
|
172
|
+
const env = {};
|
|
173
|
+
if (process.env.DB_URL) env.DATABASE_URL = process.env.DB_URL;
|
|
174
|
+
(process.env.EXTRA || "").split(/[;\n]+/).forEach((p) => {
|
|
175
|
+
const i = p.indexOf("=");
|
|
176
|
+
if (i < 1) return;
|
|
177
|
+
const k = p.slice(0, i).trim();
|
|
178
|
+
const v = p.slice(i + 1).trim();
|
|
179
|
+
if (/^[A-Z_][A-Z0-9_]*$/.test(k)) env[k] = v;
|
|
180
|
+
});
|
|
181
|
+
if (Object.keys(env).length) body.postgres = env;
|
|
182
|
+
process.stdout.write(JSON.stringify(body));
|
|
183
|
+
')"
|
|
184
|
+
curl -sf -X PATCH "$AGENT_URL/workspaces/$WID/applications/$AID/config" \
|
|
185
|
+
-H 'content-type: application/json' -d "$CONFIG_BODY" >/dev/null 2>&1 \
|
|
186
|
+
|| fail "Failed to set Kerno target config (PATCH /workspaces/$WID/applications/$AID/config)."
|
|
187
|
+
|
|
188
|
+
log "Validating against SUT $API_URL${DB_URL:+ (greybox DB configured)}"
|
|
189
|
+
log "Workspace: $WID Application: $AID"
|
|
190
|
+
|
|
191
|
+
# ── Enumerate unique endpoints (METHOD + path) from the on-disk suite ──────────
|
|
192
|
+
# Each *.scenario.ts lives in .kerno/scenarios/endpoints/<METHOD>/<path>/ ; the
|
|
193
|
+
# endpoint is that parent dir. No `mapfile` — stock macOS bash is 3.2.
|
|
194
|
+
ENDPOINTS=""
|
|
195
|
+
while IFS= read -r scen; do
|
|
196
|
+
[ -n "$scen" ] || continue
|
|
197
|
+
d="$(dirname "$scen")"
|
|
198
|
+
rel="${d#"$SCEN_DIR"/}" # e.g. GET/api/health
|
|
199
|
+
method="${rel%%/*}" # GET
|
|
200
|
+
rest="${rel#*/}" # api/health (== rel when no sub-path)
|
|
201
|
+
if [ "$rest" = "$rel" ]; then rest=""; fi
|
|
202
|
+
path="/${rest}"; path="${path%/}"; [ -z "$path" ] && path="/"
|
|
203
|
+
ENDPOINTS="${ENDPOINTS}${method} ${path}
|
|
204
|
+
"
|
|
205
|
+
done <<< "$SCEN_FILES"
|
|
206
|
+
ENDPOINTS="$(printf '%s' "$ENDPOINTS" | sort -u | sed '/^[[:space:]]*$/d')"
|
|
207
|
+
|
|
208
|
+
# ── Run each endpoint (synchronous REST; the response is the source of truth) ──
|
|
209
|
+
mkdir -p "$ARTIFACT_DIR"
|
|
210
|
+
MARKER="$(mktemp)" # reference for "fresh" report.json (macOS find has -newer, not -newermt)
|
|
211
|
+
START_TOTAL=$(now_ms)
|
|
212
|
+
OVERALL_PASS=true
|
|
213
|
+
ENDPOINT_RESULTS="[]"
|
|
214
|
+
|
|
215
|
+
while IFS=$'\t' read -r method path; do
|
|
216
|
+
[ -n "$method" ] || continue
|
|
217
|
+
printf " → %-6s %-40s" "$method" "$path" >&2
|
|
218
|
+
|
|
219
|
+
REQ_BODY="$(METHOD="$method" EPATH="$path" node -e 'process.stdout.write(JSON.stringify({method:process.env.METHOD,path:process.env.EPATH}))')"
|
|
220
|
+
|
|
221
|
+
set +e
|
|
222
|
+
RESP="$(curl -s -X POST "$AGENT_URL/workspaces/$WID/applications/$AID/scenarios/run" \
|
|
223
|
+
-H 'content-type: application/json' -d "$REQ_BODY" 2>/dev/null)"
|
|
224
|
+
CURL_EXIT=$?
|
|
225
|
+
set -e
|
|
226
|
+
|
|
227
|
+
if [ "$CURL_EXIT" != "0" ]; then
|
|
228
|
+
# Connection reset mid-run usually means the Kerno agent was killed (another
|
|
229
|
+
# agent booted). Surface it as a failure rather than a silent pass.
|
|
230
|
+
echo "✗ (agent unreachable)" >&2
|
|
231
|
+
SUMMARY="$(METHOD="$method" EPATH="$path" node -e 'process.stdout.write(JSON.stringify({method:process.env.METHOD,path:process.env.EPATH,ok:false,total:0,passed:0,error:"agent unreachable mid-run (curl exit '"$CURL_EXIT"')"}))')"
|
|
232
|
+
OVERALL_PASS=false
|
|
233
|
+
else
|
|
234
|
+
SAFE="$(printf '%s' "${method}_${path}" | tr '/ ' '__' | tr -cd 'A-Za-z0-9_.-')"
|
|
235
|
+
printf '%s\n' "$RESP" > "$ARTIFACT_DIR/${SAFE}.run.json"
|
|
236
|
+
|
|
237
|
+
SUMMARY="$(printf '%s' "$RESP" | METHOD="$method" EPATH="$path" node -e '
|
|
238
|
+
const fs = require("fs");
|
|
239
|
+
let raw = ""; try { raw = fs.readFileSync(0, "utf8"); } catch (e) {}
|
|
240
|
+
let data; try { data = JSON.parse(raw); } catch (e) {
|
|
241
|
+
process.stdout.write(JSON.stringify({ method: process.env.METHOD, path: process.env.EPATH, ok: false, total: 0, passed: 0, error: "unparseable run response" }));
|
|
242
|
+
process.exit(0);
|
|
243
|
+
}
|
|
244
|
+
const results = Array.isArray(data.results) ? data.results : [];
|
|
245
|
+
const passed = results.filter(r => r.verdict === "passed").length;
|
|
246
|
+
const failing = results.filter(r => r.verdict !== "passed");
|
|
247
|
+
const bugs = results.filter(r => r.potentialBug).map(r => ({ title: r.potentialBug.title, severity: r.potentialBug.severity || null, scenario: r.scenarioId }));
|
|
248
|
+
const ok = results.length > 0 && failing.length === 0;
|
|
249
|
+
process.stdout.write(JSON.stringify({
|
|
250
|
+
method: process.env.METHOD, path: process.env.EPATH, ok,
|
|
251
|
+
total: results.length, passed,
|
|
252
|
+
failing: failing.map(r => ({ scenario: r.scenarioId, verdict: r.verdict, status: r.responseStatus })),
|
|
253
|
+
bugs,
|
|
254
|
+
}));
|
|
255
|
+
')"
|
|
256
|
+
|
|
257
|
+
EP_OK="$(printf '%s' "$SUMMARY" | node -e 'const d=JSON.parse(require("fs").readFileSync(0,"utf8"));process.stdout.write(d.ok?"1":"0")' 2>/dev/null || echo 0)"
|
|
258
|
+
EP_PASS="$(printf '%s' "$SUMMARY" | node -e 'const d=JSON.parse(require("fs").readFileSync(0,"utf8"));process.stdout.write(String(d.passed)+"/"+String(d.total))' 2>/dev/null || echo "?")"
|
|
259
|
+
if [ "$EP_OK" = "1" ]; then
|
|
260
|
+
echo "✓ ($EP_PASS)" >&2
|
|
261
|
+
else
|
|
262
|
+
echo "✗ ($EP_PASS)" >&2
|
|
263
|
+
OVERALL_PASS=false
|
|
264
|
+
fi
|
|
265
|
+
fi
|
|
266
|
+
|
|
267
|
+
ENDPOINT_RESULTS="$(printf '%s' "$ENDPOINT_RESULTS" | SUMMARY="$SUMMARY" node -e 'const fs=require("fs");let a=JSON.parse(fs.readFileSync(0,"utf8"));a.push(JSON.parse(process.env.SUMMARY));process.stdout.write(JSON.stringify(a))')"
|
|
268
|
+
done <<< "$ENDPOINTS"
|
|
269
|
+
|
|
270
|
+
END_TOTAL=$(now_ms)
|
|
271
|
+
TOTAL_MS=$(( END_TOTAL - START_TOTAL ))
|
|
272
|
+
|
|
273
|
+
# ── Copy Kerno's own report.json files, but only ones written by THIS run ──────
|
|
274
|
+
if [ -d "$WORK_DIR/.kerno" ]; then
|
|
275
|
+
while IFS= read -r rep; do
|
|
276
|
+
[ -n "$rep" ] || continue
|
|
277
|
+
rel="${rep#"$WORK_DIR"/}"
|
|
278
|
+
safe="$(printf '%s' "$rel" | tr '/ ' '__')"
|
|
279
|
+
cp "$rep" "$ARTIFACT_DIR/$safe" 2>/dev/null || true
|
|
280
|
+
done < <(find "$WORK_DIR/.kerno" -name 'report.json' -newer "$MARKER" 2>/dev/null || true)
|
|
281
|
+
fi
|
|
282
|
+
|
|
283
|
+
# ── Emit aggregated result ─────────────────────────────────────────────────────
|
|
284
|
+
RESULTS="$ENDPOINT_RESULTS" AID="$AGENT_ID" TMS="$TOTAL_MS" SUT="$API_URL" node -e "
|
|
285
|
+
const results = JSON.parse(process.env.RESULTS || '[]');
|
|
286
|
+
const overall = results.length > 0 && results.every(r => r.ok);
|
|
287
|
+
const scenarios = {
|
|
288
|
+
total: results.reduce((a, r) => a + (r.total || 0), 0),
|
|
289
|
+
passed: results.reduce((a, r) => a + (r.passed || 0), 0),
|
|
290
|
+
};
|
|
291
|
+
const bugs = results.reduce((a, r) => a.concat(r.bugs || []), []);
|
|
292
|
+
const out = {
|
|
293
|
+
status: overall ? 'pass' : 'fail',
|
|
294
|
+
stageId: 'backend-validation',
|
|
295
|
+
kind: 'test',
|
|
296
|
+
agent_id: Number(process.env.AID),
|
|
297
|
+
total_ms: Number(process.env.TMS),
|
|
298
|
+
urls: [{ label: 'sut', url: process.env.SUT }],
|
|
299
|
+
scenarios,
|
|
300
|
+
endpoints: results,
|
|
301
|
+
bugs,
|
|
302
|
+
artifacts: [{ path: '.har/artifacts/backend-validation', kind: 'directory' }],
|
|
303
|
+
};
|
|
304
|
+
process.stdout.write(JSON.stringify(out, null, 2) + '\n');
|
|
305
|
+
"
|
|
306
|
+
|
|
307
|
+
[ "$OVERALL_PASS" = "true" ] || exit 1
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# Kerno plugin
|
|
2
|
+
|
|
3
|
+
Add deterministic backend/API validation to your harness with [Kerno](https://kerno.io).
|
|
4
|
+
Kerno re-runs a committed scenario suite against the app running in an agent slot, using the
|
|
5
|
+
slot's own database for greybox checks, and reports a pass/fail with a full evidence trail.
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
har env add-plugin kerno
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## What this installs
|
|
12
|
+
|
|
13
|
+
| File | Purpose |
|
|
14
|
+
|------|---------|
|
|
15
|
+
| `.har/stages/backend-validation.sh` | Stage runner. Re-runs the Kerno suite against the running slot over REST |
|
|
16
|
+
| `.har/stages/KERNO.md` | Setup and adaptation guide: prerequisites, the one-agent rule, artifacts |
|
|
17
|
+
| `tests/kerno/README.md` | This file |
|
|
18
|
+
| `.github/workflows/kerno.yml` | Optional CI workflow (skip with `--skip-ci`) |
|
|
19
|
+
|
|
20
|
+
## Workflow
|
|
21
|
+
|
|
22
|
+
```
|
|
23
|
+
Code change
|
|
24
|
+
│
|
|
25
|
+
launch slot + app running
|
|
26
|
+
│
|
|
27
|
+
typecheck + unit-tests + lint
|
|
28
|
+
│
|
|
29
|
+
backend-validation ◄── your committed .kerno/scenarios/
|
|
30
|
+
│
|
|
31
|
+
pass/fail + evidence
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Requirements
|
|
35
|
+
|
|
36
|
+
- Kerno CLI: `npm install -g @kerno/cli`
|
|
37
|
+
- Docker running (Kerno executes scenarios in a sandbox container)
|
|
38
|
+
- A Kerno agent bound to the slot's worktree (`kerno init` inside it)
|
|
39
|
+
- A committed suite under `.kerno/scenarios/` (validate re-runs, it does not generate)
|
|
40
|
+
|
|
41
|
+
Kerno runs one agent per machine, so backend validation is serialized across slots by a
|
|
42
|
+
fail-fast lock. See `.har/stages/KERNO.md` for the full explanation.
|
|
43
|
+
|
|
44
|
+
## Quick start after install
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
# 1. Install the stage
|
|
48
|
+
har env add-plugin kerno
|
|
49
|
+
|
|
50
|
+
# 2. Install the Kerno CLI and bind an agent to the worktree
|
|
51
|
+
npm install -g @kerno/cli
|
|
52
|
+
kerno init # run inside the slot worktree
|
|
53
|
+
|
|
54
|
+
# 3. Ensure a suite exists (ask your Kerno agent to generate one, then commit it)
|
|
55
|
+
ls .kerno/scenarios/endpoints/
|
|
56
|
+
|
|
57
|
+
# 4. Launch the slot and start your app
|
|
58
|
+
./.har/launch.sh 1
|
|
59
|
+
|
|
60
|
+
# 5. Run backend validation
|
|
61
|
+
./.har/stages/backend-validation.sh 1
|
|
62
|
+
|
|
63
|
+
# 6. Validate everything
|
|
64
|
+
./.har/verify.sh 1 --full
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Per-endpoint run responses and verdicts land in `.har/artifacts/backend-validation/`.
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "kerno",
|
|
3
|
+
"stageId": "backend-validation",
|
|
4
|
+
"verificationStages": [
|
|
5
|
+
"typecheck",
|
|
6
|
+
"unit-tests",
|
|
7
|
+
"api-health",
|
|
8
|
+
"lint",
|
|
9
|
+
"backend-validation"
|
|
10
|
+
],
|
|
11
|
+
"stage": {
|
|
12
|
+
"id": "backend-validation",
|
|
13
|
+
"kind": "test",
|
|
14
|
+
"description": "Kerno deterministic backend/API scenario validation against the running slot",
|
|
15
|
+
"script": "stages/backend-validation.sh",
|
|
16
|
+
"requiresAgentId": true,
|
|
17
|
+
"artifacts": [
|
|
18
|
+
{
|
|
19
|
+
"path": ".har/artifacts/backend-validation",
|
|
20
|
+
"kind": "directory",
|
|
21
|
+
"description": "Per-endpoint run responses, aggregated verdicts, and copied Kerno report.json files"
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
"path": ".har/artifacts/backend-validation/report.json",
|
|
25
|
+
"kind": "report"
|
|
26
|
+
}
|
|
27
|
+
]
|
|
28
|
+
},
|
|
29
|
+
"files": [
|
|
30
|
+
{ "src": ".har/stages/backend-validation.sh", "dest": ".har/stages/backend-validation.sh", "executable": true },
|
|
31
|
+
{ "src": ".har/stages/KERNO.md", "dest": ".har/stages/KERNO.md" },
|
|
32
|
+
{ "src": "README.md", "dest": "tests/kerno/README.md" }
|
|
33
|
+
],
|
|
34
|
+
"optionalFiles": [
|
|
35
|
+
{ "src": ".github/workflows/kerno.yml", "dest": ".github/workflows/kerno.yml", "skipFlag": "skipCi" }
|
|
36
|
+
],
|
|
37
|
+
"nextSteps": [
|
|
38
|
+
"npm install -g @kerno/cli",
|
|
39
|
+
"kerno init # bind the single Kerno agent to THIS worktree",
|
|
40
|
+
"Ask your Kerno agent to generate a scenario suite (or ensure .kerno/scenarios/ is committed)",
|
|
41
|
+
"./.har/launch.sh 1 # then start your app in the slot",
|
|
42
|
+
"./.har/stages/backend-validation.sh 1",
|
|
43
|
+
"Open .har/artifacts/backend-validation/ for verdicts and per-endpoint run responses"
|
|
44
|
+
],
|
|
45
|
+
"docsPath": ".har/stages/KERNO.md"
|
|
46
|
+
}
|