@aion0/forge 0.10.20 → 0.10.23

package/docs/tp-automation-api.md

@@ -0,0 +1,617 @@
+# TP `/automation` Page — API Reference
+
+Reference for the HTTP endpoints used by TP's Automation page
+(`/automation`, source: `frontend/src/pages/Automation/Automation.jsx`)
+and the related upgrade / testbed workflows the page invokes through
+shared infrastructure.
+
+All endpoints are mounted under the `adc` Django app:
+
+```
+<TP-base-url>/adc/<endpoint>
+```
+
+`<TP-base-url>` examples:
+- Production: `https://nac-tp.fortinet-us.com`
+- Test:       `http://10.15.33.25:8000`
+- Dev (.11):  `http://10.15.33.11:8000`
+
+## Authentication
+
+Every endpoint requires a JWT in the `Authorization` header:
+
+```
+Authorization: JWT <token>
+```
+
+Mint a token:
+
+```bash
+T=$(curl -s -X POST <TP-base-url>/token-auth/ \
+       -H 'Content-Type: application/json' \
+       -d '{"username":"<user>","password":"<pw>"}' | jq -r .token)
+```
+
+In the examples below, `$T` stands for the JWT. Tokens are
+short-lived; on 401 the frontend redirects to SSO, and scripts should
+re-mint.
+
+---
+
+## Endpoints called by the `/automation` page
+
+### `GET /adc/automation-verion/`
+
+Returns the FortiNAC versions the automation pipeline tracks.
+Populates the version dropdown.
+
+Response:
+```json
+{"versions": ["7.4.6", "7.4.7", "7.6.5", "7.6.6"]}
+```
+
+Backed by: `adc.views.dashboard.dashboardviews.get_automation_version`
+
+### `GET /adc/get_testcases`
+
+Walks the cloned test-framework repo on TP, parses every Python test
+file, and returns a hierarchical tree of modules → test cases.
+
+**Side-effect:** pulls latest commits on the `main` branch of the local
+clone before parsing. Calling from a script will rebase TP's local
+repo on `main` — fine in normal use, but worth knowing if a developer
+is hand-testing branches on the TP host.
+
+Response shape (truncated):
+```json
+{
+  "tests": {
+    "L2": {
+      "test_l2_radius.py": [
+        "test_basic_auth",
+        "test_radius_attributes"
+      ]
+    },
+    "L3": {}
+  }
+}
+```
+
+Backed by: `automationview.get_testcases`
+
+### `POST /adc/pytest_run`
+
+Kicks off a pytest execution on the chosen automation testbed. Creates
+a `PytestExecution` row and returns its id; the actual run is
+asynchronous.
+
+Body:
+```json
+{
+  "user":     "alice",
+  "lab":      "L2Mode_7",
+  "testcase": [
+    "tests/L2/test_l2_radius.py::test_basic_auth",
+    "tests/L2/test_l2_radius.py::test_radius_attributes"
+  ],
+  "argument": "-k 'radius and not flaky' --tb=short -vv"
+}
+```
+
+| Field | Type | Notes |
+|---|---|---|
+| `user` | string | TP username of the caller. |
+| `lab` | string | AT lab name from `AutomationTBUser` (the same names returned by `get_automation_lab`). Comma-separate multiple labs (`"L2Mode_7,L2Mode_9"`). The caller must already own or be a member of `usedby` on the lab, and no other execution can be `Running`/`Initiating` against it. |
+| `testcase` | **list of strings** | Each entry is a pytest test-id path. The handler iterates the list, prefixes each with the test-framework repo path on TP, and joins them with spaces before invoking pytest. |
+| `argument` | string | **Raw pytest CLI arguments**, injected verbatim between the testcase paths and the framework's `--html=...`/`--rack-file ...` flags. Use this for filters, verbosity, fail-fast, collect-only, marker expressions, etc. |
+
+The handler also accepts any **extra** fields in the body — they're
+preserved on the execution record. You can attach tracking metadata
+(`mantis_id`, `jenkins_job`, etc.) without backend changes.
+
+#### What gets executed on the testbed
+
+The worker generates a shell script of the form:
+
+```bash
+git pull origin main
+cd <repo>
+source venv/bin/activate
+export PYTHONPATH=<test-framework>:<tests>
+export DISPLAY=:99
+pytest <testcase paths> <argument> --collect-only
+pytest <testcase paths> <argument> --html=<report> --rack-file <rack> <pytest_options>
+```
+
+So `argument` is literally the slot for any pytest flag. The worker
+runs `--collect-only` first as a dry-run check, then the real run.
+
+#### Examples
+
+**Run one test:**
+```json
+{
+  "user":     "alice",
+  "lab":      "L2Mode_7",
+  "testcase": ["tests/L2/test_l2_radius.py::test_basic_auth"],
+  "argument": ""
+}
+```
+
+**Run several specific tests, with verbose output and fail-fast:**
+```json
+{
+  "user":     "alice",
+  "lab":      "L2Mode_7",
+  "testcase": [
+    "tests/L2/test_l2_radius.py::test_basic_auth",
+    "tests/L3/test_l3_vpn.py::test_vpn_login"
+  ],
+  "argument": "-vv -x"
+}
+```
+
+**Run every test in a file, filtered by keyword:**
+```json
+{
+  "user":     "alice",
+  "lab":      "L2Mode_7",
+  "testcase": ["tests/L2/test_l2_radius.py"],
+  "argument": "-k 'radius and not flaky'"
+}
+```
+
+**Dry-run / list-only (no test bodies executed by the second pytest invocation):**
+```json
+{
+  "user":     "alice",
+  "lab":      "L2Mode_7",
+  "testcase": ["tests/L2/"],
+  "argument": "--collect-only"
+}
+```
+
+(Note: `--collect-only` runs twice in this case — once by the worker's
+hardcoded first-line check, once by your explicit flag. Functionally
+identical to a normal collect-only.)
+
+#### Response
+
+```json
+{"exec_id": "53"}
+```
+Or on error:
+```json
+{"error": "..."}
+```
+
+On lab conflict (someone else's run is in progress on that lab), the
+response contains `conflict_labs` and `usable_labs` instead of
+`exec_id`.
+
+Backed by: `automationview.pytest_run` → `PytestAction.create_execution_entry`
+
+### `POST /adc/get_automation_lab`
+
+Lists the automation testbeds the calling user can see (owned or
+shared). Each entry includes a live `status` field computed from
+in-progress executions.
+
+Body:
+```json
+{"user": "alice"}
+```
+
+Response (truncated):
+```json
+[
+  {
+    "name": "L2Mode_7",
+    "usedby": "alice,bob",
+    "status": "Available"
+  }
+]
+```
+
+`status` is `"Running"` when any `PytestExecution` for this lab has
+status `Running`, else `"Available"`.
+
+### `POST /adc/get_test_result`
+
+Returns the calling user's recent test executions with computed
+progress percentages.
+
+Body:
+```json
+{"user": "alice"}
+```
+
+Response:
+```json
+[
+  {
+    "id":               53,
+    "name":             "test001",
+    "report_file_path": "media/automation/53/report.html",
+    "log_file_path":    "media/automation/53/log.txt",
+    "status":           "Running",
+    "progress":         42
+  }
+]
+```
+
+`progress = (pass + fail + skip + error) / total * 100`, rounded.
+
+`report_file_path` and `log_file_path` are server-relative; open them
+by prepending `<TP-base-url>` and sending the JWT.
+
+### `POST /adc/get_test_execution_by_id`
+
+Returns one execution's full record (steps, counts, environment,
+timestamps) for the detail panel.
+
+Body:
+```json
+{"id": 53}
+```
+
+Response: serialized `PytestExecution`. Returns 404 if not found.
+
+### `POST /adc/test_exec_kill`
+
+Aborts a running execution. The row stays in the DB with a terminal
+status; the underlying pytest subprocess is sent SIGTERM.
+
+Body:
+```json
+{"id": 53}
+```
+
+### `POST /adc/test_exec_delete`
+
+Deletes one or many `PytestExecution` rows and their on-disk
+artifacts. Accepts either a single id or a list.
+
+Body (single):
+```json
+{"id": 53}
+```
+
+Body (bulk):
+```json
+{"historyresults": [50, 51, 52]}
+```
+
+### `POST /adc/automation_lock`
+
+Marks a testbed as in-use, or releases it. Used to serialize tests
+that need exclusive access to shared hardware.
+
+Body:
+```json
+{"action": "lock", "user": "alice", "name": "L2Mode_7"}
+```
+
+`action` is `"lock"` or `"unlock"`.
+
+### `POST /adc/automation_tb_set_viewers/`
+
+Edits who can *see* a testbed (separate from ownership/usage).
+
+Body:
+```json
+{
+  "action":  "add",
+  "tb_name": "L2Mode_7",
+  "viewers": ["bob", "carol"]
+}
+```
+
+`action` is `"add"`, `"remove"`, or `"set"`.
+
+### `GET /adc/status-check/`
+
+Generic shared-lab status payload used by the status widget rendered
+on the page. Not specific to automation but called from it.
+
+Backed by: `adc.views.sharedlab.tbviews.ClassicCombinedStatus`
+
+### `GET /user/get_user/<username>`
+
+Returns the calling user's profile (role, group, default project) so
+the page can decide what controls to render. Note: outside the `adc`
+namespace.
+
+---
+
+## NAC upgrade APIs
+
+The `/automation` page does not call these directly, but every
+automation run that targets a specific build relies on the testbed
+having been upgraded first. There are **two modes** of upgrade:
+
+| Mode | What it targets | Endpoints |
+|---|---|---|
+| **Device mode** | One specific `ip` | `nac-upgrade/`, `nac-upgrade-version-snapshot/`, `nac-upgrade-snapshot/`, `check-upgrade-status/` |
+| **Testbed mode** | Every NAC/NCM device in a testbed (discovered from `deployinfo`) | `nac-upgrade-testbed/` |
+
+Both modes share the same `upgrade_type` axis:
+
+| `upgrade_type` | Required extra field | Meaning |
+|---|---|---|
+| `GA`      | (none)              | Pull the latest GA image from the build server |
+| `build`   | `build_number`      | Download a specific build (e.g. `7.6.6.0123`) and install it |
+| `file`    | `file_uploaded` (multipart) | Install from an uploaded image |
+| `command` | `command`           | Run a raw upgrade CLI command (advanced/manual) |
+
+Upgrade work is **asynchronous** in both modes — the call returns once
+the job is queued. Use the device-mode status endpoint to poll.
+
+### Device mode
+
+#### `POST /adc/nac-upgrade/`
+
+Upgrade a single FortiNAC device. Accepts `multipart/form-data` so it
+can carry the uploaded image file.
+
+Form fields:
+- `ip` — target device IP (required)
+- `upgrade_type` — `GA` / `build` / `file` / `command`
+- `build_number` — when `upgrade_type=build`
+- `file_uploaded` — when `upgrade_type=file` (image file as multipart)
+- `command` — when `upgrade_type=command`
+
+Response:
+```json
+{"status": "success", "message": "..."}
+```
+HTTP `202` on success, `400` on failure.
+
+Backed by: `nacviews.nac_upgrade`
+
+#### `POST /adc/nac-upgrade-version-snapshot/`
+
+Take a pre-upgrade snapshot of the device's current image so it can be
+reverted later. Returns once the snapshot job has been queued.
+
+Body:
+```json
+{
+  "ip":            "10.15.40.42",
+  "dev_name":      "fortinac01",
+  "build_number":  "7.6.6.0123",
+  "upgrade_type":  "build",
+  "major_version": "7.6",
+  "minor_version": "6",
+  "rp_prefix":     "rp",
+  "command":       "<optional, when upgrade_type=command>"
+}
+```
+
+Backed by: `nacupgradeview.nac_upgrade_version_snapshot`
+
+#### `POST /adc/nac-upgrade-snapshot/`
+
+Upgrade *with* automatic snapshot/revert. Same payload as
+`nac-upgrade-version-snapshot/` plus:
+
+- `revert_snapshot` — `true` / `false`; when `true`, the device is
+  reverted to the snapshot if the upgrade fails.
+
+Backed by: `nacupgradeview.nac_upgrade_snapshot`
+
+#### `POST /adc/check-upgrade-status/`
+
+Poll the upgrade status of a single device.
+
+Body:
+```json
+{"ip": "10.15.40.42"}
+```
+
+Response: state of the most recent upgrade task for that IP. Shape:
+```json
+{
+  "is_upgrading":       false,
+  "last_task_status":   "SUCCESS",
+  "last_task_type":     "build",
+  "last_build_number":  "7.6.6.0123",
+  "last_image_name":    "FortiNAC-F-7.6.6.0123.out",
+  "last_major_version": "7.6",
+  "last_minor_version": "6",
+  "updated_at":         "2026-05-28T11:42:01.123456+00:00",
+  "log":                "...full log tail...",
+  "ip":                 "10.15.40.42"
+}
+```
+
+`is_upgrading` is `true` when `last_task_status` is `PENDING` or
+`PROGRESS`. The endpoint works for both modes — see the table at the
+top of the section for which testbed-mode `upgrade_type`s actually
+write to this table — but read the polling caveats below carefully.
+
+##### Polling caveat — testbed-mode `build`
+
+`run_download_async` (the worker that handles `upgrade_type=build` in
+testbed mode) does **not** set `last_task_status` to `PENDING` or
+`PROGRESS` while running. It only writes `SUCCESS` on completion.
+Effect: while a testbed-mode build upgrade is in flight,
+`is_upgrading` returns `false` even though the worker is actively
+downloading + restoring on the device.
+
+To get live progress mid-run for testbed-mode build upgrades, read
+the `log` field instead — `run_download_async` appends to it
+continuously (download start, restore start, VM CLI output, success).
+
+> **Known bug (worth fixing as a separate task):** `run_download_async`
+> in `backend/adc/views/sharedlab/nacviews.py` should set
+> `last_task_status = "PROGRESS"` immediately after `get_or_create`
+> at the start of the worker, and `"FAILURE"` in its `except` branch
+> (it currently only writes `"SUCCESS"` on the happy path). With that
+> fix, `check-upgrade-status/` would return an accurate `is_upgrading`
+> flag for testbed-mode build upgrades and surface failures without
+> requiring the caller to parse the log field.
+
+##### Polling caveat — testbed-mode `GA`
+
+`upgrade_type=GA` in `nac_upgrade_testbed` is a stub — it returns
+`{"upgrade_type": "GA"}` immediately and never touches `SingleDevice`.
+This endpoint will report `"Device not found"` (404) for IPs that have
+never been upgraded by another mode. There is no work to poll.
+
+##### Polling caveat — testbed-mode `file` and `command`
+
+These branches run synchronously inside the HTTP request and only
+write to `SingleDevice` *after* the work finishes. The request itself
+blocks until then (file restore can take minutes), so polling
+`check-upgrade-status/` for in-flight progress isn't useful for these
+types — by the time you can issue a separate poll, the upgrade is
+already done.
+
+Backed by: `nacupgradeview.check_upgrade_status`
+
+### Testbed mode
+
+#### `POST /adc/nac-upgrade-testbed/`
+
+Upgrade every NAC/NCM device in a testbed at once. Accepts
+`multipart/form-data` to carry an optional uploaded image.
+
+The handler reads `deployinfo` (JSON), finds every key matching
+`dev<N>` whose value contains `nac` or `ncm`, and pairs it with the
+corresponding `ip<N>` to build the upgrade list — then runs the
+chosen `upgrade_type` against each in parallel.
+
+Form fields:
+- `deployinfo` — JSON string with the testbed's device map. Shape:
+  `{"dev1":"fortinac01","ip1":"10.15.40.42","dev2":"ncm01","ip2":"10.15.40.43", ...}`
+  (same as `AutomationTBSerializer.deployinfo`)
+- `upgrade_type` — `GA` / `build` / `file` / `command`
+- `build_number` — when `upgrade_type=build`
+- `file_uploaded` — when `upgrade_type=file`
+- `command` — when `upgrade_type=command`
+
+Response:
+```json
+{"upgrade_type": "build"}
+```
+
+To monitor progress, poll `check-upgrade-status/` per-IP for each NAC
+in the testbed.
+
+Backed by: `nacviews.nac_upgrade_testbed`
+
+### Diagnostic
+
+#### `POST /adc/test_thread_db/`
+
+Diagnostic helper that exercises threaded DB writes against the
+`SingleDevice` model — used to verify upgrade-worker threading on a
+host. Not used by the UI; do not call in production.
+
+Backed by: `nacupgradeview.test_thread_db`
+
+---
+
+## Other related APIs the `/automation` workflow touches
+
+These belong to other pages (Shared Lab, Automation Testbed) but are
+part of the same domain. Listed here so the dev team can find them
+without spelunking.
+
+### Automation testbed management — `adc.views.automation.automation_tbview`
+
+| Endpoint | Method | Purpose |
+|---|---|---|
+| `/adc/create_automation_testbed/` | POST | Create a new automation testbed entry |
+| `/adc/automationtb-list/` | POST | List testbeds by `category` + `keyword` filter |
+| `/adc/automationtb-moduletitle/` | GET | Distinct module titles across testbeds (dropdown source) |
+| `/adc/automationtb-update/` | POST | Update a testbed's metadata, `deployinfo`, or `new_name` |
+
+### Performance lab (parallel of automation, used by `/performance`)
+
+Mirror endpoints exist for performance-test workflows; they share the
+same shape as the automation ones above. Listed for completeness in
+case the dev team needs to script performance runs the same way.
+
+| Endpoint | Method | Mirrors |
+|---|---|---|
+| `/adc/get_perf_testcases/` | GET | `get_testcases` |
+| `/adc/get_perf_testcases_module/` | GET | (module-level filter) |
+| `/adc/perf_robot_run/` | POST | `pytest_run` (Robot Framework instead of pytest) |
+| `/adc/get_perf_lab/` | POST | `get_automation_lab` |
+| `/adc/get_perf_test_result/` | POST | `get_test_result` |
+| `/adc/get_perf_test_execution_by_id/` | POST | `get_test_execution_by_id` |
+| `/adc/performance_lock/` | POST | `automation_lock` |
+| `/adc/perf_exec_kill/` | POST | `test_exec_kill` |
+| `/adc/perf_exec_delete/` | POST | `test_exec_delete` |
+
+---
+
+## Quick smoke test from the command line
+
+```bash
+T=$(curl -s -X POST http://10.15.33.11:8000/token-auth/ \
+       -H 'Content-Type: application/json' \
+       -d '{"username":"admin","password":"<your-pw>"}' | jq -r .token)
+
+# 1. Pull versions
+curl -s -H "Authorization: JWT $T" \
+     http://10.15.33.11:8000/adc/automation-verion/ | jq .
+
+# 2. List your testbeds
+curl -s -X POST -H "Authorization: JWT $T" \
+     -H 'Content-Type: application/json' \
+     -d '{"user":"admin"}' \
+     http://10.15.33.11:8000/adc/get_automation_lab | jq '.[] | {name, status}'
+
+# 3. Fire a pytest run
+EXEC_ID=$(curl -s -X POST -H "Authorization: JWT $T" \
+     -H 'Content-Type: application/json' \
+     -d '{
+           "user":"admin",
+           "lab":"L2Mode_7",
+           "testcase":["tests/L2/test_l2_radius.py::test_basic_auth"],
+           "argument":"-vv"
+         }' \
+     http://10.15.33.11:8000/adc/pytest_run | jq -r .exec_id)
+echo "Started exec $EXEC_ID"
+
+# 4. Poll progress
+curl -s -X POST -H "Authorization: JWT $T" \
+     -H 'Content-Type: application/json' \
+     -d "{\"id\":$EXEC_ID}" \
+     http://10.15.33.11:8000/adc/get_test_execution_by_id | jq .status,.progress
+
+# 5. Upgrade a NAC device to a specific build (out-of-band)
+curl -s -X POST -H "Authorization: JWT $T" \
+     -F "ip=10.15.40.42" \
+     -F "upgrade_type=build" \
+     -F "build_number=7.6.6.0123" \
+     http://10.15.33.11:8000/adc/nac-upgrade/
+
+# 6. Poll upgrade status
+curl -s -X POST -H "Authorization: JWT $T" \
+     -H 'Content-Type: application/json' \
+     -d '{"ip":"10.15.40.42"}' \
+     http://10.15.33.11:8000/adc/check-upgrade-status/ | jq .
+```
+
+---
+
+## Source files
+
+| Concern | File |
+|---|---|
+| URL routes | `backend/adc/urls.py` |
+| Run / lab / lock | `backend/adc/views/automation/automationview.py` |
+| Testbed management | `backend/adc/views/automation/automation_tbview.py` |
+| FortiNAC upgrade (single device + testbed) | `backend/adc/views/sharedlab/nacviews.py` |
+| Upgrade with snapshot / status polling | `backend/adc/views/sharedlab/nacupgradeview.py` |
+| Status widget | `backend/adc/views/sharedlab/tbviews.py` |
+| Version dropdown | `backend/adc/views/dashboard/dashboardviews.py` |
+| Frontend page | `frontend/src/pages/Automation/Automation.jsx` |
+
+When adding a new endpoint, follow the pattern: route in
+`adc/urls.py` → handler in the appropriate view module → serializer
+if returning a model → consumer in `Automation.jsx`.

package/lib/browser-bridge-standalone.ts

@@ -48,6 +48,11 @@ import { randomUUID, createHash } from 'node:crypto';
 const PORT = Number(process.env.BRIDGE_PORT) || 8407;
 const FORGE_PORT = Number(process.env.PORT) || 8403;
 const RPC_TIMEOUT_MS = 60_000;
+// Ceiling for per-call overrides. Kept just under undici's default 300s
+// headersTimeout on the loopback fetch in bridge-client.ts — past that the
+// client fetch dies first with an opaque error. Genuinely long backend work
+// (multi-minute NAC upgrades) should fire-and-poll, not hold the RPC open.
+const RPC_TIMEOUT_MAX_MS = 280_000;
 const TOKEN_CACHE_TTL_MS = 60_000;
 
 // ─── Forge-token validation (with short-lived cache) ──────
@@ -112,17 +117,21 @@ interface PendingRpc {
 
 const pendingRpcs = new Map<string, PendingRpc>(); // rpc_id → callbacks
 
-function callExtension(method: string, params: unknown): Promise<unknown> {
+function callExtension(method: string, params: unknown, timeoutMs?: number): Promise<unknown> {
   const client = pickAnyClient();
   if (!client) {
     return Promise.reject(new Error('No extension connected to the bridge.'));
   }
   const id = randomUUID();
+  const effectiveTimeout = Math.min(
+    Math.max(1000, Number(timeoutMs) || RPC_TIMEOUT_MS),
+    RPC_TIMEOUT_MAX_MS,
+  );
   return new Promise<unknown>((resolve, reject) => {
     const timer = setTimeout(() => {
       pendingRpcs.delete(id);
-      reject(new Error(`RPC ${method} timed out after ${RPC_TIMEOUT_MS / 1000}s`));
-    }, RPC_TIMEOUT_MS);
+      reject(new Error(`RPC ${method} timed out after ${effectiveTimeout / 1000}s`));
+    }, effectiveTimeout);
     pendingRpcs.set(id, { resolve, reject, timer });
     client.ws.send(JSON.stringify({ type: 'rpc_request', id, method, params }));
   });
@@ -248,7 +257,7 @@ async function handleHttp(req: IncomingMessage, res: ServerResponse): Promise<vo
   if (req.method === 'POST' && url.pathname === '/api/rpc') {
     try {
       const body = JSON.parse(await readBody(req));
-      const value = await callExtension(body.method, body.params);
+      const value = await callExtension(body.method, body.params, body.timeout_ms);
       return sendJson(res, 200, { ok: true, value });
     } catch (e) {
       return sendJson(res, 200, { ok: false, error: (e as Error).message });