@andrian.yablonskyy/thub-coordinator 1.1.16 → 1.1.18

package/views/help/_storage.pug

@@ -0,0 +1,218 @@
++section('storage', 'Artifact storage and certificates: Artifactory, S3, Google Drive, (S)FTP', 'cloud-arrow-up')
+  p.
+    TestHub stores no files. A job's #[strong inputs] come from wherever their URLs point, and its #[strong outputs]
+    (reports, binaries, core dumps) go wherever its command uploads them. The patterns are the same for every kind of storage:
+  .table-responsive
+    table.table.table-sm.small.align-middle
+      thead
+        tr
+          th(style="width: 22%") Direction
+          th How
+          th(style="width: 30%") Credentials
+      tbody
+        tr
+          td Input, anonymous HTTP(S)
+          td #[code --download-file <url>]: fetched by the Client before the command (#[code $THUB_DOWNLOAD_1] …)
+          td None. A plain GET with no headers
+        tr
+          td Input, signed URL
+          td #[code --download-file "<pre-signed URL>"] (S3, GCS, Azure SAS, Artifactory signed URLs)
+          td In the URL. It shows on the job page, so keep its expiry short
+        tr
+          td Input, private
+          td Fetched in #[code --command] (#[code curl], #[code aws], #[code rclone], …)
+          td #[code --env NAME]: masked, dropped when the job ends
+        tr
+          td Input, client certificate
+          td Fetched in #[code --command] with #[code curl --cert] (#[code --download-file] can't present one)
+          td The certificate and key as #[code --env]
+        tr
+          td Output
+          td Uploaded in #[code --command], then listed in #[code $THUB_ARTIFACTS_FILE] so it shows on the job page
+          td #[code --env NAME]
+  ul.small
+    li The tools run on the #[strong Client host], which must reach the storage. Install #[code awscli], #[code rclone], … there, or run them in a container.
+    li Keep secrets off the command line (visible in #[code ps]): #[code curl -K -] reads them from standard input, and #[code aws] and #[code rclone] read them from the environment.
+    li Artifact links must be #[code http(s)]: an #[code s3://] or #[code sftp://] link is dropped from the list.
+    li Keep the verdict: run the tests, save #[code rc=$?], upload, then #[code exit $rc].
+    li Name outputs after the job (#[code …/$THUB_JOB_ID/…]) or a CI id, so a CI step can find them.
+
+  h3.h6 Helper: list what was published
+  p.small The examples below call #[code ci/artifacts.sh] from the tests repository, and assume the command has cloned it.
+  +code('ci/artifacts.sh').
+    #!/bin/sh
+    # ci/artifacts.sh name link [file] — append one entry to $THUB_ARTIFACTS_FILE (§7.3)
+    name=$1 link=$2 file=${3:-}
+    size=null; [ -n "$file" ] && size=$(wc -c < "$file" | tr -d ' ')
+    entry=$(printf '{"name":"%s","size":%s,"link":"%s","timestamp":%s}' "$name" "$size" "$link" "$(date +%s)")
+    if [ -s "$THUB_ARTIFACTS_FILE" ]; then   # plain shell, not sed: signed URLs contain & and |
+      list=$(cat "$THUB_ARTIFACTS_FILE")
+      printf '%s,%s]\n' "${list%]}" "$entry" > "$THUB_ARTIFACTS_FILE"
+    else
+      printf '[%s]\n' "$entry" > "$THUB_ARTIFACTS_FILE"
+    fi
+
+  h3.h6 HTTPS certificates: private CAs and client certificates
+  p.small.
+    #[strong A private CA.] #[code --download-file] is fetched by the Client's Node.js process, which doesn't read the
+    system CA store, so the download fails with #[code SELF_SIGNED_CERT_IN_CHAIN]. Give the Client the CA with
+    #[code NODE_EXTRA_CA_CERTS], once per host, in a drop-in for the template unit, which applies to every instance.
+    #[code update-ca-certificates] covers #[code curl] and #[code git] in jobs, and Docker reads
+    #[code /etc/docker/certs.d/<registry>/ca.crt]. Never switch verification off in a job.
+  +code('Client host').
+    sudo cp lab-ca.crt /usr/local/share/ca-certificates/ && sudo update-ca-certificates   # curl, git, wget, apt…
+    sudo systemctl edit thub-client@.service          # every instance on this host
+    #   [Service]
+    #   Environment=NODE_EXTRA_CA_CERTS=/usr/local/share/ca-certificates/lab-ca.crt
+    sudo systemctl restart 'thub-client@*'
+  p.small.
+    #[strong A client certificate (mutual TLS).] #[code --download-file] can't present one. Fetch the file in
+    #[code --command] with #[code curl --cert], with the certificate and key passed as #[code --env] and written
+    (#[code umask 077]) to the job's directory, which is deleted with the job. For PKCS#12, use
+    #[code curl --cert-type P12 --cert file.p12:password]. One certificate for the whole lab can instead live on the host
+    (readable by the service user only), named in a drop-in: every job on that host can then use it.
+  +code('mTLS download in the command').
+    export CLIENT_CERT="$(cat thub-ci.crt)" CLIENT_KEY="$(cat thub-ci.key)"     # PEM, from your CI's secret store
+    thub run --type hw --env CLIENT_CERT --env CLIENT_KEY \
+      --command 'umask 077 && printf "%s\n" "$CLIENT_CERT" > "$THUB_WORK_DIR/c.pem" && printf "%s\n" "$CLIENT_KEY" > "$THUB_WORK_DIR/k.pem" &&
+                 curl -fsSL --cert "$THUB_WORK_DIR/c.pem" --key "$THUB_WORK_DIR/k.pem" -o app.bin https://artifactory.lab/fw-local/app/1.4.0/app.bin &&
+                 st-flash --reset write app.bin 0x08000000 && ./ci/test.sh' --wait
+
+  h3.h6 JFrog Artifactory: service authentication
+  .table-responsive
+    table.table.table-sm.small.align-middle
+      thead
+        tr
+          th Credential
+          th Sent as
+          th Notes
+      tbody
+        tr
+          td #[strong Access token] (scoped, expiring)
+          td: code Authorization: Bearer <token>
+          td Recommended. A service user with read access to inputs and deploy access to outputs.
+        tr
+          td Reference token
+          td: code Authorization: Bearer <token>
+          td Short form of an access token, for CI variables with a length limit.
+        tr
+          td Identity token / password
+          td: code -u svc-thub:<token>
+          td For tools that only speak basic auth: #[code docker login], #[code pip].
+        tr
+          td API key
+          td: code X-JFrog-Art-Api
+          td Deprecated and turned off in current versions. Migrate.
+        tr
+          td Anonymous read
+          td —
+          td Public repositories only. Then #[code --download-file] is enough.
+  p.small.
+    Best: a #[strong short-lived token per run], minted in the CI step from a service token, so a leaked token expires on its own.
+    On GitHub Actions, JFrog's OIDC integration (#[code jfrog/setup-jfrog-cli] with an #[code oidc-provider-name]) needs no stored secret at all.
+  +code('CI step: a one-hour token for the job').
+    # CI step: ART_SERVICE_TOKEN is a CI secret for the service user svc-thub
+    export ART_TOKEN=$(curl -fsS -H "Authorization: Bearer $ART_SERVICE_TOKEN" -X POST \
+      -d scope=applied-permissions/user -d expires_in=3600 \
+      https://artifactory.example.com/access/api/v1/tokens | jq -r .access_token)
+    thub run --type hw --env ART_TOKEN --command '…' --wait
+  +code('curl: download, test, upload with checksum, list').
+    thub run --type hw --env ART_TOKEN \
+      --command 'ART=https://artifactory.example.com/artifactory
+                 H() { printf "header = \"Authorization: Bearer %s\"\n" "$ART_TOKEN"; }     # via curl -K -, so not visible in ps
+                 H | curl -fsSL -K - -o app.bin "$ART/fw-local/app/1.4.0-42/app.bin" &&
+                 st-flash --reset write app.bin 0x08000000 && ./ci/test.sh; rc=$?
+                 url="$ART/qa-local/$THUB_JOB_ID/junit.xml"
+                 H | curl -fsS -K - -H "X-Checksum-Sha256: $(sha256sum results/junit.xml | cut -d" " -f1)" -T results/junit.xml "$url" &&
+                   sh ci/artifacts.sh junit.xml "$url" results/junit.xml
+                 exit $rc' --wait
+  +code('JFrog CLI, configured from --env, nothing left in the home directory').
+    export JF_ACCESS_TOKEN="$ART_TOKEN"
+    thub run --type sw --env JF_URL=https://artifactory.example.com --env JF_ACCESS_TOKEN \
+      --command 'export JFROG_CLI_HOME_DIR="$THUB_WORK_DIR/.jfrog" CI=true
+                 jf rt dl "fw-local/app/1.4.0-42/*.bin" downloads/ --flat && ./ci/test.sh; rc=$?
+                 jf rt u "results/*.xml" "qa-local/$THUB_JOB_ID/" --flat
+                 exit $rc' --wait
+  +code('Signed URL (Enterprise+): --download-file with nothing else').
+    IMAGE_URL=$(curl -fsS -H "Authorization: Bearer $ART_SERVICE_TOKEN" -H "Content-Type: application/json" -X POST \
+      -d '{"repo_path":"/fw-local/app/1.4.0-42/app.bin","valid_for_secs":3600}' \
+      https://artifactory.example.com/artifactory/api/signed/url)
+    thub run --type hw --download-file "$IMAGE_URL" --command 'st-flash --reset write "$THUB_DOWNLOAD_1" 0x08000000 && ./ci/test.sh' --wait
+  +code('The same token for Docker, PyPI and npm repositories').
+    # Docker registry (see §7.2): the token as the password
+    echo "$ART_TOKEN" | docker login artifactory.example.com -u svc-thub --password-stdin
+    docker pull artifactory.example.com/docker-local/team/test-runner:1.4
+    # PyPI remote/virtual repository
+    pip install -r requirements.txt --index-url "https://svc-thub:$ART_TOKEN@artifactory.example.com/artifactory/api/pypi/pypi/simple"
+    # npm: a project-local .npmrc in the work directory, not ~/.npmrc
+    printf '//artifactory.example.com/artifactory/api/npm/npm/:_authToken=%s\n' "$ART_TOKEN" > .npmrc
+    npm ci --registry https://artifactory.example.com/artifactory/api/npm/npm/
+  p.small.
+    Nexus raw repositories take #[code curl -u user:password --upload-file]. GitLab's generic packages take #[code curl -H "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file].
+
+  h3.h6 AWS S3 (and MinIO, Ceph, Cloudflare R2, Wasabi)
+  p.small.
+    Pre-sign inputs in CI, so the job needs no AWS credentials. Pass short-lived session credentials from the CI step's role
+    (OIDC) for outputs. A pre-signed link lasts at most 7 days. For S3-compatible storage, add #[code --endpoint-url] to #[code aws].
+  +code('Inputs: pre-signed in the CI step').
+    # in the CI step, which has AWS access (e.g. GitHub/GitLab OIDC → an IAM role)
+    IMAGE_URL=$(aws s3 presign "s3://fw-builds/app/$SHA/app.bin" --expires-in 3600)
+    thub run --type hw --download-file "$IMAGE_URL" --command 'st-flash --reset write "$THUB_DOWNLOAD_1" 0x08000000 && ./ci/test.sh' --wait
+  +code('Outputs: aws s3 cp with session credentials').
+    thub run --type sw --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --env AWS_SESSION_TOKEN --env AWS_DEFAULT_REGION=eu-central-1 \
+      --command 'aws s3 cp "s3://fw-builds/app/1.4.0/app.elf" . && ./ci/test.sh; rc=$?
+                 aws s3 cp --recursive results/ "s3://qa-results/$THUB_JOB_ID/" &&
+                 sh ci/artifacts.sh report.html "$(aws s3 presign "s3://qa-results/$THUB_JOB_ID/report.html" --expires-in 604800)"
+                 exit $rc' --wait
+
+  h3.h6 Google Drive
+  p.small.
+    Use #[a(href="https://rclone.org/drive/" target="_blank" rel="noopener") rclone] with a #[strong service account] that's a
+    member of a #[strong shared drive] (service accounts have no storage of their own). rclone takes its configuration from
+    environment variables. #[code rclone link] makes a file readable by anyone with the link; leave it out for private reports.
+    Public #[code uc?export=download] links work with #[code --download-file] only for small files.
+  +code('rclone with a service account').
+    # GDRIVE_SA: the service account's JSON key, as one line (a CI secret)
+    thub run --type sw --env GDRIVE_SA --env RCLONE_CONFIG_GD_TYPE=drive --env RCLONE_CONFIG_GD_SCOPE=drive \
+      --env RCLONE_CONFIG_GD_TEAM_DRIVE=0ABCdEfGhIjKlUk9PVA \
+      --command 'export RCLONE_CONFIG_GD_SERVICE_ACCOUNT_CREDENTIALS="$GDRIVE_SA"
+                 rclone copyto "gd:firmware/1.4.0/app.bin" app.bin && ./ci/test.sh app.bin; rc=$?
+                 rclone copy results/ "gd:qa/$THUB_JOB_ID/" &&
+                 sh ci/artifacts.sh report.html "$(rclone link "gd:qa/$THUB_JOB_ID/report.html")"
+                 exit $rc' --wait
+  +code('Or the Drive API with curl and an OAuth token from the CI step').
+    curl -fsSL -H "Authorization: Bearer $GDRIVE_TOKEN" -o app.bin "https://www.googleapis.com/drive/v3/files/$FILE_ID?alt=media&supportsAllDrives=true"
+    curl -fsS -H "Authorization: Bearer $GDRIVE_TOKEN" \
+      -F "metadata={\"name\":\"report.html\",\"parents\":[\"$FOLDER_ID\"]};type=application/json" -F "file=@results/report.html" \
+      "https://www.googleapis.com/upload/drive/v3/files?uploadType=multipart&supportsAllDrives=true"
+
+  h3.h6 FTP, FTPS and SFTP
+  p.small.
+    #[code curl] speaks all three. Prefer SFTP or FTPS (#[code --ssl-reqd]); plain FTP sends the password in clear text. Pin the
+    SFTP host key with #[code --hostpubsha256] (from #[code ssh-keyscan host | ssh-keygen -lf -], without #[code SHA256:]).
+    FTP and SFTP links aren't #[code http(s)], so list the server's HTTPS view, if it has one.
+  +code('SFTP download and upload (FTPS and key auth in the comments)').
+    thub run --type hw --env SFTP_USER --env SFTP_PASSWORD --env SFTP_HOSTKEY \
+      --command 'auth() { printf "user = \"%s:%s\"\n" "$SFTP_USER" "$SFTP_PASSWORD"; }   # read by curl -K -, so not visible in ps
+                 auth | curl -fsS -K - --hostpubsha256 "$SFTP_HOSTKEY" -o app.bin "sftp://files.lab/fw/1.4.0/app.bin" &&
+                 st-flash --reset write app.bin 0x08000000 && ./ci/test.sh; rc=$?
+                 auth | curl -fsS -K - --hostpubsha256 "$SFTP_HOSTKEY" --ftp-create-dirs -T results/junit.xml "sftp://files.lab/qa/$THUB_JOB_ID/"
+                 exit $rc' --wait
+    # FTPS:         auth | curl -fsS -K - --ssl-reqd --ftp-create-dirs -T results/junit.xml "ftp://files.lab/qa/$THUB_JOB_ID/"
+    # SSH key auth: printf '%s\n' "$SFTP_KEY" > "$THUB_WORK_DIR/.key" && chmod 600 "$THUB_WORK_DIR/.key" &&
+    #               curl --key "$THUB_WORK_DIR/.key" -u "$SFTP_USER:" --hostpubsha256 "$SFTP_HOSTKEY" …
+
+  h3.h6 Any other HTTP storage: curl and your own authentication
+  +code('curl authentication patterns').
+    H() { printf 'header = "%s"\n' "$1"; }                                     # headers via curl -K -, kept out of ps
+    H "Authorization: Bearer $API_TOKEN" | curl -fsSL -K - -o app.bin "$URL"   # bearer token
+    H "X-API-Key: $API_KEY"              | curl -fsSL -K - -o app.bin "$URL"   # API-key header
+    printf 'user = "%s:%s"\n' "$USER_NAME" "$USER_PASS" | curl -fsSL -K - -o app.bin "$URL"   # basic auth
+    curl -fsSL --netrc-file "$THUB_WORK_DIR/.netrc" -o app.bin "$URL"         # a .netrc the command wrote from --env values
+    curl -fsSL --cert "$THUB_WORK_DIR/c.pem" --key "$THUB_WORK_DIR/k.pem" -o app.bin "$URL"   # mutual TLS
+    # OAuth2 client credentials: get a token, then use it as a bearer token
+    TOKEN=$(curl -fsS -d grant_type=client_credentials -u "$CLIENT_ID:$CLIENT_SECRET" https://auth.example.com/oauth/token | jq -r .access_token)
+    # uploads: PUT (-T file), or a form POST (-F "file=@results/report.html")
+  p.small.mb-0.
+    Once the patterns settle, move them into #[code ci/fetch.sh] / #[code ci/publish.sh] in the tests repository, so every job's
+    #[code --command] stays one line. The CI examples in #[a(href="#ci") CI/CD integration] use this storage to bring JUnit reports back.

package/views/help/_troubleshooting.pug

@@ -9,10 +9,18 @@
       ['The job is rejected with `422`', 'No registered Client can ever match its type, labels, group or client. Compare `thub resources` with your `--type` and `--label`s; your group (set on the dashboard, see `thub whoami`) must have a Client of that kind.'],
       ['The job stays `QUEUED`', 'Matching Clients are all `BUSY`, locked (`local`), in `MAINTENANCE` or offline, or the job is pinned with `--client`. It times out after `--timeout`.'],
       ['`ERROR` during prepare', 'A download failed; the job log names the URL and the reason.'],
+      ['`Download failed … (SELF_SIGNED_CERT_IN_CHAIN)` or `UNABLE_TO_GET_ISSUER_CERT_LOCALLY`', 'The server\'s certificate is from a CA the Client doesn\'t trust. Node.js ignores the system CA store: add `Environment=NODE_EXTRA_CA_CERTS=/path/ca.crt` with `sudo systemctl edit thub-client@.service` and restart the Clients. A download that needs a client certificate can\'t use `--download-file` at all: fetch it with `curl --cert` in `--command` (Artifact storage and certificates).'],
+      ['`Host key verification failed` on a git clone', 'The job\'s `known_hosts` has no key for that server (or a different one). Pass the output of `ssh-keyscan <host>` as `--env GIT_KNOWN_HOSTS` and use it with `UserKnownHostsFile` (Using git). `Permission denied (publickey)` means the host is fine, but the deploy key isn\'t on that repository.'],
+      ['`docker login`: unauthorized', 'Check the user the registry expects: `AWS` for ECR, `oauth2accesstoken` for Google, `gitlab-ci-token` with `CI_JOB_TOKEN` for GitLab (Using Docker). ECR and Google tokens expire after 12 h and 1 h, so mint them in the CI step that submits the job.'],
       ['A git clone or docker pull in the command fails', 'They run in your `--command`, with the credentials you pass as `--env` (see Using git / Using Docker). `git` or `docker` must be installed on that Client host; the Client itself needs neither. Try the command with `--dry-run` to see its environment.'],
       ['`--git-repo` / `--docker-image`: unknown option, or the job is refused', 'They were removed: clone the repository and run containers in `--command`, passing tokens with `--env`. An older Agent that still sends them is refused with that message; update it (`thub self-update`).'],
       ['Permission denied on `/dev/ttyUSB*` or `docker.sock`', 'Docker or the device groups were added after the Client was installed. Re-run `sudo npm i -g @andrian.yablonskyy/thub-client` so the unit gets the `dialout`/`plugdev`/`docker` groups, then restart.'],
       ['A configured device is shown as missing', 'Check the `devpath` with `udevadm info -a -n <dev>`, then `thub-client udev --print`; restart the Client to regenerate the udev rules.'],
+      ['`thub power` is refused', 'It says why: only the job\'s owner may switch power (`403`, even for an admin; `thub whoami` shows whose key you use); the job must be `PREPARING` or `RUNNING`; the Client must list ports in `hw-devices.usbPower.ports` and have `uhubctl` installed (restart it after installing, so it reports it). `--port` is a position in that list, not the hub\'s port number.'],
+      ['USB power: `No compatible devices detected!`', 'Either the hub can\'t switch per-port power (`sudo uhubctl` doesn\'t list it), or the Client\'s user may not switch it: install its udev rules with `sudo thub-client --config <path> udev` and check its service has the `plugdev` group (reinstall the Client if `plugdev` was created after it).'],
+      ['USB power is off, but the board stays on (or half-boots after a reset)', 'Something else powers it: a second USB cable (ST-Link and the board\'s own USB), a debugger or an external supply. Every cable that powers it needs a switched port in `usbPower.ports`. Some hubs also feed power back from upstream. A board that half-boots needs a longer delay: `--delay 3`.'],
+      ['A job can\'t reach its smart socket or PDU', 'The job\'s command runs on the Client host, so the device must be reachable from there: `curl -sI http://<device>/` as the Client\'s user. Tools such as `curl` or `snmpset` must be installed on that host. `BENCH_POWER is not set`: add the instance\'s systemd drop-in and restart it (`systemctl show thub-client@<instance> -p Environment`).'],
+      ['A board on a socket or PDU stays on after a canceled job', 'On cancel the Client sends `SIGTERM` to the job\'s shell only, then `SIGKILL` after 10 s. Start the wrapper with `exec`, run the tests in the background with `wait`, trap `TERM`, and finish within 10 s. Add a timer on the device (Shelly `toggle_after`, Tasmota `PulseTime`) as a backstop.'],
       ['Job `LOST`', 'The Client missed 3 heartbeats (network, reboot, crash). With `requeueOnLost` the job is retried once on another Client.'],
       ['A job can\'t write to `~` (read-only file system)', 'Under systemd the Client can only write to its state directory. Write to `$THUB_WORK_DIR`, and set `DOCKER_CONFIG` there for `docker login`.'],
       ['Live logs don\'t stream through the proxy', 'Turn off response buffering for the Coordinator (nginx: `proxy_buffering off;`, Caddy: `flush_interval -1`).'],

package/views/help/_usb-power.pug

@@ -0,0 +1,147 @@
++section('usb-power', 'USB port power (uhubctl)', 'power')
+  p.
+    An HW Client can switch the power of the USB ports its boards hang off, with
+    #[a(href="https://github.com/mvp/uhubctl" target="_blank" rel="noopener") uhubctl]. Use it to power-cycle a
+    hung board, start every job from a cold boot, or keep boards off between jobs. It needs a USB hub with
+    #[strong per-port power switching] (uhubctl's README lists hubs known to work) and #[code sudo apt install uhubctl]
+    on the Client host.
+
+  h3.h6 Setting it up
+  ol.small
+    li
+      | Find each board's hub and port. Run #[code uhubctl] with no arguments. The hub's #[strong location] is the
+      | word after #[code hub], and the #[strong port] is the number after #[code Port]. #[code ppps] means the hub
+      | switches each port on its own, and #[code ganged] means it switches all its ports together. A USB3 hub shows up twice
+      | (#[code 1-1.4] and #[code 2-1.4] below): use the half the board is on, and uhubctl switches the other with it.
+      +code('Client host').
+        $ sudo uhubctl
+        Current status for hub 1-1.4 [2109:2817 VIA Labs, Inc. USB2.0 Hub, USB 2.10, 4 ports, ppps]
+          Port 1: 0100 power
+          Port 2: 0103 power enable connect [0483:3748 STMicroelectronics STM32 STLink 066DFF485457725187092834]
+          Port 3: 0103 power enable connect [0483:3748 STMicroelectronics STM32 STLink 0670FF505055877267183229]
+          Port 4: 0000 off
+        Current status for hub 2-1.4 [2109:0817 VIA Labs, Inc. USB3.0 Hub, USB 3.00, 4 ports, ppps]
+          Port 1: 02a0 power 5gbps Rx.Detect
+    li
+      | Try it by hand:
+      +code('Client host').
+        sudo uhubctl -l 1-1.4 -p 2 -a off    # the board on port 2 goes dark
+        sudo uhubctl -l 1-1.4 -p 2 -a on
+    li
+      | List the ports under #[code hw-devices.usbPower.ports] in the Client's config, up to 8. Each port's number is
+      | its #[strong position in this list] (from 1), and that's what #[code --port] takes. It isn't the hub's own
+      | port number: below, #[code --port 2] is hub #[code 1-1.4] port 3.
+      +code('~/.config/thub/dut1.json').
+        "hw-devices": {
+          "stlinks": [{ "index": 1, "devpath": "1.4.2" }, { "index": 2, "devpath": "1.4.3" }],
+          "usbPower": {
+            "ports": [
+              { "hub": "1-1.4", "port": 2 },
+              { "hub": "1-1.4", "port": 3 }
+            ]
+          }
+        }
+    li
+      | Restart the Client. It installs a udev rule that lets the #[code plugdev] group (which its service has)
+      | switch hub ports without root, and reports the ports to the Coordinator, along with whether uhubctl is
+      | installed. Then check:
+      +code('Client host').
+        sudo systemctl restart thub-client@dut1
+        thub-client --config ~/.config/thub/dut1.json power status
+        #   1. hub 1-1.4 port 2: on  (Port 2: 0103 power enable connect [...])
+        #   2. hub 1-1.4 port 3: on  (Port 3: 0103 power enable connect [...])
+  +note.
+    Saving a runner's devices on its #[strong Config] tab keeps #[code usbPower] as it is. The ports are only edited in the
+    config file.
+
+  h3.h6 What the actions do
+  p.small.
+    Every action switches all the listed ports, unless #[code --port] picks one. The Client runs one power request at a
+    time, so a job's own reset, its owner's and one from the bench never overlap.
+  .table-responsive
+    table.table.table-sm.small.align-middle
+      thead
+        tr
+          th(style="width: 12%") Action
+          th What the Client runs
+      tbody
+        tr
+          td: code on
+          td: code uhubctl -l 1-1.4 -p 2,3 -a on
+        tr
+          td: code off
+          td: code uhubctl -l 1-1.4 -p 2,3 -a off
+        tr
+          td: code reset
+          td
+            | #[code uhubctl … -a off], then a wait of the #[strong reset delay] (1 s unless given, 0–60 s, fractions
+            | allowed), then #[code uhubctl … -a on]. Give boards that are slow to discharge a longer delay.
+        tr
+          td: code status
+          td #[code uhubctl -l 1-1.4 -p 2,3], each port's state from it (Client host only)
+
+  h3.h6 Three ways to switch the power
+  .row.g-3.mb-3
+    .col-md-4
+      .card.h-100
+        .card-header.small
+          i.bi.bi-pc.me-1
+          strong On the Client host
+        .card-body.small
+          p.
+            #[code thub-client power on|off|reset|status [--port N] [--delay SEC]]. It goes through the running Client.
+            If a job is running, its log records the action.
+    .col-md-4
+      .card.h-100
+        .card-header.small
+          i.bi.bi-play-circle.me-1
+          strong At a job's start and end
+        .card-body.small
+          p.mb-2.
+            #[code thub run --power-on-start … --power-on-end … [--power-reset-delay SEC]] (HW jobs). The start action runs
+            after the downloads, before the command; if it fails, the job ends #[code ERROR].
+          p.mb-0.
+            The end action runs however the job ends (failed and canceled jobs too), before the result is reported.
+            If it fails, the failure is logged and the verdict stays as it was.
+    .col-md-4
+      .card.h-100
+        .card-header.small
+          i.bi.bi-lightning.me-1
+          strong While your job runs
+        .card-body.small
+          p.mb-2.
+            #[code thub power on|off|reset <jobId> [--delay SEC] [--port N]]. Power-cycle a hung board straight away,
+            without canceling the job.
+          p.mb-0.
+            Only the job's owner can do this, not even an admin, and only while the job is #[code PREPARING] or
+            #[code RUNNING]. It reaches the Client within about a second.
+
+  +code('Cold-boot the board before the job, leave it off afterwards').
+    thub run --type hw --label board:nucleo-f401re \
+      --download-file "$IMAGE_URL" \
+      --command 'st-flash --reset write "$THUB_DOWNLOAD_1" 0x08000000 && ./ci/test.sh' \
+      --power-on-start reset --power-reset-delay 2 \
+      --power-on-end off \
+      --wait
+  +code('The board hung: power-cycle it from another terminal').
+    $ thub power reset M-00131
+    USB power reset, 1 s off sent to lab-hw-01 for job M-00131 — the job's log shows when it's done.
+    $ thub power reset M-00131 --port 2 --delay 3     # just the Client's second port, 3 s off
+    $ thub power off M-00131 && thub power on M-00131
+  +code('What the job log shows').
+    [runner] job start: USB power reset
+    [runner] USB power reset: 1-1.4:2, 1-1.4:3 (off 2 s)
+    [runner] preparing DUT
+    [runner] USB power reset (port 2) requested by alice (thub power)
+    [runner] USB power reset: 1-1.4:3 (off 3 s)
+    [runner] job end: USB power off
+    [runner] USB power off: 1-1.4:2, 1-1.4:3
+  +code('From the test script itself, between test cases (the Agent must be installed on the Client host)').
+    thub run --type hw --env AGENT_URL --env AGENT_KEY --command './ci/test.sh' --wait
+    # in ci/test.sh — the key must be the one that submitted the job:
+    thub --url "$AGENT_URL" --key "$AGENT_KEY" power reset "$THUB_JOB_ID" --delay 2
+  p.small.mb-0.
+    A dry run (#[code --dry-run]) lists the uhubctl calls instead of making them. The command sees what was asked for
+    as #[code JOB_POWER_ON_START], #[code JOB_POWER_ON_END] and #[code JOB_POWER_RESET_DELAY]. Problems are covered
+    under #[a(href="#troubleshooting") Troubleshooting]. Boards on a smart socket or a PDU outlet are switched by the job
+    itself: see #[a(href="#lab-devices") Smart sockets, PDUs and other lab devices].

package/views/help/index.pug

@@ -12,11 +12,14 @@ block content
       { id: 'agent-setup', label: 'Agent setup' },
       { id: 'client-setup', label: 'Client setup' },
       { id: 'client-machines', label: 'Client machines (HW / SW)' },
+      { id: 'usb-power', label: 'USB port power (uhubctl)' },
+      { id: 'lab-devices', label: 'Smart sockets, PDUs, devices' },
       { id: 'docker', label: 'Using Docker' },
       { id: 'git', label: 'Using git' },
       { id: 'agent-cli', label: 'Agent CLI reference' },
       { id: 'env', label: 'Environment variables' },
-      { id: 'ci', label: 'CI/CD (GitHub Actions)' },
+      { id: 'storage', label: 'Artifact storage & certificates' },
+      { id: 'ci', label: 'CI/CD (GitHub, GitLab, Bitbucket, Jenkins)' },
       { id: 'troubleshooting', label: 'Troubleshooting' }
     ]
 
@@ -58,10 +61,13 @@ block content
       include _agent-setup
       include _client-setup
       include _client-machines
+      include _usb-power
+      include _lab-devices
       include _docker
       include _git
       include _agent-cli
       include _env
+      include _storage
       include _ci
       include _troubleshooting
 

package/views/layout.pug

@@ -29,12 +29,13 @@ html(lang="en" data-bs-theme=(user && user.theme) || "auto")
             data-bs-title=`Coordinator v${coordinatorVersion} · thub-common v${commonVersion} (validates job specs)`
           )
             | ver. #{coordinatorVersion}
-            //- Without --self-update (or for non-admins): just the notice.
-            if user && updates.coordinatorUpdate && !(can.admin && updates.selfUpdate)
+            //- With --self-update, for those who can't update it themselves.
+            //- Without it, nothing about Coordinator updates is shown.
+            if user && updates.selfUpdate && updates.coordinatorUpdate && !can.admin
               span.badge.text-bg-warning.ms-1(
                 data-bs-toggle="tooltip"
                 data-bs-placement="bottom"
-                data-bs-title=updates.selfUpdate ? 'An admin can update the Coordinator from the navbar' : 'Update the Coordinator by hand (thub-admin self-update, npm i -g, or a new container image); self-update from the dashboard is off'
+                data-bs-title="An admin can update the Coordinator from the navbar"
               )= `v${updates.coordinatorUpdate} available`
         button.navbar-toggler(type="button" data-bs-toggle="collapse" data-bs-target="#nav")
           span.navbar-toggler-icon
@@ -80,7 +81,9 @@ html(lang="en" data-bs-theme=(user && user.theme) || "auto")
                 )
                   i.bi.bi-question-circle.me-1
                   | Help
-            if user && can.operate
+            //- The Coordinator's own update check: only with --self-update.
+            //- (Agents and Clients are checked from their own pages, always.)
+            if user && can.operate && updates.selfUpdate
               li.nav-item.d-flex.align-items-center.me-2
                 form(method="post" action="/updates/check" data-submit-busy)
                   input(type="hidden" name="returnTo" value=currentPath)

package/views/mixins/client-config.pug

@@ -234,10 +234,13 @@ mixin configPanes(r, returnTo, groupsById)
       +configPane(p, t[0])
         p.small.text-body-secondary.mb-0 Devices can't be edited here: #{noConfigReport}
   else
+    //- What the tabs don't edit (usbPower, README §8.7) goes back as it was.
+    - const keep = Object.fromEntries(Object.entries(current).filter(([k]) => !['stlinks', 'uarts', 'usbs'].includes(k)))
     form(
       method="post"
       action=`/runners/${r.id}/config`
       data-client-config="hw"
+      data-keep=JSON.stringify(keep)
       data-confirm=`Apply these devices to ${r.name}? The Client writes them to its config file on its next heartbeat and restarts once it has no job running.`
       data-confirm-title="Save Client devices?"
       data-confirm-ok="Save"