@andrian.yablonskyy/thub-coordinator 1.0.37 → 1.0.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/thub-admin.js +3 -5
- package/config.json +1 -6
- package/package.json +2 -2
- package/public/css/thub.css +129 -0
- package/public/js/config-import.js +37 -33
- package/public/js/copy-to-clipboard.js +21 -18
- package/public/js/help.js +93 -0
- package/public/js/list-search.js +51 -0
- package/public/js/live.js +287 -0
- package/public/js/log-viewer.js +289 -45
- package/public/js/remove-resource.js +5 -3
- package/public/js/resource-card.js +14 -13
- package/public/js/tooltips.js +3 -1
- package/scripts/install-default-config.js +1 -1
- package/scripts/install-target.js +2 -2
- package/src/api/admin.js +1 -1
- package/src/api/agent.js +4 -10
- package/src/api/resource.js +6 -11
- package/src/api/sse.js +13 -10
- package/src/config.js +16 -9
- package/src/db/migrations/022_add_sessions.sql +10 -0
- package/src/db/migrations/023_drop_artifacts.sql +4 -0
- package/src/dev/virtual.js +1 -1
- package/src/server.js +55 -21
- package/src/services/cleanup.js +2 -2
- package/src/services/jobs.js +33 -19
- package/src/services/list-prefs.js +4 -0
- package/src/services/live.js +124 -0
- package/src/services/logs.js +12 -14
- package/src/services/retention.js +5 -14
- package/src/services/search.js +59 -0
- package/src/services/session-store.js +127 -0
- package/src/web/routes.js +205 -34
- package/test/cleanup.test.js +1 -5
- package/test/list-prefs.test.js +6 -0
- package/test/live.test.js +153 -0
- package/test/logs.test.js +150 -0
- package/test/scheduler.test.js +10 -12
- package/test/search.test.js +97 -0
- package/test/session.test.js +132 -0
- package/test/views.test.js +30 -0
- package/views/admin/agents.pug +28 -10
- package/views/groups/list.pug +14 -1
- package/views/help/_agent-cli.pug +103 -0
- package/views/help/_agent-setup.pug +72 -0
- package/views/help/_ci.pug +44 -0
- package/views/help/_client-machines.pug +87 -0
- package/views/help/_client-setup.pug +122 -0
- package/views/help/_coordinator.pug +137 -0
- package/views/help/_docker.pug +124 -0
- package/views/help/_env.pug +127 -0
- package/views/help/_git.pug +130 -0
- package/views/help/_mixins.pug +24 -0
- package/views/help/_overview.pug +118 -0
- package/views/help/_quick-start.pug +37 -0
- package/views/help/_troubleshooting.pug +44 -0
- package/views/help/index.pug +67 -0
- package/views/index.pug +3 -2
- package/views/jobs/list.pug +8 -5
- package/views/jobs/show.pug +0 -10
- package/views/layout.pug +24 -1
- package/views/mixins/list-controls.pug +36 -0
- package/views/mixins/log-viewer.pug +39 -3
- package/views/mixins/resource.pug +1 -1
- package/views/resources/list.pug +7 -4
- package/src/services/artifacts.js +0 -149
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
+section('agent-cli', 'Agent CLI reference', 'code-square')
|
|
2
|
+
h3.h6 Commands
|
|
3
|
+
+code('thub --help').
|
|
4
|
+
thub run [options] Submit a job and follow its log
|
|
5
|
+
thub status <jobId> [--json] Status; follow the log if running, verdict and test counts if done
|
|
6
|
+
thub cancel <jobId> Cancel a job
|
|
7
|
+
thub resources [--json] Clients and their status
|
|
8
|
+
thub jobs [--mine] [--state <s>] [--json] Recent jobs
|
|
9
|
+
thub config set <url|token|group|user> <value>
|
|
10
|
+
thub check-update Compare with the latest published Agent
|
|
11
|
+
thub self-update [--to <x.y.z>] Update with npm i -g
|
|
12
|
+
thub --version
|
|
13
|
+
Global: --url <url> --token <token> (override THUB_URL / THUB_TOKEN / config file)
|
|
14
|
+
|
|
15
|
+
h3.h6 #[code thub run] options
|
|
16
|
+
-
|
|
17
|
+
const groups = [
|
|
18
|
+
{ title: 'Where to run', rows: [
|
|
19
|
+
['--type hw|sw', 'Required. Resource type.', '--type hw'],
|
|
20
|
+
['--board <name>', 'Shorthand for --label board:<name>.', '--board nucleo-f401re'],
|
|
21
|
+
['--label <l>', 'Required label (repeatable). The Client must have all of them.', '--label uart --label stlink'],
|
|
22
|
+
['--group <id>', 'Only Clients in this group. Default: THUB_GROUP / config.', '--group 548ae4ae-…'],
|
|
23
|
+
['--client <name|id>', 'Only this Client; waits for it even if others are idle.', '--client lab-hw-01']
|
|
24
|
+
] },
|
|
25
|
+
{ title: 'What to run', rows: [
|
|
26
|
+
['--command <string>', 'Required. Run with sh -c in the work directory. Exit code 0 = PASSED.', "--command './ci/test.sh'"],
|
|
27
|
+
['--arg <value>', 'Argument for the command, as "$@" (repeatable).', '--arg --junit --arg -v'],
|
|
28
|
+
['--suite <name>', 'Passed as THUB_SUITE (default: default).', '--suite smoke'],
|
|
29
|
+
['--download-file <url>', 'File downloaded before the command (repeatable, http(s), no credentials). THUB_DOWNLOAD_1…', '--download-file https://…/app.bin'],
|
|
30
|
+
['--git-repo <url> [ref]', 'Repository cloned first; the command runs in it.', '--git-repo git@github.com:org/tests.git v1.4.0'],
|
|
31
|
+
['--depth <n>', 'Commits to fetch with --git-repo (default 1, 0 = full).', '--depth 0'],
|
|
32
|
+
['--git-options <string>', 'Extra git options between git and its subcommand. Stored with the job.', '--git-options \'-c core.sshCommand="ssh -i ~/.ssh/k"\''],
|
|
33
|
+
['--docker-image <ref>', 'SW only: DUT container image, pulled with the Client host’s Docker login.', '--docker-image registry.lab:5000/emu:1'],
|
|
34
|
+
['--env <vars>', 'NAME=value[,NAME=value] for git and the command (repeatable). --env NAME takes the value from your shell. Masked, dropped at job end.', '--env TARGET=staging --env API_TOKEN']
|
|
35
|
+
] },
|
|
36
|
+
{ title: 'Job settings', rows: [
|
|
37
|
+
['--timeout <dur>', 'e.g. 30m, 1h (default 30m, capped by the Coordinator).', '--timeout 1h'],
|
|
38
|
+
['--priority <n>', '0–100. Defaults: ci 50, cli 60.', '--priority 80'],
|
|
39
|
+
['--user <name>', 'Free-text owner label. Default: THUB_USER / config.', '--user alice'],
|
|
40
|
+
['--meta <key=value>', 'Metadata stored on the job (repeatable). The command sees THUB_META_<KEY>.', '--meta ciJobId=$GITHUB_RUN_ID'],
|
|
41
|
+
['--dry-run', 'Schedule for real, but only log what the Client would run.', '--dry-run']
|
|
42
|
+
] },
|
|
43
|
+
{ title: 'Agent behavior', rows: [
|
|
44
|
+
['--wait', 'Follow to the end and exit with the verdict code (CI). Ctrl-C/SIGINT cancels the job.', '--wait'],
|
|
45
|
+
['--detach', 'Print the job id and exit.', '--detach --json'],
|
|
46
|
+
['--json', 'Machine-readable output.', '--json']
|
|
47
|
+
] }
|
|
48
|
+
]
|
|
49
|
+
each g in groups
|
|
50
|
+
h4.h6.text-body-secondary.mt-3= g.title
|
|
51
|
+
.table-responsive
|
|
52
|
+
table.table.table-sm.small.align-middle
|
|
53
|
+
thead
|
|
54
|
+
tr
|
|
55
|
+
th(style="width: 22%") Option
|
|
56
|
+
th Description
|
|
57
|
+
th(style="width: 32%") Example
|
|
58
|
+
tbody
|
|
59
|
+
each r in g.rows
|
|
60
|
+
tr
|
|
61
|
+
td: code.text-nowrap= r[0]
|
|
62
|
+
td= r[1]
|
|
63
|
+
td: code= r[2]
|
|
64
|
+
|
|
65
|
+
h3.h6 Exit codes
|
|
66
|
+
table.table.table-sm.small.w-auto
|
|
67
|
+
tbody
|
|
68
|
+
each e in [['0', 'PASSED'], ['1', 'FAILED'], ['2', 'ERROR, TIMEOUT or LOST'], ['3', 'CANCELED'], ['4', 'Usage, auth or connection error'], ['5', 'thub status --json: still queued or running'], ['130', 'Detached with Ctrl-C (job keeps running)']]
|
|
69
|
+
tr
|
|
70
|
+
td: code= e[0]
|
|
71
|
+
td= e[1]
|
|
72
|
+
|
|
73
|
+
h3.h6 Examples
|
|
74
|
+
+code('Flash and test a board, fail CI on failure').
|
|
75
|
+
thub run --type hw --board nucleo-f401re \
|
|
76
|
+
--download-file https://artifactory.example.com/fw-local/app/1.4.0-42/app.bin \
|
|
77
|
+
--git-repo https://github.com/yourorg/firmware-tests.git v1.4.0 \
|
|
78
|
+
--command 'st-flash --serial "$THUB_DUT_STLINK" --reset write "$THUB_DOWNLOAD_1" 0x08000000 && ./ci/test.sh "$@"' \
|
|
79
|
+
--arg --junit --timeout 45m --wait
|
|
80
|
+
+code('Reproduce a failure on the exact bench, labeled with your name').
|
|
81
|
+
thub run --type hw --board nucleo-f401re --client lab-hw-01 --user "$USER" \
|
|
82
|
+
--download-file "$IMAGE_URL" --git-repo "$TESTS_REPO" --command ./ci/test.sh
|
|
83
|
+
# Ctrl-C detaches; the job keeps running. Re-attach:
|
|
84
|
+
thub status M-00126
|
|
85
|
+
+code('Verify a download before using it').
|
|
86
|
+
thub run --type hw --download-file https://artifactory.example.com/fw-local/app/1.4.0-42/app.bin \
|
|
87
|
+
--command 'echo "8f15bf27… $THUB_DOWNLOAD_1" | sha256sum -c && ./flash.sh "$THUB_DOWNLOAD_1"' --wait
|
|
88
|
+
+code('Secrets and parameters').
|
|
89
|
+
export API_TOKEN=…
|
|
90
|
+
thub run --type sw --git-repo "$TESTS_REPO" \
|
|
91
|
+
--env TARGET=staging,LOG_LEVEL=debug --env API_TOKEN \
|
|
92
|
+
--command './run-tests.sh --target "$TARGET"' --suite regression --wait
|
|
93
|
+
+code('Preview exactly what would run (nothing is executed)').
|
|
94
|
+
thub run --type sw --docker-image alpine --git-repo https://example.invalid/tests.git main \
|
|
95
|
+
--command ./ci/test.sh --dry-run --wait
|
|
96
|
+
+code('Scripting: submit, poll, read the result').
|
|
97
|
+
JOB=$(thub run --type sw --git-repo "$TESTS_REPO" --command ./ci/test.sh --detach --json | jq -r .jobId)
|
|
98
|
+
while thub status "$JOB" --json > job.json; [ $? -eq 5 ]; do sleep 10; done
|
|
99
|
+
jq -r '"\(.state) exit=\(.exit_code) tests=\(.summary.total // 0) failed=\(.summary.failed // 0)"' job.json
|
|
100
|
+
+code('Lists').
|
|
101
|
+
thub resources
|
|
102
|
+
thub jobs --mine --state FAILED
|
|
103
|
+
thub cancel M-00126
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
+section('agent-setup', 'Agent setup', 'terminal')
|
|
2
|
+
p.
|
|
3
|
+
The Agent (#[code thub]) is a stateless Node.js CLI. Install it wherever jobs are submitted from:
|
|
4
|
+
a developer machine (macOS, Linux or Windows with Node.js 20+) or a CI runner.
|
|
5
|
+
|
|
6
|
+
h3.h6 Install
|
|
7
|
+
+code('Developer machine').
|
|
8
|
+
npm i -g @andrian.yablonskyy/thub-agent
|
|
9
|
+
thub --version
|
|
10
|
+
p.small.
|
|
11
|
+
In CI you don't need a global install: #[code npx -y @andrian.yablonskyy/thub-agent run …].
|
|
12
|
+
|
|
13
|
+
h3.h6 Get a token
|
|
14
|
+
p.small.
|
|
15
|
+
An admin registers an agent on #[a(href="/admin/agents") Agents] (or with #[code thub-admin agent add]) and passes you
|
|
16
|
+
its token, which is shown only once. The agent's #[strong kind] decides the job id prefix and the default priority:
|
|
17
|
+
table.table.table-sm.small.w-auto
|
|
18
|
+
thead
|
|
19
|
+
tr
|
|
20
|
+
th Kind
|
|
21
|
+
th For
|
|
22
|
+
th Job ids
|
|
23
|
+
th Default priority
|
|
24
|
+
tbody
|
|
25
|
+
tr
|
|
26
|
+
td: code ci
|
|
27
|
+
td Pipelines
|
|
28
|
+
td: code A-00001…
|
|
29
|
+
td 50
|
|
30
|
+
tr
|
|
31
|
+
td: code cli
|
|
32
|
+
td Developers
|
|
33
|
+
td: code M-00001…
|
|
34
|
+
td 60 (so a busy pipeline doesn't starve them)
|
|
35
|
+
|
|
36
|
+
h3.h6 Configure
|
|
37
|
+
+code('Save the settings once').
|
|
38
|
+
thub config set url #{coordinatorUrl}
|
|
39
|
+
thub config set token agt_…
|
|
40
|
+
thub config set user "Your Name" # optional: job owner label
|
|
41
|
+
thub config set group <groupId> # optional: default --group
|
|
42
|
+
p.small.
|
|
43
|
+
These are saved in #[code ~/.config/thub/agent.json]. Order of precedence: flags (#[code --url], #[code --token]) →
|
|
44
|
+
environment → config file.
|
|
45
|
+
table.table.table-sm.small
|
|
46
|
+
thead
|
|
47
|
+
tr
|
|
48
|
+
th Variable
|
|
49
|
+
th Meaning
|
|
50
|
+
tbody
|
|
51
|
+
tr
|
|
52
|
+
td: code THUB_URL
|
|
53
|
+
td Coordinator URL, e.g. #[code #{coordinatorUrl}]
|
|
54
|
+
tr
|
|
55
|
+
td: code THUB_TOKEN
|
|
56
|
+
td Agent token. Keep it in your CI's secret store.
|
|
57
|
+
tr
|
|
58
|
+
td: code THUB_GROUP
|
|
59
|
+
td Default #[code --group]
|
|
60
|
+
tr
|
|
61
|
+
td: code THUB_USER
|
|
62
|
+
td Default #[code --user]
|
|
63
|
+
tr
|
|
64
|
+
td: code THUB_NO_SELF_UPDATE=1
|
|
65
|
+
td Don't install admin-requested Agent updates on this machine
|
|
66
|
+
+code('Check the connection').
|
|
67
|
+
thub resources # lists the Clients and their status
|
|
68
|
+
thub jobs --mine # your recent jobs
|
|
69
|
+
p.small.mb-0.
|
|
70
|
+
When an admin requests an Agent update, the next #[code thub] command installs it and then runs on
|
|
71
|
+
the new version. If it can't install (e.g. no permission in CI), it prints the manual command and
|
|
72
|
+
carries on with the current version.
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
+section('ci', 'CI/CD integration (GitHub Actions)', 'github')
|
|
2
|
+
p.
|
|
3
|
+
The test job only needs the Agent and HTTPS access to the Coordinator, so it runs on an ordinary hosted runner.
|
|
4
|
+
Create a #[strong ci]-kind agent on #[a(href="/admin/agents") Agents] and store its token as the repository secret
|
|
5
|
+
#[code THUB_AGENT_TOKEN].
|
|
6
|
+
+code('.github/workflows/firmware.yml').
|
|
7
|
+
name: firmware
|
|
8
|
+
on: [push, pull_request]
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
build:
|
|
12
|
+
runs-on: [self-hosted, fw-build]
|
|
13
|
+
outputs:
|
|
14
|
+
image_url: ${{ steps.upload.outputs.image_url }}
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v4
|
|
17
|
+
- run: cmake -B build -G Ninja && cmake --build build
|
|
18
|
+
- id: upload
|
|
19
|
+
env: { ART_TOKEN: "${{ secrets.ARTIFACTORY_TOKEN }}" }
|
|
20
|
+
run: |
|
|
21
|
+
URL="https://artifactory.example.com/fw-local/app/${GITHUB_SHA::7}-${GITHUB_RUN_NUMBER}/app.bin"
|
|
22
|
+
curl -fsS -H "Authorization: Bearer $ART_TOKEN" -T build/app.bin "$URL"
|
|
23
|
+
echo "image_url=$URL" >> "$GITHUB_OUTPUT"
|
|
24
|
+
|
|
25
|
+
test-hw:
|
|
26
|
+
needs: build
|
|
27
|
+
runs-on: ubuntu-latest
|
|
28
|
+
env:
|
|
29
|
+
THUB_URL: #{coordinatorUrl}
|
|
30
|
+
THUB_TOKEN: ${{ secrets.THUB_AGENT_TOKEN }}
|
|
31
|
+
steps:
|
|
32
|
+
- run: |
|
|
33
|
+
npx -y @andrian.yablonskyy/thub-agent run \
|
|
34
|
+
--type hw --board nucleo-f401re \
|
|
35
|
+
--download-file "${{ needs.build.outputs.image_url }}" \
|
|
36
|
+
--git-repo "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git" "$GITHUB_SHA" \
|
|
37
|
+
--command 'st-flash --serial "$THUB_DUT_STLINK" --reset write "$THUB_DOWNLOAD_1" 0x08000000 && ./ci/hw-tests.sh' \
|
|
38
|
+
--suite smoke --timeout 30m --wait \
|
|
39
|
+
--meta runUrl="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
|
|
40
|
+
ul.small.mb-0
|
|
41
|
+
li The step's exit code is the verdict (#[a(href="#agent-cli") exit codes]), so the workflow fails exactly when the tests do.
|
|
42
|
+
li If the workflow is canceled, the Agent (in #[code --wait] mode) cancels the TestHub job, so abandoned runs don't hold hardware.
|
|
43
|
+
li Downloads carry no credentials. Publish the image to a URL the Client can read, or fetch it in #[code --command] with a token passed as #[code --env].
|
|
44
|
+
li Other CI systems (GitLab CI, Jenkins, Azure Pipelines) work the same way: install Node.js, set #[code THUB_URL]/#[code THUB_TOKEN] and run #[code thub run … --wait].
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
+section('client-machines', 'Client machines: HW and SW', 'pc')
|
|
2
|
+
p.
|
|
3
|
+
Which machine you need depends on what the jobs run against. Clients are supported on
|
|
4
|
+
#[strong Ubuntu 26.04]; other systemd-based Linux distributions work the same way in practice.
|
|
5
|
+
Every Client machine needs outbound HTTPS to the Coordinator (and to wherever jobs download from:
|
|
6
|
+
Artifactory, git servers, Docker registries). Nothing inbound.
|
|
7
|
+
|
|
8
|
+
.row.g-3.mb-3
|
|
9
|
+
.col-md-6
|
|
10
|
+
.card.h-100.border-primary-subtle
|
|
11
|
+
.card-header
|
|
12
|
+
i.bi.bi-cpu.me-1
|
|
13
|
+
strong HW Client: real machine with a physical DUT
|
|
14
|
+
.card-body.small
|
|
15
|
+
ul.mb-0
|
|
16
|
+
li A physical machine (a mini-PC, NUC or Raspberry-class ARM64 board) next to the bench.
|
|
17
|
+
li USB for the DUT's debugger (ST-Link), USB-serial adapters (UART) and DUT USB. A #[strong powered USB hub] is recommended.
|
|
18
|
+
li #[code stlink-tools] and/or #[code openocd], #[code usbutils] (for the dashboard's USB scan).
|
|
19
|
+
li Docker only if jobs run containers in their #[code --command].
|
|
20
|
+
li One Client instance per board. One machine can drive several boards (up to 8 ST-Links, UARTs and USB devices per instance).
|
|
21
|
+
.col-md-6
|
|
22
|
+
.card.h-100.border-success-subtle
|
|
23
|
+
.card-header
|
|
24
|
+
i.bi.bi-pc-display.me-1
|
|
25
|
+
strong SW Client: real or virtual machine, software-only jobs
|
|
26
|
+
.card-body.small
|
|
27
|
+
ul.mb-0
|
|
28
|
+
li Any Linux host: bare metal, a VM (Proxmox, VMware, Hyper-V, KVM) or a cloud instance.
|
|
29
|
+
li Docker Engine (#[code docker.io]), with the Client's service user in the #[code docker] group (the installer arranges this).
|
|
30
|
+
li Each job's DUT container is limited to 2 CPUs and 2 GB RAM. Size the VM for one job plus your test tools: 4 vCPU, 8 GB RAM and 40 GB+ disk for images and workspaces is a good start.
|
|
31
|
+
li For KVM-accelerated emulators (QEMU) inside a VM, turn on #[strong nested virtualization].
|
|
32
|
+
li No USB or udev setup.
|
|
33
|
+
|
|
34
|
+
h3.h6 Preparing an HW machine
|
|
35
|
+
ol.small
|
|
36
|
+
li Install the OS and the Client (#[a(href="#client-setup") Client setup]), then plug in the board's ST-Link, UART adapter and DUT USB.
|
|
37
|
+
li
|
|
38
|
+
| Find each device's USB #[strong port path] (#[code devpath]). It's stable across replugs as long as the device stays in the same port:
|
|
39
|
+
+code('HW machine').
|
|
40
|
+
lsusb -t # USB tree
|
|
41
|
+
udevadm info -a -n /dev/ttyUSB0 | grep -m1 'ATTRS{devpath}'
|
|
42
|
+
st-info --probe # ST-Link serials
|
|
43
|
+
| Or open the resource card → #[strong Connected USB devices] → #[strong Refresh], then #[strong Import to config].
|
|
44
|
+
li
|
|
45
|
+
| Put the devpaths into #[code hw-devices] and restart. On every start the Client writes
|
|
46
|
+
| #[code /etc/udev/rules.d/99-thub-<instance>.rules], so the devices appear as #[code /dev/thub/dut<N>-stlink|uart|usb]:
|
|
47
|
+
+code('HW machine').
|
|
48
|
+
sudo systemctl restart thub-client@client
|
|
49
|
+
ls -l /dev/thub/
|
|
50
|
+
li Check the resource card's #[strong Capabilities]: each configured device must be present (a missing one is flagged).
|
|
51
|
+
li Your job's #[code --command] flashes the board. The Client never flashes by itself; it passes the ST-Link serials and the downloaded files in the environment (#[a(href="#env") Environment variables]).
|
|
52
|
+
+code('A typical HW job').
|
|
53
|
+
thub run --type hw --board nucleo-f401re \
|
|
54
|
+
--download-file https://artifactory.example.com/fw-local/app/1.4.0-42/app.bin \
|
|
55
|
+
--git-repo https://github.com/yourorg/firmware-tests.git v1.4.0 \
|
|
56
|
+
--command 'st-flash --serial "$THUB_DUT_STLINK" --reset write "$THUB_DOWNLOAD_1" 0x08000000 && ./ci/test.sh' \
|
|
57
|
+
--wait
|
|
58
|
+
|
|
59
|
+
+note('warning').
|
|
60
|
+
#[strong HW in a VM?] USB passthrough works, but the #[code devpath] then reflects the #[em virtual] USB topology and
|
|
61
|
+
can change when the hypervisor reassigns devices. Serial timing-sensitive tests may also suffer. Use a physical machine for HW Clients.
|
|
62
|
+
|
|
63
|
+
h3.h6 Preparing an SW machine (real or virtual)
|
|
64
|
+
+code('SW machine / VM').
|
|
65
|
+
sudo apt install -y nodejs npm docker.io
|
|
66
|
+
sudo npm i -g @andrian.yablonskyy/thub-client
|
|
67
|
+
nano ~/.config/thub/client.json # "type": "sw", coordinatorUrl, joinKey
|
|
68
|
+
sudo systemctl enable --now thub-client@client
|
|
69
|
+
# a private registry for DUT images? log in once, as the Client's user:
|
|
70
|
+
docker login registry.lab.local:5000
|
|
71
|
+
p.small.
|
|
72
|
+
To scale out, clone the VM template. In the clone, #[strong before its first start], run
|
|
73
|
+
#[code rm -f ~/var/lib/thub/client/*/.client-id ~/var/lib/thub/client/*.token] and give it its own #[code name].
|
|
74
|
+
Otherwise both VMs register as the same resource.
|
|
75
|
+
+code('A typical SW job').
|
|
76
|
+
thub run --type sw \
|
|
77
|
+
--docker-image registry.lab.local:5000/dut-emulator:2026.08 \
|
|
78
|
+
--git-repo git@github.com:yourorg/firmware-tests.git main \
|
|
79
|
+
--command 'make test DUT="$THUB_DUT_HOST"' --wait
|
|
80
|
+
|
|
81
|
+
h3.h6 Checklist for any Client machine
|
|
82
|
+
ul.small.mb-0
|
|
83
|
+
li Correct time (NTP). Timestamps are anchored to the Coordinator's clock, but TLS needs a sane clock.
|
|
84
|
+
li Outbound HTTPS to the Coordinator; test it with #[code curl -sI #{coordinatorUrl}/login].
|
|
85
|
+
li The Client's service user has SSH keys and git credentials for private repos (#[a(href="#git") Git repositories]) and Docker logins for private registries (#[a(href="#docker") Docker registry]).
|
|
86
|
+
li Under systemd, the service can only write to its state directory. Jobs write to #[code $THUB_WORK_DIR], not to #[code ~].
|
|
87
|
+
li Optional: a nightly reboot from the resource card's #[strong Reboot] tab (cron, host-local time). It waits for running jobs.
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
+section('client-setup', 'Client setup', 'motherboard')
|
|
2
|
+
p.
|
|
3
|
+
A Client (#[code thub-client]) is a daemon on a lab machine. One #[strong instance] owns one DUT slot. A machine with
|
|
4
|
+
several boards runs several instances (#[code dut0], #[code dut1], …), each with its own config file.
|
|
5
|
+
Clients register themselves with the shared join key; there's no admin step.
|
|
6
|
+
|
|
7
|
+
h3.h6 1. Install
|
|
8
|
+
+code('Lab machine, as the user the Client should run as (via sudo)').
|
|
9
|
+
# Node.js and HW tools (stlink-tools/openocd only for HW Clients)
|
|
10
|
+
sudo apt install -y nodejs npm stlink-tools openocd usbutils
|
|
11
|
+
# Docker: needed on SW Clients, and on HW Clients whose jobs use docker.
|
|
12
|
+
# Install it BEFORE thub-client so the service gets the docker group.
|
|
13
|
+
sudo apt install -y docker.io
|
|
14
|
+
sudo npm i -g @andrian.yablonskyy/thub-client
|
|
15
|
+
p.small.
|
|
16
|
+
This creates #[code ~/.config/thub/client.json] and #[code ~/var/lib/thub/client] for the user who ran
|
|
17
|
+
#[code sudo], and installs the #[code thub-client@.service] systemd unit running as that user, with the
|
|
18
|
+
#[code dialout], #[code plugdev] and #[code docker] groups. It also installs the root helpers that
|
|
19
|
+
apply self-updates and scheduled reboots.
|
|
20
|
+
|
|
21
|
+
h3.h6 2. Configure and start
|
|
22
|
+
.row.g-3
|
|
23
|
+
.col-md-6
|
|
24
|
+
+code('SW Client — ~/.config/thub/client.json').
|
|
25
|
+
{
|
|
26
|
+
"coordinatorUrl": "#{coordinatorUrl}",
|
|
27
|
+
"name": "lab-sw-01",
|
|
28
|
+
"type": "sw",
|
|
29
|
+
"labels": [],
|
|
30
|
+
"groups": [],
|
|
31
|
+
"joinKey": "<the Coordinator's clientJoinKey>",
|
|
32
|
+
"varDir": "/home/thub/var/lib/thub/client"
|
|
33
|
+
}
|
|
34
|
+
.col-md-6
|
|
35
|
+
+code('HW Client — ~/.config/thub/client.json').
|
|
36
|
+
{
|
|
37
|
+
"coordinatorUrl": "#{coordinatorUrl}",
|
|
38
|
+
"name": "lab-hw-01",
|
|
39
|
+
"type": "hw",
|
|
40
|
+
"labels": ["board:nucleo-f401re", "uart", "stlink"],
|
|
41
|
+
"groups": ["<group id>"],
|
|
42
|
+
"joinKey": "<the Coordinator's clientJoinKey>",
|
|
43
|
+
"varDir": "/home/thub/var/lib/thub/client",
|
|
44
|
+
"hw-devices": {
|
|
45
|
+
"stlinks": [{ "index": 1, "devpath": "3.3.4.3.1" }],
|
|
46
|
+
"uarts": [{ "index": 1, "devpath": "3.3.3.2", "baudRate": 115200 }],
|
|
47
|
+
"usbs": [{ "index": 1, "devpath": "3.2.4" }]
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
+code('Start it and follow the log').
|
|
51
|
+
sudo systemctl enable --now thub-client@client
|
|
52
|
+
journalctl -u thub-client@client -f
|
|
53
|
+
|
|
54
|
+
h3.h6 Config reference
|
|
55
|
+
.table-responsive
|
|
56
|
+
table.table.table-sm.small
|
|
57
|
+
thead
|
|
58
|
+
tr
|
|
59
|
+
th Field
|
|
60
|
+
th Required
|
|
61
|
+
th Meaning
|
|
62
|
+
tbody
|
|
63
|
+
tr
|
|
64
|
+
td: code coordinatorUrl
|
|
65
|
+
td Yes
|
|
66
|
+
td The Coordinator's #[code publicUrl].
|
|
67
|
+
tr
|
|
68
|
+
td: code type
|
|
69
|
+
td Yes
|
|
70
|
+
td #[code hw] (physical DUT) or #[code sw] (DUT image in Docker).
|
|
71
|
+
tr
|
|
72
|
+
td: code joinKey
|
|
73
|
+
td Yes
|
|
74
|
+
td Must equal the Coordinator's #[code clientJoinKey]. Or set #[code THUB_CLIENT_JOIN_KEY].
|
|
75
|
+
tr
|
|
76
|
+
td: code name
|
|
77
|
+
td No
|
|
78
|
+
td Resource name. Under systemd, the instance name (#[code thub-client@dut1] → #[code dut1]) is used unless an admin renames it on the dashboard.
|
|
79
|
+
tr
|
|
80
|
+
td: code labels
|
|
81
|
+
td No
|
|
82
|
+
td Matched against a job's #[code --board]/#[code --label]. Use #[code board:<name>] for the board.
|
|
83
|
+
tr
|
|
84
|
+
td: code groups
|
|
85
|
+
td No
|
|
86
|
+
td Group ids (from #[a(href="/groups") Groups]) this Client belongs to, for #[code thub run --group].
|
|
87
|
+
tr
|
|
88
|
+
td: code varDir
|
|
89
|
+
td No
|
|
90
|
+
td State directory: tokens, the client id, job workspaces, sockets. Keep the installer's default under systemd.
|
|
91
|
+
tr
|
|
92
|
+
td: code hw-devices.stlinks / uarts / usbs
|
|
93
|
+
td HW
|
|
94
|
+
td Up to 8 of each. An entry is a udev index #[code N] (→ #[code /dev/thub/dut<N>-stlink|uart|usb]), a path, or an object with #[code index]/#[code path], #[code devpath] (USB port), #[code serial] (ST-Link) and #[code baudRate] (UART, default 115200).
|
|
95
|
+
tr
|
|
96
|
+
td: code heartbeatIntervalSec / longPollWaitSec
|
|
97
|
+
td No
|
|
98
|
+
td 10 s / 30 s by default.
|
|
99
|
+
tr
|
|
100
|
+
td: code clientId
|
|
101
|
+
td No
|
|
102
|
+
td An explicit identity UUID (or #[code THUB_CLIENT_ID]). Normally generated once and kept in #[code .client-id].
|
|
103
|
+
p.small.
|
|
104
|
+
SW Clients have no settings of their own: they run whatever DUT image a job brings (#[code --docker-image]).
|
|
105
|
+
Admins can also edit an HW Client's devices from its resource card (#[strong ST-Link | UART | DUT USB]
|
|
106
|
+
tabs). #[strong Connected USB devices → Import to config] fills them from the host's #[code lsusb].
|
|
107
|
+
|
|
108
|
+
h3.h6 Several DUT slots on one machine
|
|
109
|
+
+code('One instance per slot').
|
|
110
|
+
thub-client register --name dut1 --type hw # creates ~/.config/thub/dut1.json, starts thub-client@dut1
|
|
111
|
+
nano ~/.config/thub/dut1.json # set that slot's hw-devices
|
|
112
|
+
sudo systemctl restart thub-client@dut1
|
|
113
|
+
thub-client deregister --name dut1 # when the slot goes away
|
|
114
|
+
|
|
115
|
+
h3.h6 Controlling a Client
|
|
116
|
+
+code('As the Client\'s user (not with sudo)').
|
|
117
|
+
thub-client status
|
|
118
|
+
thub-client lock --reason "debugging I2C" # BUSY (local): nothing gets scheduled here
|
|
119
|
+
thub-client unlock
|
|
120
|
+
thub-client restart
|
|
121
|
+
thub-client --config ~/.config/thub/dut1.json status # a specific instance
|
|
122
|
+
thub-client udev --print # the udev rules this config generates
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
+section('coordinator', 'Coordinator setup', 'hdd-network')
|
|
2
|
+
p.
|
|
3
|
+
One Coordinator serves the whole network. It's a single Node.js process with SQLite, which is
|
|
4
|
+
enough for tens of Clients and hundreds of jobs a day. Run it on a small Linux VM (Ubuntu recommended)
|
|
5
|
+
with Node.js 20+ (24 LTS recommended), reachable over HTTPS by the Agents and Clients.
|
|
6
|
+
|
|
7
|
+
h3.h6 1. Install as a systemd service
|
|
8
|
+
+code('Coordinator host').
|
|
9
|
+
sudo apt install -y nodejs npm
|
|
10
|
+
sudo npm i -g @andrian.yablonskyy/thub-coordinator
|
|
11
|
+
p.small.
|
|
12
|
+
#[code sudo npm i -g] does the whole setup for the user who ran #[code sudo]. It creates
|
|
13
|
+
#[code ~/.config/thub/coordinator.json] (with a random #[code sessionSecret]) and
|
|
14
|
+
#[code ~/var/lib/thub] (database, avatars). It installs, enables and starts
|
|
15
|
+
#[code thub-coordinator.service], plus the root helper behind the navbar's #[strong Update app] button.
|
|
16
|
+
|
|
17
|
+
h3.h6 2. Configure
|
|
18
|
+
p.small.
|
|
19
|
+
Edit #[code ~/.config/thub/coordinator.json], then #[code sudo systemctl restart thub-coordinator].
|
|
20
|
+
Logs: #[code journalctl -u thub-coordinator -f].
|
|
21
|
+
+code('~/.config/thub/coordinator.json').
|
|
22
|
+
{
|
|
23
|
+
"listen": "127.0.0.1:8080",
|
|
24
|
+
"publicUrl": "#{coordinatorUrl}",
|
|
25
|
+
"trustProxy": "loopback",
|
|
26
|
+
"dataDir": "/home/thub/var/lib/thub",
|
|
27
|
+
"sessionSecret": "<long random string>",
|
|
28
|
+
"session": { "secureCookie": "auto" },
|
|
29
|
+
"clientJoinKey": "<output of: thub-admin join-key generate>",
|
|
30
|
+
"heartbeat": { "intervalSec": 10, "missedLimit": 3, "sweepIntervalSec": 5 },
|
|
31
|
+
"scheduler": { "assignAckTimeoutSec": 15, "requeueOnLost": true, "maxQueuedPerAgent": 20, "tickIntervalSec": 10 },
|
|
32
|
+
"jobs": { "defaultTimeoutSec": 1800, "maxTimeoutSec": 14400 },
|
|
33
|
+
"retention": { "jobRetention": "forever" },
|
|
34
|
+
"updates": { "checkIntervalMin": 15, "registry": null }
|
|
35
|
+
}
|
|
36
|
+
.table-responsive
|
|
37
|
+
table.table.table-sm.small
|
|
38
|
+
thead
|
|
39
|
+
tr
|
|
40
|
+
th Field
|
|
41
|
+
th Meaning
|
|
42
|
+
tbody
|
|
43
|
+
tr
|
|
44
|
+
td: code listen
|
|
45
|
+
td Address the Node.js process binds to. Keep it on #[code 127.0.0.1] behind a reverse proxy.
|
|
46
|
+
tr
|
|
47
|
+
td: code publicUrl
|
|
48
|
+
td The URL Agents, Clients and job links use, e.g. #[code https://thub.example.com].
|
|
49
|
+
tr
|
|
50
|
+
td: code trustProxy
|
|
51
|
+
td Which proxies' #[code X-Forwarded-For] to trust for a Client's external address: #[code loopback] (default), a CIDR such as #[code "10.0.0.0/8"], a hop count, or #[code false].
|
|
52
|
+
tr
|
|
53
|
+
td: code sessionSecret
|
|
54
|
+
td Signs dashboard session cookies. Keep it secret and never use the #[code change-me-…] placeholder.
|
|
55
|
+
tr
|
|
56
|
+
td: code session.secureCookie
|
|
57
|
+
td #[code "auto"] (default): the session cookie is Secure (HTTPS-only) whenever #[code publicUrl] is https://. #[code true]/#[code false] force it. Sessions are stored in the database and survive restarts.
|
|
58
|
+
tr
|
|
59
|
+
td: code clientJoinKey
|
|
60
|
+
td The shared secret Clients self-register with. Unset or #[code null] turns registration off (#[code 503]).
|
|
61
|
+
tr
|
|
62
|
+
td: code heartbeat.*
|
|
63
|
+
td Clients heartbeat every #[code intervalSec]. After #[code missedLimit] misses a resource goes #[code OUT_OF_SERVICE] and its job becomes #[code LOST].
|
|
64
|
+
tr
|
|
65
|
+
td: code scheduler.*
|
|
66
|
+
td #[code assignAckTimeoutSec]: how long an assignment waits for the Client to accept. #[code requeueOnLost]: retry a #[code LOST] job once. #[code maxQueuedPerAgent]: queue limit per agent.
|
|
67
|
+
tr
|
|
68
|
+
td: code jobs.*
|
|
69
|
+
td Default and maximum job timeout, in seconds.
|
|
70
|
+
tr
|
|
71
|
+
td: code retention.*
|
|
72
|
+
td #[code jobRetention]: #[code 1w], #[code 2w], #[code 1m], #[code 3m], #[code 6m] or #[code forever]. Older finished jobs and their logs are deleted hourly.
|
|
73
|
+
tr
|
|
74
|
+
td: code updates.*
|
|
75
|
+
td How often to check npm for new versions (#[code 0] turns the periodic check off), and an optional private registry.
|
|
76
|
+
p.small.
|
|
77
|
+
Resolution order: #[code THUB_COORDINATOR_CONFIG] (a path) → #[code ~/.config/thub/coordinator.json] → the bundled
|
|
78
|
+
default. These environment variables override the file: #[code THUB_LISTEN], #[code THUB_PUBLIC_URL],
|
|
79
|
+
#[code THUB_DATA_DIR], #[code THUB_SESSION_SECRET] and #[code THUB_CLIENT_JOIN_KEY].
|
|
80
|
+
|
|
81
|
+
h3.h6 3. HTTPS reverse proxy
|
|
82
|
+
p.small.
|
|
83
|
+
Expose only port 443. Log streams use Server-Sent Events, so the proxy must not buffer responses.
|
|
84
|
+
.row.g-3
|
|
85
|
+
.col-md-6
|
|
86
|
+
+code('Caddyfile').
|
|
87
|
+
thub.example.com {
|
|
88
|
+
reverse_proxy 127.0.0.1:8080 {
|
|
89
|
+
flush_interval -1
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
.col-md-6
|
|
93
|
+
+code('nginx').
|
|
94
|
+
location / {
|
|
95
|
+
proxy_pass http://127.0.0.1:8080;
|
|
96
|
+
proxy_set_header Host $host;
|
|
97
|
+
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
|
98
|
+
proxy_set_header X-Forwarded-Proto $scheme;
|
|
99
|
+
proxy_http_version 1.1;
|
|
100
|
+
proxy_buffering off; # SSE log streams
|
|
101
|
+
proxy_read_timeout 1h;
|
|
102
|
+
client_max_body_size 10m; # log batches, avatars
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
h3.h6 4. Users, agents and the join key
|
|
106
|
+
p.small.
|
|
107
|
+
#[code thub-admin] works directly on the database. Run it on the Coordinator host as the service user.
|
|
108
|
+
+code('Coordinator host').
|
|
109
|
+
thub-admin create-admin alice 's3cret' --role admin # or --role viewer (read-only dashboard)
|
|
110
|
+
thub-admin join-key generate # → clientJoinKey here, joinKey on every Client
|
|
111
|
+
thub-admin agent add ci-firmware --kind ci # CI token: job ids A-00001…
|
|
112
|
+
thub-admin agent add alice-laptop --kind cli # developer token: job ids M-00001…
|
|
113
|
+
thub-admin group add ci-nightly --comment "nightly pool" # → a group id for thub run --group
|
|
114
|
+
thub-admin resource maintenance <resourceId> --on
|
|
115
|
+
thub-admin jobs reset --yes # cancel every active job
|
|
116
|
+
thub-admin jobs clean --yes --before "01/09/2026 00:00:00"
|
|
117
|
+
+note('warning').
|
|
118
|
+
Forgot the admin password? Start the Coordinator once with #[code THUB_BOOTSTRAP_ADMIN_PASSWORD=…]
|
|
119
|
+
(and optionally #[code THUB_BOOTSTRAP_ADMIN_USER], default #[code admin]). The password is reset on
|
|
120
|
+
#[em every] start while the variable is set, so unset it once you're signed in again.
|
|
121
|
+
|
|
122
|
+
h3.h6 Running from a checkout / DEV mode
|
|
123
|
+
+code('Repository checkout').
|
|
124
|
+
git clone --recurse-submodules git@github.com:andrianyablonskyy/thub.git && cd thub && npm install
|
|
125
|
+
THUB_BOOTSTRAP_ADMIN_PASSWORD=admin ./bin/coordinator
|
|
126
|
+
# Dashboard without hardware: virtual agent, two virtual Clients and demo jobs
|
|
127
|
+
DEV_MODE=1 THUB_BOOTSTRAP_ADMIN_PASSWORD=admin npm run coordinator
|
|
128
|
+
p.small.text-body-secondary.
|
|
129
|
+
In DEV mode the agent token is #[code agt_dev-virtual-agent-token], which is public. Never set
|
|
130
|
+
#[code DEV_MODE] on a real Coordinator.
|
|
131
|
+
|
|
132
|
+
h3.h6 Updates
|
|
133
|
+
p.small.mb-0.
|
|
134
|
+
The Coordinator checks npm for new Coordinator, Agent and Client versions. Admins update the Coordinator
|
|
135
|
+
with #[strong Update app] in the navbar, and Agents and Clients from the #[a(href="/admin/agents") Agents]
|
|
136
|
+
and #[a(href="/resources") Resources] pages. Clients install an update only between jobs.
|
|
137
|
+
Manual alternatives: #[code thub-admin self-update], #[code thub self-update] and #[code thub-client self-update].
|