@aion0/forge 0.10.23 → 0.10.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/RELEASE_NOTES.md +4 -19
- package/components/SettingsModal.tsx +0 -6
- package/docs/tp-automation-api-v2.md +482 -0
- package/lib/pipeline.ts +2 -2
- package/lib/settings.ts +17 -6
- package/lib/task-manager.ts +4 -4
- package/lib/telegram-bot.ts +3 -3
- package/lib/watch/watch-runner.ts +20 -2
- package/package.json +1 -1
package/RELEASE_NOTES.md
CHANGED
|
@@ -1,26 +1,11 @@
|
|
|
1
|
-
# Forge v0.10.
|
|
1
|
+
# Forge v0.10.25
|
|
2
2
|
|
|
3
3
|
Released: 2026-06-01
|
|
4
4
|
|
|
5
|
-
## Changes since v0.10.
|
|
6
|
-
|
|
7
|
-
### Features
|
|
8
|
-
- feat: long-task watch — lightweight async background tool-call primitive for chat
|
|
9
|
-
|
|
10
|
-
### Documentation
|
|
11
|
-
- docs: mark watch backend implemented on feat/watch
|
|
5
|
+
## Changes since v0.10.24
|
|
12
6
|
|
|
13
7
|
### Other
|
|
14
|
-
-
|
|
15
|
-
- feat(dashboard): expandable Alerts/notifications — click a row to see full title+body (was truncated, no way to read in full); auto-marks read on expand
|
|
16
|
-
- docs(help): add 24-watch.md (background watches: async block + start_watch, where to see/cancel) + index/routing
|
|
17
|
-
- feat(watch): start_watch builtin — LLM-driven dynamic watch (poll/done/interval), coexists with declarative async blocks
|
|
18
|
-
- feat(chat): show per-message timestamp in /chat (time-of-day, or date+time if older)
|
|
19
|
-
- docs(watch): document terminal watch_status → immediate chip removal (prune is fallback)
|
|
20
|
-
- fix(watch): emit terminal watch_status on finish so the progress chip clears immediately (was lingering until 150s prune)
|
|
21
|
-
- feat(watch): Background Watches section in MonitorPanel (list + cancel/delete)
|
|
22
|
-
- fix(watch): pre-resolve {args.*}/{result.*}/{settings.*} in watch messages at register time
|
|
23
|
-
- feat(watch): /chat UI — ambient progress chip (watch_status) + Background watches panel (list/cancel/delete); help doc for async block
|
|
8
|
+
- refactor(settings): drop unused taskModel / pipelineModel / telegramModel
|
|
24
9
|
|
|
25
10
|
|
|
26
|
-
**Full Changelog**: https://github.com/aiwatching/forge/compare/v0.10.
|
|
11
|
+
**Full Changelog**: https://github.com/aiwatching/forge/compare/v0.10.24...v0.10.25
|
|
@@ -200,9 +200,6 @@ interface Settings {
|
|
|
200
200
|
notifyOnFailure: boolean;
|
|
201
201
|
tunnelAutoStart: boolean;
|
|
202
202
|
telegramTunnelPassword: string;
|
|
203
|
-
taskModel: string;
|
|
204
|
-
pipelineModel: string;
|
|
205
|
-
telegramModel: string;
|
|
206
203
|
skipPermissions: boolean;
|
|
207
204
|
notificationRetentionDays: number;
|
|
208
205
|
maxConcurrentPipelines: number;
|
|
@@ -228,9 +225,6 @@ export default function SettingsModal({ onClose }: { onClose: () => void }) {
|
|
|
228
225
|
notifyOnFailure: true,
|
|
229
226
|
tunnelAutoStart: false,
|
|
230
227
|
telegramTunnelPassword: '',
|
|
231
|
-
taskModel: 'sonnet',
|
|
232
|
-
pipelineModel: 'sonnet',
|
|
233
|
-
telegramModel: 'sonnet',
|
|
234
228
|
skipPermissions: false,
|
|
235
229
|
notificationRetentionDays: 30,
|
|
236
230
|
maxConcurrentPipelines: 5,
|
|
@@ -0,0 +1,482 @@
|
|
|
1
|
+
# TP Automation API — 3-Step Workflow for Dev/QA
|
|
2
|
+
|
|
3
|
+
Minimum HTTP contract for driving a `build → upgrade → run tests →
|
|
4
|
+
collect results` cycle against an automation testbed on TP. All
|
|
5
|
+
endpoints are stateless except for the durable `PytestExecution` /
|
|
6
|
+
`SingleDevice` rows the celery workers update — callers just POST then
|
|
7
|
+
poll, no websockets/sse/callbacks.
|
|
8
|
+
|
|
9
|
+
```
|
|
10
|
+
<TP-base-url>/<endpoint>
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
`<TP-base-url>`:
|
|
14
|
+
- Production: `https://nac-tp.fortinet-us.com`
|
|
15
|
+
- Test: `http://10.15.33.25:8000`
|
|
16
|
+
- Dev (.11): `http://10.15.33.11:8000`
|
|
17
|
+
|
|
18
|
+
## Authentication
|
|
19
|
+
|
|
20
|
+
Every endpoint requires a JWT in the `Authorization` header:
|
|
21
|
+
|
|
22
|
+
```
|
|
23
|
+
Authorization: JWT <token>
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
Mint a token:
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
T=$(curl -s -X POST <TP-base-url>/token-auth/ \
|
|
30
|
+
-H 'Content-Type: application/json' \
|
|
31
|
+
--data-binary @- <<JSON | jq -r .token
|
|
32
|
+
{"username":"<user>","password":"<pw>"}
|
|
33
|
+
JSON
|
|
34
|
+
)
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
In examples below `$T` stands for the JWT.
|
|
38
|
+
|
|
39
|
+
> **Note on prod:** `/token-auth/` is **not** exposed at the public
|
|
40
|
+
> reverse proxy on prod (SSO-only). Automation scripts on prod-facing
|
|
41
|
+
> endpoints need to be run from a host that can reach the internal
|
|
42
|
+
> port, or they need an SSO-issued token. On the dev `.11` server,
|
|
43
|
+
> `/token-auth/` works directly with username + password.
|
|
44
|
+
|
|
45
|
+
---
|
|
46
|
+
|
|
47
|
+
## Step 1 — Upgrade the testbed
|
|
48
|
+
|
|
49
|
+
```
|
|
50
|
+
POST /adc/automation/upgrade/ — kick off (HTTP 202)
|
|
51
|
+
GET /adc/automation/upgrade/<testbed>/ — poll status
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
The dev team has three modes available; pick whichever matches your
|
|
55
|
+
build pipeline:
|
|
56
|
+
|
|
57
|
+
| `mode` | When to use it | Required extra fields |
|
|
58
|
+
|---|---|---|
|
|
59
|
+
| `command` | You have a Jenkins-built image and Jenkins already produces the full `execute restore image scp ...` command string. | `command` |
|
|
60
|
+
| `build` | You have a build number from the official build server. | `build_number` |
|
|
61
|
+
| `ga` | You want the latest GA build for a specific FortiNAC version. The system resolves the build number via `BuildHistory.GA=True`. | `version` |
|
|
62
|
+
|
|
63
|
+
### Request
|
|
64
|
+
|
|
65
|
+
Content-Type: `application/json`.
|
|
66
|
+
|
|
67
|
+
```json
|
|
68
|
+
{
|
|
69
|
+
"testbed": "AT16_Combined_FSW",
|
|
70
|
+
"mode": "command",
|
|
71
|
+
"command": "execute restore image scp /var/lib/jenkins/jobs/fortinac-build-7.6-1/builds/3983/archive/nacos/FNAC_ESX-v7-build6956-FORTINET_job1_3983.out 10.15.33.5 jenkins fortinet"
|
|
72
|
+
}
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
```json
|
|
76
|
+
{
|
|
77
|
+
"testbed": "AT16_Combined_FSW",
|
|
78
|
+
"mode": "build",
|
|
79
|
+
"build_number": "0815"
|
|
80
|
+
}
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
```json
|
|
84
|
+
{
|
|
85
|
+
"testbed": "AT16_Combined_FSW",
|
|
86
|
+
"mode": "ga",
|
|
87
|
+
"version": "7.6.5"
|
|
88
|
+
}
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
Internal behavior:
|
|
92
|
+
|
|
93
|
+
1. Looks up the testbed's `deployinfo`, extracts every device whose
|
|
94
|
+
name contains `nac` or `ncm`. A multi-NAC testbed gets one celery
|
|
95
|
+
task per device, running in parallel.
|
|
96
|
+
2. `mode=ga` resolves `version` → `build_number` via `BuildHistory`,
|
|
97
|
+
then dispatches the same path as `mode=build`.
|
|
98
|
+
3. Concurrency guard: HTTP **409** if any target IP is already
|
|
99
|
+
`PROGRESS`.
|
|
100
|
+
4. `mode=command` validates the command starts with
|
|
101
|
+
`execute restore image` (security guard — keeps arbitrary CLI from
|
|
102
|
+
riding through this endpoint).
|
|
103
|
+
|
|
104
|
+
### Response (HTTP 202)
|
|
105
|
+
|
|
106
|
+
```json
|
|
107
|
+
{
|
|
108
|
+
"testbed": "AT16_Combined_FSW",
|
|
109
|
+
"mode": "command",
|
|
110
|
+
"build_number": "",
|
|
111
|
+
"target_ips": ["10.15.52.152"],
|
|
112
|
+
"tasks": [
|
|
113
|
+
{"ip": "10.15.52.152", "celery_id": "f62a4cb7-d435-41de-adb6-12cca405d507"}
|
|
114
|
+
]
|
|
115
|
+
}
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
Returns in <500 ms. The actual upgrade work runs asynchronously in
|
|
119
|
+
celery workers — the HTTP call returns the moment the task is queued.
|
|
120
|
+
|
|
121
|
+
### Poll for status
|
|
122
|
+
|
|
123
|
+
```
|
|
124
|
+
GET /adc/automation/upgrade/<testbed>/
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
```json
|
|
128
|
+
{
|
|
129
|
+
"testbed": "AT16_Combined_FSW",
|
|
130
|
+
"status": "PROGRESS",
|
|
131
|
+
"target_ips": ["10.15.52.152"],
|
|
132
|
+
"per_device": [
|
|
133
|
+
{
|
|
134
|
+
"ip": "10.15.52.152",
|
|
135
|
+
"status": "PROGRESS",
|
|
136
|
+
"last_task_type": "command",
|
|
137
|
+
"last_build_number": null,
|
|
138
|
+
"updated_at": "2026-05-28T18:14:19+00:00",
|
|
139
|
+
"log_tail": "...last 2 KB of the NAC's SSH output..."
|
|
140
|
+
}
|
|
141
|
+
]
|
|
142
|
+
}
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
Aggregate `status` values:
|
|
146
|
+
|
|
147
|
+
| Value | Meaning |
|
|
148
|
+
|---|---|
|
|
149
|
+
| `UNKNOWN` | No upgrade has been dispatched against this testbed (yet). |
|
|
150
|
+
| `PROGRESS` | At least one device is still mid-upgrade. |
|
|
151
|
+
| `SUCCESS` | Every device finished with `SUCCESS`. |
|
|
152
|
+
| `FAILURE` | At least one device finished with `FAILURE` / `TIMEOUT` (and no device is still in flight). |
|
|
153
|
+
|
|
154
|
+
`per_device[].log_tail` is the last ~2 KB of the device's SSH session
|
|
155
|
+
output, updated every ~3 s by the worker. Useful for live debugging
|
|
156
|
+
without flooding the response.
|
|
157
|
+
|
|
158
|
+
### Concrete polling loop
|
|
159
|
+
|
|
160
|
+
```bash
|
|
161
|
+
while :; do
|
|
162
|
+
R=$(curl -sH "Authorization: JWT $T" $TP/adc/automation/upgrade/AT16_Combined_FSW/)
|
|
163
|
+
STATUS=$(echo "$R" | jq -r .status)
|
|
164
|
+
echo "$(date +%H:%M:%S) $STATUS"
|
|
165
|
+
case "$STATUS" in
|
|
166
|
+
PROGRESS) sleep 15 ;;
|
|
167
|
+
SUCCESS) echo "upgrade complete"; break ;;
|
|
168
|
+
FAILURE) echo "upgrade FAILED"; echo "$R" | jq .per_device; exit 1 ;;
|
|
169
|
+
*) echo "$R"; exit 1 ;;
|
|
170
|
+
esac
|
|
171
|
+
done
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
A real upgrade takes 2-15 minutes depending on the image and how many
|
|
175
|
+
devices the testbed has.
|
|
176
|
+
|
|
177
|
+
---
|
|
178
|
+
|
|
179
|
+
## Step 2 — Run pytest cases on that testbed
|
|
180
|
+
|
|
181
|
+
```
|
|
182
|
+
POST /adc/automation/pytest/ — kick off (HTTP 202)
|
|
183
|
+
GET /adc/automation/pytest/<exec_id>/ — poll status + results
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
### Request
|
|
187
|
+
|
|
188
|
+
Content-Type: `application/json`.
|
|
189
|
+
|
|
190
|
+
```json
|
|
191
|
+
{
|
|
192
|
+
"user": "<TP username>",
|
|
193
|
+
"lab": "AT16_Combined_FSW",
|
|
194
|
+
"testcase": [
|
|
195
|
+
"/Tests_CLI/test_cli_sanity.py::TestGetBasic::test_get_system_status"
|
|
196
|
+
],
|
|
197
|
+
"argument": "-vv --tb=short",
|
|
198
|
+
"extra_pytest_options": ""
|
|
199
|
+
}
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
| Field | Type | Notes |
|
|
203
|
+
|---|---|---|
|
|
204
|
+
| `user` | string | TP username of the caller. Must own the lab (be in `users` or `usedby` on the `AutomationTBUser` row). |
|
|
205
|
+
| `lab` | string | AT lab name. Same `<testbed>` value as Step 1. |
|
|
206
|
+
| `testcase` | **list of strings** | Each entry is a pytest test-id path **relative to the tests repo root** (`/root/fnac_auto/tests` on the controller). Leading slash is OK; both `/Tests_CLI/...` and `Tests_CLI/...` work. The handler iterates the list, joins with spaces. |
|
|
207
|
+
| `argument` | string | Raw pytest CLI args injected between the testcase paths and the framework's `--html`/`--rack-file` flags. `-k`, `-m`, `-vv`, `--tb=short`, `--maxfail=N`, etc. |
|
|
208
|
+
| `extra_pytest_options` | string | (optional) Extra options appended *after* `--html`/`--rack-file` on the real run. Use for env-specific flags that shouldn't apply to the `--collect-only` dry run. |
|
|
209
|
+
|
|
210
|
+
### Response (HTTP 202)
|
|
211
|
+
|
|
212
|
+
```json
|
|
213
|
+
{
|
|
214
|
+
"exec_id": 446,
|
|
215
|
+
"status": "Initiating",
|
|
216
|
+
"lab": "AT16_Combined_FSW",
|
|
217
|
+
"testcase_count": 1
|
|
218
|
+
}
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
Returns in ~300 ms. Save `exec_id` — it's the handle for Step 3.
|
|
222
|
+
|
|
223
|
+
### What runs on the controller
|
|
224
|
+
|
|
225
|
+
The celery worker SFTP-uploads this script to the controller VM and
|
|
226
|
+
launches it under `nohup setsid`:
|
|
227
|
+
|
|
228
|
+
```bash
|
|
229
|
+
#!/bin/bash
|
|
230
|
+
set +e
|
|
231
|
+
cd /root/fnac_auto/tests && git pull origin main || true
|
|
232
|
+
cd /root/fnac_auto/test-framework && git pull origin main || true
|
|
233
|
+
cd /root/fnac_auto
|
|
234
|
+
source venv/bin/activate
|
|
235
|
+
export PYTHONPATH=/root/fnac_auto/test-framework:/root/fnac_auto/tests
|
|
236
|
+
export DISPLAY=:99
|
|
237
|
+
pytest <expanded testcase paths> <argument> --collect-only
|
|
238
|
+
pytest <expanded testcase paths> <argument> --html=<report> --rack-file <rack> <extra_pytest_options>
|
|
239
|
+
echo $? > /tmp/pytest_<exec_id>.exit
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
Key properties:
|
|
243
|
+
|
|
244
|
+
- **Pytest is fully detached** from the launching ssh session via
|
|
245
|
+
`nohup setsid`. The ssh session can drop, the network can blip, the
|
|
246
|
+
TP backend can restart — pytest keeps running on the controller.
|
|
247
|
+
- **TP polls via short, separate ssh round-trips** to read the log
|
|
248
|
+
file and check for the terminal `/tmp/pytest_<exec_id>.exit` marker.
|
|
249
|
+
- **Surviving a TP backend restart** is automatic — the celery worker
|
|
250
|
+
is a separate process and just keeps polling. Verified end-to-end on
|
|
251
|
+
`.11` (exec_id 447: killed runserver mid-run, exec completed
|
|
252
|
+
normally and report was fetched back).
|
|
253
|
+
|
|
254
|
+
### Concurrency guard
|
|
255
|
+
|
|
256
|
+
If another `PytestExecution` on the same lab has `status` =
|
|
257
|
+
`Running` or `Initiating`, the POST returns HTTP **409**:
|
|
258
|
+
|
|
259
|
+
```json
|
|
260
|
+
{"error": "another execution is already in flight on 'AT16_Combined_FSW'"}
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
---
|
|
264
|
+
|
|
265
|
+
## Step 3 — Read per-test results
|
|
266
|
+
|
|
267
|
+
```
|
|
268
|
+
GET /adc/automation/pytest/<exec_id>/
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
Same endpoint as the Step-2 poll — there's nothing distinct to call.
|
|
272
|
+
|
|
273
|
+
### Response
|
|
274
|
+
|
|
275
|
+
```json
|
|
276
|
+
{
|
|
277
|
+
"exec_id": 446,
|
|
278
|
+
"status": "Done",
|
|
279
|
+
"lab": "AT16_Combined_FSW",
|
|
280
|
+
"controller": "10.15.52.159",
|
|
281
|
+
"remote_pid": "177854",
|
|
282
|
+
"pass_count": 1,
|
|
283
|
+
"fail_count": 0,
|
|
284
|
+
"skip_count": 0,
|
|
285
|
+
"error_count": 0,
|
|
286
|
+
"total_count": 1,
|
|
287
|
+
"start_timestamp": "1780090011",
|
|
288
|
+
"end_timestamp": "1780090041",
|
|
289
|
+
"report_file_path": "data/AutomationTest/test_446_report_2026-05-29_21:26:50.html",
|
|
290
|
+
"log_file_path": "/tmp/pytest_446.log",
|
|
291
|
+
"log_size": 25338,
|
|
292
|
+
"log_tail": "...last 4 KB of pytest stdout..."
|
|
293
|
+
}
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
| Field | Meaning |
|
|
297
|
+
|---|---|
|
|
298
|
+
| `status` | One of `Initiating`, `Running`, `Done`, `Failed`, `Cancelled`. **Treat `Done` as terminal — success/failure is inferred from counts.** |
|
|
299
|
+
| `pass_count`, `fail_count`, `skip_count`, `error_count`, `total_count` | Parsed from pytest's own summary line. Populated **mid-run** (updated every 15 s), so you can see partial progress without waiting for the run to finish. |
|
|
300
|
+
| `start_timestamp` / `end_timestamp` | Unix epoch seconds, written by the worker. |
|
|
301
|
+
| `report_file_path` | Server-relative path to the pytest-html report. Fetch via `<TP>/<report_file_path>` with the JWT for per-test rows + tracebacks. |
|
|
302
|
+
| `log_file_path` | On-controller path to the live log. Mostly informational — use `log_tail` to read the last 4 KB without an extra round-trip. |
|
|
303
|
+
| `log_tail` | Last 4 KB of pytest stdout. |
|
|
304
|
+
| `remote_pid` | The detached pytest tree's pid on the controller. Useful if you ever need to ssh to the controller and kill it manually. |
|
|
305
|
+
|
|
306
|
+
### Concrete polling loop
|
|
307
|
+
|
|
308
|
+
```bash
|
|
309
|
+
EXEC=$(curl -sX POST -H "Authorization: JWT $T" \
|
|
310
|
+
-H 'Content-Type: application/json' \
|
|
311
|
+
-d '{"user":"alice","lab":"AT16_Combined_FSW","testcase":["/Tests_CLI/test_cli_sanity.py::TestGetBasic::test_get_system_status"],"argument":"-vv"}' \
|
|
312
|
+
$TP/adc/automation/pytest/ | jq -r .exec_id)
|
|
313
|
+
echo "started exec $EXEC"
|
|
314
|
+
|
|
315
|
+
while :; do
|
|
316
|
+
R=$(curl -sH "Authorization: JWT $T" $TP/adc/automation/pytest/$EXEC/)
|
|
317
|
+
STATUS=$(echo "$R" | jq -r .status)
|
|
318
|
+
COUNTS=$(echo "$R" | jq -r '"P\(.pass_count)/F\(.fail_count)/S\(.skip_count)/E\(.error_count) of T\(.total_count)"')
|
|
319
|
+
echo "$(date +%H:%M:%S) $STATUS $COUNTS"
|
|
320
|
+
case "$STATUS" in
|
|
321
|
+
Initiating|Running) sleep 15 ;;
|
|
322
|
+
Done)
|
|
323
|
+
FAIL=$(echo "$R" | jq -r .fail_count)
|
|
324
|
+
ERR=$(echo "$R" | jq -r .error_count)
|
|
325
|
+
if [ "$FAIL" -gt 0 ] || [ "$ERR" -gt 0 ]; then
|
|
326
|
+
echo "tests had failures — fetch report at $TP/$(echo "$R" | jq -r .report_file_path)"
|
|
327
|
+
exit 1
|
|
328
|
+
fi
|
|
329
|
+
echo "all pass"; break ;;
|
|
330
|
+
Failed|Cancelled)
|
|
331
|
+
echo "execution terminated: $STATUS"
|
|
332
|
+
echo "$R" | jq -r .log_tail; exit 1 ;;
|
|
333
|
+
esac
|
|
334
|
+
done
|
|
335
|
+
```
|
|
336
|
+
|
|
337
|
+
---
|
|
338
|
+
|
|
339
|
+
## End-to-end script
|
|
340
|
+
|
|
341
|
+
```bash
|
|
342
|
+
#!/usr/bin/env bash
|
|
343
|
+
set -euo pipefail
|
|
344
|
+
TP=http://10.15.33.11:8000
|
|
345
|
+
USER=admin
|
|
346
|
+
TESTBED=AT16_Combined_FSW
|
|
347
|
+
|
|
348
|
+
T=$(curl -s -X POST $TP/token-auth/ -H 'Content-Type: application/json' \
|
|
349
|
+
--data-binary @- <<JSON | jq -r .token
|
|
350
|
+
{"username":"$USER","password":"<your-pw>"}
|
|
351
|
+
JSON
|
|
352
|
+
)
|
|
353
|
+
|
|
354
|
+
# ---- Step 1: upgrade testbed ----
|
|
355
|
+
COMMAND='execute restore image scp /var/lib/jenkins/jobs/fortinac-build-7.6-1/builds/3983/archive/nacos/FNAC_ESX-v7-build6956-FORTINET_job1_3983.out 10.15.33.5 jenkins fortinet'
|
|
356
|
+
|
|
357
|
+
curl -s -X POST -H "Authorization: JWT $T" \
|
|
358
|
+
-H 'Content-Type: application/json' \
|
|
359
|
+
-d "$(jq -n --arg cmd "$COMMAND" --arg tb "$TESTBED" \
|
|
360
|
+
'{testbed:$tb, mode:"command", command:$cmd}')" \
|
|
361
|
+
$TP/adc/automation/upgrade/ > /dev/null
|
|
362
|
+
|
|
363
|
+
while :; do
|
|
364
|
+
R=$(curl -sH "Authorization: JWT $T" $TP/adc/automation/upgrade/$TESTBED/)
|
|
365
|
+
S=$(echo "$R" | jq -r .status)
|
|
366
|
+
echo "upgrade: $S"
|
|
367
|
+
[ "$S" = "SUCCESS" ] && break
|
|
368
|
+
[ "$S" = "FAILURE" ] && { echo "$R" | jq .per_device; exit 1; }
|
|
369
|
+
sleep 30
|
|
370
|
+
done
|
|
371
|
+
|
|
372
|
+
# ---- Step 2 + 3: run tests, collect results ----
|
|
373
|
+
EXEC=$(curl -s -X POST -H "Authorization: JWT $T" \
|
|
374
|
+
-H 'Content-Type: application/json' \
|
|
375
|
+
-d "$(jq -n --arg tb "$TESTBED" --arg user "$USER" \
|
|
376
|
+
'{user:$user, lab:$tb,
|
|
377
|
+
testcase:["/Tests_CLI/test_cli_sanity.py::TestGetBasic::test_get_system_status"],
|
|
378
|
+
argument:"-vv --tb=short"}')" \
|
|
379
|
+
$TP/adc/automation/pytest/ | jq -r .exec_id)
|
|
380
|
+
echo "started exec $EXEC"
|
|
381
|
+
|
|
382
|
+
while :; do
|
|
383
|
+
R=$(curl -sH "Authorization: JWT $T" $TP/adc/automation/pytest/$EXEC/)
|
|
384
|
+
S=$(echo "$R" | jq -r .status)
|
|
385
|
+
echo "pytest: $S $(echo "$R" | jq -r '"P\(.pass_count)/F\(.fail_count) of T\(.total_count)"')"
|
|
386
|
+
case "$S" in
|
|
387
|
+
Done)
|
|
388
|
+
F=$(echo "$R" | jq -r .fail_count)
|
|
389
|
+
E=$(echo "$R" | jq -r .error_count)
|
|
390
|
+
[ "$F" -gt 0 ] || [ "$E" -gt 0 ] && {
|
|
391
|
+
echo "report: $TP/$(echo "$R" | jq -r .report_file_path)"
|
|
392
|
+
exit 1
|
|
393
|
+
}
|
|
394
|
+
echo "PASS"; break ;;
|
|
395
|
+
Failed|Cancelled)
|
|
396
|
+
echo "exec terminated: $S"; echo "$R" | jq -r .log_tail; exit 1 ;;
|
|
397
|
+
*) sleep 15 ;;
|
|
398
|
+
esac
|
|
399
|
+
done
|
|
400
|
+
```
|
|
401
|
+
|
|
402
|
+
---
|
|
403
|
+
|
|
404
|
+
## Behaviors worth knowing
|
|
405
|
+
|
|
406
|
+
### Concurrency
|
|
407
|
+
|
|
408
|
+
For each `<testbed>`:
|
|
409
|
+
- **Upgrade**: HTTP 409 if any target IP currently has
|
|
410
|
+
`SingleDevice.last_task_status='PROGRESS'`.
|
|
411
|
+
- **Pytest**: HTTP 409 if any `PytestExecution` with this `lab` has
|
|
412
|
+
`status` in `{Running, Initiating}`.
|
|
413
|
+
|
|
414
|
+
You can have an upgrade AND a pytest running simultaneously on
|
|
415
|
+
*different* testbeds. The guards are per-testbed.
|
|
416
|
+
|
|
417
|
+
### Survival semantics
|
|
418
|
+
|
|
419
|
+
| Failure | Upgrade | Pytest v2 |
|
|
420
|
+
|---|---|---|
|
|
421
|
+
| ssh session drops mid-run | N/A — TP only ssh's briefly | Pytest keeps running (`nohup setsid` on controller) |
|
|
422
|
+
| TP backend restart mid-run | Celery worker keeps going, completes the run, writes terminal state | Same — celery `pytest_poll` keeps short-polling the controller |
|
|
423
|
+
| Controller VM reboot mid-run | Upgrade reboots the NAC; this is expected/desired. | Pytest is lost (controller died); polling will eventually fail with the max-runtime safety net (6 h cap). |
|
|
424
|
+
| Celery worker restart | In-flight task may be lost (celery uses ack_late=false by default). Worth a follow-up. | Same. |
|
|
425
|
+
|
|
426
|
+
### Per-test pass/fail visibility
|
|
427
|
+
|
|
428
|
+
The `passed_nodes` / `failed_nodes` fields on the legacy `pytest_run`
|
|
429
|
+
response shape don't apply to v2. v2 surfaces:
|
|
430
|
+
|
|
431
|
+
- **aggregate counts** (`pass_count`, `fail_count`, etc.) in the GET
|
|
432
|
+
response — populated mid-run as pytest emits them
|
|
433
|
+
- **the HTML report** at `report_file_path` — the standard
|
|
434
|
+
pytest-html document with per-test rows, durations, tracebacks, and
|
|
435
|
+
captured logs. Fetch via `<TP>/<report_file_path>` with the JWT.
|
|
436
|
+
|
|
437
|
+
If you need per-test JSON instead of the HTML report, ask — it's a
|
|
438
|
+
trivial extra endpoint that parses the HTML and returns
|
|
439
|
+
`{passed: [...], failed: [...]}`.
|
|
440
|
+
|
|
441
|
+
### Test-case path format
|
|
442
|
+
|
|
443
|
+
Whatever you'd type after `pytest` on the command line, prefixed (or
|
|
444
|
+
not) with `/`. Relative to `/root/fnac_auto/tests/` on the controller.
|
|
445
|
+
|
|
446
|
+
| Want to | Use |
|
|
447
|
+
|---|---|
|
|
448
|
+
| Run one test | `["/Tests_CLI/test_cli_sanity.py::TestGetBasic::test_get_system_status"]` |
|
|
449
|
+
| Run several | `["/Tests_CLI/test_cli_sanity.py::TestGetBasic::test_get_system_status", "/Tests_CLI/test_cli_sanity.py::TestShowFullConfig::test_nacos_show_config"]` |
|
|
450
|
+
| Run a whole file | `["/Tests_CLI/test_cli_sanity.py"]` |
|
|
451
|
+
| Run by marker | `["/Tests_CLI"]` + `argument: "-m smoke"` |
|
|
452
|
+
| Dry-run / list | `["/Tests_CLI"]` + `argument: "--collect-only"` |
|
|
453
|
+
|
|
454
|
+
Tip: discover paths via `GET /adc/get_testcases` (legacy endpoint;
|
|
455
|
+
returns the full test tree). Note: this endpoint does a `git pull` on
|
|
456
|
+
TP's local mirror of the tests repo as a side effect — fine in normal
|
|
457
|
+
use, just be aware.
|
|
458
|
+
|
|
459
|
+
---
|
|
460
|
+
|
|
461
|
+
## Source files
|
|
462
|
+
|
|
463
|
+
| Concern | File |
|
|
464
|
+
|---|---|
|
|
465
|
+
| URL routes | `backend/adc/urls.py` |
|
|
466
|
+
| Upgrade API | `backend/adc/views/automation/upgrade_api.py` |
|
|
467
|
+
| Upgrade tasks | `backend/adc/tasks.py` (`upgrade_nac_command`, `upgrade_nac_build`) |
|
|
468
|
+
| Pytest v2 API | `backend/adc/views/automation/pytest_api.py` |
|
|
469
|
+
| Pytest v2 tasks | `backend/adc/tasks.py` (`pytest_launch`, `pytest_poll`) |
|
|
470
|
+
| Per-device state | `backend/adc/models.py` (`SingleDevice`, `PytestExecution`) |
|
|
471
|
+
| GA-build resolution | `backend/adc/models.py` (`BuildHistory.GA=True`) |
|
|
472
|
+
|
|
473
|
+
Legacy endpoints (still live, used by the `/automation` UI page, kept
|
|
474
|
+
for backward compatibility):
|
|
475
|
+
|
|
476
|
+
- `POST /adc/nac-upgrade-testbed/`
|
|
477
|
+
- `POST /adc/pytest_run`
|
|
478
|
+
- `POST /adc/get_test_execution_by_id`
|
|
479
|
+
|
|
480
|
+
Avoid these for new automation — they're synchronous-in-the-request,
|
|
481
|
+
don't survive TP restarts, and the upgrade variant blocks the HTTP
|
|
482
|
+
request for the full restore duration.
|
package/lib/pipeline.ts
CHANGED
|
@@ -1930,8 +1930,8 @@ async function scheduleReadyNodes(pipeline: Pipeline, workflow: Workflow) {
|
|
|
1930
1930
|
if (skillsAppend) {
|
|
1931
1931
|
taskAppendSystemPromptOverrides.set(task.id, skillsAppend);
|
|
1932
1932
|
}
|
|
1933
|
-
// Pipeline tasks use the same model selection as normal tasks
|
|
1934
|
-
//
|
|
1933
|
+
// Pipeline tasks use the same model selection as normal tasks
|
|
1934
|
+
// (per-task override > agent scene model > agent default).
|
|
1935
1935
|
|
|
1936
1936
|
nodeState.status = 'running';
|
|
1937
1937
|
nodeState.taskId = task.id;
|
package/lib/settings.ts
CHANGED
|
@@ -94,9 +94,6 @@ export interface Settings {
|
|
|
94
94
|
notifyOnFailure: boolean;
|
|
95
95
|
tunnelAutoStart: boolean;
|
|
96
96
|
telegramTunnelPassword: string;
|
|
97
|
-
taskModel: string;
|
|
98
|
-
pipelineModel: string;
|
|
99
|
-
telegramModel: string;
|
|
100
97
|
skipPermissions: boolean;
|
|
101
98
|
manageClaudeConfig: boolean;
|
|
102
99
|
notificationRetentionDays: number;
|
|
@@ -206,9 +203,6 @@ const defaults: Settings = {
|
|
|
206
203
|
notifyOnFailure: true,
|
|
207
204
|
tunnelAutoStart: false,
|
|
208
205
|
telegramTunnelPassword: '',
|
|
209
|
-
taskModel: 'default',
|
|
210
|
-
pipelineModel: 'default',
|
|
211
|
-
telegramModel: 'sonnet',
|
|
212
206
|
skipPermissions: false,
|
|
213
207
|
manageClaudeConfig: true,
|
|
214
208
|
notificationRetentionDays: 30,
|
|
@@ -310,6 +304,23 @@ export function loadSettings(): Settings {
|
|
|
310
304
|
);
|
|
311
305
|
}
|
|
312
306
|
}
|
|
307
|
+
// taskModel / pipelineModel / telegramModel were removed: no UI ever
|
|
308
|
+
// exposed them, and the resolution chain now goes per-task override
|
|
309
|
+
// → agent scene model → agent default. Drop leftover keys; warn when
|
|
310
|
+
// they held a non-default value so the user knows their override
|
|
311
|
+
// stopped applying.
|
|
312
|
+
for (const k of ['taskModel', 'pipelineModel', 'telegramModel'] as const) {
|
|
313
|
+
if (k in parsed) {
|
|
314
|
+
const old = parsed[k];
|
|
315
|
+
delete parsed[k];
|
|
316
|
+
if (old && old !== '' && old !== 'default' && old !== 'sonnet') {
|
|
317
|
+
console.warn(
|
|
318
|
+
`[settings] ${k}="${old}" is no longer read. ` +
|
|
319
|
+
`Set the model on the relevant agent (settings.agents.<id>.models.task) instead.`,
|
|
320
|
+
);
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
}
|
|
313
324
|
// Decrypt top-level secret fields
|
|
314
325
|
for (const field of SECRET_FIELDS) {
|
|
315
326
|
if (parsed[field] && isEncrypted(parsed[field])) {
|
package/lib/task-manager.ts
CHANGED
|
@@ -54,7 +54,7 @@ function db() {
|
|
|
54
54
|
return getDb(getDbPath());
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
-
// Per-task model overrides
|
|
57
|
+
// Per-task model overrides — top of the model-resolution chain.
|
|
58
58
|
export const taskModelOverrides = new Map<string, string>();
|
|
59
59
|
|
|
60
60
|
/**
|
|
@@ -514,12 +514,12 @@ function executeTask(task: Task): Promise<void> {
|
|
|
514
514
|
const agentId = (task as any).agent || settings.defaultAgent || 'claude';
|
|
515
515
|
const adapter = getAgent(agentId);
|
|
516
516
|
|
|
517
|
-
// Model priority: per-task override > agent scene model >
|
|
518
|
-
// "default" means "no override"
|
|
517
|
+
// Model priority: per-task override > agent scene model > 'default'
|
|
518
|
+
// (agent picks its own default). "default" means "no override".
|
|
519
519
|
const agentCfg = settings.agents?.[agentId];
|
|
520
520
|
const agentModel = agentCfg?.models?.task;
|
|
521
521
|
const effectiveAgentModel = agentModel && agentModel !== 'default' ? agentModel : null;
|
|
522
|
-
const model = taskModelOverrides.get(task.id) || effectiveAgentModel ||
|
|
522
|
+
const model = taskModelOverrides.get(task.id) || effectiveAgentModel || 'default';
|
|
523
523
|
const supportsModel = adapter.config.capabilities?.supportsModel;
|
|
524
524
|
const spawnOpts = adapter.buildTaskSpawn({
|
|
525
525
|
projectPath: task.projectPath,
|
package/lib/telegram-bot.ts
CHANGED
|
@@ -736,7 +736,7 @@ async function handlePeek(chatId: number, projectArg?: string, sessionArg?: stri
|
|
|
736
736
|
contextLen += line.length;
|
|
737
737
|
}
|
|
738
738
|
|
|
739
|
-
const telegramModel =
|
|
739
|
+
const telegramModel = 'sonnet';
|
|
740
740
|
const summary = contextEntries.length > 3
|
|
741
741
|
? await aiSummarize(contextEntries.join('\n'), 'Summarize this Claude Code session in 2-3 sentences. What was the user working on? What is the current status? Answer in the same language as the content.')
|
|
742
742
|
: '';
|
|
@@ -1353,7 +1353,7 @@ async function aiSummarize(content: string, instruction: string): Promise<string
|
|
|
1353
1353
|
try {
|
|
1354
1354
|
const settings = loadSettings();
|
|
1355
1355
|
const claudePath = settings.claudePath || process.env.CLAUDE_PATH || 'claude';
|
|
1356
|
-
const model =
|
|
1356
|
+
const model = 'sonnet';
|
|
1357
1357
|
const { execSync } = require('child_process');
|
|
1358
1358
|
const { realpathSync } = require('fs');
|
|
1359
1359
|
|
|
@@ -1509,7 +1509,7 @@ async function handleDocs(chatId: number, input: string) {
|
|
|
1509
1509
|
} catch {}
|
|
1510
1510
|
|
|
1511
1511
|
const recent = entries.slice(-8).join('\n\n');
|
|
1512
|
-
const tModel =
|
|
1512
|
+
const tModel = 'sonnet';
|
|
1513
1513
|
const summary = entries.length > 3
|
|
1514
1514
|
? await aiSummarize(entries.slice(-15).join('\n'), 'Summarize this Claude Code session in 2-3 sentences. What was the user working on? What is the current status? Answer in the same language as the content.')
|
|
1515
1515
|
: '';
|
|
@@ -89,7 +89,11 @@ export function startWatchRunner(hooks: WatchRunnerHooks = {}): void {
|
|
|
89
89
|
}
|
|
90
90
|
let res;
|
|
91
91
|
try {
|
|
92
|
-
|
|
92
|
+
// noTruncation=true so http-protocol polls skip the "HTTP 200 OK · GET …"
|
|
93
|
+
// preamble — the body comes back as raw JSON so done_path/done_match can
|
|
94
|
+
// see it. (Without this, every http-protocol watch would silently never
|
|
95
|
+
// hit its done condition, e.g. jenkins.get_build never resolving.)
|
|
96
|
+
res = await dispatchTool({ id: `watch-${w.id}-${w.polls}`, name: `${w.connector_id}.${w.poll_tool}`, input: w.poll_args }, { noTruncation: true } as any);
|
|
93
97
|
} catch (e) {
|
|
94
98
|
res = { content: String(e), is_error: true };
|
|
95
99
|
}
|
|
@@ -129,7 +133,21 @@ export function startWatchRunner(hooks: WatchRunnerHooks = {}): void {
|
|
|
129
133
|
if (polls >= w.max_polls || now - w.created_at > w.timeout_sec * 1000) {
|
|
130
134
|
return finish(w, 'timed_out', obj, `${w.label}: not done within ${w.max_polls} polls / ${w.timeout_sec}s — please verify manually.`);
|
|
131
135
|
}
|
|
132
|
-
|
|
136
|
+
// Persist the latest poll result + a tiny preview text on EVERY poll
|
|
137
|
+
// (not just terminal) so the Monitor / DB shows what the watch is
|
|
138
|
+
// actually seeing — crucial for diagnosing "polling forever, no done":
|
|
139
|
+
// usually means the done condition refers to a field the poll's
|
|
140
|
+
// result doesn't actually have.
|
|
141
|
+
const previewKeys = obj && typeof obj === 'object' && !Array.isArray(obj)
|
|
142
|
+
? Object.keys(obj as Record<string, unknown>).slice(0, 8).join(', ')
|
|
143
|
+
: typeof obj;
|
|
144
|
+
updateWatch(w.id, {
|
|
145
|
+
polls,
|
|
146
|
+
err_count: 0,
|
|
147
|
+
next_poll_at: now + w.interval_sec * 1000,
|
|
148
|
+
last_result: obj,
|
|
149
|
+
last_text: `poll ${polls}/${w.max_polls} · keys: ${previewKeys}`,
|
|
150
|
+
}, now);
|
|
133
151
|
emitProgress(w, obj, polls);
|
|
134
152
|
};
|
|
135
153
|
|
package/package.json
CHANGED