@amsterdamdatalabs/adl-agent-control-darwin-arm64 0.1.69 → 0.1.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/adl +2 -2
- package/bin/job-watchdog +38 -10
- package/bin/setup +15 -0
- package/bin/validate-manifest +13 -3
- package/package.json +1 -1
- package/packages/fleet/manifest.json +19 -5
- package/packages/fleet/manifest.schema.json +26 -5
package/bin/adl
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
[diffend] Oversized file quarantined before diffing.
|
|
2
2
|
name: package/bin/adl
|
|
3
|
-
size:
|
|
4
|
-
sha256:
|
|
3
|
+
size: 63753136 bytes
|
|
4
|
+
sha256: d68024582f65fde1c78a67d63997aecccd11f7c2a71499dd0e98eb6cd1253951
|
package/bin/job-watchdog
CHANGED
|
@@ -55,27 +55,54 @@ if [ "$os" = Linux ] && [ -f "$systemd_dir/$pool_unit" ]; then
|
|
|
55
55
|
fi
|
|
56
56
|
fi
|
|
57
57
|
|
|
58
|
-
# services.pool_reaper (mac):
|
|
58
|
+
# services.pool_reaper (mac): re-arm the Hetzner dead-man switch when it is down or its last run failed; see docs/operations/agent-pool.md.
|
|
59
59
|
reaper_label=com.adl.pool-reaper
|
|
60
|
-
|
|
60
|
+
reaper_plist="$root/Library/LaunchDaemons/$reaper_label.plist"
|
|
61
|
+
reaper_failed=0
|
|
62
|
+
reaper_error() { log "ERROR watchdog: $*; run: sudo adl fleet setup NAME --rearm"; reaper_failed=1; }
|
|
63
|
+
rearm_reaper() {
|
|
64
|
+
log "WARN watchdog: re-arming $reaper_label: $1"
|
|
65
|
+
if [ "$2" = loaded ]; then
|
|
66
|
+
# Re-check right before bootout: a run may have started since the first look.
|
|
67
|
+
reaper_recheck="$(launchctl print "system/$reaper_label" 2>&1)" || reaper_recheck=""
|
|
68
|
+
if [ "$(printf '%s\n' "$reaper_recheck" | sed -n 's/^[[:space:]]*state = //p' | head -n 1)" = running ]; then
|
|
69
|
+
log "WARN watchdog: $reaper_label started running meanwhile; not booting it out"
|
|
70
|
+
return
|
|
71
|
+
fi
|
|
72
|
+
launchctl bootout "system/$reaper_label" || { reaper_error "launchctl bootout system/$reaper_label failed"; return; }
|
|
73
|
+
reaper_gone=0
|
|
74
|
+
for _ in 1 2 3 4 5 6 7 8 9 10; do
|
|
75
|
+
if ! launchctl print "system/$reaper_label" >/dev/null 2>&1; then reaper_gone=1; break; fi
|
|
76
|
+
sleep 0.5
|
|
77
|
+
done
|
|
78
|
+
[ "$reaper_gone" -eq 1 ] || { reaper_error "$reaper_label is still loaded after bootout (waited 5s); not bootstrapping"; return; }
|
|
79
|
+
fi
|
|
80
|
+
launchctl enable "system/$reaper_label" || { reaper_error "launchctl enable system/$reaper_label failed"; return; }
|
|
81
|
+
launchctl bootstrap system "$reaper_plist" || { reaper_error "launchctl bootstrap system $reaper_plist failed"; return; }
|
|
82
|
+
log "WARN watchdog: re-armed $reaper_label"
|
|
83
|
+
}
|
|
84
|
+
if [ "$os" = Darwin ] && [ -f "$reaper_plist" ]; then
|
|
61
85
|
if reaper_printed="$(launchctl print "system/$reaper_label" 2>&1)"; then
|
|
86
|
+
reaper_state="$(printf '%s\n' "$reaper_printed" | sed -n 's/^[[:space:]]*state = //p' | head -n 1)"
|
|
62
87
|
reaper_exit="$(printf '%s\n' "$reaper_printed" | sed -n 's/^[[:space:]]*last exit code = \([0-9][0-9]*\).*/\1/p' | head -n 1)"
|
|
63
|
-
if [ -n "$reaper_exit" ] && [ "$reaper_exit" != 0 ]; then
|
|
64
|
-
|
|
88
|
+
if [ "$reaper_state" != running ] && [ -n "$reaper_exit" ] && [ "$reaper_exit" != 0 ]; then
|
|
89
|
+
rearm_reaper "its last run failed (exit $reaper_exit); see /var/log/adl/pool-reaper.log" loaded
|
|
65
90
|
else
|
|
66
91
|
log "INFO watchdog: $reaper_label is loaded"
|
|
67
92
|
fi
|
|
68
93
|
else
|
|
69
|
-
|
|
94
|
+
rearm_reaper "it is not loaded; the Hetzner dead-man switch is off" missing
|
|
70
95
|
fi
|
|
71
96
|
fi
|
|
97
|
+
# Every exit below goes through finish, so a failed reaper re-arm still exits 1.
|
|
98
|
+
finish() { exit "$reaper_failed"; }
|
|
72
99
|
|
|
73
100
|
case "$os" in
|
|
74
101
|
Darwin)
|
|
75
102
|
[ -f "$plist" ] || fail "$plist is missing; run: sudo adl fleet setup NAME --rearm"
|
|
76
103
|
if printed="$(launchctl print "system/$label" 2>&1)"; then
|
|
77
104
|
job_state="$(printf '%s\n' "$printed" | sed -n 's/^[[:space:]]*state = //p' | head -n 1)"
|
|
78
|
-
[ "$job_state" = running ] &&
|
|
105
|
+
[ "$job_state" = running ] && finish
|
|
79
106
|
exit_code="$(printf '%s\n' "$printed" | sed -n 's/^[[:space:]]*last exit code = \([0-9][0-9]*\).*/\1/p' | head -n 1)"
|
|
80
107
|
if [ "$exit_code" = 78 ]; then cause="launchd could not spawn the program (last exit code 78 EX_CONFIG)"; fi
|
|
81
108
|
# Incident state as launchd reports it (m3, 2026-10-24): packages/fleet/README.md#self-healing-the-watchdog
|
|
@@ -87,7 +114,7 @@ case "$os" in
|
|
|
87
114
|
Linux)
|
|
88
115
|
[ -f "$systemd_dir/$service_unit" ] && [ -f "$systemd_dir/$timer_unit" ] || fail "$service_unit / $timer_unit are missing in $systemd_dir; run: sudo adl fleet setup NAME --rearm"
|
|
89
116
|
service_state="$(systemctl is-active "$service_unit" 2>/dev/null)"
|
|
90
|
-
case "$service_state" in active|activating)
|
|
117
|
+
case "$service_state" in active|activating) finish ;; esac
|
|
91
118
|
timer_state="$(systemctl is-active "$timer_unit" 2>/dev/null)"
|
|
92
119
|
timer_enabled="$(systemctl is-enabled "$timer_unit" 2>/dev/null)"
|
|
93
120
|
if [ "$timer_state" != active ]; then cause="$timer_unit is $timer_state"
|
|
@@ -99,13 +126,13 @@ esac
|
|
|
99
126
|
|
|
100
127
|
stale_only=0
|
|
101
128
|
if [ -z "$cause" ]; then
|
|
102
|
-
if fresh "$last_success" "$max_age_minutes"; then
|
|
129
|
+
if fresh "$last_success" "$max_age_minutes"; then finish; fi
|
|
103
130
|
cause="no successful reconcile in the last 25 hours ($last_success)"
|
|
104
131
|
stale_only=1
|
|
105
132
|
fi
|
|
106
133
|
|
|
107
134
|
# Stale last-success alone re-arms at most once per 6 hours: packages/fleet/README.md#self-healing-the-watchdog
|
|
108
|
-
if [ "$stale_only" -eq 1 ] && fresh "$kick_stamp" "$kick_gap_minutes"; then
|
|
135
|
+
if [ "$stale_only" -eq 1 ] && fresh "$kick_stamp" "$kick_gap_minutes"; then finish; fi
|
|
109
136
|
|
|
110
137
|
log "WARN watchdog: re-arming the scheduled job: $cause"
|
|
111
138
|
|
|
@@ -116,7 +143,7 @@ case "$os" in
|
|
|
116
143
|
recheck_state="$(printf '%s\n' "$rechecked" | sed -n 's/^[[:space:]]*state = //p' | head -n 1)"
|
|
117
144
|
if [ "$recheck_state" = running ]; then
|
|
118
145
|
log "WARN watchdog: $label started running meanwhile; not booting it out"
|
|
119
|
-
|
|
146
|
+
finish
|
|
120
147
|
fi
|
|
121
148
|
launchctl bootout "system/$label" || fail "launchctl bootout system/$label failed"
|
|
122
149
|
gone=0
|
|
@@ -144,3 +171,4 @@ case "$os" in
|
|
|
144
171
|
Linux) systemctl start --no-block "$service_unit" || fail "systemctl start $service_unit failed" ;;
|
|
145
172
|
esac
|
|
146
173
|
log "WARN watchdog: re-armed and started $label"
|
|
174
|
+
finish
|
package/bin/setup
CHANGED
|
@@ -404,6 +404,21 @@ EOF
|
|
|
404
404
|
systemctl enable --now adl-global-tools-watchdog.timer
|
|
405
405
|
systemctl restart adl-global-tools-watchdog.timer
|
|
406
406
|
|
|
407
|
+
# Azure Pipelines agents (svc.sh units vsts.agent.*.service) restart after any exit, an OOM kill included.
|
|
408
|
+
# Named zzz- so it sorts after the hand-installed zz-ci-retention.conf (Restart=on-success): the last Restart= wins.
|
|
409
|
+
# Takes effect at daemon-reload; a running agent is not restarted, so no job is cut short.
|
|
410
|
+
agent_drop_ins=0
|
|
411
|
+
for agent_unit in "$systemd_dir"/vsts.agent.*.service; do
|
|
412
|
+
[[ -f "$agent_unit" ]] || continue
|
|
413
|
+
mkdir -p "$agent_unit.d"
|
|
414
|
+
printf '%s\n' '# Written by adl fleet setup: restart the agent after any exit, an OOM kill included.' \
|
|
415
|
+
'[Service]' 'Restart=always' 'RestartSec=10' >"$agent_unit.d/zzz-adl-restart.conf"
|
|
416
|
+
chmod 0644 "$agent_unit.d/zzz-adl-restart.conf"
|
|
417
|
+
echo "adl fleet setup: agent restart drop-in: ${agent_unit##*/}"
|
|
418
|
+
agent_drop_ins=1
|
|
419
|
+
done
|
|
420
|
+
if [[ "$agent_drop_ins" -eq 1 ]]; then systemctl daemon-reload; fi
|
|
421
|
+
|
|
407
422
|
# Long-running services the manifest declares for this host (Feature 758):
|
|
408
423
|
# services.agent_pool => adl-agent-pool.service runs `adl pool run --host NAME`
|
|
409
424
|
# always on. Everything the poller needs is read from the manifest by the
|
package/bin/validate-manifest
CHANGED
|
@@ -20,14 +20,24 @@ if ! jq -e '
|
|
|
20
20
|
def posint: type == "number" and . >= 1 and floor == .;
|
|
21
21
|
def nonneg: type == "number" and . >= 0 and floor == .;
|
|
22
22
|
def nes: type == "string" and length > 0;
|
|
23
|
+
def guid: type == "string" and test("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$");
|
|
24
|
+
def aca_pool: type == "object" and
|
|
25
|
+
((keys - ["capabilities"]) | sort) == ["cpu", "disk", "disk_image_id", "memory", "region", "resource_group", "sandbox_group", "sp_client_id", "sp_tenant_id", "subscription_id"] and
|
|
26
|
+
(.subscription_id | guid) and (.resource_group | nes) and (.sandbox_group | nes) and (.region | nes) and
|
|
27
|
+
(.disk_image_id | nes) and (.cpu | nes) and (.memory | nes) and (.disk | nes) and
|
|
28
|
+
(.sp_client_id | . == "UNSET" or guid) and (.sp_tenant_id | guid) and
|
|
29
|
+
((has("capabilities") | not) or (.capabilities | type == "object" and all(.[]; type == "string")));
|
|
23
30
|
def agent_pool: type == "object" and
|
|
24
|
-
(keys | sort) == ["agents_per_server", "capabilities", "hour_end_window_minutes", "identity", "location", "max_lifetime_minutes", "max_servers", "org", "poll_seconds", "project", "queue_id", "
|
|
31
|
+
((keys - ["aca"]) | sort) == ["agents_per_server", "backend", "capabilities", "hour_end_window_minutes", "identity", "location", "max_lifetime_minutes", "max_servers", "org", "parallel_jobs", "poll_seconds", "project", "queue_id", "queue_wait_minutes", "register_timeout_minutes", "server_type", "snapshot_label_selector", "sp_client_id", "sp_tenant_id"] and
|
|
32
|
+
(.backend | . == "hetzner" or . == "aca-sandbox") and
|
|
33
|
+
((.backend != "aca-sandbox") or (has("aca") and (.aca | has("capabilities")))) and
|
|
34
|
+
((has("aca") | not) or (.aca | aca_pool)) and
|
|
25
35
|
(.org | nes) and (.project | nes) and (.queue_id | posint) and
|
|
26
36
|
(.identity | . == "agent-tarun" or . == "agent-vivek" or . == "agent-reviewer") and
|
|
27
37
|
(.server_type | nes) and (.location | nes) and (.snapshot_label_selector | nes) and
|
|
28
38
|
(.agents_per_server | posint) and (.max_servers | posint) and (.max_lifetime_minutes | posint) and
|
|
29
|
-
(.register_timeout_minutes | posint) and (.poll_seconds | posint) and (.
|
|
30
|
-
(.
|
|
39
|
+
(.register_timeout_minutes | posint) and (.poll_seconds | posint) and (.queue_wait_minutes | nonneg) and
|
|
40
|
+
(.parallel_jobs | posint) and (.hour_end_window_minutes | nonneg and . <= 59) and
|
|
31
41
|
(.capabilities | type == "object" and all(.[]; type == "string")) and
|
|
32
42
|
(.sp_client_id | type == "string" and (. == "UNSET" or test("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"))) and
|
|
33
43
|
(.sp_tenant_id | type == "string" and test("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"));
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@amsterdamdatalabs/adl-agent-control-darwin-arm64",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.71",
|
|
4
4
|
"description": "adl CLI binary for darwin/arm64 (agent identity + Azure DevOps front door — see docs/architecture/cli-design.md)",
|
|
5
5
|
"license": "UNLICENSED",
|
|
6
6
|
"os": [
|
|
@@ -218,20 +218,34 @@
|
|
|
218
218
|
"project": "Enact-Now-AI-AzDo",
|
|
219
219
|
"queue_id": 80,
|
|
220
220
|
"identity": "agent-tarun",
|
|
221
|
+
"backend": "aca-sandbox",
|
|
221
222
|
"server_type": "cpx52",
|
|
222
223
|
"location": "fsn1",
|
|
223
224
|
"snapshot_label_selector": "adl-pool-image=enact",
|
|
224
|
-
"agents_per_server":
|
|
225
|
-
"max_servers":
|
|
225
|
+
"agents_per_server": 1,
|
|
226
|
+
"max_servers": 2,
|
|
226
227
|
"max_lifetime_minutes": 360,
|
|
227
228
|
"register_timeout_minutes": 15,
|
|
228
229
|
"poll_seconds": 60,
|
|
229
|
-
"
|
|
230
|
-
"
|
|
230
|
+
"queue_wait_minutes": 5,
|
|
231
|
+
"parallel_jobs": 6,
|
|
231
232
|
"hour_end_window_minutes": 10,
|
|
232
233
|
"capabilities": {"Agent.OS": "Linux", "docker": "present"},
|
|
233
234
|
"sp_client_id": "f34d1cd9-4fa1-47bb-9e82-ad9f5cb78a11",
|
|
234
|
-
"sp_tenant_id": "49d4be4b-06f2-42b7-91bc-a17ccde6f0ce"
|
|
235
|
+
"sp_tenant_id": "49d4be4b-06f2-42b7-91bc-a17ccde6f0ce",
|
|
236
|
+
"aca": {
|
|
237
|
+
"subscription_id": "3ca1de39-2e87-4ba3-8a47-b3ecc4775697",
|
|
238
|
+
"resource_group": "enact-neu-d-rg",
|
|
239
|
+
"sandbox_group": "enact-neu-d-space-sandboxes",
|
|
240
|
+
"region": "northeurope",
|
|
241
|
+
"disk_image_id": "b9a40fe6-791b-400e-bdcf-c8145514b7e5",
|
|
242
|
+
"cpu": "2",
|
|
243
|
+
"memory": "4Gi",
|
|
244
|
+
"disk": "40Gi",
|
|
245
|
+
"sp_client_id": "f34d1cd9-4fa1-47bb-9e82-ad9f5cb78a11",
|
|
246
|
+
"sp_tenant_id": "49d4be4b-06f2-42b7-91bc-a17ccde6f0ce",
|
|
247
|
+
"capabilities": {"Agent.OS": "Linux"}
|
|
248
|
+
}
|
|
235
249
|
}
|
|
236
250
|
},
|
|
237
251
|
"surface_clis": [
|
|
@@ -298,14 +298,16 @@
|
|
|
298
298
|
"properties": {
|
|
299
299
|
"agent_pool": {
|
|
300
300
|
"type": "object",
|
|
301
|
-
"description": "Feature 758: `adl pool run --host NAME`, the elastic
|
|
301
|
+
"description": "Feature 758: `adl pool run --host NAME`, the elastic Azure Pipelines agent poller (Hetzner servers or Azure Container Apps Sandboxes, per `backend`). Maps 1:1 onto PoolConfig (packages/agent-pool/src/types.ts) in snake_case, plus `backend`, `aca`, `project` (the Azure DevOps project of the queue), `sp_client_id` and `sp_tenant_id` (the service principal agents register with).",
|
|
302
302
|
"additionalProperties": false,
|
|
303
|
-
"
|
|
303
|
+
"allOf": [{"if": {"properties": {"backend": {"const": "aca-sandbox"}}}, "then": {"required": ["aca"], "properties": {"aca": {"required": ["capabilities"]}}}}],
|
|
304
|
+
"required": ["org", "project", "queue_id", "identity", "backend", "server_type", "location", "snapshot_label_selector", "agents_per_server", "max_servers", "max_lifetime_minutes", "register_timeout_minutes", "poll_seconds", "queue_wait_minutes", "parallel_jobs", "hour_end_window_minutes", "capabilities", "sp_client_id", "sp_tenant_id"],
|
|
304
305
|
"properties": {
|
|
305
306
|
"org": {"type": "string", "minLength": 1},
|
|
306
307
|
"project": {"type": "string", "minLength": 1},
|
|
307
308
|
"queue_id": {"type": "integer", "minimum": 1},
|
|
308
309
|
"identity": {"enum": ["agent-tarun", "agent-vivek", "agent-reviewer"]},
|
|
310
|
+
"backend": {"description": "Where pool machines run: Hetzner servers or ephemeral Azure Container Apps Sandboxes (one sandbox = one agent = one job; needs the `aca` block).", "enum": ["hetzner", "aca-sandbox"]},
|
|
309
311
|
"server_type": {"type": "string", "minLength": 1},
|
|
310
312
|
"location": {"type": "string", "minLength": 1},
|
|
311
313
|
"snapshot_label_selector": {"type": "string", "minLength": 1},
|
|
@@ -314,12 +316,31 @@
|
|
|
314
316
|
"max_lifetime_minutes": {"type": "integer", "minimum": 1},
|
|
315
317
|
"register_timeout_minutes": {"type": "integer", "minimum": 1},
|
|
316
318
|
"poll_seconds": {"type": "integer", "minimum": 1},
|
|
317
|
-
"
|
|
318
|
-
"
|
|
319
|
+
"queue_wait_minutes": {"description": "A pending job counts as demand only after it has queued this many minutes.", "type": "integer", "minimum": 0},
|
|
320
|
+
"parallel_jobs": {"description": "Azure DevOps runs at most this many self-hosted jobs at once across all pools; the pool never creates past the free slots.", "type": "integer", "minimum": 1},
|
|
319
321
|
"hour_end_window_minutes": {"type": "integer", "minimum": 0, "maximum": 59},
|
|
320
322
|
"capabilities": {"type": "object", "additionalProperties": {"type": "string"}},
|
|
321
323
|
"sp_client_id": {"description": "Application (client) id of the identity's Entra service principal, used by `config.sh --auth SP`; the literal UNSET until Tarun fills it (`pool run` refuses while UNSET).", "type": "string", "pattern": "^(UNSET|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$"},
|
|
322
|
-
"sp_tenant_id": {"description": "Entra tenant id of that service principal (AGENT_TENANTS in apps/cli/src/ado-token.ts).", "type": "string", "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"}
|
|
324
|
+
"sp_tenant_id": {"description": "Entra tenant id of that service principal (AGENT_TENANTS in apps/cli/src/ado-token.ts).", "type": "string", "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"},
|
|
325
|
+
"aca": {
|
|
326
|
+
"type": "object",
|
|
327
|
+
"description": "Azure Container Apps Sandboxes backend settings; required when `backend` is `aca-sandbox` (checked by bin/validate-manifest and `adl pool`). `disk_image_id` and `sp_client_id` are the literal UNSET until Tarun fills them (`pool run` refuses while UNSET).",
|
|
328
|
+
"additionalProperties": false,
|
|
329
|
+
"required": ["subscription_id", "resource_group", "sandbox_group", "region", "disk_image_id", "cpu", "memory", "disk", "sp_client_id", "sp_tenant_id"],
|
|
330
|
+
"properties": {
|
|
331
|
+
"subscription_id": {"type": "string", "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"},
|
|
332
|
+
"resource_group": {"type": "string", "minLength": 1},
|
|
333
|
+
"sandbox_group": {"type": "string", "minLength": 1},
|
|
334
|
+
"region": {"type": "string", "minLength": 1},
|
|
335
|
+
"disk_image_id": {"type": "string", "minLength": 1},
|
|
336
|
+
"cpu": {"type": "string", "minLength": 1},
|
|
337
|
+
"memory": {"type": "string", "minLength": 1},
|
|
338
|
+
"disk": {"type": "string", "minLength": 1},
|
|
339
|
+
"sp_client_id": {"description": "Application (client) id of the Entra service principal that calls the sandbox data plane (scope https://dynamicsessions.io/.default).", "type": "string", "pattern": "^(UNSET|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$"},
|
|
340
|
+
"sp_tenant_id": {"type": "string", "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"},
|
|
341
|
+
"capabilities": {"description": "Capabilities the sandbox image really has (no docker); required when `backend` is `aca-sandbox`, where it replaces the pool `capabilities` for the servable-job filter.", "type": "object", "additionalProperties": {"type": "string"}}
|
|
342
|
+
}
|
|
343
|
+
}
|
|
323
344
|
}
|
|
324
345
|
},
|
|
325
346
|
"pool_reaper": {
|