@amsterdamdatalabs/adl-agent-control-darwin-arm64 0.1.69 → 0.1.70

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/bin/adl CHANGED
@@ -1,4 +1,4 @@
1
1
  [diffend] Oversized file quarantined before diffing.
2
2
  name: package/bin/adl
3
- size: 63736720 bytes
4
- sha256: d5cf54c48c04b06dbd603d5bb03449bf705a65777fe5579abf138c041edbdfa1
3
+ size: 63753136 bytes
4
+ sha256: 03c23d250dd0c873e7e5f606966860be53152ade6c2f42bd0739e979d048ddee
package/bin/job-watchdog CHANGED
@@ -55,27 +55,54 @@ if [ "$os" = Linux ] && [ -f "$systemd_dir/$pool_unit" ]; then
55
55
  fi
56
56
  fi
57
57
 
58
- # services.pool_reaper (mac): report-only check of the Hetzner dead-man switch; see docs/operations/agent-pool.md.
58
+ # services.pool_reaper (mac): re-arm the Hetzner dead-man switch when it is down or its last run failed; see docs/operations/agent-pool.md.
59
59
  reaper_label=com.adl.pool-reaper
60
- if [ "$os" = Darwin ] && [ -f "$root/Library/LaunchDaemons/$reaper_label.plist" ]; then
60
+ reaper_plist="$root/Library/LaunchDaemons/$reaper_label.plist"
61
+ reaper_failed=0
62
+ reaper_error() { log "ERROR watchdog: $*; run: sudo adl fleet setup NAME --rearm"; reaper_failed=1; }
63
+ rearm_reaper() {
64
+ log "WARN watchdog: re-arming $reaper_label: $1"
65
+ if [ "$2" = loaded ]; then
66
+ # Re-check right before bootout: a run may have started since the first look.
67
+ reaper_recheck="$(launchctl print "system/$reaper_label" 2>&1)" || reaper_recheck=""
68
+ if [ "$(printf '%s\n' "$reaper_recheck" | sed -n 's/^[[:space:]]*state = //p' | head -n 1)" = running ]; then
69
+ log "WARN watchdog: $reaper_label started running meanwhile; not booting it out"
70
+ return
71
+ fi
72
+ launchctl bootout "system/$reaper_label" || { reaper_error "launchctl bootout system/$reaper_label failed"; return; }
73
+ reaper_gone=0
74
+ for _ in 1 2 3 4 5 6 7 8 9 10; do
75
+ if ! launchctl print "system/$reaper_label" >/dev/null 2>&1; then reaper_gone=1; break; fi
76
+ sleep 0.5
77
+ done
78
+ [ "$reaper_gone" -eq 1 ] || { reaper_error "$reaper_label is still loaded after bootout (waited 5s); not bootstrapping"; return; }
79
+ fi
80
+ launchctl enable "system/$reaper_label" || { reaper_error "launchctl enable system/$reaper_label failed"; return; }
81
+ launchctl bootstrap system "$reaper_plist" || { reaper_error "launchctl bootstrap system $reaper_plist failed"; return; }
82
+ log "WARN watchdog: re-armed $reaper_label"
83
+ }
84
+ if [ "$os" = Darwin ] && [ -f "$reaper_plist" ]; then
61
85
  if reaper_printed="$(launchctl print "system/$reaper_label" 2>&1)"; then
86
+ reaper_state="$(printf '%s\n' "$reaper_printed" | sed -n 's/^[[:space:]]*state = //p' | head -n 1)"
62
87
  reaper_exit="$(printf '%s\n' "$reaper_printed" | sed -n 's/^[[:space:]]*last exit code = \([0-9][0-9]*\).*/\1/p' | head -n 1)"
63
- if [ -n "$reaper_exit" ] && [ "$reaper_exit" != 0 ]; then
64
- log "WARN watchdog: $reaper_label last run failed (exit $reaper_exit); the Hetzner dead-man switch may not be reaping: see /var/log/adl/pool-reaper.log"
88
+ if [ "$reaper_state" != running ] && [ -n "$reaper_exit" ] && [ "$reaper_exit" != 0 ]; then
89
+ rearm_reaper "its last run failed (exit $reaper_exit); see /var/log/adl/pool-reaper.log" loaded
65
90
  else
66
91
  log "INFO watchdog: $reaper_label is loaded"
67
92
  fi
68
93
  else
69
- log "WARN watchdog: $reaper_label is not loaded; the Hetzner dead-man switch is off; run: sudo adl fleet setup NAME --rearm"
94
+ rearm_reaper "it is not loaded; the Hetzner dead-man switch is off" missing
70
95
  fi
71
96
  fi
97
+ # Every exit below goes through finish, so a failed reaper re-arm still exits 1.
98
+ finish() { exit "$reaper_failed"; }
72
99
 
73
100
  case "$os" in
74
101
  Darwin)
75
102
  [ -f "$plist" ] || fail "$plist is missing; run: sudo adl fleet setup NAME --rearm"
76
103
  if printed="$(launchctl print "system/$label" 2>&1)"; then
77
104
  job_state="$(printf '%s\n' "$printed" | sed -n 's/^[[:space:]]*state = //p' | head -n 1)"
78
- [ "$job_state" = running ] && exit 0
105
+ [ "$job_state" = running ] && finish
79
106
  exit_code="$(printf '%s\n' "$printed" | sed -n 's/^[[:space:]]*last exit code = \([0-9][0-9]*\).*/\1/p' | head -n 1)"
80
107
  if [ "$exit_code" = 78 ]; then cause="launchd could not spawn the program (last exit code 78 EX_CONFIG)"; fi
81
108
  # Incident state as launchd reports it (m3, 2026-10-24): packages/fleet/README.md#self-healing-the-watchdog
@@ -87,7 +114,7 @@ case "$os" in
87
114
  Linux)
88
115
  [ -f "$systemd_dir/$service_unit" ] && [ -f "$systemd_dir/$timer_unit" ] || fail "$service_unit / $timer_unit are missing in $systemd_dir; run: sudo adl fleet setup NAME --rearm"
89
116
  service_state="$(systemctl is-active "$service_unit" 2>/dev/null)"
90
- case "$service_state" in active|activating) exit 0 ;; esac
117
+ case "$service_state" in active|activating) finish ;; esac
91
118
  timer_state="$(systemctl is-active "$timer_unit" 2>/dev/null)"
92
119
  timer_enabled="$(systemctl is-enabled "$timer_unit" 2>/dev/null)"
93
120
  if [ "$timer_state" != active ]; then cause="$timer_unit is $timer_state"
@@ -99,13 +126,13 @@ esac
99
126
 
100
127
  stale_only=0
101
128
  if [ -z "$cause" ]; then
102
- if fresh "$last_success" "$max_age_minutes"; then exit 0; fi
129
+ if fresh "$last_success" "$max_age_minutes"; then finish; fi
103
130
  cause="no successful reconcile in the last 25 hours ($last_success)"
104
131
  stale_only=1
105
132
  fi
106
133
 
107
134
  # Stale last-success alone re-arms at most once per 6 hours: packages/fleet/README.md#self-healing-the-watchdog
108
- if [ "$stale_only" -eq 1 ] && fresh "$kick_stamp" "$kick_gap_minutes"; then exit 0; fi
135
+ if [ "$stale_only" -eq 1 ] && fresh "$kick_stamp" "$kick_gap_minutes"; then finish; fi
109
136
 
110
137
  log "WARN watchdog: re-arming the scheduled job: $cause"
111
138
 
@@ -116,7 +143,7 @@ case "$os" in
116
143
  recheck_state="$(printf '%s\n' "$rechecked" | sed -n 's/^[[:space:]]*state = //p' | head -n 1)"
117
144
  if [ "$recheck_state" = running ]; then
118
145
  log "WARN watchdog: $label started running meanwhile; not booting it out"
119
- exit 0
146
+ finish
120
147
  fi
121
148
  launchctl bootout "system/$label" || fail "launchctl bootout system/$label failed"
122
149
  gone=0
@@ -144,3 +171,4 @@ case "$os" in
144
171
  Linux) systemctl start --no-block "$service_unit" || fail "systemctl start $service_unit failed" ;;
145
172
  esac
146
173
  log "WARN watchdog: re-armed and started $label"
174
+ finish
package/bin/setup CHANGED
@@ -404,6 +404,21 @@ EOF
404
404
  systemctl enable --now adl-global-tools-watchdog.timer
405
405
  systemctl restart adl-global-tools-watchdog.timer
406
406
 
407
+ # Azure Pipelines agents (svc.sh units vsts.agent.*.service) restart after any exit, an OOM kill included.
408
+ # Named zzz- so it sorts after the hand-installed zz-ci-retention.conf (Restart=on-success): the last Restart= wins.
409
+ # Takes effect at daemon-reload; a running agent is not restarted, so no job is cut short.
410
+ agent_drop_ins=0
411
+ for agent_unit in "$systemd_dir"/vsts.agent.*.service; do
412
+ [[ -f "$agent_unit" ]] || continue
413
+ mkdir -p "$agent_unit.d"
414
+ printf '%s\n' '# Written by adl fleet setup: restart the agent after any exit, an OOM kill included.' \
415
+ '[Service]' 'Restart=always' 'RestartSec=10' >"$agent_unit.d/zzz-adl-restart.conf"
416
+ chmod 0644 "$agent_unit.d/zzz-adl-restart.conf"
417
+ echo "adl fleet setup: agent restart drop-in: ${agent_unit##*/}"
418
+ agent_drop_ins=1
419
+ done
420
+ if [[ "$agent_drop_ins" -eq 1 ]]; then systemctl daemon-reload; fi
421
+
407
422
  # Long-running services the manifest declares for this host (Feature 758):
408
423
  # services.agent_pool => adl-agent-pool.service runs `adl pool run --host NAME`
409
424
  # always on. Everything the poller needs is read from the manifest by the
@@ -20,14 +20,24 @@ if ! jq -e '
20
20
  def posint: type == "number" and . >= 1 and floor == .;
21
21
  def nonneg: type == "number" and . >= 0 and floor == .;
22
22
  def nes: type == "string" and length > 0;
23
+ def guid: type == "string" and test("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$");
24
+ def aca_pool: type == "object" and
25
+ ((keys - ["capabilities"]) | sort) == ["cpu", "disk", "disk_image_id", "memory", "region", "resource_group", "sandbox_group", "sp_client_id", "sp_tenant_id", "subscription_id"] and
26
+ (.subscription_id | guid) and (.resource_group | nes) and (.sandbox_group | nes) and (.region | nes) and
27
+ (.disk_image_id | nes) and (.cpu | nes) and (.memory | nes) and (.disk | nes) and
28
+ (.sp_client_id | . == "UNSET" or guid) and (.sp_tenant_id | guid) and
29
+ ((has("capabilities") | not) or (.capabilities | type == "object" and all(.[]; type == "string")));
23
30
  def agent_pool: type == "object" and
24
- (keys | sort) == ["agents_per_server", "capabilities", "hour_end_window_minutes", "identity", "location", "max_lifetime_minutes", "max_servers", "org", "poll_seconds", "project", "queue_id", "queue_threshold", "register_timeout_minutes", "server_type", "snapshot_label_selector", "sp_client_id", "sp_tenant_id", "trigger_ticks"] and
31
+ ((keys - ["aca"]) | sort) == ["agents_per_server", "backend", "capabilities", "hour_end_window_minutes", "identity", "location", "max_lifetime_minutes", "max_servers", "org", "parallel_jobs", "poll_seconds", "project", "queue_id", "queue_wait_minutes", "register_timeout_minutes", "server_type", "snapshot_label_selector", "sp_client_id", "sp_tenant_id"] and
32
+ (.backend | . == "hetzner" or . == "aca-sandbox") and
33
+ ((.backend != "aca-sandbox") or (has("aca") and (.aca | has("capabilities")))) and
34
+ ((has("aca") | not) or (.aca | aca_pool)) and
25
35
  (.org | nes) and (.project | nes) and (.queue_id | posint) and
26
36
  (.identity | . == "agent-tarun" or . == "agent-vivek" or . == "agent-reviewer") and
27
37
  (.server_type | nes) and (.location | nes) and (.snapshot_label_selector | nes) and
28
38
  (.agents_per_server | posint) and (.max_servers | posint) and (.max_lifetime_minutes | posint) and
29
- (.register_timeout_minutes | posint) and (.poll_seconds | posint) and (.queue_threshold | nonneg) and
30
- (.trigger_ticks | posint) and (.hour_end_window_minutes | nonneg and . <= 59) and
39
+ (.register_timeout_minutes | posint) and (.poll_seconds | posint) and (.queue_wait_minutes | nonneg) and
40
+ (.parallel_jobs | posint) and (.hour_end_window_minutes | nonneg and . <= 59) and
31
41
  (.capabilities | type == "object" and all(.[]; type == "string")) and
32
42
  (.sp_client_id | type == "string" and (. == "UNSET" or test("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"))) and
33
43
  (.sp_tenant_id | type == "string" and test("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"));
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@amsterdamdatalabs/adl-agent-control-darwin-arm64",
3
- "version": "0.1.69",
3
+ "version": "0.1.70",
4
4
  "description": "adl CLI binary for darwin/arm64 (agent identity + Azure DevOps front door — see docs/architecture/cli-design.md)",
5
5
  "license": "UNLICENSED",
6
6
  "os": [
@@ -218,6 +218,7 @@
218
218
  "project": "Enact-Now-AI-AzDo",
219
219
  "queue_id": 80,
220
220
  "identity": "agent-tarun",
221
+ "backend": "hetzner",
221
222
  "server_type": "cpx52",
222
223
  "location": "fsn1",
223
224
  "snapshot_label_selector": "adl-pool-image=enact",
@@ -226,12 +227,25 @@
226
227
  "max_lifetime_minutes": 360,
227
228
  "register_timeout_minutes": 15,
228
229
  "poll_seconds": 60,
229
- "queue_threshold": 1,
230
- "trigger_ticks": 3,
230
+ "queue_wait_minutes": 5,
231
+ "parallel_jobs": 6,
231
232
  "hour_end_window_minutes": 10,
232
233
  "capabilities": {"Agent.OS": "Linux", "docker": "present"},
233
234
  "sp_client_id": "f34d1cd9-4fa1-47bb-9e82-ad9f5cb78a11",
234
- "sp_tenant_id": "49d4be4b-06f2-42b7-91bc-a17ccde6f0ce"
235
+ "sp_tenant_id": "49d4be4b-06f2-42b7-91bc-a17ccde6f0ce",
236
+ "aca": {
237
+ "subscription_id": "3ca1de39-2e87-4ba3-8a47-b3ecc4775697",
238
+ "resource_group": "enact-neu-d-rg",
239
+ "sandbox_group": "enact-neu-d-space-sandboxes",
240
+ "region": "northeurope",
241
+ "disk_image_id": "UNSET",
242
+ "cpu": "2",
243
+ "memory": "4Gi",
244
+ "disk": "40Gi",
245
+ "sp_client_id": "f34d1cd9-4fa1-47bb-9e82-ad9f5cb78a11",
246
+ "sp_tenant_id": "49d4be4b-06f2-42b7-91bc-a17ccde6f0ce",
247
+ "capabilities": {"Agent.OS": "Linux"}
248
+ }
235
249
  }
236
250
  },
237
251
  "surface_clis": [
@@ -298,14 +298,16 @@
298
298
  "properties": {
299
299
  "agent_pool": {
300
300
  "type": "object",
301
- "description": "Feature 758: `adl pool run --host NAME`, the elastic Hetzner Azure Pipelines agent poller. Maps 1:1 onto PoolConfig (packages/agent-pool/src/types.ts) in snake_case, plus `project` (the Azure DevOps project of the queue), `sp_client_id` and `sp_tenant_id` (the service principal agents register with).",
301
+ "description": "Feature 758: `adl pool run --host NAME`, the elastic Azure Pipelines agent poller (Hetzner servers or Azure Container Apps Sandboxes, per `backend`). Maps 1:1 onto PoolConfig (packages/agent-pool/src/types.ts) in snake_case, plus `backend`, `aca`, `project` (the Azure DevOps project of the queue), `sp_client_id` and `sp_tenant_id` (the service principal agents register with).",
302
302
  "additionalProperties": false,
303
- "required": ["org", "project", "queue_id", "identity", "server_type", "location", "snapshot_label_selector", "agents_per_server", "max_servers", "max_lifetime_minutes", "register_timeout_minutes", "poll_seconds", "queue_threshold", "trigger_ticks", "hour_end_window_minutes", "capabilities", "sp_client_id", "sp_tenant_id"],
303
+ "allOf": [{"if": {"properties": {"backend": {"const": "aca-sandbox"}}}, "then": {"required": ["aca"], "properties": {"aca": {"required": ["capabilities"]}}}}],
304
+ "required": ["org", "project", "queue_id", "identity", "backend", "server_type", "location", "snapshot_label_selector", "agents_per_server", "max_servers", "max_lifetime_minutes", "register_timeout_minutes", "poll_seconds", "queue_wait_minutes", "parallel_jobs", "hour_end_window_minutes", "capabilities", "sp_client_id", "sp_tenant_id"],
304
305
  "properties": {
305
306
  "org": {"type": "string", "minLength": 1},
306
307
  "project": {"type": "string", "minLength": 1},
307
308
  "queue_id": {"type": "integer", "minimum": 1},
308
309
  "identity": {"enum": ["agent-tarun", "agent-vivek", "agent-reviewer"]},
310
+ "backend": {"description": "Where pool machines run: Hetzner servers or ephemeral Azure Container Apps Sandboxes (one sandbox = one agent = one job; needs the `aca` block).", "enum": ["hetzner", "aca-sandbox"]},
309
311
  "server_type": {"type": "string", "minLength": 1},
310
312
  "location": {"type": "string", "minLength": 1},
311
313
  "snapshot_label_selector": {"type": "string", "minLength": 1},
@@ -314,12 +316,31 @@
314
316
  "max_lifetime_minutes": {"type": "integer", "minimum": 1},
315
317
  "register_timeout_minutes": {"type": "integer", "minimum": 1},
316
318
  "poll_seconds": {"type": "integer", "minimum": 1},
317
- "queue_threshold": {"type": "integer", "minimum": 0},
318
- "trigger_ticks": {"type": "integer", "minimum": 1},
319
+ "queue_wait_minutes": {"description": "A pending job counts as demand only after it has queued this many minutes.", "type": "integer", "minimum": 0},
320
+ "parallel_jobs": {"description": "Azure DevOps runs at most this many self-hosted jobs at once across all pools; the pool never creates past the free slots.", "type": "integer", "minimum": 1},
319
321
  "hour_end_window_minutes": {"type": "integer", "minimum": 0, "maximum": 59},
320
322
  "capabilities": {"type": "object", "additionalProperties": {"type": "string"}},
321
323
  "sp_client_id": {"description": "Application (client) id of the identity's Entra service principal, used by `config.sh --auth SP`; the literal UNSET until Tarun fills it (`pool run` refuses while UNSET).", "type": "string", "pattern": "^(UNSET|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$"},
322
- "sp_tenant_id": {"description": "Entra tenant id of that service principal (AGENT_TENANTS in apps/cli/src/ado-token.ts).", "type": "string", "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"}
324
+ "sp_tenant_id": {"description": "Entra tenant id of that service principal (AGENT_TENANTS in apps/cli/src/ado-token.ts).", "type": "string", "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"},
325
+ "aca": {
326
+ "type": "object",
327
+ "description": "Azure Container Apps Sandboxes backend settings; required when `backend` is `aca-sandbox` (checked by bin/validate-manifest and `adl pool`). `disk_image_id` and `sp_client_id` are the literal UNSET until Tarun fills them (`pool run` refuses while UNSET).",
328
+ "additionalProperties": false,
329
+ "required": ["subscription_id", "resource_group", "sandbox_group", "region", "disk_image_id", "cpu", "memory", "disk", "sp_client_id", "sp_tenant_id"],
330
+ "properties": {
331
+ "subscription_id": {"type": "string", "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"},
332
+ "resource_group": {"type": "string", "minLength": 1},
333
+ "sandbox_group": {"type": "string", "minLength": 1},
334
+ "region": {"type": "string", "minLength": 1},
335
+ "disk_image_id": {"type": "string", "minLength": 1},
336
+ "cpu": {"type": "string", "minLength": 1},
337
+ "memory": {"type": "string", "minLength": 1},
338
+ "disk": {"type": "string", "minLength": 1},
339
+ "sp_client_id": {"description": "Application (client) id of the Entra service principal that calls the sandbox data plane (scope https://dynamicsessions.io/.default).", "type": "string", "pattern": "^(UNSET|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$"},
340
+ "sp_tenant_id": {"type": "string", "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"},
341
+ "capabilities": {"description": "Capabilities the sandbox image really has (no docker); required when `backend` is `aca-sandbox`, where it replaces the pool `capabilities` for the servable-job filter.", "type": "object", "additionalProperties": {"type": "string"}}
342
+ }
343
+ }
323
344
  }
324
345
  },
325
346
  "pool_reaper": {