@turmind/halo 2.0.1 → 2.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/admin-out/404.html +1 -1
- package/admin-out/_next/static/chunks/241-75bac3bb8e7aca1b.js +1 -0
- package/admin-out/_next/static/chunks/989-5c90f88e8111b360.js +1 -0
- package/admin-out/_next/static/chunks/app/_not-found/{page-4271518f2746cf73.js → page-484fd244e4bc7b70.js} +1 -1
- package/admin-out/_next/static/chunks/app/layout-3e437a977abd0081.js +1 -0
- package/admin-out/_next/static/chunks/app/page-faee6fbd17dc7b80.js +1 -0
- package/admin-out/_next/static/chunks/{main-app-340a5b58ed229c4d.js → main-app-42f48cb3fd4811f4.js} +1 -1
- package/admin-out/_next/static/chunks/{webpack-74f577c1b094f942.js → webpack-b7907e0d16faa4ca.js} +1 -1
- package/admin-out/_next/static/css/{e890aa0d13e94290.css → 2e6b00182ac44ae1.css} +1 -1
- package/admin-out/index.html +1 -1
- package/admin-out/index.txt +5 -5
- package/bundled-docs/design/a2a.md +1 -1
- package/bundled-docs/design/agentcore.md +385 -34
- package/bundled-docs/design/canvas-extensions.md +76 -29
- package/bundled-docs/design/cron.md +1 -1
- package/bundled-docs/design/observability.md +4 -2
- package/bundled-docs/design/relay.md +2 -2
- package/bundled-docs/design/session.md +1 -1
- package/bundled-docs/design/storage.md +25 -3
- package/bundled-docs/design/stt-local.md +2 -2
- package/bundled-docs/design/ws.md +2 -1
- package/bundled-docs/dev/api.md +93 -13
- package/bundled-docs/dev/deploy.md +6 -0
- package/bundled-docs/dev/env.md +8 -4
- package/bundled-docs/dev/tools.md +1 -1
- package/bundled-docs/guide/cli.md +25 -11
- package/bundled-docs/guide/skills.md +2 -2
- package/bundled-docs/guide/workspace.md +2 -2
- package/bundled-docs/requirements/editor.md +4 -4
- package/bundled-docs/requirements/settings.md +16 -4
- package/dist/index.js +4011 -2135
- package/package.json +2 -1
- package/templates/skills/extension/SKILL.md +21 -16
- package/templates/skills/extension/config.yaml +6 -2
- package/templates/skills/extension/templates/ext.sh +137 -57
- package/admin-out/_next/static/chunks/87-6a2716c98a819c4e.js +0 -1
- package/admin-out/_next/static/chunks/989-cb287787a4aca288.js +0 -1
- package/admin-out/_next/static/chunks/app/layout-ea161287d934ba96.js +0 -1
- package/admin-out/_next/static/chunks/app/page-c9fb252d0cf34c20.js +0 -1
- /package/admin-out/_next/static/{52ayDC9phpvb-95S7qHHq → P7yMYf16Q3-XqGtzCJNnw}/_buildManifest.js +0 -0
- /package/admin-out/_next/static/{52ayDC9phpvb-95S7qHHq → P7yMYf16Q3-XqGtzCJNnw}/_ssgManifest.js +0 -0
package/admin-out/index.txt
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
1:"$Sreact.fragment"
|
|
2
|
-
2:I[58321,["989","static/chunks/989-
|
|
3
|
-
3:I[24554,["989","static/chunks/989-
|
|
2
|
+
2:I[58321,["989","static/chunks/989-5c90f88e8111b360.js","177","static/chunks/app/layout-3e437a977abd0081.js"],"ThemeProvider"]
|
|
3
|
+
3:I[24554,["989","static/chunks/989-5c90f88e8111b360.js","177","static/chunks/app/layout-3e437a977abd0081.js"],"AppI18nProvider"]
|
|
4
4
|
4:I[63693,[],""]
|
|
5
5
|
5:I[74713,[],""]
|
|
6
6
|
6:I[5444,[],"ClientPageRoot"]
|
|
7
|
-
7:I[
|
|
7
|
+
7:I[48891,["560","static/chunks/1d2d5650-d8d130c85727fdbb.js","241","static/chunks/241-75bac3bb8e7aca1b.js","989","static/chunks/989-5c90f88e8111b360.js","974","static/chunks/app/page-faee6fbd17dc7b80.js"],"default"]
|
|
8
8
|
a:I[23632,[],"OutletBoundary"]
|
|
9
9
|
c:I[86889,[],"AsyncMetadataOutlet"]
|
|
10
10
|
e:I[23632,[],"ViewportBoundary"]
|
|
@@ -13,9 +13,9 @@ e:I[23632,[],"ViewportBoundary"]
|
|
|
13
13
|
13:I[1935,[],""]
|
|
14
14
|
:HL["/_next/static/media/e4af272ccee01ff0-s.p.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
|
|
15
15
|
:HL["/_next/static/css/7b736202f0b1dbfd.css","style"]
|
|
16
|
-
:HL["/_next/static/css/
|
|
16
|
+
:HL["/_next/static/css/2e6b00182ac44ae1.css","style"]
|
|
17
17
|
:HL["/_next/static/css/659eccb5db697b76.css","style"]
|
|
18
|
-
0:{"P":null,"b":"
|
|
18
|
+
0:{"P":null,"b":"P7yMYf16Q3-XqGtzCJNnw","p":"","c":["",""],"i":false,"f":[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/7b736202f0b1dbfd.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/_next/static/css/2e6b00182ac44ae1.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}]],["$","html",null,{"lang":"en","className":"dark","suppressHydrationWarning":true,"children":[["$","head",null,{"children":["$","script",null,{"dangerouslySetInnerHTML":{"__html":"try{var t=localStorage.getItem('halo_theme');if(t==='light'||t==='midnight'||t==='warm')document.documentElement.dataset.theme=t}catch(e){}"}}]}],["$","body",null,{"className":"__variable_d8ebd1 font-sans antialiased","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]]}]]}],{"children":["__PAGE__",["$","$1","c",{"children":[["$","$L6",null,{"Component":"$7","searchParams":{},"params":{},"promises":["$@8","$@9"]}],[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/659eccb5db697b76.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}]],["$","$La",null,{"children":["$Lb",["$","$Lc",null,{"promise":"$@d"}]]}]]}],{},null,false]},null,false],["$","$1","h",{"children":[null,[["$","$Le",null,{"children":"$Lf"}],["$","meta",null,{"name":"next-size-adjust","content":""}]],["$","$L10",null,{"children":["$","div",null,{"hidden":true,"children":["$","$11",null,{"fallback":null,"children":"$L12"}]}]}]]}],false]],"m":"$undefined","G":["$13",[]],"s":false,"S":true}
|
|
19
19
|
8:{}
|
|
20
20
|
9:"$0:f:0:1:2:children:1:props:children:0:props:params"
|
|
21
21
|
f:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]
|
|
@@ -169,7 +169,7 @@ remotes:
|
|
|
169
169
|
# push_base: http://172.31.7.121:9527 # optional; overrides general.a2a.public_url for this remote's webhooks
|
|
170
170
|
```
|
|
171
171
|
|
|
172
|
-
The bearer token is the settings secret `a2a.secrets.<remote>` (`getServerSecret('a2a', name, ws)`: the workspace `settings.yaml` overrides the global one; `<<ENV>>` placeholders work), resolved at call time so it never enters a prompt. **When non-full sessions use the tools, keep it in the global `~/.halo/secrets/settings.yaml`**: a workspace / readonly session can read `<ws>/.halo/settings.yaml` (it is readable workspace knowledge), but `~/.halo/secrets` is hidden from it. `auth: sigv4` signs instead — see [AgentCore](#agentcore).
|
|
172
|
+
The bearer token is the settings secret `a2a.secrets.<remote>` (`getServerSecret('a2a', name, ws)`: the workspace `settings.yaml` overrides the global one; `<<ENV>>` placeholders work; edited in Settings → System → A2A remotes, one field per bearer remote), resolved at call time so it never enters a prompt. **When non-full sessions use the tools, keep it in the global `~/.halo/secrets/settings.yaml`**: a workspace / readonly session can read `<ws>/.halo/settings.yaml` (it is readable workspace knowledge), but `~/.halo/secrets` is hidden from it. `auth: sigv4` signs instead — see [AgentCore](#agentcore).
|
|
173
173
|
|
|
174
174
|
**The list is full-only.** A remote's `card:` decides where its token is sent, on the card fetch and on every RPC. A session that could edit the list could point a remote at a host it controls (public https passes the URL policy) and collect the token from the next `a2a_send`, made by any session. So the list lives under `~/.halo/secrets`, which is in the built-in hidden dirs: bwrap, seatbelt and `assertPathAllowed` mask it from workspace / readonly sessions, whether the file exists or not. Those sessions use the remotes only through the tools. Every workspace's `a2a_list` shows the same list.
|
|
175
175
|
|
|
@@ -1,27 +1,43 @@
|
|
|
1
1
|
# AgentCore Runtime Mode
|
|
2
2
|
|
|
3
|
-
Halo server as an **Amazon Bedrock AgentCore Runtime** container
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
3
|
+
Halo server as an **Amazon Bedrock AgentCore Runtime** container — a fourth
|
|
4
|
+
way to run halo (server / CLI / desktop / AgentCore). `halo agentcore
|
|
5
|
+
--workspace <path>` sets `HALO_RUNTIME_MODE=agentcore-a2a` and runs the server
|
|
6
|
+
in the foreground; the implementation is `packages/server/src/a2a/agentcore.ts`
|
|
7
|
+
(+ `invocations.ts` for jobs, `agentcore-front.ts` in session-dirs mode). One
|
|
8
|
+
process serves both AgentCore server protocols, and AgentCore forwards only the
|
|
9
|
+
paths of the runtime's own protocol:
|
|
10
|
+
|
|
11
|
+
- **A2A** (`serverProtocol: A2A`, :9000) — `POST /` JSON-RPC and the agent
|
|
12
|
+
card. Callers are halo servers with an `auth: sigv4` remote, or any A2A
|
|
13
|
+
client that SigV4-signs.
|
|
14
|
+
- **HTTP** (`serverProtocol: HTTP`, :8080) — [`POST /invocations`](#post-invocations),
|
|
15
|
+
a prompt job, for anything that signs `InvokeAgentRuntime`: EventBridge
|
|
16
|
+
Scheduler (which can't send A2A), a script.
|
|
17
|
+
|
|
18
|
+
The repo ships no deploy package (Dockerfile / IaC): the hub skill
|
|
19
|
+
`agentcore-deploy` ([turmind/halo-hub](https://github.com/turmind/halo-hub))
|
|
20
|
+
builds the image and creates the runtime, or you do it yourself. An earlier
|
|
21
|
+
HTTP mode (`/invocations` with WS `/ws` and per-user workspaces) and its demo
|
|
22
|
+
package were removed 2026-10-09 (git history `7e98369`); today's
|
|
23
|
+
`/invocations` is a different, job-shaped entry.
|
|
11
24
|
|
|
12
25
|
```
|
|
13
|
-
Caller
|
|
14
|
-
|
|
15
|
-
│
|
|
16
|
-
│
|
|
26
|
+
Caller — a halo server / SigV4 A2A client (A2A runtime), or anything that
|
|
27
|
+
signs InvokeAgentRuntime: EventBridge Scheduler, a script (HTTP runtime)
|
|
28
|
+
│ POST …/runtimes/<escaped ARN>/invocations (A2A JSON-RPC | job payload, SigV4)
|
|
29
|
+
│ GET …/invocations/.well-known/agent-card.json (A2A)
|
|
30
|
+
│ X-Amzn-Bedrock-AgentCore-Runtime-Session-Id = session id
|
|
17
31
|
▼
|
|
18
32
|
AgentCore Runtime (per-session microVM, verifies the signed request)
|
|
19
33
|
▼
|
|
20
|
-
halo agentcore :9000
|
|
34
|
+
halo agentcore :9000 (A2A) | :8080 (HTTP) (HALO_RUNTIME_MODE=agentcore-a2a)
|
|
21
35
|
├── GET /ping Healthy | HealthyBusy
|
|
22
36
|
├── POST / A2A JSON-RPC
|
|
23
|
-
|
|
37
|
+
├── GET /.well-known/agent-card.json card (no securitySchemes)
|
|
38
|
+
└── POST /invocations prompt job: sync | SSE | async | get
|
|
24
39
|
one fixed workspace (EFS) — <ws>/.halo/{a2a.db, runs.db, agentcore.lease}
|
|
40
|
+
session-dirs mode: one per session id, <base>/<sid> seen as /workspace
|
|
25
41
|
```
|
|
26
42
|
|
|
27
43
|
## Running it
|
|
@@ -33,14 +49,19 @@ the image build (seeds `~/.halo/global` — agents, skills, models). `halo
|
|
|
33
49
|
agentcore` skips the CLI's setup gate, but the server still exits at boot with
|
|
34
50
|
"~/.halo/global/ not initialized" without it. Edits to the seeded files (e.g.
|
|
35
51
|
pointing the agents' Bedrock endpoint at the runtime's region) go after it.
|
|
52
|
+
In session-dirs mode the worker's home is `/home/node`, so seed that one
|
|
53
|
+
(`HOME=/home/node halo setup --non-interactive`, as user `node`).
|
|
36
54
|
|
|
37
55
|
| Setting | Meaning |
|
|
38
56
|
|---|---|
|
|
39
|
-
| `-w, --workspace` / `HALO_WORKSPACE` | The one workspace served — an EFS mount in practice. |
|
|
40
|
-
| `-p, --port` | Listen port
|
|
57
|
+
| `-w, --workspace` / `HALO_WORKSPACE` | The one workspace served — an EFS mount in practice. In session-dirs mode, the base holding one dir per session id. |
|
|
58
|
+
| `-p, --port` / `HALO_PORT` | Listen port: `-p`, else `HALO_PORT`, else `9000`. A2A runtimes listen on 9000, HTTP runtimes on 8080 (the AgentCore contracts); halo serves every route on either. Session-dirs mode refuses 9100 (the worker's loopback port). |
|
|
41
59
|
| `HALO_A2A_PUBLIC_URL` | The runtime's invoke URL (`…/runtimes/<escaped ARN>/invocations`); becomes the card's interface URL (`<url>/`). Unset → warning, and the card advertises the container's own origin. |
|
|
42
|
-
| `HALO_A2A_ACCESS` | Access level of every session: `workspace` (default) \| `full` \| `readonly`. Anything else is a boot error. |
|
|
60
|
+
| `HALO_A2A_ACCESS` | Access level of every session (A2A and `/invocations`): `workspace` (default) \| `full` \| `readonly`. Anything else is a boot error. |
|
|
61
|
+
| `HALO_AGENTCORE_NOTIFY` | Shell command run when an `/invocations` job finishes — see [the notify hook](#completion-and-the-notify-hook). |
|
|
62
|
+
| `HALO_AGENTCORE_SESSION_DIRS=1` | One workspace per runtime session id — see [Session dirs](#session-dirs). |
|
|
43
63
|
| `HALO_LOG_LEVEL=info` | Needed for CloudWatch — see the ops notes. |
|
|
64
|
+
| `OTEL_*` | Telemetry from the runtime env, no settings edit — see [Observability](#observability). |
|
|
44
65
|
|
|
45
66
|
## What the mode changes (packages/server/src/index.ts)
|
|
46
67
|
|
|
@@ -51,9 +72,9 @@ pointing the agents' Bedrock endpoint at the runtime's region) go after it.
|
|
|
51
72
|
- **Channels, cron, evolution, archive daemon, extensions watcher, admin WS
|
|
52
73
|
and transcribe proxy** (every WS upgrade gets the router's 400).
|
|
53
74
|
- **The whole `/api/*` surface** (auth middleware included) **and the admin
|
|
54
|
-
static bundle / SPA fallback.** The container answers only `/ping`, `POST
|
|
55
|
-
|
|
56
|
-
plain JSON 404, no HTML.
|
|
75
|
+
static bundle / SPA fallback.** The container answers only `/ping`, `POST /`,
|
|
76
|
+
the card and `POST /invocations`, exact paths (AgentCore passes through only
|
|
77
|
+
its protocol's); everything else is a plain JSON 404, no HTML.
|
|
57
78
|
- The normal `/a2a/<path>` mount and the `/a2a-push` receiver. There is no
|
|
58
79
|
outbound A2A from inside the container (`a2a_send` reports itself
|
|
59
80
|
unavailable): a remote's webhook can't reach a microVM.
|
|
@@ -74,7 +95,7 @@ Everything the mode keeps on the workspace (EFS), under `<ws>/.halo/`:
|
|
|
74
95
|
| File | What | Written |
|
|
75
96
|
|---|---|---|
|
|
76
97
|
| `halo.db` (+ `-wal` / `-shm`) | session rows (title, counts, tokens, output, `reply_to`, `stopped_at`) | every turn and tool call — see [SQLite durability](#sqlite-durability) |
|
|
77
|
-
| `a2a.db` (+ `-wal` / `-shm`) | A2A tasks, push configs + outbox | task create / touch / transition, push delivery |
|
|
98
|
+
| `a2a.db` (+ `-wal` / `-shm`) | A2A tasks, push configs + outbox; `/invocations` jobs (`agentcore_invocations`, see [Job rows](#job-rows-a2adb)) | task create / touch / transition, push delivery; job accept / finish / supersede |
|
|
78
99
|
| `runs.db` (+ `-wal` / `-shm`) | run ledger, see [Resume after a mid-task kill](#resume-after-a-mid-task-kill) | entry / exit of every run |
|
|
79
100
|
| `sessions/<agentId>/<seg>.{ui,raw}.jsonl` | session logs ([session.md](session.md)) | appended at every save, rewritten whole on compaction / archive |
|
|
80
101
|
| `sessions/<agentId>/<seg>.ui.arch.<N>.<h>.jsonl.gz` | archived UI history | turn end, past 5 MB |
|
|
@@ -95,8 +116,9 @@ ops only:
|
|
|
95
116
|
invocation — at boot `HALO_WORKSPACE` is still the image's empty dir. So
|
|
96
117
|
nothing touches the workspace at boot: the first request other than `/ping`
|
|
97
118
|
creates `.halo/` and seeds the card.
|
|
98
|
-
- **Acquire on request.** Each `POST /`
|
|
99
|
-
lease is held (concurrent POSTs share one
|
|
119
|
+
- **Acquire on request.** Each `POST /` and `POST /invocations` makes one
|
|
120
|
+
acquisition attempt unless the lease is held (concurrent POSTs share one
|
|
121
|
+
attempt). There is no background
|
|
100
122
|
retry: a refused microVM never polls, so only the microVM callers are actually
|
|
101
123
|
talking to competes. AgentCore can leave microVMs it no longer routes to
|
|
102
124
|
(see gotchas), and a polling one would take the workspace away from the
|
|
@@ -108,8 +130,9 @@ ops only:
|
|
|
108
130
|
- `a2a.db`, the SessionManager and the push sender start only once the lease
|
|
109
131
|
is held; `runtime.lock` is then deleted (its pid probe can't see other microVMs).
|
|
110
132
|
- Without the lease: `POST /` answers JSON-RPC `-32603` "workspace is in use by
|
|
111
|
-
another runtime session — call with the fixed runtime session id"
|
|
112
|
-
|
|
133
|
+
another runtime session — call with the fixed runtime session id"
|
|
134
|
+
(`POST /invocations`: HTTP 503 with the same text), and `/ping` reports
|
|
135
|
+
`Healthy` (never busy) so AgentCore can reap that microVM.
|
|
113
136
|
- **Handoff takes ~45 s in practice, not a graceful release.** Graceful stop
|
|
114
137
|
does try to delete the lease, but on AgentCore the EFS mount is usually
|
|
115
138
|
already detached when SIGTERM arrives (`StopRuntimeSession`, idle reap). The
|
|
@@ -125,8 +148,10 @@ ops only:
|
|
|
125
148
|
The [run ledger](session.md#run-ledger--restart-nudge-for-interrupted-roots-haloglobalrunsdb) lives on the workspace in this mode — `<ws>/.halo/runs.db` (hidden from non-full sessions, like `a2a.db`), not the container's `HALO_HOME`, which dies with the microVM. So a task cut off mid-run (the 8 h `maxLifetime`, a crash, a recycle) resumes on the next microVM:
|
|
126
149
|
|
|
127
150
|
- `onAcquired` opens the ledger (`setRunsDb(createRunsDb(<ws>/.halo))`) **before** `registry.getOrCreate`; `index.ts` opens none in this mode. The SessionManager constructor then runs the server's boot chain — claim `runtime.lock`, orphan reconcile, goal sweep, run-ledger sweep — which drains the rows the dead microVM left and nudges each interrupted root.
|
|
128
|
-
- The `
|
|
129
|
-
-
|
|
151
|
+
- The lease counts as held only once `onAcquired` has returned. If it throws (an `a2a.db` migration, `createRunsDb`, `getOrCreate`), the lease file is released, the half-opened db handles are closed and unset, `[AgentCore] workspace lease released: opening the workspace failed (…)` is logged, and the request is refused. The next request retries the whole acquisition. Otherwise the process would hold a lease with no heartbeat, and a second writer would take over 45 s later.
|
|
152
|
+
- `/invocations` sessions resume the same way: a root with an `{invocation}` `reply_to` gets the nudge, and its resumed run finishes the job (state, result, notify). That includes `cron-*` session ids — the `agentcore-deploy` skill passes the local cron session id as `session`. The sweep skips a `cron-*` root only when it has no `reply_to` (a cli-driven cron tree); with one, a caller is waiting on it.
|
|
153
|
+
- The request that acquired the lease — a `POST /` (A2A) or a `POST /invocations`, a `get` included — triggers that sweep. A `POST /` and a new `/invocations` job then wait for the nudges to reach their sessions (`nudgesSettled`) before they are handled, so a GetTask on the interrupted task sees its root running → `WORKING`, not idle. A `get` answers at once: a job's row has no lazy reconcile and stays `running` until the resumed run finishes it.
|
|
154
|
+
- Nothing wakes a new microVM by itself: the resume happens on the **next invocation** — the caller's `a2a_read` / GetTask, a new send, or an `/invocations` call (a new job or a `get`). No caller-side polling.
|
|
130
155
|
- `ownsRuntimes` stays `false`, so the lazy FAILED in `routes.ts` `reconcileStale` remains the fallback for a task the sweep didn't resume (root idle, quiet subtree): "Interrupted by a server restart on the remote. Send again on the same context to resume."
|
|
131
156
|
- Verified locally: `kill -9` mid-task (2 of 6 `sleep 30` calls done) → new process refused for ~45 s, then the first accepted GetTask read `WORKING` and the task completed with all six results.
|
|
132
157
|
|
|
@@ -136,17 +161,326 @@ The [run ledger](session.md#run-ledger--restart-nudge-for-interrupted-roots-halo
|
|
|
136
161
|
|
|
137
162
|
Cost: one fsync per commit that changes a row; an UPDATE that changes nothing writes no frame and syncs nothing. Measured (strace, scripted A2A SendMessage): a follow-up turn with no tool call commits 10 times (a2a.db 2 — create + complete; runs.db 2 — insert + delete; halo.db 6 — `reply_to` set + clear, output / last-activity twice (first event, release), counts / tokens, `updated_at`); each tool call adds one (the next model call's token UPDATE), plus the output / last-activity write at most once per 5 s mid-turn. Checkpoints fsync the main db file on top.
|
|
138
163
|
|
|
139
|
-
**Known risk — `-shm` and dead-host locks on NFS (
|
|
164
|
+
**Known risk — `-shm` and dead-host locks on NFS (measured 2026-10-17, low).** SQLite doesn't support WAL on network filesystems; here the lease keeps a single writer, so normal running is fine, and the risk sits at a crash handoff. When a db opens, SQLite rebuilds the wal-index only if no other connection holds the DMS lock byte on `-shm`. A dead microVM's NFS locks persist until its lock lease expires on the server; while they stand, the next holder keeps the old `-shm` contents, which could in theory lag the WAL. Measured on the `halo_a2a` runtime (2.0.1, EFS through the microVM's loopback NFS proxy):
|
|
165
|
+
|
|
166
|
+
- `StopRuntimeSession` and `kill -9`: the old locks are gone ~1 s later — no overlap with the ~45 s lease handoff. Three mid-load handoffs (2–3 tasks committing on every tool call): every task resumed and completed with no step missing or repeated, `integrity_check` ok on all three dbs at every checkpoint, no acked commit lost.
|
|
167
|
+
- **Idle reap** (no stop, the 60 s idle timeout): the dead microVM's locks stand **78–88 s** after its last write, i.e. 30–40 s past the 45 s stale window (consistent with a ~90 s NFSv4 lock lease, inferred). Real halo took over at 48–50 s under the standing locks and ran normally (3/3); a write-heavy probe db saw a transient `SQLITE_BUSY` / `SQLITE_PROTOCOL` on 2 of 16 opens and no corruption. A stale `-shm` behind its WAL was never observed.
|
|
168
|
+
|
|
169
|
+
So `journal_mode=TRUNCATE` is not warranted. If the transient BUSY ever matters, the cheaper knobs are a longer busy timeout on the workspace dbs or a stale window past the ~90 s lock lease — neither tried.
|
|
140
170
|
|
|
141
171
|
**Known cost — synchronous IO on the event loop.** Session-log writes, SQLite commits and the bwrap mask lookups are synchronous by design (they keep UI / raw writes and the tombstone check in one synchronous slice), so each EFS round trip blocks the loop: estimated at ~40 ms per tool call and ~70 ms per turn before the 2.0 IO trimming, during which other sessions and `/ping` wait. Moving to async IO would reorder the write paths' guarantees; not planned.
|
|
142
172
|
|
|
173
|
+
## `POST /invocations`
|
|
174
|
+
|
|
175
|
+
A prompt job over plain HTTP — the entry of an HTTP-protocol runtime, for
|
|
176
|
+
callers that can't speak A2A (EventBridge Scheduler → `InvokeAgentRuntime`,
|
|
177
|
+
a script). Routes in `agentcore.ts`, job state in `invocations.ts`. Wire shapes
|
|
178
|
+
and status codes: [dev/api.md](../dev/api.md#post-invocations).
|
|
179
|
+
|
|
180
|
+
### Request and dispatch
|
|
181
|
+
|
|
182
|
+
The body (≤ 1 MiB) is a JSON object: exactly one of `prompt` / `promptFile`,
|
|
183
|
+
plus optional `session`, `agent`, `id`, `async`; unknown fields are ignored.
|
|
184
|
+
`{"get": "<key>"}` alone reads a job instead (below). Shape errors are 400 and
|
|
185
|
+
answered before the lease.
|
|
186
|
+
|
|
187
|
+
- **Lease first.** Like `POST /`, the request takes the workspace lease (busy
|
|
188
|
+
→ 503 with the `LEASE_BUSY` text) and waits for the acquisition's run-ledger
|
|
189
|
+
nudges (`nudgesSettled`) before dispatching.
|
|
190
|
+
- **`promptFile`** is read from the workspace: its realpath must be a regular
|
|
191
|
+
file inside it, not a hidden runtime path (sessions, dbs — `isHiddenWorkspacePath`),
|
|
192
|
+
non-empty, ≤ 1 MiB. The skill points a job at `.halo/jobs/<name>.md`, so the
|
|
193
|
+
prompt is edited by syncing a file, not by recreating the schedule.
|
|
194
|
+
- **`session`** (`^[A-Za-z0-9_:-]{1,200}$`, a root session id) names the halo
|
|
195
|
+
session: a fixed one keeps the job's history across runs (one per schedule),
|
|
196
|
+
an existing one is resumed with its own agent. Absent → a fresh
|
|
197
|
+
`inv_<time><hex>` session. **`agent`** applies only when the session is
|
|
198
|
+
created (unknown or internal agent → 400); default is the workspace's default
|
|
199
|
+
agent. Sessions run at `HALO_A2A_ACCESS`.
|
|
200
|
+
- The prompt reaches the model behind cron's unattended preamble ("nobody will
|
|
201
|
+
answer questions…"); the chat log shows the prompt alone. A busy session
|
|
202
|
+
queues it, as for any user message.
|
|
203
|
+
- **`reply_to = {"invocation": <id>}`** on the root marks it as owing the job —
|
|
204
|
+
the third kind of [relay](relay.md) pointer beside `{workspace, sessionId}`
|
|
205
|
+
and `{a2a}`. A session already owing an A2A task or a relay report is refused
|
|
206
|
+
(409).
|
|
207
|
+
- A dispatch that throws before the message reached the session deletes the
|
|
208
|
+
row and clears the pointer, so the caller's retry with the same `id` runs (500).
|
|
209
|
+
|
|
210
|
+
### Response modes
|
|
211
|
+
|
|
212
|
+
A new job answers by mode, all from the same receipt
|
|
213
|
+
`{status: "accepted", invocation, session, id}`:
|
|
214
|
+
|
|
215
|
+
- **sync** (default) — waits until the job leaves `running`, at most 10 min
|
|
216
|
+
(A2A blocking SendMessage's cap), then answers the job body. Past the cap the
|
|
217
|
+
body still reads `running`; the run goes on and the caller `get`s it later.
|
|
218
|
+
- **SSE** (`Accept: text/event-stream`) — `accepted` (the receipt, so a client
|
|
219
|
+
that sent no `id` can still `get` after a drop), `delta {text}` events with
|
|
220
|
+
the root's live text (sub-sessions' deltas are not forwarded), a
|
|
221
|
+
`: keepalive` comment every 15 s, then one `result` event (the job body) and
|
|
222
|
+
close. No time cap.
|
|
223
|
+
- **async** (`"async": true`) — the receipt at once.
|
|
224
|
+
- **get** (`{"get": "<key>"}`) — the job body now; the key is an invocation id,
|
|
225
|
+
else a caller `id`. Unknown → 404.
|
|
226
|
+
|
|
227
|
+
A client that disconnects only stops waiting (sync) or unsubscribes (SSE); the
|
|
228
|
+
run is never tied to the request. Waiters are woken in-process
|
|
229
|
+
(`onInvocationDone`) — no polling of the db.
|
|
230
|
+
|
|
231
|
+
The **job body**: `{status, invocation, session, id, result, files,
|
|
232
|
+
startedAt, finishedAt, supersededBy?}`. `status` ∈ `running` \| `ok` \| `error`
|
|
233
|
+
\| `superseded`; `result` is the final reply (`finalOutput || output`, the text
|
|
234
|
+
`halo cli` prints, `MEDIA:` lines kept), null until finished; `files` is
|
|
235
|
+
`[{path, size}]` for the reply's `MEDIA:` paths that were regular files inside
|
|
236
|
+
the workspace at finish (realpath checked; paths as the agent wrote them,
|
|
237
|
+
`/workspace/…` in session-dirs mode) — a manifest, never the bytes.
|
|
238
|
+
|
|
239
|
+
### Idempotency and supersede
|
|
240
|
+
|
|
241
|
+
- **`id`** (≤ 256 chars) is the caller's idempotency key, unique across the
|
|
242
|
+
workspace's jobs (partial unique index). A repeat answers that job's current
|
|
243
|
+
body plus `"duplicate": true` at once, in any mode (JSON even when SSE was
|
|
244
|
+
asked for), and runs nothing — whatever the new payload says. The re-check
|
|
245
|
+
right before the insert sits in the same synchronous slice as the insert, so
|
|
246
|
+
two concurrent retries can't both run; the unique index is the cross-process
|
|
247
|
+
backstop. Keys persist with the rows (no expiry).
|
|
248
|
+
- **Supersede.** A new job on a session still owing an earlier one (still
|
|
249
|
+
running, or cut off and never resumed) takes the pointer: the earlier row
|
|
250
|
+
becomes `superseded` with `supersededBy` = the new id, its sync waiter and SSE
|
|
251
|
+
stream answer that at once, and one completion — the new job's — covers both
|
|
252
|
+
(the message folded into the same session). The same happens when an A2A
|
|
253
|
+
message or a `relay_send` / `relay_interrupt` lands on a session owing a job:
|
|
254
|
+
the job is superseded (`supersededBy` = `a2a:<task id>` /
|
|
255
|
+
`relay:<caller session>`), then their pointer is written (`relay.ts`
|
|
256
|
+
`takeReplyTo`). A job on a session owing an A2A task or a relay report is
|
|
257
|
+
refused (409). For a new session that is checked again right after it is
|
|
258
|
+
created, in the same synchronous slice as the write, so a concurrent request
|
|
259
|
+
can't slip in between.
|
|
260
|
+
|
|
261
|
+
### Completion and the notify hook
|
|
262
|
+
|
|
263
|
+
The job ends exactly like an A2A task: relay's turn-end hook
|
|
264
|
+
(`deliverRelayReport`, after the quiet gate — the root's whole subtree idle)
|
|
265
|
+
calls `finishInvocation`, then clears `reply_to`. `finishInvocation` is a
|
|
266
|
+
guarded UPDATE on `running` (first writer wins): it stores `state` (`error`
|
|
267
|
+
when the turn ended in an error, else `ok`), `finished_at`, `result` and
|
|
268
|
+
`result_files` in one statement, wakes the waiters, then starts the hook.
|
|
269
|
+
Interim reports are not sent — a job notifies its final reply only.
|
|
270
|
+
|
|
271
|
+
`HALO_AGENTCORE_NOTIFY` (unset = no hook) runs as `sh -c <cmd>` in its own
|
|
272
|
+
process group: stdin = the final reply, env = the process env plus
|
|
273
|
+
`HALO_SESSION_ID`, `HALO_RUN_STATUS` (`ok` \| `error`) and `HALO_WORKSPACE`,
|
|
274
|
+
cwd = `HALO_WORKSPACE`. It is a server child, outside the agent sandbox; it is
|
|
275
|
+
killed (whole group) after 300 s, and its outcome is logged as one
|
|
276
|
+
`[AgentCore] notify <session>: exit N (ms) stdout: … stderr: …` line (tails of
|
|
277
|
+
1000 chars). `/ping` stays `HealthyBusy` while a hook runs. The hub skill's
|
|
278
|
+
`notify.py` is the intended hook: it pushes the text and its `MEDIA:` files
|
|
279
|
+
to Feishu / Slack / Telegram / WeCom from secrets in the runtime env.
|
|
280
|
+
|
|
281
|
+
In single-process mode the hook's env and `HALO_WORKSPACE` are the server's own
|
|
282
|
+
(the workspace). In [session-dirs mode](#notify-hand-off) the worker hands the
|
|
283
|
+
reply to the root front instead, which runs the hook on vetted input.
|
|
284
|
+
|
|
285
|
+
**At most once.** The job's row is final before the hook starts, and nothing
|
|
286
|
+
re-runs a hook. A hook that is cut off loses that push, and the row still
|
|
287
|
+
reads `ok` / `error` (`get` has the result). A hook is cut off by the 300 s
|
|
288
|
+
kill, or when the microVM goes away mid-hook: a stop, or the 8 h `maxLifetime`.
|
|
289
|
+
`/ping` reports `HealthyBusy` while a hook runs, so an idle reap doesn't cut
|
|
290
|
+
one off. On SIGTERM, single-process mode's `gracefulShutdown` does not wait for
|
|
291
|
+
a running hook; in session-dirs mode the front does, within its 30 s stop
|
|
292
|
+
budget ([front and worker](#front-and-worker)).
|
|
293
|
+
|
|
294
|
+
### Job rows (`a2a.db`)
|
|
295
|
+
|
|
296
|
+
`agentcore_invocations` in the workspace's `a2a.db` (durable on EFS, so a key
|
|
297
|
+
and an outcome survive microVM recycling): `id` (the invocation id, a UUID),
|
|
298
|
+
`request_id` (the caller's `id`, partial unique index), `session_id`, `state`,
|
|
299
|
+
`created_at`, `finished_at`, `result`, `result_files` (JSON `[{path, size}]`),
|
|
300
|
+
`superseded_by`. Sync, SSE and `get` answer from the row, never from the
|
|
301
|
+
session log (a fixed job session keeps growing). `A2A_MIGRATIONS` v2 adds the
|
|
302
|
+
last three columns to a table created by an earlier build (`user_version` 2
|
|
303
|
+
after it). Rows are not pruned.
|
|
304
|
+
|
|
305
|
+
## Session dirs
|
|
306
|
+
|
|
307
|
+
`HALO_AGENTCORE_SESSION_DIRS=1` gives each runtime session id its own
|
|
308
|
+
workspace `<base>/<sid>` on the EFS base (`HALO_WORKSPACE`), so callers on one
|
|
309
|
+
runtime don't share files. One microVM serves one session id, so isolation is
|
|
310
|
+
the microVM's own mount view — no extra namespace. Code:
|
|
311
|
+
`a2a/agentcore-front.ts` (front), `agentcore-notify.ts` (hook + worker → front
|
|
312
|
+
calls).
|
|
313
|
+
|
|
314
|
+
```
|
|
315
|
+
AgentCore ──► front (PID 1, root, :9000 | :8080; node builtins only — no config / dbs / SessionManager)
|
|
316
|
+
│ session header check, view setup once, proxy (streams both ways)
|
|
317
|
+
▼
|
|
318
|
+
worker (uid/gid 1000, no caps, no_new_privs, env allowlist)
|
|
319
|
+
= plain `halo agentcore --workspace /workspace -p 9100` on 127.0.0.1
|
|
320
|
+
│ /ready, /notify ──► front loopback (per-boot token)
|
|
321
|
+
```
|
|
322
|
+
|
|
323
|
+
### Front and worker
|
|
324
|
+
|
|
325
|
+
- `halo agentcore` with the env set runs only the front: it needs root, refuses
|
|
326
|
+
port 9100 (the worker's) and a base overlapping `/workspace`.
|
|
327
|
+
- Exact contract paths, else 404. `/ping` is the front's: `Healthy` until a
|
|
328
|
+
worker is ready, then the worker's answer (5 s timeout; no answer →
|
|
329
|
+
`HealthyBusy`, so a worker whose loop is blocked on EFS IO isn't reaped
|
|
330
|
+
mid-run), `HealthyBusy` while the front runs a hook.
|
|
331
|
+
- Every other request needs `X-Amzn-Bedrock-AgentCore-Runtime-Session-Id`
|
|
332
|
+
matching `^[A-Za-z0-9][A-Za-z0-9_-]{32,254}$` — 33–255 chars: AgentCore
|
|
333
|
+
allows 256, but the id is the dir name and NAME_MAX is 255, so a 256-char id
|
|
334
|
+
is a 400 rather than a dir that can never be made. A leading dot can't
|
|
335
|
+
match, so `.template` / `.tmp-*` are never served. Else 400. Once a microVM
|
|
336
|
+
serves a session id, any other id is 409: it never switches workspace.
|
|
337
|
+
- **First request:** the base must be a mount point (`/proc/self/mountinfo`;
|
|
338
|
+
EFS arrives only at the first invocation — not yet → 503 "`<base>` is not a
|
|
339
|
+
mount point …", retried by the next request; a `HALO_WORKSPACE` that isn't
|
|
340
|
+
the EFS mount path keeps answering it). `<base>/<sid>` is created if
|
|
341
|
+
missing: built in a dot-named `.tmp-<16 hex>` (0700 — no id in the name, so
|
|
342
|
+
a long id can't push it past NAME_MAX) from `<base>/.template` minus `.halo/sessions`,
|
|
343
|
+
sqlite files (+ `-wal` / `-shm` / `-journal`), `runtime.lock` and
|
|
344
|
+
`agentcore.lease*`, chowned to 1000:1000 (best effort — an access point that
|
|
345
|
+
enforces 0:0 refuses it and access still works), then renamed in, so a
|
|
346
|
+
half-copied dir is never served and two microVMs racing on one id end with
|
|
347
|
+
one complete dir. A failure here is a 500, retried by the next request.
|
|
348
|
+
`<base>/.template` is the deployer's (the skill's `sync --template`); a new
|
|
349
|
+
caller's first call costs a full copy of it.
|
|
350
|
+
- Then the [view setup](#view-setup), then the worker:
|
|
351
|
+
`setpriv --reuid=1000 --regid=1000 --clear-groups --inh-caps=-all
|
|
352
|
+
--bounding-set=-all --no-new-privs env -i <allowlist> <node> <halo>
|
|
353
|
+
agentcore --workspace /workspace -p 9100`, cwd `/workspace`. The allowlist:
|
|
354
|
+
`PATH`, `AWS_REGION`, `AWS_DEFAULT_REGION`, `AWS_ACCESS_KEY_ID`,
|
|
355
|
+
`AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN`, `HALO_LOG_LEVEL`,
|
|
356
|
+
`HALO_A2A_ACCESS`, `HALO_A2A_PUBLIC_URL`, Halo's tuning knobs
|
|
357
|
+
(`HALO_MAX_CONTEXT_TOKENS`, `HALO_SHELL_TIMEOUT`, `HALO_WEB_FETCH_TIMEOUT`,
|
|
358
|
+
`HALO_MODEL_TIMEOUT`, `HALO_SESSION_GRACE`, `HALO_TERMINAL_GRACE`,
|
|
359
|
+
`HALO_MAX_CACHED_SESSIONS`, `HALO_LOG_MAX_SIZE`, `HALO_LOG_MAX_FILES`), every
|
|
360
|
+
`OTEL_*`, `AGENT_OBSERVABILITY_*` and `AWS_CONTAINER_*`, plus
|
|
361
|
+
`HOME=/home/node` and the front's loopback URL + token — each named, no
|
|
362
|
+
`HALO_*` wildcard. Channel secrets, `HALO_AGENTCORE_NOTIFY` and
|
|
363
|
+
`HALO_PASSWORD` stay with the front.
|
|
364
|
+
- **Models: Bedrock on the runtime role only.** Model keys don't pass either:
|
|
365
|
+
a `<<OPENAI_API_KEY>>`-style placeholder in `settings.yaml` resolves to
|
|
366
|
+
nothing in the worker. The worker's credentials are the AWS ones above, so
|
|
367
|
+
its agents use Bedrock through the runtime's execution role (as the
|
|
368
|
+
`agentcore-deploy` skill deploys them). Another provider needs its key
|
|
369
|
+
written as a literal value into the image's `/home/node/.halo` — and is then
|
|
370
|
+
readable by the agent.
|
|
371
|
+
- The worker is the single-workspace mode on `/workspace`, unchanged (lease,
|
|
372
|
+
run ledger, dbs, `/invocations`), bound to 127.0.0.1; once listening it posts
|
|
373
|
+
`/ready` to the front (push, no probe). No `/ready` within 30 s → 503 for
|
|
374
|
+
good. The request that triggered the start waits for it, then is proxied
|
|
375
|
+
(unless its client has gone meanwhile — then nothing reaches the worker);
|
|
376
|
+
hop-by-hop headers dropped, bodies streamed (SSE chunk by chunk), a closed
|
|
377
|
+
client aborts the upstream request, and a worker response cut off mid-body
|
|
378
|
+
closes the client's too (logged `[AgentCore] front: worker response cut off
|
|
379
|
+
mid-stream`), so no client hangs on a dead stream. A handler that throws is
|
|
380
|
+
logged and answered 500, never fatal to the front.
|
|
381
|
+
- The worker exiting ends the front: with the worker's exit code, or 1 when
|
|
382
|
+
that was 0 or a signal (an unasked-for worker exit is a failure). SIGTERM /
|
|
383
|
+
SIGINT to the front go to the worker — its graceful stop releases the lease.
|
|
384
|
+
Either way the front first waits for its own notify work (a MEDIA copy or a
|
|
385
|
+
hook still running), within the same 30 s budget: SIGKILL to the worker and
|
|
386
|
+
exit once it is spent.
|
|
387
|
+
|
|
388
|
+
### View setup
|
|
389
|
+
|
|
390
|
+
Once per process, as root in the front's own (the microVM's main) mount
|
|
391
|
+
namespace, every command with a 30 s timeout:
|
|
392
|
+
|
|
393
|
+
1. `mkdir -p /workspace`
|
|
394
|
+
2. `mount --bind <base>/<sid> /workspace`
|
|
395
|
+
3. `umount -l <base>` — repeated while mountinfo still lists the base, at most
|
|
396
|
+
5 times. `/mnt/efs` can carry stacked mounts (the platform's overlay bind
|
|
397
|
+
under the nfs4 one) and each `umount` pops only the top one.
|
|
398
|
+
4. Check: the base is no longer a mount point and `<base>/<sid>` is no longer
|
|
399
|
+
visible. Layers left after 5 rounds → "`<base>` still has N mount(s) after
|
|
400
|
+
5 × umount -l".
|
|
401
|
+
|
|
402
|
+
Fail closed: any failing step or check answers 503 for that request and every
|
|
403
|
+
later one, the worker is never spawned and the setup never re-run (a half-done
|
|
404
|
+
setup must not be served). There is no network step — see
|
|
405
|
+
[the limits](#known-limitations-session-isolation-within-a-runtime).
|
|
406
|
+
|
|
407
|
+
**503 means "not serving right now", not "give up"** — the reason is the
|
|
408
|
+
body's `error` text, the status alone doesn't say which. Only two are for good
|
|
409
|
+
on this microVM: the view setup failed, and the worker wasn't ready in 30 s.
|
|
410
|
+
The base not being a mount point yet and the workspace lease being held by
|
|
411
|
+
another runtime session (`/invocations`) clear on their own — retry with the
|
|
412
|
+
same id. A caller can't decide to stop retrying from a 503 alone.
|
|
413
|
+
|
|
414
|
+
### Notify hand-off
|
|
415
|
+
|
|
416
|
+
In a worker, `finishInvocation` posts `{id, session, status, text}` to the
|
|
417
|
+
front's loopback `/notify` instead of running a hook. The token is per boot
|
|
418
|
+
but readable by the agent (same uid as the worker), so the front treats every
|
|
419
|
+
field as agent-controlled:
|
|
420
|
+
|
|
421
|
+
- token compared in constant time (else 403); body ≤ 16 MiB; `session` must
|
|
422
|
+
match `^[A-Za-z0-9_:-]{1,200}$`, `status` be `ok` \| `error`, `text` a string
|
|
423
|
+
(else 400). `/ready` works only while the worker starts; any other path, or
|
|
424
|
+
`/notify` before the view is set up, is 404.
|
|
425
|
+
- 202, then — only if `HALO_AGENTCORE_NOTIFY` is set — a root-only (0700)
|
|
426
|
+
temp dir per notify, which is also the hook's `HALO_WORKSPACE` and cwd. Each
|
|
427
|
+
`MEDIA:<path>` line becomes a copy in it (`<dir>/<n>/<name>`): the path must
|
|
428
|
+
be absolute; it is opened `O_PATH` (no side effects, never blocks on a FIFO),
|
|
429
|
+
the file actually opened must resolve (`/proc/self/fd/N`) under `/workspace/`,
|
|
430
|
+
be a regular file and ≤ 20 MB, and its bytes are read through that same fd.
|
|
431
|
+
So no symlink or swap between the checks can point root at another file. A
|
|
432
|
+
refused line is dropped with `[AgentCore] notify <session>: MEDIA dropped
|
|
433
|
+
(<why>): "<path>"`.
|
|
434
|
+
- The hook runs with the front's env (the channel secrets) plus the validated
|
|
435
|
+
`HALO_SESSION_ID` / `HALO_RUN_STATUS`, and the temp dir is removed after it.
|
|
436
|
+
`notify.py`'s own "file under `$HALO_WORKSPACE`" rule then admits only the
|
|
437
|
+
copies.
|
|
438
|
+
|
|
439
|
+
### Known limitations: session isolation within a runtime
|
|
440
|
+
|
|
441
|
+
Session dirs keep callers on one runtime from **reading each other by
|
|
442
|
+
accident**; they don't stop a caller who can make the agent read another
|
|
443
|
+
session on purpose. This is an accepted limitation; the hard boundary is the
|
|
444
|
+
runtime.
|
|
445
|
+
|
|
446
|
+
- **No network isolation inside the microVM.** Root has `CAP_SYS_ADMIN` but
|
|
447
|
+
not `CAP_NET_ADMIN` (no packet-filter rules, nft or legacy); Landlock
|
|
448
|
+
returns `ENOSYS` (and kernel 6.1 has no Landlock network rules); the cgroup
|
|
449
|
+
fs is read-only (no cgroup-bpf); there is no `/dev/net/tun`. So halo can't
|
|
450
|
+
keep the worker off the NFS port, and doesn't try.
|
|
451
|
+
- **Shared storage paths stay reachable.** EFS is mounted through a
|
|
452
|
+
platform-side NFS proxy on the microVM's loopback (`127.0.0.1:/`, a high
|
|
453
|
+
port) outside the container, and uid 1000 can get the execution role's
|
|
454
|
+
credentials from the microVM metadata service (MMDS). An agent that writes
|
|
455
|
+
an NFS client can mount the access point through either path and read every
|
|
456
|
+
session's dir on that runtime.
|
|
457
|
+
- **The session id is the access key.** Anyone allowed to invoke the runtime
|
|
458
|
+
with a given session id gets that session's dir.
|
|
459
|
+
- **The worker can call `/notify` itself** (it holds the token): only the
|
|
460
|
+
deployer's hook runs, on validated input, but nothing rate-limits it.
|
|
461
|
+
|
|
462
|
+
What to do instead:
|
|
463
|
+
|
|
464
|
+
- **Separate runtimes for callers that don't trust each other** — the simplest
|
|
465
|
+
hard isolation. The hub skill
|
|
466
|
+
[`agentcore-deploy`](https://github.com/turmind/halo-hub) builds each deploy
|
|
467
|
+
that way: its own encrypted EFS, access point and execution role, with a
|
|
468
|
+
file-system policy of three statements — `DenyNonTls`,
|
|
469
|
+
`DenyWithoutAccessPoint`, `AllowRuntimeRoleViaAP` — so a runtime can mount
|
|
470
|
+
only its own access point, over TLS, and never the fs root. Storage limits
|
|
471
|
+
live at that infra layer, not in halo.
|
|
472
|
+
- **Possible later hardening (not done):** run the worker in a network
|
|
473
|
+
namespace with only `lo`, and give it a CONNECT proxy on a unix socket that
|
|
474
|
+
allows only the Bedrock endpoints.
|
|
475
|
+
|
|
143
476
|
## Access model
|
|
144
477
|
|
|
145
478
|
"Upstream" strategy: AgentCore verifies the caller's SigV4 before forwarding,
|
|
146
479
|
so the container does no token auth and the card carries no `securitySchemes`.
|
|
147
480
|
Every call is one fixed caller (`accountId: agentcore`) on the fixed workspace,
|
|
148
481
|
mounted at `/` (any sub-path is 404), with the session level from
|
|
149
|
-
`HALO_A2A_ACCESS` — default `workspace`, not `full`.
|
|
482
|
+
`HALO_A2A_ACCESS` — default `workspace`, not `full`. `/invocations` sessions
|
|
483
|
+
run at the same level. See
|
|
150
484
|
[guide/delegation-and-access.md](../guide/delegation-and-access.md).
|
|
151
485
|
|
|
152
486
|
## Caller side (any halo server)
|
|
@@ -187,8 +521,9 @@ remotes:
|
|
|
187
521
|
## Session lifecycle
|
|
188
522
|
|
|
189
523
|
- `/ping` returns `HealthyBusy` while the lease is held and an agent session is
|
|
190
|
-
running (`hasRunningSessions()`)
|
|
191
|
-
|
|
524
|
+
running (`hasRunningSessions()`), a push is pending or a notify hook (or its
|
|
525
|
+
hand-off to the front) is in flight — the keep-alive for long tool chains
|
|
526
|
+
with no open request.
|
|
192
527
|
- Idle timeout (`idleRuntimeSessionTimeout`, **60 s**) terminates a session
|
|
193
528
|
whose `/ping` reports `Healthy` — the microVM exits on its own once idle (no
|
|
194
529
|
self-stop, no extra IAM). So the 8 h `maxLifetime`, which force-terminates
|
|
@@ -197,9 +532,10 @@ remotes:
|
|
|
197
532
|
immediately.
|
|
198
533
|
- Termination is cheap: state is on EFS; the next call cold-starts a microVM
|
|
199
534
|
and takes the lease once the old one is stale (~45 s, see above). A call in
|
|
200
|
-
that gap gets the lease-busy `-32603
|
|
201
|
-
retries
|
|
202
|
-
message prefix — keep that text
|
|
535
|
+
that gap gets the lease-busy `-32603` (`/invocations`: HTTP 503, same text);
|
|
536
|
+
the caller side (`a2a/outbound.ts`) retries the A2A one at 5 / 10 / 15 / 20 s,
|
|
537
|
+
matched on the code plus the `LEASE_BUSY` message prefix — keep that text
|
|
538
|
+
stable. An `/invocations` caller retries itself, with the same `id`.
|
|
203
539
|
There is **no list/get-runtime-sessions API** (observe via the CloudWatch
|
|
204
540
|
`Sessions` metric + runtime log filtering).
|
|
205
541
|
|
|
@@ -244,6 +580,14 @@ Found deploying `halo_a2a` (ap-northeast-1, 2026-10-09):
|
|
|
244
580
|
- `-32054` RetryableConflict never showed up in practice (concurrent cold
|
|
245
581
|
first calls, bursts after a stop); the caller retry stays as cheap insurance.
|
|
246
582
|
|
|
583
|
+
Found deploying session-dirs runtimes (2026-10-17, kernel 6.1 amzn2023, arm64):
|
|
584
|
+
|
|
585
|
+
- **No `CAP_NET_ADMIN` in the microVM** — root's CapEff is `0xa82425fb`
|
|
586
|
+
(`CAP_SYS_ADMIN` yes, `CAP_NET_ADMIN` no), so packet-filter rules can't be
|
|
587
|
+
added; a fail-closed setup step that needs them refuses every request.
|
|
588
|
+
- **`/mnt/efs` can be two mounts deep** (an overlay bind with nfs4 on top,
|
|
589
|
+
depending on the image); `umount -l` pops one layer per call.
|
|
590
|
+
|
|
247
591
|
## Observability
|
|
248
592
|
|
|
249
593
|
The runtime container can export traces/metrics/logs via `general.observability.*`
|
|
@@ -252,3 +596,10 @@ deployment. AgentCore Evaluations reads the resulting spans out of the
|
|
|
252
596
|
CloudWatch `aws/spans` log group (Transaction Search) — see
|
|
253
597
|
[design/observability.md](observability.md) for the collector config and the
|
|
254
598
|
Evaluations input contract.
|
|
599
|
+
|
|
600
|
+
The image's `settings.yaml` needn't carry any of it: the standard
|
|
601
|
+
`OTEL_EXPORTER_OTLP_ENDPOINT` / `OTEL_EXPORTER_OTLP_HEADERS` /
|
|
602
|
+
`OTEL_SERVICE_NAME` in the runtime env win over settings, and
|
|
603
|
+
`OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT` (`true` / `false`)
|
|
604
|
+
overrides `capture_content` ([details](observability.md#configuration)). In
|
|
605
|
+
session-dirs mode every `OTEL_*` var passes the worker's env allowlist.
|