@open-everywhere/create 0.7.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/package.json +3 -3
- package/src/steps/build-deploy.ts +2 -2
- package/template/.dockerignore +35 -0
- package/template/AGENTS.md +7 -0
- package/template/Dockerfile +60 -0
- package/template/README.md +13 -0
- package/template/agent/agent.ts +25 -8
- package/template/agent/instructions.md +6 -0
- package/template/agent/lib/register-config.ts +5 -1
- package/template/agent/memory/config/domains/org.ts +2 -2
- package/template/agent/model/models.ts +17 -0
- package/template/agent/model/stacks.ts +23 -6
- package/template/agent/settings/settings.ts +21 -2
- package/template/agent/spend/model-pricing.ts +3 -1
- package/template/claude/skills/quickstart-open-everywhere/SKILL.md +96 -9
- package/template/claude/skills/upgrade-open-everywhere/SKILL.md +16 -3
- package/template/deploy/egress-proxy/Dockerfile +30 -0
- package/template/deploy/egress-proxy/control/acl.ts +228 -0
- package/template/deploy/egress-proxy/control/handlers.ts +174 -0
- package/template/deploy/egress-proxy/control/helper.cjs +108 -0
- package/template/deploy/egress-proxy/control/server.ts +122 -0
- package/template/deploy/egress-proxy/entrypoint.sh +62 -0
- package/template/deploy/egress-proxy/gen-ca.sh +42 -0
- package/template/deploy/egress-proxy/install-ca.sh +39 -0
- package/template/deploy/egress-proxy/squid.conf +93 -0
- package/template/deploy/egress-proxy/tier-b.conf +42 -0
- package/template/docker-compose.yml +135 -0
- package/template/gitignore +4 -0
- package/template/package.json +4 -3
- package/template/roster.lock.json +9 -1
- package/template/scripts/goldens/roster-plan.txt +8 -0
- package/template/scripts/model-config-check.ts +81 -0
- package/template/scripts/ops-mcp.ts +1 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: quickstart-open-everywhere
|
|
3
|
-
description: Set up, configure or deploy an Open Everywhere app — from a fresh scaffold to an agent that answers in Slack. Use when the user wants to set up Open Everywhere, deploy it, or connect Slack, Notion or GitHub. Two routes are supported, Vercel and Docker, and step 1 is to ask which. Drives the open-everywhere-ops MCP server (preflight, surface_manifest, scaffold_env, scaffold_credential_key, secret_set/list/rm)
|
|
3
|
+
description: Set up, configure or deploy an Open Everywhere app — from a fresh scaffold to an agent that answers in Slack. Use when the user wants to set up Open Everywhere, deploy it, or connect Slack, Notion or GitHub. Two routes are supported, Vercel and Docker, and step 1 is to ask which. Drives the open-everywhere-ops MCP server (preflight, surface_manifest, surface_app_create, scaffold_env, scaffold_credential_key, secret_set/list/rm, sovereignty), the `@open-everywhere/create --check --json` status report and the `npm run check:model-config` model check.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# Quickstart Open Everywhere
|
|
@@ -79,6 +79,10 @@ To turn it off, `OE_TELEMETRY=off` (or `DO_NOT_TRACK=1`) must be set on the DEPL
|
|
|
79
79
|
`printf off | npx vercel env add OE_TELEMETRY production`. On the Docker route, put it in the
|
|
80
80
|
container's environment. Every boot prints a line that says whether telemetry is on.
|
|
81
81
|
|
|
82
|
+
eve's own CLI telemetry is separate: Open Everywhere sets `EVE_TELEMETRY_DISABLED=1` where it
|
|
83
|
+
starts eve (its npm scripts and `Dockerfile`), and `OE_TELEMETRY` does not change it. For an `eve`
|
|
84
|
+
command run by hand, or an app scaffolded before 0.8.0, read the eve section of the page above.
|
|
85
|
+
|
|
82
86
|
## Status, at every step
|
|
83
87
|
|
|
84
88
|
- `npx @open-everywhere/create --check --json` — the step list as data: `satisfied`, `detail`, and a
|
|
@@ -92,6 +96,56 @@ Re-run both after each step. They are the only answer to "what is left to do": t
|
|
|
92
96
|
tool. The same step list, written for a human, is
|
|
93
97
|
`https://github.com/open-everywhere/open-everywhere/blob/main/docs/deploy/setup-steps.md`.
|
|
94
98
|
|
|
99
|
+
## Check the model config, after every model setting
|
|
100
|
+
|
|
101
|
+
Run `npm run check:model-config` after the model settings are written, and before the deploy that
|
|
102
|
+
carries them. The Vercel route runs it at its build step, and the Docker route at its model route
|
|
103
|
+
step. Run it again after each change to `models` in `agent/settings/settings.ts` (`dataPolicy`,
|
|
104
|
+
`targets`, `force`), to the stack in `agent/model/stacks.ts`, or to a model route of a domain or a
|
|
105
|
+
role. The command is the same in both homes. For each model, it reads the public list of endpoints
|
|
106
|
+
from the model's target: the AI Gateway, unless `models.targets` names another URL. So it needs the
|
|
107
|
+
network and no credential. It changes nothing.
|
|
108
|
+
|
|
109
|
+
Read the result to the user in plain words:
|
|
110
|
+
|
|
111
|
+
- **Exit 0, no `✗` line and no `?` line.** Each model the app can use has a provider that meets
|
|
112
|
+
every demand. Go on.
|
|
113
|
+
- **A `?` line that says "did not answer", "answered HTTP" or "not in the expected shape".** The
|
|
114
|
+
check did not run for that model. This is not a green. Fix the network, then run it again. A
|
|
115
|
+
target URL that "does not serve the endpoints read" (a relay) cannot be checked here: say so.
|
|
116
|
+
- **A `?` line that says a `providers.only` name "serves none of the checked models".** The name is
|
|
117
|
+
most likely a typo. Show it to the user.
|
|
118
|
+
- **A `?` line that says "does not say whether".** The gateway does not publish that fact, so the
|
|
119
|
+
demand is unknown. Name the model and the demand to the user, then go on.
|
|
120
|
+
- **Exit 1, a `✗` line.** Each line names one demand and one model. Read it to the user:
|
|
121
|
+
- `region eu: no provider for deepseek/deepseek-v4.1-flash` (or `zeroRetention`, `noTraining`,
|
|
122
|
+
`providers.only`): no provider meets the demand. Under it every call of that model fails
|
|
123
|
+
closed: the call errors, and it never goes to a provider outside the demand. The choices are a
|
|
124
|
+
stack whose models meet the demand, a wider `providers.only` list, or fewer rows in
|
|
125
|
+
`models.dataPolicy` or in the target's own `dataPolicy`.
|
|
126
|
+
- `models.dataPolicy and target <name> name two different regions`: no call can meet both.
|
|
127
|
+
Keep one region.
|
|
128
|
+
- `the gateway does not know this model`: correct the model id in the stack or the route.
|
|
129
|
+
- `models.targets names no such target`: add the target to `models.targets`, or correct the
|
|
130
|
+
`via` name.
|
|
131
|
+
|
|
132
|
+
**Do not deploy on this red.** Let the user pick the fix, then run the check again. One role
|
|
133
|
+
differs: transcription tries the fallback route that `agent/model/stacks.ts` names when the
|
|
134
|
+
primary route fails. So a red on the primary transcription model sends the audio to the fallback
|
|
135
|
+
route. Check that route too.
|
|
136
|
+
|
|
137
|
+
After the deploy, `--check --host` reads the same check in the `model-route` row of the status
|
|
138
|
+
route. That row names no model, so on a `no` there, run `npm run check:model-config` for the names.
|
|
139
|
+
An app scaffolded before 0.8.0 has no `check:model-config` script: its upgrade notes say how to add it.
|
|
140
|
+
|
|
141
|
+
## Tell the user where their data lives
|
|
142
|
+
|
|
143
|
+
Before the first deploy, offer the data map once: `npx oe sovereignty --host vercel` (or
|
|
144
|
+
`--host docker`), or the `sovereignty` ops tool. For each piece of data it says where it lives,
|
|
145
|
+
which region, and which company could be compelled to hand it over. It reads the app's own config
|
|
146
|
+
and changes nothing. Give the user the short version. Run it again after a change to the model
|
|
147
|
+
settings or the store.
|
|
148
|
+
|
|
95
149
|
## The Vercel route
|
|
96
150
|
|
|
97
151
|
1. **Install.** If `node_modules` is missing: `npm install`. Its `postinstall` runs `oe-eve-patch`,
|
|
@@ -132,7 +186,8 @@ tool. The same step list, written for a human, is
|
|
|
132
186
|
`agent/memory/config/domains/org.ts` set `admins: ["slack/<team-id>/<member-id>"]`. Offer to
|
|
133
187
|
commit both edits. They take effect on the next deploy, so make them before it.
|
|
134
188
|
Then OFFER the optional window step below. Do not make it for the user unasked.
|
|
135
|
-
8. **Build.** `npm run build`. It must pass.
|
|
189
|
+
8. **Build.** `npm run build`. It must pass. Then `npm run check:model-config`, and read it as
|
|
190
|
+
"Check the model config" above says. A `✗` line stops the route here.
|
|
136
191
|
9. **Deployment env.** `.env.local` never ships, so the deployment reads only what you set ON it.
|
|
137
192
|
The names are `OE_CREDENTIAL_ENCRYPTION_KEY`, `SLACK_BOT_TOKEN` and `SLACK_SIGNING_SECRET`. The
|
|
138
193
|
Neon integration already set `DATABASE_URL` there, and that one value resolves both stores — so
|
|
@@ -170,20 +225,28 @@ Then:
|
|
|
170
225
|
1. **Install and build.** `npm install`, then `npm run build`.
|
|
171
226
|
2. **Postgres.** Provision one the container can reach. The user runs `npx oe env set DATABASE_URL`
|
|
172
227
|
and pastes its connection string at the prompt. That one value resolves both stores to postgres;
|
|
173
|
-
set no provider selector.
|
|
228
|
+
set no provider selector. The compose stack of step 6 also carries a Postgres of its own.
|
|
174
229
|
3. **Key.** `scaffold_credential_key`.
|
|
175
230
|
4. **Model route.** There is no OIDC off Vercel, so the user supplies `AI_GATEWAY_API_KEY` (Vercel
|
|
176
231
|
→ AI Gateway → API Keys issues one for any host) with `npx oe env set AI_GATEWAY_API_KEY`. Running a different model backend instead is
|
|
177
232
|
an **adapter swap in app code**
|
|
178
233
|
(`https://github.com/open-everywhere/open-everywhere/blob/main/docs/extending/adapter-model.md`),
|
|
179
234
|
which `--check` cannot see: that step stays `missing` on the Docker route either way, so read it
|
|
180
|
-
and move on rather than chasing a green it cannot give.
|
|
235
|
+
and move on rather than chasing a green it cannot give. On the gateway, run
|
|
236
|
+
`npm run check:model-config` once the model settings are written, and read it as "Check the
|
|
237
|
+
model config" above says. A `✗` line stops the route here.
|
|
181
238
|
5. **Slack app.** Exactly as steps 5–7 of the Vercel route.
|
|
182
|
-
6. **Run it.**
|
|
183
|
-
|
|
184
|
-
`
|
|
185
|
-
|
|
186
|
-
|
|
239
|
+
6. **Run it.** The app ships its own `Dockerfile` and `docker-compose.yml`: the image, the egress
|
|
240
|
+
proxy (`deploy/egress-proxy/`), Postgres and its WebSocket bridge. On the server, the user puts
|
|
241
|
+
the contract's environment in `.env`, with `OE_EGRESS_PROXY_TOKEN` (`openssl rand -hex 32`) and
|
|
242
|
+
a `POSTGRES_PASSWORD`, then runs `docker compose up --build`. The compose file sets
|
|
243
|
+
`DATABASE_URL` to its own `postgres` service. For the managed Postgres of step 2 instead, put
|
|
244
|
+
its `DATABASE_URL` in `.env`, then delete the app's `DATABASE_URL` and `OE_NEON_WS_PROXY` lines
|
|
245
|
+
and the `postgres` and `neon-wsproxy` services from the compose file. Put a TLS-terminating
|
|
246
|
+
reverse proxy in front that forwards `/eve/` and `/.well-known/workflow/`.
|
|
247
|
+
Tell the user plainly: this stack runs the `justbash` sandbox backend, so the agent's shell
|
|
248
|
+
shares the app process's filesystem and environment, every `.env` value included. That is
|
|
249
|
+
development grade, and it does not meet the contract's `sandbox-backend` requirement.
|
|
187
250
|
7. **Prove it, then the Slack URLs and the hello.** As steps 11–13 of the Vercel route, with
|
|
188
251
|
`--route docker --host https://<host>`.
|
|
189
252
|
|
|
@@ -233,6 +296,30 @@ person in chat cannot add one.
|
|
|
233
296
|
With no server declared, nothing changes: the build writes no connection for one, and only the
|
|
234
297
|
shipped connections (Notion) stay mounted.
|
|
235
298
|
|
|
299
|
+
## Optional: connect GitHub
|
|
300
|
+
|
|
301
|
+
Offer this step after the first deploy, when the user wants the agent to work in GitHub. The
|
|
302
|
+
GitHub App is one click: the ops server generates its manifest, and GitHub hands the App's keys
|
|
303
|
+
back to a page on this computer, so the user copies nothing.
|
|
304
|
+
|
|
305
|
+
1. Call `surface_app_create { surface: "github", baseUrl: "https://<host>" }`. Pass
|
|
306
|
+
`owner: "<org login>"` to create the App under an organization. The default App also answers
|
|
307
|
+
@mentions on issues and pull requests; `variant: "source-control"` creates one for checkout,
|
|
308
|
+
pull requests and ticket actions only.
|
|
309
|
+
2. Give the user the `openUrl` from the reply. It is a local page: they open it in a browser on
|
|
310
|
+
this computer. GitHub opens with the App filled in. They check the name (GitHub App names are
|
|
311
|
+
unique, so they change it if GitHub says it is taken) and confirm.
|
|
312
|
+
3. Call `surface_app_create { surface: "github", action: "status" }`. `state: "done"` lists the
|
|
313
|
+
env names written to `.env.local` and the App's install link. Keep the link for step 5.
|
|
314
|
+
4. On the Vercel route, push each name in `written` exactly as in step 9, then redeploy. On the
|
|
315
|
+
Docker route, add each one to the server's `.env` (Docker step 6), then restart the stack.
|
|
316
|
+
5. Only after that redeploy or restart, ask the user to open the install link and install the App
|
|
317
|
+
on the repositories the agent works in. The install sends one `installation` delivery, and the
|
|
318
|
+
deployment learns the installation from it. Before the redeploy the deployment does not hold the
|
|
319
|
+
App's webhook secret, so it refuses that delivery and never learns the installation.
|
|
320
|
+
|
|
321
|
+
The tool never returns a value, and it refuses to overwrite names `.env.local` already holds.
|
|
322
|
+
|
|
236
323
|
## Develop locally
|
|
237
324
|
|
|
238
325
|
Only when the user asks to change or debug the agent — never as a setup step. The user creates
|
|
@@ -55,13 +55,19 @@ nothing. Each entry names:
|
|
|
55
55
|
| CONFIG | a change to this app's own config files. A codemod applies some; the rest are yours. |
|
|
56
56
|
| ENV + / ENV - | an env NAME the deployment starts or stops needing. The admin sets the value. |
|
|
57
57
|
| MANUAL | a step outside the files — a dashboard change, a URL to re-paste, a key to rotate. |
|
|
58
|
+
| DEFAULT | behaviour the release turns on with no config change. It applies at the next deploy. The line says which setting turns it off, if one does. |
|
|
58
59
|
|
|
59
60
|
The installed copy of the record cannot know about a release published after it, so `--plan`
|
|
60
61
|
says plainly which crossed versions it has no entry for. The full window is printed again in
|
|
61
62
|
step 3, after the install puts the target's own copy on disk. The same text is in
|
|
62
63
|
`node_modules/@open-everywhere/core/UPGRADING.md`.
|
|
63
64
|
|
|
64
|
-
|
|
65
|
+
The installed core prints no DEFAULT line for the target. To see the target's new defaults before
|
|
66
|
+
the install, run the target's own `oe`:
|
|
67
|
+
`npx -p @open-everywhere/core@<target> oe upgrade --plan --to <target>`.
|
|
68
|
+
|
|
69
|
+
**Stop and ask the user** when the window holds a breaking change, an env change, a manual step or
|
|
70
|
+
a new default.
|
|
65
71
|
Those are decisions, not steps.
|
|
66
72
|
|
|
67
73
|
`--plan` also prints a **template drift** report. It compares every file the scaffold copied into
|
|
@@ -116,7 +122,12 @@ In order, and it stops at the first step that fails:
|
|
|
116
122
|
4. run the config codemods the target's notes name, and print the migrations that are yours;
|
|
117
123
|
5. `oe build --accept` — this rewrites `roster.lock.json`;
|
|
118
124
|
6. `npm run typecheck`;
|
|
119
|
-
7. `npm run build`.
|
|
125
|
+
7. `npm run build`. eve prepares the sandbox template at build time. Off Vercel an app whose
|
|
126
|
+
`sandbox.selfHostBackend` is `docker` needs a running Docker daemon for that (this template
|
|
127
|
+
declares `justbash`, which needs nothing). When the build fails, `docker version` does not
|
|
128
|
+
answer, and the app's `build` script ends in `eve build`, a core at 0.8.0 or later builds
|
|
129
|
+
again with `npm run build -- --skip-sandbox-prewarm` and says so in one line. eve says that output may not be deployable, so do not start or deploy
|
|
130
|
+
it. A Vercel deploy builds again on Vercel.
|
|
120
131
|
|
|
121
132
|
A failed step prints which one and leaves the tree where it stopped. Fix the cause and run
|
|
122
133
|
`oe upgrade` again: it RESUMES, from whichever step stopped it.
|
|
@@ -158,7 +169,8 @@ thing the agent can do. If a line surprises you, say so to the user before the p
|
|
|
158
169
|
is the review surface, and it is the only one.
|
|
159
170
|
|
|
160
171
|
Then run the app's own checks once more if anything above was fixed by hand:
|
|
161
|
-
`npm run typecheck && npm run build`.
|
|
172
|
+
`npm run typecheck && npm run build`. If the build stops because no Docker daemon runs, add
|
|
173
|
+
`-- --skip-sandbox-prewarm` to the build, as `oe upgrade` does, and do not deploy that output.
|
|
162
174
|
|
|
163
175
|
Then act on the template drift report. A core that carries it prints it again after the install;
|
|
164
176
|
if this run printed no "Template drift" heading, run `npx oe upgrade --to <target> --plan` now to get it. For
|
|
@@ -185,6 +197,7 @@ Commit on a branch, never on the default branch. The body carries:
|
|
|
185
197
|
- what changed in `roster.lock.json`, in your own words;
|
|
186
198
|
- **the env names the deployment needs before this merges**, names only;
|
|
187
199
|
- the manual steps, and who does each one.
|
|
200
|
+
- each new default, and the setting the user chose for it, or that no setting turns it off.
|
|
188
201
|
- the template files you synced or deleted, and each SYNC or DELETE line you left as it is, with the reason.
|
|
189
202
|
|
|
190
203
|
Then stop. The admin reviews the lock diff and merges. The merge deploys.
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# Open Everywhere egress proxy — Squid (Tier A/B) + the Node policy control service.
|
|
2
|
+
# The SOLE container with a WAN route in docker-compose.yml; the app reaches the internet
|
|
3
|
+
# only by proxying through here. See deploy/egress-proxy/README.md.
|
|
4
|
+
#
|
|
5
|
+
# Base on node:24-alpine (NOT plain alpine, whose `nodejs` package is too old for
|
|
6
|
+
# `--experimental-strip-types`). Node 24 runs the TypeScript control service directly via
|
|
7
|
+
# native type stripping — no build step. node:24-alpine puts node at /usr/local/bin; the
|
|
8
|
+
# squid external_acl helper references /usr/bin/node, so we symlink it below.
|
|
9
|
+
FROM node:24-alpine
|
|
10
|
+
|
|
11
|
+
# squid = the forward proxy + ssl_bump; openssl = CA generation for Tier B; wget = the
|
|
12
|
+
# entrypoint health probe. (nodejs comes from the base image.)
|
|
13
|
+
RUN apk add --no-cache squid openssl wget ca-certificates \
|
|
14
|
+
&& ln -sf "$(command -v node)" /usr/bin/node \
|
|
15
|
+
&& mkdir -p /opt/oe-egress/control /var/lib/squid /opt/oe-egress/ca
|
|
16
|
+
|
|
17
|
+
WORKDIR /opt/oe-egress
|
|
18
|
+
|
|
19
|
+
# Config + scripts + control service (kept alongside so the entrypoint copies a writable
|
|
20
|
+
# squid.conf and can drop the Tier B overlay in conditionally).
|
|
21
|
+
COPY squid.conf tier-b.conf gen-ca.sh install-ca.sh entrypoint.sh ./
|
|
22
|
+
COPY control/ ./control/
|
|
23
|
+
RUN chmod +x gen-ca.sh install-ca.sh entrypoint.sh
|
|
24
|
+
|
|
25
|
+
# 3128 = the forward proxy the app points HTTPS_PROXY at. The control service (8080) is
|
|
26
|
+
# reachable on the internal network so the app can POST policy; the ACL helper hits it on
|
|
27
|
+
# loopback. Not exposed to the host.
|
|
28
|
+
EXPOSE 3128 8080
|
|
29
|
+
|
|
30
|
+
ENTRYPOINT ["/bin/sh", "/opt/oe-egress/entrypoint.sh"]
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
// Pure ACL logic for the Open Everywhere egress-proxy control service.
|
|
2
|
+
//
|
|
3
|
+
// This is the enforcement decision the Squid `external_acl_type` helper calls per request:
|
|
4
|
+
// given a source IP + destination host (+ method/path when the request is decrypted in
|
|
5
|
+
// Tier B), decide allow/deny against the compiled SandboxNetworkPolicy the app PUSHED for
|
|
6
|
+
// that source, and — in Tier B — return the credential-injection headers to stamp.
|
|
7
|
+
//
|
|
8
|
+
// Deliberately dependency-free (no eve, no npm): it is imported by server.ts, unit-tested
|
|
9
|
+
// via tsx (acl-test.ts), and runs under Node's native TS stripping in the proxy container.
|
|
10
|
+
//
|
|
11
|
+
// The policy shape mirrors eve's SandboxNetworkPolicy (Vercel NetworkPolicy) — a structural
|
|
12
|
+
// copy so this module needs no eve import. See
|
|
13
|
+
// node_modules/eve/dist/src/compiled/@vercel/sandbox/network-policy.d.ts.
|
|
14
|
+
|
|
15
|
+
export type NetworkTransformer = { headers?: Record<string, string> };
|
|
16
|
+
export type PolicyMatch = {
|
|
17
|
+
path?: { exact?: string; startsWith?: string; regex?: string };
|
|
18
|
+
method?: string[];
|
|
19
|
+
};
|
|
20
|
+
export type PolicyRule = { match?: PolicyMatch; transform?: NetworkTransformer[] };
|
|
21
|
+
export type EgressPolicy =
|
|
22
|
+
| "allow-all"
|
|
23
|
+
| "deny-all"
|
|
24
|
+
| {
|
|
25
|
+
allow?: string[] | Record<string, PolicyRule[]>;
|
|
26
|
+
subnets?: { allow?: string[]; deny?: string[] };
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
export interface AclRequest {
|
|
30
|
+
/** Destination host (SNI for Tier A peek, CONNECT target, or Host header for Tier B). */
|
|
31
|
+
host: string;
|
|
32
|
+
/** HTTP method — only known in Tier B (decrypted). Undefined ⇒ Tier A, method un-checkable. */
|
|
33
|
+
method?: string;
|
|
34
|
+
/** Request path — only known in Tier B. Undefined ⇒ Tier A, path un-checkable. */
|
|
35
|
+
path?: string;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface AclDecision {
|
|
39
|
+
readonly allow: boolean;
|
|
40
|
+
/** The allow-map key that matched (for logging). */
|
|
41
|
+
readonly matchedHost?: string;
|
|
42
|
+
/** Tier B credential-injection headers from the first matching rule's transforms. */
|
|
43
|
+
readonly headers?: Record<string, string>;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const DENY: AclDecision = { allow: false };
|
|
47
|
+
|
|
48
|
+
/** Normalize a host for matching: lowercase, drop a trailing dot, strip any :port. */
|
|
49
|
+
export function normalizeHost(host: string): string {
|
|
50
|
+
let h = host.trim().toLowerCase();
|
|
51
|
+
if (h.endsWith(".")) h = h.slice(0, -1);
|
|
52
|
+
// Strip a trailing :port, but leave bare IPv6 (no port) alone. CONNECT targets are
|
|
53
|
+
// "host:port"; SNI has no port. Only strip when there's a single trailing :digits.
|
|
54
|
+
const m = h.match(/^(.*):(\d+)$/);
|
|
55
|
+
if (m && !h.includes("::")) h = m[1];
|
|
56
|
+
// Strip IPv6 brackets if present ("[::1]:443" → "::1").
|
|
57
|
+
if (h.startsWith("[") && h.includes("]")) h = h.slice(1, h.indexOf("]"));
|
|
58
|
+
return h;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Normalize a source IP key: strip IPv4-mapped-IPv6 prefix so "::ffff:172.20.0.3" == "172.20.0.3". */
|
|
62
|
+
export function normalizeSourceIp(ip: string): string {
|
|
63
|
+
const v = ip.trim().toLowerCase();
|
|
64
|
+
return v.startsWith("::ffff:") ? v.slice("::ffff:".length) : v;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Does an allow-map key match a destination host? Supports Vercel's `*` wildcard prefix
|
|
69
|
+
* (`*.npmjs.org` matches any subdomain of npmjs.org) and a bare `*` (match everything).
|
|
70
|
+
* A plain key matches its exact host only.
|
|
71
|
+
*/
|
|
72
|
+
export function hostKeyMatches(key: string, host: string): boolean {
|
|
73
|
+
const k = normalizeHost(key);
|
|
74
|
+
const h = normalizeHost(host);
|
|
75
|
+
if (k === "*") return true;
|
|
76
|
+
if (k.startsWith("*.")) {
|
|
77
|
+
const suffix = k.slice(1); // ".npmjs.org"
|
|
78
|
+
return h.endsWith(suffix) && h.length > suffix.length;
|
|
79
|
+
}
|
|
80
|
+
return k === h;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** Does a single rule permit this request? method/path constraints are SKIPPED when the
|
|
84
|
+
* corresponding request field is undefined (Tier A can't see them — host granularity only). */
|
|
85
|
+
function ruleAllows(rule: PolicyRule, req: AclRequest): boolean {
|
|
86
|
+
const m = rule.match;
|
|
87
|
+
if (!m) return true;
|
|
88
|
+
if (m.method && m.method.length > 0 && req.method !== undefined) {
|
|
89
|
+
const want = m.method.map((x) => x.toUpperCase());
|
|
90
|
+
if (!want.includes(req.method.toUpperCase())) return false;
|
|
91
|
+
}
|
|
92
|
+
if (m.path && req.path !== undefined) {
|
|
93
|
+
const p = req.path;
|
|
94
|
+
if (m.path.exact !== undefined && p !== m.path.exact) return false;
|
|
95
|
+
if (m.path.startsWith !== undefined && !p.startsWith(m.path.startsWith)) return false;
|
|
96
|
+
if (m.path.regex !== undefined) {
|
|
97
|
+
try {
|
|
98
|
+
if (!new RegExp(m.path.regex).test(p)) return false;
|
|
99
|
+
} catch {
|
|
100
|
+
return false; // an un-compilable regex fails closed
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return true;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function mergeHeaders(rule: PolicyRule): Record<string, string> | undefined {
|
|
108
|
+
if (!rule.transform || rule.transform.length === 0) return undefined;
|
|
109
|
+
const out: Record<string, string> = {};
|
|
110
|
+
for (const t of rule.transform) {
|
|
111
|
+
if (t.headers) for (const [k, v] of Object.entries(t.headers)) out[k] = v;
|
|
112
|
+
}
|
|
113
|
+
return Object.keys(out).length > 0 ? out : undefined;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Decide allow/deny for `req` against `policy`. FAIL-CLOSED: an unknown host, a policy with
|
|
118
|
+
* no allow map, or "deny-all" all deny. "allow-all" allows. For the record form, the first
|
|
119
|
+
* rule that permits the request wins (its transform headers ride back for Tier B injection);
|
|
120
|
+
* an empty rules array is a bare allow (no headers).
|
|
121
|
+
*/
|
|
122
|
+
export function decide(policy: EgressPolicy | undefined, req: AclRequest): AclDecision {
|
|
123
|
+
if (policy === undefined) return DENY; // no policy pushed for this source yet
|
|
124
|
+
if (policy === "deny-all") return DENY;
|
|
125
|
+
if (policy === "allow-all") return { allow: true };
|
|
126
|
+
const allow = policy.allow;
|
|
127
|
+
if (!allow) return DENY;
|
|
128
|
+
|
|
129
|
+
if (Array.isArray(allow)) {
|
|
130
|
+
// Simple string list — host-level only, no credential injection.
|
|
131
|
+
for (const key of allow) {
|
|
132
|
+
if (hostKeyMatches(key, req.host)) return { allow: true, matchedHost: normalizeHost(key) };
|
|
133
|
+
}
|
|
134
|
+
return DENY;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// Record form: find the matching host key, then a permitting rule under it.
|
|
138
|
+
for (const [key, rules] of Object.entries(allow)) {
|
|
139
|
+
if (!hostKeyMatches(key, req.host)) continue;
|
|
140
|
+
if (!rules || rules.length === 0) return { allow: true, matchedHost: normalizeHost(key) };
|
|
141
|
+
for (const rule of rules) {
|
|
142
|
+
if (ruleAllows(rule, req)) {
|
|
143
|
+
return { allow: true, matchedHost: normalizeHost(key), headers: mergeHeaders(rule) };
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
// Host is listed but no rule permitted this specific method/path (Tier B narrowing).
|
|
147
|
+
return DENY;
|
|
148
|
+
}
|
|
149
|
+
return DENY;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* In-memory policy store keyed by (normalized) source IP. The control service holds one
|
|
154
|
+
* compiled policy per active sandbox source. Unknown source ⇒ deny (fail-closed) — the
|
|
155
|
+
* internal-only Docker network is the structural backstop; this is the runtime floor.
|
|
156
|
+
*/
|
|
157
|
+
export class PolicyStore {
|
|
158
|
+
private readonly byIp = new Map<string, EgressPolicy>();
|
|
159
|
+
/** The APP's own allow list (OPE-2406), kept apart from every per-source session policy. See {@link validateHostPlanePolicy}. */
|
|
160
|
+
private hostPlanePolicy: EgressPolicy | undefined;
|
|
161
|
+
|
|
162
|
+
setHostPlane(policy: EgressPolicy): void {
|
|
163
|
+
this.hostPlanePolicy = policy;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/** Undefined until the app pushes one, which {@link decide} reads as deny. */
|
|
167
|
+
hostPlane(): EgressPolicy | undefined {
|
|
168
|
+
return this.hostPlanePolicy;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
set(sourceIp: string, policy: EgressPolicy): void {
|
|
172
|
+
this.byIp.set(normalizeSourceIp(sourceIp), policy);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
get(sourceIp: string): EgressPolicy | undefined {
|
|
176
|
+
return this.byIp.get(normalizeSourceIp(sourceIp));
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
delete(sourceIp: string): boolean {
|
|
180
|
+
return this.byIp.delete(normalizeSourceIp(sourceIp));
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
size(): number {
|
|
184
|
+
return this.byIp.size;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** Decide for a source+request in one call (fail-closed on unknown source). */
|
|
188
|
+
decide(sourceIp: string, req: AclRequest): AclDecision {
|
|
189
|
+
return decide(this.get(sourceIp), req);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* Validate a pushed policy body just enough to reject garbage before it becomes the ACL
|
|
195
|
+
* for a source. Returns the policy or throws with a reason (the server turns it into 400).
|
|
196
|
+
*/
|
|
197
|
+
export function validatePolicy(input: unknown): EgressPolicy {
|
|
198
|
+
if (input === "allow-all" || input === "deny-all") return input;
|
|
199
|
+
if (input && typeof input === "object") {
|
|
200
|
+
const obj = input as Record<string, unknown>;
|
|
201
|
+
const allow = obj.allow;
|
|
202
|
+
if (allow === undefined || Array.isArray(allow) || (allow && typeof allow === "object")) {
|
|
203
|
+
return input as EgressPolicy;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
throw new Error('policy must be "allow-all", "deny-all", or an object with an `allow` list/map');
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Validate a HOST-PLANE policy (OPE-2406): the hosts the APP process itself calls, for example the
|
|
211
|
+
* model gateway. It is narrower than {@link validatePolicy} on purpose. Only a list of exact hosts
|
|
212
|
+
* passes: no "allow-all", no wildcard, no rule map (so no Tier B credential injection on this plane).
|
|
213
|
+
* The app is the only writer, because the push needs the control token. The narrow shape keeps a
|
|
214
|
+
* wrong push from opening the whole internet to every request that carries the credential.
|
|
215
|
+
*/
|
|
216
|
+
export function validateHostPlanePolicy(input: unknown): EgressPolicy {
|
|
217
|
+
const allow = input && typeof input === "object" ? (input as Record<string, unknown>).allow : undefined;
|
|
218
|
+
if (!Array.isArray(allow) || allow.length === 0) {
|
|
219
|
+
throw new Error("host-plane policy must be an object with a non-empty `allow` list of exact hosts");
|
|
220
|
+
}
|
|
221
|
+
const hosts: string[] = [];
|
|
222
|
+
for (const h of allow) {
|
|
223
|
+
if (typeof h !== "string" || h.trim() === "") throw new Error("host-plane `allow` entries must be non-empty host strings");
|
|
224
|
+
if (h.includes("*")) throw new Error(`host-plane hosts are exact hosts, never a wildcard ("${h}")`);
|
|
225
|
+
hosts.push(normalizeHost(h));
|
|
226
|
+
}
|
|
227
|
+
return { allow: hosts };
|
|
228
|
+
}
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
// Pure request handlers for the control service — the policy-push handler, the ACL query
|
|
2
|
+
// handler, and the two auth primitives (boot-time token requirement + bearer comparison),
|
|
3
|
+
// factored out of the HTTP plumbing (server.ts) so all are unit-testable without opening a
|
|
4
|
+
// socket (acl-test.ts).
|
|
5
|
+
|
|
6
|
+
import { timingSafeEqual } from "node:crypto";
|
|
7
|
+
import { PolicyStore, decide, validatePolicy, validateHostPlanePolicy, normalizeSourceIp, type AclRequest, type AclDecision } from "./acl.ts";
|
|
8
|
+
|
|
9
|
+
export interface PushInput {
|
|
10
|
+
/** Parsed JSON body of POST /policy. */
|
|
11
|
+
readonly body: unknown;
|
|
12
|
+
/** The connection's remote address (the fallback source-IP key). */
|
|
13
|
+
readonly remoteAddr: string;
|
|
14
|
+
/** Whether the request carried the valid bearer token. Auth is never disabled — the control
|
|
15
|
+
* service refuses to boot without a token (see requireControlToken). */
|
|
16
|
+
readonly authOk: boolean;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* The fatal message printed when the control service is started without a token. Kept next to
|
|
21
|
+
* requireControlToken so the test can pin the wording that operators will actually see.
|
|
22
|
+
*/
|
|
23
|
+
export const MISSING_TOKEN_FATAL =
|
|
24
|
+
"OE_EGRESS_PROXY_TOKEN is required — the control service writes the egress policy that is the " +
|
|
25
|
+
"only thing standing between the sandbox and the open internet, and it listens on the same " +
|
|
26
|
+
"network the sandbox runs on. Refusing to start without it. Generate one with " +
|
|
27
|
+
"`openssl rand -hex 32` and set it on BOTH the proxy and the app (docker-compose.yml passes " +
|
|
28
|
+
"it to both).";
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Resolve the control-plane bearer token, or throw. There is deliberately NO tokenless mode and
|
|
32
|
+
* no opt-out flag: this proxy exists only for self-host deploys, where an unauthenticated
|
|
33
|
+
* `POST /policy` lets any peer on the internal network (including a compromised sandbox) install
|
|
34
|
+
* an allow-all policy for itself and defeat default-deny egress entirely. Fail loud at boot
|
|
35
|
+
* rather than silently ship an open control plane.
|
|
36
|
+
*/
|
|
37
|
+
export function requireControlToken(raw: string | undefined): string {
|
|
38
|
+
const token = raw?.trim();
|
|
39
|
+
if (!token) throw new Error(MISSING_TOKEN_FATAL);
|
|
40
|
+
return token;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Constant-time bearer-token check for the `Authorization` header. Length is compared first
|
|
45
|
+
* (timingSafeEqual throws on a length mismatch), so this leaks the token's length and nothing
|
|
46
|
+
* else. Returns false for a missing/duplicated/non-Bearer header — fail-closed.
|
|
47
|
+
*/
|
|
48
|
+
export function bearerMatches(header: string | string[] | undefined, token: string): boolean {
|
|
49
|
+
if (typeof header !== "string") return false;
|
|
50
|
+
const prefix = "Bearer ";
|
|
51
|
+
if (!header.startsWith(prefix)) return false;
|
|
52
|
+
const got = Buffer.from(header.slice(prefix.length), "utf-8");
|
|
53
|
+
const want = Buffer.from(token, "utf-8");
|
|
54
|
+
if (got.length !== want.length) return false;
|
|
55
|
+
return timingSafeEqual(got, want);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export interface PushResult {
|
|
59
|
+
readonly status: number;
|
|
60
|
+
readonly body: unknown;
|
|
61
|
+
/** The source-IP key the policy was stored under (undefined on rejection). */
|
|
62
|
+
readonly keyed?: string;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Apply a policy push. Keys by the body's explicit `sourceIp` when present, else the
|
|
67
|
+
* connection's remote address (correct for the justbash topology, where the app pushes
|
|
68
|
+
* from the very IP its egress uses). Fail-closed on auth/validation.
|
|
69
|
+
*
|
|
70
|
+
* `plane: "host"` writes the HOST-PLANE policy instead (OPE-2406): the app's own allow list,
|
|
71
|
+
* which only a request that carries the control credential is decided against (see
|
|
72
|
+
* {@link handleAcl}). It is never keyed by source IP, so it never reaches a session. It travels
|
|
73
|
+
* in `hostPlane`, never in `policy`: a proxy built before the host plane validates `policy`
|
|
74
|
+
* and so refuses this body (400) instead of storing the list as the app IP's session policy.
|
|
75
|
+
*/
|
|
76
|
+
export function handlePolicyPush(store: PolicyStore, input: PushInput): PushResult {
|
|
77
|
+
if (!input.authOk) return { status: 401, body: { error: "unauthorized" } };
|
|
78
|
+
const body = input.body;
|
|
79
|
+
if (!body || typeof body !== "object") {
|
|
80
|
+
return { status: 400, body: { error: "expected a JSON object body" } };
|
|
81
|
+
}
|
|
82
|
+
const obj = body as Record<string, unknown>;
|
|
83
|
+
if (obj.plane !== undefined && obj.plane !== "host" && obj.plane !== "session") {
|
|
84
|
+
return { status: 400, body: { error: 'plane must be "host" or "session"' } };
|
|
85
|
+
}
|
|
86
|
+
if (obj.plane === "host") {
|
|
87
|
+
if (obj.policy !== undefined) return { status: 400, body: { error: "a host-plane push carries `hostPlane`, never `policy`" } };
|
|
88
|
+
let hostPolicy;
|
|
89
|
+
try {
|
|
90
|
+
hostPolicy = validateHostPlanePolicy(obj.hostPlane);
|
|
91
|
+
} catch (err) {
|
|
92
|
+
return { status: 400, body: { error: String(err instanceof Error ? err.message : err) } };
|
|
93
|
+
}
|
|
94
|
+
store.setHostPlane(hostPolicy);
|
|
95
|
+
return { status: 200, body: { ok: true, plane: "host", policies: store.size() }, keyed: "host-plane" };
|
|
96
|
+
}
|
|
97
|
+
let policy;
|
|
98
|
+
try {
|
|
99
|
+
policy = validatePolicy(obj.policy);
|
|
100
|
+
} catch (err) {
|
|
101
|
+
return { status: 400, body: { error: String(err instanceof Error ? err.message : err) } };
|
|
102
|
+
}
|
|
103
|
+
const explicit = typeof obj.sourceIp === "string" && obj.sourceIp.trim() ? obj.sourceIp.trim() : undefined;
|
|
104
|
+
const key = explicit ?? input.remoteAddr;
|
|
105
|
+
if (!key) return { status: 400, body: { error: "no source IP: set body.sourceIp or connect from the sandbox network" } };
|
|
106
|
+
store.set(key, policy);
|
|
107
|
+
return { status: 200, body: { ok: true, sourceIp: key, policies: store.size() }, keyed: key };
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
export interface AclQuery {
|
|
111
|
+
src?: string;
|
|
112
|
+
host?: string;
|
|
113
|
+
method?: string;
|
|
114
|
+
path?: string;
|
|
115
|
+
/** The request's `Proxy-Authorization` value, as Squid passed it. Absent ⇒ a session request. */
|
|
116
|
+
auth?: string;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export interface AclResult {
|
|
120
|
+
readonly decision: AclDecision;
|
|
121
|
+
/** True iff the query was well-formed enough to evaluate. Malformed ⇒ fail-closed deny. */
|
|
122
|
+
readonly evaluated: boolean;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Evaluate an ACL query from the Squid helper. Missing src/host ⇒ fail-closed deny (a
|
|
127
|
+
* malformed callout must never open the firewall). "-" is Squid's empty-field sentinel.
|
|
128
|
+
*
|
|
129
|
+
* 🔴 TWO PRINCIPALS, ONE SOURCE IP (OPE-2406). Under the justbash backend the app process and
|
|
130
|
+
* the agent's sandboxed session send from the same address, so the address cannot tell them
|
|
131
|
+
* apart. The credential does: the app sends `Proxy-Authorization: Bearer <control token>` on its
|
|
132
|
+
* own model call. The token stays in the app's environment, outside the session's shell, and a
|
|
133
|
+
* session that obtained it could already push itself any policy, so this adds no reach. So:
|
|
134
|
+
* - no credential → the SESSION policy for the source IP, exactly as before;
|
|
135
|
+
* - the credential → the HOST-PLANE policy only, never the session policy;
|
|
136
|
+
* - any other credential → deny. A wrong token never falls back to the session policy.
|
|
137
|
+
* The host plane never widens what a session reaches: a session request carries no credential,
|
|
138
|
+
* so the host-plane list is never consulted for it.
|
|
139
|
+
*/
|
|
140
|
+
export function handleAcl(store: PolicyStore, q: AclQuery, token?: string): AclResult {
|
|
141
|
+
const src = clean(q.src);
|
|
142
|
+
const host = clean(q.host);
|
|
143
|
+
if (!src || !host) return { decision: { allow: false }, evaluated: false };
|
|
144
|
+
const req: AclRequest = { host };
|
|
145
|
+
const method = clean(q.method);
|
|
146
|
+
const path = clean(q.path);
|
|
147
|
+
if (method) req.method = method;
|
|
148
|
+
if (path) req.path = path;
|
|
149
|
+
const auth = clean(q.auth);
|
|
150
|
+
if (auth !== undefined) {
|
|
151
|
+
if (token === undefined || !bearerMatches(auth, token)) return { decision: { allow: false }, evaluated: true };
|
|
152
|
+
// Host grain only: the host plane carries no rule map, so method/path cannot narrow or inject.
|
|
153
|
+
return { decision: decide(store.hostPlane(), { host }), evaluated: true };
|
|
154
|
+
}
|
|
155
|
+
return { decision: store.decide(src, req), evaluated: true };
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Is a connection's remote address loopback (127.0.0.0/8 or ::1)? Gates /acl, which returns
|
|
160
|
+
* Tier B credential-injection headers and must only ever answer the in-container Squid helper
|
|
161
|
+
* (which calls over 127.0.0.1). A cross-container peer arrives under its network IP, never
|
|
162
|
+
* loopback, so it is refused. Empty/unknown ⇒ not loopback (fail-closed).
|
|
163
|
+
*/
|
|
164
|
+
export function isLoopbackAddr(addr: string | undefined): boolean {
|
|
165
|
+
if (!addr) return false;
|
|
166
|
+
const a = normalizeSourceIp(addr); // lowercases + strips the ::ffff: ipv4-mapped prefix
|
|
167
|
+
return a === "::1" || /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(a);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
function clean(v: string | undefined): string | undefined {
|
|
171
|
+
if (v === undefined) return undefined;
|
|
172
|
+
const t = v.trim();
|
|
173
|
+
return t === "" || t === "-" ? undefined : t;
|
|
174
|
+
}
|