@lanes-sh/link 0.6.8 → 0.6.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.gcloudignore +50 -0
- package/README.md +5 -0
- package/package.json +4 -3
- package/src/auth/oidc.ts +65 -12
- package/src/cli/brand.ts +16 -9
- package/src/cli/commands/operate/auth.ts +333 -0
- package/src/cli/commands/operate/desktop.ts +220 -0
- package/src/cli/commands/operate/findings.ts +9 -38
- package/src/cli/commands/operate/inspect.ts +59 -40
- package/src/cli/commands/operate/serve.ts +0 -3
- package/src/cli/commands/operate.ts +5 -3
- package/src/cli/main.ts +19 -4
- package/src/cli/selection.ts +15 -4
- package/src/cli/usage.ts +5 -1
- package/src/connectivity/auth/index.ts +1 -0
- package/src/connectivity/auth/oauth-authcode/provider.ts +17 -1
- package/src/connectivity/auth/oauth-authcode/refresh.ts +43 -11
- package/src/connectivity/auth/oauth-jwt/index.ts +12 -1
- package/src/connectivity/auth/reauth.ts +48 -0
- package/src/deployments/gcp/Dockerfile +27 -2
- package/src/deployments/gcp/bucket.ts +82 -9
- package/src/deployments/gcp/driver.ts +43 -5
- package/src/deployments/gcp/lifecycle.json +12 -0
- package/src/deployments/gcp/survey.ts +12 -1
- package/src/policy/limits.ts +76 -3
- package/src/profile/index.ts +1 -0
- package/src/profile/legacy.ts +8 -3
- package/src/profile/schema.ts +65 -0
- package/src/server/cors.ts +3 -3
- package/src/server/edge.ts +188 -1
- package/src/server/endpoint.ts +0 -17
- package/src/server/harness.ts +10 -7
- package/src/server/index.ts +62 -90
- package/src/server/mcp/index.ts +0 -1
- package/src/server/mcp/visibility.ts +0 -33
- package/src/server/oauth.ts +21 -2
- package/src/cli/commands/operate/dashboard.ts +0 -107
- package/src/cli/dashboard-page.ts +0 -293
- package/src/cli/dashboard-shell.ts +0 -125
- package/src/cli/provider-marks.ts +0 -45
- package/src/server/dashboard.ts +0 -212
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { ProviderManifest } from '#connectivity';
|
|
2
2
|
import type { SecretStore } from '#secrets';
|
|
3
3
|
import { BROKERED, BrokerError, brokerRefresh } from './broker.ts';
|
|
4
|
+
import { ReauthRequired, statusMeansGrantIsDead } from '../reauth.ts';
|
|
4
5
|
import type { CredentialOAuthProvider } from './provider.ts';
|
|
5
6
|
|
|
6
7
|
/** What a stored OAuth credential carries beyond the tokens themselves. */
|
|
@@ -23,7 +24,10 @@ export async function refreshDirectly(
|
|
|
23
24
|
const refreshToken = existing?.refresh_token;
|
|
24
25
|
|
|
25
26
|
if (!refreshToken) {
|
|
26
|
-
|
|
27
|
+
// Nothing to renew with, so this can only be settled by signing in again —
|
|
28
|
+
// the same remedy as a dead grant, and reported as the same thing.
|
|
29
|
+
throw new ReauthRequired(
|
|
30
|
+
`${manifest.id}.${provider.connectionId}`,
|
|
27
31
|
`No refresh token stored for ${manifest.id}. Connecting it again for this profile and target would store one.`,
|
|
28
32
|
);
|
|
29
33
|
}
|
|
@@ -38,9 +42,19 @@ export async function refreshDirectly(
|
|
|
38
42
|
const broker = auth?.broker;
|
|
39
43
|
const brokered = broker !== undefined && existing?.authorized_via === BROKERED;
|
|
40
44
|
|
|
45
|
+
const connectionKey = `${manifest.id}.${provider.connectionId}`;
|
|
46
|
+
|
|
41
47
|
const refreshed = brokered
|
|
42
|
-
? await viaBroker(manifest, broker.url, refreshToken, existing, fetchImpl)
|
|
43
|
-
: await viaStoredClient(
|
|
48
|
+
? await viaBroker(manifest, connectionKey, broker.url, refreshToken, existing, fetchImpl)
|
|
49
|
+
: await viaStoredClient(
|
|
50
|
+
manifest,
|
|
51
|
+
connectionKey,
|
|
52
|
+
auth?.app,
|
|
53
|
+
tokenUrl,
|
|
54
|
+
refreshToken,
|
|
55
|
+
credentials,
|
|
56
|
+
fetchImpl,
|
|
57
|
+
);
|
|
44
58
|
|
|
45
59
|
// `existing` first, so what the response does not mention survives it. Neither
|
|
46
60
|
// the vendor nor the broker echoes `refresh_token`, `id_token`, or
|
|
@@ -53,6 +67,7 @@ export async function refreshDirectly(
|
|
|
53
67
|
|
|
54
68
|
async function viaBroker(
|
|
55
69
|
manifest: ProviderManifest,
|
|
70
|
+
connectionKey: string,
|
|
56
71
|
url: string,
|
|
57
72
|
refreshToken: string,
|
|
58
73
|
existing: StoredTokens,
|
|
@@ -70,17 +85,27 @@ async function viaBroker(
|
|
|
70
85
|
// next step. Same shape as the stored-client message below, deliberately:
|
|
71
86
|
// where the credential came from is not the reader's problem here.
|
|
72
87
|
const notice = cause instanceof BrokerError && cause.notice ? `\n${cause.notice}` : '';
|
|
73
|
-
|
|
88
|
+
const message =
|
|
74
89
|
`The credential for ${manifest.id} could not be refreshed. ` +
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
90
|
+
`Re-authorise ${manifest.id} for this profile and target.\n${String(
|
|
91
|
+
cause instanceof Error ? cause.message : cause,
|
|
92
|
+
).slice(0, 200)}${notice}`;
|
|
93
|
+
|
|
94
|
+
// Only the broker refusing *this* credential means a person is needed. A
|
|
95
|
+
// broker that is down, or rate-limiting, says nothing about the grant, and
|
|
96
|
+
// reporting it as "sign in again" would send someone through a consent
|
|
97
|
+
// screen to fix an outage. A `cause` that is not a `BrokerError` never
|
|
98
|
+
// reached the broker at all, so it is the same case.
|
|
99
|
+
if (cause instanceof BrokerError && statusMeansGrantIsDead(cause.status)) {
|
|
100
|
+
throw new ReauthRequired(connectionKey, message);
|
|
101
|
+
}
|
|
102
|
+
throw new Error(message);
|
|
79
103
|
}
|
|
80
104
|
}
|
|
81
105
|
|
|
82
106
|
async function viaStoredClient(
|
|
83
107
|
manifest: ProviderManifest,
|
|
108
|
+
connectionKey: string,
|
|
84
109
|
app: string | undefined,
|
|
85
110
|
tokenUrl: string,
|
|
86
111
|
refreshToken: string,
|
|
@@ -108,10 +133,17 @@ async function viaStoredClient(
|
|
|
108
133
|
if (!response.ok) {
|
|
109
134
|
// A revoked or expired refresh token is the common case here, and the fix
|
|
110
135
|
// is always the same, so say it rather than surfacing the raw grant error.
|
|
111
|
-
|
|
136
|
+
const message =
|
|
112
137
|
`The credential for ${manifest.id} could not be refreshed (${response.status}). ` +
|
|
113
|
-
|
|
114
|
-
|
|
138
|
+
`Re-authorise ${manifest.id} for this profile and target.\n${text.slice(0, 200)}`;
|
|
139
|
+
|
|
140
|
+
// 4xx is the authorization server rejecting this credential; 5xx is it
|
|
141
|
+
// being unwell. Only the first is something a person can fix, and only the
|
|
142
|
+
// first may be reported as such.
|
|
143
|
+
if (statusMeansGrantIsDead(response.status)) {
|
|
144
|
+
throw new ReauthRequired(connectionKey, message);
|
|
145
|
+
}
|
|
146
|
+
throw new Error(message);
|
|
115
147
|
}
|
|
116
148
|
|
|
117
149
|
return JSON.parse(text) as Record<string, unknown>;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { ProviderManifest } from '#connectivity';
|
|
2
|
+
import { ReauthRequired, statusMeansGrantIsDead } from '../reauth.ts';
|
|
2
3
|
import type { SecretStore } from '#secrets';
|
|
3
4
|
import { credentialRefForConnection } from '../../manifest/credential-ref.ts';
|
|
4
5
|
import { parseAssertionKey, signAssertion } from './key.ts';
|
|
@@ -170,7 +171,17 @@ export async function resolveAssertionToken(input: {
|
|
|
170
171
|
const body = (await response.json().catch(() => ({}))) as TokenResponse;
|
|
171
172
|
|
|
172
173
|
if (!response.ok || !body.access_token) {
|
|
173
|
-
|
|
174
|
+
const message = refusalMessage(manifest, stored, body, response.status);
|
|
175
|
+
|
|
176
|
+
// A key is refused for the same two reasons a refresh token is: the grant
|
|
177
|
+
// behind it is gone, or the token endpoint is unwell. The remedy differs
|
|
178
|
+
// from a consent screen — it is an admin grant in a console, or `connect
|
|
179
|
+
// --replace` — but "the owner has to re-authorise this connection" is the
|
|
180
|
+
// same claim, and `refusalMessage` already writes the specific sentence.
|
|
181
|
+
if (statusMeansGrantIsDead(response.status)) {
|
|
182
|
+
throw new ReauthRequired(`${manifest.id}.${connectionId}`, message);
|
|
183
|
+
}
|
|
184
|
+
throw new Error(message);
|
|
174
185
|
}
|
|
175
186
|
|
|
176
187
|
minted.set(cacheKey, {
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The one failure a person has to fix, told apart from every other one.
|
|
3
|
+
*
|
|
4
|
+
* A stored credential stops working for two very different reasons, and until
|
|
5
|
+
* this existed they arrived as the same `Error`:
|
|
6
|
+
*
|
|
7
|
+
* - the grant is gone — revoked, or expired because the client's publishing
|
|
8
|
+
* status expires refresh tokens on a timer. Nothing retries its way out of
|
|
9
|
+
* this; somebody has to sign in again.
|
|
10
|
+
* - the token endpoint had a bad afternoon — a 502, a reset connection, DNS.
|
|
11
|
+
* Retrying is exactly right, and telling the owner to re-authorise would be
|
|
12
|
+
* a lie that costs them a consent screen.
|
|
13
|
+
*
|
|
14
|
+
* Both used to read as "could not be refreshed", so anything trying to *report*
|
|
15
|
+
* connection health had to match on the message text. That is why this is a
|
|
16
|
+
* class and not a string: `auth.ts` classifies on `instanceof`, and the messages
|
|
17
|
+
* stay free to be rewritten for whoever is reading them.
|
|
18
|
+
*
|
|
19
|
+
* The message is deliberately unchanged from what each throw site said before —
|
|
20
|
+
* this is a widening, not a rewrite. What is new is that the type now carries
|
|
21
|
+
* *which* connection, so a caller holding several can say which one to fix
|
|
22
|
+
* without parsing the sentence.
|
|
23
|
+
*/
|
|
24
|
+
export class ReauthRequired extends Error {
|
|
25
|
+
/** `provider.id`, e.g. `gmail.main`. The addressing form used everywhere. */
|
|
26
|
+
readonly connectionKey: string;
|
|
27
|
+
|
|
28
|
+
constructor(connectionKey: string, message: string) {
|
|
29
|
+
super(message);
|
|
30
|
+
this.name = 'ReauthRequired';
|
|
31
|
+
this.connectionKey = connectionKey;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Whether an HTTP status from a token endpoint means the grant itself is dead.
|
|
37
|
+
*
|
|
38
|
+
* 4xx is the authorization server saying no to *this credential* — `invalid_grant`
|
|
39
|
+
* for a revoked or expired refresh token, `invalid_client` for a client that no
|
|
40
|
+
* longer exists. Signing in again is the fix.
|
|
41
|
+
*
|
|
42
|
+
* 5xx is the server saying no to *everyone*, and 429 is it saying "not now".
|
|
43
|
+
* Neither is a statement about the credential, so neither may be reported as
|
|
44
|
+
* needing a human. 429 sits in the 4xx range and is excluded for that reason.
|
|
45
|
+
*/
|
|
46
|
+
export function statusMeansGrantIsDead(status: number): boolean {
|
|
47
|
+
return status >= 400 && status < 500 && status !== 429;
|
|
48
|
+
}
|
|
@@ -11,7 +11,21 @@
|
|
|
11
11
|
# There is no build step: Bun runs TypeScript directly, so the image holds
|
|
12
12
|
# source, and `bun install` only fetches the third-party dependencies.
|
|
13
13
|
|
|
14
|
-
|
|
14
|
+
# Pinned by digest, with the tag kept beside it so the next bump is legible.
|
|
15
|
+
#
|
|
16
|
+
# A tag is a pointer its publisher can move. `1.3.11-slim` is not a promise about
|
|
17
|
+
# bytes — it resolves to whatever was last pushed under that name — so a rebuild
|
|
18
|
+
# months from now can ship a base image nobody here reviewed, in a container that
|
|
19
|
+
# holds live OAuth refresh tokens. The digest is the bytes.
|
|
20
|
+
#
|
|
21
|
+
# `bunfig.toml` already makes this argument about npm packages, with a seven-day
|
|
22
|
+
# release-age floor; the base image is the larger half of the same supply chain
|
|
23
|
+
# and had nothing.
|
|
24
|
+
#
|
|
25
|
+
# To bump: change the tag, then resolve it —
|
|
26
|
+
# docker buildx imagetools inspect oven/bun:<tag> --format '{{.Manifest.Digest}}'
|
|
27
|
+
# The digest below is the multi-arch index, so it still resolves per architecture.
|
|
28
|
+
FROM oven/bun:1.3.11-slim@sha256:478281fdd196871c7e51ba6a820b7803a8ae97042ec86cdbc2e1c6b6626442d9
|
|
15
29
|
|
|
16
30
|
WORKDIR /app
|
|
17
31
|
|
|
@@ -42,7 +56,18 @@ COPY package.json bunfig.toml bun.lock* ./
|
|
|
42
56
|
# in `package.json` is an exact version, so the direct set is identical either
|
|
43
57
|
# way; the lockfile pins what those depend on in turn, and a published package
|
|
44
58
|
# has never been able to carry one.
|
|
45
|
-
|
|
59
|
+
#
|
|
60
|
+
# The fallback says so out loud. An unfrozen resolve is a real difference in what
|
|
61
|
+
# ships and it used to be silent — the build log looked identical either way, so
|
|
62
|
+
# "which transitive versions did this image get" was answerable only by opening
|
|
63
|
+
# the image. One line makes it answerable from the build.
|
|
64
|
+
RUN if [ -f bun.lock ]; then \
|
|
65
|
+
bun install --frozen-lockfile; \
|
|
66
|
+
else \
|
|
67
|
+
echo "no bun.lock in the build context: resolving transitive dependencies fresh." >&2; \
|
|
68
|
+
echo "Direct dependencies are exact in package.json, so the direct set is unchanged." >&2; \
|
|
69
|
+
bun install; \
|
|
70
|
+
fi
|
|
46
71
|
|
|
47
72
|
COPY src/ src/
|
|
48
73
|
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { installRoot, layout } from '#profile';
|
|
2
3
|
import type { DeployStep } from '../driver.ts';
|
|
3
4
|
import {
|
|
4
5
|
removalStep,
|
|
@@ -103,6 +104,68 @@ export function bucketGrants(bucket: string, profiles: readonly string[]): Condi
|
|
|
103
104
|
];
|
|
104
105
|
}
|
|
105
106
|
|
|
107
|
+
/**
|
|
108
|
+
* How long a deleted or overwritten object can still be recovered.
|
|
109
|
+
*
|
|
110
|
+
* The revision holds `objectAdmin` on everything under `data/`, and
|
|
111
|
+
* `objectAdmin` contains `storage.objects.delete`. That grant is correct — the
|
|
112
|
+
* endpoint writes state, memory, tasks, assets and the audit log, and rewriting
|
|
113
|
+
* an object is deleting the old one — but it means the process most exposed to
|
|
114
|
+
* the internet is also the one that can erase the record of what it did.
|
|
115
|
+
* `audit.tamper-evident` already says deleting a run whole is not *detectable*;
|
|
116
|
+
* without this it was not *recoverable* either.
|
|
117
|
+
*
|
|
118
|
+
* Thirty days rather than the platform's default seven, because the gap this
|
|
119
|
+
* closes is noticing late. A compromise found the same afternoon needs no
|
|
120
|
+
* retention policy at all.
|
|
121
|
+
*/
|
|
122
|
+
const SOFT_DELETE_DURATION = '30d';
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* The three protections a deploy applies to the bucket every time it runs.
|
|
126
|
+
*
|
|
127
|
+
* `update` rather than flags on `create`, so they reach a bucket that already
|
|
128
|
+
* exists — see the call site. Idempotent: setting a policy to what it already is
|
|
129
|
+
* is a no-op that costs one API call.
|
|
130
|
+
*
|
|
131
|
+
* - **Public access prevention**, enforced. Nothing in this bucket is served to
|
|
132
|
+
* a browser and nothing in it should ever be anonymous-readable, so the useful
|
|
133
|
+
* setting is the one that makes granting that impossible rather than merely
|
|
134
|
+
* absent. Uniform bucket-level access already removes per-object ACLs; this
|
|
135
|
+
* removes the bucket-level way to do the same thing.
|
|
136
|
+
* - **Soft delete**, so a deletion is recoverable — see above.
|
|
137
|
+
* - **Object versioning**, with the lifecycle rule that bounds it. Versioning
|
|
138
|
+
* covers what soft delete does not: an object *overwritten* in place, where
|
|
139
|
+
* the previous content is the thing worth keeping. The rule is a file shipped
|
|
140
|
+
* beside this one rather than written at plan time, because `--dry-run` writes
|
|
141
|
+
* nothing and prints what it would run — a temporary file would break both.
|
|
142
|
+
*/
|
|
143
|
+
function durabilitySteps(bucket: string): DeployStep[] {
|
|
144
|
+
const lifecycle = join(installRoot(import.meta.dir), 'src/deployments/gcp/lifecycle.json');
|
|
145
|
+
|
|
146
|
+
return [
|
|
147
|
+
{
|
|
148
|
+
title: 'make the bucket unable to be shared publicly, and its deletions recoverable',
|
|
149
|
+
argv: [
|
|
150
|
+
'storage',
|
|
151
|
+
'buckets',
|
|
152
|
+
'update',
|
|
153
|
+
`gs://${bucket}`,
|
|
154
|
+
'--public-access-prevention',
|
|
155
|
+
'--soft-delete-duration',
|
|
156
|
+
SOFT_DELETE_DURATION,
|
|
157
|
+
'--versioning',
|
|
158
|
+
// Without this, versioning keeps every prior copy of every state key
|
|
159
|
+
// forever, and state is the one thing here that is rewritten rather than
|
|
160
|
+
// appended. The rule bounds it by age and by count.
|
|
161
|
+
'--lifecycle-file',
|
|
162
|
+
lifecycle,
|
|
163
|
+
],
|
|
164
|
+
tolerateFailure: true,
|
|
165
|
+
},
|
|
166
|
+
];
|
|
167
|
+
}
|
|
168
|
+
|
|
106
169
|
const TITLES: Record<string, string> = {
|
|
107
170
|
'owns-its-data': 'let the revision write its own data, but not the manifests in it',
|
|
108
171
|
'reads-its-config': 'let the revision read its config, and only read it',
|
|
@@ -133,20 +196,30 @@ export async function bucketSteps(input: {
|
|
|
133
196
|
// served publicly; uniform access removes per-object ACLs as a way to
|
|
134
197
|
// get that wrong.
|
|
135
198
|
'--uniform-bucket-level-access',
|
|
136
|
-
// Autoclass rather than a lifecycle rule, because no one
|
|
137
|
-
// right for this bucket: it holds the config the endpoint
|
|
138
|
-
// boot next to assets and audit rows nobody opens again.
|
|
139
|
-
// Autoclass bucket there are no retrieval and no
|
|
140
|
-
// which is what makes ARCHIVE safe as the floor
|
|
141
|
-
// never reading the thing again — a read pulls the
|
|
142
|
-
// Standard at no charge. Objects under 128 KiB never
|
|
143
|
-
// this costs the config and the log nothing and saves
|
|
199
|
+
// Autoclass rather than a storage-class lifecycle rule, because no one
|
|
200
|
+
// fixed class is right for this bucket: it holds the config the endpoint
|
|
201
|
+
// reads on every boot next to assets and audit rows nobody opens again.
|
|
202
|
+
// Inside an Autoclass bucket there are no retrieval and no
|
|
203
|
+
// early-deletion fees, which is what makes ARCHIVE safe as the floor
|
|
204
|
+
// rather than a bet on never reading the thing again — a read pulls the
|
|
205
|
+
// object back to Standard at no charge. Objects under 128 KiB never
|
|
206
|
+
// leave Standard, so this costs the config and the log nothing and saves
|
|
207
|
+
// on attachments.
|
|
144
208
|
'--enable-autoclass',
|
|
145
209
|
'--autoclass-terminal-storage-class',
|
|
146
210
|
'ARCHIVE',
|
|
147
211
|
],
|
|
148
212
|
tolerateFailure: true,
|
|
149
213
|
},
|
|
214
|
+
// Everything about durability, applied separately from the create.
|
|
215
|
+
//
|
|
216
|
+
// **Not folded into the flags above, and that is the whole point.** Every
|
|
217
|
+
// step here tolerates failure, so the create is refused as `ALREADY_EXISTS`
|
|
218
|
+
// on the second deploy onwards — which means a protection added to that
|
|
219
|
+
// argv reaches a bucket made after this commit and no other. Every
|
|
220
|
+
// deployment that already exists is exactly the one that has an audit log
|
|
221
|
+
// worth keeping.
|
|
222
|
+
...durabilitySteps(input.bucket),
|
|
150
223
|
];
|
|
151
224
|
|
|
152
225
|
if (!input.serviceAccount) return steps;
|
|
@@ -126,12 +126,43 @@ export function deployPlan(input: PlanInput): DeployStep[] {
|
|
|
126
126
|
// harness can mint one — so a target reached by a remote MCP client
|
|
127
127
|
// declares `public` and gates the request in the application instead.
|
|
128
128
|
cloudrun.access === 'iam' ? '--no-allow-unauthenticated' : '--allow-unauthenticated',
|
|
129
|
-
//
|
|
130
|
-
//
|
|
131
|
-
//
|
|
132
|
-
//
|
|
129
|
+
// What the platform will accept a connection from at all, which is a
|
|
130
|
+
// different question from who it lets through. An `iam` target is by
|
|
131
|
+
// definition not reached by an MCP client — no agent harness can mint the
|
|
132
|
+
// identity token Cloud Run wants — so it is reached by other cloud
|
|
133
|
+
// workloads or by nothing, and neither needs an internet-facing listener.
|
|
134
|
+
// `public` is the case where the listener *is* the point.
|
|
135
|
+
'--ingress',
|
|
136
|
+
cloudrun.access === 'iam' ? 'internal-and-cloud-load-balancing' : 'all',
|
|
137
|
+
// Always passed, including the zero and including every default below
|
|
138
|
+
// it. Config is the source of truth here (ADR-004), and a flag sent only
|
|
139
|
+
// when non-zero would let a value be raised and never lowered — the
|
|
140
|
+
// revision would keep whatever the last deploy that bothered to mention
|
|
141
|
+
// it had set.
|
|
133
142
|
'--min-instances',
|
|
134
143
|
String(cloudrun.min_instances),
|
|
144
|
+
// The five that used to be absent, and absent meant the platform's own
|
|
145
|
+
// defaults: a hundred instances, eighty concurrent requests each, and
|
|
146
|
+
// 512 MiB to serve a 64 MiB upload in. Each one is argued at its field in
|
|
147
|
+
// `deployTargetSchema`; what they have in common is that a public URL
|
|
148
|
+
// with no ceiling on any of them is an endpoint whose cost and whose
|
|
149
|
+
// credential-store traffic are decided by whoever is calling it.
|
|
150
|
+
'--max-instances',
|
|
151
|
+
String(cloudrun.max_instances),
|
|
152
|
+
'--concurrency',
|
|
153
|
+
String(cloudrun.concurrency),
|
|
154
|
+
'--timeout',
|
|
155
|
+
String(cloudrun.timeout_seconds),
|
|
156
|
+
'--memory',
|
|
157
|
+
cloudrun.memory,
|
|
158
|
+
'--cpu',
|
|
159
|
+
cloudrun.cpu,
|
|
160
|
+
// Named rather than inherited, like the five above. gen2 is the current
|
|
161
|
+
// default and the one this image is tested on; pinning it means a
|
|
162
|
+
// platform migration is a commit here rather than a change under a
|
|
163
|
+
// running endpoint.
|
|
164
|
+
'--execution-environment',
|
|
165
|
+
'gen2',
|
|
135
166
|
],
|
|
136
167
|
},
|
|
137
168
|
];
|
|
@@ -151,7 +182,14 @@ export function deployPlan(input: PlanInput): DeployStep[] {
|
|
|
151
182
|
*/
|
|
152
183
|
function secretMounts(secretEnv: PlanInput['secretEnv']): string[] {
|
|
153
184
|
const entries = Object.entries(secretEnv ?? {});
|
|
154
|
-
|
|
185
|
+
|
|
186
|
+
// `--clear-secrets`, not nothing. `gcloud run deploy` leaves a setting it is
|
|
187
|
+
// not told about exactly as the last revision had it, so an empty map used to
|
|
188
|
+
// mean "keep whatever is mounted" rather than "mount nothing" — and removing a
|
|
189
|
+
// vault from config left its secret still resolved into the new revision's
|
|
190
|
+
// environment, indefinitely, with nothing in the config saying so. The same
|
|
191
|
+
// argument `--min-instances` makes about always passing the zero.
|
|
192
|
+
if (entries.length === 0) return ['--clear-secrets'];
|
|
155
193
|
|
|
156
194
|
return [
|
|
157
195
|
'--set-secrets',
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { ConfigError, type DeployConfig, type TargetConfig } from '#profile';
|
|
1
|
+
import { ConfigError, DEPLOY_DEFAULTS, type DeployConfig, type TargetConfig } from '#profile';
|
|
2
2
|
import { heading, print, style, waiting } from '#cli/output.ts';
|
|
3
3
|
import { ask, confirm } from '#cli/prompt.ts';
|
|
4
4
|
import type { SurveyInput, SurveyResult } from '../driver.ts';
|
|
@@ -130,6 +130,17 @@ export async function surveyCloudRun(input: SurveyInput): Promise<SurveyResult>
|
|
|
130
130
|
// Not asked about. Zero is right for almost every target and the question
|
|
131
131
|
// would cost every operator a decision to buy one of them a knob.
|
|
132
132
|
min_instances: current.min_instances ?? 0,
|
|
133
|
+
// Not asked about either, and for a stronger version of the same reason: a
|
|
134
|
+
// ceiling is only interesting to somebody who has already hit it, and the
|
|
135
|
+
// defaults are the ones a single-user endpoint wants. Carried through from
|
|
136
|
+
// what the target already says so that an operator who *has* edited them
|
|
137
|
+
// keeps their edit — pressing return through the survey changes nothing,
|
|
138
|
+
// which is the property every other field here has too.
|
|
139
|
+
max_instances: current.max_instances ?? DEPLOY_DEFAULTS.max_instances,
|
|
140
|
+
concurrency: current.concurrency ?? DEPLOY_DEFAULTS.concurrency,
|
|
141
|
+
timeout_seconds: current.timeout_seconds ?? DEPLOY_DEFAULTS.timeout_seconds,
|
|
142
|
+
memory: current.memory ?? DEPLOY_DEFAULTS.memory,
|
|
143
|
+
cpu: current.cpu ?? DEPLOY_DEFAULTS.cpu,
|
|
133
144
|
project,
|
|
134
145
|
region,
|
|
135
146
|
service,
|
package/src/policy/limits.ts
CHANGED
|
@@ -27,6 +27,29 @@ interface Bucket {
|
|
|
27
27
|
lastRefill: number;
|
|
28
28
|
}
|
|
29
29
|
|
|
30
|
+
/**
|
|
31
|
+
* How many keys one limiter will hold.
|
|
32
|
+
*
|
|
33
|
+
* The bound exists because a key is not always something the endpoint chose. At
|
|
34
|
+
* the HTTP edge a caller is identified by the first `X-Forwarded-For` hop, which
|
|
35
|
+
* on a public deployment is a header a stranger writes — so an unbounded map is
|
|
36
|
+
* an unbounded allocation driven from outside, in a container that now has an
|
|
37
|
+
* explicit memory limit to exceed.
|
|
38
|
+
*
|
|
39
|
+
* Ten thousand is far above any real caller count for a single-user endpoint and
|
|
40
|
+
* far below a problem: a bucket is two numbers and a string key.
|
|
41
|
+
*/
|
|
42
|
+
const DEFAULT_MAX_KEYS = 10_000;
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* How many go at once when the cap is reached.
|
|
46
|
+
*
|
|
47
|
+
* A batch rather than one, so the sort that finds them is amortised. Evicting a
|
|
48
|
+
* single key per overflow would run an O(n log n) pass on every request once the
|
|
49
|
+
* map is full, which turns the bound into its own denial of service.
|
|
50
|
+
*/
|
|
51
|
+
const EVICTION_FRACTION = 10;
|
|
52
|
+
|
|
30
53
|
/**
|
|
31
54
|
* Token bucket, refilled continuously rather than on a fixed window boundary.
|
|
32
55
|
*
|
|
@@ -37,9 +60,16 @@ interface Bucket {
|
|
|
37
60
|
export class RateLimiter {
|
|
38
61
|
readonly #buckets = new Map<string, Bucket>();
|
|
39
62
|
readonly #now: () => number;
|
|
63
|
+
readonly #maxKeys: number;
|
|
40
64
|
|
|
41
|
-
constructor(now: () => number = Date.now) {
|
|
65
|
+
constructor(now: () => number = Date.now, maxKeys: number = DEFAULT_MAX_KEYS) {
|
|
42
66
|
this.#now = now;
|
|
67
|
+
this.#maxKeys = maxKeys;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** How many callers are currently held. For tests, and for nothing else. */
|
|
71
|
+
get size(): number {
|
|
72
|
+
return this.#buckets.size;
|
|
43
73
|
}
|
|
44
74
|
|
|
45
75
|
/**
|
|
@@ -52,7 +82,13 @@ export class RateLimiter {
|
|
|
52
82
|
|
|
53
83
|
const now = this.#now();
|
|
54
84
|
const refillPerMs = perMinute / 60_000;
|
|
55
|
-
const
|
|
85
|
+
const existing = this.#buckets.get(key);
|
|
86
|
+
|
|
87
|
+
// Before the insert, not after: the cap is on what this map holds, and
|
|
88
|
+
// checking afterwards means it is briefly one over on every overflow.
|
|
89
|
+
if (!existing) this.#makeRoom();
|
|
90
|
+
|
|
91
|
+
const bucket = existing ?? { tokens: perMinute, lastRefill: now };
|
|
56
92
|
|
|
57
93
|
bucket.tokens = Math.min(perMinute, bucket.tokens + (now - bucket.lastRefill) * refillPerMs);
|
|
58
94
|
bucket.lastRefill = now;
|
|
@@ -67,11 +103,48 @@ export class RateLimiter {
|
|
|
67
103
|
return { allowed: true, retryAfterMs: 0 };
|
|
68
104
|
}
|
|
69
105
|
|
|
70
|
-
/**
|
|
106
|
+
/**
|
|
107
|
+
* Drop idle buckets so a long-lived process does not accumulate keys forever.
|
|
108
|
+
*
|
|
109
|
+
* Public because it reads as the obvious lever, and because the tests drive it
|
|
110
|
+
* directly. It is **not** what bounds the map, and nothing's correctness may
|
|
111
|
+
* depend on a caller remembering it: for most of this file's life nothing did
|
|
112
|
+
* call it, `edge.ts` claimed idle callers were dropped, and the map grew for
|
|
113
|
+
* as long as the process lived. `#makeRoom` is the bound now, and it runs on
|
|
114
|
+
* the insert path where it cannot be forgotten.
|
|
115
|
+
*/
|
|
71
116
|
prune(idleMs = 300_000): void {
|
|
72
117
|
const cutoff = this.#now() - idleMs;
|
|
73
118
|
for (const [key, bucket] of this.#buckets) {
|
|
74
119
|
if (bucket.lastRefill < cutoff) this.#buckets.delete(key);
|
|
75
120
|
}
|
|
76
121
|
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Make space for one more key, if the map is full.
|
|
125
|
+
*
|
|
126
|
+
* Idle callers first, because dropping one costs nothing — a bucket that has
|
|
127
|
+
* not been touched in five minutes has refilled to full, so re-creating it
|
|
128
|
+
* gives back exactly what was discarded.
|
|
129
|
+
*
|
|
130
|
+
* Only when that is not enough does this evict a live caller, oldest first by
|
|
131
|
+
* last use. That *does* forgive whatever the evicted caller had spent, which
|
|
132
|
+
* is the honest cost of a bounded map: an attacker who can mint ten thousand
|
|
133
|
+
* distinct keys can push their own bucket out and start again. What they
|
|
134
|
+
* cannot do is grow the map, and on the paths this limiter guards there is a
|
|
135
|
+
* second bucket keyed on nothing at all — see `edge.ts` — which is the one
|
|
136
|
+
* that holds when the per-caller key is worthless.
|
|
137
|
+
*/
|
|
138
|
+
#makeRoom(): void {
|
|
139
|
+
if (this.#buckets.size < this.#maxKeys) return;
|
|
140
|
+
|
|
141
|
+
this.prune();
|
|
142
|
+
if (this.#buckets.size < this.#maxKeys) return;
|
|
143
|
+
|
|
144
|
+
const oldest = [...this.#buckets.entries()]
|
|
145
|
+
.sort((a, b) => a[1].lastRefill - b[1].lastRefill)
|
|
146
|
+
.slice(0, Math.max(1, Math.ceil(this.#maxKeys / EVICTION_FRACTION)));
|
|
147
|
+
|
|
148
|
+
for (const [key] of oldest) this.#buckets.delete(key);
|
|
149
|
+
}
|
|
77
150
|
}
|
package/src/profile/index.ts
CHANGED
package/src/profile/legacy.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { z } from 'zod';
|
|
|
2
2
|
import {
|
|
3
3
|
auditTargetSchema,
|
|
4
4
|
credentialsTargetSchema,
|
|
5
|
+
DEPLOY_DEFAULTS,
|
|
5
6
|
deployTargetSchema,
|
|
6
7
|
storageTargetSchema,
|
|
7
8
|
vaultTargetSchema,
|
|
@@ -54,14 +55,18 @@ export const legacyTargetSchema = z
|
|
|
54
55
|
? target
|
|
55
56
|
: {
|
|
56
57
|
...target,
|
|
57
|
-
// The pre-`deploy` spelling predates
|
|
58
|
-
// same defaults the current one would: the closed door,
|
|
59
|
-
//
|
|
58
|
+
// The pre-`deploy` spelling predates all of these, so it gets the
|
|
59
|
+
// same defaults the current one would: the closed door, no instance
|
|
60
|
+
// kept warm, and the ceilings a public URL is deployed under. Built by
|
|
61
|
+
// hand here rather than parsed, so `DEPLOY_DEFAULTS` is spread rather
|
|
62
|
+
// than left to zod — a block that reaches `deployPlan` without them
|
|
63
|
+
// sends `undefined` to `gcloud` as the string "undefined".
|
|
60
64
|
deploy: {
|
|
61
65
|
...cloudrun,
|
|
62
66
|
platform: 'cloudrun' as const,
|
|
63
67
|
access: 'iam' as const,
|
|
64
68
|
min_instances: 0,
|
|
69
|
+
...DEPLOY_DEFAULTS,
|
|
65
70
|
},
|
|
66
71
|
},
|
|
67
72
|
);
|
package/src/profile/schema.ts
CHANGED
|
@@ -158,6 +158,25 @@ export const vaultTargetSchema = z.object({
|
|
|
158
158
|
* discriminated union per platform — buys precision this file cannot use and
|
|
159
159
|
* costs a schema edit on every field any host ever adds.
|
|
160
160
|
*/
|
|
161
|
+
/**
|
|
162
|
+
* The ceilings a deployed revision runs under, in one place.
|
|
163
|
+
*
|
|
164
|
+
* A named constant rather than five literals inside `z.default()` because two
|
|
165
|
+
* callers need the same numbers and neither can read a zod default out of a
|
|
166
|
+
* schema: the survey writes a target's block field by field, and the deploy plan
|
|
167
|
+
* sends every one of them on every rollout. Two spellings of a ceiling is a
|
|
168
|
+
* ceiling that is one value in a fresh profile and another in a surveyed one.
|
|
169
|
+
*
|
|
170
|
+
* Each is argued at its own field below.
|
|
171
|
+
*/
|
|
172
|
+
export const DEPLOY_DEFAULTS = {
|
|
173
|
+
max_instances: 4,
|
|
174
|
+
concurrency: 40,
|
|
175
|
+
timeout_seconds: 300,
|
|
176
|
+
memory: '1Gi',
|
|
177
|
+
cpu: '1',
|
|
178
|
+
} as const;
|
|
179
|
+
|
|
161
180
|
export const deployTargetSchema = z.object({
|
|
162
181
|
platform: z.enum(['cloudrun']),
|
|
163
182
|
// Non-empty here rather than in a referential check further down. Under
|
|
@@ -210,6 +229,52 @@ export const deployTargetSchema = z.object({
|
|
|
210
229
|
* Raise it if a re-authorization ever lines up with a cold `/token`.
|
|
211
230
|
*/
|
|
212
231
|
min_instances: z.number().int().min(0).max(10).default(0),
|
|
232
|
+
/**
|
|
233
|
+
* The ceiling on instances, and the only thing bounding what a public URL can
|
|
234
|
+
* spend.
|
|
235
|
+
*
|
|
236
|
+
* Four, not the platform's hundred. `access: public` is a routable address on
|
|
237
|
+
* the internet, and every instance that starts reads the credential store and
|
|
238
|
+
* lists the bucket — so scaling out multiplies cost *and* traffic against the
|
|
239
|
+
* two things this endpoint most wants kept quiet. A single-user endpoint that
|
|
240
|
+
* genuinely needs a fifth concurrent instance has an agent in a loop, which is
|
|
241
|
+
* the case the ceiling is for.
|
|
242
|
+
*
|
|
243
|
+
* It is also what makes `limits.requests_per_minute` mean something in
|
|
244
|
+
* aggregate. Those limits are per instance and always were; with no ceiling
|
|
245
|
+
* the aggregate had no value at all, and `docs/detailed/deployment-cloudrun.md`
|
|
246
|
+
* told the reader to cap this themselves because nothing here did.
|
|
247
|
+
*/
|
|
248
|
+
max_instances: z.number().int().min(1).max(100).default(DEPLOY_DEFAULTS.max_instances),
|
|
249
|
+
/**
|
|
250
|
+
* Requests one instance serves at once.
|
|
251
|
+
*
|
|
252
|
+
* Forty rather than the platform's eighty, paired with `memory` below: an
|
|
253
|
+
* attachment upload is capped at 64 MiB and is buffered before it is written,
|
|
254
|
+
* so what bounds this is memory per instance rather than CPU.
|
|
255
|
+
*/
|
|
256
|
+
concurrency: z.number().int().min(1).max(1000).default(DEPLOY_DEFAULTS.concurrency),
|
|
257
|
+
/**
|
|
258
|
+
* How long one request may run before the platform cuts it.
|
|
259
|
+
*
|
|
260
|
+
* The platform's own default, stated rather than inherited — the point is that
|
|
261
|
+
* it is written down and always sent, so a platform changing its default is
|
|
262
|
+
* not a silent change to what this serves.
|
|
263
|
+
*/
|
|
264
|
+
timeout_seconds: z.number().int().min(1).max(3600).default(DEPLOY_DEFAULTS.timeout_seconds),
|
|
265
|
+
/**
|
|
266
|
+
* Memory per instance.
|
|
267
|
+
*
|
|
268
|
+
* A gigabyte because 512 MiB is not enough for what the endpoint already
|
|
269
|
+
* accepts: `MAX_UPLOAD_BYTES` is 64 MiB, and staging one costs roughly twice
|
|
270
|
+
* that at peak — the chunks as they arrive, and the single buffer they are
|
|
271
|
+
* copied into. At the platform default that is one upload away from an
|
|
272
|
+
* out-of-memory kill, and an OOM is a 503 for every request the instance was
|
|
273
|
+
* also serving.
|
|
274
|
+
*/
|
|
275
|
+
memory: z.string().min(1).default(DEPLOY_DEFAULTS.memory),
|
|
276
|
+
/** CPU per instance. Explicit for the same reason `timeout_seconds` is. */
|
|
277
|
+
cpu: z.string().min(1).default(DEPLOY_DEFAULTS.cpu),
|
|
213
278
|
});
|
|
214
279
|
|
|
215
280
|
/**
|