@fabrika/provider-cloudflare 0.8.2 → 0.9.0-cloudflare.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +16 -15
- package/src/bounds.ts +142 -4
- package/src/cli.ts +14 -0
- package/src/command.ts +8 -4
- package/src/contract-support.ts +118 -0
- package/src/control.ts +37 -119
- package/src/index.ts +1 -1
- package/src/installation/init.ts +11 -18
- package/src/installation/templates/README.md +21 -4
- package/src/installation/templates/platform.yml +6 -0
- package/src/managed-secrets.ts +61 -0
- package/src/plan.ts +11 -3
- package/src/provider.ts +309 -54
- package/src/proxy-worker.ts +33 -13
- package/src/proxy.ts +48 -16
- package/src/runner-protocol.ts +16 -1
- package/src/spa-assets.ts +150 -0
- package/types/src/assets.d.ts +8 -0
- package/types/src/authoring.d.ts +17 -0
- package/types/src/bounds.d.ts +81 -0
- package/types/src/checkout.d.ts +7 -0
- package/types/src/cli-args.d.ts +25 -0
- package/types/src/cli.d.ts +8 -0
- package/types/src/codec.d.ts +8 -0
- package/types/src/collaborators.d.ts +52 -0
- package/types/src/command.d.ts +49 -0
- package/types/src/config-schema.d.ts +9 -0
- package/types/src/contract-support.d.ts +39 -0
- package/types/src/control.d.ts +468 -0
- package/types/src/index.d.ts +18 -0
- package/types/src/installation/cloudflare.d.ts +40 -0
- package/types/src/installation/envfile.d.ts +39 -0
- package/types/src/installation/index.d.ts +2 -0
- package/types/src/installation/init-options.d.ts +59 -0
- package/types/src/installation/init.d.ts +114 -0
- package/types/src/installation/installation.d.ts +2 -0
- package/types/src/installation/scaffold.d.ts +39 -0
- package/types/src/managed-secrets.d.ts +19 -0
- package/types/src/plan.d.ts +21 -0
- package/types/src/provider.d.ts +58 -0
- package/types/src/proxy-worker.d.ts +15 -0
- package/types/src/proxy.d.ts +72 -0
- package/types/src/runner-protocol.d.ts +121 -0
- package/types/src/schema.d.ts +17 -0
- package/types/src/spa-assets.d.ts +27 -0
package/src/provider.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { CLOUDFLARE_APPLICATION_BINDING, CLOUDFLARE_APPLICATION_CLASS, CLOUDFLARE_FILES_BINDING, GENERATED_ENTRY_FILE } from '@fabrika/app/build'
|
|
1
2
|
import {
|
|
2
3
|
type AppEnvironment,
|
|
3
4
|
appEnvironmentId,
|
|
@@ -5,18 +6,25 @@ import {
|
|
|
5
6
|
connectVariables,
|
|
6
7
|
type ContractArtifact,
|
|
7
8
|
contractDigest,
|
|
9
|
+
DEFAULT_APPLICATION_OBJECT,
|
|
10
|
+
FABRIKA_APPLICATION_OBJECT,
|
|
11
|
+
FABRIKA_APPLICATION_OBJECTS,
|
|
8
12
|
FABRIKA_CONTRACT_DIGEST,
|
|
9
13
|
FABRIKA_IAM_ISSUER,
|
|
10
14
|
isFrameworkVariableName,
|
|
15
|
+
isManagedCredential,
|
|
16
|
+
managedValue,
|
|
11
17
|
} from '@fabrika/auth-core'
|
|
12
18
|
import { type DeployStepResult, type ProviderManagedEnvironment, runSteps, type SchemaReconciler } from '@fabrika/provider-contract'
|
|
13
19
|
import { isPolicyOrigin } from '@fabrika/proxy-core/headers'
|
|
14
20
|
import { contractProxyTable } from '@fabrika/proxy-core/manifest'
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
21
|
+
import { stat } from 'node:fs/promises'
|
|
22
|
+
import { basename, dirname, isAbsolute, relative, resolve } from 'node:path'
|
|
23
|
+
import { DurableObject, type GeneratedConfig, R2Bucket, ServiceReference, Worker } from 'oblaka-iac'
|
|
17
24
|
import { assertLiteralAssets } from './assets'
|
|
18
25
|
import type { CloudflareAppConfig } from './authoring'
|
|
19
26
|
import {
|
|
27
|
+
type ApplicationRuntime,
|
|
20
28
|
assertGeneratedConfigs,
|
|
21
29
|
assertGeneration,
|
|
22
30
|
assertStateNamespace,
|
|
@@ -27,10 +35,13 @@ import {
|
|
|
27
35
|
import { runnerWorkerGraph } from './checkout'
|
|
28
36
|
import { type CloudflareCollaborators, defaultCloudflareCollaborators } from './collaborators'
|
|
29
37
|
import { configSchema } from './config-schema'
|
|
38
|
+
import { assertCloudflareContract, needsMinuteTick, pinsGeneratedEntry, runsInObject, servesOps } from './contract-support'
|
|
39
|
+
import { parseSecretNames, splitManagedEnvironment, staleManagedSecrets, withSecretsFile } from './managed-secrets'
|
|
30
40
|
import { buildPlan, type CloudflareStep, findMigratableDatabases } from './plan'
|
|
31
|
-
import { createCloudflareProxyWorker } from './proxy'
|
|
41
|
+
import { createCloudflareProxyWorker, IamApplicationReference, PROXY_MANIFEST_DEFINE } from './proxy'
|
|
32
42
|
import type { RunnerApprovedAccess } from './runner-protocol'
|
|
33
43
|
import { defaultReconcileSchema } from './schema'
|
|
44
|
+
import { type AssetMount, assetMounts, SPA_ASSETS, SPA_ASSETS_DIRECTORY, stageSpaAssets } from './spa-assets'
|
|
34
45
|
|
|
35
46
|
const CANCELLED = 'deploy cancelled'
|
|
36
47
|
|
|
@@ -60,42 +71,183 @@ const assertRunning = (signal: AbortSignal): void => {
|
|
|
60
71
|
}
|
|
61
72
|
}
|
|
62
73
|
|
|
74
|
+
/** The bindings the platform writes into a contract application's Worker (ADR-0065). */
|
|
75
|
+
const PLATFORM_BINDINGS: ReadonlySet<string> = new Set([CLOUDFLARE_APPLICATION_BINDING, CLOUDFLARE_FILES_BINDING])
|
|
76
|
+
|
|
63
77
|
/** An application never names a variable only the platform writes, whether or not this deploy writes it. */
|
|
64
78
|
const assertNoFrameworkVariables = (config: CloudflareAppConfig, application: Worker, run: CloudflareDeployRun): void => {
|
|
65
79
|
const authored = [
|
|
66
80
|
...(config.pipeline?.vars ?? []),
|
|
81
|
+
...(config.pipeline?.secrets ?? []),
|
|
67
82
|
...Object.keys(application.options.vars ?? {}),
|
|
68
83
|
...Object.keys(application.options.bindings ?? {}),
|
|
69
84
|
...Object.keys(run.vars),
|
|
85
|
+
...Object.keys(run.secrets),
|
|
70
86
|
]
|
|
71
|
-
const reserved = authored.find(isFrameworkVariableName)
|
|
87
|
+
const reserved = authored.find((name) => isFrameworkVariableName(name) || PLATFORM_BINDINGS.has(name))
|
|
72
88
|
if (reserved !== undefined) {
|
|
73
89
|
throw new Error(`cloudflare: application variable \`${reserved}\` is managed by Fabrika`)
|
|
74
90
|
}
|
|
91
|
+
if (application.options.triggers !== undefined) {
|
|
92
|
+
throw new Error(`cloudflare: application Worker \`${application.options.name}\` sets \`triggers\`; the platform writes its cron triggers`)
|
|
93
|
+
}
|
|
75
94
|
}
|
|
76
95
|
|
|
96
|
+
/**
|
|
97
|
+
* Write the managed values into the application Worker: the public ones as vars, and the credentials as Worker secrets
|
|
98
|
+
* the deploy uploads with the version (`withSecretsFile`), never as plain text in a generated config.
|
|
99
|
+
*/
|
|
77
100
|
const withManagedEnvironment = (
|
|
78
101
|
config: CloudflareAppConfig,
|
|
79
102
|
authored: Worker,
|
|
80
103
|
run: CloudflareDeployRun,
|
|
81
104
|
managed: ProviderManagedEnvironment,
|
|
82
|
-
): Worker => {
|
|
83
|
-
const
|
|
84
|
-
const declared = new Set(config.pipeline?.vars ?? [])
|
|
105
|
+
): { readonly worker: Worker; readonly secrets: Readonly<Record<string, string>> } => {
|
|
106
|
+
const declared = new Set([...(config.pipeline?.vars ?? []), ...(config.pipeline?.secrets ?? [])])
|
|
85
107
|
const application = applicationWorker(authored)
|
|
86
|
-
for (const
|
|
87
|
-
if (
|
|
108
|
+
for (const name of Object.keys(managed)) {
|
|
109
|
+
if (
|
|
110
|
+
declared.has(name) || application.options.vars?.[name] !== undefined || run.vars[name] !== undefined || run.secrets[name] !== undefined
|
|
111
|
+
) {
|
|
88
112
|
throw new Error(`cloudflare: application variable \`${name}\` is managed by Fabrika`)
|
|
89
113
|
}
|
|
90
|
-
if (value !== null) present[name] = value
|
|
91
114
|
}
|
|
92
|
-
|
|
115
|
+
const { vars, secrets } = splitManagedEnvironment(managed)
|
|
116
|
+
if (Object.keys(vars).length === 0) return { worker: authored, secrets }
|
|
93
117
|
const configured = new Worker({
|
|
94
118
|
...application.options,
|
|
95
119
|
vars: {
|
|
96
120
|
...(application.options.vars ?? {}),
|
|
97
|
-
...
|
|
121
|
+
...vars,
|
|
122
|
+
},
|
|
123
|
+
})
|
|
124
|
+
return { worker: replaceApplicationWorker(authored, configured), secrets }
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** R2's bucket names: 3 to 63 lowercase letters, digits and hyphens, starting and ending with a letter or a digit. */
|
|
128
|
+
const R2_BUCKET_NAME = /^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$/
|
|
129
|
+
|
|
130
|
+
/** The environment's files bucket, before oblaka's environment prefix. */
|
|
131
|
+
const filesBucketName = (appId: string): string => `${appId}-files`
|
|
132
|
+
|
|
133
|
+
/** The remote name oblaka gives the files bucket; it prefixes a bucket with the environment, `local` included. */
|
|
134
|
+
const filesBucketRemote = (run: CloudflareDeployRun): string => {
|
|
135
|
+
const remote = `${run.env}-${filesBucketName(run.appId)}`
|
|
136
|
+
if (!R2_BUCKET_NAME.test(remote)) {
|
|
137
|
+
throw new Error(
|
|
138
|
+
`cloudflare: the files bucket \`${remote}\` of \`${run.appId}@${run.env}\` is not a valid R2 bucket name; shorten the app id or the environment`,
|
|
139
|
+
)
|
|
140
|
+
}
|
|
141
|
+
return remote
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const MINUTE_TICK = '* * * * *'
|
|
145
|
+
|
|
146
|
+
/** What the platform gives a contract application's Worker; the deployer writes exactly this and the bounds check it. */
|
|
147
|
+
const applicationRuntime = (run: CloudflareDeployRun, contract: ContractArtifact, entry: string | null): ApplicationRuntime => ({
|
|
148
|
+
filesBucket: filesBucketRemote(run),
|
|
149
|
+
crons: needsMinuteTick(contract) ? [MINUTE_TICK] : [],
|
|
150
|
+
entry,
|
|
151
|
+
})
|
|
152
|
+
|
|
153
|
+
/** Where `fabrika app build` runs: the application directory, beside its config and contract, and the config it reads. */
|
|
154
|
+
interface EntryBuild {
|
|
155
|
+
readonly cwd: string
|
|
156
|
+
readonly config: string
|
|
157
|
+
/** The directory the runner installed the checkout's dependencies in; the CLI is looked for no higher. */
|
|
158
|
+
readonly root: string
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/** The application's own CLI, where the checkout's `bun install` links it, as the Zerops build runs it. */
|
|
162
|
+
const APPLICATION_CLI = ['node_modules', '.bin', 'fabrika']
|
|
163
|
+
|
|
164
|
+
const ENTRY_BUILD_ARGS = ['app', 'build', '--provider=cloudflare']
|
|
165
|
+
|
|
166
|
+
/** All the CLI gets of the executor's environment: what Bun needs to run, as the generated tests of `fabrika db` get. */
|
|
167
|
+
const ENTRY_BUILD_VARIABLES: readonly string[] = ['PATH', 'HOME', 'TMPDIR']
|
|
168
|
+
|
|
169
|
+
const isFile = async (path: string): Promise<boolean> => {
|
|
170
|
+
try {
|
|
171
|
+
return (await stat(path)).isFile()
|
|
172
|
+
} catch {
|
|
173
|
+
return false
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** The first `node_modules/.bin/fabrika` from the application directory up to `root`: a workspace links it at its root. */
|
|
178
|
+
const applicationCli = async ({ cwd, root }: EntryBuild): Promise<string> => {
|
|
179
|
+
for (let dir = cwd; isWithin(root, dir); dir = dirname(dir)) {
|
|
180
|
+
const candidate = resolve(dir, ...APPLICATION_CLI)
|
|
181
|
+
if (await isFile(candidate)) return candidate
|
|
182
|
+
if (dir === root) break
|
|
183
|
+
}
|
|
184
|
+
throw new Error(
|
|
185
|
+
`fabrika app build: no \`${APPLICATION_CLI.join('/')}\` from \`${cwd}\` up to \`${root}\`; the application installs \`@fabrika/cli\``,
|
|
186
|
+
)
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
const isWithin = (root: string, dir: string): boolean => {
|
|
190
|
+
const path = relative(root, dir)
|
|
191
|
+
return path === '' || (!path.startsWith('..') && !isAbsolute(path))
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* The executor's environment carries the application's and the managed secrets, and the CLI is checkout code whose
|
|
196
|
+
* failed output reaches the log: it inherits nothing and gets only `ENTRY_BUILD_VARIABLES`.
|
|
197
|
+
*/
|
|
198
|
+
const entryBuildEnvironment = (): { readonly env: Record<string, string>; readonly omitEnv: readonly string[] } => {
|
|
199
|
+
const env: Record<string, string> = {}
|
|
200
|
+
for (const name of ENTRY_BUILD_VARIABLES) {
|
|
201
|
+
const value = process.env[name]
|
|
202
|
+
if (value !== undefined) env[name] = value
|
|
203
|
+
}
|
|
204
|
+
return { env, omitEnv: Object.keys(process.env) }
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/** Regenerate the entry and the inlined migrations from the checkout, with the application's own release of the CLI. */
|
|
208
|
+
const buildEntry = async (build: EntryBuild, run: CloudflareDeployRun, cf: CloudflareCollaborators): Promise<void> => {
|
|
209
|
+
const args = [...ENTRY_BUILD_ARGS, `--config=${build.config}`]
|
|
210
|
+
if (run.dryRun) {
|
|
211
|
+
run.log(` [dry-run] would run: fabrika ${args.join(' ')} in ${build.cwd}`)
|
|
212
|
+
return
|
|
213
|
+
}
|
|
214
|
+
assertRunning(run.signal)
|
|
215
|
+
const command = await applicationCli(build)
|
|
216
|
+
const result = await cf.runCommand({ command, args, cwd: build.cwd, ...entryBuildEnvironment(), signal: run.signal })
|
|
217
|
+
if (result.exitCode !== 0) throw commandError('fabrika app build', result)
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/** The object variables of a release that serves ops (ADR-0069): ordinary traffic and ops requests reach `application`. */
|
|
221
|
+
const OBJECT_VARIABLES: Readonly<Record<string, string>> = {
|
|
222
|
+
[FABRIKA_APPLICATION_OBJECT]: DEFAULT_APPLICATION_OBJECT,
|
|
223
|
+
[FABRIKA_APPLICATION_OBJECTS]: DEFAULT_APPLICATION_OBJECT,
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* Bind the application's Durable Object, which runs the application over its SQLite, the environment's files bucket,
|
|
228
|
+
* and the minute tick (ADR-0065), and on a release that serves ops name its objects. The object is bound first, so
|
|
229
|
+
* oblaka numbers its migration `v0001` whatever else the application binds, and the tag never moves. With `assets`,
|
|
230
|
+
* the Worker also gets the Workers Assets the deploy stages the mounts' files into (`SPA_ASSETS`).
|
|
231
|
+
*/
|
|
232
|
+
const withApplicationRuntime = (
|
|
233
|
+
authored: Worker,
|
|
234
|
+
run: CloudflareDeployRun,
|
|
235
|
+
runtime: ApplicationRuntime,
|
|
236
|
+
ops: boolean,
|
|
237
|
+
assets: boolean,
|
|
238
|
+
): Worker => {
|
|
239
|
+
const application = applicationWorker(authored)
|
|
240
|
+
const configured = new Worker({
|
|
241
|
+
...application.options,
|
|
242
|
+
...(ops ? { vars: { ...(application.options.vars ?? {}), ...OBJECT_VARIABLES } } : {}),
|
|
243
|
+
...(assets ? { assets: { ...SPA_ASSETS } } : {}),
|
|
244
|
+
bindings: {
|
|
245
|
+
[CLOUDFLARE_APPLICATION_BINDING]: new DurableObject({ name: `${run.appId}-application`, className: CLOUDFLARE_APPLICATION_CLASS }),
|
|
246
|
+
...(application.options.bindings ?? {}),
|
|
247
|
+
[CLOUDFLARE_FILES_BINDING]: new R2Bucket({ name: filesBucketName(run.appId) }),
|
|
98
248
|
},
|
|
249
|
+
// Always written: wrangler leaves a Worker's schedules alone when `triggers` is absent, and clears them on `[]`.
|
|
250
|
+
triggers: { crons: [...runtime.crons] },
|
|
99
251
|
})
|
|
100
252
|
return replaceApplicationWorker(authored, configured)
|
|
101
253
|
}
|
|
@@ -115,6 +267,19 @@ const connectorBindings = (contract: ContractArtifact | null): Readonly<Record<s
|
|
|
115
267
|
Object.entries(contract?.imports ?? {}).map(([name, imported]) => [connectVariables(name).binding, proxyWorkerName(imported.app)]),
|
|
116
268
|
)
|
|
117
269
|
|
|
270
|
+
/** The application Worker's `IAM` binding, written by the deployer so that it names this deployment (ADR-0067 §3). */
|
|
271
|
+
const withIamIdentity = (authored: Worker, iamAppId: string): Worker => {
|
|
272
|
+
const application = applicationWorker(authored)
|
|
273
|
+
const declared = application.options.bindings?.['IAM']
|
|
274
|
+
// A resource under the name is no IAM binding; oblaka provisions it, and the bounds check whatever it emits.
|
|
275
|
+
if (declared === undefined || 'getId' in declared) return authored
|
|
276
|
+
const configured = new Worker({
|
|
277
|
+
...application.options,
|
|
278
|
+
bindings: { ...application.options.bindings, IAM: new IamApplicationReference(iamAppId, declared) },
|
|
279
|
+
})
|
|
280
|
+
return replaceApplicationWorker(authored, configured)
|
|
281
|
+
}
|
|
282
|
+
|
|
118
283
|
const withConnectorBindings = (authored: Worker, bindings: Readonly<Record<string, string>>): Worker => {
|
|
119
284
|
if (Object.keys(bindings).length === 0) return authored
|
|
120
285
|
const application = applicationWorker(authored)
|
|
@@ -134,15 +299,13 @@ interface ServedContract {
|
|
|
134
299
|
readonly digest: string
|
|
135
300
|
}
|
|
136
301
|
|
|
137
|
-
/**
|
|
302
|
+
/**
|
|
303
|
+
* A contract application starts only as the contract it was deployed with (ADR-0022 rule 5), so the platform writes its
|
|
304
|
+
* digest. Control never hands one: `parseManagedEnvironment` refuses the name.
|
|
305
|
+
*/
|
|
138
306
|
const contractEnvironment = (managed: ProviderManagedEnvironment, served: ServedContract | null): ProviderManagedEnvironment => {
|
|
139
307
|
if (served === null) return managed
|
|
140
|
-
|
|
141
|
-
const sent = managed[FABRIKA_CONTRACT_DIGEST]
|
|
142
|
-
if (sent !== undefined && sent !== null && sent !== digest) {
|
|
143
|
-
throw new Error(`cloudflare: ${FABRIKA_CONTRACT_DIGEST} is \`${sent}\`, but the contract this deployment serves has digest \`${digest}\``)
|
|
144
|
-
}
|
|
145
|
-
return { ...managed, [FABRIKA_CONTRACT_DIGEST]: digest }
|
|
308
|
+
return { ...managed, [FABRIKA_CONTRACT_DIGEST]: { value: served.digest, credential: isManagedCredential(FABRIKA_CONTRACT_DIGEST) } }
|
|
146
309
|
}
|
|
147
310
|
|
|
148
311
|
/**
|
|
@@ -162,8 +325,9 @@ const assertContractSchema = (schema: AppSchema | null, contract: ContractArtifa
|
|
|
162
325
|
/**
|
|
163
326
|
* The proxy Worker is the platform's (ADR-0048): built here, from this package's own entrypoint, around the
|
|
164
327
|
* application Worker the config returns, never taken from the checkout. A config that returns a proxy is refused.
|
|
328
|
+
* `ops` adds Control's ops row to the table.
|
|
165
329
|
*/
|
|
166
|
-
const platformProxied = (authored: Worker, served: ServedContract, run: CloudflareDeployRun): Worker => {
|
|
330
|
+
const platformProxied = (authored: Worker, served: ServedContract, run: CloudflareDeployRun, ops: boolean): Worker => {
|
|
167
331
|
if (authored.options.bindings?.['APP'] !== undefined) {
|
|
168
332
|
throw new Error(
|
|
169
333
|
`cloudflare: \`resources\` returns the Worker \`${authored.options.name}\` binding \`APP\`, a proxy of its own; return the application Worker, and the platform builds the proxy Worker from its contract`,
|
|
@@ -180,16 +344,18 @@ const platformProxied = (authored: Worker, served: ServedContract, run: Cloudfla
|
|
|
180
344
|
appHost: run.domain ?? 'localhost',
|
|
181
345
|
...(run.domain === undefined ? {} : { domain: run.domain }),
|
|
182
346
|
...(run.crossOrigins === undefined ? {} : { crossOrigins: run.crossOrigins }),
|
|
183
|
-
endpoints: contractProxyTable(served.contract, served.digest),
|
|
184
|
-
...operationsOrigin(run
|
|
185
|
-
iamUrl: run.managedEnvironment
|
|
347
|
+
endpoints: contractProxyTable(served.contract, served.digest, { ops }),
|
|
348
|
+
...operationsOrigin(run),
|
|
349
|
+
iamUrl: managedValue(run.managedEnvironment, FABRIKA_IAM_ISSUER) ?? '',
|
|
186
350
|
})
|
|
187
351
|
}
|
|
188
352
|
|
|
189
|
-
/** The
|
|
190
|
-
const operationsOrigin = (
|
|
191
|
-
|
|
192
|
-
|
|
353
|
+
/** The origin of the environment's Operations ingest, which the proxy's `connect-src` admits, as the framework's does. */
|
|
354
|
+
const operationsOrigin = (run: CloudflareDeployRun): { readonly operationsOrigin?: string } => {
|
|
355
|
+
if (run.operationsOrigin !== undefined) return { operationsOrigin: run.operationsOrigin }
|
|
356
|
+
// A job from an older Control names no origin; derive it from the DSN until no release in the window omits it.
|
|
357
|
+
const dsn = managedValue(run.managedEnvironment, 'FABRIKA_OPERATIONS_DSN')
|
|
358
|
+
if (dsn === null) return {}
|
|
193
359
|
let origin: string
|
|
194
360
|
try {
|
|
195
361
|
origin = new URL(dsn).origin
|
|
@@ -249,9 +415,81 @@ interface StepEnv {
|
|
|
249
415
|
readonly platformIam: PlatformIamTarget | undefined
|
|
250
416
|
readonly reconcileSchema: SchemaReconciler
|
|
251
417
|
readonly bounds: DeploymentBounds
|
|
418
|
+
/** The managed credentials the application Worker's version is uploaded with, as Worker secrets. */
|
|
419
|
+
readonly managedSecrets: Readonly<Record<string, string>>
|
|
420
|
+
/** Where the build step regenerates the entry, after `pipeline.build`; null for a release that keeps its authored `main`. */
|
|
421
|
+
readonly entryBuild: EntryBuild | null
|
|
422
|
+
/**
|
|
423
|
+
* The mounts whose files the deploy stages into the application Worker's assets, from the application's root; null
|
|
424
|
+
* when the Worker declares its own `assets` or no mount serves files.
|
|
425
|
+
*/
|
|
426
|
+
readonly spaAssets: { readonly mounts: readonly AssetMount[]; readonly root: string } | null
|
|
252
427
|
generatedConfigs: readonly GeneratedConfig[]
|
|
253
428
|
}
|
|
254
429
|
|
|
430
|
+
/**
|
|
431
|
+
* `wrangler deploy`, with the managed secrets uploaded in the same version when there are any, so the version never
|
|
432
|
+
* serves without them, and a plain var of the same name an older release wrote goes with the old version.
|
|
433
|
+
*/
|
|
434
|
+
const wranglerDeploy = async (env: StepEnv, cwd: string, secrets: Readonly<Record<string, string>>): Promise<void> => {
|
|
435
|
+
const { run, credentials, cf } = env
|
|
436
|
+
const names = Object.keys(secrets)
|
|
437
|
+
if (run.dryRun) {
|
|
438
|
+
run.log(` [dry-run] would run: wrangler deploy${names.length === 0 ? '' : ` with the secrets ${names.join(', ')}`} in ${cwd}`)
|
|
439
|
+
return
|
|
440
|
+
}
|
|
441
|
+
assertRunning(run.signal)
|
|
442
|
+
await withSecretsFile(secrets, async (path) => {
|
|
443
|
+
const args = path === null ? ['deploy'] : ['deploy', '--secrets-file', path]
|
|
444
|
+
const result = await cf.runCommand({ command: 'wrangler', args, cwd, env: wranglerEnv(credentials), signal: run.signal })
|
|
445
|
+
if (result.exitCode !== 0) throw commandError('wrangler deploy', result)
|
|
446
|
+
})
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
/** Delete the framework secrets the application Worker holds that this deploy no longer writes, as Zerops removes them. */
|
|
450
|
+
const deleteStaleSecrets = async (env: StepEnv, cwd: string): Promise<void> => {
|
|
451
|
+
const { run, credentials, cf } = env
|
|
452
|
+
if (run.dryRun) {
|
|
453
|
+
run.log(` [dry-run] would delete the managed secrets this deploy no longer writes in ${cwd}`)
|
|
454
|
+
return
|
|
455
|
+
}
|
|
456
|
+
assertRunning(run.signal)
|
|
457
|
+
const listed = await cf.runCommand({
|
|
458
|
+
command: 'wrangler',
|
|
459
|
+
args: ['secret', 'list', '--format', 'json'],
|
|
460
|
+
cwd,
|
|
461
|
+
env: wranglerEnv(credentials),
|
|
462
|
+
signal: run.signal,
|
|
463
|
+
})
|
|
464
|
+
if (listed.exitCode !== 0) throw commandError('wrangler secret list', listed)
|
|
465
|
+
for (const name of staleManagedSecrets(parseSecretNames(listed.stdout), env.managedSecrets)) {
|
|
466
|
+
assertRunning(run.signal)
|
|
467
|
+
// A piped stdin keeps wrangler non-interactive, so its confirmation takes its fallback answer.
|
|
468
|
+
const result = await cf.runCommand({
|
|
469
|
+
command: 'wrangler',
|
|
470
|
+
args: ['secret', 'delete', name],
|
|
471
|
+
cwd,
|
|
472
|
+
env: wranglerEnv(credentials),
|
|
473
|
+
stdin: '',
|
|
474
|
+
signal: run.signal,
|
|
475
|
+
})
|
|
476
|
+
if (result.exitCode !== 0) throw commandError(`wrangler secret delete ${name}`, result)
|
|
477
|
+
run.log(` deleted the managed secret ${name}, which this deploy no longer writes`)
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
/** Stage the mounts' files beside the application Worker's generated config, where its `assets.directory` points. */
|
|
482
|
+
const stageAssets = async (env: StepEnv, assets: NonNullable<StepEnv['spaAssets']>, generatedDir: string): Promise<void> => {
|
|
483
|
+
const mounts = assets.mounts.map((mount) => `${mount.prefix} from ${mount.dir}`).join(', ')
|
|
484
|
+
if (env.run.dryRun) {
|
|
485
|
+
env.run.log(` [dry-run] would stage the SPA files into ${SPA_ASSETS_DIRECTORY}: ${mounts}`)
|
|
486
|
+
return
|
|
487
|
+
}
|
|
488
|
+
assertRunning(env.run.signal)
|
|
489
|
+
const staged = await stageSpaAssets(assets.mounts, assets.root, generatedDir)
|
|
490
|
+
env.run.log(` staged ${staged} SPA files into ${SPA_ASSETS_DIRECTORY}: ${mounts}`)
|
|
491
|
+
}
|
|
492
|
+
|
|
255
493
|
const runStep = async (spec: CloudflareStep, env: StepEnv): Promise<void> => {
|
|
256
494
|
const { config, run, credentials, worker, dir, cf } = env
|
|
257
495
|
const { signal, dryRun } = run
|
|
@@ -259,14 +497,14 @@ const runStep = async (spec: CloudflareStep, env: StepEnv): Promise<void> => {
|
|
|
259
497
|
switch (spec.kind) {
|
|
260
498
|
case 'build': {
|
|
261
499
|
const build = config.pipeline?.build
|
|
262
|
-
if (build
|
|
263
|
-
if (dryRun) {
|
|
500
|
+
if (build !== undefined && dryRun) {
|
|
264
501
|
run.log(` [dry-run] would run build: \`${build}\` in ${dir}`)
|
|
265
|
-
|
|
502
|
+
} else if (build !== undefined) {
|
|
503
|
+
assertRunning(signal)
|
|
504
|
+
const result = await cf.runCommand({ command: 'sh', args: ['-c', build], cwd: dir, omitEnv: PLATFORM_CREDENTIAL_NAMES, signal })
|
|
505
|
+
if (result.exitCode !== 0) throw commandError(`build (\`${build}\`)`, result)
|
|
266
506
|
}
|
|
267
|
-
|
|
268
|
-
const result = await cf.runCommand({ command: 'sh', args: ['-c', build], cwd: dir, omitEnv: PLATFORM_CREDENTIAL_NAMES, signal })
|
|
269
|
-
if (result.exitCode !== 0) throw commandError(`build (\`${build}\`)`, result)
|
|
507
|
+
if (env.entryBuild !== null) await buildEntry(env.entryBuild, run, cf)
|
|
270
508
|
return
|
|
271
509
|
}
|
|
272
510
|
case 'provision-resources': {
|
|
@@ -311,26 +549,20 @@ const runStep = async (spec: CloudflareStep, env: StepEnv): Promise<void> => {
|
|
|
311
549
|
// The configs oblaka wrote, not the generation checked before the run: a stateful binding can emit differently twice.
|
|
312
550
|
assertGeneratedConfigs(configs, env.bounds)
|
|
313
551
|
if (configs.length === 0) {
|
|
314
|
-
|
|
315
|
-
run.log(` [dry-run] would run: wrangler deploy in ${dir}`)
|
|
316
|
-
return
|
|
317
|
-
}
|
|
318
|
-
assertRunning(signal)
|
|
319
|
-
const result = await cf.runCommand({ command: 'wrangler', args: ['deploy'], cwd: dir, env: wranglerEnv(credentials), signal })
|
|
320
|
-
if (result.exitCode !== 0) throw commandError('wrangler deploy', result)
|
|
552
|
+
await wranglerDeploy(env, dir, env.managedSecrets)
|
|
321
553
|
return
|
|
322
554
|
}
|
|
555
|
+
const application = `${run.env}-${applicationWorker(worker).options.name}`
|
|
323
556
|
for (const generated of configs) {
|
|
324
557
|
const generatedDir = dirname(resolve(dir, generated.path))
|
|
558
|
+
const isApplication = generated.config.name === application
|
|
559
|
+
// The checkout's build has written the mounts' directories; the walk below checks the staged copy.
|
|
560
|
+
if (isApplication && env.spaAssets !== null) await stageAssets(env, env.spaAssets, generatedDir)
|
|
325
561
|
// The checkout's own build step has run by now, so this is the last moment the tree is still ours to check.
|
|
326
562
|
if (env.platformResources === 'refuse') await assertLiteralAssets(generated, generatedDir)
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
}
|
|
331
|
-
assertRunning(signal)
|
|
332
|
-
const result = await cf.runCommand({ command: 'wrangler', args: ['deploy'], cwd: generatedDir, env: wranglerEnv(credentials), signal })
|
|
333
|
-
if (result.exitCode !== 0) throw commandError('wrangler deploy', result)
|
|
563
|
+
await wranglerDeploy(env, generatedDir, isApplication ? env.managedSecrets : {})
|
|
564
|
+
// A platform component's own secrets carry framework names (IAM's mail transport), so only an application's are swept.
|
|
565
|
+
if (isApplication && env.platformResources === 'refuse') await deleteStaleSecrets(env, generatedDir)
|
|
334
566
|
}
|
|
335
567
|
return
|
|
336
568
|
}
|
|
@@ -424,6 +656,8 @@ export interface CloudflareDeployRun {
|
|
|
424
656
|
readonly domain?: string
|
|
425
657
|
/** The environment's cross-origin list from Control's registry (ADR-0022); the proxy Worker's manifest entry lists it. */
|
|
426
658
|
readonly crossOrigins?: readonly string[]
|
|
659
|
+
/** The origin of the environment's Operations ingest, from Control; the proxy Worker's `connect-src` admits it. */
|
|
660
|
+
readonly operationsOrigin?: string
|
|
427
661
|
/** Every origin IAM may hand this environment a session at, registered under `<app>@<env>`. */
|
|
428
662
|
readonly returnOrigins?: readonly string[]
|
|
429
663
|
readonly cwd: string
|
|
@@ -475,18 +709,32 @@ export const createCloudflareDeployer = (options: CloudflareDeployerOptions) =>
|
|
|
475
709
|
if (platformResources === 'refuse' && served === null) {
|
|
476
710
|
throw new Error('cloudflare: the checkout has no `fabrika.contract.ts`; Fabrika deploys an application only with a contract')
|
|
477
711
|
}
|
|
712
|
+
if (platformResources === 'refuse' && contract !== null) assertCloudflareContract(contract)
|
|
478
713
|
// Read once, before `resources()` runs the checkout's code again: nothing below reads the config's schema.
|
|
479
714
|
const schema = configSchema(loaded.config)
|
|
480
715
|
assertContractSchema(schema, contract)
|
|
481
716
|
const authored = runnerWorkerGraph(loaded.config.resources({ env: run.env, domain: run.domain }))
|
|
482
|
-
//
|
|
483
|
-
const
|
|
717
|
+
// Only a release Control named pins the entry or serves ops; a local deploy names none and keeps its authored `main`.
|
|
718
|
+
const release = run.approved?.framework
|
|
719
|
+
const opsRelease = release !== undefined && servesOps(release)
|
|
720
|
+
// Fabrika's own components compose their own proxies; every application gets the platform's. Without a domain no
|
|
721
|
+
// Custom Domain carries the ops channel, so the proxy gets no ops row (ADR-0069).
|
|
722
|
+
const proxied = platformResources === 'refuse' && served !== null
|
|
723
|
+
? platformProxied(authored, served, run, opsRelease && run.domain !== undefined)
|
|
724
|
+
: authored
|
|
484
725
|
if (platformResources === 'refuse') assertNoFrameworkVariables(loaded.config, applicationWorker(proxied), run)
|
|
485
726
|
const connectors = platformResources === 'refuse' ? connectorBindings(contract) : {}
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
)
|
|
727
|
+
// An older release's entry exports no object, and a Control older than the runner names no release (ADR-0062).
|
|
728
|
+
const objectRuntime = run.approved === undefined || (run.approved.framework !== undefined && runsInObject(run.approved.framework))
|
|
729
|
+
const entry = release !== undefined && pinsGeneratedEntry(release) ? resolve(loaded.cwd, GENERATED_ENTRY_FILE) : null
|
|
730
|
+
const runtime = platformResources === 'refuse' && contract !== null && objectRuntime ? applicationRuntime(run, contract, entry) : null
|
|
731
|
+
const managed = withManagedEnvironment(loaded.config, proxied, run, contractEnvironment(run.managedEnvironment, served))
|
|
732
|
+
const identified = platformResources === 'refuse' ? withIamIdentity(managed.worker, appEnvironmentId(run.appId, run.env)) : managed.worker
|
|
733
|
+
const connected = withConnectorBindings(identified, connectors)
|
|
734
|
+
// An application Worker that declares its own `assets` keeps them; otherwise the platform stages the mounts' files.
|
|
735
|
+
const mounts = runtime !== null && contract !== null ? assetMounts(contract) : []
|
|
736
|
+
const spaAssets = mounts.length > 0 && applicationWorker(connected).options.assets === undefined ? { mounts, root: loaded.cwd } : null
|
|
737
|
+
const worker = runtime === null ? connected : withApplicationRuntime(connected, run, runtime, opsRelease, spaAssets !== null)
|
|
490
738
|
const dir = workerDir(loaded.config, loaded.cwd)
|
|
491
739
|
const bounds: DeploymentBounds = {
|
|
492
740
|
env: run.env,
|
|
@@ -501,6 +749,8 @@ export const createCloudflareDeployer = (options: CloudflareDeployerOptions) =>
|
|
|
501
749
|
application: frameworkValues(applicationWorker(worker).options.vars),
|
|
502
750
|
proxy: proxied === authored ? null : frameworkValues(worker.options.vars),
|
|
503
751
|
},
|
|
752
|
+
proxyManifest: proxied === authored ? null : (worker.options.define?.[PROXY_MANIFEST_DEFINE] ?? null),
|
|
753
|
+
application: runtime,
|
|
504
754
|
}
|
|
505
755
|
assertStateNamespace(stateNamespace(loaded.config, run), platformResources)
|
|
506
756
|
assertGeneration(await generateWranglerConfigs(worker, run.env), bounds)
|
|
@@ -516,8 +766,13 @@ export const createCloudflareDeployer = (options: CloudflareDeployerOptions) =>
|
|
|
516
766
|
platformIam: options.platformIam,
|
|
517
767
|
reconcileSchema: options.reconcileSchema ?? defaultReconcileSchema,
|
|
518
768
|
bounds,
|
|
769
|
+
managedSecrets: managed.secrets,
|
|
770
|
+
entryBuild: runtime !== null && runtime.entry !== null
|
|
771
|
+
? { cwd: loaded.cwd, config: basename(resolve(run.cwd, run.configPath)), root: resolve(run.cwd) }
|
|
772
|
+
: null,
|
|
773
|
+
spaAssets,
|
|
519
774
|
generatedConfigs: [],
|
|
520
775
|
}
|
|
521
|
-
const result = await runSteps(buildPlan(loaded.config, worker), run, (step) => runStep(step, env))
|
|
776
|
+
const result = await runSteps(buildPlan(loaded.config, worker, env.entryBuild !== null), run, (step) => runStep(step, env))
|
|
522
777
|
return { appId: run.appId, env: run.env, ...result }
|
|
523
778
|
}
|
package/src/proxy-worker.ts
CHANGED
|
@@ -7,11 +7,14 @@ import {
|
|
|
7
7
|
DEFAULT_VERIFY_PATH,
|
|
8
8
|
DELEGATION_HEADER,
|
|
9
9
|
ENDPOINT_HEADER,
|
|
10
|
+
errorAnswer,
|
|
11
|
+
type ErrorPageStatus,
|
|
10
12
|
FORWARD_AUTHORIZATION_HEADER,
|
|
11
13
|
FORWARDED_HOST_HEADER,
|
|
12
14
|
FORWARDED_METHOD_HEADER,
|
|
13
15
|
FORWARDED_PROTO_HEADER,
|
|
14
16
|
FORWARDED_URI_HEADER,
|
|
17
|
+
isPageNavigation,
|
|
15
18
|
MemoryTokenCache,
|
|
16
19
|
OAUTH_METADATA_VERIFY_PATH,
|
|
17
20
|
PREFLIGHT_HEADER,
|
|
@@ -49,6 +52,7 @@ const CLOUDFLARE_CLIENT_ADDRESS_HEADER = 'CF-Connecting-IP'
|
|
|
49
52
|
|
|
50
53
|
export interface CloudflareProxyEnv {
|
|
51
54
|
readonly IAM?: unknown
|
|
55
|
+
/** The manifest JSON the handler serves; the deployed Worker passes the one baked into its bundle. */
|
|
52
56
|
readonly FABRIKA_PROXY_MANIFEST_JSON?: string
|
|
53
57
|
readonly FABRIKA_IAM_ISSUER?: string
|
|
54
58
|
readonly [binding: string]: unknown
|
|
@@ -60,10 +64,13 @@ interface WorkerService {
|
|
|
60
64
|
|
|
61
65
|
type ProxyFetch = (request: Request) => Promise<Response>
|
|
62
66
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
+
/**
|
|
68
|
+
* A failure the Worker answers itself: the status's page to a navigation that does not select a download endpoint
|
|
69
|
+
* (whose file treatment would save the page as a file), its fixed text to everything else.
|
|
70
|
+
*/
|
|
71
|
+
function failure(status: ErrorPageStatus, text: string, request: Request, download: boolean, manifest: ProxyManifest | null): Response {
|
|
72
|
+
return errorAnswer(status, text, !download && isPageNavigation(request.headers), manifest?.brand ?? null)
|
|
73
|
+
}
|
|
67
74
|
|
|
68
75
|
let cached: {
|
|
69
76
|
readonly manifestJson: string
|
|
@@ -123,7 +130,7 @@ export function createCloudflareProxyHandler(env: CloudflareProxyEnv, logger: Pr
|
|
|
123
130
|
function unavailable(manifest: ProxyManifest | null): ProxyFetch {
|
|
124
131
|
return (request) => {
|
|
125
132
|
const app = manifest === null ? null : findApp(manifest, new URL(request.url))
|
|
126
|
-
const answer =
|
|
133
|
+
const answer = failure(503, 'proxy unavailable', request, false, manifest)
|
|
127
134
|
return Promise.resolve(
|
|
128
135
|
app === null || !isApplicationEntry(app) ? answer : withResponseHeaders(answer, contractAnswerHeaders(app)(null, answer.status)),
|
|
129
136
|
)
|
|
@@ -164,14 +171,14 @@ function proxyHandler(env: CloudflareProxyEnv, manifest: ProxyManifest, verify:
|
|
|
164
171
|
// An admitted preflight: the decision's 2xx is the whole answer, and the application never sees the request.
|
|
165
172
|
if (verification.headers.get(PREFLIGHT_HEADER) === '1') return withResponseHeaders(verification, (headers) => headers.delete(PREFLIGHT_HEADER))
|
|
166
173
|
|
|
167
|
-
if (app === null) return
|
|
174
|
+
if (app === null) return failure(404, 'not found', request, false, manifest)
|
|
168
175
|
const treatment = answerHeaders.get(app.id)
|
|
176
|
+
const endpoint = verification.headers.get(ENDPOINT_HEADER)
|
|
177
|
+
const download = selectsDownload(app, endpoint)
|
|
169
178
|
const upstream = env[app.upstream]
|
|
170
179
|
if (!isWorkerService(upstream)) {
|
|
171
|
-
const unavailable =
|
|
172
|
-
return treatment === undefined
|
|
173
|
-
? unavailable
|
|
174
|
-
: withResponseHeaders(unavailable, treatment(verification.headers.get(ENDPOINT_HEADER), unavailable.status))
|
|
180
|
+
const unavailable = failure(503, 'proxy unavailable', request, download, manifest)
|
|
181
|
+
return treatment === undefined ? unavailable : withResponseHeaders(unavailable, treatment(endpoint, unavailable.status))
|
|
175
182
|
}
|
|
176
183
|
|
|
177
184
|
// Built from the ORIGINAL headers, so it is stripped again here — the set above was applied to the
|
|
@@ -198,13 +205,13 @@ function proxyHandler(env: CloudflareProxyEnv, manifest: ProxyManifest, verify:
|
|
|
198
205
|
answer = await upstream.fetch(new Request(request, { headers: upstreamHeaders }))
|
|
199
206
|
} catch {
|
|
200
207
|
// Nothing of the error is answered or logged: it can carry the upstream request, a token header included.
|
|
201
|
-
answer =
|
|
208
|
+
answer = failure(502, 'bad gateway', request, download, manifest)
|
|
202
209
|
}
|
|
203
210
|
if (treatment === undefined) return answer
|
|
204
211
|
// The treatment deleted the application's own `Access-Control-*`; only the origin the decision admitted is set.
|
|
205
212
|
const allowOrigin = verification.headers.get(ALLOW_ORIGIN_HEADER)
|
|
206
213
|
return withResponseHeaders(answer, (headers) => {
|
|
207
|
-
treatment(
|
|
214
|
+
treatment(endpoint, answer.status)(headers)
|
|
208
215
|
if (allowOrigin !== null) applyCrossOriginHeaders(headers, crossOriginHeaders(allowOrigin))
|
|
209
216
|
})
|
|
210
217
|
}
|
|
@@ -238,6 +245,13 @@ function contractAnswerHeaders(app: ContractProxyApp): AnswerHeaders {
|
|
|
238
245
|
}
|
|
239
246
|
}
|
|
240
247
|
|
|
248
|
+
/** Whether the decision selected a download endpoint of a contract application. */
|
|
249
|
+
function selectsDownload(app: ProxyApp, endpoint: string | null): boolean {
|
|
250
|
+
if (!isApplicationEntry(app) || endpoint === null) return false
|
|
251
|
+
const id = parseEndpointHeader(endpoint)?.id
|
|
252
|
+
return app.endpoints.some((row) => row.kind === 'download' && row.id === id)
|
|
253
|
+
}
|
|
254
|
+
|
|
241
255
|
/** What only the decision may assert to the app; each is copied from an allow, the way Caddy's `copy_headers` does. */
|
|
242
256
|
const PROXY_ASSERTED_HEADERS = [PROXY_TOKEN_HEADER, REQUEST_ID_HEADER, ENDPOINT_HEADER, DELEGATION_HEADER]
|
|
243
257
|
|
|
@@ -296,8 +310,14 @@ function isObject(value: unknown): value is Record<string, unknown> {
|
|
|
296
310
|
return typeof value === 'object' && value !== null
|
|
297
311
|
}
|
|
298
312
|
|
|
313
|
+
/** Replaced with the manifest's string literal when wrangler bundles the Worker (`PROXY_MANIFEST_DEFINE` in `./proxy`). */
|
|
314
|
+
declare const __FABRIKA_PROXY_MANIFEST__: string | undefined
|
|
315
|
+
|
|
316
|
+
// `typeof` reads an identifier no bundle defined without throwing: an unbaked bundle serves its fixed 503.
|
|
317
|
+
const BAKED_MANIFEST_JSON: string = typeof __FABRIKA_PROXY_MANIFEST__ === 'string' ? __FABRIKA_PROXY_MANIFEST__ : ''
|
|
318
|
+
|
|
299
319
|
export default {
|
|
300
320
|
fetch(request: Request, env: CloudflareProxyEnv): Promise<Response> {
|
|
301
|
-
return createCloudflareProxyHandler(env)(request)
|
|
321
|
+
return createCloudflareProxyHandler({ ...env, FABRIKA_PROXY_MANIFEST_JSON: BAKED_MANIFEST_JSON })(request)
|
|
302
322
|
},
|
|
303
323
|
}
|