@fabrika/provider-zerops 0.4.6 → 0.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -8
- package/src/api/client.ts +10 -0
- package/src/api/readers.ts +38 -0
- package/src/api/types.ts +47 -0
- package/src/api.ts +1 -0
- package/src/control.ts +3 -1
- package/src/execution.ts +15 -1
- package/src/index.ts +2 -0
- package/src/installation/admin-options.ts +2 -2
- package/src/installation/guide.md +46 -12
- package/src/installation/index.ts +5 -2
- package/src/installation/init.ts +27 -5
- package/src/installation/install.ts +1 -1
- package/src/namespace/errors.ts +5 -0
- package/src/namespace/operator.ts +8 -1
- package/src/plan.ts +6 -0
- package/src/provider.ts +68 -1
- package/src/source/port.ts +11 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fabrika/provider-zerops",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.7",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -43,13 +43,13 @@
|
|
|
43
43
|
},
|
|
44
44
|
"dependencies": {
|
|
45
45
|
"zod": "^3.25.76",
|
|
46
|
-
"@fabrika/auth": "
|
|
47
|
-
"@fabrika/auth-core": "
|
|
48
|
-
"@fabrika/iam-contract": "
|
|
49
|
-
"@fabrika/installation-init": "
|
|
50
|
-
"@fabrika/operations-contract": "
|
|
51
|
-
"@fabrika/provider-contract": "
|
|
52
|
-
"@fabrika/proxy-core": "
|
|
46
|
+
"@fabrika/auth": "0.4.7",
|
|
47
|
+
"@fabrika/auth-core": "0.4.7",
|
|
48
|
+
"@fabrika/iam-contract": "0.4.7",
|
|
49
|
+
"@fabrika/installation-init": "0.4.7",
|
|
50
|
+
"@fabrika/operations-contract": "0.4.7",
|
|
51
|
+
"@fabrika/provider-contract": "0.4.7",
|
|
52
|
+
"@fabrika/proxy-core": "0.4.7",
|
|
53
53
|
"yaml": "^2.9.1"
|
|
54
54
|
}
|
|
55
55
|
}
|
package/src/api/client.ts
CHANGED
|
@@ -354,6 +354,16 @@ export const createZeropsApi = (options: ZeropsApiOptions): ZeropsApi => {
|
|
|
354
354
|
return readResponse('configure autoscaling', processNilSchema, payload).process ?? null
|
|
355
355
|
},
|
|
356
356
|
|
|
357
|
+
configureCustomAutoscaling: async ({ serviceId, autoscaling, signal }) => {
|
|
358
|
+
const payload = await request('configure custom autoscaling', 'PUT', `/service-stack/${serviceId}/autoscaling`, {
|
|
359
|
+
body: {
|
|
360
|
+
customAutoscaling: { verticalAutoscaling: autoscaling.vertical, horizontalAutoscaling: autoscaling.horizontal },
|
|
361
|
+
},
|
|
362
|
+
signal,
|
|
363
|
+
})
|
|
364
|
+
return readResponse('configure custom autoscaling', processNilSchema, payload).process ?? null
|
|
365
|
+
},
|
|
366
|
+
|
|
357
367
|
getService: async ({ serviceId, signal }) =>
|
|
358
368
|
readResponse('get service', serviceSchema, await request('get service', 'GET', `/service-stack/${serviceId}`, { signal })),
|
|
359
369
|
|
package/src/api/readers.ts
CHANGED
|
@@ -12,6 +12,7 @@ import type {
|
|
|
12
12
|
ZeropsProjectStatus,
|
|
13
13
|
ZeropsRoleCode,
|
|
14
14
|
ZeropsService,
|
|
15
|
+
ZeropsServiceAutoscaling,
|
|
15
16
|
ZeropsServiceEnv,
|
|
16
17
|
ZeropsServiceStatus,
|
|
17
18
|
ZeropsUserInfo,
|
|
@@ -121,6 +122,41 @@ export const processSchema = z.object({
|
|
|
121
122
|
appVersion: nestedIdSchema,
|
|
122
123
|
}).transform(({ appVersion, ...process }): ZeropsProcess => ({ ...process, appVersionId: appVersion?.id }))
|
|
123
124
|
|
|
125
|
+
const nullableNumber = z.number().nullish().transform((value) => value ?? null)
|
|
126
|
+
|
|
127
|
+
const scalingResourceSchema = z.object({ cpuCoreCount: nullableNumber, memoryGBytes: nullableNumber, diskGBytes: nullableNumber }).nullish()
|
|
128
|
+
.transform((resource) => resource ?? { cpuCoreCount: null, memoryGBytes: null, diskGBytes: null })
|
|
129
|
+
|
|
130
|
+
const scalingFreeResourceSchema = z.object({
|
|
131
|
+
cpuCoreCount: nullableNumber,
|
|
132
|
+
cpuCorePercent: nullableNumber,
|
|
133
|
+
memoryGBytes: nullableNumber,
|
|
134
|
+
memoryPercent: nullableNumber,
|
|
135
|
+
}).nullish().transform((resource) => resource ?? { cpuCoreCount: null, cpuCorePercent: null, memoryGBytes: null, memoryPercent: null })
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Strict where `optional` is lenient: the custom autoscaling is sent back whole, so a field read as absent would be
|
|
139
|
+
* reset. One that does not parse makes the whole object absent, and nothing is sent.
|
|
140
|
+
*/
|
|
141
|
+
const verticalAutoscalingSchema = z.object({
|
|
142
|
+
cpuMode: z.enum(['DEDICATED', 'SHARED']).nullish().transform((value) => value ?? null),
|
|
143
|
+
minResource: scalingResourceSchema,
|
|
144
|
+
maxResource: scalingResourceSchema,
|
|
145
|
+
minFreeResource: scalingFreeResourceSchema,
|
|
146
|
+
startCpuCoreCount: nullableNumber,
|
|
147
|
+
swapEnabled: z.boolean().nullish().transform((value) => value ?? null),
|
|
148
|
+
})
|
|
149
|
+
|
|
150
|
+
const serviceAutoscalingSchema = z.object({
|
|
151
|
+
// A service with no custom vertical autoscaling reads `null` here (measured 2026-10-15): every field inherits.
|
|
152
|
+
verticalAutoscaling: verticalAutoscalingSchema.nullish().transform((vertical) => vertical ?? verticalAutoscalingSchema.parse({})),
|
|
153
|
+
horizontalAutoscaling: z.object({ minContainerCount: nullableNumber, maxContainerCount: nullableNumber }).nullish()
|
|
154
|
+
.transform((horizontal) => horizontal ?? { minContainerCount: null, maxContainerCount: null }),
|
|
155
|
+
}).transform(({ verticalAutoscaling, horizontalAutoscaling }): ZeropsServiceAutoscaling => ({
|
|
156
|
+
vertical: verticalAutoscaling,
|
|
157
|
+
horizontal: horizontalAutoscaling,
|
|
158
|
+
}))
|
|
159
|
+
|
|
124
160
|
export const serviceSchema = z.object({
|
|
125
161
|
id: idSchema,
|
|
126
162
|
name: z.string(),
|
|
@@ -130,6 +166,8 @@ export const serviceSchema = z.object({
|
|
|
130
166
|
activeAppVersion: nestedIdSchema,
|
|
131
167
|
subdomainAccess: optional(z.boolean()),
|
|
132
168
|
autoscalingProfileId: optional(z.string()),
|
|
169
|
+
customAutoscaling: optional(serviceAutoscalingSchema),
|
|
170
|
+
currentAutoscaling: optional(serviceAutoscalingSchema),
|
|
133
171
|
}).transform(({ activeAppVersion, ...service }): ZeropsService => ({ ...service, activeAppVersionId: activeAppVersion?.id }))
|
|
134
172
|
|
|
135
173
|
export const projectSchema = z.object({
|
package/src/api/types.ts
CHANGED
|
@@ -162,6 +162,40 @@ export type ZeropsServiceStatus =
|
|
|
162
162
|
| 'SCALING'
|
|
163
163
|
| 'REPAIR_FAILED'
|
|
164
164
|
|
|
165
|
+
/** One bound of vertical autoscaling. `null` inherits the service type's value. */
|
|
166
|
+
export interface ZeropsScalingResource {
|
|
167
|
+
cpuCoreCount: number | null
|
|
168
|
+
memoryGBytes: number | null
|
|
169
|
+
diskGBytes: number | null
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** The free capacity below which vertical autoscaling scales up. `null` inherits the service type's value. */
|
|
173
|
+
export interface ZeropsScalingFreeResource {
|
|
174
|
+
cpuCoreCount: number | null
|
|
175
|
+
cpuCorePercent: number | null
|
|
176
|
+
memoryGBytes: number | null
|
|
177
|
+
memoryPercent: number | null
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* A service's autoscaling, as `ResponseServiceStack.customAutoscaling` (what was set) and `.currentAutoscaling` (what
|
|
182
|
+
* applies) carry it, every field nullable. VERIFIED live 2026-10-15 (`docs/reference/zerops-platform.md`).
|
|
183
|
+
*/
|
|
184
|
+
export interface ZeropsServiceAutoscaling {
|
|
185
|
+
vertical: {
|
|
186
|
+
cpuMode: 'DEDICATED' | 'SHARED' | null
|
|
187
|
+
minResource: ZeropsScalingResource
|
|
188
|
+
maxResource: ZeropsScalingResource
|
|
189
|
+
minFreeResource: ZeropsScalingFreeResource
|
|
190
|
+
startCpuCoreCount: number | null
|
|
191
|
+
swapEnabled: boolean | null
|
|
192
|
+
}
|
|
193
|
+
horizontal: {
|
|
194
|
+
minContainerCount: number | null
|
|
195
|
+
maxContainerCount: number | null
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
165
199
|
/** One service. VERIFIED: `ResponseServiceStack` (narrowed to the fields fabrika reads). */
|
|
166
200
|
export interface ZeropsService {
|
|
167
201
|
id: string
|
|
@@ -175,6 +209,10 @@ export interface ZeropsService {
|
|
|
175
209
|
subdomainAccess?: boolean
|
|
176
210
|
/** Provider profile identifier, e.g. `oltp-production`. VERIFIED: `ResponseServiceStack.autoscalingProfileId`. */
|
|
177
211
|
autoscalingProfileId?: string
|
|
212
|
+
/** The autoscaling set on the service; absent when the platform sent none or one this client cannot read whole. */
|
|
213
|
+
customAutoscaling?: ZeropsServiceAutoscaling
|
|
214
|
+
/** The autoscaling that applies, the service type's values included; absent like `customAutoscaling`. */
|
|
215
|
+
currentAutoscaling?: ZeropsServiceAutoscaling
|
|
178
216
|
}
|
|
179
217
|
|
|
180
218
|
/** One project. VERIFIED: `ResponseProject` (narrowed). */
|
|
@@ -447,6 +485,15 @@ export interface ZeropsApi {
|
|
|
447
485
|
*/
|
|
448
486
|
configureAutoscaling(input: { serviceId: string; autoscalingProfileId: string; signal: AbortSignal }): Promise<ZeropsProcess | null>
|
|
449
487
|
|
|
488
|
+
/**
|
|
489
|
+
* Replace a service's custom autoscaling: the same `PUT /service-stack/{id}/autoscaling`, body
|
|
490
|
+
* `{ customAutoscaling: { verticalAutoscaling, horizontalAutoscaling } }`. VERIFIED live 2026-10-15: it answers
|
|
491
|
+
* `{ process: null }` and `currentAutoscaling` reads the change at once. The platform replaces the whole object: a
|
|
492
|
+
* field left out, or under a key it does not know, is reset to the service type's value and the call still answers
|
|
493
|
+
* 200, so the caller sends what `getService().customAutoscaling` read, changed where it means to.
|
|
494
|
+
*/
|
|
495
|
+
configureCustomAutoscaling(input: { serviceId: string; autoscaling: ZeropsServiceAutoscaling; signal: AbortSignal }): Promise<ZeropsProcess | null>
|
|
496
|
+
|
|
450
497
|
// ── service-stack / project: reading state ────────────────────────────────────
|
|
451
498
|
|
|
452
499
|
/** One service by id. VERIFIED: `GET /service-stack/{id}`. */
|
package/src/api.ts
CHANGED
package/src/control.ts
CHANGED
|
@@ -434,7 +434,9 @@ export const createZeropsControlProvider = (options: ZeropsControlProviderOption
|
|
|
434
434
|
if (!checkpoint.triggered) {
|
|
435
435
|
// A throw keeps the run running; the next maintenance pass retries the deletion.
|
|
436
436
|
await deleteVersion(api, checkpoint.appVersionId, signal)
|
|
437
|
-
await log(
|
|
437
|
+
await log(
|
|
438
|
+
`deploy interrupted: Control stopped before it requested the build; deleted app-version ${checkpoint.appVersionId}; run the deploy again`,
|
|
439
|
+
)
|
|
438
440
|
return { state: 'failed' }
|
|
439
441
|
}
|
|
440
442
|
let version: ZeropsAppVersion | null
|
package/src/execution.ts
CHANGED
|
@@ -55,6 +55,20 @@ const FILE_SWEEP_SCHEDULE = '41 * * * *'
|
|
|
55
55
|
*/
|
|
56
56
|
export const ZEROPS_SCAN_MIN_RAM_GB = 2
|
|
57
57
|
|
|
58
|
+
/**
|
|
59
|
+
* The RAM in GB a container of an application with a database starts with. Its start script runs `fabrika db
|
|
60
|
+
* test-isolation`, whose CLI and `bun test` child together peak at 161 MiB of anonymous memory for the Notes example
|
|
61
|
+
* and 225 MiB for the CRM example (measured locally, 2026-10-15), beside Bun's own mapped binary. Zerops' 0.125 GB
|
|
62
|
+
* floor kills that within a second, faster than vertical autoscaling reacts.
|
|
63
|
+
*/
|
|
64
|
+
export const ZEROPS_START_MIN_RAM_GB = 0.5
|
|
65
|
+
|
|
66
|
+
/** The minimum RAM in GB the deploy service's containers need to start, or null when Zerops' floor suffices. */
|
|
67
|
+
export function zeropsStartMinRam(execution: ZeropsLockedExecution): number | null {
|
|
68
|
+
if (execution.contract.scan === true) return ZEROPS_SCAN_MIN_RAM_GB
|
|
69
|
+
return execution.databaseUrl === null ? null : ZEROPS_START_MIN_RAM_GB
|
|
70
|
+
}
|
|
71
|
+
|
|
58
72
|
/** What a scanning application adds: ClamAV in the image with its first signatures, refreshed at start and daily. */
|
|
59
73
|
const SCANNER_PACKAGE = 'clamav'
|
|
60
74
|
const FRESHCLAM = 'sudo freshclam --stdout --quiet'
|
|
@@ -113,7 +127,7 @@ export function zeropsContractRun(artifact: ContractArtifact): ZeropsContractRun
|
|
|
113
127
|
/**
|
|
114
128
|
* A scanning application's deploy service must start its containers with `ZEROPS_SCAN_MIN_RAM_GB`: below it the
|
|
115
129
|
* scanner is killed for memory, and every upload stays quarantined. Zerops ignores a changed `verticalAutoscaling` of
|
|
116
|
-
* an existing service on a re-import, so
|
|
130
|
+
* an existing service on a re-import, so the deploy's `raise-start-memory` step raises an existing service.
|
|
117
131
|
*/
|
|
118
132
|
export function assertScanMemory(services: readonly { readonly [key: string]: JsonValue }[], deployService: string, run: ZeropsContractRun): void {
|
|
119
133
|
if (run.scan !== true) return
|
package/src/index.ts
CHANGED
|
@@ -29,6 +29,7 @@ export type {
|
|
|
29
29
|
ZeropsProjectStatus,
|
|
30
30
|
ZeropsRoleCode,
|
|
31
31
|
ZeropsService,
|
|
32
|
+
ZeropsServiceAutoscaling,
|
|
32
33
|
ZeropsServiceEnv,
|
|
33
34
|
ZeropsServiceEnvWrite,
|
|
34
35
|
ZeropsServiceStatus,
|
|
@@ -108,6 +109,7 @@ export { ZEROPS_PROXY_MANIFEST_PATH, type ZeropsProxyManifestCarrier, zeropsProx
|
|
|
108
109
|
export type * from './schema.generated'
|
|
109
110
|
export { isZeropsUploadDestination, normalizeZeropsSourceRepository } from './source'
|
|
110
111
|
export type { ZeropsSourceRepository } from './source'
|
|
112
|
+
export { ZeropsSourceRateLimitedError } from './source/port'
|
|
111
113
|
export type {
|
|
112
114
|
ZeropsSourceArchive,
|
|
113
115
|
ZeropsSourcePort,
|
|
@@ -19,7 +19,7 @@ export interface PlatformAdminInput {
|
|
|
19
19
|
readonly iamHost: string
|
|
20
20
|
/** The scheme a browser speaks to this installation. With `iamHost` it composes IAM's public origin. */
|
|
21
21
|
readonly scheme: 'http' | 'https'
|
|
22
|
-
/** The `px_` provisioning key `platform install`
|
|
22
|
+
/** The `px_` provisioning key `platform install` wrote to its secrets file. Environment only. */
|
|
23
23
|
readonly provisioningKey: string
|
|
24
24
|
/** Replace an outstanding enrollment instead of reporting it. */
|
|
25
25
|
readonly reissue: boolean
|
|
@@ -78,7 +78,7 @@ const provisioningKey = (env: Record<string, string | undefined>): string => {
|
|
|
78
78
|
const value = readEnv(env, 'FABRIKA_IAM_PROVISIONING_KEY')
|
|
79
79
|
if (value === undefined) {
|
|
80
80
|
throw new Error(
|
|
81
|
-
'FABRIKA_IAM_PROVISIONING_KEY is required. It has no flag: the
|
|
81
|
+
'FABRIKA_IAM_PROVISIONING_KEY is required. It has no flag: load it from the secrets file platform install wrote, into the environment only.',
|
|
82
82
|
)
|
|
83
83
|
}
|
|
84
84
|
return value
|
|
@@ -152,8 +152,10 @@ URL stops working.
|
|
|
152
152
|
|
|
153
153
|
The sidecar is a GitHub repository the operator owns. Its workflow runs `platform deploy` at the pinned tag.
|
|
154
154
|
|
|
155
|
-
|
|
156
|
-
|
|
155
|
+
The installation from step 1 already runs, and the sidecar is what upgrades it later (step 6). But `init`
|
|
156
|
+
triggers the workflow, and its run redeploys every platform service, control included. A deploy that is in
|
|
157
|
+
flight when control restarts fails, and its log names no reason. So run this step BEFORE step 5, and wait for
|
|
158
|
+
its run to end before the first `control deploy` (see the check below). Step 4 can run meanwhile.
|
|
157
159
|
|
|
158
160
|
`init` takes the integration token step 1 minted, never the personal one. The secrets file holds it as
|
|
159
161
|
`FABRIKA_ZEROPS_ACCESS_TOKEN`, beside the provisioning key. Run `init` from a directory that is NOT inside
|
|
@@ -169,9 +171,16 @@ one too. Otherwise pass `--checkout=<dir>` outside one:
|
|
|
169
171
|
- Without `--repo` the repository is `contember/fabrika-zerops-<installation>`. Pass `--repo` for any
|
|
170
172
|
other organization.
|
|
171
173
|
- Keep the local checkout path. `platform upgrade` needs it later (`--sidecar=<dir>`).
|
|
174
|
+
- `--tag` is written only into a NEW sidecar. An existing one keeps its `fabrika.ref`. When that pins another
|
|
175
|
+
tag, `init` writes the GitHub Environment and then refuses to trigger the run, which would deploy the other
|
|
176
|
+
tag. Its message names the fix: `fabrika platform upgrade --provider=zerops --to=<tag> --sidecar=<dir>`.
|
|
172
177
|
|
|
173
|
-
**Check.** The `platform` workflow run in the sidecar repository ends green:
|
|
174
|
-
|
|
178
|
+
**Check.** The `platform` workflow run in the sidecar repository ends green. Wait for it before step 5:
|
|
179
|
+
|
|
180
|
+
```bash
|
|
181
|
+
gh run list --repo <owner>/<name> --workflow=platform.yml --limit=1 # its id, status and conclusion
|
|
182
|
+
gh run watch <run id> --repo <owner>/<name> --exit-status # returns when the run ends; non-zero if it failed
|
|
183
|
+
```
|
|
175
184
|
|
|
176
185
|
## Step 4. Mint a machine key for `fabrika control`
|
|
177
186
|
|
|
@@ -210,17 +219,22 @@ places it on the namespace's shared PostgreSQL (`cheap` preset). Use the config
|
|
|
210
219
|
**Repository access decides how you register** (step 5.2). Check it first:
|
|
211
220
|
`curl -s -o /dev/null -w '%{http_code}\n' https://api.github.com/repos/<owner>/<name>`.
|
|
212
221
|
|
|
213
|
-
- `200`: the repository is public. Register it with `--installation-id=none`. Control reads it without a token
|
|
222
|
+
- `200`: the repository is public. Register it with `--installation-id=none`. Control reads it without a token,
|
|
223
|
+
or with the organization's GitHub App once that App is connected and installed on the repository.
|
|
214
224
|
- `404`: GitHub hides it from an anonymous caller, so it is private (or the name is wrong). `register` refuses
|
|
215
225
|
it with `--installation-id=none`. Ask the human to connect the GitHub App in the console first (Settings →
|
|
216
226
|
Source, a browser flow), then register without `--installation-id`.
|
|
217
227
|
|
|
218
228
|
**Use the installation's release in the application.** The application's `@fabrika/*` packages, the CLI
|
|
219
|
-
among them, must be the version of the tag you installed: for `v0.4.3`,
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
229
|
+
among them, must be exactly the version of the tag you installed: for `v0.4.3`, `0.4.3`, not a range such as
|
|
230
|
+
`^0.4.3`, which resolves a later patch. Another version can build a manifest this installation's control
|
|
231
|
+
refuses or reads differently. The example's `main` pins the latest release; for an older installation, from
|
|
232
|
+
the fabrika clone of step 0, run `bun scripts/release.ts example-pin <application checkout> --tag=<tag>`. It
|
|
233
|
+
pins every `@fabrika/*` package exactly and commits nothing. Run `bun install` in the application repository,
|
|
234
|
+
then commit and push `package.json` and `bun.lock`. Check that every `@fabrika/*` package, the transitive ones
|
|
235
|
+
included, is the tag's version: `bun pm ls --all | grep @fabrika/`. Releases up to v0.4.6 name their own
|
|
236
|
+
`@fabrika/*` dependencies by a range, so a transitive one can still resolve a later patch of that line. Then
|
|
237
|
+
install the newest patch instead (step 6 upgrades an installation).
|
|
224
238
|
|
|
225
239
|
1. **Create a namespace** with the preset and core package the human chose (see Decisions). Without a
|
|
226
240
|
custom domain bound to its proxy, pass `--public-access=zerops-subdomain`; the default is
|
|
@@ -232,6 +246,9 @@ Check with `bun pm ls | grep @fabrika/cli`.
|
|
|
232
246
|
fabrika control namespaces get --namespace=<id>
|
|
233
247
|
```
|
|
234
248
|
|
|
249
|
+
The namespace's Zerops project is `fabrika-<env>-<namespace id>`, beside the installation's
|
|
250
|
+
`fabrika-<env>`. Pass `--project=<name>` for another name; it cannot change after `create`.
|
|
251
|
+
|
|
235
252
|
`create` returns while the state is `pending`. Provisioning takes several minutes. Poll `get` every
|
|
236
253
|
30 seconds until it is `ready`.
|
|
237
254
|
|
|
@@ -263,8 +280,12 @@ Check with `bun pm ls | grep @fabrika/cli`.
|
|
|
263
280
|
once and refuses one that is not publicly readable, so the failure shows here and not at deploy. A
|
|
264
281
|
private repository needs the GitHub App connected in the console first (Settings → Source, a browser
|
|
265
282
|
flow for the human); then omit the flag.
|
|
283
|
+
- The public origin is where IAM returns a person after sign-in; without one, sign-in to the application
|
|
284
|
+
fails with "… is not a registered return origin for this app". For a host the namespace lists, `register`
|
|
285
|
+
sets `https://<free host>` itself and prints `public origin https://<host>` on stderr. For a custom domain
|
|
286
|
+
the namespace cannot list, add `--public-origin=https://<domain>`.
|
|
266
287
|
- `register` CREATES and refuses a second registration. A changed manifest goes through
|
|
267
|
-
`control apps environments put` with the same placement flags.
|
|
288
|
+
`control apps environments put` with the same placement flags. `put` derives no public origin.
|
|
268
289
|
|
|
269
290
|
3. **Deploy and follow it.**
|
|
270
291
|
|
|
@@ -275,7 +296,13 @@ Check with `bun pm ls | grep @fabrika/cli`.
|
|
|
275
296
|
```
|
|
276
297
|
|
|
277
298
|
A failed run's log ends with `deploy failed: <reason>`. For example, `source resolve failed
|
|
278
|
-
(ref_not_found)` means control could not read the repository or the ref.
|
|
299
|
+
(ref_not_found)` means control could not read the repository or the ref. `GitHub rate limit for
|
|
300
|
+
anonymous reads reached; resets at <time>` means GitHub refused control's tokenless reads: every tenant
|
|
301
|
+
behind the cloud's egress address shares that limit. Deploy again after the reset, or connect the GitHub
|
|
302
|
+
App in the console so reads of that organization's repositories carry a token. A namespace that hits the
|
|
303
|
+
limit stays `provisioning` and retries on its own. A run that ends with
|
|
304
|
+
`deploy interrupted: …; run the deploy again` lost Control mid-run, typically to step 3's workflow
|
|
305
|
+
redeploying it: deploy again once Control is back.
|
|
279
306
|
|
|
280
307
|
The deploy fails when the repository at the deployed commit does not match the registered manifest.
|
|
281
308
|
Commit and push, rebuild the manifest, `apps environments put` it, and deploy again.
|
|
@@ -283,6 +310,13 @@ Check with `bun pm ls | grep @fabrika/cli`.
|
|
|
283
310
|
On an environment named `prod`, a change that widens access waits for a person to approve it in the
|
|
284
311
|
console before it deploys.
|
|
285
312
|
|
|
313
|
+
4. **Give the administrator a role in the application.** The step 2 grant administers the installation; it
|
|
314
|
+
holds none of the application's actions, so the application answers `forbidden` to the administrator.
|
|
315
|
+
No `fabrika` command grants an application role. The human does it in the console: Access → Users → the
|
|
316
|
+
administrator → Add grant. Pick the application `<id>@<env>` (the example: `notes@<env>`), Named role / policy,
|
|
317
|
+
a role the application declares (the example's `editor` reads, writes and deletes notes), the global scope,
|
|
318
|
+
and Add grant. It takes effect at the next sign-in.
|
|
319
|
+
|
|
286
320
|
**Check.** The run ends `succeeded`. `https://<host>/healthz` answers 200, and `/` redirects to the IAM
|
|
287
321
|
sign-in. A green run proves only that Zerops accepted the build: have the human sign in and use the app.
|
|
288
322
|
|
|
@@ -172,13 +172,16 @@ It writes TWO secrets into that Environment and GENERATES NEITHER — both alrea
|
|
|
172
172
|
installation, so a value invented here would not be one it accepts. Both are environment only:
|
|
173
173
|
|
|
174
174
|
FABRIKA_ZEROPS_ACCESS_TOKEN required the INTEGRATION token install minted (the \`control\` service's env)
|
|
175
|
-
FABRIKA_IAM_PROVISIONING_KEY required the px_ key install
|
|
175
|
+
FABRIKA_IAM_PROVISIONING_KEY required the px_ key install wrote to its --secrets-file
|
|
176
176
|
FABRIKA_ZEROPS_API_URL optional region API base, when not the default
|
|
177
177
|
|
|
178
178
|
Each is sent to GitHub over \`gh\` stdin, and never printed and never written to disk.
|
|
179
179
|
|
|
180
180
|
The generated pipeline pins a published TAG of contember/fabrika-platform and refuses a branch, at
|
|
181
181
|
init and again in the workflow. It uploads that checkout's exact committed archive to every installation service.
|
|
182
|
+
\`--tag\` is written only to a new sidecar: an existing \`fabrika.ref\` is the operator's and stays. When it pins
|
|
183
|
+
another tag, init writes the Environment and then refuses to trigger the run, which would deploy that other tag;
|
|
184
|
+
\`platform upgrade --to=<tag>\` moves the pin and starts the run.
|
|
182
185
|
|
|
183
186
|
── platform deploy ───────────────────────────────────────────────────────────────────────────────
|
|
184
187
|
|
|
@@ -275,7 +278,7 @@ Options (a flag beats the environment variable beside it):
|
|
|
275
278
|
--scheme=<http|https> FABRIKA_PLATFORM_SCHEME default https
|
|
276
279
|
${REISSUE_FLAG} replace an outstanding enrollment
|
|
277
280
|
|
|
278
|
-
FABRIKA_IAM_PROVISIONING_KEY required the px_ admin key \`install\`
|
|
281
|
+
FABRIKA_IAM_PROVISIONING_KEY required the px_ admin key \`install\` wrote to its --secrets-file, environment only
|
|
279
282
|
|
|
280
283
|
── platform upgrade ──────────────────────────────────────────────────────────────────────────────
|
|
281
284
|
|
package/src/installation/init.ts
CHANGED
|
@@ -258,6 +258,12 @@ const readPlatform = async (input: InitInput, { log, prompts, effects }: InitCol
|
|
|
258
258
|
})
|
|
259
259
|
}
|
|
260
260
|
|
|
261
|
+
/** The local checkout and the tag its `fabrika.ref` pins, which an existing sidecar keeps whatever `--tag` says. */
|
|
262
|
+
interface ScaffoldedSidecar {
|
|
263
|
+
readonly dir: string
|
|
264
|
+
readonly pinned: string
|
|
265
|
+
}
|
|
266
|
+
|
|
261
267
|
/**
|
|
262
268
|
* Create or refresh the sidecar repository — the first step that leaves this disk.
|
|
263
269
|
*
|
|
@@ -265,7 +271,7 @@ const readPlatform = async (input: InitInput, { log, prompts, effects }: InitCol
|
|
|
265
271
|
* outward step: an Environment cannot be written to a repository that does not exist, and a workflow
|
|
266
272
|
* cannot be triggered without one.
|
|
267
273
|
*/
|
|
268
|
-
const scaffold = async (input: InitInput, { log, prompts, effects }: InitCollaborators): Promise<
|
|
274
|
+
const scaffold = async (input: InitInput, { log, prompts, effects }: InitCollaborators): Promise<ScaffoldedSidecar | undefined> => {
|
|
269
275
|
log.step('Scaffold the sidecar repository')
|
|
270
276
|
const exists = await effects.repositoryExists(input.repo)
|
|
271
277
|
const question = exists
|
|
@@ -293,7 +299,22 @@ const scaffold = async (input: InitInput, { log, prompts, effects }: InitCollabo
|
|
|
293
299
|
if (pinned !== input.fabrikaRef) {
|
|
294
300
|
log.warn(`the existing pin ${pinned} was kept; ${input.fabrikaRef} was not written, because fabrika.ref is operator-owned`)
|
|
295
301
|
}
|
|
296
|
-
return result.dir
|
|
302
|
+
return { dir: result.dir, pinned }
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/**
|
|
306
|
+
* Refuse to trigger a run that would deploy a tag other than `--tag`: the run deploys the kept pin, so a sidecar reused
|
|
307
|
+
* for a new installation would roll an older release over it. The Environment is already written, so the pin moves by
|
|
308
|
+
* `upgrade`, whose push triggers the run with it.
|
|
309
|
+
*/
|
|
310
|
+
const assertPinnedAsRequested = (input: InitInput, sidecar: ScaffoldedSidecar): void => {
|
|
311
|
+
if (sidecar.pinned === input.fabrikaRef) return
|
|
312
|
+
throw new Error(
|
|
313
|
+
`the platform workflow was NOT triggered: ${input.repo} pins ${sidecar.pinned}, not ${input.fabrikaRef}, and a run deploys the pin. `
|
|
314
|
+
+ `The ${input.installation} Environment is written. To deploy ${input.fabrikaRef}, move the pin: `
|
|
315
|
+
+ `fabrika platform upgrade --provider=zerops --to=${input.fabrikaRef} --sidecar=${sidecar.dir}. `
|
|
316
|
+
+ `To deploy ${sidecar.pinned}, run init again with --tag=${sidecar.pinned}`,
|
|
317
|
+
)
|
|
297
318
|
}
|
|
298
319
|
|
|
299
320
|
/** Run the whole thing for `<installation>`. */
|
|
@@ -320,8 +341,8 @@ export const runInit = async (input: InitInput, collaborators: InitCollaborators
|
|
|
320
341
|
: [url(settingsUrl)],
|
|
321
342
|
)
|
|
322
343
|
|
|
323
|
-
const
|
|
324
|
-
if (
|
|
344
|
+
const sidecar = await scaffold(input, collaborators)
|
|
345
|
+
if (sidecar === undefined) {
|
|
325
346
|
return
|
|
326
347
|
}
|
|
327
348
|
|
|
@@ -338,6 +359,7 @@ export const runInit = async (input: InitInput, collaborators: InitCollaborators
|
|
|
338
359
|
log.ok(`Environment ${input.installation} written: ${Object.keys(config.secrets).join(', ')} and ${Object.keys(config.vars).join(', ')}.`)
|
|
339
360
|
|
|
340
361
|
log.step('Trigger the deploy')
|
|
362
|
+
assertPinnedAsRequested(input, sidecar)
|
|
341
363
|
log.info("The pipeline runs the whole ordered sequence in the operator's CI; this machine deploys nothing.")
|
|
342
364
|
if (!(await prompts.confirm(`Run the platform workflow on ${input.repo} now?`, true))) {
|
|
343
365
|
log.action('OPERATOR ACTION — run it when ready', [
|
|
@@ -351,7 +373,7 @@ export const runInit = async (input: InitInput, collaborators: InitCollaborators
|
|
|
351
373
|
}
|
|
352
374
|
|
|
353
375
|
log.step('Done')
|
|
354
|
-
log.ok(`Sidecar repository: ${input.repo} (local checkout: ${dir})`)
|
|
376
|
+
log.ok(`Sidecar repository: ${input.repo} (local checkout: ${sidecar.dir})`)
|
|
355
377
|
log.info('Roll this installation forward by bumping fabrika.ref to a newer published tag and pushing.')
|
|
356
378
|
log.info("Who may administer the installation is IAM's to say: this repository holds no admission list to close later.")
|
|
357
379
|
// The origin comes off the live control service, never from what was typed here: an operator who
|
|
@@ -131,7 +131,7 @@ const installPreparedPlatform = async (
|
|
|
131
131
|
log.ok(
|
|
132
132
|
'IAM signing keys (1 × ES256), the IAM RPC key, the proxy key, the provisioning key, the Operations sync key and the vault key',
|
|
133
133
|
)
|
|
134
|
-
log.info(
|
|
134
|
+
log.info(`None of them is printed: every one goes to ${input.secretsFile} (mode 0600) before the first is placed.`)
|
|
135
135
|
|
|
136
136
|
log.step("Mint the control plane's Zerops token")
|
|
137
137
|
log.info('An INTEGRATION token, client role NO_ACCESS with ADMIN on this one project — never the personal token you authenticated with.')
|
package/src/namespace/errors.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { ProviderNamespaceError } from '@fabrika/provider-contract'
|
|
2
2
|
import { ZeropsApiError, ZeropsServiceEnvValueTooLargeError } from '../api'
|
|
3
|
+
import { ZeropsSourceRateLimitedError } from '../source/port'
|
|
3
4
|
|
|
4
5
|
/**
|
|
5
6
|
* Lifecycle codes this provider defines where the platform has none of its own. They are an
|
|
@@ -16,6 +17,8 @@ export const ZEROPS_PROXY_DEPLOY_FAILED = 'proxyDeployFailed'
|
|
|
16
17
|
|
|
17
18
|
export const ZEROPS_SUBDOMAIN_NOT_PUBLISHED = 'subdomainNotPublished'
|
|
18
19
|
|
|
20
|
+
export const ZEROPS_SOURCE_RATE_LIMITED = 'githubRateLimited'
|
|
21
|
+
|
|
19
22
|
export const namespaceError = (code: string, message: string, retryable: boolean, detail?: string): ProviderNamespaceError =>
|
|
20
23
|
new ProviderNamespaceError(message, code, retryable, detail)
|
|
21
24
|
|
|
@@ -43,6 +46,8 @@ export const asNamespaceError = (cause: unknown, mode: 'provision' | 'reconcile'
|
|
|
43
46
|
if (cause instanceof ProviderNamespaceError) return cause
|
|
44
47
|
// The same value is refused again on every retry; only a smaller manifest or a new carrier fixes it.
|
|
45
48
|
if (cause instanceof ZeropsServiceEnvValueTooLargeError) return namespaceError(cause.code, cause.message, false)
|
|
49
|
+
// The limit resets on its own: the namespace stays `provisioning` and the cron's reconcile retries it.
|
|
50
|
+
if (cause instanceof ZeropsSourceRateLimitedError) return namespaceError(ZEROPS_SOURCE_RATE_LIMITED, cause.message, true)
|
|
46
51
|
if (cause instanceof ZeropsApiError) {
|
|
47
52
|
const { summary, detail } = splitApiMessage(cause.message, cause.code)
|
|
48
53
|
return namespaceError(
|
|
@@ -103,6 +103,13 @@ const namespacePresentation = (
|
|
|
103
103
|
}
|
|
104
104
|
}
|
|
105
105
|
|
|
106
|
+
/**
|
|
107
|
+
* The Zerops project a namespace creates when the plan names none: `fabrika-<env>-<id>`, beside the `fabrika-<env>` that
|
|
108
|
+
* `platform install` creates for the installation, so a namespace reads as fabrika's in the Zerops GUI.
|
|
109
|
+
*/
|
|
110
|
+
export const defaultZeropsNamespaceProjectName = (namespace: { readonly id: string; readonly env: string }): string =>
|
|
111
|
+
`fabrika-${namespace.env}-${namespace.id}`
|
|
112
|
+
|
|
106
113
|
/** Build the provider-owned namespace planning surface without any network collaborators. */
|
|
107
114
|
export const createZeropsNamespaceOperator = (
|
|
108
115
|
options: ZeropsNamespaceOperatorOptions,
|
|
@@ -130,7 +137,7 @@ export const createZeropsNamespaceOperator = (
|
|
|
130
137
|
const target = zeropsNamespacePreset({
|
|
131
138
|
preset: input.preset,
|
|
132
139
|
env: input.env,
|
|
133
|
-
projectName: planOptions.projectName ?? input
|
|
140
|
+
projectName: planOptions.projectName ?? defaultZeropsNamespaceProjectName(input),
|
|
134
141
|
proxySource: options.proxySource,
|
|
135
142
|
...(planOptions.corePackage === undefined ? {} : { corePackage: planOptions.corePackage }),
|
|
136
143
|
...(planOptions.publicAccess === undefined ? {} : { publicAccess: planOptions.publicAccess }),
|
package/src/plan.ts
CHANGED
|
@@ -5,6 +5,7 @@ export type ZeropsStepKind =
|
|
|
5
5
|
| 'apply-import'
|
|
6
6
|
| 'verify-exposure'
|
|
7
7
|
| 'write-service-env'
|
|
8
|
+
| 'raise-start-memory'
|
|
8
9
|
| 'reserve-version'
|
|
9
10
|
| 'upload-source'
|
|
10
11
|
| 'trigger-build'
|
|
@@ -43,6 +44,11 @@ export const buildZeropsPlan = (manifest: FabrikaManifest, target: ZeropsRuntime
|
|
|
43
44
|
description: 'Read back that no service has subdomain access and every bucket is one the platform created private',
|
|
44
45
|
})
|
|
45
46
|
steps.push({ id: 'write-service-env', kind: 'write-service-env', description: 'Write the managed service variables' })
|
|
47
|
+
steps.push({
|
|
48
|
+
id: 'raise-start-memory',
|
|
49
|
+
kind: 'raise-start-memory',
|
|
50
|
+
description: "Raise the deploy service's minimum RAM to what its start script needs",
|
|
51
|
+
})
|
|
46
52
|
steps.push({ id: 'reserve-version', kind: 'reserve-version', description: 'Reserve a Zerops app version' })
|
|
47
53
|
steps.push({ id: 'upload-source', kind: 'upload-source', description: 'Upload the resolved repository snapshot' })
|
|
48
54
|
steps.push({ id: 'trigger-build', kind: 'trigger-build', description: 'Trigger the Zerops build+deploy pipeline' })
|
package/src/provider.ts
CHANGED
|
@@ -33,10 +33,11 @@ import {
|
|
|
33
33
|
type ZeropsAppVersion,
|
|
34
34
|
type ZeropsLogAccess,
|
|
35
35
|
type ZeropsProcess,
|
|
36
|
+
type ZeropsServiceAutoscaling,
|
|
36
37
|
} from './api'
|
|
37
38
|
import { defaultZeropsCollaborators, type ZeropsCollaboratorFactory, type ZeropsCollaborators } from './collaborators'
|
|
38
39
|
import { assertZeropsInvariants, renderYaml } from './compile'
|
|
39
|
-
import { FABRIKA_CHECK_VERSION, PLATFORM_TYPESCRIPT_VERSION, renderLockedExecutionDescriptor } from './execution'
|
|
40
|
+
import { FABRIKA_CHECK_VERSION, PLATFORM_TYPESCRIPT_VERSION, renderLockedExecutionDescriptor, zeropsStartMinRam } from './execution'
|
|
40
41
|
import {
|
|
41
42
|
createdServices,
|
|
42
43
|
type ExposureFinding,
|
|
@@ -642,6 +643,67 @@ const writeServiceEnv = async (env: StepEnv): Promise<void> => {
|
|
|
642
643
|
}
|
|
643
644
|
}
|
|
644
645
|
|
|
646
|
+
/** The custom autoscaling with its minimum RAM raised to `minRam`; every other field is sent back as it was read. */
|
|
647
|
+
const withMinRam = (custom: ZeropsServiceAutoscaling, minRam: number): ZeropsServiceAutoscaling => ({
|
|
648
|
+
...custom,
|
|
649
|
+
vertical: { ...custom.vertical, minResource: { ...custom.vertical.minResource, memoryGBytes: minRam } },
|
|
650
|
+
})
|
|
651
|
+
|
|
652
|
+
/**
|
|
653
|
+
* Raises the deploy service's minimum RAM to what the generated start script needs (`zeropsStartMinRam`), before the
|
|
654
|
+
* version's containers start. An import cannot: Zerops ignores a changed `verticalAutoscaling` of an existing service,
|
|
655
|
+
* so an existing service converges here, on every deploy. It only raises, and decides by reading the service back.
|
|
656
|
+
*/
|
|
657
|
+
const raiseStartMemory = async (env: StepEnv): Promise<void> => {
|
|
658
|
+
const { run, zerops, log, signal, dryRun } = env
|
|
659
|
+
const locked = lockedExecutionFor(run.artifact, run, dryRun)
|
|
660
|
+
const minRam = locked === null ? null : zeropsStartMinRam(locked)
|
|
661
|
+
const hostname = run.artifact.target.deployService
|
|
662
|
+
if (minRam === null) {
|
|
663
|
+
log(` ${hostname} needs no more than the platform's minimum RAM`)
|
|
664
|
+
return
|
|
665
|
+
}
|
|
666
|
+
if (dryRun) {
|
|
667
|
+
log(` [dry-run] would raise the minimum RAM of ${hostname} to ${minRam} GB where it is lower`)
|
|
668
|
+
return
|
|
669
|
+
}
|
|
670
|
+
const serviceId = run.target.serviceId
|
|
671
|
+
assertRunning(signal)
|
|
672
|
+
const service = await zerops.api.getService({ serviceId, signal })
|
|
673
|
+
const custom = service.customAutoscaling
|
|
674
|
+
const current = service.currentAutoscaling
|
|
675
|
+
if (custom === undefined || current === undefined) {
|
|
676
|
+
throw new Error(`zerops: ${hostname} did not report its autoscaling whole, so its minimum RAM cannot be raised to ${minRam} GB`)
|
|
677
|
+
}
|
|
678
|
+
const before = current.vertical.minResource.memoryGBytes
|
|
679
|
+
if (before !== null && before >= minRam) {
|
|
680
|
+
log(` minimum RAM of ${hostname}: ${before} GB, unchanged (its start needs ${minRam} GB)`)
|
|
681
|
+
return
|
|
682
|
+
}
|
|
683
|
+
const maxRam = current.vertical.maxResource.memoryGBytes
|
|
684
|
+
if (maxRam !== null && maxRam < minRam) {
|
|
685
|
+
throw new Error(`zerops: ${hostname} may use at most ${maxRam} GB of RAM and its start needs ${minRam} GB; raise verticalAutoscaling.maxRam`)
|
|
686
|
+
}
|
|
687
|
+
assertRunning(signal)
|
|
688
|
+
let process: ZeropsProcess | null
|
|
689
|
+
try {
|
|
690
|
+
process = await zerops.api.configureCustomAutoscaling({ serviceId, autoscaling: withMinRam(custom, minRam), signal })
|
|
691
|
+
} catch (error) {
|
|
692
|
+
if (error instanceof ZeropsApiError && error.status >= 400 && error.status < 500 && error.status !== 408) {
|
|
693
|
+
throw new Error(`zerops: raising the minimum RAM of ${hostname} to ${minRam} GB was refused (${error.code === '' ? error.status : error.code})`)
|
|
694
|
+
}
|
|
695
|
+
throw error
|
|
696
|
+
}
|
|
697
|
+
if (process !== null) {
|
|
698
|
+
await waitForProcess({ api: zerops.api, processId: process.id, sleep: zerops.sleep, signal, label: `the autoscaling change of ${hostname}` })
|
|
699
|
+
}
|
|
700
|
+
const after = (await zerops.api.getService({ serviceId, signal })).currentAutoscaling?.vertical.minResource.memoryGBytes ?? null
|
|
701
|
+
if (after === null || after < minRam) {
|
|
702
|
+
throw new Error(`zerops: ${hostname} reads back a minimum RAM of ${after ?? 'nothing'} GB after it was raised to ${minRam} GB`)
|
|
703
|
+
}
|
|
704
|
+
log(` minimum RAM of ${hostname}: ${before ?? 'the platform default'} → ${after} GB`)
|
|
705
|
+
}
|
|
706
|
+
|
|
645
707
|
/** Run one step's effect. Resolves on success, throws on failure (caught + recorded by `runSteps`). */
|
|
646
708
|
const runStep = async (spec: ZeropsStep, env: StepEnv): Promise<void> => {
|
|
647
709
|
const { run, zerops, compiled, log, signal, dryRun, attempt } = env
|
|
@@ -696,6 +758,11 @@ const runStep = async (spec: ZeropsStep, env: StepEnv): Promise<void> => {
|
|
|
696
758
|
return
|
|
697
759
|
}
|
|
698
760
|
|
|
761
|
+
case 'raise-start-memory': {
|
|
762
|
+
await raiseStartMemory(env)
|
|
763
|
+
return
|
|
764
|
+
}
|
|
765
|
+
|
|
699
766
|
case 'reserve-version': {
|
|
700
767
|
if (dryRun) {
|
|
701
768
|
log(` [dry-run] would reserve an app version on service ${target.serviceId}`)
|
package/src/source/port.ts
CHANGED
|
@@ -55,3 +55,14 @@ export interface ZeropsSourcePort {
|
|
|
55
55
|
resolve(input: ZeropsSourceResolveInput): Promise<ZeropsSourceResolution>
|
|
56
56
|
upload(input: ZeropsSourceUploadInput): Promise<ZeropsSourceUploadResult>
|
|
57
57
|
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* The source refused a read for GitHub's rate limit. The read succeeds again once the limit resets, so a namespace
|
|
61
|
+
* mutation that hits it stays retryable. The message is the user's: whose limit, when it resets, how to avoid it.
|
|
62
|
+
*/
|
|
63
|
+
export class ZeropsSourceRateLimitedError extends Error {
|
|
64
|
+
constructor(message: string) {
|
|
65
|
+
super(message)
|
|
66
|
+
this.name = 'ZeropsSourceRateLimitedError'
|
|
67
|
+
}
|
|
68
|
+
}
|