@reventlessdev/reventless-seed-aws 1.0.0-alpha.24 → 1.0.0-alpha.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/lib/bs/.compiler.log +2 -2
- package/lib/bs/compiler-info.json +1 -1
- package/lib/bs/src/ReventlessSeedAws.cmi +0 -0
- package/lib/bs/src/ReventlessSeedAws.cmt +0 -0
- package/lib/bs/src/ReventlessSeedAws_Quiesce.ast +0 -0
- package/lib/bs/src/ReventlessSeedAws_Quiesce.cmi +0 -0
- package/lib/bs/src/ReventlessSeedAws_Quiesce.cmj +0 -0
- package/lib/bs/src/ReventlessSeedAws_Quiesce.cmt +0 -0
- package/lib/bs/src/ReventlessSeedAws_Quiesce.res +23 -5
- package/lib/bs/src/ReventlessSeedAws_Quiesce.res.mjs +13 -1
- package/lib/bs/src/ReventlessSeedAws_Reconcile.cmi +0 -0
- package/lib/bs/src/ReventlessSeedAws_Reconcile.cmt +0 -0
- package/lib/bs/src/ReventlessSeedAws_Reset.cmi +0 -0
- package/lib/bs/src/ReventlessSeedAws_Reset.cmt +0 -0
- package/lib/bs/tests/EndpointResolutionTest.cmi +0 -0
- package/lib/bs/tests/EndpointResolutionTest.cmt +0 -0
- package/lib/bs/tests/ObjectStoreReconcileTest.cmi +0 -0
- package/lib/bs/tests/ObjectStoreReconcileTest.cmt +0 -0
- package/lib/bs/tests/ObjectStoreResetTest.cmi +0 -0
- package/lib/bs/tests/ObjectStoreResetTest.cmt +0 -0
- package/lib/bs/tests/QuiesceResetTest.ast +0 -0
- package/lib/bs/tests/QuiesceResetTest.cmi +0 -0
- package/lib/bs/tests/QuiesceResetTest.cmt +0 -0
- package/lib/bs/tests/QuiesceResetTest.res +24 -0
- package/lib/bs/tests/QuiesceResetTest.res.mjs +16 -0
- package/lib/ocaml/.compiler.log +2 -2
- package/lib/ocaml/EndpointResolutionTest.cmi +0 -0
- package/lib/ocaml/EndpointResolutionTest.cmt +0 -0
- package/lib/ocaml/ObjectStoreReconcileTest.cmi +0 -0
- package/lib/ocaml/ObjectStoreReconcileTest.cmt +0 -0
- package/lib/ocaml/ObjectStoreResetTest.cmi +0 -0
- package/lib/ocaml/ObjectStoreResetTest.cmt +0 -0
- package/lib/ocaml/QuiesceResetTest.ast +0 -0
- package/lib/ocaml/QuiesceResetTest.cmi +0 -0
- package/lib/ocaml/QuiesceResetTest.cmt +0 -0
- package/lib/ocaml/QuiesceResetTest.res +24 -0
- package/lib/ocaml/ReventlessSeedAws.cmi +0 -0
- package/lib/ocaml/ReventlessSeedAws.cmt +0 -0
- package/lib/ocaml/ReventlessSeedAws_Quiesce.ast +0 -0
- package/lib/ocaml/ReventlessSeedAws_Quiesce.cmi +0 -0
- package/lib/ocaml/ReventlessSeedAws_Quiesce.cmj +0 -0
- package/lib/ocaml/ReventlessSeedAws_Quiesce.cmt +0 -0
- package/lib/ocaml/ReventlessSeedAws_Quiesce.res +23 -5
- package/lib/ocaml/ReventlessSeedAws_Reconcile.cmi +0 -0
- package/lib/ocaml/ReventlessSeedAws_Reconcile.cmt +0 -0
- package/lib/ocaml/ReventlessSeedAws_Reset.cmi +0 -0
- package/lib/ocaml/ReventlessSeedAws_Reset.cmt +0 -0
- package/package.json +3 -3
- package/src/ReventlessSeedAws_Quiesce.res +23 -5
- package/src/ReventlessSeedAws_Quiesce.res.mjs +13 -1
- package/tests/QuiesceResetTest.res +24 -0
- package/tests/QuiesceResetTest.res.mjs +16 -0
package/CHANGELOG.md
CHANGED
|
@@ -3,6 +3,21 @@
|
|
|
3
3
|
All notable changes to this project will be documented in this file.
|
|
4
4
|
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
|
5
5
|
|
|
6
|
+
# 1.0.0-alpha.26 (2026-08-27)
|
|
7
|
+
|
|
8
|
+
**Note:** Version bump only for package @reventlessdev/reventless-seed-aws
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
# 1.0.0-alpha.25 (2026-08-27)
|
|
15
|
+
|
|
16
|
+
### Bug Fixes
|
|
17
|
+
|
|
18
|
+
* **seed-aws:** a throttled control plane aborts the reset hold and blames IAM ([95af263](https://github.com/ReventlessDev/reventless-core/commit/95af263afba7dfdfa9a7fc45f494b302dba4baf8))
|
|
19
|
+
|
|
20
|
+
|
|
6
21
|
# 1.0.0-alpha.24 (2026-08-23)
|
|
7
22
|
|
|
8
23
|
**Note:** Version bump only for package @reventlessdev/reventless-seed-aws
|
package/lib/bs/.compiler.log
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
#Start(
|
|
2
|
-
#Done(
|
|
1
|
+
#Start(1787841680994)
|
|
2
|
+
#Done(1787841681025)
|
|
@@ -4,5 +4,5 @@
|
|
|
4
4
|
"bsc_hash": "75e3a59c95cc953608fd3dd6ea6ed68141bba4414a20a52f114261b8a48385fa",
|
|
5
5
|
"rescript_config_hash": "c18a0067c363fd1290f6e7f6fe6212d2e16e201c1702fac667962047f88ed468",
|
|
6
6
|
"runtime_path": "/home/runner/work/reventless-core/reventless-core/node_modules/@rescript/runtime",
|
|
7
|
-
"generated_at": "
|
|
7
|
+
"generated_at": "1787841681029"
|
|
8
8
|
}
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -86,6 +86,17 @@ let mapBounded = async (items: array<'a>, ~limit: int, f: 'a => promise<'b>): ar
|
|
|
86
86
|
let errorText = (exn: exn): string =>
|
|
87
87
|
exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("unknown error")
|
|
88
88
|
|
|
89
|
+
/** A throttle and a missing permission both surface as a failure to hold, and
|
|
90
|
+
they call for opposite responses — wait and re-run, versus go and fix the
|
|
91
|
+
policy. The control plane says "Rate exceeded"; the SDK says
|
|
92
|
+
"TooManyRequestsException". Match on either. */
|
|
93
|
+
let isThrottle = (reason: string): bool => {
|
|
94
|
+
let r = reason->String.toLowerCase
|
|
95
|
+
r->String.includes("rate exceeded") ||
|
|
96
|
+
r->String.includes("toomanyrequests") ||
|
|
97
|
+
r->String.includes("throttl")
|
|
98
|
+
}
|
|
99
|
+
|
|
89
100
|
// An update is rejected while another is still settling, and a stack that was
|
|
90
101
|
// deployed moments ago can still be settling. Poll rather than assume.
|
|
91
102
|
let awaitSettled = async (~client, ~functionName: string): unit => {
|
|
@@ -269,14 +280,21 @@ let hold = async (~client, ~functionNames: array<string>): array<held> => {
|
|
|
269
280
|
| Some((functionName, reason)) =>
|
|
270
281
|
let rollback = await taken->mapBounded(~limit=concurrency, h => restoreConcurrency(~client, ~h))
|
|
271
282
|
let stranded = rollback->Array.filterMap(w => w)
|
|
283
|
+
let advice = if reason->isThrottle {
|
|
284
|
+
`The Lambda control plane throttled the hold. That budget is per account and region and is ` ++
|
|
285
|
+
`shared with anything else using it — a deploy, an inspector sync, another stack in the same ` ++
|
|
286
|
+
`region — so this is transient and nothing is wrong with the credentials. Wait for whatever ` ++
|
|
287
|
+
`else is running to finish and re-run.`
|
|
288
|
+
} else {
|
|
289
|
+
`The reset holds every runtime in scope for the length of the wipe, which needs ` ++
|
|
290
|
+
`lambda:GetFunctionConcurrency, lambda:GetFunctionConfiguration, lambda:PutFunctionConcurrency, ` ++
|
|
291
|
+
`lambda:DeleteFunctionConcurrency and lambda:UpdateFunctionConfiguration.`
|
|
292
|
+
}
|
|
272
293
|
throw(
|
|
273
294
|
ReventlessSeed.Seed.Failed(
|
|
274
295
|
`could not hold Lambda function ${functionName} at zero concurrency (${reason}) — nothing was ` ++
|
|
275
|
-
`deleted.
|
|
276
|
-
`
|
|
277
|
-
`lambda:DeleteFunctionConcurrency and lambda:UpdateFunctionConfiguration. Set ` ++
|
|
278
|
-
`SEED_RESET_NO_QUIESCE=1 to wipe without the hold — but a running runtime can then write back ` ++
|
|
279
|
-
`what the wipe removes.` ++
|
|
296
|
+
`deleted. ${advice} Set SEED_RESET_NO_QUIESCE=1 to wipe without the hold — but a running ` ++
|
|
297
|
+
`runtime can then write back what the wipe removes.` ++
|
|
280
298
|
switch stranded {
|
|
281
299
|
| [] => ""
|
|
282
300
|
| messages => `\n\n ` ++ messages->Array.join("\n ")
|
|
@@ -37,6 +37,15 @@ function errorText(exn) {
|
|
|
37
37
|
return Stdlib_Option.getOr(Stdlib_Option.flatMap(Stdlib_JsExn.fromException(exn), Stdlib_JsExn.message), "unknown error");
|
|
38
38
|
}
|
|
39
39
|
|
|
40
|
+
function isThrottle(reason) {
|
|
41
|
+
let r = reason.toLowerCase();
|
|
42
|
+
if (r.includes("rate exceeded") || r.includes("toomanyrequests")) {
|
|
43
|
+
return true;
|
|
44
|
+
} else {
|
|
45
|
+
return r.includes("throttl");
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
40
49
|
async function awaitSettled(client, functionName) {
|
|
41
50
|
let poll = async attempt => {
|
|
42
51
|
if (attempt >= 60) {
|
|
@@ -174,11 +183,13 @@ async function hold(client, functionNames) {
|
|
|
174
183
|
if (match === undefined) {
|
|
175
184
|
return taken;
|
|
176
185
|
}
|
|
186
|
+
let reason = match[1];
|
|
177
187
|
let rollback = await mapBounded(taken, 6, h => restoreConcurrency(client, h));
|
|
178
188
|
let stranded = Stdlib_Array.filterMap(rollback, w => w);
|
|
189
|
+
let advice = isThrottle(reason) ? `The Lambda control plane throttled the hold. That budget is per account and region and is shared with anything else using it — a deploy, an inspector sync, another stack in the same region — so this is transient and nothing is wrong with the credentials. Wait for whatever else is running to finish and re-run.` : `The reset holds every runtime in scope for the length of the wipe, which needs lambda:GetFunctionConcurrency, lambda:GetFunctionConfiguration, lambda:PutFunctionConcurrency, lambda:DeleteFunctionConcurrency and lambda:UpdateFunctionConfiguration.`;
|
|
179
190
|
throw {
|
|
180
191
|
RE_EXN_ID: Seed$ReventlessSeed.Failed,
|
|
181
|
-
_1: `could not hold Lambda function ` + match[0] + ` at zero concurrency (` +
|
|
192
|
+
_1: `could not hold Lambda function ` + match[0] + ` at zero concurrency (` + reason + `) — nothing was ` + (`deleted. ` + advice + ` Set SEED_RESET_NO_QUIESCE=1 to wipe without the hold — but a running `) + `runtime can then write back what the wipe removes.` + (
|
|
182
193
|
stranded.length !== 0 ? `\n\n ` + stranded.join("\n ") : ""
|
|
183
194
|
),
|
|
184
195
|
Error: new Error()
|
|
@@ -195,6 +206,7 @@ export {
|
|
|
195
206
|
concurrency,
|
|
196
207
|
mapBounded,
|
|
197
208
|
errorText,
|
|
209
|
+
isThrottle,
|
|
198
210
|
awaitSettled,
|
|
199
211
|
updateVariables,
|
|
200
212
|
restoreConcurrency,
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -109,6 +109,30 @@ describe("refillMessage", () => {
|
|
|
109
109
|
})
|
|
110
110
|
})
|
|
111
111
|
|
|
112
|
+
describe("isThrottle", () => {
|
|
113
|
+
// A hold that fails because the control plane is busy and a hold that fails
|
|
114
|
+
// because the credentials are short a policy read identically at the call
|
|
115
|
+
// site, and the fixes point in opposite directions: wait, versus go and edit
|
|
116
|
+
// an IAM policy. Sending an operator to the policy for a throttle costs them
|
|
117
|
+
// the whole trip.
|
|
118
|
+
testSync("recognises the wordings a throttled hold actually comes back with", () =>
|
|
119
|
+
[
|
|
120
|
+
"Rate exceeded",
|
|
121
|
+
"TooManyRequestsException: Rate exceeded",
|
|
122
|
+
"Request throttled, please try again",
|
|
123
|
+
]->Array.forEach(reason => expect(Quiesce.isThrottle(reason))->toBe(true))
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
testSync("leaves a permission failure to the IAM advice", () => {
|
|
127
|
+
expect(
|
|
128
|
+
Quiesce.isThrottle(
|
|
129
|
+
"AccessDeniedException: User is not authorized to perform: lambda:PutFunctionConcurrency",
|
|
130
|
+
),
|
|
131
|
+
)->toBe(false)
|
|
132
|
+
expect(Quiesce.isThrottle("ResourceNotFoundException: Function not found"))->toBe(false)
|
|
133
|
+
})
|
|
134
|
+
})
|
|
135
|
+
|
|
112
136
|
describe("noQuiesce", () => {
|
|
113
137
|
let set = v => NodeProcess.env->Dict.set("SEED_RESET_NO_QUIESCE", v)
|
|
114
138
|
|
|
@@ -104,6 +104,22 @@ globalThis.describe("refillMessage", () => {
|
|
|
104
104
|
});
|
|
105
105
|
});
|
|
106
106
|
|
|
107
|
+
globalThis.describe("isThrottle", () => {
|
|
108
|
+
globalThis.test("recognises the wordings a throttled hold actually comes back with", () => {
|
|
109
|
+
[
|
|
110
|
+
"Rate exceeded",
|
|
111
|
+
"TooManyRequestsException: Rate exceeded",
|
|
112
|
+
"Request throttled, please try again"
|
|
113
|
+
].forEach(reason => {
|
|
114
|
+
globalThis.expect(ReventlessSeedAws_Quiesce.isThrottle(reason)).toBe(true);
|
|
115
|
+
});
|
|
116
|
+
});
|
|
117
|
+
globalThis.test("leaves a permission failure to the IAM advice", () => {
|
|
118
|
+
globalThis.expect(ReventlessSeedAws_Quiesce.isThrottle("AccessDeniedException: User is not authorized to perform: lambda:PutFunctionConcurrency")).toBe(false);
|
|
119
|
+
globalThis.expect(ReventlessSeedAws_Quiesce.isThrottle("ResourceNotFoundException: Function not found")).toBe(false);
|
|
120
|
+
});
|
|
121
|
+
});
|
|
122
|
+
|
|
107
123
|
globalThis.describe("noQuiesce", () => {
|
|
108
124
|
let set = v => {
|
|
109
125
|
process.env["SEED_RESET_NO_QUIESCE"] = v;
|
package/lib/ocaml/.compiler.log
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
#Start(
|
|
2
|
-
#Done(
|
|
1
|
+
#Start(1787841680994)
|
|
2
|
+
#Done(1787841681025)
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -109,6 +109,30 @@ describe("refillMessage", () => {
|
|
|
109
109
|
})
|
|
110
110
|
})
|
|
111
111
|
|
|
112
|
+
describe("isThrottle", () => {
|
|
113
|
+
// A hold that fails because the control plane is busy and a hold that fails
|
|
114
|
+
// because the credentials are short a policy read identically at the call
|
|
115
|
+
// site, and the fixes point in opposite directions: wait, versus go and edit
|
|
116
|
+
// an IAM policy. Sending an operator to the policy for a throttle costs them
|
|
117
|
+
// the whole trip.
|
|
118
|
+
testSync("recognises the wordings a throttled hold actually comes back with", () =>
|
|
119
|
+
[
|
|
120
|
+
"Rate exceeded",
|
|
121
|
+
"TooManyRequestsException: Rate exceeded",
|
|
122
|
+
"Request throttled, please try again",
|
|
123
|
+
]->Array.forEach(reason => expect(Quiesce.isThrottle(reason))->toBe(true))
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
testSync("leaves a permission failure to the IAM advice", () => {
|
|
127
|
+
expect(
|
|
128
|
+
Quiesce.isThrottle(
|
|
129
|
+
"AccessDeniedException: User is not authorized to perform: lambda:PutFunctionConcurrency",
|
|
130
|
+
),
|
|
131
|
+
)->toBe(false)
|
|
132
|
+
expect(Quiesce.isThrottle("ResourceNotFoundException: Function not found"))->toBe(false)
|
|
133
|
+
})
|
|
134
|
+
})
|
|
135
|
+
|
|
112
136
|
describe("noQuiesce", () => {
|
|
113
137
|
let set = v => NodeProcess.env->Dict.set("SEED_RESET_NO_QUIESCE", v)
|
|
114
138
|
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -86,6 +86,17 @@ let mapBounded = async (items: array<'a>, ~limit: int, f: 'a => promise<'b>): ar
|
|
|
86
86
|
let errorText = (exn: exn): string =>
|
|
87
87
|
exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("unknown error")
|
|
88
88
|
|
|
89
|
+
/** A throttle and a missing permission both surface as a failure to hold, and
|
|
90
|
+
they call for opposite responses — wait and re-run, versus go and fix the
|
|
91
|
+
policy. The control plane says "Rate exceeded"; the SDK says
|
|
92
|
+
"TooManyRequestsException". Match on either. */
|
|
93
|
+
let isThrottle = (reason: string): bool => {
|
|
94
|
+
let r = reason->String.toLowerCase
|
|
95
|
+
r->String.includes("rate exceeded") ||
|
|
96
|
+
r->String.includes("toomanyrequests") ||
|
|
97
|
+
r->String.includes("throttl")
|
|
98
|
+
}
|
|
99
|
+
|
|
89
100
|
// An update is rejected while another is still settling, and a stack that was
|
|
90
101
|
// deployed moments ago can still be settling. Poll rather than assume.
|
|
91
102
|
let awaitSettled = async (~client, ~functionName: string): unit => {
|
|
@@ -269,14 +280,21 @@ let hold = async (~client, ~functionNames: array<string>): array<held> => {
|
|
|
269
280
|
| Some((functionName, reason)) =>
|
|
270
281
|
let rollback = await taken->mapBounded(~limit=concurrency, h => restoreConcurrency(~client, ~h))
|
|
271
282
|
let stranded = rollback->Array.filterMap(w => w)
|
|
283
|
+
let advice = if reason->isThrottle {
|
|
284
|
+
`The Lambda control plane throttled the hold. That budget is per account and region and is ` ++
|
|
285
|
+
`shared with anything else using it — a deploy, an inspector sync, another stack in the same ` ++
|
|
286
|
+
`region — so this is transient and nothing is wrong with the credentials. Wait for whatever ` ++
|
|
287
|
+
`else is running to finish and re-run.`
|
|
288
|
+
} else {
|
|
289
|
+
`The reset holds every runtime in scope for the length of the wipe, which needs ` ++
|
|
290
|
+
`lambda:GetFunctionConcurrency, lambda:GetFunctionConfiguration, lambda:PutFunctionConcurrency, ` ++
|
|
291
|
+
`lambda:DeleteFunctionConcurrency and lambda:UpdateFunctionConfiguration.`
|
|
292
|
+
}
|
|
272
293
|
throw(
|
|
273
294
|
ReventlessSeed.Seed.Failed(
|
|
274
295
|
`could not hold Lambda function ${functionName} at zero concurrency (${reason}) — nothing was ` ++
|
|
275
|
-
`deleted.
|
|
276
|
-
`
|
|
277
|
-
`lambda:DeleteFunctionConcurrency and lambda:UpdateFunctionConfiguration. Set ` ++
|
|
278
|
-
`SEED_RESET_NO_QUIESCE=1 to wipe without the hold — but a running runtime can then write back ` ++
|
|
279
|
-
`what the wipe removes.` ++
|
|
296
|
+
`deleted. ${advice} Set SEED_RESET_NO_QUIESCE=1 to wipe without the hold — but a running ` ++
|
|
297
|
+
`runtime can then write back what the wipe removes.` ++
|
|
280
298
|
switch stranded {
|
|
281
299
|
| [] => ""
|
|
282
300
|
| messages => `\n\n ` ++ messages->Array.join("\n ")
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/package.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@reventlessdev/reventless-seed-aws",
|
|
3
|
-
"version": "1.0.0-alpha.
|
|
3
|
+
"version": "1.0.0-alpha.26",
|
|
4
4
|
"description": "AWS connect + guarded reset for the Reventless seed harness — stack discovery, Cognito login, and a tag-scoped, allowlisted store wipe",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"dependencies": {
|
|
7
|
+
"@reventlessdev/rescript-aws-sdk": "3.0.0-alpha.14",
|
|
7
8
|
"@reventlessdev/rescript-node": "2.0.0-alpha.8",
|
|
8
|
-
"@reventlessdev/reventless-seed": "1.0.0-alpha.
|
|
9
|
-
"@reventlessdev/rescript-aws-sdk": "3.0.0-alpha.13"
|
|
9
|
+
"@reventlessdev/reventless-seed": "1.0.0-alpha.18"
|
|
10
10
|
},
|
|
11
11
|
"devDependencies": {
|
|
12
12
|
"rescript": "12.3.0",
|
|
@@ -86,6 +86,17 @@ let mapBounded = async (items: array<'a>, ~limit: int, f: 'a => promise<'b>): ar
|
|
|
86
86
|
let errorText = (exn: exn): string =>
|
|
87
87
|
exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("unknown error")
|
|
88
88
|
|
|
89
|
+
/** A throttle and a missing permission both surface as a failure to hold, and
|
|
90
|
+
they call for opposite responses — wait and re-run, versus go and fix the
|
|
91
|
+
policy. The control plane says "Rate exceeded"; the SDK says
|
|
92
|
+
"TooManyRequestsException". Match on either. */
|
|
93
|
+
let isThrottle = (reason: string): bool => {
|
|
94
|
+
let r = reason->String.toLowerCase
|
|
95
|
+
r->String.includes("rate exceeded") ||
|
|
96
|
+
r->String.includes("toomanyrequests") ||
|
|
97
|
+
r->String.includes("throttl")
|
|
98
|
+
}
|
|
99
|
+
|
|
89
100
|
// An update is rejected while another is still settling, and a stack that was
|
|
90
101
|
// deployed moments ago can still be settling. Poll rather than assume.
|
|
91
102
|
let awaitSettled = async (~client, ~functionName: string): unit => {
|
|
@@ -269,14 +280,21 @@ let hold = async (~client, ~functionNames: array<string>): array<held> => {
|
|
|
269
280
|
| Some((functionName, reason)) =>
|
|
270
281
|
let rollback = await taken->mapBounded(~limit=concurrency, h => restoreConcurrency(~client, ~h))
|
|
271
282
|
let stranded = rollback->Array.filterMap(w => w)
|
|
283
|
+
let advice = if reason->isThrottle {
|
|
284
|
+
`The Lambda control plane throttled the hold. That budget is per account and region and is ` ++
|
|
285
|
+
`shared with anything else using it — a deploy, an inspector sync, another stack in the same ` ++
|
|
286
|
+
`region — so this is transient and nothing is wrong with the credentials. Wait for whatever ` ++
|
|
287
|
+
`else is running to finish and re-run.`
|
|
288
|
+
} else {
|
|
289
|
+
`The reset holds every runtime in scope for the length of the wipe, which needs ` ++
|
|
290
|
+
`lambda:GetFunctionConcurrency, lambda:GetFunctionConfiguration, lambda:PutFunctionConcurrency, ` ++
|
|
291
|
+
`lambda:DeleteFunctionConcurrency and lambda:UpdateFunctionConfiguration.`
|
|
292
|
+
}
|
|
272
293
|
throw(
|
|
273
294
|
ReventlessSeed.Seed.Failed(
|
|
274
295
|
`could not hold Lambda function ${functionName} at zero concurrency (${reason}) — nothing was ` ++
|
|
275
|
-
`deleted.
|
|
276
|
-
`
|
|
277
|
-
`lambda:DeleteFunctionConcurrency and lambda:UpdateFunctionConfiguration. Set ` ++
|
|
278
|
-
`SEED_RESET_NO_QUIESCE=1 to wipe without the hold — but a running runtime can then write back ` ++
|
|
279
|
-
`what the wipe removes.` ++
|
|
296
|
+
`deleted. ${advice} Set SEED_RESET_NO_QUIESCE=1 to wipe without the hold — but a running ` ++
|
|
297
|
+
`runtime can then write back what the wipe removes.` ++
|
|
280
298
|
switch stranded {
|
|
281
299
|
| [] => ""
|
|
282
300
|
| messages => `\n\n ` ++ messages->Array.join("\n ")
|
|
@@ -37,6 +37,15 @@ function errorText(exn) {
|
|
|
37
37
|
return Stdlib_Option.getOr(Stdlib_Option.flatMap(Stdlib_JsExn.fromException(exn), Stdlib_JsExn.message), "unknown error");
|
|
38
38
|
}
|
|
39
39
|
|
|
40
|
+
function isThrottle(reason) {
|
|
41
|
+
let r = reason.toLowerCase();
|
|
42
|
+
if (r.includes("rate exceeded") || r.includes("toomanyrequests")) {
|
|
43
|
+
return true;
|
|
44
|
+
} else {
|
|
45
|
+
return r.includes("throttl");
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
40
49
|
async function awaitSettled(client, functionName) {
|
|
41
50
|
let poll = async attempt => {
|
|
42
51
|
if (attempt >= 60) {
|
|
@@ -174,11 +183,13 @@ async function hold(client, functionNames) {
|
|
|
174
183
|
if (match === undefined) {
|
|
175
184
|
return taken;
|
|
176
185
|
}
|
|
186
|
+
let reason = match[1];
|
|
177
187
|
let rollback = await mapBounded(taken, 6, h => restoreConcurrency(client, h));
|
|
178
188
|
let stranded = Stdlib_Array.filterMap(rollback, w => w);
|
|
189
|
+
let advice = isThrottle(reason) ? `The Lambda control plane throttled the hold. That budget is per account and region and is shared with anything else using it — a deploy, an inspector sync, another stack in the same region — so this is transient and nothing is wrong with the credentials. Wait for whatever else is running to finish and re-run.` : `The reset holds every runtime in scope for the length of the wipe, which needs lambda:GetFunctionConcurrency, lambda:GetFunctionConfiguration, lambda:PutFunctionConcurrency, lambda:DeleteFunctionConcurrency and lambda:UpdateFunctionConfiguration.`;
|
|
179
190
|
throw {
|
|
180
191
|
RE_EXN_ID: Seed$ReventlessSeed.Failed,
|
|
181
|
-
_1: `could not hold Lambda function ` + match[0] + ` at zero concurrency (` +
|
|
192
|
+
_1: `could not hold Lambda function ` + match[0] + ` at zero concurrency (` + reason + `) — nothing was ` + (`deleted. ` + advice + ` Set SEED_RESET_NO_QUIESCE=1 to wipe without the hold — but a running `) + `runtime can then write back what the wipe removes.` + (
|
|
182
193
|
stranded.length !== 0 ? `\n\n ` + stranded.join("\n ") : ""
|
|
183
194
|
),
|
|
184
195
|
Error: new Error()
|
|
@@ -195,6 +206,7 @@ export {
|
|
|
195
206
|
concurrency,
|
|
196
207
|
mapBounded,
|
|
197
208
|
errorText,
|
|
209
|
+
isThrottle,
|
|
198
210
|
awaitSettled,
|
|
199
211
|
updateVariables,
|
|
200
212
|
restoreConcurrency,
|
|
@@ -109,6 +109,30 @@ describe("refillMessage", () => {
|
|
|
109
109
|
})
|
|
110
110
|
})
|
|
111
111
|
|
|
112
|
+
describe("isThrottle", () => {
|
|
113
|
+
// A hold that fails because the control plane is busy and a hold that fails
|
|
114
|
+
// because the credentials are short a policy read identically at the call
|
|
115
|
+
// site, and the fixes point in opposite directions: wait, versus go and edit
|
|
116
|
+
// an IAM policy. Sending an operator to the policy for a throttle costs them
|
|
117
|
+
// the whole trip.
|
|
118
|
+
testSync("recognises the wordings a throttled hold actually comes back with", () =>
|
|
119
|
+
[
|
|
120
|
+
"Rate exceeded",
|
|
121
|
+
"TooManyRequestsException: Rate exceeded",
|
|
122
|
+
"Request throttled, please try again",
|
|
123
|
+
]->Array.forEach(reason => expect(Quiesce.isThrottle(reason))->toBe(true))
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
testSync("leaves a permission failure to the IAM advice", () => {
|
|
127
|
+
expect(
|
|
128
|
+
Quiesce.isThrottle(
|
|
129
|
+
"AccessDeniedException: User is not authorized to perform: lambda:PutFunctionConcurrency",
|
|
130
|
+
),
|
|
131
|
+
)->toBe(false)
|
|
132
|
+
expect(Quiesce.isThrottle("ResourceNotFoundException: Function not found"))->toBe(false)
|
|
133
|
+
})
|
|
134
|
+
})
|
|
135
|
+
|
|
112
136
|
describe("noQuiesce", () => {
|
|
113
137
|
let set = v => NodeProcess.env->Dict.set("SEED_RESET_NO_QUIESCE", v)
|
|
114
138
|
|
|
@@ -104,6 +104,22 @@ globalThis.describe("refillMessage", () => {
|
|
|
104
104
|
});
|
|
105
105
|
});
|
|
106
106
|
|
|
107
|
+
globalThis.describe("isThrottle", () => {
|
|
108
|
+
globalThis.test("recognises the wordings a throttled hold actually comes back with", () => {
|
|
109
|
+
[
|
|
110
|
+
"Rate exceeded",
|
|
111
|
+
"TooManyRequestsException: Rate exceeded",
|
|
112
|
+
"Request throttled, please try again"
|
|
113
|
+
].forEach(reason => {
|
|
114
|
+
globalThis.expect(ReventlessSeedAws_Quiesce.isThrottle(reason)).toBe(true);
|
|
115
|
+
});
|
|
116
|
+
});
|
|
117
|
+
globalThis.test("leaves a permission failure to the IAM advice", () => {
|
|
118
|
+
globalThis.expect(ReventlessSeedAws_Quiesce.isThrottle("AccessDeniedException: User is not authorized to perform: lambda:PutFunctionConcurrency")).toBe(false);
|
|
119
|
+
globalThis.expect(ReventlessSeedAws_Quiesce.isThrottle("ResourceNotFoundException: Function not found")).toBe(false);
|
|
120
|
+
});
|
|
121
|
+
});
|
|
122
|
+
|
|
107
123
|
globalThis.describe("noQuiesce", () => {
|
|
108
124
|
let set = v => {
|
|
109
125
|
process.env["SEED_RESET_NO_QUIESCE"] = v;
|