@reventlessdev/reventless-seed-aws 1.0.0-alpha.2 → 1.0.0-alpha.21

package/package.json

@@ -1,13 +1,16 @@
 {
   "name": "@reventlessdev/reventless-seed-aws",
-  "version": "1.0.0-alpha.2",
-  "description": "AWS connect for the Reventless seed harness — stack discovery and Cognito login, no AWS SDK",
+  "version": "1.0.0-alpha.21",
+  "description": "AWS connect + guarded reset for the Reventless seed harness — stack discovery, Cognito login, and a tag-scoped, allowlisted store wipe",
   "license": "Apache-2.0",
   "dependencies": {
-    "@reventlessdev/reventless-seed": "1.0.0-alpha.2"
+    "@reventlessdev/rescript-aws-sdk": "3.0.0-alpha.11",
+    "@reventlessdev/rescript-node": "2.0.0-alpha.7",
+    "@reventlessdev/reventless-seed": "1.0.0-alpha.15"
   },
   "devDependencies": {
-    "rescript": "12.3.0"
+    "rescript": "12.3.0",
+    "@reventlessdev/rescript-jest": "1.0.0-alpha.10"
   },
   "peerDependencies": {
     "rescript": "12.3.0"

package/rescript.json

@@ -11,10 +11,18 @@
     {
       "dir": "src",
       "subdirs": true
+    },
+    {
+      "dir": "tests",
+      "subdirs": true,
+      "type": "dev"
     }
   ],
   "dependencies": [
-    "@reventlessdev/reventless-seed"
+    "@reventlessdev/reventless-seed",
+    "@reventlessdev/rescript-aws-sdk",
+    "@reventlessdev/rescript-node",
+    "@reventlessdev/rescript-jest"
   ],
   "compiler-flags": [],
   "suffix": ".res.mjs"

package/src/ReventlessSeedAws.res

@@ -14,11 +14,6 @@ open ReventlessSeed
 
 // ── Node bindings ─────────────────────────────────────────────────────────────
 
-@val @scope("process") external processEnv: dict<string> = "env"
-
-type execOptions = {cwd: string, encoding: string, env?: dict<string>}
-@module("node:child_process")
-external execFileSync: (string, array<string>, execOptions) => string = "execFileSync"
 
 type fetchInit = {method: string, headers: dict<string>, body?: string}
 type response
@@ -53,13 +48,17 @@ let envForBackend = (backend: option<string>): option<dict<string>> =>
   switch backend {
   | None => None
   | Some(url) =>
-    let copy = processEnv->Dict.toArray->Dict.fromArray
+    let copy = NodeProcess.env->Dict.toArray->Dict.fromArray
     copy->Dict.set("PULUMI_BACKEND_URL", url)
     Some(copy)
   }
 
 let pulumi = (~projectDir: string, ~backend: option<string>, args: array<string>): string =>
-  execFileSync("pulumi", args, {cwd: projectDir, encoding: "utf8", env: ?envForBackend(backend)})
+  NodeChildProcess.execFileSync(
+    "pulumi",
+    args,
+    {cwd: projectDir, encoding: "utf8", env: ?envForBackend(backend)},
+  )
 
 // Stacks that actually exist in the Pulumi backend for this project — the names
 // `pulumi stack output --stack <name>` accepts. A `Pulumi.<stack>.yaml` config
@@ -147,70 +146,88 @@ let resolveField = (~envKey: string, ~fromSource: option<string>, ~human: string
     }
   }
 
-// Returns (graphqlEndpoint, uploadEndpoint, cognitoRegion, cognitoClientId).
-// `uploadEndpoint` is "" when the deployment publishes none — the data set's
-// upload phase no-ops on empty.
-let resolveEndpoints = async (~projectDir: string, ~backend: option<string>, ~stack: string): (
-  string,
-  string,
-  string,
-  string,
-) => {
-  let outputs = stackOutputs(~projectDir, ~backend, stack)
-  switch outputs->field("hostShellUrl")->Option.flatMap(asString) {
-  | Some(hostShellUrl) =>
-    let cfg = await fetchConfig(hostShellUrl)
+// Which document publishes the deployment's endpoints. A stack that serves a
+// host shell publishes them in the shell's `config.json` under the client's key
+// names; one that does not publishes them as stack outputs under Pulumi's. The
+// two arms name different keys for the same values, which is why this is a
+// variant and not a merged lookup.
+type source = HostShellConfig(JSON.t) | StackOutputs(JSON.t)
+
+type endpoints = {
+  graphql: string,
+  cognitoRegion: string,
+  cognitoClientId: string,
+}
+
+/**
+ * The endpoints a source document publishes — pure, so the branch selection and
+ * every key it reads can be exercised with synthetic documents.
+ *
+ * Upload endpoints are no longer resolved here: under route B the seed mints through
+ * the domain API's `Upload_Presign` mutation on `graphql`, passing the store per asset,
+ * so there is no per-store URL to discover.
+ */
+let endpointsFrom = (~stack: string, source: source): endpoints =>
+  switch source {
+  | HostShellConfig(cfg) =>
     let fromCfg = key => cfg->field(key)->Option.flatMap(asString)
-    let endpoint = resolveField(
-      ~envKey="REVENTLESS_GRAPHQL_ENDPOINT",
-      ~fromSource=fromCfg("apiEndpoint"),
-      ~human="apiEndpoint",
-    )
-    let uploadEndpoint = resolveField(
-      ~envKey="REVENTLESS_UPLOAD_ENDPOINT",
-      ~fromSource=fromCfg("uploadEndpoint"),
-      ~human="uploadEndpoint",
-    )
-    let region = resolveField(
-      ~envKey="AWS_REGION",
-      ~fromSource=fromCfg("region"),
-      ~human="region",
-    )
-    let clientId = resolveField(
-      ~envKey="COGNITO_CLIENT_ID",
-      ~fromSource=fromCfg("cognitoClientId"),
-      ~human="cognitoClientId",
-    )
-    (endpoint, uploadEndpoint, region, clientId)
-  | None =>
+    {
+      graphql: resolveField(
+        ~envKey="REVENTLESS_GRAPHQL_ENDPOINT",
+        ~fromSource=fromCfg("apiEndpoint"),
+        ~human="apiEndpoint",
+      ),
+      cognitoRegion: resolveField(
+        ~envKey="AWS_REGION",
+        ~fromSource=fromCfg("region"),
+        ~human="region",
+      ),
+      cognitoClientId: resolveField(
+        ~envKey="COGNITO_CLIENT_ID",
+        ~fromSource=fromCfg("cognitoClientId"),
+        ~human="cognitoClientId",
+      ),
+    }
+  | StackOutputs(outputs) =>
     let out = key => outputs->field(key)->Option.flatMap(asString)
-    let endpoint = switch (
-      Seed.Prompt.envValue("REVENTLESS_GRAPHQL_ENDPOINT"),
-      out("domainMergedApiEndpoint"),
-      out("domainApiEndpoint"),
-    ) {
-    | (Some(v), _, _) | (_, Some(v), _) | (_, _, Some(v)) => v
-    | _ =>
-      throw(
-        Seed.Failed(
-          `stack "${stack}" exports neither domainMergedApiEndpoint nor domainApiEndpoint`,
-        ),
-      )
+    {
+      graphql: switch (
+        Seed.Prompt.envValue("REVENTLESS_GRAPHQL_ENDPOINT"),
+        out("domainMergedApiEndpoint"),
+        out("domainApiEndpoint"),
+      ) {
+      | (Some(v), _, _) | (_, Some(v), _) | (_, _, Some(v)) => v
+      | _ =>
+        throw(
+          Seed.Failed(
+            `stack "${stack}" exports neither domainMergedApiEndpoint nor domainApiEndpoint`,
+          ),
+        )
+      },
+      cognitoRegion: resolveField(
+        ~envKey="AWS_REGION",
+        ~fromSource=out("cognitoRegion"),
+        ~human="cognitoRegion",
+      ),
+      cognitoClientId: resolveField(
+        ~envKey="COGNITO_CLIENT_ID",
+        ~fromSource=out("cognitoUserPoolClientId"),
+        ~human="cognitoUserPoolClientId",
+      ),
     }
-    let region = resolveField(
-      ~envKey="AWS_REGION",
-      ~fromSource=out("cognitoRegion"),
-      ~human="cognitoRegion",
-    )
-    let clientId = resolveField(
-      ~envKey="COGNITO_CLIENT_ID",
-      ~fromSource=out("cognitoUserPoolClientId"),
-      ~human="cognitoUserPoolClientId",
-    )
-    // Absent → "" → the data set skips its upload phase.
-    let uploadEndpoint = Seed.Prompt.envValue("REVENTLESS_UPLOAD_ENDPOINT")->Option.getOr("")
-    (endpoint, uploadEndpoint, region, clientId)
   }
+
+let resolveEndpoints = async (
+  ~projectDir: string,
+  ~backend: option<string>,
+  ~stack: string,
+): endpoints => {
+  let outputs = stackOutputs(~projectDir, ~backend, stack)
+  let source = switch outputs->field("hostShellUrl")->Option.flatMap(asString) {
+  | Some(hostShellUrl) => HostShellConfig(await fetchConfig(hostShellUrl))
+  | None => StackOutputs(outputs)
+  }
+  endpointsFrom(~stack, source)
 }
 
 // ── Cognito login ─────────────────────────────────────────────────────────────
@@ -302,15 +319,10 @@ let connect = (~projectDir: string=".", ~stack=?, ~backend=?, ()): (
     | None => backend
     }
     let stackName = await resolveStack(~projectDir, ~backend, ~stack)
-    let (endpoint, uploadEndpoint, region, clientId) = await resolveEndpoints(
-      ~projectDir,
-      ~backend,
-      ~stack=stackName,
-    )
+    let eps = await resolveEndpoints(~projectDir, ~backend, ~stack=stackName)
     await Seed.Connect.make(
       ~label=stackName,
-      ~endpoint,
-      ~uploadEndpoint,
-      ~login=cognito(~region, ~clientId),
+      ~endpoint=eps.graphql,
+      ~login=cognito(~region=eps.cognitoRegion, ~clientId=eps.cognitoClientId),
     )
   }

package/src/ReventlessSeedAws.res.mjs

@@ -152,34 +152,28 @@ function resolveField(envKey, fromSource, human) {
   };
 }
 
-async function resolveEndpoints(projectDir, backend, stack) {
-  let outputs = stackOutputs(projectDir, backend, stack);
-  let hostShellUrl = Stdlib_Option.flatMap(field(outputs, "hostShellUrl"), asString);
-  if (hostShellUrl !== undefined) {
-    let cfg = await fetchConfig(hostShellUrl);
+function endpointsFrom(stack, source) {
+  if (source.TAG === "HostShellConfig") {
+    let cfg = source._0;
     let fromCfg = key => Stdlib_Option.flatMap(field(cfg, key), asString);
-    let endpoint = resolveField("REVENTLESS_GRAPHQL_ENDPOINT", fromCfg("apiEndpoint"), "apiEndpoint");
-    let uploadEndpoint = resolveField("REVENTLESS_UPLOAD_ENDPOINT", fromCfg("uploadEndpoint"), "uploadEndpoint");
-    let region = resolveField("AWS_REGION", fromCfg("region"), "region");
-    let clientId = resolveField("COGNITO_CLIENT_ID", fromCfg("cognitoClientId"), "cognitoClientId");
-    return [
-      endpoint,
-      uploadEndpoint,
-      region,
-      clientId
-    ];
+    return {
+      graphql: resolveField("REVENTLESS_GRAPHQL_ENDPOINT", fromCfg("apiEndpoint"), "apiEndpoint"),
+      cognitoRegion: resolveField("AWS_REGION", fromCfg("region"), "region"),
+      cognitoClientId: resolveField("COGNITO_CLIENT_ID", fromCfg("cognitoClientId"), "cognitoClientId")
+    };
   }
+  let outputs = source._0;
   let out = key => Stdlib_Option.flatMap(field(outputs, key), asString);
   let match = Seed_Prompt$ReventlessSeed.envValue("REVENTLESS_GRAPHQL_ENDPOINT");
   let match$1 = out("domainMergedApiEndpoint");
   let match$2 = out("domainApiEndpoint");
-  let endpoint$1;
+  let tmp;
   if (match !== undefined) {
-    endpoint$1 = match;
+    tmp = match;
   } else if (match$1 !== undefined) {
-    endpoint$1 = match$1;
+    tmp = match$1;
   } else if (match$2 !== undefined) {
-    endpoint$1 = match$2;
+    tmp = match$2;
   } else {
     throw {
       RE_EXN_ID: Seed$ReventlessSeed.Failed,
@@ -187,15 +181,24 @@ async function resolveEndpoints(projectDir, backend, stack) {
       Error: new Error()
     };
   }
-  let region$1 = resolveField("AWS_REGION", out("cognitoRegion"), "cognitoRegion");
-  let clientId$1 = resolveField("COGNITO_CLIENT_ID", out("cognitoUserPoolClientId"), "cognitoUserPoolClientId");
-  let uploadEndpoint$1 = Stdlib_Option.getOr(Seed_Prompt$ReventlessSeed.envValue("REVENTLESS_UPLOAD_ENDPOINT"), "");
-  return [
-    endpoint$1,
-    uploadEndpoint$1,
-    region$1,
-    clientId$1
-  ];
+  return {
+    graphql: tmp,
+    cognitoRegion: resolveField("AWS_REGION", out("cognitoRegion"), "cognitoRegion"),
+    cognitoClientId: resolveField("COGNITO_CLIENT_ID", out("cognitoUserPoolClientId"), "cognitoUserPoolClientId")
+  };
+}
+
+async function resolveEndpoints(projectDir, backend, stack) {
+  let outputs = stackOutputs(projectDir, backend, stack);
+  let hostShellUrl = Stdlib_Option.flatMap(field(outputs, "hostShellUrl"), asString);
+  let source = hostShellUrl !== undefined ? ({
+      TAG: "HostShellConfig",
+      _0: await fetchConfig(hostShellUrl)
+    }) : ({
+      TAG: "StackOutputs",
+      _0: outputs
+    });
+  return endpointsFrom(stack, source);
 }
 
 function cognito(region, clientId) {
@@ -282,8 +285,8 @@ function connect($staropt$star, stack, backend, param) {
     let url = Seed_Prompt$ReventlessSeed.envValue("SEED_PULUMI_BACKEND");
     let backend$1 = url !== undefined ? url : backend;
     let stackName = await resolveStack(projectDir, backend$1, stack);
-    let match = await resolveEndpoints(projectDir, backend$1, stackName);
-    return await Seed_Connect$ReventlessSeed.make(stackName, match[0], match[1], cognito(match[2], match[3]), undefined);
+    let eps = await resolveEndpoints(projectDir, backend$1, stackName);
+    return await Seed_Connect$ReventlessSeed.make(stackName, eps.graphql, cognito(eps.cognitoRegion, eps.cognitoClientId), undefined);
   };
 }
 
@@ -298,6 +301,7 @@ export {
   resolveStack,
   fetchConfig,
   resolveField,
+  endpointsFrom,
   resolveEndpoints,
   cognito,
   connect,

package/src/ReventlessSeedAws_Quiesce.res

@@ -0,0 +1,287 @@
+// Taking a deployed stack's runtimes out of service for the length of a wipe,
+// and putting them back exactly as they were.
+//
+// Why a wipe needs this at all: a truncate is not durable while the runtimes
+// that own the data are running. A slice runtime keeps its TODO list in a
+// module-level dict for the lifetime of its execution environment and re-saves
+// every row it holds at the end of each invocation — including the scheduled
+// sweep, which carries no events. Emptying the table under a warm container
+// therefore succeeds and is then undone, byte for byte, by the next sweep: the
+// rows are not recomputed from anything, so their ids and timestamps come back
+// identical. Emptying the upstream stores first does not help, because nothing
+// upstream is read.
+//
+// The answer is to remove the contention rather than to outrun it, which takes
+// two distinct steps, in this order:
+//
+//   1. HOLD — reserve 0 concurrency on every function in scope, so no new
+//      invocation can start while the wipe and its verification run. This alone
+//      is not enough: a reservation stops invocations, it does not discard the
+//      execution environment that holds the state.
+//   2. RECYCLE — change each function's configuration once. A configuration
+//      change is the documented way to make Lambda discard existing execution
+//      environments, so whatever a warm container was holding is gone before it
+//      can be invoked again. Done while still held, so the container that
+//      eventually replaces it starts from the emptied store.
+//
+// The recycle is a round trip — a marker variable added, then the original map
+// put back — so the function ends the run byte-identical to how it started and
+// `pulumi preview` reports no drift. The second update recycles again, which is
+// harmless: it happens while the hold is still on and the store is already
+// empty.
+//
+// Everything acquired here is released on the failure path as well as the
+// success path, and the release never throws: a reset that aborted mid-wipe
+// must still hand the stack back. Release failures come back as warnings for
+// the caller to print, because a function left at zero concurrency is a stack
+// left switched off, and that has to be said out loud rather than raised as one
+// more exception on top of whatever already went wrong.
+
+module Lambda = AwsSdk.Lambda
+
+/** The variable a recycle adds and then removes. Named for what it is, so an
+    operator who finds one left behind by a killed run knows it is inert and can
+    be deleted (or left for the next `pulumi up` to remove). */
+let markerKey = "REVENTLESS_RESET_RECYCLE"
+
+/** What one function looked like before the reset touched it. */
+type held = {
+  functionName: string,
+  /** `None` = the function had no reservation at all. That is a different state
+      from a reservation of 0, and restoring it needs
+      `DeleteFunctionConcurrency` rather than a put. */
+  priorReserved: option<int>,
+  /** The variable map as it was, flattened: a function with no `Environment`
+      block at all reads as an empty map and is restored to an empty map. */
+  priorVariables: dict<string>,
+}
+
+/** The Lambda control plane is rate-limited well below the data plane, and a
+    reset touches every function in a project at once. Six at a time keeps a
+    forty-function stack under a minute per pass without risking a throttle. */
+let concurrency = 6
+
+let mapBounded = async (items: array<'a>, ~limit: int, f: 'a => promise<'b>): array<'b> => {
+  let results = Array.make(~length=items->Array.length, None)
+  let next = ref(0)
+  let worker = async () => {
+    let running = ref(true)
+    while running.contents {
+      let i = next.contents
+      if i >= items->Array.length {
+        running := false
+      } else {
+        next := i + 1
+        let r = await f(items->Array.getUnsafe(i))
+        results->Array.set(i, Some(r))
+      }
+    }
+  }
+  let _ = await Array.fromInitializer(~length=Math.Int.min(limit, items->Array.length), _ =>
+    worker()
+  )->Promise.all
+  results->Array.filterMap(x => x)
+}
+
+let errorText = (exn: exn): string =>
+  exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("unknown error")
+
+// An update is rejected while another is still settling, and a stack that was
+// deployed moments ago can still be settling. Poll rather than assume.
+let awaitSettled = async (~client, ~functionName: string): unit => {
+  let deadline = 60
+  let rec poll = async (attempt: int): unit =>
+    if attempt < deadline {
+      let out = await Lambda.GetFunctionConfigurationCommand.send(
+        client,
+        Lambda.GetFunctionConfigurationCommand.make({functionName: functionName}),
+      )
+      switch out.lastUpdateStatus {
+      | Some("InProgress") =>
+        await ReventlessSeed.Seed.Client.sleep(1000)
+        await poll(attempt + 1)
+      | _ => ()
+      }
+    }
+  await poll(0)
+}
+
+let updateVariables = async (~client, ~functionName: string, ~variables: dict<string>): unit => {
+  await awaitSettled(~client, ~functionName)
+  let _ = await Lambda.UpdateFunctionConfigurationCommand.send(
+    client,
+    Lambda.UpdateFunctionConfigurationCommand.make({
+      functionName,
+      environment: {variables: variables},
+    }),
+  )
+  await awaitSettled(~client, ~functionName)
+}
+
+/** Put one function's reservation back exactly as it was — a value it had, or no
+    reservation at all. Never throws: a function left reserved at 0 is a function
+    left switched off, which has to be said rather than raised. */
+let restoreConcurrency = async (~client, ~h: held): option<string> =>
+  switch await (
+    switch h.priorReserved {
+    | Some(n) =>
+      Lambda.PutFunctionConcurrencyCommand.send(
+        client,
+        Lambda.PutFunctionConcurrencyCommand.make({
+          functionName: h.functionName,
+          reservedConcurrentExecutions: n,
+        }),
+      )->Promise.thenResolve(_ => ())
+    | None =>
+      Lambda.DeleteFunctionConcurrencyCommand.send(
+        client,
+        Lambda.DeleteFunctionConcurrencyCommand.make({functionName: h.functionName}),
+      )->Promise.thenResolve(_ => ())
+    }
+  ) {
+  | () => None
+  | exception exn =>
+    Some(
+      `LEFT SWITCHED OFF: could not restore concurrency on ${h.functionName} (${errorText(exn)}). ` ++
+      `It is still reserved at 0 and will not run until that is undone.`,
+    )
+  }
+
+/**
+Hand the stack back: original environment first (which also drops the marker),
+then the original concurrency.
+
+That order matters. The environment restore is itself a configuration change, so
+it recycles a second time; doing it while the function is still held means the
+container that finally serves traffic is created after the restore, not during
+it.
+
+Never throws, for either step and for any function — this runs on the abort path
+too, where an exception would replace the reason the reset failed with a
+secondary one.
+*/
+let release = async (~client, ~held: array<held>): array<string> => {
+  let warnings = await held->mapBounded(~limit=concurrency, async h => {
+    let envWarning = switch await updateVariables(
+      ~client,
+      ~functionName=h.functionName,
+      ~variables=h.priorVariables,
+    ) {
+    | () => None
+    | exception exn =>
+      Some(
+        `could not restore the environment of ${h.functionName} (${errorText(exn)}) — it may still ` ++
+        `carry ${markerKey}, which is inert and is removed by the next deploy.`,
+      )
+    }
+    let concurrencyWarning = await restoreConcurrency(~client, ~h)
+    [envWarning, concurrencyWarning]->Array.filterMap(w => w)
+  })
+  warnings->Array.flat
+}
+
+/**
+Discard every held function's execution environments, so nothing that loaded
+state before the wipe can write it back afterwards.
+
+Never throws. A function that cannot be recycled is the one case where the reset
+genuinely cannot promise the store stays empty, so it comes back as a warning
+naming the function — the caller reports it rather than swallowing it.
+*/
+let recycle = async (~client, ~held: array<held>): array<string> => {
+  let warnings = await held->mapBounded(~limit=concurrency, async h => {
+    let marked = h.priorVariables->Dict.copy
+    marked->Dict.set(markerKey, "1")
+    switch await updateVariables(~client, ~functionName=h.functionName, ~variables=marked) {
+    | () => None
+    | exception exn =>
+      Some(
+        `could not recycle ${h.functionName} (${errorText(exn)}) — a warm container may still hold ` ++
+        `pre-wipe state and write it back. Redeploy or update that function to clear it.`,
+      )
+    }
+  })
+  warnings->Array.filterMap(w => w)
+}
+
+/**
+Take every named function out of service, recording what to put back.
+
+Throws if any function cannot be held — before a single delete has been issued,
+so a reset that cannot hold the platform still has the option of not starting.
+The message names the IAM actions involved, because a missing permission is much
+the likeliest cause and "AccessDenied" on its own does not say which one.
+
+Crucially, it hands back whatever it did manage to take before throwing. The
+functions are held concurrently, so a failure part-way through the batch leaves
+earlier ones already at zero — and a reset that aborted while silently switching
+off half a stack would be a worse failure than the one it was reporting. Only the
+reservations are rolled back: this runs before `recycle`, so no environment has
+been touched yet.
+*/
+let hold = async (~client, ~functionNames: array<string>): array<held> => {
+  let attempts = await functionNames->mapBounded(~limit=concurrency, async functionName =>
+    switch await (
+      async () => {
+        let concurrencyOut = await Lambda.GetFunctionConcurrencyCommand.send(
+          client,
+          Lambda.GetFunctionConcurrencyCommand.make({functionName: functionName}),
+        )
+        let configOut = await Lambda.GetFunctionConfigurationCommand.send(
+          client,
+          Lambda.GetFunctionConfigurationCommand.make({functionName: functionName}),
+        )
+        let held = {
+          functionName,
+          priorReserved: concurrencyOut.reservedConcurrentExecutions,
+          priorVariables: configOut.environment
+          ->Option.flatMap(e => e.variables)
+          ->Option.getOr(Dict.make()),
+        }
+        let _ = await Lambda.PutFunctionConcurrencyCommand.send(
+          client,
+          Lambda.PutFunctionConcurrencyCommand.make({
+            functionName,
+            reservedConcurrentExecutions: 0,
+          }),
+        )
+        held
+      }
+    )() {
+    | held => Ok(held)
+    | exception exn => Error((functionName, errorText(exn)))
+    }
+  )
+
+  let taken = attempts->Array.filterMap(a =>
+    switch a {
+    | Ok(h) => Some(h)
+    | Error(_) => None
+    }
+  )
+  switch attempts->Array.findMap(a =>
+    switch a {
+    | Error(failure) => Some(failure)
+    | Ok(_) => None
+    }
+  ) {
+  | None => taken
+  | Some((functionName, reason)) =>
+    let rollback = await taken->mapBounded(~limit=concurrency, h => restoreConcurrency(~client, ~h))
+    let stranded = rollback->Array.filterMap(w => w)
+    throw(
+      ReventlessSeed.Seed.Failed(
+        `could not hold Lambda function ${functionName} at zero concurrency (${reason}) — nothing was ` ++
+        `deleted. The reset holds every runtime in scope for the length of the wipe, which needs ` ++
+        `lambda:GetFunctionConcurrency, lambda:GetFunctionConfiguration, lambda:PutFunctionConcurrency, ` ++
+        `lambda:DeleteFunctionConcurrency and lambda:UpdateFunctionConfiguration. Set ` ++
+        `SEED_RESET_NO_QUIESCE=1 to wipe without the hold — but a running runtime can then write back ` ++
+        `what the wipe removes.` ++
+        switch stranded {
+        | [] => ""
+        | messages => `\n\n  ` ++ messages->Array.join("\n  ")
+        },
+      ),
+    )
+  }
+}