@reventlessdev/reventless-seed-aws 1.0.0-alpha.4 → 1.0.0-alpha.40

package/lib/bs/src/ReventlessSeedAws_Reset.res

@@ -7,7 +7,7 @@
 //
 // A wipe is irreversible, so every step is fail-closed and production must be
 // unreachable at several at once (the separate-AWS-account guarantee is the one
-// deferred layer — see docs/plans/seed-reset-and-fresh-store-guard.md):
+// deferred layer — see docs/plans/done/seed-reset-and-fresh-store-guard.md):
 //
 //   1. name allowlist    — the stack name must match ^(alpha|dev|pr-.+)$
 //   2. wipeable flag      — the target's own `Pulumi.<stack>.yaml` must declare
@@ -47,15 +47,8 @@ open ReventlessSeed
 module Ddb = AwsSdk.DynamoDb
 module Rgt = AwsSdk.ResourceGroupsTaggingApi
 module S3 = AwsSdk.S3
-
-@scope("process") @val external exit: int => unit = "exit"
-@scope("process") @val external processEnv: dict<string> = "env"
-@module("node:fs") external readFileSync: (string, string) => string = "readFileSync"
-
-type stream
-@scope("process") @val external stdin: stream = "stdin"
-// `process.stdin.isTTY` is `true` interactively and `undefined` under a pipe/CI.
-@get external isTTY: stream => option<bool> = "isTTY"
+module Lambda = AwsSdk.Lambda
+module Quiesce = ReventlessSeedAws_Quiesce
 
 // The Pulumi project name, which the framework stamps as `reventless:platform` on
 // every resource (`Plugin.res`: `platformName = getProjectName()`). Discovery
@@ -66,7 +59,7 @@ type stream
 // the target project's Pulumi.yaml — the same dir the pulumi subprocess uses.
 let projectName = (~projectDir: string): string => {
   let path = `${projectDir}/Pulumi.yaml`
-  let raw = try readFileSync(path, "utf8") catch {
+  let raw = try NodeFs.readFileSync(path) catch {
   | _ => throw(Seed.Failed(`could not read ${path} to scope the wipe to this Pulumi project.`))
   }
   switch raw
@@ -77,7 +70,7 @@ let projectName = (~projectDir: string): string => {
     trimmed
     ->String.slice(~start=String.length("name:"), ~end=String.length(trimmed))
     ->String.trim
-    ->String.replaceRegExp(%re("/^[\"']|[\"']$/g"), "")
+    ->String.replaceRegExp(/^[\"']|[\"']$/g, "")
   | None => throw(Seed.Failed(`could not find a \`name:\` field in ${path}.`))
   }
 }
@@ -111,7 +104,7 @@ let chunk = (arr: array<'a>, size: int): array<array<'a>> => {
 
 // Fail-closed name allowlist. A denylist ("everything except prod") fails open
 // the day a new prod-like stack is added and forgotten; this fails closed.
-let nameAllowlist = %re("/^(alpha|dev|pr-.+)$/")
+let nameAllowlist = /^(alpha|dev|pr-.+)$/
 
 // The stack's fully-resolved Pulumi config, read once. Reading it whole (rather
 // than `pulumi config get <key>`, which exits non-zero for BOTH a missing key
@@ -119,21 +112,19 @@ let nameAllowlist = %re("/^(alpha|dev|pr-.+)$/")
 // unreadable config is a pulumi/backend problem; an absent flag is a missing
 // opt-in. `pulumi config --json` maps each key to `{value, secret}`.
 let readStackConfig = (~projectDir, ~backend, ~stack): result<dict<JSON.t>, string> =>
-  switch (
-    try Some(ReventlessSeedAws.pulumi(~projectDir, ~backend, ["config", "--json", "--stack", stack])) catch {
-    | _ => None
-    }
-  ) {
+  switch try Some(
+    ReventlessSeedAws.pulumi(~projectDir, ~backend, ["config", "--json", "--stack", stack]),
+  ) catch {
+  | _ => None
+  } {
   | None =>
     Error(
       `could not read the Pulumi config for stack "${stack}" — is pulumi logged in to the right backend, and is the stack deployed?`,
     )
   | Some(raw) =>
-    switch (
-      try Some(JSON.parseOrThrow(raw)) catch {
-      | _ => None
-      }
-    ) {
+    switch try Some(JSON.parseOrThrow(raw)) catch {
+    | _ => None
+    } {
     | Some(Object(obj)) => Ok(obj)
     | _ => Error(`could not parse \`pulumi config --json\` output for stack "${stack}".`)
     }
@@ -147,9 +138,16 @@ let configValue = (cfg: dict<JSON.t>, key: string): option<string> =>
 type resource =
   | Table(string)
   | Bucket(string)
+  | Function(string)
   | Other
 
 // arn:aws:dynamodb:<region>:<acct>:table/<name>  |  arn:aws:s3:::<bucket>
+//   |  arn:aws:lambda:<region>:<acct>:function:<name>
+//
+// A Lambda ARN can carry a trailing `:<version|alias>` qualifier. The tagging
+// API returns the unqualified form, but taking the segment after `:function:`
+// up to the next colon rather than to the end of the string means a qualified
+// one still yields the function name instead of a name no API accepts.
 let classify = (arn: string): resource =>
   if arn->String.startsWith("arn:aws:dynamodb:") {
     switch arn->String.split(":table/") {
@@ -158,6 +156,16 @@ let classify = (arn: string): resource =>
     }
   } else if arn->String.startsWith("arn:aws:s3:::") {
     Bucket(arn->String.slice(~start=String.length("arn:aws:s3:::"), ~end=String.length(arn)))
+  } else if arn->String.startsWith("arn:aws:lambda:") {
+    switch arn->String.split(":function:") {
+    | [_, rest] =>
+      switch rest->String.split(":") {
+      | [name] => Function(name)
+      | [name, _qualifier] => Function(name)
+      | _ => Other
+      }
+    | _ => Other
+    }
   } else {
     Other
   }
@@ -165,15 +173,29 @@ let classify = (arn: string): resource =>
 let tagValue = (tags: array<Rgt.GetResourcesCommand.tag>, key: string): option<string> =>
   tags->Array.findMap(t => t.key == key ? Some(t.value) : None)
 
+/** What one project's tags resolve to: the stores to empty, and the runtimes
+    that have to be held still while that happens. */
+type discovered = {
+  tables: array<string>,
+  buckets: array<string>,
+  functions: array<string>,
+}
+
 // Discovery is scoped by BOTH tags (platform AND environment) — the Tagging API
 // ANDs multiple TagFilters — so it only ever sees the target project's own stack,
 // never a same-named stack from another project. Each returned resource is then
 // re-checked to carry both tags (a mismatch aborts the whole run rather than
 // being skipped) as a second, per-resource guard.
-let discover = async (~region, ~stack, ~platform): (array<string>, array<string>) => {
+//
+// Lambda functions are discovered alongside the stores, under the same tags and
+// the same per-resource re-check. They are never deleted from — they are what
+// the wipe has to hold still, and the tag scope that decides what may be emptied
+// is exactly the scope that decides what may be held.
+let discover = async (~region, ~stack, ~platform): discovered => {
   let client = Rgt.client(~region, ())
   let tables = []
   let buckets = []
+  let functions = []
   let rec loop = async (token: option<string>): unit => {
     let out = await Rgt.GetResourcesCommand.send(
       client,
@@ -182,7 +204,7 @@ let discover = async (~region, ~stack, ~platform): (array<string>, array<string>
           {key: "reventless:platform", values: [platform]},
           {key: "reventless:environment", values: [stack]},
         ],
-        resourceTypeFilters: ["dynamodb:table", "s3"],
+        resourceTypeFilters: ["dynamodb:table", "s3", "lambda:function"],
         resourcesPerPage: 100,
         paginationToken: ?token,
       }),
@@ -208,6 +230,7 @@ let discover = async (~region, ~stack, ~platform): (array<string>, array<string>
       switch classify(m.resourceARN) {
       | Table(name) => tables->Array.push(name)
       | Bucket(name) => buckets->Array.push(name)
+      | Function(name) => functions->Array.push(name)
       | Other => ()
       }
     })
@@ -217,9 +240,118 @@ let discover = async (~region, ~stack, ~platform): (array<string>, array<string>
     }
   }
   await loop(None)
-  (tables, buckets)
+  {tables, buckets, functions}
+}
+
+// ── Declared object stores ────────────────────────────────────────────────────
+//
+// A store declared by a `@storageRef` field is provisioned by the PLATFORM
+// deploy — the serving CDN and the presign services live there, and a
+// shared-layout bucket holds several plugins' stores, so no plugin stack can own
+// it. Its bucket therefore carries the platform project's `reventless:platform`
+// tag, which makes it invisible to a domain-scoped tag discovery and, worse,
+// wholesale-emptiable by a platform-scoped one.
+//
+// Tag discovery cannot fix that: the tag says which project *built* the bucket,
+// and the answer needed here is which plugin's data is *in* it — at prefix
+// granularity, since one bucket holds several plugins' stores. The platform
+// already publishes exactly that mapping as its `objectStores` stack output, so
+// the reset reads ownership from the declaration rather than inferring it.
+
+/** One provisioned store: which plugin owns it, and where its objects live. */
+type objectStore = {
+  qualified: string,
+  plugin: string,
+  store: string,
+  bucketName: string,
+  keyPrefix: string,
 }
 
+// A store's key is `{plugin}.{store}`, split at the FIRST dot: a registered
+// plugin name cannot contain one, a store name could in principle.
+let splitQualified = (key: string): option<(string, string)> =>
+  key
+  ->String.indexOfOpt(".")
+  ->Option.map(i => (
+    key->String.slice(~start=0, ~end=i),
+    key->String.slice(~start=i + 1, ~end=key->String.length),
+  ))
+
+/** Parse the platform stack's `objectStores` output. Absent is normal — a
+    platform may declare no stores — so only a malformed entry is an error, and
+    it is an error rather than a skip because a store the reset cannot read is a
+    store it would silently leave behind. */
+let parseObjectStores = (json: option<JSON.t>): result<array<objectStore>, string> =>
+  switch json {
+  | None => Ok([])
+  | Some(Object(entries)) =>
+    entries
+    ->Dict.toArray
+    ->Array.reduce(Ok([]), (acc, (qualified, entry)) =>
+      switch acc {
+      | Error(_) as failed => failed
+      | Ok(stores) =>
+        switch (
+          splitQualified(qualified),
+          entry->field("bucketName")->Option.flatMap(asString),
+          entry->field("keyPrefix")->Option.flatMap(asString),
+        ) {
+        | (Some((plugin, store)), Some(bucketName), Some(keyPrefix)) =>
+          Ok(Array.concat(stores, [{qualified, plugin, store, bucketName, keyPrefix}]))
+        | _ =>
+          Error(
+            `the platform stack's \`objectStores\` output has a malformed entry for "${qualified}" — ` ++ `expected a {plugin}.{store} key carrying bucketName and keyPrefix.`,
+          )
+        }
+      }
+    )
+  | Some(_) => Error("the platform stack's `objectStores` output is not an object.")
+  }
+
+/** Refuse any store set a prefix-scoped wipe cannot separate.
+
+    Equality is the cross-plugin collision: two plugins declaring one store name
+    land on one prefix inside a shared bucket, and nothing distinguishes their
+    objects. Containment is the same problem one level up — the delete prefix is
+    `{keyPrefix}/`, so a store rooted at `a` encloses one rooted at `a/b`.
+    Comparing containment rather than equality is what keeps this correct once a
+    prefix carries path structure.
+
+    Fail-closed, and before anything is counted: this is the tool that destroys
+    data, so it refuses rather than skipping, and it does not assume an upstream
+    deploy-time check ran. */
+let validateStores = (stores: array<objectStore>): result<unit, string> =>
+  switch stores->Array.find(s => s.keyPrefix == "" || s.store->String.includes("/")) {
+  | Some(s) =>
+    Error(
+      `store "${s.qualified}" has an unusable key prefix ("${s.keyPrefix}") — ` ++ `a store name may not be empty or contain "/".`,
+    )
+  | None =>
+    switch stores->Array.findMap(a =>
+      stores->Array.findMap(b =>
+        if a.qualified == b.qualified || a.bucketName != b.bucketName {
+          None
+        } else if a.keyPrefix == b.keyPrefix {
+          Some(
+            `stores "${a.qualified}" and "${b.qualified}" both live at ` ++
+            `${a.bucketName}/${a.keyPrefix}/ — a prefix-scoped wipe cannot tell their objects ` ++
+            `apart. Rename one store, or qualify the \`@storageRef\` annotation if they were ` ++ `meant to be one shared store.`,
+          )
+        } else if b.keyPrefix->String.startsWith(a.keyPrefix ++ "/") {
+          Some(
+            `store "${a.qualified}" (${a.bucketName}/${a.keyPrefix}/) encloses "${b.qualified}" ` ++
+            `(${b.keyPrefix}/) — wiping the first would delete the second's objects. Rename one.`,
+          )
+        } else {
+          None
+        }
+      )
+    ) {
+    | Some(message) => Error(message)
+    | None => Ok()
+    }
+  }
+
 // ── Counting (dry-run) ─────────────────────────────────────────────────────────
 
 let countTable = async (table: string): int => {
@@ -240,12 +372,15 @@ let countTable = async (table: string): int => {
   await loop(None, 0)
 }
 
-let countBucket = async (bucket: string): int => {
+// `~prefix` narrows the count to one declared store inside a shared bucket;
+// omitted, it counts the whole bucket.
+let countBucket = async (bucket: string, ~prefix: option<string>=?): int => {
   let rec loop = async (keyMarker, versionMarker, acc): int => {
     let out = await S3.ListObjectVersionsCommand.send(
       S3.ListObjectVersionsCommand.make({
         bucket,
-        keyMarker: ?keyMarker,
+        ?prefix,
+        ?keyMarker,
         versionIdMarker: ?versionMarker,
       }),
     )
@@ -273,7 +408,13 @@ let rec sendBatch = async (
 ): unit =>
   if requests->Array.length > 0 {
     if attempt > 8 {
-      throw(Seed.Failed(`table ${table}: ${(requests->Array.length)->Int.toString} item(s) still unprocessed after 8 retries.`))
+      throw(
+        Seed.Failed(
+          `table ${table}: ${requests
+            ->Array.length
+            ->Int.toString} item(s) still unprocessed after 8 retries.`,
+        ),
+      )
     }
     let out = await Ddb.DocumentClient.BatchWriteCommand.send(
       Ddb.DocumentClient.BatchWriteCommand.make({
@@ -314,7 +455,9 @@ let truncateTable = async (table: string): unit => {
       ->Option.getOr([])
       ->Array.map(item => {
         let key =
-          keyAttrs->Array.filterMap(attr => item->field(attr)->Option.map(v => (attr, v)))->Dict.fromArray
+          keyAttrs
+          ->Array.filterMap(attr => item->field(attr)->Option.map(v => (attr, v)))
+          ->Dict.fromArray
         ({deleteRequest: {key: key}}: Ddb.DocumentClient.BatchWriteCommand.writeRequest)
       })
     let batches = chunk(requests, Ddb.DocumentClient.BatchWriteCommand.maxBatchSize)
@@ -334,19 +477,20 @@ let truncateTable = async (table: string): unit => {
 
 // One ListObjectVersions page returns ≤ 1000 entries (versions + delete
 // markers), and DeleteObjects takes ≤ 1000, so one list page maps to one delete.
-let emptyBucket = async (bucket: string): unit => {
+let emptyBucket = async (bucket: string, ~prefix: option<string>=?): unit => {
   let rec loop = async (keyMarker, versionMarker): unit => {
     let out = await S3.ListObjectVersionsCommand.send(
       S3.ListObjectVersionsCommand.make({
         bucket,
-        keyMarker: ?keyMarker,
+        ?prefix,
+        ?keyMarker,
         versionIdMarker: ?versionMarker,
       }),
     )
     let ids =
-      Array.concat(out.versions->Option.getOr([]), out.deleteMarkers->Option.getOr([]))->Array.map(v => (
-        {key: v.key, versionId: v.versionId}: S3.DeleteObjectsCommand.objectIdentifier
-      ))
+      Array.concat(out.versions->Option.getOr([]), out.deleteMarkers->Option.getOr([]))->Array.map((
+        v
+      ): S3.DeleteObjectsCommand.objectIdentifier => {key: v.key, versionId: v.versionId})
     if ids->Array.length > 0 {
       let res = await S3.DeleteObjectsCommand.send(
         S3.DeleteObjectsCommand.make({
@@ -358,7 +502,7 @@ let emptyBucket = async (bucket: string): unit => {
       | Some(errs) if errs->Array.length > 0 =>
         throw(
           Seed.Failed(
-            `failed to delete ${(errs->Array.length)->Int.toString} object(s) from ${bucket}: ${errs
+            `failed to delete ${errs->Array.length->Int.toString} object(s) from ${bucket}: ${errs
               ->Array.get(0)
               ->Option.flatMap(e => e.message)
               ->Option.getOr("unknown")}`,
@@ -390,8 +534,16 @@ type target = {
   projectDir: string,
   label: string,
   group: group,
+  // The name this project's plugin REGISTERS, when it differs from the
+  // operator-facing `label` (`Catalog` against `catalog`). It is what the
+  // platform's `objectStores` keys are qualified by, so it is how a declared
+  // store is attributed to a target. Declared rather than case-folded from the
+  // label: the caller states the topology, the reset never guesses it.
+  plugin?: string,
 }
 
+let pluginOf = (t: target): string => t.plugin->Option.getOr(t.label)
+
 // A target resolved to its discovered, counted stores, ready to report and wipe.
 type resolved = {
   target: target,
@@ -399,6 +551,9 @@ type resolved = {
   tables: array<string>,
   tableCounts: array<int>,
   bucketCounts: array<(string, int)>,
+  storeCounts: array<(objectStore, int)>,
+  /** The project's Lambda functions — not wiped, held still while it is. */
+  functions: array<string>,
 }
 
 // Picks which targets to wipe. `domain` (every domain plugin) leads and is the
@@ -429,6 +584,7 @@ let chooseScope = async (~targets: array<target>): array<target> => {
     let options = []
     if domain->Array.length > 0 {
       options->Array.push((`domain — ${labelsOf(domain)}`, domain))
+
       // Single-plugin entries only when there is more than one, else they just
       // duplicate the `domain` entry.
       if domain->Array.length > 1 {
@@ -456,7 +612,9 @@ let gateTarget = (~target: target, ~backend, ~stack): string => {
   | Ok(c) => c
   | Error(message) => throw(Seed.Failed(message))
   }
-  switch configValue(cfg, "reventless:wipeable")->Option.map(v => v->String.trim->String.toLowerCase) {
+  switch configValue(cfg, "reventless:wipeable")->Option.map(v =>
+    v->String.trim->String.toLowerCase
+  ) {
   | Some("true") => ()
   | _ =>
     throw(
@@ -480,6 +638,47 @@ let gateTarget = (~target: target, ~backend, ~stack): string => {
   }
 }
 
+/** Opt out of holding the platform still for the wipe. The hold needs Lambda
+    permissions the reset did not previously ask for, so credentials that lack
+    them have a way through — at the cost of the emptiness guarantee, which the
+    note printed alongside it says plainly. */
+let noQuiesce = (): bool =>
+  switch Seed.Prompt.envValue("SEED_RESET_NO_QUIESCE")->Option.map(v =>
+    v->String.trim->String.toLowerCase
+  ) {
+  | Some("1") | Some("true") | Some("yes") => true
+  | _ => false
+  }
+
+/** What to say when a store is not empty after the wipe.
+
+    The old message reported only a total and advised re-running. That advice was
+    wrong in exactly the case that occurs: a runtime holding its state in memory
+    rewrites the store on every invocation, so every re-run reproduces the same
+    failure and the message sends the operator round the loop again.
+
+    So name the stores, and separate the two situations that are actually
+    different. Under a hold, nothing could start during the wipe, so the only
+    possible writer is an invocation that was already in flight — and a re-run
+    does clear that. Without a hold there is no such bound, and re-running is the
+    wrong advice. */
+let refillMessage = (~refilled: array<(string, int)>, ~heldCount: int): string => {
+  let total = refilled->Array.reduce(0, (acc, (_, n)) => acc + n)
+  let lines =
+    refilled
+    ->Array.map(((label, n)) => `\n      ${n->Int.toString->String.padStart(8, " ")}  ${label}`)
+    ->Array.join("")
+  let diagnosis = if heldCount > 0 {
+    `\n\n  The wipe deleted these and something wrote them back. All ${heldCount->Int.toString} runtime(s) ` ++
+    `in scope were held at zero concurrency for the whole wipe, so the writer was an invocation already ` ++ `in flight when the hold took effect. Re-running clears it.`
+  } else {
+    `\n\n  The wipe deleted these and a running runtime wrote them back. The stack was NOT held ` ++
+    `(SEED_RESET_NO_QUIESCE), so nothing stopped it, and re-running will not help: the writer keeps the ` ++
+    `state in memory and restores it on every invocation, identical each time. Re-run without ` ++ `SEED_RESET_NO_QUIESCE so the runtimes are held and recycled.`
+  }
+  `${total->Int.toString} item(s)/object(s) came back after the wipe:${lines}${diagnosis}`
+}
+
 let reportAll = (resolvedList: array<resolved>, ~stack, ~region): int => {
   Seed.Runner.heading(`Reset target: stack "${stack}" in ${region}`)
   let total = ref(0)
@@ -503,6 +702,20 @@ let reportAll = (resolvedList: array<resolved>, ~stack, ~region): int => {
     if r.bucketCounts->Array.length == 0 {
       Console.log("      (none)")
     }
+
+    // Declared stores get their own section rather than being folded in with the
+    // plain buckets: the unit is a prefix inside a bucket that other plugins also
+    // write to, and the operator needs to see which is which before confirming.
+    if r.storeCounts->Array.length > 0 {
+      Console.log("    Object stores:")
+      r.storeCounts->Array.forEach(((s, c)) => {
+        total := total.contents + c
+        Console.log(
+          `      ${c->Int.toString->String.padStart(8, " ")}  ${s.qualified}   ` ++
+          `${s.bucketName}/${s.keyPrefix}/`,
+        )
+      })
+    }
   })
   total.contents
 }
@@ -548,9 +761,52 @@ let run = (~stack=?, ~backend=?, ~targets: array<target>, ()): unit => {
 
       let selected = await chooseScope(~targets)
 
+      // Declared object stores live in a bucket the PLATFORM project owns, so
+      // resolving them reads the platform's stack output whichever scope was
+      // picked. Reading is side-effect free; the gates below decide whether
+      // anything may be deleted from it.
+      let platformTarget = targets->Array.find(t => t.group == Platform)
+      let allStores = switch platformTarget {
+      | None =>
+        // Not silent: a topology with no platform target is exactly the case
+        // where declared stores would be missed, and missing them is the bug
+        // this resolution exists to fix.
+        Console.log("")
+        Console.log(
+          "Note: no `platform` target is declared, so declared object stores could not be " ++ "resolved — any uploaded objects will be left in place.",
+        )
+        []
+      | Some(pt) =>
+        let output = ReventlessSeedAws.stackOutputs(~projectDir=pt.projectDir, ~backend, stack)
+        switch parseObjectStores(output->field("objectStores")) {
+        | Ok(stores) => stores
+        | Error(message) => throw(Seed.Failed(message))
+        }
+      }
+      switch validateStores(allStores) {
+      | Ok() => ()
+      | Error(message) => throw(Seed.Failed(`refusing: ${message}`))
+      }
+
+      // Every bucket that holds a declared store, selected or not. These are
+      // excluded from the plain per-target bucket lists below so a store is
+      // reachable ONLY through the plugin that declared it — otherwise the
+      // platform scope would empty every plugin's objects wholesale.
+      let storeBucketNames = allStores->Array.map(s => s.bucketName)
+      let selectedStores =
+        allStores->Array.filter(s => selected->Array.some(t => pluginOf(t) == s.plugin))
+
       // Gate every selected target and collect its region; all must agree, since
-      // the DynamoDB/S3 clients read one region from the environment.
-      let regions = selected->Array.map(t => gateTarget(~target=t, ~backend, ~stack))
+      // the DynamoDB/S3 clients read one region from the environment. When a
+      // store is in scope, the platform's own stack must also declare itself
+      // wipeable — the objects belong to a plugin, but the bucket is the
+      // platform's, and both consents are needed to delete from it.
+      let gated = switch (selectedStores->Array.length > 0, platformTarget) {
+      | (true, Some(pt)) if !(selected->Array.some(t => t.projectDir == pt.projectDir)) =>
+        Array.concat(selected, [pt])
+      | _ => selected
+      }
+      let regions = gated->Array.map(t => gateTarget(~target=t, ~backend, ~stack))
       let region = regions->Array.getUnsafe(0)
       if regions->Array.some(r => r != region) {
         throw(
@@ -561,7 +817,31 @@ let run = (~stack=?, ~backend=?, ~targets: array<target>, ()): unit => {
           ),
         )
       }
-      processEnv->Dict.set("AWS_REGION", region)
+      NodeProcess.env->Dict.set("AWS_REGION", region)
+
+      // A store bucket arrives by stack output, not by tag discovery, so the
+      // per-resource tag re-check has to be applied to it explicitly: confirm it
+      // carries the platform project's own `platform`+`environment` tags before
+      // anything is deleted from it. One extra tagging-API call, and only when a
+      // store is actually in scope.
+      if selectedStores->Array.length > 0 {
+        switch platformTarget {
+        | Some(pt) =>
+          let platformProject = projectName(~projectDir=pt.projectDir)
+          let platformFound = await discover(~region, ~stack, ~platform=platformProject)
+          selectedStores->Array.forEach(s =>
+            if !(platformFound.buckets->Array.includes(s.bucketName)) {
+              throw(
+                Seed.Failed(
+                  `refusing: store "${s.qualified}" names bucket ${s.bucketName}, which does not ` ++
+                  `carry reventless:platform=${platformProject} + reventless:environment=${stack}.`,
+                ),
+              )
+            }
+          )
+        | None => ()
+        }
+      }
 
       // Discover + count each target, scoped to its own project via the platform
       // tag so a same-named stack from another project is never touched.
@@ -570,9 +850,12 @@ let run = (~stack=?, ~backend=?, ~targets: array<target>, ()): unit => {
         switch selected->Array.get(i) {
         | Some(target) =>
           let platform = projectName(~projectDir=target.projectDir)
-          let (tables, buckets) = await discover(~region, ~stack, ~platform)
-          let tables = tables->Array.toSorted(String.compare)
-          let buckets = buckets->Array.toSorted(String.compare)
+          let found = await discover(~region, ~stack, ~platform)
+          let tables = found.tables->Array.toSorted(String.compare)
+          let buckets =
+            found.buckets
+            ->Array.filter(b => !(storeBucketNames->Array.includes(b)))
+            ->Array.toSorted(String.compare)
           let tableCounts = []
           for j in 0 to tables->Array.length - 1 {
             switch tables->Array.get(j) {
@@ -587,7 +870,27 @@ let run = (~stack=?, ~backend=?, ~targets: array<target>, ()): unit => {
             | None => ()
             }
           }
-          resolvedList->Array.push({target, platform, tables, tableCounts, bucketCounts})
+          let stores = selectedStores->Array.filter(s => s.plugin == pluginOf(target))
+          let storeCounts = []
+          for j in 0 to stores->Array.length - 1 {
+            switch stores->Array.get(j) {
+            | Some(s) =>
+              storeCounts->Array.push((
+                s,
+                await countBucket(s.bucketName, ~prefix=`${s.keyPrefix}/`),
+              ))
+            | None => ()
+            }
+          }
+          resolvedList->Array.push({
+            target,
+            platform,
+            tables,
+            tableCounts,
+            bucketCounts,
+            storeCounts,
+            functions: found.functions->Array.toSorted(String.compare),
+          })
         | None => ()
         }
       }
@@ -597,7 +900,7 @@ let run = (~stack=?, ~backend=?, ~targets: array<target>, ()): unit => {
         Seed.Prompt.close()
         Console.log("")
         Console.log(`Nothing to reset — the selected scope already reads empty in ${region}.`)
-        exit(0)
+        NodeProcess.exit(0)
       }
 
       // Confirmation. Interactively, re-typing the exact stack name IS the
@@ -606,11 +909,10 @@ let run = (~stack=?, ~backend=?, ~targets: array<target>, ()): unit => {
       // TTY (CI/scripts) there is nothing to type into, so
       // REVENTLESS_WIPE_CONFIRM=<stack> is the equivalent opt-in. Dry-run is the
       // default either way.
-      let interactive = stdin->isTTY->Option.getOr(false)
+      let interactive = NodeProcess.stdin->NodeProcess.isTTY->Option.getOr(false)
       let confirmed = if interactive {
         let typed = await Seed.Prompt.ask(
-          `About to permanently empty ${total->Int.toString} item(s)/object(s) across the selected scope of "${stack}". ` ++
-          `Type the stack name to confirm, or press Enter to keep this a dry run: `,
+          `About to permanently empty ${total->Int.toString} item(s)/object(s) across the selected scope of "${stack}". ` ++ `Type the stack name to confirm, or press Enter to keep this a dry run: `,
         )
         typed->String.trim == stack
       } else {
@@ -624,76 +926,177 @@ let run = (~stack=?, ~backend=?, ~targets: array<target>, ()): unit => {
             ? "Dry run — nothing was deleted."
             : `Dry run — nothing was deleted. Set REVENTLESS_WIPE_CONFIRM=${stack} to empty this scope non-interactively.`,
         )
-        exit(0)
+        NodeProcess.exit(0)
       }
 
-      for i in 0 to resolvedList->Array.length - 1 {
-        switch resolvedList->Array.get(i) {
-        | Some(r) =>
-          Seed.Runner.heading(`Emptying ${r.target.label} …`)
-          for j in 0 to r.tables->Array.length - 1 {
-            switch r.tables->Array.get(j) {
-            | Some(t) =>
-              await truncateTable(t)
-              Console.log(`  truncated ${t}`)
-            | None => ()
+      // Hold the platform still before the first delete. A truncate is not
+      // durable while the runtimes that own the data are running — see
+      // ReventlessSeedAws_Quiesce for the mechanism and for why emptying the
+      // upstream stores first does not substitute for this.
+      let functionNames =
+        resolvedList->Array.reduce([], (acc, r) =>
+          Array.concat(acc, r.functions->Array.filter(f => !(acc->Array.includes(f))))
+        )
+      let quiesce = !noQuiesce() && functionNames->Array.length > 0
+      let lambdaClient = Lambda.client(~region, ())
+      let held = if quiesce {
+        Seed.Runner.heading(
+          `Holding ${functionNames->Array.length->Int.toString} runtime(s) at zero concurrency …`,
+        )
+        await Quiesce.hold(~client=lambdaClient, ~functionNames)
+      } else {
+        if functionNames->Array.length > 0 {
+          Console.log("")
+          Console.log(
+            "Note: SEED_RESET_NO_QUIESCE is set — the stack's runtimes keep running through the wipe. " ++ "A runtime that holds state across invocations can write it straight back over an emptied store.",
+          )
+        }
+        []
+      }
+
+      // From here the reset owns the platform's concurrency, so every exit has
+      // to run through the release below — including the abort path. ReScript
+      // has no `finally`, so the wipe's outcome is captured as a value and
+      // re-raised after the stack has been handed back.
+      let outcome = try {
+        for i in 0 to resolvedList->Array.length - 1 {
+          switch resolvedList->Array.get(i) {
+          | Some(r) =>
+            Seed.Runner.heading(`Emptying ${r.target.label} …`)
+            for j in 0 to r.tables->Array.length - 1 {
+              switch r.tables->Array.get(j) {
+              | Some(t) =>
+                await truncateTable(t)
+                Console.log(`  truncated ${t}`)
+              | None => ()
+              }
             }
-          }
-          for j in 0 to r.bucketCounts->Array.length - 1 {
-            switch r.bucketCounts->Array.get(j) {
-            | Some((b, _)) =>
-              await emptyBucket(b)
-              Console.log(`  emptied ${b}`)
-            | None => ()
+            for j in 0 to r.bucketCounts->Array.length - 1 {
+              switch r.bucketCounts->Array.get(j) {
+              | Some((b, _)) =>
+                await emptyBucket(b)
+                Console.log(`  emptied ${b}`)
+              | None => ()
+              }
+            }
+            // Report the count with the store, not just in the run total: "how
+            // many images went" is the question an operator is actually asking.
+            for j in 0 to r.storeCounts->Array.length - 1 {
+              switch r.storeCounts->Array.get(j) {
+              | Some((s, count)) =>
+                await emptyBucket(s.bucketName, ~prefix=`${s.keyPrefix}/`)
+                Console.log(
+                  `  emptied ${s.qualified} — ${count->Int.toString} object(s) removed from ` ++
+                  `${s.bucketName}/${s.keyPrefix}/`,
+                )
+              | None => ()
+              }
             }
+          | None => ()
           }
-        | None => ()
         }
-      }
 
-      // Prove empty — the inverse of the post-seed verify.
-      let remaining = ref(0)
-      for i in 0 to resolvedList->Array.length - 1 {
-        switch resolvedList->Array.get(i) {
-        | Some(r) =>
-          for j in 0 to r.tables->Array.length - 1 {
-            switch r.tables->Array.get(j) {
-            | Some(t) => remaining := remaining.contents + (await countTable(t))
-            | None => ()
+        // Prove empty — the inverse of the post-seed verify. Every store that
+        // still reads non-zero is named, because "12 items remain" without
+        // saying where is the difference between a diagnosis and a mystery.
+        let refilled = []
+        for i in 0 to resolvedList->Array.length - 1 {
+          switch resolvedList->Array.get(i) {
+          | Some(r) =>
+            for j in 0 to r.tables->Array.length - 1 {
+              switch r.tables->Array.get(j) {
+              | Some(t) =>
+                let left = await countTable(t)
+                if left != 0 {
+                  refilled->Array.push((t, left))
+                }
+              | None => ()
+              }
             }
-          }
-          for j in 0 to r.bucketCounts->Array.length - 1 {
-            switch r.bucketCounts->Array.get(j) {
-            | Some((b, _)) => remaining := remaining.contents + (await countBucket(b))
-            | None => ()
+            for j in 0 to r.bucketCounts->Array.length - 1 {
+              switch r.bucketCounts->Array.get(j) {
+              | Some((b, _)) =>
+                let left = await countBucket(b)
+                if left != 0 {
+                  refilled->Array.push((b, left))
+                }
+              | None => ()
+              }
             }
+            // Per-store confirmation, so "every uploaded object is gone" is an
+            // observed fact rather than an inference from a global total.
+            for j in 0 to r.storeCounts->Array.length - 1 {
+              switch r.storeCounts->Array.get(j) {
+              | Some((s, _)) =>
+                let left = await countBucket(s.bucketName, ~prefix=`${s.keyPrefix}/`)
+                if left != 0 {
+                  refilled->Array.push((`${s.qualified} (${s.bucketName}/${s.keyPrefix}/)`, left))
+                }
+                Console.log(
+                  `  verified empty: ${s.qualified} — ${left->Int.toString} object(s) remain under ` ++
+                  `${s.bucketName}/${s.keyPrefix}/`,
+                )
+              | None => ()
+              }
+            }
+          | None => ()
           }
-        | None => ()
         }
-      }
-      if remaining.contents != 0 {
-        throw(
-          Seed.Failed(
-            `${remaining.contents->Int.toString} item(s)/object(s) remain after the wipe — re-run to finish.`,
-          ),
+        Ok(refilled)
+      } catch {
+      | Seed.Failed(message) => Error(message)
+      | exn =>
+        Error(
+          `the wipe failed: ${exn
+            ->JsExn.fromException
+            ->Option.flatMap(JsExn.message)
+            ->Option.getOr("unknown error")}`,
         )
       }
 
+      // Recycle while still held, so the container that eventually serves
+      // traffic is created after the stores were emptied — then hand the stack
+      // back. Both are best-effort and report rather than raise: they run on the
+      // abort path too, where throwing would bury the reason the reset failed.
+      let warnings = if held->Array.length > 0 {
+        Seed.Runner.heading(`Recycling ${held->Array.length->Int.toString} runtime(s) …`)
+        let recycled = await Quiesce.recycle(~client=lambdaClient, ~held)
+        let released = await Quiesce.release(~client=lambdaClient, ~held)
+        Array.concat(recycled, released)
+      } else {
+        []
+      }
+      if warnings->Array.length > 0 {
+        Console.error("")
+        warnings->Array.forEach(w => Console.error(`  warning: ${w}`))
+      }
+
+      switch outcome {
+      | Error(message) => throw(Seed.Failed(message))
+      | Ok(refilled) if refilled->Array.length > 0 =>
+        throw(Seed.Failed(refillMessage(~refilled, ~heldCount=held->Array.length)))
+      | Ok(_) => ()
+      }
+
       Console.log("")
-      Console.log(`Reset complete — the selected scope of "${stack}" reads empty and is re-seedable.`)
-      exit(0)
+      Console.log(
+        `Reset complete — the selected scope of "${stack}" reads empty and is re-seedable.`,
+      )
+      NodeProcess.exit(0)
     } catch {
     | Seed.Failed(message) =>
       Seed.Prompt.close()
       Console.error("")
       Console.error(`Reset aborted — ${message}`)
-      exit(1)
+      NodeProcess.exit(1)
     | exn =>
       Seed.Prompt.close()
       Console.error("")
       Console.error("Reset aborted with an unexpected error:")
-      Console.error(exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("unknown"))
-      exit(1)
+      Console.error(
+        exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("unknown"),
+      )
+      NodeProcess.exit(1)
     }
   }
   go()->ignore