@reventlessdev/reventless-seed-aws 1.0.0-alpha.11 → 1.0.0-alpha.13

package/lib/bs/tests/QuiesceResetTest.res.mjs

@@ -0,0 +1,141 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
+import * as ReventlessSeedAws_Reset from "../src/ReventlessSeedAws_Reset.res.mjs";
+import * as ReventlessSeedAws_Quiesce from "../src/ReventlessSeedAws_Quiesce.res.mjs";
+import * as Seed_Client$ReventlessSeed from "@reventlessdev/reventless-seed/src/Seed_Client.res.mjs";
+
+globalThis.describe("classify", () => {
+  globalThis.test("reads the function name out of a Lambda ARN", () => {
+    globalThis.expect(ReventlessSeedAws_Reset.classify("arn:aws:lambda:eu-west-1:123456789012:function:AllAutomationSlices-28b116c")).toEqual({
+      TAG: "Function",
+      _0: "AllAutomationSlices-28b116c"
+    });
+  });
+  globalThis.test("drops a version or alias qualifier", () => {
+    globalThis.expect(ReventlessSeedAws_Reset.classify("arn:aws:lambda:eu-west-1:123456789012:function:AllReadModels-0287438:42")).toEqual({
+      TAG: "Function",
+      _0: "AllReadModels-0287438"
+    });
+  });
+  globalThis.test("still classifies tables and buckets", () => {
+    globalThis.expect(ReventlessSeedAws_Reset.classify("arn:aws:dynamodb:eu-west-1:123456789012:table/Products-0ba6849")).toEqual({
+      TAG: "Table",
+      _0: "Products-0ba6849"
+    });
+    globalThis.expect(ReventlessSeedAws_Reset.classify("arn:aws:s3:::alpha-product-images")).toEqual({
+      TAG: "Bucket",
+      _0: "alpha-product-images"
+    });
+  });
+  globalThis.test("ignores anything else the tag scope returns", () => {
+    globalThis.expect(ReventlessSeedAws_Reset.classify("arn:aws:sqs:eu-west-1:123456789012:OrderingDcbCmdTopic-4b3658c")).toEqual("Other");
+  });
+});
+
+globalThis.describe("mapBounded", () => {
+  globalThis.test("keeps results in input order under out-of-order completion", async () => {
+    let delays = [
+      40,
+      0,
+      30,
+      10,
+      20,
+      5
+    ];
+    let out = await ReventlessSeedAws_Quiesce.mapBounded(delays, 3, async ms => {
+      await Seed_Client$ReventlessSeed.sleep(ms);
+      return ms;
+    });
+    globalThis.expect(out).toEqual(delays);
+  });
+  globalThis.test("never runs more than the limit at once", async () => {
+    let inFlight = {
+      contents: 0
+    };
+    let peak = {
+      contents: 0
+    };
+    await ReventlessSeedAws_Quiesce.mapBounded(Stdlib_Array.make(12, 0), 4, async param => {
+      inFlight.contents = inFlight.contents + 1 | 0;
+      peak.contents = Math.max(peak.contents, inFlight.contents);
+      await Seed_Client$ReventlessSeed.sleep(5);
+      inFlight.contents = inFlight.contents - 1 | 0;
+    });
+    globalThis.expect(peak.contents).toBe(4);
+  });
+  globalThis.test("an empty list spawns no workers and returns nothing", async () => {
+    let out = await ReventlessSeedAws_Quiesce.mapBounded([], 4, async x => x);
+    globalThis.expect(out.length).toBe(0);
+  });
+});
+
+globalThis.describe("refillMessage", () => {
+  globalThis.test("names every store that came back, with its count", () => {
+    let message = ReventlessSeedAws_Reset.refillMessage([
+      [
+        "AutoShipOrderTodo-61f39a2",
+        12
+      ],
+      [
+        "GeocodeCustomerAddressTodo-50d9488",
+        1
+      ]
+    ], 41);
+    globalThis.expect(message.includes("AutoShipOrderTodo-61f39a2")).toBe(true);
+    globalThis.expect(message.includes("GeocodeCustomerAddressTodo-50d9488")).toBe(true);
+    globalThis.expect(message.startsWith("13 item(s)/object(s) came back")).toBe(true);
+  });
+  globalThis.test("advises re-running only when the runtimes were held", () => {
+    let message = ReventlessSeedAws_Reset.refillMessage([[
+        "AutoShipOrderTodo-61f39a2",
+        12
+      ]], 41);
+    globalThis.expect(message.includes("already in flight")).toBe(true);
+    globalThis.expect(message.includes("Re-running clears it")).toBe(true);
+  });
+  globalThis.test("refuses to advise re-running when the stack was not held", () => {
+    let message = ReventlessSeedAws_Reset.refillMessage([[
+        "AutoShipOrderTodo-61f39a2",
+        12
+      ]], 0);
+    globalThis.expect(message.includes("re-running will not help".toLowerCase())).toBe(true);
+    globalThis.expect(message.includes("SEED_RESET_NO_QUIESCE")).toBe(true);
+  });
+});
+
+globalThis.describe("noQuiesce", () => {
+  let set = v => {
+    process.env["SEED_RESET_NO_QUIESCE"] = v;
+  };
+  globalThis.test("defaults to holding the platform", () => {
+    set("");
+    globalThis.expect(ReventlessSeedAws_Reset.noQuiesce()).toBe(false);
+    set("0");
+    globalThis.expect(ReventlessSeedAws_Reset.noQuiesce()).toBe(false);
+    set("no");
+    globalThis.expect(ReventlessSeedAws_Reset.noQuiesce()).toBe(false);
+  });
+  globalThis.test("accepts the spellings an operator would actually type", () => {
+    [
+      "1",
+      "true",
+      "TRUE",
+      " yes "
+    ].forEach(v => {
+      set(v);
+      globalThis.expect(ReventlessSeedAws_Reset.noQuiesce()).toBe(true);
+    });
+    set("");
+  });
+});
+
+let Reset;
+
+let Quiesce;
+
+export {
+  Reset,
+  Quiesce,
+}
+/*  Not a pure module */

package/lib/ocaml/.compiler.log

@@ -1,2 +1,2 @@
-#Start(1785775483095)
-#Done(1785775483128)
+#Start(1786243797857)
+#Done(1786243797920)

package/lib/ocaml/QuiesceResetTest.res

@@ -0,0 +1,133 @@
+// Holding a stack's runtimes still for the length of a wipe: what gets held, and
+// what the reset says when a store comes back anyway.
+//
+// A truncate is not durable while the runtimes that own the data are running — a
+// slice runtime keeps its TODO list in memory for the life of its execution
+// environment and re-saves every row it holds at the end of each invocation,
+// including the scheduled sweep that carries no events. So the reset now
+// discovers the stack's Lambda functions alongside its stores and holds them at
+// zero concurrency across the wipe.
+//
+// The pure half of that is what these pin: which ARNs resolve to a function to
+// hold, the bounded fan-out the control-plane calls go through, and the failure
+// message — which used to say "re-run to finish" for a condition re-running
+// could not clear. The impure half is the Lambda control-plane calls themselves.
+
+open JestGlobals
+
+module Reset = ReventlessSeedAws_Reset
+module Quiesce = ReventlessSeedAws_Quiesce
+
+describe("classify", () => {
+  testSync("reads the function name out of a Lambda ARN", () =>
+    expect(Reset.classify("arn:aws:lambda:eu-west-1:123456789012:function:AllAutomationSlices-28b116c"))
+    ->toEqual(Reset.Function("AllAutomationSlices-28b116c"))
+  )
+
+  // The tagging API returns unqualified ARNs, but a qualified one must still
+  // yield the function name — every Lambda control-plane call the hold makes
+  // takes a name, and "Name:1" is not one.
+  testSync("drops a version or alias qualifier", () =>
+    expect(Reset.classify("arn:aws:lambda:eu-west-1:123456789012:function:AllReadModels-0287438:42"))
+    ->toEqual(Reset.Function("AllReadModels-0287438"))
+  )
+
+  testSync("still classifies tables and buckets", () => {
+    expect(Reset.classify("arn:aws:dynamodb:eu-west-1:123456789012:table/Products-0ba6849"))
+    ->toEqual(Reset.Table("Products-0ba6849"))
+    expect(Reset.classify("arn:aws:s3:::alpha-product-images"))->toEqual(
+      Reset.Bucket("alpha-product-images"),
+    )
+  })
+
+  testSync("ignores anything else the tag scope returns", () =>
+    expect(Reset.classify("arn:aws:sqs:eu-west-1:123456789012:OrderingDcbCmdTopic-4b3658c"))
+    ->toEqual(Reset.Other)
+  )
+})
+
+describe("mapBounded", () => {
+  // Results must line up with their inputs regardless of which worker finished
+  // first: the hold pairs each result with the function it came from, and a
+  // shuffled array would restore the wrong concurrency to the wrong function.
+  test("keeps results in input order under out-of-order completion", async () => {
+    let delays = [40, 0, 30, 10, 20, 5]
+    let out = await delays->Quiesce.mapBounded(~limit=3, async ms => {
+      await ReventlessSeed.Seed.Client.sleep(ms)
+      ms
+    })
+    expect(out)->toEqual(delays)
+  })
+
+  test("never runs more than the limit at once", async () => {
+    let inFlight = ref(0)
+    let peak = ref(0)
+    let _ = await Array.make(~length=12, 0)->Quiesce.mapBounded(~limit=4, async _ => {
+      inFlight := inFlight.contents + 1
+      peak := Math.Int.max(peak.contents, inFlight.contents)
+      await ReventlessSeed.Seed.Client.sleep(5)
+      inFlight := inFlight.contents - 1
+    })
+    expect(peak.contents)->toBe(4)
+  })
+
+  test("an empty list spawns no workers and returns nothing", async () => {
+    let out = await []->Quiesce.mapBounded(~limit=4, async x => x)
+    expect(out->Array.length)->toBe(0)
+  })
+})
+
+describe("refillMessage", () => {
+  // The whole point of the change: the operator has to be able to see WHICH
+  // store came back, not just a total.
+  testSync("names every store that came back, with its count", () => {
+    let message = Reset.refillMessage(
+      ~refilled=[("AutoShipOrderTodo-61f39a2", 12), ("GeocodeCustomerAddressTodo-50d9488", 1)],
+      ~heldCount=41,
+    )
+    expect(message->String.includes("AutoShipOrderTodo-61f39a2"))->toBe(true)
+    expect(message->String.includes("GeocodeCustomerAddressTodo-50d9488"))->toBe(true)
+    expect(message->String.startsWith("13 item(s)/object(s) came back"))->toBe(true)
+  })
+
+  // Under a hold nothing could START during the wipe, so the only writer left is
+  // an invocation already running when the hold landed — and that one really
+  // does go away. This is the only case where "re-run" is honest advice.
+  testSync("advises re-running only when the runtimes were held", () => {
+    let message = Reset.refillMessage(~refilled=[("AutoShipOrderTodo-61f39a2", 12)], ~heldCount=41)
+    expect(message->String.includes("already in flight"))->toBe(true)
+    expect(message->String.includes("Re-running clears it"))->toBe(true)
+  })
+
+  // Without a hold, the writer holds the rows in memory and restores them every
+  // invocation — the exact condition the old "re-run to finish" sent operators
+  // round in circles on. It must not appear here.
+  testSync("refuses to advise re-running when the stack was not held", () => {
+    let message = Reset.refillMessage(~refilled=[("AutoShipOrderTodo-61f39a2", 12)], ~heldCount=0)
+    expect(message->String.includes("re-running will not help"->String.toLowerCase))->toBe(true)
+    expect(message->String.includes("SEED_RESET_NO_QUIESCE"))->toBe(true)
+  })
+})
+
+describe("noQuiesce", () => {
+  let set = v => NodeProcess.env->Dict.set("SEED_RESET_NO_QUIESCE", v)
+
+  // Fail-closed: the hold is what makes the wipe durable, so anything other than
+  // a deliberate opt-out keeps it on.
+  testSync("defaults to holding the platform", () => {
+    set("")
+    expect(Reset.noQuiesce())->toBe(false)
+    set("0")
+    expect(Reset.noQuiesce())->toBe(false)
+    set("no")
+    expect(Reset.noQuiesce())->toBe(false)
+  })
+
+  testSync("accepts the spellings an operator would actually type", () => {
+    ["1", "true", "TRUE", " yes "]->Array.forEach(v => {
+      set(v)
+      expect(Reset.noQuiesce())->toBe(true)
+    })
+    set("")
+  })
+})

package/lib/ocaml/ReventlessSeedAws_Quiesce.res

@@ -0,0 +1,287 @@
+// Taking a deployed stack's runtimes out of service for the length of a wipe,
+// and putting them back exactly as they were.
+//
+// Why a wipe needs this at all: a truncate is not durable while the runtimes
+// that own the data are running. A slice runtime keeps its TODO list in a
+// module-level dict for the lifetime of its execution environment and re-saves
+// every row it holds at the end of each invocation — including the scheduled
+// sweep, which carries no events. Emptying the table under a warm container
+// therefore succeeds and is then undone, byte for byte, by the next sweep: the
+// rows are not recomputed from anything, so their ids and timestamps come back
+// identical. Emptying the upstream stores first does not help, because nothing
+// upstream is read.
+//
+// The answer is to remove the contention rather than to outrun it, which takes
+// two distinct steps, in this order:
+//
+//   1. HOLD — reserve 0 concurrency on every function in scope, so no new
+//      invocation can start while the wipe and its verification run. This alone
+//      is not enough: a reservation stops invocations, it does not discard the
+//      execution environment that holds the state.
+//   2. RECYCLE — change each function's configuration once. A configuration
+//      change is the documented way to make Lambda discard existing execution
+//      environments, so whatever a warm container was holding is gone before it
+//      can be invoked again. Done while still held, so the container that
+//      eventually replaces it starts from the emptied store.
+//
+// The recycle is a round trip — a marker variable added, then the original map
+// put back — so the function ends the run byte-identical to how it started and
+// `pulumi preview` reports no drift. The second update recycles again, which is
+// harmless: it happens while the hold is still on and the store is already
+// empty.
+//
+// Everything acquired here is released on the failure path as well as the
+// success path, and the release never throws: a reset that aborted mid-wipe
+// must still hand the stack back. Release failures come back as warnings for
+// the caller to print, because a function left at zero concurrency is a stack
+// left switched off, and that has to be said out loud rather than raised as one
+// more exception on top of whatever already went wrong.
+
+module Lambda = AwsSdk.Lambda
+
+/** The variable a recycle adds and then removes. Named for what it is, so an
+    operator who finds one left behind by a killed run knows it is inert and can
+    be deleted (or left for the next `pulumi up` to remove). */
+let markerKey = "REVENTLESS_RESET_RECYCLE"
+
+/** What one function looked like before the reset touched it. */
+type held = {
+  functionName: string,
+  /** `None` = the function had no reservation at all. That is a different state
+      from a reservation of 0, and restoring it needs
+      `DeleteFunctionConcurrency` rather than a put. */
+  priorReserved: option<int>,
+  /** The variable map as it was, flattened: a function with no `Environment`
+      block at all reads as an empty map and is restored to an empty map. */
+  priorVariables: dict<string>,
+}
+
+/** The Lambda control plane is rate-limited well below the data plane, and a
+    reset touches every function in a project at once. Six at a time keeps a
+    forty-function stack under a minute per pass without risking a throttle. */
+let concurrency = 6
+
+let mapBounded = async (items: array<'a>, ~limit: int, f: 'a => promise<'b>): array<'b> => {
+  let results = Array.make(~length=items->Array.length, None)
+  let next = ref(0)
+  let worker = async () => {
+    let running = ref(true)
+    while running.contents {
+      let i = next.contents
+      if i >= items->Array.length {
+        running := false
+      } else {
+        next := i + 1
+        let r = await f(items->Array.getUnsafe(i))
+        results->Array.set(i, Some(r))
+      }
+    }
+  }
+  let _ = await Array.fromInitializer(~length=Math.Int.min(limit, items->Array.length), _ =>
+    worker()
+  )->Promise.all
+  results->Array.filterMap(x => x)
+}
+
+let errorText = (exn: exn): string =>
+  exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("unknown error")
+
+// An update is rejected while another is still settling, and a stack that was
+// deployed moments ago can still be settling. Poll rather than assume.
+let awaitSettled = async (~client, ~functionName: string): unit => {
+  let deadline = 60
+  let rec poll = async (attempt: int): unit =>
+    if attempt < deadline {
+      let out = await Lambda.GetFunctionConfigurationCommand.send(
+        client,
+        Lambda.GetFunctionConfigurationCommand.make({functionName: functionName}),
+      )
+      switch out.lastUpdateStatus {
+      | Some("InProgress") =>
+        await ReventlessSeed.Seed.Client.sleep(1000)
+        await poll(attempt + 1)
+      | _ => ()
+      }
+    }
+  await poll(0)
+}
+
+let updateVariables = async (~client, ~functionName: string, ~variables: dict<string>): unit => {
+  await awaitSettled(~client, ~functionName)
+  let _ = await Lambda.UpdateFunctionConfigurationCommand.send(
+    client,
+    Lambda.UpdateFunctionConfigurationCommand.make({
+      functionName,
+      environment: {variables: variables},
+    }),
+  )
+  await awaitSettled(~client, ~functionName)
+}
+
+/** Put one function's reservation back exactly as it was — a value it had, or no
+    reservation at all. Never throws: a function left reserved at 0 is a function
+    left switched off, which has to be said rather than raised. */
+let restoreConcurrency = async (~client, ~h: held): option<string> =>
+  switch await (
+    switch h.priorReserved {
+    | Some(n) =>
+      Lambda.PutFunctionConcurrencyCommand.send(
+        client,
+        Lambda.PutFunctionConcurrencyCommand.make({
+          functionName: h.functionName,
+          reservedConcurrentExecutions: n,
+        }),
+      )->Promise.thenResolve(_ => ())
+    | None =>
+      Lambda.DeleteFunctionConcurrencyCommand.send(
+        client,
+        Lambda.DeleteFunctionConcurrencyCommand.make({functionName: h.functionName}),
+      )->Promise.thenResolve(_ => ())
+    }
+  ) {
+  | () => None
+  | exception exn =>
+    Some(
+      `LEFT SWITCHED OFF: could not restore concurrency on ${h.functionName} (${errorText(exn)}). ` ++
+      `It is still reserved at 0 and will not run until that is undone.`,
+    )
+  }
+
+/**
+Hand the stack back: original environment first (which also drops the marker),
+then the original concurrency.
+
+That order matters. The environment restore is itself a configuration change, so
+it recycles a second time; doing it while the function is still held means the
+container that finally serves traffic is created after the restore, not during
+it.
+
+Never throws, for either step and for any function — this runs on the abort path
+too, where an exception would replace the reason the reset failed with a
+secondary one.
+*/
+let release = async (~client, ~held: array<held>): array<string> => {
+  let warnings = await held->mapBounded(~limit=concurrency, async h => {
+    let envWarning = switch await updateVariables(
+      ~client,
+      ~functionName=h.functionName,
+      ~variables=h.priorVariables,
+    ) {
+    | () => None
+    | exception exn =>
+      Some(
+        `could not restore the environment of ${h.functionName} (${errorText(exn)}) — it may still ` ++
+        `carry ${markerKey}, which is inert and is removed by the next deploy.`,
+      )
+    }
+    let concurrencyWarning = await restoreConcurrency(~client, ~h)
+    [envWarning, concurrencyWarning]->Array.filterMap(w => w)
+  })
+  warnings->Array.flat
+}
+
+/**
+Discard every held function's execution environments, so nothing that loaded
+state before the wipe can write it back afterwards.
+
+Never throws. A function that cannot be recycled is the one case where the reset
+genuinely cannot promise the store stays empty, so it comes back as a warning
+naming the function — the caller reports it rather than swallowing it.
+*/
+let recycle = async (~client, ~held: array<held>): array<string> => {
+  let warnings = await held->mapBounded(~limit=concurrency, async h => {
+    let marked = h.priorVariables->Dict.copy
+    marked->Dict.set(markerKey, "1")
+    switch await updateVariables(~client, ~functionName=h.functionName, ~variables=marked) {
+    | () => None
+    | exception exn =>
+      Some(
+        `could not recycle ${h.functionName} (${errorText(exn)}) — a warm container may still hold ` ++
+        `pre-wipe state and write it back. Redeploy or update that function to clear it.`,
+      )
+    }
+  })
+  warnings->Array.filterMap(w => w)
+}
+
+/**
+Take every named function out of service, recording what to put back.
+
+Throws if any function cannot be held — before a single delete has been issued,
+so a reset that cannot hold the platform still has the option of not starting.
+The message names the IAM actions involved, because a missing permission is much
+the likeliest cause and "AccessDenied" on its own does not say which one.
+
+Crucially, it hands back whatever it did manage to take before throwing. The
+functions are held concurrently, so a failure part-way through the batch leaves
+earlier ones already at zero — and a reset that aborted while silently switching
+off half a stack would be a worse failure than the one it was reporting. Only the
+reservations are rolled back: this runs before `recycle`, so no environment has
+been touched yet.
+*/
+let hold = async (~client, ~functionNames: array<string>): array<held> => {
+  let attempts = await functionNames->mapBounded(~limit=concurrency, async functionName =>
+    switch await (
+      async () => {
+        let concurrencyOut = await Lambda.GetFunctionConcurrencyCommand.send(
+          client,
+          Lambda.GetFunctionConcurrencyCommand.make({functionName: functionName}),
+        )
+        let configOut = await Lambda.GetFunctionConfigurationCommand.send(
+          client,
+          Lambda.GetFunctionConfigurationCommand.make({functionName: functionName}),
+        )
+        let held = {
+          functionName,
+          priorReserved: concurrencyOut.reservedConcurrentExecutions,
+          priorVariables: configOut.environment
+          ->Option.flatMap(e => e.variables)
+          ->Option.getOr(Dict.make()),
+        }
+        let _ = await Lambda.PutFunctionConcurrencyCommand.send(
+          client,
+          Lambda.PutFunctionConcurrencyCommand.make({
+            functionName,
+            reservedConcurrentExecutions: 0,
+          }),
+        )
+        held
+      }
+    )() {
+    | held => Ok(held)
+    | exception exn => Error((functionName, errorText(exn)))
+    }
+  )
+
+  let taken = attempts->Array.filterMap(a =>
+    switch a {
+    | Ok(h) => Some(h)
+    | Error(_) => None
+    }
+  )
+  switch attempts->Array.findMap(a =>
+    switch a {
+    | Error(failure) => Some(failure)
+    | Ok(_) => None
+    }
+  ) {
+  | None => taken
+  | Some((functionName, reason)) =>
+    let rollback = await taken->mapBounded(~limit=concurrency, h => restoreConcurrency(~client, ~h))
+    let stranded = rollback->Array.filterMap(w => w)
+    throw(
+      ReventlessSeed.Seed.Failed(
+        `could not hold Lambda function ${functionName} at zero concurrency (${reason}) — nothing was ` ++
+        `deleted. The reset holds every runtime in scope for the length of the wipe, which needs ` ++
+        `lambda:GetFunctionConcurrency, lambda:GetFunctionConfiguration, lambda:PutFunctionConcurrency, ` ++
+        `lambda:DeleteFunctionConcurrency and lambda:UpdateFunctionConfiguration. Set ` ++
+        `SEED_RESET_NO_QUIESCE=1 to wipe without the hold — but a running runtime can then write back ` ++
+        `what the wipe removes.` ++
+        switch stranded {
+        | [] => ""
+        | messages => `\n\n  ` ++ messages->Array.join("\n  ")
+        },
+      ),
+    )
+  }
+}

package/lib/ocaml/ReventlessSeedAws_Reconcile.res

@@ -333,12 +333,8 @@ let run = (~stack=?, ~backend=?, ~targets: array<ReventlessSeedAws_Reset.target>
         await targets->Array.reduce(Promise.resolve(), (acc, target) =>
           acc->Promise.then(async _ => {
             let platform = ReventlessSeedAws_Reset.projectName(~projectDir=target.projectDir)
-            let (found, _buckets) = await ReventlessSeedAws_Reset.discover(
-              ~region,
-              ~stack,
-              ~platform,
-            )
-            found->Array.forEach(t => tables->Array.push(t))
+            let found = await ReventlessSeedAws_Reset.discover(~region, ~stack, ~platform)
+            found.tables->Array.forEach(t => tables->Array.push(t))
           })
         )