@reventlessdev/reventless-seed 1.0.0-alpha.17 → 1.0.0-alpha.19

package/CHANGELOG.md

@@ -3,6 +3,21 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+# 1.0.0-alpha.19 (2026-09-04)
+
+**Note:** Version bump only for package @reventlessdev/reventless-seed
+
+
+
+
+
+# 1.0.0-alpha.18 (2026-08-27)
+
+### Bug Fixes
+
+* **seed:** one endpoint-side fault throws away every command that landed ([6126dca](https://github.com/ReventlessDev/reventless-core/commit/6126dca7c62d193cf3a4efe379160af8207faa86))
+
+
 # 1.0.0-alpha.17 (2026-08-20)
 
 ### Features

package/lib/bs/.compiler.log

@@ -1,2 +1,2 @@
-#Start(1787252810629)
-#Done(1787252810671)
+#Start(1788547299414)
+#Done(1788547299452)

package/lib/bs/ReventlessSeed.mlmap

@@ -1,6 +1,7 @@
 randjbuildsystem
 Seed
 Seed_Client
+Seed_ClientTest
 Seed_Connect
 Seed_Prompt
 Seed_PromptTest

package/lib/bs/compiler-info.json

@@ -4,5 +4,5 @@
   "bsc_hash": "75e3a59c95cc953608fd3dd6ea6ed68141bba4414a20a52f114261b8a48385fa",
   "rescript_config_hash": "25cfff836d200084949fdfdfec2f6875901bfee1f04d84634eca252ef1ca1ef1",
   "runtime_path": "/home/runner/work/reventless-core/reventless-core/node_modules/@rescript/runtime",
-  "generated_at": "1787252810674"
+  "generated_at": "1788547299455"
 }

package/lib/bs/src/Seed_Client.res

@@ -106,10 +106,57 @@ let login = async (t: t): unit => {
   t.token = json->field("token")->Option.flatMap(asString)
 }
 
+/** Error types the endpoint returns for a fault on its own side rather than a
+    problem with the document. They say nothing about the request, so the same
+    request sent again can succeed — and across the few hundred commands a seed
+    issues, one of these is ordinary rather than exceptional.
+
+    Everything else — a validation error, an unknown field, a refused token — is
+    an answer, and the same answer comes back however many times it is asked. */
+let transientErrorTypes = [
+  "InternalFailure",
+  "ServiceUnavailable",
+  "ServiceUnavailableError",
+  "Throttling",
+  "ThrottlingException",
+  "TooManyRequestsException",
+  "RequestTimeout",
+]
+
+/** Every entry must be transient for the document to be worth resending: one
+    real error alongside a transient one still means the request was wrong. An
+    empty `errors` array is not a reason to retry either. */
+let isTransient = (errors: JSON.t): bool =>
+  switch errors->JSON.Decode.array {
+  | None | Some([]) => false
+  | Some(entries) =>
+    entries->Array.every(e =>
+      switch e->nodeString("errorType") {
+      | Some(errorType) => transientErrorTypes->Array.includes(errorType)
+      | None => false
+      }
+    )
+  }
+
+let attempts = 4
+
 /**
  * Sends one GraphQL document and returns its `data`. A GraphQL-level error
  * aborts the run: a half-seeded store is worse than an empty one, because it
  * looks like a working dataset.
+ *
+ * Which is exactly why a fault on the endpoint's own side must not abort it. A
+ * seed issues hundreds of commands, many of them concurrently and all of them
+ * against runtimes that a preceding reset left cold, so a single
+ * `InternalFailure` somewhere in the batch is a normal event. Treating it as
+ * fatal threw away every command that had already landed and demanded a full
+ * wipe and re-run — for a request that would have succeeded on being asked
+ * again. So resend those, backing off between tries, and keep failing fast on
+ * everything that is an answer rather than a fault.
+ *
+ * The backoff is jittered because the commands go out in concurrent batches: a
+ * fixed delay would have the whole batch retry in lockstep and rebuild the same
+ * burst that provoked the fault.
  */
 let gql = async (t: t, ~query: string, ~label: string): JSON.t => {
   let headers = Dict.fromArray([("content-type", "application/json")])
@@ -120,18 +167,27 @@ let gql = async (t: t, ~query: string, ~label: string): JSON.t => {
   let body = JSON.stringify(
     JSON.Encode.object(Dict.fromArray([("query", JSON.Encode.string(query))])),
   )
-  let res = try await fetch(t.config.endpoint, {method: "POST", headers, body}) catch {
-  | _ =>
-    throw(Failed(`${label}: cannot reach ${t.config.endpoint} — is the platform running?`))
-  }
-  let json = await res->responseJson
-  switch json->field("errors") {
-  | Some(errors) =>
-    throw(
-      Failed(`${label} failed\n  query: ${query}\n  response: ${JSON.stringify(errors)}`),
-    )
-  | None => json->field("data")->Option.getOr(JSON.Encode.null)
+  let rec attempt = async (n: int): JSON.t => {
+    let res = try await fetch(t.config.endpoint, {method: "POST", headers, body}) catch {
+    | _ =>
+      throw(Failed(`${label}: cannot reach ${t.config.endpoint} — is the platform running?`))
+    }
+    let json = await res->responseJson
+    switch json->field("errors") {
+    | Some(errors) =>
+      if errors->isTransient && n < attempts {
+        await sleep(250 * Int.shiftLeft(1, n - 1) + Int.fromFloat(Math.random() *. 250.))
+        await attempt(n + 1)
+      } else {
+        let tried = n > 1 ? `\n  gave up after ${n->Int.toString} attempts` : ""
+        throw(
+          Failed(`${label} failed\n  query: ${query}\n  response: ${JSON.stringify(errors)}${tried}`),
+        )
+      }
+    | None => json->field("data")->Option.getOr(JSON.Encode.null)
+    }
   }
+  await attempt(1)
 }
 
 // ── Commands ────────────────────────────────────────────────────────────────

package/lib/bs/src/Seed_Client.res.mjs

@@ -1,5 +1,6 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
+import * as Stdlib_JSON from "@rescript/runtime/lib/es6/Stdlib_JSON.js";
 import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Seed_Types$ReventlessSeed from "./Seed_Types.res.mjs";
@@ -98,6 +99,32 @@ async function login(t) {
   t.token = Stdlib_Option.flatMap(field(json, "token"), asString);
 }
 
+let transientErrorTypes = [
+  "InternalFailure",
+  "ServiceUnavailable",
+  "ServiceUnavailableError",
+  "Throttling",
+  "ThrottlingException",
+  "TooManyRequestsException",
+  "RequestTimeout"
+];
+
+function isTransient(errors) {
+  let entries = Stdlib_JSON.Decode.array(errors);
+  if (entries !== undefined && entries.length !== 0) {
+    return entries.every(e => {
+      let errorType = nodeString(e, "errorType");
+      if (errorType !== undefined) {
+        return transientErrorTypes.includes(errorType);
+      } else {
+        return false;
+      }
+    });
+  } else {
+    return false;
+  }
+}
+
 async function gql(t, query, label) {
   let headers = Object.fromEntries([[
       "content-type",
@@ -111,30 +138,38 @@ async function gql(t, query, label) {
       "query",
       query
     ]]));
-  let res;
-  try {
-    res = await fetch(t.config.endpoint, {
-      method: "POST",
-      headers: headers,
-      body: body
-    });
-  } catch (exn) {
-    throw {
-      RE_EXN_ID: Seed_Types$ReventlessSeed.Failed,
-      _1: label + `: cannot reach ` + t.config.endpoint + ` — is the platform running?`,
-      Error: new Error()
-    };
-  }
-  let json = await res.json();
-  let errors = field(json, "errors");
-  if (errors !== undefined) {
+  let attempt = async n => {
+    let res;
+    try {
+      res = await fetch(t.config.endpoint, {
+        method: "POST",
+        headers: headers,
+        body: body
+      });
+    } catch (exn) {
+      throw {
+        RE_EXN_ID: Seed_Types$ReventlessSeed.Failed,
+        _1: label + `: cannot reach ` + t.config.endpoint + ` — is the platform running?`,
+        Error: new Error()
+      };
+    }
+    let json = await res.json();
+    let errors = field(json, "errors");
+    if (errors === undefined) {
+      return Stdlib_Option.getOr(field(json, "data"), null);
+    }
+    if (isTransient(errors) && n < 4) {
+      await sleep((250 * (1 << (n - 1 | 0)) | 0) + (Math.random() * 250 | 0) | 0);
+      return await attempt(n + 1 | 0);
+    }
+    let tried = n > 1 ? `\n  gave up after ` + n.toString() + ` attempts` : "";
     throw {
       RE_EXN_ID: Seed_Types$ReventlessSeed.Failed,
-      _1: label + ` failed\n  query: ` + query + `\n  response: ` + JSON.stringify(errors),
+      _1: label + ` failed\n  query: ` + query + `\n  response: ` + JSON.stringify(errors) + tried,
       Error: new Error()
     };
-  }
-  return Stdlib_Option.getOr(field(json, "data"), null);
+  };
+  return await attempt(1);
 }
 
 let commandResultSelection = `__typename
@@ -269,6 +304,8 @@ async function waitForIds(t, fieldName, ids, timeoutMsOpt) {
   return Stdlib_Option.getOr(result, 0);
 }
 
+let attempts = 4;
+
 export {
   sleep,
   make,
@@ -279,6 +316,9 @@ export {
   asString,
   nodeString,
   login,
+  transientErrorTypes,
+  isTransient,
+  attempts,
   gql,
   commandResultSelection,
   send,

package/lib/bs/tests/Seed_ClientTest.res

@@ -0,0 +1,59 @@
+open JestGlobals
+
+// A seed issues hundreds of commands against runtimes a preceding reset left
+// cold, so the endpoint faulting on one of them is a normal event rather than an
+// exceptional one. Before the retry, a single `InternalFailure` in that batch
+// aborted the run and left a half-seeded store that could only be recovered by a
+// full wipe and a re-run — for a request that would have succeeded on being
+// asked again.
+//
+// What makes the retry safe is the classifier, so that is what these pin. A
+// fault on the endpoint's side carries no information about the document and is
+// worth resending; anything that is an *answer* — a validation error, an unknown
+// field, a refused token — comes back identical however many times it is asked,
+// and resending it only delays the failure the operator needs to see.
+
+let errors = entries => JSON.Encode.array(entries->Array.map(JSON.Encode.object))
+
+let errorOfType = errorType =>
+  Dict.fromArray([("errorType", JSON.Encode.string(errorType))])
+
+describe("Seed_Client.isTransient:", () => {
+  testSync("resends a fault on the endpoint's own side", () =>
+    ["InternalFailure", "ServiceUnavailable", "Throttling", "TooManyRequestsException"]
+    ->Array.forEach(t =>
+      expect(Seed_Client.isTransient(errors([errorOfType(t)])))->toBe(true)
+    )
+  )
+
+  // The exact payload the seed aborted on.
+  testSync("resends the shape AppSync actually returned", () => {
+    let payload = JSON.parseOrThrow(`[{"errorType":"InternalFailure","message":"An internal failure occurred."}]`)
+    expect(Seed_Client.isTransient(payload))->toBe(true)
+  })
+
+  testSync("fails fast on an error that is an answer", () =>
+    ["ValidationException", "Unauthorized", "FieldUndefined"]->Array.forEach(t =>
+      expect(Seed_Client.isTransient(errors([errorOfType(t)])))->toBe(false)
+    )
+  )
+
+  // A real error sitting alongside a transient one still means the request was
+  // wrong, and resending it cannot make it right.
+  testSync("refuses to resend a batch that mixes a real error in", () =>
+    expect(
+      Seed_Client.isTransient(
+        errors([errorOfType("InternalFailure"), errorOfType("ValidationException")]),
+      ),
+    )->toBe(false)
+  )
+
+  // `Array.every` is vacuously true on an empty array, which would turn "the
+  // endpoint said errors but named none" into an endless resend.
+  testSync("treats an empty or untyped error list as final", () => {
+    expect(Seed_Client.isTransient(errors([])))->toBe(false)
+    expect(Seed_Client.isTransient(errors([Dict.fromArray([("message", JSON.Encode.string("boom"))])])))
+    ->toBe(false)
+    expect(Seed_Client.isTransient(JSON.Encode.null))->toBe(false)
+  })
+})

package/lib/bs/tests/Seed_ClientTest.res.mjs

@@ -0,0 +1,64 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Seed_Client$ReventlessSeed from "../src/Seed_Client.res.mjs";
+
+function errors(entries) {
+  return entries.map(prim => prim);
+}
+
+function errorOfType(errorType) {
+  return Object.fromEntries([[
+      "errorType",
+      errorType
+    ]]);
+}
+
+globalThis.describe("Seed_Client.isTransient:", () => {
+  globalThis.test("resends a fault on the endpoint's own side", () => {
+    [
+      "InternalFailure",
+      "ServiceUnavailable",
+      "Throttling",
+      "TooManyRequestsException"
+    ].forEach(t => {
+      let entries = [errorOfType(t)];
+      globalThis.expect(Seed_Client$ReventlessSeed.isTransient(entries.map(prim => prim))).toBe(true);
+    });
+  });
+  globalThis.test("resends the shape AppSync actually returned", () => {
+    let payload = JSON.parse(`[{"errorType":"InternalFailure","message":"An internal failure occurred."}]`);
+    globalThis.expect(Seed_Client$ReventlessSeed.isTransient(payload)).toBe(true);
+  });
+  globalThis.test("fails fast on an error that is an answer", () => {
+    [
+      "ValidationException",
+      "Unauthorized",
+      "FieldUndefined"
+    ].forEach(t => {
+      let entries = [errorOfType(t)];
+      globalThis.expect(Seed_Client$ReventlessSeed.isTransient(entries.map(prim => prim))).toBe(false);
+    });
+  });
+  globalThis.test("refuses to resend a batch that mixes a real error in", () => {
+    let entries = [
+      errorOfType("InternalFailure"),
+      errorOfType("ValidationException")
+    ];
+    globalThis.expect(Seed_Client$ReventlessSeed.isTransient(entries.map(prim => prim))).toBe(false);
+  });
+  globalThis.test("treats an empty or untyped error list as final", () => {
+    globalThis.expect(Seed_Client$ReventlessSeed.isTransient([].map(prim => prim))).toBe(false);
+    let entries = [Object.fromEntries([[
+          "message",
+          "boom"
+        ]])];
+    globalThis.expect(Seed_Client$ReventlessSeed.isTransient(entries.map(prim => prim))).toBe(false);
+    globalThis.expect(Seed_Client$ReventlessSeed.isTransient(null)).toBe(false);
+  });
+});
+
+export {
+  errors,
+  errorOfType,
+}
+/*  Not a pure module */

package/lib/ocaml/.compiler.log

@@ -1,2 +1,2 @@
-#Start(1787252810629)
-#Done(1787252810671)
+#Start(1788547299414)
+#Done(1788547299452)

package/lib/ocaml/ReventlessSeed.mlmap

@@ -1,6 +1,7 @@
 randjbuildsystem
 Seed
 Seed_Client
+Seed_ClientTest
 Seed_Connect
 Seed_Prompt
 Seed_PromptTest

package/lib/ocaml/Seed_Client.res

@@ -106,10 +106,57 @@ let login = async (t: t): unit => {
   t.token = json->field("token")->Option.flatMap(asString)
 }
 
+/** Error types the endpoint returns for a fault on its own side rather than a
+    problem with the document. They say nothing about the request, so the same
+    request sent again can succeed — and across the few hundred commands a seed
+    issues, one of these is ordinary rather than exceptional.
+
+    Everything else — a validation error, an unknown field, a refused token — is
+    an answer, and the same answer comes back however many times it is asked. */
+let transientErrorTypes = [
+  "InternalFailure",
+  "ServiceUnavailable",
+  "ServiceUnavailableError",
+  "Throttling",
+  "ThrottlingException",
+  "TooManyRequestsException",
+  "RequestTimeout",
+]
+
+/** Every entry must be transient for the document to be worth resending: one
+    real error alongside a transient one still means the request was wrong. An
+    empty `errors` array is not a reason to retry either. */
+let isTransient = (errors: JSON.t): bool =>
+  switch errors->JSON.Decode.array {
+  | None | Some([]) => false
+  | Some(entries) =>
+    entries->Array.every(e =>
+      switch e->nodeString("errorType") {
+      | Some(errorType) => transientErrorTypes->Array.includes(errorType)
+      | None => false
+      }
+    )
+  }
+
+let attempts = 4
+
 /**
  * Sends one GraphQL document and returns its `data`. A GraphQL-level error
  * aborts the run: a half-seeded store is worse than an empty one, because it
  * looks like a working dataset.
+ *
+ * Which is exactly why a fault on the endpoint's own side must not abort it. A
+ * seed issues hundreds of commands, many of them concurrently and all of them
+ * against runtimes that a preceding reset left cold, so a single
+ * `InternalFailure` somewhere in the batch is a normal event. Treating it as
+ * fatal threw away every command that had already landed and demanded a full
+ * wipe and re-run — for a request that would have succeeded on being asked
+ * again. So resend those, backing off between tries, and keep failing fast on
+ * everything that is an answer rather than a fault.
+ *
+ * The backoff is jittered because the commands go out in concurrent batches: a
+ * fixed delay would have the whole batch retry in lockstep and rebuild the same
+ * burst that provoked the fault.
  */
 let gql = async (t: t, ~query: string, ~label: string): JSON.t => {
   let headers = Dict.fromArray([("content-type", "application/json")])
@@ -120,18 +167,27 @@ let gql = async (t: t, ~query: string, ~label: string): JSON.t => {
   let body = JSON.stringify(
     JSON.Encode.object(Dict.fromArray([("query", JSON.Encode.string(query))])),
   )
-  let res = try await fetch(t.config.endpoint, {method: "POST", headers, body}) catch {
-  | _ =>
-    throw(Failed(`${label}: cannot reach ${t.config.endpoint} — is the platform running?`))
-  }
-  let json = await res->responseJson
-  switch json->field("errors") {
-  | Some(errors) =>
-    throw(
-      Failed(`${label} failed\n  query: ${query}\n  response: ${JSON.stringify(errors)}`),
-    )
-  | None => json->field("data")->Option.getOr(JSON.Encode.null)
+  let rec attempt = async (n: int): JSON.t => {
+    let res = try await fetch(t.config.endpoint, {method: "POST", headers, body}) catch {
+    | _ =>
+      throw(Failed(`${label}: cannot reach ${t.config.endpoint} — is the platform running?`))
+    }
+    let json = await res->responseJson
+    switch json->field("errors") {
+    | Some(errors) =>
+      if errors->isTransient && n < attempts {
+        await sleep(250 * Int.shiftLeft(1, n - 1) + Int.fromFloat(Math.random() *. 250.))
+        await attempt(n + 1)
+      } else {
+        let tried = n > 1 ? `\n  gave up after ${n->Int.toString} attempts` : ""
+        throw(
+          Failed(`${label} failed\n  query: ${query}\n  response: ${JSON.stringify(errors)}${tried}`),
+        )
+      }
+    | None => json->field("data")->Option.getOr(JSON.Encode.null)
+    }
   }
+  await attempt(1)
 }
 
 // ── Commands ────────────────────────────────────────────────────────────────

package/lib/ocaml/Seed_ClientTest.res

@@ -0,0 +1,59 @@
+open JestGlobals
+
+// A seed issues hundreds of commands against runtimes a preceding reset left
+// cold, so the endpoint faulting on one of them is a normal event rather than an
+// exceptional one. Before the retry, a single `InternalFailure` in that batch
+// aborted the run and left a half-seeded store that could only be recovered by a
+// full wipe and a re-run — for a request that would have succeeded on being
+// asked again.
+//
+// What makes the retry safe is the classifier, so that is what these pin. A
+// fault on the endpoint's side carries no information about the document and is
+// worth resending; anything that is an *answer* — a validation error, an unknown
+// field, a refused token — comes back identical however many times it is asked,
+// and resending it only delays the failure the operator needs to see.
+
+let errors = entries => JSON.Encode.array(entries->Array.map(JSON.Encode.object))
+
+let errorOfType = errorType =>
+  Dict.fromArray([("errorType", JSON.Encode.string(errorType))])
+
+describe("Seed_Client.isTransient:", () => {
+  testSync("resends a fault on the endpoint's own side", () =>
+    ["InternalFailure", "ServiceUnavailable", "Throttling", "TooManyRequestsException"]
+    ->Array.forEach(t =>
+      expect(Seed_Client.isTransient(errors([errorOfType(t)])))->toBe(true)
+    )
+  )
+
+  // The exact payload the seed aborted on.
+  testSync("resends the shape AppSync actually returned", () => {
+    let payload = JSON.parseOrThrow(`[{"errorType":"InternalFailure","message":"An internal failure occurred."}]`)
+    expect(Seed_Client.isTransient(payload))->toBe(true)
+  })
+
+  testSync("fails fast on an error that is an answer", () =>
+    ["ValidationException", "Unauthorized", "FieldUndefined"]->Array.forEach(t =>
+      expect(Seed_Client.isTransient(errors([errorOfType(t)])))->toBe(false)
+    )
+  )
+
+  // A real error sitting alongside a transient one still means the request was
+  // wrong, and resending it cannot make it right.
+  testSync("refuses to resend a batch that mixes a real error in", () =>
+    expect(
+      Seed_Client.isTransient(
+        errors([errorOfType("InternalFailure"), errorOfType("ValidationException")]),
+      ),
+    )->toBe(false)
+  )
+
+  // `Array.every` is vacuously true on an empty array, which would turn "the
+  // endpoint said errors but named none" into an endless resend.
+  testSync("treats an empty or untyped error list as final", () => {
+    expect(Seed_Client.isTransient(errors([])))->toBe(false)
+    expect(Seed_Client.isTransient(errors([Dict.fromArray([("message", JSON.Encode.string("boom"))])])))
+    ->toBe(false)
+    expect(Seed_Client.isTransient(JSON.Encode.null))->toBe(false)
+  })
+})

package/package.json

@@ -1,11 +1,11 @@
 {
   "name": "@reventlessdev/reventless-seed",
-  "version": "1.0.0-alpha.17",
+  "version": "1.0.0-alpha.19",
   "description": "Reusable harness for seeding a Reventless platform through its public GraphQL command API",
   "license": "Apache-2.0",
   "dependencies": {
     "yaml": "^2.8.3",
-    "@reventlessdev/rescript-node": "2.0.0-alpha.8"
+    "@reventlessdev/rescript-node": "2.0.0-alpha.9"
   },
   "devDependencies": {
     "rescript": "12.3.0",

package/src/Seed_Client.res

@@ -106,10 +106,57 @@ let login = async (t: t): unit => {
   t.token = json->field("token")->Option.flatMap(asString)
 }
 
+/** Error types the endpoint returns for a fault on its own side rather than a
+    problem with the document. They say nothing about the request, so the same
+    request sent again can succeed — and across the few hundred commands a seed
+    issues, one of these is ordinary rather than exceptional.
+
+    Everything else — a validation error, an unknown field, a refused token — is
+    an answer, and the same answer comes back however many times it is asked. */
+let transientErrorTypes = [
+  "InternalFailure",
+  "ServiceUnavailable",
+  "ServiceUnavailableError",
+  "Throttling",
+  "ThrottlingException",
+  "TooManyRequestsException",
+  "RequestTimeout",
+]
+
+/** Every entry must be transient for the document to be worth resending: one
+    real error alongside a transient one still means the request was wrong. An
+    empty `errors` array is not a reason to retry either. */
+let isTransient = (errors: JSON.t): bool =>
+  switch errors->JSON.Decode.array {
+  | None | Some([]) => false
+  | Some(entries) =>
+    entries->Array.every(e =>
+      switch e->nodeString("errorType") {
+      | Some(errorType) => transientErrorTypes->Array.includes(errorType)
+      | None => false
+      }
+    )
+  }
+
+let attempts = 4
+
 /**
  * Sends one GraphQL document and returns its `data`. A GraphQL-level error
  * aborts the run: a half-seeded store is worse than an empty one, because it
  * looks like a working dataset.
+ *
+ * Which is exactly why a fault on the endpoint's own side must not abort it. A
+ * seed issues hundreds of commands, many of them concurrently and all of them
+ * against runtimes that a preceding reset left cold, so a single
+ * `InternalFailure` somewhere in the batch is a normal event. Treating it as
+ * fatal threw away every command that had already landed and demanded a full
+ * wipe and re-run — for a request that would have succeeded on being asked
+ * again. So resend those, backing off between tries, and keep failing fast on
+ * everything that is an answer rather than a fault.
+ *
+ * The backoff is jittered because the commands go out in concurrent batches: a
+ * fixed delay would have the whole batch retry in lockstep and rebuild the same
+ * burst that provoked the fault.
  */
 let gql = async (t: t, ~query: string, ~label: string): JSON.t => {
   let headers = Dict.fromArray([("content-type", "application/json")])
@@ -120,18 +167,27 @@ let gql = async (t: t, ~query: string, ~label: string): JSON.t => {
   let body = JSON.stringify(
     JSON.Encode.object(Dict.fromArray([("query", JSON.Encode.string(query))])),
   )
-  let res = try await fetch(t.config.endpoint, {method: "POST", headers, body}) catch {
-  | _ =>
-    throw(Failed(`${label}: cannot reach ${t.config.endpoint} — is the platform running?`))
-  }
-  let json = await res->responseJson
-  switch json->field("errors") {
-  | Some(errors) =>
-    throw(
-      Failed(`${label} failed\n  query: ${query}\n  response: ${JSON.stringify(errors)}`),
-    )
-  | None => json->field("data")->Option.getOr(JSON.Encode.null)
+  let rec attempt = async (n: int): JSON.t => {
+    let res = try await fetch(t.config.endpoint, {method: "POST", headers, body}) catch {
+    | _ =>
+      throw(Failed(`${label}: cannot reach ${t.config.endpoint} — is the platform running?`))
+    }
+    let json = await res->responseJson
+    switch json->field("errors") {
+    | Some(errors) =>
+      if errors->isTransient && n < attempts {
+        await sleep(250 * Int.shiftLeft(1, n - 1) + Int.fromFloat(Math.random() *. 250.))
+        await attempt(n + 1)
+      } else {
+        let tried = n > 1 ? `\n  gave up after ${n->Int.toString} attempts` : ""
+        throw(
+          Failed(`${label} failed\n  query: ${query}\n  response: ${JSON.stringify(errors)}${tried}`),
+        )
+      }
+    | None => json->field("data")->Option.getOr(JSON.Encode.null)
+    }
   }
+  await attempt(1)
 }
 
 // ── Commands ────────────────────────────────────────────────────────────────

package/src/Seed_Client.res.mjs

@@ -1,5 +1,6 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
+import * as Stdlib_JSON from "@rescript/runtime/lib/es6/Stdlib_JSON.js";
 import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Seed_Types$ReventlessSeed from "./Seed_Types.res.mjs";
@@ -98,6 +99,32 @@ async function login(t) {
   t.token = Stdlib_Option.flatMap(field(json, "token"), asString);
 }
 
+let transientErrorTypes = [
+  "InternalFailure",
+  "ServiceUnavailable",
+  "ServiceUnavailableError",
+  "Throttling",
+  "ThrottlingException",
+  "TooManyRequestsException",
+  "RequestTimeout"
+];
+
+function isTransient(errors) {
+  let entries = Stdlib_JSON.Decode.array(errors);
+  if (entries !== undefined && entries.length !== 0) {
+    return entries.every(e => {
+      let errorType = nodeString(e, "errorType");
+      if (errorType !== undefined) {
+        return transientErrorTypes.includes(errorType);
+      } else {
+        return false;
+      }
+    });
+  } else {
+    return false;
+  }
+}
+
 async function gql(t, query, label) {
   let headers = Object.fromEntries([[
       "content-type",
@@ -111,30 +138,38 @@ async function gql(t, query, label) {
       "query",
       query
     ]]));
-  let res;
-  try {
-    res = await fetch(t.config.endpoint, {
-      method: "POST",
-      headers: headers,
-      body: body
-    });
-  } catch (exn) {
-    throw {
-      RE_EXN_ID: Seed_Types$ReventlessSeed.Failed,
-      _1: label + `: cannot reach ` + t.config.endpoint + ` — is the platform running?`,
-      Error: new Error()
-    };
-  }
-  let json = await res.json();
-  let errors = field(json, "errors");
-  if (errors !== undefined) {
+  let attempt = async n => {
+    let res;
+    try {
+      res = await fetch(t.config.endpoint, {
+        method: "POST",
+        headers: headers,
+        body: body
+      });
+    } catch (exn) {
+      throw {
+        RE_EXN_ID: Seed_Types$ReventlessSeed.Failed,
+        _1: label + `: cannot reach ` + t.config.endpoint + ` — is the platform running?`,
+        Error: new Error()
+      };
+    }
+    let json = await res.json();
+    let errors = field(json, "errors");
+    if (errors === undefined) {
+      return Stdlib_Option.getOr(field(json, "data"), null);
+    }
+    if (isTransient(errors) && n < 4) {
+      await sleep((250 * (1 << (n - 1 | 0)) | 0) + (Math.random() * 250 | 0) | 0);
+      return await attempt(n + 1 | 0);
+    }
+    let tried = n > 1 ? `\n  gave up after ` + n.toString() + ` attempts` : "";
     throw {
       RE_EXN_ID: Seed_Types$ReventlessSeed.Failed,
-      _1: label + ` failed\n  query: ` + query + `\n  response: ` + JSON.stringify(errors),
+      _1: label + ` failed\n  query: ` + query + `\n  response: ` + JSON.stringify(errors) + tried,
       Error: new Error()
     };
-  }
-  return Stdlib_Option.getOr(field(json, "data"), null);
+  };
+  return await attempt(1);
 }
 
 let commandResultSelection = `__typename
@@ -269,6 +304,8 @@ async function waitForIds(t, fieldName, ids, timeoutMsOpt) {
   return Stdlib_Option.getOr(result, 0);
 }
 
+let attempts = 4;
+
 export {
   sleep,
   make,
@@ -279,6 +316,9 @@ export {
   asString,
   nodeString,
   login,
+  transientErrorTypes,
+  isTransient,
+  attempts,
   gql,
   commandResultSelection,
   send,

package/tests/Seed_ClientTest.res

@@ -0,0 +1,59 @@
+open JestGlobals
+
+// A seed issues hundreds of commands against runtimes a preceding reset left
+// cold, so the endpoint faulting on one of them is a normal event rather than an
+// exceptional one. Before the retry, a single `InternalFailure` in that batch
+// aborted the run and left a half-seeded store that could only be recovered by a
+// full wipe and a re-run — for a request that would have succeeded on being
+// asked again.
+//
+// What makes the retry safe is the classifier, so that is what these pin. A
+// fault on the endpoint's side carries no information about the document and is
+// worth resending; anything that is an *answer* — a validation error, an unknown
+// field, a refused token — comes back identical however many times it is asked,
+// and resending it only delays the failure the operator needs to see.
+
+let errors = entries => JSON.Encode.array(entries->Array.map(JSON.Encode.object))
+
+let errorOfType = errorType =>
+  Dict.fromArray([("errorType", JSON.Encode.string(errorType))])
+
+describe("Seed_Client.isTransient:", () => {
+  testSync("resends a fault on the endpoint's own side", () =>
+    ["InternalFailure", "ServiceUnavailable", "Throttling", "TooManyRequestsException"]
+    ->Array.forEach(t =>
+      expect(Seed_Client.isTransient(errors([errorOfType(t)])))->toBe(true)
+    )
+  )
+
+  // The exact payload the seed aborted on.
+  testSync("resends the shape AppSync actually returned", () => {
+    let payload = JSON.parseOrThrow(`[{"errorType":"InternalFailure","message":"An internal failure occurred."}]`)
+    expect(Seed_Client.isTransient(payload))->toBe(true)
+  })
+
+  testSync("fails fast on an error that is an answer", () =>
+    ["ValidationException", "Unauthorized", "FieldUndefined"]->Array.forEach(t =>
+      expect(Seed_Client.isTransient(errors([errorOfType(t)])))->toBe(false)
+    )
+  )
+
+  // A real error sitting alongside a transient one still means the request was
+  // wrong, and resending it cannot make it right.
+  testSync("refuses to resend a batch that mixes a real error in", () =>
+    expect(
+      Seed_Client.isTransient(
+        errors([errorOfType("InternalFailure"), errorOfType("ValidationException")]),
+      ),
+    )->toBe(false)
+  )
+
+  // `Array.every` is vacuously true on an empty array, which would turn "the
+  // endpoint said errors but named none" into an endless resend.
+  testSync("treats an empty or untyped error list as final", () => {
+    expect(Seed_Client.isTransient(errors([])))->toBe(false)
+    expect(Seed_Client.isTransient(errors([Dict.fromArray([("message", JSON.Encode.string("boom"))])])))
+    ->toBe(false)
+    expect(Seed_Client.isTransient(JSON.Encode.null))->toBe(false)
+  })
+})

package/tests/Seed_ClientTest.res.mjs

@@ -0,0 +1,64 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Seed_Client$ReventlessSeed from "../src/Seed_Client.res.mjs";
+
+function errors(entries) {
+  return entries.map(prim => prim);
+}
+
+function errorOfType(errorType) {
+  return Object.fromEntries([[
+      "errorType",
+      errorType
+    ]]);
+}
+
+globalThis.describe("Seed_Client.isTransient:", () => {
+  globalThis.test("resends a fault on the endpoint's own side", () => {
+    [
+      "InternalFailure",
+      "ServiceUnavailable",
+      "Throttling",
+      "TooManyRequestsException"
+    ].forEach(t => {
+      let entries = [errorOfType(t)];
+      globalThis.expect(Seed_Client$ReventlessSeed.isTransient(entries.map(prim => prim))).toBe(true);
+    });
+  });
+  globalThis.test("resends the shape AppSync actually returned", () => {
+    let payload = JSON.parse(`[{"errorType":"InternalFailure","message":"An internal failure occurred."}]`);
+    globalThis.expect(Seed_Client$ReventlessSeed.isTransient(payload)).toBe(true);
+  });
+  globalThis.test("fails fast on an error that is an answer", () => {
+    [
+      "ValidationException",
+      "Unauthorized",
+      "FieldUndefined"
+    ].forEach(t => {
+      let entries = [errorOfType(t)];
+      globalThis.expect(Seed_Client$ReventlessSeed.isTransient(entries.map(prim => prim))).toBe(false);
+    });
+  });
+  globalThis.test("refuses to resend a batch that mixes a real error in", () => {
+    let entries = [
+      errorOfType("InternalFailure"),
+      errorOfType("ValidationException")
+    ];
+    globalThis.expect(Seed_Client$ReventlessSeed.isTransient(entries.map(prim => prim))).toBe(false);
+  });
+  globalThis.test("treats an empty or untyped error list as final", () => {
+    globalThis.expect(Seed_Client$ReventlessSeed.isTransient([].map(prim => prim))).toBe(false);
+    let entries = [Object.fromEntries([[
+          "message",
+          "boom"
+        ]])];
+    globalThis.expect(Seed_Client$ReventlessSeed.isTransient(entries.map(prim => prim))).toBe(false);
+    globalThis.expect(Seed_Client$ReventlessSeed.isTransient(null)).toBe(false);
+  });
+});
+
+export {
+  errors,
+  errorOfType,
+}
+/*  Not a pure module */