@reventlessdev/reventless-seed-aws 1.0.0-alpha.10

package/lib/bs/src/ReventlessSeedAws_Reconcile.res.mjs

@@ -0,0 +1,318 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as S3$AwsSdk from "@reventlessdev/rescript-aws-sdk/src/S3.res.mjs";
+import * as Stdlib_JSON from "@rescript/runtime/lib/es6/Stdlib_JSON.js";
+import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
+import * as Stdlib_JsExn from "@rescript/runtime/lib/es6/Stdlib_JsExn.js";
+import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
+import * as ReventlessSeedAws from "./ReventlessSeedAws.res.mjs";
+import * as ClientS3 from "@aws-sdk/client-s3";
+import * as Seed$ReventlessSeed from "@reventlessdev/reventless-seed/src/Seed.res.mjs";
+import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
+import * as LibDynamodb from "@aws-sdk/lib-dynamodb";
+import * as ReventlessSeedAws_Reset from "./ReventlessSeedAws_Reset.res.mjs";
+import * as Seed_Prompt$ReventlessSeed from "@reventlessdev/reventless-seed/src/Seed_Prompt.res.mjs";
+import * as Seed_Runner$ReventlessSeed from "@reventlessdev/reventless-seed/src/Seed_Runner.res.mjs";
+import * as DynamoDb_DocumentClient$AwsSdk from "@reventlessdev/rescript-aws-sdk/src/DynamoDb_DocumentClient.res.mjs";
+
+let pendingTagKey = "reventless:pending";
+
+function collectRefs(prefix, json, into) {
+  if (Array.isArray(json)) {
+    json.forEach(item => collectRefs(prefix, item, into));
+    return;
+  }
+  if (json === null) {
+    return;
+  }
+  switch (typeof json) {
+    case "string" :
+      if (json.startsWith(`/` + prefix + `/`)) {
+        into.add(json);
+        return;
+      } else {
+        return;
+      }
+    case "object" :
+      Object.values(json).forEach(v => collectRefs(prefix, v, into));
+      return;
+    default:
+      return;
+  }
+}
+
+async function refsInTable(table, prefix, into) {
+  let loop = async (start, rows) => {
+    let out = await DynamoDb_DocumentClient$AwsSdk.ScanCommand.send(new LibDynamodb.ScanCommand({
+      TableName: table,
+      ExclusiveStartKey: start
+    }));
+    let items = Stdlib_Option.getOr(out.Items, []);
+    items.forEach(item => {
+      let row = Stdlib_JSON.Decode.object(item);
+      if (row === undefined) {
+        return;
+      }
+      let match = row["event"];
+      let match$1 = row["data"];
+      if (typeof match === "string" && match$1 !== undefined) {
+        return collectRefs(prefix, match$1, into);
+      }
+    });
+    let rows$1 = rows + items.length | 0;
+    let k = out.LastEvaluatedKey;
+    if (k !== undefined) {
+      return await loop(k, rows$1);
+    } else {
+      return rows$1;
+    }
+  };
+  return await loop(undefined, 0);
+}
+
+async function listObjects(bucket, prefix) {
+  let keys = [];
+  let loop = async (keyMarker, versionMarker) => {
+    let out = await S3$AwsSdk.ListObjectVersionsCommand.send(new ClientS3.ListObjectVersionsCommand({
+      Bucket: bucket,
+      Prefix: prefix + `/`,
+      KeyMarker: keyMarker,
+      VersionIdMarker: versionMarker
+    }));
+    Stdlib_Option.getOr(out.Versions, []).forEach(v => {
+      if (v.IsLatest) {
+        keys.push(v.Key);
+        return;
+      }
+    });
+    let match = out.IsTruncated;
+    let match$1 = out.NextKeyMarker;
+    if (match !== undefined && match && match$1 !== undefined) {
+      return await loop(out.NextKeyMarker, out.NextVersionIdMarker);
+    }
+  };
+  await loop(undefined, undefined);
+  let objects = [];
+  await Stdlib_Array.reduce(keys, Promise.resolve(), (acc, key) => acc.then(async () => {
+    let tagging = await S3$AwsSdk.GetObjectTaggingCommand.send(new ClientS3.GetObjectTaggingCommand({
+      Bucket: bucket,
+      Key: key
+    }));
+    let pending = Stdlib_Option.getOr(tagging.TagSet, []).some(t => t.Key === pendingTagKey);
+    objects.push({
+      key: key,
+      ref: `/` + key,
+      pending: pending
+    });
+  }));
+  return objects;
+}
+
+async function reconcileStore(store, bucket, prefix, tables) {
+  let refs = new Set();
+  let eventRows = {
+    contents: 0
+  };
+  await Stdlib_Array.reduce(tables, Promise.resolve(), (acc, table) => acc.then(async () => {
+    let rows = await refsInTable(table, prefix, refs);
+    eventRows.contents = eventRows.contents + rows | 0;
+  }));
+  let objects = await listObjects(bucket, prefix);
+  let present = new Set();
+  objects.forEach(o => {
+    present.add(o.ref);
+  });
+  return {
+    store: store,
+    bucket: bucket,
+    prefix: prefix,
+    eventRows: eventRows.contents,
+    referencedButPending: objects.filter(o => {
+      if (o.pending) {
+        return refs.has(o.ref);
+      } else {
+        return false;
+      }
+    }),
+    unreferencedPending: objects.filter(o => {
+      if (o.pending) {
+        return !refs.has(o.ref);
+      } else {
+        return false;
+      }
+    }),
+    unreferencedUntagged: objects.filter(o => {
+      if (o.pending) {
+        return false;
+      } else {
+        return !refs.has(o.ref);
+      }
+    }),
+    danglingRefs: Array.from(refs).filter(r => !present.has(r))
+  };
+}
+
+function sample(objects, limitOpt) {
+  let limit = limitOpt !== undefined ? limitOpt : 10;
+  return objects.slice(0, limit);
+}
+
+function print(r) {
+  console.log("");
+  console.log(`  ` + r.store + `   ` + r.bucket + `/` + r.prefix + `/`);
+  console.log(`    event rows scanned: ` + r.eventRows.toString());
+  let line = (label, n) => {
+    console.log(`      ` + n.toString().padStart(8, " ") + `  ` + label);
+  };
+  line("referenced, still tagged pending  ← must be 0", r.referencedButPending.length);
+  line("unreferenced, tagged pending      ← what expiry would delete", r.unreferencedPending.length);
+  line("unreferenced, untagged            ← outside the rule", r.unreferencedUntagged.length);
+  line("referenced but missing            ← dangling refs", r.danglingRefs.length);
+  let objectCount = (r.referencedButPending.length + r.unreferencedPending.length | 0) + r.unreferencedUntagged.length | 0;
+  if (objectCount > 0 && r.referencedButPending.length === 0 && r.unreferencedPending.length === 0) {
+    console.log("");
+    console.log(`    Note: none of the ` + objectCount.toString() + ` objects carry "` + pendingTagKey + `". That is ` + "what a fully-claimed store looks like — and also what it looks like if the mint side never wrote the tag (a store whose objects all predate it) or if this tool's tag key has drifted from the deploy's.");
+  }
+  if (r.referencedButPending.length !== 0) {
+    console.log("");
+    console.log("    Referenced objects that are still tagged pending:");
+    sample(r.referencedButPending, undefined).forEach(o => {
+      console.log(`      ` + o.ref);
+    });
+    if (r.referencedButPending.length > 10) {
+      console.log(`      … and ` + (r.referencedButPending.length - 10 | 0).toString() + ` more`);
+    }
+    console.log("");
+    console.log("    Do NOT enable this store's expiry rule. Either the claim component is behind or stopped (check its IteratorAge alarm), or a field holding these refs is missing its `@storageRef` annotation — in which case the claimer never sees it and never will.");
+  }
+  return r.referencedButPending.length === 0;
+}
+
+function run(stack, backend, targets, param) {
+  let go = async () => {
+    try {
+      if (targets.length === 0) {
+        throw {
+          RE_EXN_ID: Seed$ReventlessSeed.Failed,
+          _1: "no targets were declared — nothing to reconcile.",
+          Error: new Error()
+        };
+      }
+      let url = Seed_Prompt$ReventlessSeed.envValue("SEED_PULUMI_BACKEND");
+      let backend$1 = url !== undefined ? url : backend;
+      let t = targets.find(t => t.group === "Platform");
+      let platformTarget;
+      if (t !== undefined) {
+        platformTarget = t;
+      } else {
+        throw {
+          RE_EXN_ID: Seed$ReventlessSeed.Failed,
+          _1: "no `platform` target is declared — declared object stores live in the platform project's stack output and cannot be resolved without it.",
+          Error: new Error()
+        };
+      }
+      let stack$1 = await ReventlessSeedAws.resolveStack(platformTarget.projectDir, backend$1, stack);
+      let output = ReventlessSeedAws.stackOutputs(platformTarget.projectDir, backend$1, stack$1);
+      let stores = ReventlessSeedAws_Reset.parseObjectStores(ReventlessSeedAws_Reset.field(output, "objectStores"));
+      let stores$1;
+      if (stores.TAG === "Ok") {
+        stores$1 = stores._0;
+      } else {
+        throw {
+          RE_EXN_ID: Seed$ReventlessSeed.Failed,
+          _1: stores._0,
+          Error: new Error()
+        };
+      }
+      if (stores$1.length === 0) {
+        Seed_Runner$ReventlessSeed.heading(`Reconcile: stack "` + stack$1 + `" declares no object stores.`);
+      } else {
+        let r = Seed_Prompt$ReventlessSeed.envValue("AWS_REGION");
+        let region;
+        if (r !== undefined) {
+          if (r !== "") {
+            region = r;
+          } else {
+            throw {
+              RE_EXN_ID: Seed$ReventlessSeed.Failed,
+              _1: "could not resolve the AWS region — set AWS_REGION.",
+              Error: new Error()
+            };
+          }
+        } else {
+          throw {
+            RE_EXN_ID: Seed$ReventlessSeed.Failed,
+            _1: "could not resolve the AWS region — set AWS_REGION.",
+            Error: new Error()
+          };
+        }
+        Seed_Runner$ReventlessSeed.heading(`Reconcile: stack "` + stack$1 + `" in ` + region);
+        let tables = [];
+        await Stdlib_Array.reduce(targets, Promise.resolve(), (acc, target) => acc.then(async () => {
+          let platform = ReventlessSeedAws_Reset.projectName(target.projectDir);
+          let match = await ReventlessSeedAws_Reset.discover(region, stack$1, platform);
+          match[0].forEach(t => {
+            tables.push(t);
+          });
+        }));
+        let allClear = {
+          contents: true
+        };
+        await Stdlib_Array.reduce(stores$1, Promise.resolve(), (acc, s) => acc.then(async () => {
+          let report = await reconcileStore(s.qualified, s.bucketName, s.keyPrefix, tables);
+          if (!print(report)) {
+            allClear.contents = false;
+            return;
+          }
+        }));
+        console.log("");
+        if (allClear.contents) {
+          console.log("Every referenced object has been claimed. Enabling a store's expiry rule (`pendingUploadExpiryDays: \"<plugin>.<store>=<days>\"`) would delete only objects nothing references.");
+        } else {
+          throw {
+            RE_EXN_ID: Seed$ReventlessSeed.Failed,
+            _1: "at least one referenced object is still tagged pending — see above. No store's expiry rule should be enabled while this is true.",
+            Error: new Error()
+          };
+        }
+      }
+      process.exit(0);
+      return;
+    } catch (raw_message) {
+      let message = Primitive_exceptions.internalToException(raw_message);
+      if (message.RE_EXN_ID === Seed$ReventlessSeed.Failed) {
+        console.error("");
+        console.error(`Reconcile failed — ` + message._1);
+        process.exit(1);
+      } else {
+        console.error("");
+        console.error("Reconcile failed with an unexpected error:");
+        console.error(Stdlib_Option.getOr(Stdlib_Option.flatMap(Stdlib_JsExn.fromException(message), Stdlib_JsExn.message), "unknown"));
+        process.exit(1);
+      }
+      return;
+    }
+  };
+  go();
+}
+
+let Ddb;
+
+let Rgt;
+
+let S3;
+
+export {
+  Ddb,
+  Rgt,
+  S3,
+  pendingTagKey,
+  collectRefs,
+  refsInTable,
+  listObjects,
+  reconcileStore,
+  sample,
+  print,
+  run,
+}
+/* S3-AwsSdk Not a pure module */