@auth-craft/aws-cf-stack 1.9.0 → 1.10.0

package/lib/sync-table-gsis.sh

@@ -0,0 +1,257 @@
+#!/usr/bin/env bash
+# Stage 1a — bring an existing table's GSIs up to the schema before the normal deploy.
+#
+# CloudFormation creates or deletes at most ONE GSI per table update, so a table deployed
+# by an older package version (missing several GSIs) cannot jump to the full schema in one
+# `cdk deploy`. This adds the missing GSIs one per data-stack update, in schema order,
+# waiting for each to finish backfilling. A new table needs none of this (CreateTable takes
+# every GSI at once), and an up-to-date table costs one DescribeTable.
+#
+# The schema comes from cdk/lib/table-schema.json (generated from database-plugin-dynamodb).
+# Each rollout step passes the GSI subset to the CDK app as context (`-c dataGsis=...`) on
+# that command line only — never through the environment, so it cannot leak into the full
+# deploy that follows.
+#
+# Needs, besides the CDK deploy roles: dynamodb:DescribeTable on the table and
+# cloudformation:GetTemplate on the data stack, for the identity running this CLI.
+
+# Poll interval / ceiling while a GSI backfills (populated tables take minutes).
+GSI_POLL_SECONDS="${AUTH_CRAFT_GSI_POLL_SECONDS:-15}"
+GSI_WAIT_MINUTES="${AUTH_CRAFT_GSI_WAIT_MINUTES:-60}"
+
+# describe-table JSON on stdout; returns 3 when the table does not exist.
+_describe_table() {
+  local table="$1" out err
+  err="$(mktemp)"
+  if out="$(aws dynamodb describe-table --table-name "$table" --region "$AWS_REGION" --output json 2>"$err")"; then
+    rm -f "$err"
+    printf '%s' "$out"
+    return 0
+  fi
+  if grep -q ResourceNotFoundException "$err"; then
+    rm -f "$err"
+    return 3
+  fi
+  error "$(cat "$err")"
+  rm -f "$err"
+  die "Could not describe table $table (the deploy identity needs dynamodb:DescribeTable on it)."
+}
+
+# The DynamoDB table's Properties from the data stack's deployed template (JSON); returns 3
+# when the stack does not exist.
+_template_table() {
+  local stack="$1" out err
+  err="$(mktemp)"
+  if ! out="$(aws cloudformation get-template --stack-name "$stack" --region "$AWS_REGION" \
+    --query TemplateBody --output json 2>"$err")"; then
+    if grep -q 'does not exist' "$err"; then
+      rm -f "$err"
+      return 3
+    fi
+    error "$(cat "$err")"
+    rm -f "$err"
+    die "Could not read the template of stack $stack (the deploy identity needs cloudformation:GetTemplate)."
+  fi
+  rm -f "$err"
+  # TemplateBody comes back as an object for JSON templates, as a string otherwise.
+  jq -c '(if type == "string" then fromjson else . end)
+    | first(.Resources[] | select(.Type == "AWS::DynamoDB::Table")) | .Properties' <<<"$out"
+}
+
+# `aws dynamodb update-table` command that recreates GSI $2 exactly as template table JSON $1
+# declares it (keys, attribute types, projection) — so the table matches CloudFormation again.
+_recreate_gsi_command() {
+  local props="$1" name="$2" attrs gsi
+  attrs="$(jq -c --arg n "$name" '
+    (.GlobalSecondaryIndexes[] | select(.IndexName == $n) | [.KeySchema[].AttributeName]) as $keys
+    | [.AttributeDefinitions[] | select(.AttributeName as $a | $keys | index($a))]' <<<"$props")"
+  gsi="$(jq -c --arg n "$name" '[{Create: (.GlobalSecondaryIndexes[] | select(.IndexName == $n)
+    | {IndexName, KeySchema, Projection})}]' <<<"$props")"
+  printf 'aws dynamodb update-table --table-name %s --region %s \\\n       --attribute-definitions %s \\\n       --global-secondary-index-updates %s' \
+    "$table" "$AWS_REGION" "'$attrs'" "'$gsi'"
+}
+
+# One line per GSI: "<name>\t<definition>", the definition being keys (with attribute
+# types) and projection in a canonical JSON form, so a live index and a schema entry compare
+# equal exactly when CloudFormation would leave the index alone. Comparing names only misses
+# a changed projection/key — CloudFormation replaces such a GSI (delete + create = two GSI
+# operations), which fails a single update.
+_GSI_SIG_JQ='def sig: "\(.name)\t\({keys: .keys, projection: .projection} | tojson)";
+  def proj($type; $nonKey): {type: $type, nonKey: (($nonKey // []) | sort)};'
+
+# Definitions of the live GSIs, from describe-table JSON on stdin.
+_live_gsi_sigs() {
+  jq -r "$_GSI_SIG_JQ"'
+    (.Table.AttributeDefinitions // [] | map({key: .AttributeName, value: .AttributeType}) | from_entries) as $types
+    | .Table.GlobalSecondaryIndexes[]?
+    | {name: .IndexName,
+       keys: [.KeySchema[] | {name: .AttributeName, keyType: .KeyType, type: ($types[.AttributeName] // "?")}]
+             | sort_by(.keyType),
+       projection: proj(.Projection.ProjectionType; .Projection.NonKeyAttributes)}
+    | sig'
+}
+
+# Definitions of the schema GSIs (table-schema.json given as $1), in schema order.
+_schema_gsi_sigs() {
+  jq -r "$_GSI_SIG_JQ"'
+    .globalSecondaryIndexes[]
+    | {name: .indexName,
+       keys: ([{name: .partitionKey.name, keyType: "HASH", type: .partitionKey.type}]
+              + (if .sortKey then [{name: .sortKey.name, keyType: "RANGE", type: .sortKey.type}] else [] end)),
+       projection: proj(.projection.type; .projection.nonKeyAttributes)}
+    | sig' "$1"
+}
+
+# Wait until the table and all its GSIs are ACTIVE; print the describe-table JSON.
+_wait_table_active() {
+  local table="$1" deadline desc pending
+  deadline=$(( $(date +%s) + GSI_WAIT_MINUTES * 60 ))
+  local rc
+  while :; do
+    rc=0
+    desc="$(_describe_table "$table")" || rc=$?
+    [[ $rc -eq 3 ]] && die "Table $table disappeared while waiting for it (deleted mid-deploy?). Re-run the deploy."
+    [[ $rc -eq 0 ]] || exit 1
+    pending="$(jq -r '[.Table.TableStatus] + [.Table.GlobalSecondaryIndexes[]?.IndexStatus]
+      | map(select(. != "ACTIVE")) | length' <<<"$desc")"
+    if [[ "$pending" == "0" ]]; then
+      printf '%s' "$desc"
+      return 0
+    fi
+    (( $(date +%s) < deadline )) \
+      || die "Table $table still not ACTIVE after ${GSI_WAIT_MINUTES} min (AUTH_CRAFT_GSI_WAIT_MINUTES). Re-run the deploy once it is."
+    info "  waiting for $table: $(jq -r '[.Table.GlobalSecondaryIndexes[]? | select(.IndexStatus != "ACTIVE")
+      | "\(.IndexName)=\(.IndexStatus)"] | join(", ") | if . == "" then "table updating" else . end' <<<"$desc")" >&2
+    sleep "$GSI_POLL_SECONDS"
+  done
+}
+
+# Read newline-separated text into the array named $1 (empty text → empty array).
+_to_array() {
+  local -n _arr="$1"
+  _arr=()
+  [[ -n "$2" ]] && mapfile -t _arr <<<"$2"
+  return 0
+}
+
+_contains() {
+  local needle="$1"
+  shift
+  local item
+  for item in "$@"; do [[ "$item" == "$needle" ]] && return 0; done
+  return 1
+}
+
+sync_table_gsis() {
+  local table="${RESOURCE_NAME}-data"
+  local schema="$CDK_DIR/lib/table-schema.json"
+  require_cmd jq
+  [[ -f "$schema" ]] || die "Table schema missing: $schema (was the package built?)"
+
+  local -a desired live managed include
+  local -A want_sig live_sig
+  local name sig
+  while IFS=$'\t' read -r name sig; do
+    desired+=("$name")
+    want_sig[$name]="$sig"
+  done < <(_schema_gsi_sigs "$schema")
+
+  local rc=0
+  _describe_table "$table" >/dev/null || rc=$?
+  if [[ $rc -eq 3 ]]; then
+    info "Table $table does not exist yet — it will be created with all ${#desired[@]} GSIs."
+    return 0
+  fi
+
+  local added=0 missing="" previous="" out template
+  while :; do
+    out="$(_wait_table_active "$table")" || exit 1
+    live=()
+    live_sig=()
+    while IFS=$'\t' read -r name sig; do
+      [[ -n "$name" ]] || continue
+      live+=("$name")
+      live_sig[$name]="$sig"
+    done < <(_live_gsi_sigs <<<"$out")
+
+    rc=0
+    template="$(_template_table "$table")" || rc=$?
+    [[ $rc -eq 3 ]] \
+      && die "Table $table exists but stack $table does not — CloudFormation cannot create a table that already exists. Import the table into the stack or delete it."
+    [[ $rc -eq 0 ]] || exit 1
+    _to_array managed "$(jq -r '.GlobalSecondaryIndexes[]?.IndexName' <<<"$template")"
+
+    # A GSI on the table that the deployed template does not declare was created by hand.
+    # CloudFormation would try to create it again and fail halfway — stop before that.
+    for name in "${live[@]}"; do
+      _contains "$name" "${managed[@]}" && continue
+      die "GSI $name exists on $table but not in stack $table (created outside CloudFormation).
+   Delete it first (aws dynamodb update-table --table-name $table --region $AWS_REGION \\
+     --global-secondary-index-updates '[{\"Delete\":{\"IndexName\":\"$name\"}}]'),
+   wait until the table is ACTIVE, then re-run the deploy — it will be recreated under CloudFormation."
+    done
+
+    # The reverse: the template declares a GSI the table lacks (deleted by hand). CloudFormation
+    # thinks it exists, so no update would ever recreate it. Recreating it exactly as the
+    # template declares it puts the table back in line with the stack (needs only the AWS CLI,
+    # one GSI per call — re-running reports the next one).
+    for name in "${managed[@]}"; do
+      _contains "$name" "${live[@]}" && continue
+      die "GSI $name is declared by stack $table but missing from the table (deleted outside CloudFormation).
+   CloudFormation will not recreate it. Recreate it as the stack declares it:
+     $(_recreate_gsi_command "$template" "$name")
+   wait until the table is ACTIVE (backfill can take minutes), then re-run the deploy."
+    done
+
+    # A deployed GSI the schema no longer has would be deleted in the same update as the
+    # next addition (two GSI changes) — CloudFormation rejects that.
+    for name in "${live[@]}"; do
+      _contains "$name" "${desired[@]}" && continue
+      die "GSI $name is on $table but not in this package's schema. Removing it while adding missing GSIs needs two CloudFormation updates; deploy with a package version that still declares it, or remove it in its own update first."
+    done
+
+    # Same name, different keys or projection: CloudFormation would replace the index
+    # (delete + create) in one update and fail. GSIs are append-only — a changed access
+    # pattern gets a new index name in TABLE_SCHEMA, the old one is retired separately.
+    for name in "${live[@]}"; do
+      [[ "${live_sig[$name]}" == "${want_sig[$name]}" ]] && continue
+      die "GSI $name on $table differs from this package's schema:
+     table : ${live_sig[$name]}
+     schema: ${want_sig[$name]}
+   CloudFormation cannot change a GSI's keys or projection in place (it replaces it — two GSI
+   operations in one update). Give the new definition a new index name in TABLE_SCHEMA instead."
+    done
+
+    missing=""
+    for name in "${desired[@]}"; do
+      _contains "$name" "${live[@]}" || { missing="$name"; break; }
+    done
+    [[ -z "$missing" ]] && break
+
+    # Progress guard: the previous update reported success but the GSI is still not on the
+    # table — looping would redeploy the same no-op forever.
+    [[ "$missing" != "$previous" ]] \
+      || die "Deploying stack $table did not add GSI $missing to the table. Check the stack events and the table, then re-run."
+    previous="$missing"
+
+    include=()
+    for name in "${desired[@]}"; do
+      if _contains "$name" "${live[@]}" || [[ "$name" == "$missing" ]]; then include+=("$name"); fi
+    done
+
+    added=$((added + 1))
+    info "Adding GSI $missing to $table (one GSI per CloudFormation update; backfill can take minutes)…"
+    (
+      cd "$CDK_DIR"
+      CDK_DEFAULT_ACCOUNT="$AWS_ACCOUNT_ID" \
+      CDK_DEFAULT_REGION="$AWS_REGION" \
+        cdk deploy "$table" --exclusively --require-approval never -c "dataGsis=$(IFS=,; echo "${include[*]}")"
+    ) || die "Adding GSI $missing to $table failed — fix the cause above and re-run; GSIs already added are kept."
+  done
+
+  if [[ $added -gt 0 ]]; then
+    success "Table $table now has all ${#desired[@]} GSIs (added $added)."
+  else
+    info "Table $table already has all ${#desired[@]} GSIs, matching the schema."
+  fi
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@auth-craft/aws-cf-stack",
-  "version": "1.9.0",
+  "version": "1.10.0",
   "description": "Self-contained, versioned distribution of the Auth Craft AWS (DynamoDB + Lambda) + Cloudflare gateway stack. Bundles prebuilt Lambda/worker artifacts + CDK app so consumers deploy without cloning auth-craft.",
   "type": "module",
   "bin": {
@@ -17,20 +17,22 @@
     ".env.example"
   ],
   "dependencies": {
-    "aws-cdk-lib": "^2.265.0",
+    "aws-cdk-lib": "^2.273.0",
     "constructs": "^10.8.1"
   },
   "devDependencies": {
-    "@types/node": "^26.2.0",
-    "tsx": "^4.23.12",
+    "@types/node": "^26.6.5",
+    "tsx": "^4.23.15",
     "typescript": "^7.0.2",
+    "vitest": "^5.0.3",
     "@auth-craft/auth-admin-cli": "2.0.0",
+    "@auth-craft/cloudflare-auth-gateway": "1.0.0",
     "@auth-craft/auth-hono-dynamodb-lambda": "1.0.0",
-    "@auth-craft/cloudflare-auth-gateway": "1.0.0"
+    "@auth-craft/database-plugin-dynamodb": "1.0.1"
   },
   "peerDependencies": {
-    "aws-cdk": "^2.1128.0",
-    "wrangler": "^4.101.0"
+    "aws-cdk": "^2.1145.0",
+    "wrangler": "^4.149.0"
   },
   "peerDependenciesMeta": {
     "aws-cdk": {
@@ -44,8 +46,10 @@
     "node": ">=20"
   },
   "scripts": {
+    "sync:table-schema": "node ./scripts/sync-table-schema.mjs",
     "prebuild:assets": "node ./scripts/prebuild-assets.mjs",
     "build": "pnpm prebuild:assets",
-    "type-check": "tsc --noEmit -p cdk/tsconfig.json"
+    "type-check": "tsc --noEmit -p cdk/tsconfig.json",
+    "test": "vitest run"
   }
 }