alchemy 2.0.0-beta.30 → 2.0.0-beta.32

package/src/AWS/Kinesis/StreamConsumer.ts

@@ -1,6 +1,8 @@
 import * as kinesis from "@distilled.cloud/aws/kinesis";
+import * as Data from "effect/Data";
 import * as Effect from "effect/Effect";
 import * as Schedule from "effect/Schedule";
+import { Unowned } from "../../AdoptPolicy.ts";
 import { isResolved } from "../../Diff.ts";
 import type { Input } from "../../Input.ts";
 import { createPhysicalName } from "../../PhysicalName.ts";
@@ -149,31 +151,45 @@ const readConsumer = Effect.fn(function* ({
   });
 });
 
-const resolveOwnedConsumer = Effect.fn(function* (
-  id: string,
+// Kinesis tells us "in use" via ResourceInUseException, but the consumer
+// registry is eventually consistent — describeStreamConsumer can briefly
+// return ResourceNotFoundException for a consumer the registry just
+// confirmed exists. Poll up to ~10s before giving up.
+class ConsumerRegistryNotConsistent extends Data.TaggedError(
+  "ConsumerRegistryNotConsistent",
+)<{ consumerName: string }> {}
+
+const adoptExistingConsumer = Effect.fn(function* (
   streamArn: string,
   consumerName: string,
 ) {
-  const state = yield* readConsumer({
-    streamArn,
-    consumerName,
-  });
+  return yield* Effect.gen(function* () {
+    const state = yield* readConsumer({
+      streamArn,
+      consumerName,
+    });
 
-  if (!state) {
-    return yield* Effect.fail(
-      new Error(`consumer ${consumerName} exists but could not be read`),
-    );
-  }
+    if (!state) {
+      return yield* new ConsumerRegistryNotConsistent({ consumerName });
+    }
 
-  if (!(yield* hasAlchemyTags(id, state.tags as Tags))) {
-    return yield* Effect.fail(
-      new Error(
-        `consumer ${consumerName} already exists but is not owned by this stack`,
+    return state;
+  }).pipe(
+    Effect.retry({
+      while: (e) => e._tag === "ConsumerRegistryNotConsistent",
+      schedule: Schedule.exponential(250).pipe(
+        Schedule.both(Schedule.recurs(8)),
       ),
-    );
-  }
-
-  return state;
+    }),
+    Effect.catchTag("ConsumerRegistryNotConsistent", () =>
+      Effect.fail(
+        new Error(
+          `consumer ${consumerName} exists but could not be read after ` +
+            `~10s of registry-consistency retries`,
+        ),
+      ),
+    ),
+  );
 });
 
 const waitForConsumerStatus = (
@@ -222,11 +238,23 @@ export const StreamConsumerProvider = () =>
     read: Effect.fn(function* ({ id, olds, output }) {
       const consumerName =
         output?.consumerName ?? (yield* createConsumerName(id, olds ?? {}));
-      return yield* readConsumer({
-        streamArn: olds.streamArn as string | undefined,
+      const streamArn = output?.streamArn ?? olds?.streamArn;
+      // describeStreamConsumer rejects with InvalidArgumentException unless
+      // either consumerARN, or both streamARN + consumerName, are provided.
+      // If the engine probed for adoption before our upstream Stream was
+      // created we'll have neither — treat that as "doesn't exist".
+      if (typeof streamArn !== "string" && !output?.consumerArn) {
+        return undefined;
+      }
+      const state = yield* readConsumer({
+        streamArn: typeof streamArn === "string" ? streamArn : undefined,
         consumerName,
         consumerArn: output?.consumerArn,
       });
+      if (!state) return undefined;
+      return (yield* hasAlchemyTags(id, state.tags as Tags))
+        ? state
+        : Unowned(state);
     }),
     diff: Effect.fn(function* ({ id, news, olds }) {
       if (!isResolved(news)) return;
@@ -243,53 +271,67 @@ export const StreamConsumerProvider = () =>
         return { action: "replace" } as const;
       }
     }),
-    create: Effect.fn(function* ({ id, news, session }) {
-      const consumerName = yield* createConsumerName(id, news);
+    reconcile: Effect.fn(function* ({ id, news, output, session }) {
+      const consumerName =
+        output?.consumerName ?? (yield* createConsumerName(id, news));
       const streamArn = news.streamArn as string;
       const internalTags = yield* createInternalTags(id);
-      const allTags = { ...internalTags, ...news.tags };
+      const desiredTags = { ...internalTags, ...news.tags };
 
-      const consumerArn = yield* kinesis
-        .registerStreamConsumer({
-          StreamARN: streamArn,
-          ConsumerName: consumerName,
-          Tags: allTags,
-        })
-        .pipe(
-          Effect.map((response) => response.Consumer.ConsumerARN),
-          Effect.catchTag("ResourceInUseException", () =>
-            resolveOwnedConsumer(id, streamArn, consumerName).pipe(
-              Effect.map((state) => state.consumerArn),
-            ),
-          ),
-        );
-
-      yield* waitForConsumerStatus(consumerArn, "ACTIVE");
-
-      const state = yield* readConsumer({
-        consumerArn,
+      // Observe — fetch live cloud state. `output` is treated as a cache
+      // for the consumer ARN; the consumer's actual existence and tags are
+      // fetched fresh so the reconciler converges regardless of drift,
+      // adoption, or a partially-completed prior run.
+      let state = yield* readConsumer({
         streamArn,
         consumerName,
+        consumerArn: output?.consumerArn,
       });
 
-      if (!state) {
-        return yield* Effect.fail(
-          new Error(`failed to read created consumer ${consumerName}`),
-        );
+      // Ensure — register the consumer if it's missing. Tolerate
+      // `ResourceInUseException` as a race with a peer reconciler: a
+      // brief registry-consistency wait, then re-read and continue the
+      // sync path.
+      if (state === undefined) {
+        yield* kinesis
+          .registerStreamConsumer({
+            StreamARN: streamArn,
+            ConsumerName: consumerName,
+            Tags: desiredTags,
+          })
+          .pipe(
+            Effect.asVoid,
+            Effect.catchTag("ResourceInUseException", () =>
+              adoptExistingConsumer(streamArn, consumerName).pipe(
+                Effect.asVoid,
+              ),
+            ),
+          );
+
+        state = yield* adoptExistingConsumer(streamArn, consumerName);
+        yield* waitForConsumerStatus(state.consumerArn, "ACTIVE");
+
+        state = yield* readConsumer({
+          consumerArn: state.consumerArn,
+          streamArn,
+          consumerName,
+        });
+        if (state === undefined) {
+          return yield* Effect.fail(
+            new Error(`failed to read created consumer ${consumerName}`),
+          );
+        }
       }
 
-      yield* session.note(state.consumerArn);
-      return state;
-    }),
-    update: Effect.fn(function* ({ id, news, olds, output, session }) {
-      const internalTags = yield* createInternalTags(id);
-      const oldTags = { ...internalTags, ...olds.tags };
-      const newTags = { ...internalTags, ...news.tags };
-      const { removed, upsert } = diffTags(oldTags, newTags);
+      // Sync tags — diff observed cloud tags against desired. Adoption
+      // may bring us a consumer that already has its own tag set; diffing
+      // against `state.tags` (fetched fresh) lets the reconciler converge
+      // ownership without fighting whatever was there before.
+      const { removed, upsert } = diffTags(state.tags, desiredTags);
 
       if (removed.length > 0) {
         yield* kinesis.untagResource({
-          ResourceARN: output.consumerArn,
+          ResourceARN: state.consumerArn,
           TagKeys: removed,
         });
       }
@@ -300,22 +342,24 @@ export const StreamConsumerProvider = () =>
           tagsToAdd[Key] = Value;
         }
         yield* kinesis.tagResource({
-          ResourceARN: output.consumerArn,
+          ResourceARN: state.consumerArn,
           Tags: tagsToAdd,
         });
       }
 
-      const state = yield* readConsumer({
-        consumerArn: output.consumerArn,
+      // Re-read final state so the returned attributes reflect what's
+      // actually in the cloud after all sync steps.
+      const final = yield* readConsumer({
+        consumerArn: state.consumerArn,
       });
-      if (!state) {
+      if (!final) {
         return yield* Effect.fail(
-          new Error(`failed to read updated consumer ${output.consumerName}`),
+          new Error(`failed to read reconciled consumer ${consumerName}`),
         );
       }
 
-      yield* session.note(output.consumerArn);
-      return state;
+      yield* session.note(final.consumerArn);
+      return final;
     }),
     delete: Effect.fn(function* ({ output }) {
       yield* kinesis

package/src/AWS/Lambda/EventSourceMapping.ts

@@ -292,68 +292,117 @@ export const EventSourceMappingProvider = () =>
             return { action: "replace" } as const;
           }
         }),
-        create: Effect.fn(function* ({ id, news, session }) {
+        reconcile: Effect.fn(function* ({ id, news, output, session }) {
           const expectedInternalTags = yield* createEventSourceMappingTags(id);
-          const allTags = { ...expectedInternalTags, ...news.tags };
+          const desiredTags = { ...expectedInternalTags, ...news.tags };
 
           const functionName = news.functionName as string;
           const eventSourceArn = news.eventSourceArn as string;
 
-          const config = yield* lambda
-            .createEventSourceMapping(toCreateRequest(news, allTags))
-            .pipe(
-              Effect.catchTags({
-                ResourceConflictException: () =>
-                  lambda.listEventSourceMappings
-                    .pages({
-                      FunctionName: functionName,
-                    })
-                    .pipe(
-                      // TODO(sam): maybe process chunks to avoid linear scanning of Event Sources
-                      Stream.mapEffect(
-                        Effect.fn(function* (page) {
-                          const mapping = page.EventSourceMappings?.find(
-                            (m) => m.EventSourceArn === eventSourceArn,
-                          );
-
-                          if (mapping?.UUID) {
-                            const { Tags } = yield* lambda
-                              .listTags({
-                                Resource: `arn:aws:lambda:${region}:${accountId}:event-source-mapping:${mapping.UUID}`,
-                              })
-                              .pipe(retryTransient);
+          // Observe — find the existing mapping. UUIDs are server-assigned
+          // so we either trust `output.uuid` (fast path) or scan
+          // `listEventSourceMappings` and confirm ownership via tags
+          // (recovery from a state-persistence failure or adoption).
+          let config: lambda.EventSourceMappingConfiguration | undefined;
+          if (output?.uuid) {
+            config = yield* lambda
+              .getEventSourceMapping({ UUID: output.uuid })
+              .pipe(
+                Effect.catchTag("ResourceNotFoundException", () =>
+                  Effect.succeed(undefined),
+                ),
+              );
+          }
+          if (!config?.UUID) {
+            config = yield* lambda.listEventSourceMappings
+              .pages({ FunctionName: functionName })
+              .pipe(
+                Stream.mapEffect(
+                  Effect.fn(function* (page) {
+                    const mapping = page.EventSourceMappings?.find(
+                      (m) => m.EventSourceArn === eventSourceArn,
+                    );
+                    if (mapping?.UUID) {
+                      const { Tags } = yield* lambda
+                        .listTags({
+                          Resource: `arn:aws:lambda:${region}:${accountId}:event-source-mapping:${mapping.UUID}`,
+                        })
+                        .pipe(retryTransient);
+                      if (hasTags(expectedInternalTags, Tags)) {
+                        return mapping;
+                      }
+                    }
+                  }),
+                ),
+                Stream.filter((item) => item !== undefined),
+                Stream.runHead,
+                Effect.map(Option.getOrUndefined),
+              );
+          }
 
-                            if (hasTags(expectedInternalTags, Tags)) {
-                              return mapping;
+          // Ensure — create if no live mapping exists. Tolerate
+          // `ResourceConflictException` (peer reconciler raced ahead) by
+          // re-scanning to find the mapping by tag ownership.
+          if (!config?.UUID) {
+            config = yield* lambda
+              .createEventSourceMapping(toCreateRequest(news, desiredTags))
+              .pipe(
+                Effect.catchTags({
+                  ResourceConflictException: () =>
+                    lambda.listEventSourceMappings
+                      .pages({ FunctionName: functionName })
+                      .pipe(
+                        Stream.mapEffect(
+                          Effect.fn(function* (page) {
+                            const mapping = page.EventSourceMappings?.find(
+                              (m) => m.EventSourceArn === eventSourceArn,
+                            );
+                            if (mapping?.UUID) {
+                              const { Tags } = yield* lambda
+                                .listTags({
+                                  Resource: `arn:aws:lambda:${region}:${accountId}:event-source-mapping:${mapping.UUID}`,
+                                })
+                                .pipe(retryTransient);
+                              if (hasTags(expectedInternalTags, Tags)) {
+                                return mapping;
+                              }
                             }
-                          }
-                        }),
-                      ),
-                      Stream.filter((item) => item !== undefined),
-                      Stream.runHead,
-                      Effect.map(Option.getOrUndefined),
-                      Effect.flatMap((mapping) =>
-                        mapping
-                          ? Effect.succeed(mapping)
-                          : Effect.die(
-                              new Error(
-                                `EventSourceMapping(${id}) not found on function ${functionName}`,
+                          }),
+                        ),
+                        Stream.filter((item) => item !== undefined),
+                        Stream.runHead,
+                        Effect.map(Option.getOrUndefined),
+                        Effect.flatMap((mapping) =>
+                          mapping
+                            ? Effect.succeed(mapping)
+                            : Effect.die(
+                                new Error(
+                                  `EventSourceMapping(${id}) not found on function ${functionName}`,
+                                ),
                               ),
-                            ),
+                        ),
                       ),
-                    ),
-              }),
-              retryPermissionsPropagation,
-              retryTransient,
+                }),
+                retryPermissionsPropagation,
+                retryTransient,
+              );
+          }
+
+          if (!config?.UUID) {
+            return yield* Effect.die(
+              new Error(`EventSourceMapping(${id}) could not be reconciled`),
             );
+          }
 
-          yield* session.note(config.EventSourceMappingArn!);
+          const uuid = config.UUID;
+          const mappingArn = `arn:aws:lambda:${region}:${accountId}:event-source-mapping:${uuid}`;
 
-          return configToAttrs(config);
-        }),
-        update: Effect.fn(function* ({ id, news, olds, output, session }) {
-          const config = yield* lambda
-            .updateEventSourceMapping(toUpdateRequest(output.uuid, news))
+          // Sync configuration — `updateEventSourceMapping` is a full PUT
+          // for mutable fields. We always send the full desired config so
+          // observed state converges. Retry `ResourceInUseException`
+          // (mapping is transitioning) and known IAM-propagation errors.
+          config = yield* lambda
+            .updateEventSourceMapping(toUpdateRequest(uuid, news))
             .pipe(
               Effect.retry({
                 while: (e: any) =>
@@ -367,12 +416,18 @@ export const EventSourceMappingProvider = () =>
               retryTransient,
             );
 
-          const internalTags = yield* createEventSourceMappingTags(id);
-          const oldTags = { ...internalTags, ...olds.tags };
-          const newTags = { ...internalTags, ...news.tags };
-          const { removed, upsert } = diffTags(oldTags, newTags);
-
-          const mappingArn = `arn:aws:lambda:${region}:${accountId}:event-source-mapping:${output.uuid}`;
+          // Sync tags — diff observed cloud tags against desired so
+          // adoption rewrites ownership tags correctly.
+          const observedTagsResp = yield* lambda
+            .listTags({ Resource: mappingArn })
+            .pipe(retryTransient);
+          const observedTags: Record<string, string> = Object.fromEntries(
+            Object.entries(observedTagsResp.Tags ?? {}).filter(
+              (entry): entry is [string, string] =>
+                typeof entry[1] === "string",
+            ),
+          );
+          const { removed, upsert } = diffTags(observedTags, desiredTags);
 
           if (removed.length > 0) {
             yield* lambda
@@ -389,7 +444,7 @@ export const EventSourceMappingProvider = () =>
               .pipe(retryTransient);
           }
 
-          yield* session.note(`Updated event source mapping ${output.uuid}`);
+          yield* session.note(config.EventSourceMappingArn ?? uuid);
 
           return configToAttrs(config);
         }),