alchemy 2.0.0-beta.84 → 2.0.0-beta.85

package/src/Drift.ts

@@ -1,6 +1,7 @@
 /** @effect-diagnostics anyUnknownInErrorContext:off */
 import * as Cause from "effect/Cause";
 import * as Data from "effect/Data";
+import * as Deferred from "effect/Deferred";
 import * as Effect from "effect/Effect";
 import * as Layer from "effect/Layer";
 import * as Option from "effect/Option";
@@ -14,15 +15,11 @@ import {
   makeScopedArtifacts,
 } from "./Artifacts.ts";
 import { deepEqual } from "./Diff.ts";
+import { captureEmit, emit } from "./Events.ts";
 import type { Apply, Plan } from "./Plan.ts";
 import { findProviderByType, Provider } from "./Provider.ts";
 import { stampedMode } from "./ProviderMode.ts";
-import {
-  noopSession,
-  Progress,
-  type PlanStatusSession,
-  type ScopedPlanStatusSession,
-} from "./Report.ts";
+import { resourceSession } from "./Report.ts";
 import type { ResourceLike } from "./Resource.ts";
 import { ResourceContext } from "./ResourceContext.ts";
 import {
@@ -33,6 +30,7 @@ import {
   type UpdatedResourceState,
 } from "./State/index.ts";
 import { type ResourceOp, recordResourceOp } from "./Telemetry/Metrics.ts";
+import { findCycleMembers } from "./Util/scc.ts";
 
 /**
  * The outcome of checking or repairing drift for a single resource.
@@ -77,11 +75,6 @@ export interface DriftResult {
   resources: Record<string, DriftResourceResult>;
 }
 
-export interface DriftOptions {
-  /** Optional progress session (the CLI passes one; tests usually don't). */
-  session?: PlanStatusSession;
-}
-
 /** A provider lifecycle failure annotated with the resource being checked. */
 export class DriftResourceError extends Data.TaggedError("DriftResourceError")<{
   readonly message: string;
@@ -114,11 +107,14 @@ export class DriftResourceError extends Data.TaggedError("DriftResourceError")<{
  *    deterministic physical names regenerate identically.
  * 4. **Persist** — write the fresh attributes back to the state store.
  *
- * Resources are checked concurrently and independently — persisted props are
- * fully resolved values, so there are no upstream/downstream data edges to
- * order by. A failure checking one resource does not interrupt the others;
- * all failures are aggregated into a single combined cause after every
- * resource has been attempted.
+ * Resources are observed concurrently. Repairs follow the persisted
+ * `downstream` edges: a recreated topic exists before its subscription is
+ * recreated. Missing resources are recreated without their bindings first and
+ * bound once every missing resource exists again, because binding data (an S3
+ * notification's target topic) can point at a resource recreated later. A
+ * failure checking one resource does not interrupt the others; all failures
+ * are aggregated into a single combined cause after every resource has been
+ * attempted.
  *
  * Resources that cannot be checked are reported as `skipped` rather than
  * failing the run: providers without `read` (nothing to observe), and
@@ -129,16 +125,18 @@ export class DriftResourceError extends Data.TaggedError("DriftResourceError")<{
  */
 const runDrift = (
   stack: { name: string; stage: string },
-  options: DriftOptions & { readonly dryRun: boolean },
+  options: {
+    readonly dryRun: boolean;
+    /** Check and repair only these resources; the rest of the stage is untouched. */
+    readonly only?: ReadonlyArray<string>;
+  },
 ): Effect.Effect<DriftResult, any, State> =>
   Effect.gen(function* () {
-    // Phase markers + per-resource observation progress go through the
-    // ambient Progress reporter (a no-op by default) so renderers can show the
-    // drift check as it runs.
-    const reportPlanned = yield* Progress;
-    yield* reportPlanned({ _tag: "plan.phase", phase: "loading-state" });
+    // Phase markers and per-resource progress are emitted as events so
+    // renderers can show the drift check as it runs.
+    yield* emit({ type: "plan.phase", phase: "loading-state" });
     const state = yield* yield* State;
-    const session = options.session ?? noopSession;
+    const reportEvent = yield* captureEmit;
     const stackName = stack.name;
     const stage = stack.stage;
     const dryRun = options.dryRun;
@@ -161,24 +159,14 @@ const runDrift = (
         ...partial,
       });
 
-      const scopedSession = {
-        ...session,
-        note: (note, options?) =>
-          session.emit({
-            fqn,
-            id: logicalId,
-            _tag: "apply.resource.note",
-            message: note,
-            kind: options?.kind,
-          }),
-      } satisfies ScopedPlanStatusSession;
+      const scopedSession = resourceSession(reportEvent, fqn, logicalId);
 
       const report = (status: "updating" | "updated" | "creating" | "created" | "skipped") =>
-        session.emit({
-          _tag: "apply.resource.status",
+        reportEvent({
+          type: "apply.resource.status",
           fqn,
           id: logicalId,
-          type: resourceType,
+          resourceType,
           status,
         });
 
@@ -211,20 +199,48 @@ const runDrift = (
         return yield* skip(`provider '${resourceType}' does not implement read`);
       }
 
-      const commit = (value: Omit<ResourceState, "namespace">) =>
-        state.set({
+      // Persist a repaired row. Everything but the status, attributes and
+      // bindings carries over from the persisted row — notably the
+      // `providerMode` stamp, without which a repaired local row would be
+      // treated as live.
+      const commit = (
+        status: "created" | "updated",
+        attr: ResourceState["attr"],
+        bindings: ResourceState["bindings"],
+      ) => {
+        const row = {
+          kind: old.kind,
+          namespace,
+          fqn,
+          logicalId,
+          instanceId,
+          resourceType,
+          props: old.props!,
+          attr: attr!,
+          providerVersion: provider.version ?? 0,
+          bindings,
+          downstream: old.downstream,
+          removalPolicy: old.removalPolicy,
+          providerMode: old.providerMode,
+          adoptionBlocked: old.adoptionBlocked,
+        };
+        return state.set({
           stack: stackName,
           stage,
           fqn,
-          value: { ...value, namespace } as ResourceState,
+          value:
+            status === "created"
+              ? ({ ...row, status } satisfies CreatedResourceState)
+              : ({ ...row, status } satisfies UpdatedResourceState),
         });
+      };
 
       // Paired with the resource-planned completion so renderers can show
       // which rows' cloud reads are actually in flight (the slowest part of
       // a drift check). Emitted after the skip gates: skipped rows settle
       // instantly and never read the cloud.
-      yield* reportPlanned({
-        _tag: "plan.resource.started",
+      yield* emit({
+        type: "plan.resource.started",
         fqn,
         logicalId,
         resourceType,
@@ -240,39 +256,11 @@ const runDrift = (
         })
         .pipe(instrumentLifecycle("read", fqn, resourceType, logicalId, instanceId));
 
-      // ── missing — recreate under the same instance id ──
       if (observed === undefined) {
         if (dryRun) {
           return result({ action: "missing" });
         }
-        yield* report("creating");
-        const attr = yield* provider
-          .reconcile({
-            id: logicalId,
-            fqn,
-            instanceId,
-            news: old.props as never,
-            olds: undefined,
-            output: undefined,
-            session: scopedSession,
-            bindings: old.bindings as never,
-          })
-          .pipe(instrumentLifecycle("create", fqn, resourceType, logicalId, instanceId));
-        yield* commit({
-          status: "created",
-          fqn,
-          logicalId,
-          instanceId,
-          resourceType,
-          props: old.props!,
-          attr,
-          providerVersion: provider.version ?? 0,
-          bindings: old.bindings,
-          downstream: old.downstream,
-          removalPolicy: old.removalPolicy,
-        } satisfies Omit<CreatedResourceState, "namespace">);
-        yield* report("created");
-        return result({ action: "recreated", attr });
+        return { fqn, old, provider, live: undefined, result, scopedSession, report, commit };
       }
 
       // `read` may brand the attributes as Unowned when ownership markers
@@ -289,9 +277,60 @@ const runDrift = (
       if (dryRun) {
         return result({ action: "drifted", attr: live });
       }
+      return { fqn, old, provider, live, result, scopedSession, report, commit };
+    });
+    type PendingRepair = Extract<
+      Effect.Success<ReturnType<typeof driftResource>>,
+      { old: unknown }
+    >;
+    const isPendingRepair = (
+      observed: DriftResourceResult | PendingRepair | undefined,
+    ): observed is PendingRepair => observed !== undefined && "old" in observed;
+
+    // ── missing — recreate under the same instance id, without bindings ──
+    //
+    // Binding data often points at other resources (an S3 notification
+    // targets a topic), and a binding edge inside a cycle is not recorded in
+    // `downstream`. So, like Apply's precreate, missing resources are first
+    // created bare in `downstream` order; their bindings are attached in the
+    // second phase, once every missing resource exists again. The bare
+    // create is committed with no bindings so a crash in between leaves a
+    // row the next deploy re-binds.
+    const recreate = Effect.fn("drift.recreate")(function* (pending: PendingRepair) {
+      const { fqn, old, provider, report, commit, scopedSession, result } = pending;
+      const { logicalId, instanceId, resourceType } = old;
+      const bindings = old.bindings ?? [];
+      yield* report("creating");
+      const attr = yield* provider
+        .reconcile({
+          id: logicalId,
+          fqn,
+          instanceId,
+          news: old.props as never,
+          olds: undefined,
+          output: undefined,
+          session: scopedSession,
+          bindings: [] as never,
+        })
+        .pipe(instrumentLifecycle("create", fqn, resourceType, logicalId, instanceId));
+      yield* commit("created", attr, []);
+      if (bindings.length === 0) {
+        yield* report("created");
+        return { done: result({ action: "recreated", attr }) };
+      }
+      return { rebind: { ...pending, live: attr, recreated: true } };
+    });
 
-      // ── drifted — converge the cloud back to the last-deployed props ──
-      yield* report("updating");
+    // ── drifted — converge the cloud back to the last-deployed props ──
+    //
+    // Also attaches the bindings of resources recreated bare above.
+    const reconcile = Effect.fn("drift.reconcile")(function* (
+      pending: PendingRepair & { recreated: boolean },
+    ) {
+      const { fqn, old, provider, live, report, commit, scopedSession, result, recreated } =
+        pending;
+      const { logicalId, instanceId, resourceType } = old;
+      if (!recreated) yield* report("updating");
       const attr = yield* provider
         .reconcile({
           id: logicalId,
@@ -305,48 +344,51 @@ const runDrift = (
           session: scopedSession,
           bindings: old.bindings as never,
         })
-        .pipe(instrumentLifecycle("update", fqn, resourceType, logicalId, instanceId));
-      yield* commit({
-        status: "updated",
-        fqn,
-        logicalId,
-        instanceId,
-        resourceType,
-        props: old.props!,
-        attr,
-        providerVersion: provider.version ?? 0,
-        bindings: old.bindings,
-        downstream: old.downstream,
-        removalPolicy: old.removalPolicy,
-      } satisfies Omit<UpdatedResourceState, "namespace">);
-      yield* report("updated");
-      return result({ action: "repaired", attr });
+        .pipe(
+          instrumentLifecycle(
+            recreated ? "create" : "update",
+            fqn,
+            resourceType,
+            logicalId,
+            instanceId,
+          ),
+        );
+      yield* commit(recreated ? "created" : "updated", attr, old.bindings);
+      yield* report(recreated ? "created" : "updated");
+      return result({ action: recreated ? "recreated" : "repaired", attr });
     });
 
-    const fqns = yield* state.list({ stack: stackName, stage });
-    yield* reportPlanned({ _tag: "plan.phase", phase: "computing-plan" });
+    const listed = yield* state.list({ stack: stackName, stage });
+    const fqns =
+      options.only === undefined ? listed : listed.filter((fqn) => options.only!.includes(fqn));
+    yield* emit({ type: "plan.phase", phase: "computing-plan" });
 
     // Per-resource observation progress: each row is reported through the
-    // ambient Progress reporter as its cloud read lands (drift checks are the
+    // events as its cloud read lands (drift checks are the
     // slowest planning path). Failed or action rows advance the count
     // without an event, so progress stays monotonic.
     const observedCount = yield* Ref.make(0);
-    const resourceObserved = (result: DriftResourceResult | undefined) =>
+    const resourceObserved = (observed: DriftResourceResult | PendingRepair | undefined) =>
       Ref.updateAndGet(observedCount, (count) => count + 1).pipe(
         Effect.flatMap((completed) =>
-          result === undefined
+          observed === undefined
             ? Effect.void
-            : reportPlanned({
-                _tag: "plan.resource.completed",
-                fqn: result.fqn,
-                logicalId: result.logicalId,
-                resourceType: result.resourceType,
+            : emit({
+                type: "plan.resource.completed",
+                fqn: observed.fqn,
+                logicalId: isPendingRepair(observed) ? observed.old.logicalId : observed.logicalId,
+                resourceType: isPendingRepair(observed)
+                  ? observed.old.resourceType
+                  : observed.resourceType,
                 // Drift observation compares attributes, not binding rows.
                 bindings: [],
-                action:
-                  result.action === "drifted" || result.action === "repaired"
+                action: isPendingRepair(observed)
+                  ? observed.live === undefined
+                    ? "create"
+                    : "update"
+                  : observed.action === "drifted"
                     ? "update"
-                    : result.action === "missing"
+                    : observed.action === "missing"
                       ? "create"
                       : "noop",
                 completed,
@@ -358,34 +400,116 @@ const runDrift = (
     // Drift every resource even if some fail, then surface all failures as
     // one combined cause (mirrors Apply's failure aggregation).
     const failures: Cause.Cause<unknown>[] = [];
-    const results = yield* Effect.all(
-      fqns.map((fqn) =>
+    const recordFailure = (fqn: string) => (cause: Cause.Cause<unknown>) =>
+      Effect.gen(function* () {
+        failures.push(cause);
+        const persisted = yield* state
+          .get({ stack: stackName, stage, fqn })
+          .pipe(Effect.orElseSucceed(() => undefined));
+        if (persisted && !isActionState(persisted)) {
+          yield* reportEvent({
+            type: "apply.resource.status",
+            fqn,
+            id: persisted.logicalId,
+            resourceType: persisted.resourceType,
+            status: "fail",
+          });
+        }
+        return undefined;
+      });
+
+    const observed = yield* Effect.forEach(
+      fqns,
+      (fqn) =>
         driftResource(fqn).pipe(
-          Effect.catchCause((cause) =>
-            Effect.gen(function* () {
-              failures.push(cause);
-              const persisted = yield* state
-                .get({ stack: stackName, stage, fqn })
-                .pipe(Effect.orElseSucceed(() => undefined));
-              if (persisted && !isActionState(persisted)) {
-                yield* session.emit({
-                  _tag: "apply.resource.status",
-                  fqn,
-                  id: persisted.logicalId,
-                  type: persisted.resourceType,
-                  status: "fail",
-                });
-              }
-              return undefined;
-            }),
-          ),
+          Effect.catchCause(recordFailure(fqn)),
           Effect.tap(resourceObserved),
         ),
-      ),
       { concurrency: "unbounded" },
     );
 
-    yield* session.done(failures.length === 0 ? "success" : "failure");
+    const pending = observed.filter(isPendingRepair);
+
+    // Upstreams per the persisted `downstream` edges, among the resources
+    // being repaired (an unchanged or skipped upstream never blocks). Edges
+    // inside a cycle — only possible through merged historical edges — are
+    // dropped.
+    const upstream: Record<string, string[]> = Object.fromEntries(
+      pending.map((node) => [node.fqn, [] as string[]]),
+    );
+    for (const node of pending) {
+      for (const down of node.old.downstream ?? []) {
+        if (down !== node.fqn) upstream[down]?.push(node.fqn);
+      }
+    }
+    const cyclic = findCycleMembers(upstream);
+    const upstreamOf = (fqn: string) =>
+      upstream[fqn]!.filter((up) => !(cyclic.has(up) && cyclic.has(fqn)));
+
+    // Resources whose repair failed, or was skipped because an upstream's
+    // did. Like Apply, a dependent never runs against a failed upstream.
+    const failed = new Set<string>();
+
+    // Run `run` over `nodes`, each node after its upstreams in `nodes` have
+    // settled. Upstreams outside `nodes` settled in an earlier phase.
+    const inDependencyOrder = Effect.fn(function* <N extends PendingRepair, A>(
+      nodes: ReadonlyArray<N>,
+      run: (node: N) => Effect.Effect<A, unknown>,
+    ) {
+      const settled = new Map<string, Deferred.Deferred<void>>();
+      for (const node of nodes) settled.set(node.fqn, yield* Deferred.make<void>());
+      return yield* Effect.forEach(
+        nodes,
+        (node) =>
+          Effect.gen(function* () {
+            const ups = upstreamOf(node.fqn);
+            yield* Effect.all(
+              ups.flatMap((up) => {
+                const deferred = settled.get(up);
+                return deferred ? [Deferred.await(deferred)] : [];
+              }),
+            );
+            const blocked = ups.filter((up) => failed.has(up));
+            if (blocked.length > 0) {
+              failed.add(node.fqn);
+              yield* node.scopedSession.note(
+                `not repaired: upstream ${blocked.map((up) => `'${up}'`).join(", ")} failed`,
+              );
+              yield* node.report("skipped");
+              return undefined;
+            }
+            return yield* run(node).pipe(
+              Effect.catchCause((cause) =>
+                Effect.sync(() => failed.add(node.fqn)).pipe(
+                  Effect.andThen(recordFailure(node.fqn)(cause)),
+                ),
+              ),
+            );
+          }).pipe(Effect.ensuring(Deferred.succeed(settled.get(node.fqn)!, undefined))),
+        { concurrency: "unbounded" },
+      );
+    });
+
+    const recreated = yield* inDependencyOrder(
+      pending.filter((node) => node.live === undefined),
+      recreate,
+    );
+    const rebind = recreated.flatMap((node) => (node?.rebind ? [node.rebind] : []));
+    const repaired = yield* inDependencyOrder(
+      [
+        ...pending
+          .filter((node) => node.live !== undefined)
+          .map((node) => ({ ...node, recreated: false })),
+        ...rebind,
+      ],
+      reconcile,
+    );
+
+    const results = [
+      ...observed.filter((r) => !isPendingRepair(r)),
+      ...recreated.map((r) => r?.done),
+      ...repaired,
+    ].filter((r): r is DriftResourceResult => r !== undefined);
 
     if (failures.length > 0) {
       return yield* Effect.failCause(failures.reduce(Cause.combine) as Cause.Cause<never>);
@@ -410,9 +534,14 @@ const runDrift = (
 /** Detect drift without reconciling resources or updating state. */
 export const detect = (stack: { name: string; stage: string }) => runDrift(stack, { dryRun: true });
 
-/** Reconcile resources back to their last-deployed desired state. */
-export const repair = (stack: { name: string; stage: string }, options: DriftOptions = {}) =>
-  runDrift(stack, { ...options, dryRun: false });
+/**
+ * Reconcile resources back to their last-deployed desired state. With
+ * `only`, just those resources are re-checked and repaired.
+ */
+export const repair = (
+  stack: { name: string; stage: string },
+  options: { readonly only?: ReadonlyArray<string> } = {},
+) => runDrift(stack, { dryRun: false, only: options.only });
 
 /**
  * Same shape as Apply's lifecycle instrumentation: scoped artifacts +

package/src/EvalStack.ts

@@ -0,0 +1,97 @@
+import { ConfigProvider } from "effect/ConfigProvider";
+import type * as Context from "effect/Context";
+import * as Effect from "effect/Effect";
+import * as FetchHttpClient from "effect/http/FetchHttpClient";
+import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
+import * as Scope from "effect/Scope";
+import { AlchemyContext, AlchemyContextLive } from "./AlchemyContext.ts";
+import { provideFreshArtifactStore } from "./Artifacts.ts";
+import { CredentialsStoreLive } from "./Auth/Credentials.ts";
+import { ProfileStoreLive } from "./Auth/Profile.ts";
+import { type StackEffect, StateScope } from "./Stack.ts";
+import { Stage } from "./Stage.ts";
+import { loadConfigProvider } from "./Util/ConfigProvider.ts";
+import { PlatformServices } from "./Util/PlatformServices.ts";
+
+// TODO(unify stack builds): this is one of several ways to build a stack;
+// the others are `Alchemist/Operations.ts` (`buildStack`,
+// `buildWithoutProgram`) and the test harness's scratch stacks
+// (`alchemy-test/src/Core.ts`). They differ in config (this one reloads `.env`
+// and drops the caller's ConfigProvider), adopt/updateStateStore, and how
+// the state-store lease is scoped. Migrate everything to `Alchemist.plan` /
+// `Alchemist.apply`: give Alchemist this file's per-call lease scope
+// (`StateScope`), reimplement `Deploy.deploy` / `Destroy.destroy` on it,
+// move scratch stacks onto it, then delete this file.
+
+// Deploy-time only: kept out of `Stack.ts` because every runtime imports the
+// `Stack` tag, and this module's top-level platform layer (Node + Bun
+// platforms, `ws`, profile and credential stores) survives tree-shaking.
+
+const platform = Layer.mergeAll(
+  PlatformServices,
+  FetchHttpClient.layer,
+  Layer.provide(ProfileStoreLive, PlatformServices),
+  Layer.provide(CredentialsStoreLive, PlatformServices),
+);
+// override alchemy state store, CLI/reporting, state, and Config
+const alchemy = (overrides?: { dev?: boolean }) =>
+  Layer.mergeAll(
+    // optional
+    overrides?.dev
+      ? Layer.provide(
+          Layer.effect(
+            AlchemyContext,
+            AlchemyContext.useSync((ctx) => ({ ...ctx, dev: overrides.dev! })),
+          ),
+          AlchemyContextLive,
+        )
+      : AlchemyContextLive,
+  );
+
+export const evalStack = <S extends { services: Context.Context<any> }, B, StackErr, Err, Req>(
+  effect: StackEffect<S, StackErr, Stage | AlchemyContext>,
+  fn: (stack: S) => Effect.Effect<B, Err, Req>,
+  options: {
+    stage: string;
+    dev?: boolean;
+    /**
+     * Optional caller-supplied scope. When provided, scoped resources
+     * (e.g. the dev sidecar process) live until the caller closes this
+     * scope instead of being torn down when `evalStack` resolves. Used
+     * by the test harness so the sidecar survives across `beforeAll`,
+     * tests, and `afterAll`. When omitted, behaves as before with a
+     * private scope closed on completion. The state store never outlives
+     * the call.
+     */
+    scope?: Scope.Scope;
+  },
+) => {
+  const body = Effect.gen(function* () {
+    const stack = yield* effect;
+    const configProvider = yield* loadConfigProvider(Option.none());
+    return yield* fn(stack).pipe(
+      provideFreshArtifactStore,
+      Effect.provide(
+        Layer.succeedContext(stack.services).pipe(
+          Layer.provideMerge(Layer.succeed(ConfigProvider, configProvider)),
+        ),
+      ),
+    );
+  }).pipe(
+    Effect.provide(
+      Layer.succeed(Stage, options.stage).pipe(
+        Layer.provideMerge(Layer.provideMerge(alchemy({ dev: options.dev }), platform)),
+      ),
+    ),
+  );
+
+  const scope = options.scope;
+  return scope === undefined
+    ? Effect.scoped(body)
+    : // The state store closes with this call and releases its lease, so
+      // the next call on the same long-lived scope can take it.
+      Effect.scopedWith((stateScope) =>
+        Scope.provide(body.pipe(Effect.provideService(StateScope, stateScope)), scope),
+      );
+};