alchemy 2.0.0-beta.3 → 2.0.0-beta.31

package/src/Output.ts

@@ -3,10 +3,11 @@ import type { Yieldable } from "effect/Effect";
 import * as Effect from "effect/Effect";
 import { pipe } from "effect/Function";
 import type { Pipeable } from "effect/Pipeable";
+import * as Redacted from "effect/Redacted";
 import { SingleShotGen } from "effect/Utils";
-import { ExecutionContext } from "./ExecutionContext.ts";
 import { getRefMetadata, isRef, ref as stageRef, type Ref } from "./Ref.ts";
 import { isResource, type Resource, type ResourceLike } from "./Resource.ts";
+import { RuntimeContext } from "./RuntimeContext.ts";
 import { Stack } from "./Stack.ts";
 import { Stage } from "./Stage.ts";
 import * as State from "./State/State.ts";
@@ -51,7 +52,7 @@ export interface Output<A = any, Req = any> extends Pipeable {
     Accessor<A>,
     void
   >;
-  bind(id: string): Effect.Effect<Effect.Effect<A>, never, ExecutionContext>;
+  bind(id: string): Effect.Effect<Effect.Effect<A>, never, RuntimeContext>;
   asEffect(): Effect.Effect<Accessor<A>, never, Req>;
   as<T>(): Output<T, Req>;
 }
@@ -90,7 +91,7 @@ export abstract class BaseExpr<A = any, Req = any> implements Output<A, Req> {
   declare readonly A: A;
   declare readonly src: ResourceLike;
   declare readonly req: Req;
-  // we use a kind tag instead of instanceof to protect ourselves from duplicate alchemy-effect module imports
+  // we use a kind tag instead of instanceof to protect ourselves from duplicate alchemy module imports
   constructor() {}
   as<T>(): Output<T, Req> {
     return this as any;
@@ -110,7 +111,7 @@ export abstract class BaseExpr<A = any, Req = any> implements Output<A, Req> {
   }
 
   public bind(id: string): any {
-    return ExecutionContext.asEffect().pipe(
+    return RuntimeContext.asEffect().pipe(
       Effect.flatMap((ctx) =>
         Effect.map(ctx.set(id, this), (key) => ctx.get<A>(key)),
       ),
@@ -208,8 +209,6 @@ export const map: {
         new ApplyExpr(output as Expr<A, Req>, args[0]) as any
     : new ApplyExpr(args[0] as any, args[1])) as any;
 
-type _ = Parameters<typeof map>;
-
 //Output.ApplyExpr<any, any, ResourceLike, any>
 export const isApplyExpr = <In = any, Out = any, Req = any>(
   node: Output<Out, Req>,
@@ -354,7 +353,7 @@ function proxy(self: any): any {
       prop === ExprSymbol || prop === inspect ? true : prop in self,
     get: (target, prop) =>
       prop === Symbol.toPrimitive
-        ? (hint: string) => (hint === "string" ? self.toString() : self)
+        ? (hint: string) => (hint === "number" ? Number.NaN : self.toString())
         : prop === ExprSymbol
           ? self
           : prop === inspect
@@ -470,6 +469,8 @@ export const evaluate: <A, Req = never>(
     }
     if (Array.isArray(expr)) {
       return yield* Effect.all(expr.map((item) => evaluate(item, upstream)));
+    } else if (Redacted.isRedacted(expr)) {
+      return expr;
     } else if (typeof expr === "object" && expr !== null) {
       return Object.fromEntries(
         yield* Effect.all(
@@ -498,7 +499,9 @@ export const upstreamAny = (
 ): {
   [ID in string]: Resource;
 } => {
-  if (isExpr(value)) {
+  if (isResource(value)) {
+    return { [value.FQN]: value as Resource };
+  } else if (isExpr(value)) {
     return upstream(value);
   } else if (Array.isArray(value)) {
     return Object.assign({}, ...value.map(resolveUpstream));
@@ -516,7 +519,11 @@ export const upstreamAny = (
 
 // TODO(sam): add a type
 export const upstream = <E extends Output<any, any>>(expr: E): any => {
-  if (isResourceExpr(expr)) {
+  if (isResource(expr)) {
+    return {
+      [(expr as unknown as Resource).FQN]: expr,
+    };
+  } else if (isResourceExpr(expr)) {
     return {
       [expr.src.FQN]: expr.src,
     };
@@ -540,6 +547,8 @@ export const upstream = <E extends Output<any, any>>(expr: E): any => {
 export const resolveUpstream = <const A>(value: A): any => {
   if (isPrimitive(value)) {
     return {} as any;
+  } else if (isResource(value)) {
+    return { [(value as unknown as Resource).FQN]: value } as any;
   } else if (isOutput(value)) {
     return upstream(value) as any;
   } else if (Array.isArray(value)) {

package/src/Plan.ts

@@ -1,7 +1,15 @@
 import * as Data from "effect/Data";
 import * as Effect from "effect/Effect";
 import * as Option from "effect/Option";
+import * as Redacted from "effect/Redacted";
 import { asEffect } from ".//Util/types.ts";
+import {
+  AdoptPolicy,
+  OwnedBySomeoneElse,
+  stripUnowned,
+  Unowned,
+} from "./AdoptPolicy.ts";
+import { AlchemyContext } from "./AlchemyContext.ts";
 import {
   Artifacts,
   ArtifactStore,
@@ -12,11 +20,12 @@ import {
 import {
   diffBindings,
   havePropsChanged,
+  isResolved,
   type NoopDiff,
   type UpdateDiff,
 } from "./Diff.ts";
 import { parseFqn } from "./FQN.ts";
-import { InstanceId } from "./InstanceId.ts";
+import { generateInstanceId, InstanceId } from "./InstanceId.ts";
 import * as Output from "./Output.ts";
 import {
   findProviderByType,
@@ -39,6 +48,7 @@ import {
   type UpdatedResourceState,
   type UpdatingReourceState,
 } from "./State/index.ts";
+import { findCycleMembers } from "./Util/scc.ts";
 
 export type PlanError = never;
 
@@ -146,6 +156,14 @@ export type Plan<Output = any> = {
     [id in string]?: Delete<ResourceLike>;
   };
   output: Output;
+  /**
+   * FQNs of resources that participate in a strongly-connected component
+   * of the upstream dependency graph (or have a self-edge). The scheduler
+   * uses this to decide whether an `update` node must publish its prior
+   * attr early to break the cycle, or can simply wait for upstreams and
+   * publish a fresh attr (the common, linear case).
+   */
+  cycleMembers: ReadonlySet<string>;
 };
 
 export interface MakePlanOptions {
@@ -166,6 +184,20 @@ export const make = <A>(
     const stackName = stack.name;
     const stage = stack.stage;
 
+    // Resolve the effective adoption setting for this plan. AdoptPolicy
+    // (provided by the CLI or scoped via `adopt(...)`) takes precedence; if
+    // unset, fall back to the AlchemyContext's `adopt` default; otherwise
+    // adoption is disabled.
+    const shouldAdopt = Effect.gen(function* () {
+      const fromService = yield* Effect.serviceOption(AdoptPolicy);
+      if (Option.isSome(fromService)) return fromService.value;
+      const ctx = yield* Effect.serviceOption(AlchemyContext);
+      return Option.match(ctx, {
+        onNone: () => false,
+        onSome: (c) => c.adopt,
+      });
+    });
+
     const resourceFqns = yield* state.list({
       stack: stackName,
       stage: stage,
@@ -271,6 +303,8 @@ export const make = <A>(
           return input;
         } else if (Output.isExpr(input)) {
           return yield* resolveOutput(input);
+        } else if (Redacted.isRedacted(input)) {
+          return input;
         } else if (Array.isArray(input)) {
           return yield* Effect.all(input.map(resolveInput), {
             concurrency: "unbounded",
@@ -394,7 +428,7 @@ export const make = <A>(
     const resourceGraph = Object.fromEntries(
       (yield* Effect.all(
         resources.map(
-          Effect.fn(function* (resource) {
+          Effect.fn("plan.diff.resource")(function* (resource) {
             const provider = yield* findProviderByType(resource.Type);
             const id = resource.LogicalId;
             const fqn = resource.FQN;
@@ -404,11 +438,98 @@ export const make = <A>(
             const newBindings: ResourceBinding[] = yield* resolveInput(
               stack.bindings[fqn] ?? [],
             );
-            const oldState = yield* state.get({
+            let oldState = yield* state.get({
               stack: stackName,
               stage: stage,
               fqn,
             });
+
+            // Engine-level adoption. When there is no prior state, always
+            // consult `provider.read` (if implemented) so the engine — not
+            // each lifecycle method — owns the existence/ownership decision.
+            //
+            // The provider returns one of:
+            //   - `undefined`        → resource doesn't exist; create it
+            //   - plain attrs        → exists and is owned by us; silent adopt
+            //   - `Unowned(attrs)`   → exists but is *not* ours
+            //
+            // Routing:
+            //   - owned                          → persist `created` state
+            //                                      from attrs and continue
+            //                                      through the normal diff
+            //                                      path (so subsequent props
+            //                                      drift produces an update).
+            //   - unowned + adopt enabled        → take over: persist `created`
+            //                                      state and let the next
+            //                                      update overwrite tags / etc.
+            //   - unowned + adopt disabled       → fail with
+            //                                      `OwnedBySomeoneElse`.
+            // After a cold-start adoption (engine just discovered an
+            // existing cloud resource via `read`), force the engine's
+            // normal `update` path so the provider can re-sync ownership
+            // tags, configuration, etc. against the desired props.
+            // Adoption persists state with `props: news`, so the default
+            // diff sees no drift and would noop — which would leave any
+            // foreign-owned tags / divergent config in place. Forcing
+            // update keeps the deploy idempotent: if cloud state already
+            // matches news, the provider's update is a no-op write.
+            //
+            // Skip the adoption probe entirely when `news` still contains
+            // unresolved upstream Outputs (e.g. a `streamArn` referencing
+            // a stream being created in the same plan). Calling `read` with
+            // an unresolved value would surface as `ParseError` from the
+            // SDK protocol layer. Resources whose props depend on
+            // not-yet-created upstreams cannot themselves be pre-existing
+            // — there's nothing to adopt.
+            let forceUpdateAfterAdoption = false;
+            if (oldState === undefined && provider.read && isResolved(news)) {
+              const adoptInstanceId = yield* generateInstanceId();
+              const readResult = yield* provider
+                .read({
+                  id,
+                  instanceId: adoptInstanceId,
+                  olds: news,
+                  output: undefined,
+                })
+                .pipe(providePlanScope(fqn, adoptInstanceId));
+              if (readResult !== undefined) {
+                const isUnowned = Unowned.is(readResult);
+                if (isUnowned && !(yield* shouldAdopt)) {
+                  return yield* new OwnedBySomeoneElse({
+                    message:
+                      `Cannot adopt resource '${fqn}' (${resource.Type}): ` +
+                      "it exists in the cloud but is not owned by this " +
+                      "stack/stage/logical-id. Re-run with `--adopt` (or " +
+                      "wrap the effect in `adopt(true)`) to take it over.",
+                    resourceType: resource.Type,
+                    logicalId: id,
+                  });
+                }
+                const adoptedState = {
+                  status: "created" as const,
+                  fqn,
+                  logicalId: id,
+                  instanceId: adoptInstanceId,
+                  namespace: resource.Namespace,
+                  resourceType: resource.Type,
+                  props: news,
+                  attr: stripUnowned(readResult),
+                  providerVersion: provider.version ?? 0,
+                  bindings: [],
+                  downstream,
+                  removalPolicy: resource.RemovalPolicy,
+                } satisfies CreatedResourceState;
+                yield* state.set({
+                  stack: stackName,
+                  stage: stage,
+                  fqn,
+                  value: adoptedState,
+                });
+                oldState = adoptedState;
+                forceUpdateAfterAdoption = true;
+              }
+            }
+
             const oldBindings = oldState?.bindings ?? [];
             const bindingDiffs = diffBindings(oldBindings, newBindings);
 
@@ -497,6 +618,17 @@ export const make = <A>(
                     } satisfies UpdateDiff)
                   : diff,
               ),
+              // After a cold-start adoption (silent or takeover), force at
+              // least an update so the provider re-syncs ownership tags /
+              // config against the desired props (otherwise the engine
+              // would noop and any drift between the existing cloud
+              // resource and `news` — including foreign-owned tags after a
+              // takeover — would persist).
+              Effect.map((diff) =>
+                forceUpdateAfterAdoption && diff.action === "noop"
+                  ? ({ action: "update" } satisfies UpdateDiff)
+                  : diff,
+              ),
             );
 
             if (oldState.status === "creating") {
@@ -657,12 +789,18 @@ export const make = <A>(
       )).map((update) => [update.resource.FQN, update]),
     ) as Plan["resources"];
 
+    // Compute SCC membership once. Apply uses it to decide whether an
+    // update node must publish its prior attr early to break a cycle, or
+    // can simply wait for upstreams like a DAG node (the common case).
+    const cycleMembers = findCycleMembers(allUpstreamDependencies);
+
     // Detect unsatisfiable dependency cycles among create/replace nodes.
-    // Update/noop nodes signal their Deferred before waitForDeps so they
-    // cannot deadlock. Create/replace nodes only signal early when they
-    // have a precreate handler. Simulate the concurrent execution:
-    // precreate nodes are immediately "resolved", then iteratively resolve
-    // any node whose deps are all resolved. Remaining nodes would deadlock.
+    // Update/noop nodes signal their Deferred before waitForDeps when in a
+    // cycle so they cannot deadlock. Create/replace nodes only signal
+    // early when they have a precreate handler. Simulate the concurrent
+    // execution: precreate nodes are immediately "resolved", then
+    // iteratively resolve any node whose deps are all resolved. Remaining
+    // nodes would deadlock.
     {
       const createReplaceNodes = new Set(
         Object.entries(resourceGraph)
@@ -720,8 +858,7 @@ export const make = <A>(
     const deletions = Object.fromEntries(
       (yield* Effect.all(
         (yield* state.list({ stack: stackName, stage: stage })).map(
-          Effect.fn(function* (fqn) {
-            // Check if this FQN is in the new resources
+          Effect.fn("plan.diff.deletion")(function* (fqn) {
             if (newResourceFqns.has(fqn)) {
               return;
             }
@@ -763,7 +900,7 @@ export const make = <A>(
                     Binding: undefined!,
                     Provider: Provider(resourceType),
                     RemovalPolicy: oldState.removalPolicy,
-                    ExecutionContext: undefined!,
+                    RuntimeContext: undefined!,
                     Providers: undefined,
                   } as ResourceLike,
                   downstream: oldDownstreamDependencies[fqn] ?? [],
@@ -801,8 +938,19 @@ export const make = <A>(
       resources: resourceGraph,
       deletions,
       output: stack.output,
+      cycleMembers,
     } satisfies Plan<A> as Plan<A>;
-  }).pipe(ensureArtifactStore);
+  }).pipe(
+    ensureArtifactStore,
+    Effect.withSpan("plan.make", {
+      attributes: {
+        "alchemy.stack": stack.name,
+        "alchemy.stage": stack.stage,
+        "alchemy.resources.count": Object.keys(stack.resources).length,
+        "alchemy.force": !!options.force,
+      },
+    }),
+  );
 
 const providePlanScope =
   (fqn: string, instanceId: string) =>

package/src/Platform.ts

@@ -7,14 +7,11 @@ import type { Scope } from "effect/Scope";
 import type { HttpClient } from "effect/unstable/http/HttpClient";
 import { SingleShotGen } from "effect/Utils";
 import type { PolicyLike } from "./Binding.ts";
-import {
-  ExecutionContext,
-  type BaseExecutionContext,
-} from "./ExecutionContext.ts";
 import type { HttpEffect } from "./Http.ts";
 import type { InputProps } from "./Input.ts";
 import type { Provider, ProviderCollectionLike } from "./Provider.ts";
 import { Resource, type ResourceLike } from "./Resource.ts";
+import { RuntimeContext, type BaseRuntimeContext } from "./RuntimeContext.ts";
 import { Self } from "./Self.ts";
 import type { Stack, StackServices } from "./Stack.ts";
 import type { Stage } from "./Stage.ts";
@@ -28,17 +25,23 @@ export interface PlatformProps {
   isExternal?: boolean;
 }
 
-export type Main<Services = never> = void | {
-  fetch: HttpEffect<Services | PlatformServices>;
+export type Main<InitServices = never> = void | {
+  fetch:
+    | HttpEffect<InitServices | PlatformServices>
+    | Effect.Effect<
+        HttpEffect<InitServices | PlatformServices>,
+        never,
+        InitServices | PlatformServices
+      >;
 };
 
 export type Rpc<Shape> = {
-  "~alchemy-effect/rpc": Shape;
+  "~alchemy/rpc": Shape;
 };
 
 // services provided to the Resource
 export type PlatformServices =
-  | ExecutionContext
+  | RuntimeContext
   | HttpClient
   | PolicyLike
   | Provider<any>
@@ -52,10 +55,10 @@ export interface Platform<
   Resource extends ResourceLike<string, PlatformProps>,
   Services,
   MainShape,
-  ExecutionContext extends BaseExecutionContext,
+  RuntimeContext extends BaseRuntimeContext,
   BaseShape = {},
 > extends Effect.Effect<
-  Resource & ExecutionContext,
+  Resource & RuntimeContext,
   never,
   Services | PlatformServices
 > {
@@ -159,7 +162,7 @@ export interface Platform<
   ): Effect.Effect<
     Resource & Rpc<Shape>,
     never,
-    | Provider<Resource>
+    | Resource["Providers"]
     | PropsReq
     | Exclude<InitReq, Services | PlatformServices>
   >;
@@ -174,14 +177,14 @@ export const Platform = <
     string,
     | {
         env?: Record<string, any>;
-        exports?: string[];
+        exports?: string[] | Record<string, any>;
       }
     | undefined
   >,
 >(
   type: R["Type"],
   hooks: {
-    createExecutionContext: (id: string) => BaseExecutionContext;
+    createRuntimeContext: (id: string) => BaseRuntimeContext;
     onCreate?: (resource: R, props: any) => Effect.Effect<void>;
   },
   methods?: { [key: string]: any },
@@ -190,7 +193,7 @@ export const Platform = <
   type Impl = Effect.Effect<any>;
 
   const resource = Resource(type);
-  const PlatformContext = ExecutionContext(type);
+  const PlatformContext = RuntimeContext(type);
 
   const constructor = (
     id?: string,
@@ -220,10 +223,15 @@ export const Platform = <
             //     return new Response("Hello, world!");
             //   }
             // }
-            resource(id, {
-              ...props,
-              isExternal: true,
-            })
+            resource(
+              id,
+              Effect.isEffect(props)
+                ? Effect.map(props, (p: any) => ({ ...p, isExternal: true }))
+                : {
+                    ...props,
+                    isExternal: true,
+                  },
+            )
           : Effect.flatMap(
               // this is a tagged resource
               Effect.serviceOption(cls.Self),
@@ -292,12 +300,12 @@ export const Platform = <
           Effect.flatMap(
             Effect.all([
               Effect.isEffect(props) ? props : Effect.succeed(props ?? {}),
-              Effect.sync(() => hooks.createExecutionContext(id)),
+              Effect.sync(() => hooks.createRuntimeContext(id)),
               Effect.context<never>(),
             ]),
             Effect.fnUntraced(function* ([
               props,
-              executionContext,
+              runtimeContext,
               outerServices,
             ]) {
               const instance = Object.assign(
@@ -310,22 +318,22 @@ export const Platform = <
                       Effect.succeed(resource),
                   ),
                 ),
-                executionContext,
+                runtimeContext,
               );
 
               yield* impl.pipe(
                 Effect.flatMap((impl) =>
-                  impl
-                    ? (executionContext.serve?.(impl.fetch) ??
+                  impl?.fetch
+                    ? (runtimeContext.serve?.(impl.fetch) ??
                       Effect.die("No serve handler"))
                     : Effect.void,
                 ),
                 Effect.provide(
                   Layer.provideMerge(
                     Layer.mergeAll(
-                      Layer.succeed(Platform.Platform, executionContext),
-                      Layer.succeed(PlatformContext, executionContext),
-                      Layer.succeed(ExecutionContext, executionContext),
+                      Layer.succeed(Platform.Platform, runtimeContext),
+                      Layer.succeed(PlatformContext, runtimeContext),
+                      Layer.succeed(RuntimeContext, runtimeContext),
                       Layer.succeed(resource.Self, instance),
                       Layer.succeed(Platform.Self, instance),
                       Layer.succeed(Self, instance),
@@ -339,15 +347,15 @@ export const Platform = <
                 ...props,
                 env: {
                   ...props?.env,
-                  ...executionContext.env,
+                  ...runtimeContext.env,
                 },
-                exports: executionContext.exports
-                  ? yield* executionContext.exports
+                exports: runtimeContext.exports
+                  ? yield* runtimeContext.exports
                   : undefined,
               };
 
               return Object.assign(instance, {
-                ExecutionContext: executionContext,
+                RuntimeContext: runtimeContext,
               }) as R;
             }),
           ),