alchemy-effect 0.2.0 → 0.4.0

package/lib/apply.js

@@ -1,45 +1,51 @@
 import * as Context from "effect/Context";
 import * as Effect from "effect/Effect";
-import * as Option from "effect/Option";
-import { PlanReviewer } from "./approve.js";
+import { App } from "./app.js";
+import { CLI, } from "./cli/service.js";
+import { generateInstanceId } from "./instance-id.js";
+import * as Output from "./output.js";
 import { plan, } from "./plan.js";
-import { State } from "./state.js";
-export class PlanStatusReporter extends Context.Tag("PlanStatusReporter")() {
-}
-export const apply = (...args) => Effect.isEffect(args[0])
-    ? applyPlan(args[0])
-    : args.length === 1 && "phase" in args[0]
-        ? applyResourcesPhase(args[0])
-        : applyResources(...args);
-export const applyResourcesPhase = (props) => applyPlan(plan(props));
-export const applyResources = (...resources) => applyPlan(plan({
-    phase: "update",
-    resources,
-}));
-export const applyPlan = (plan) => plan.pipe(Effect.flatMap((plan) => Effect.gen(function* () {
+import { State, StateStoreError, } from "./state.js";
+import { asEffect } from "./util.js";
+import { getProviderByType } from "./provider.js";
+export const apply = (...resources) => plan(...resources).pipe(Effect.flatMap((p) => applyPlan(p)));
+export const applyPlan = (plan) => Effect.gen(function* () {
+    const cli = yield* CLI;
+    const session = yield* cli.startApplySession(plan);
+    // 1. expand the graph (create new resources, update existing and create replacements)
+    const resources = yield* expandAndPivot(plan, session);
+    // TODO(sam): support roll back to previous state if errors occur during expansion
+    // -> RISK: some UPDATEs may not be reverisble (i.e. trigger replacements)
+    // TODO(sam): should pivot be done separately? E.g shift traffic?
+    // 2. delete orphans and replaced resources
+    yield* collectGarbage(plan, session);
+    yield* session.done();
+    if (Object.keys(plan.resources).length === 0) {
+        // all resources are deleted, return undefined
+        return undefined;
+    }
+    return resources;
+});
+const expandAndPivot = Effect.fnUntraced(function* (plan, session) {
     const state = yield* State;
+    const app = yield* App;
     const outputs = {};
-    const reviewer = yield* Effect.serviceOption(PlanReviewer);
-    if (Option.isSome(reviewer)) {
-        yield* reviewer.value.approve(plan);
-    }
-    const events = yield* Effect.serviceOption(PlanStatusReporter);
-    const session = Option.isSome(events)
-        ? yield* events.value.start(plan)
-        : {
-            emit: () => Effect.void,
-            done: () => Effect.void,
+    const resolveUpstream = Effect.fn(function* (resourceId) {
+        const upstreamNode = plan.resources[resourceId];
+        const upstreamAttr = upstreamNode
+            ? yield* apply(upstreamNode)
+            : yield* Effect.dieMessage(`Resource ${resourceId} not found`);
+        return {
+            resourceId,
+            upstreamAttr,
+            upstreamNode,
         };
-    const { emit, done } = session;
-    const constOrEffect = (effect) => Effect.isEffect(effect) ? effect : Effect.succeed(effect);
-    const resolveBindingUpstream = Effect.fn(function* ({ node, resource, }) {
+    });
+    const resolveBindingUpstream = Effect.fn(function* ({ node, }) {
         const binding = node.binding;
         const provider = yield* binding.Tag;
         const resourceId = node.binding.capability.resource.id;
-        const upstreamNode = plan.resources[resourceId];
-        const upstreamAttr = resource
-            ? yield* apply(upstreamNode)
-            : yield* Effect.dieMessage(`Resource ${resourceId} not found`);
+        const { upstreamAttr, upstreamNode } = yield* resolveUpstream(resourceId);
         return {
             resourceId,
             upstreamAttr,
@@ -60,14 +66,14 @@ export const applyPlan = (plan) => plan.pipe(Effect.flatMap((plan) => Effect.gen
             target,
         };
         if (node.action === "attach") {
-            return yield* constOrEffect(provider.attach(input));
+            return yield* asEffect(provider.attach(input));
         }
         else if (node.action === "reattach") {
             // reattach is optional, we fall back to attach if it's not available
-            return yield* constOrEffect((provider.reattach ? provider.reattach : provider.attach)(input));
+            return yield* asEffect((provider.reattach ? provider.reattach : provider.attach)(input));
         }
         else if (node.action === "detach" && provider.detach) {
-            return yield* constOrEffect(provider.detach({
+            return yield* asEffect(provider.detach({
                 ...input,
                 target,
             }));
@@ -79,7 +85,7 @@ export const applyPlan = (plan) => plan.pipe(Effect.flatMap((plan) => Effect.gen
         const oldBindingOutput = bindingOutputs[i];
         if (provider.postattach &&
             (node.action === "attach" || node.action === "reattach")) {
-            const bindingOutput = yield* constOrEffect(provider.postattach({
+            const bindingOutput = yield* asEffect(provider.postattach({
                 source: {
                     id: resourceId,
                     attr: upstreamAttr,
@@ -97,17 +103,12 @@ export const applyPlan = (plan) => plan.pipe(Effect.flatMap((plan) => Effect.gen
         return oldBindingOutput;
     })));
     const apply = (node) => Effect.gen(function* () {
-        const checkpoint = (effect) => effect.pipe(Effect.flatMap((output) => saveState({ output })));
-        const saveState = ({ output, bindings = node.bindings, }) => state
-            .set(node.resource.id, {
-            id: node.resource.id,
-            type: node.resource.type,
-            status: node.action === "create" ? "created" : "updated",
-            props: node.resource.props,
-            output,
-            bindings,
-        })
-            .pipe(Effect.map(() => output));
+        const commit = (value) => state.set({
+            stack: app.name,
+            stage: app.stage,
+            resourceId: node.resource.id,
+            value,
+        });
         const id = node.resource.id;
         const resource = node.resource;
         const scopedSession = {
@@ -119,159 +120,443 @@ export const applyPlan = (plan) => plan.pipe(Effect.flatMap((plan) => Effect.gen
             }),
         };
         return yield* (outputs[id] ??= yield* Effect.cached(Effect.gen(function* () {
-            const report = (status) => emit({
+            const report = (status) => session.emit({
                 kind: "status-change",
                 id,
                 type: node.resource.type,
                 status,
             });
-            const createOrUpdate = Effect.fn(function* ({ node, attr, phase, }) {
-                yield* report(phase === "create" ? "creating" : "updating");
+            if (node.action === "noop") {
+                return node.state.attr;
+            }
+            // resolve upstream dependencies before committing any changes to state
+            const upstream = Object.fromEntries(yield* Effect.all(Object.entries(Output.resolveUpstream(node.props)).map(([id]) => resolveUpstream(id).pipe(Effect.map(({ upstreamAttr }) => [id, upstreamAttr])))));
+            const instanceId = yield* Effect.gen(function* () {
+                if (node.action === "create" && !node.state?.instanceId) {
+                    const instanceId = yield* generateInstanceId();
+                    yield* commit({
+                        status: "creating",
+                        instanceId,
+                        logicalId: id,
+                        downstream: node.downstream,
+                        props: node.props,
+                        providerVersion: node.provider.version ?? 0,
+                        resourceType: node.resource.type,
+                        bindings: node.bindings,
+                    });
+                    return instanceId;
+                }
+                else if (node.action === "replace") {
+                    if (node.state.status === "replaced" ||
+                        node.state.status === "replacing") {
+                        // replace has already begun and we have the new instanceId, do not re-create it
+                        return node.state.instanceId;
+                    }
+                    const instanceId = yield* generateInstanceId();
+                    yield* commit({
+                        status: "replacing",
+                        instanceId,
+                        logicalId: id,
+                        downstream: node.downstream,
+                        props: node.props,
+                        providerVersion: node.provider.version ?? 0,
+                        resourceType: node.resource.type,
+                        bindings: node.bindings,
+                        old: node.state,
+                        deleteFirst: node.deleteFirst,
+                    });
+                    return instanceId;
+                }
+                else if (node.state?.instanceId) {
+                    // we're in a create, update or delete state with a stable instanceId, use it
+                    return node.state.instanceId;
+                }
+                // this should never happen
+                return yield* Effect.dieMessage(`Instance ID not found for resource '${id}' and action is '${node.action}'`);
+            });
+            if (node.action === "create") {
+                const news = (yield* Output.evaluate(node.props, upstream));
+                const checkpoint = (attr) => commit({
+                    status: "creating",
+                    logicalId: id,
+                    instanceId,
+                    resourceType: node.resource.type,
+                    props: news,
+                    attr,
+                    providerVersion: node.provider.version ?? 0,
+                    bindings: node.bindings,
+                    downstream: node.downstream,
+                });
+                if (!node.state) {
+                    yield* checkpoint(undefined);
+                }
+                let attr;
+                if (node.action === "create" &&
+                    node.provider.precreate &&
+                    // pre-create is only designed to ensure the resource exists, if we have state.attr, then it already exists and should be skipped
+                    node.state?.attr === undefined) {
+                    yield* report("pre-creating");
+                    // stub the resource prior to resolving upstream resources or bindings if a stub is available
+                    attr = yield* node.provider.precreate({
+                        id,
+                        news: node.props,
+                        session: scopedSession,
+                        instanceId,
+                    });
+                    yield* checkpoint(attr);
+                }
+                yield* report("attaching");
                 let bindingOutputs = yield* attachBindings({
                     resource,
                     bindings: node.bindings,
                     target: {
                         id,
-                        props: node.news,
+                        props: news,
                         attr,
                     },
                 });
-                const output = yield* (phase === "create"
-                    ? node.provider.create
-                    : node.provider.update)({
+                yield* report("creating");
+                attr = yield* node.provider.create({
                     id,
-                    news: node.news,
+                    news,
+                    instanceId,
                     bindings: bindingOutputs,
                     session: scopedSession,
-                    ...(node.action === "update"
-                        ? {
-                            output: node.output,
-                            olds: node.olds,
-                        }
-                        : {}),
-                }).pipe(
-                // TODO(sam): partial checkpoints
-                // checkpoint,
-                Effect.tap(() => report(phase === "create" ? "created" : "updated")));
+                });
+                yield* checkpoint(attr);
+                yield* report("post-attach");
                 bindingOutputs = yield* postAttachBindings({
                     resource,
                     bindings: node.bindings,
                     bindingOutputs,
                     target: {
                         id,
-                        props: node.news,
+                        props: news,
                         attr,
                     },
                 });
-                yield* saveState({
-                    output,
+                yield* commit({
+                    status: "created",
+                    logicalId: id,
+                    instanceId,
+                    resourceType: node.resource.type,
+                    props: news,
+                    attr,
                     bindings: node.bindings.map((binding, i) => ({
                         ...binding,
                         attr: bindingOutputs[i],
                     })),
+                    providerVersion: node.provider.version ?? 0,
+                    downstream: node.downstream,
                 });
-                return output;
-            });
-            if (node.action === "noop") {
-                return (yield* state.get(id))?.output;
+                yield* report("created");
+                return attr;
+            }
+            else if (node.action === "update") {
+                const upstream = Object.fromEntries(yield* Effect.all(Object.entries(Output.resolveUpstream(node.props)).map(([id]) => resolveUpstream(id).pipe(Effect.map(({ upstreamAttr }) => [id, upstreamAttr])))));
+                const news = (yield* Output.evaluate(node.props, upstream));
+                const checkpoint = (attr) => {
+                    if (node.state.status === "replaced") {
+                        return commit({
+                            ...node.state,
+                            attr,
+                            props: news,
+                        });
+                    }
+                    else {
+                        return commit({
+                            status: "updating",
+                            logicalId: id,
+                            instanceId,
+                            resourceType: node.resource.type,
+                            props: news,
+                            attr,
+                            providerVersion: node.provider.version ?? 0,
+                            bindings: node.bindings,
+                            downstream: node.downstream,
+                            old: node.state.status === "updating"
+                                ? node.state.old
+                                : node.state,
+                        });
+                    }
+                };
+                yield* checkpoint(node.state.attr);
+                yield* report("attaching");
+                let bindingOutputs = yield* attachBindings({
+                    resource,
+                    bindings: node.bindings,
+                    target: {
+                        id,
+                        props: news,
+                        attr: node.state.attr,
+                    },
+                });
+                yield* report("updating");
+                const attr = yield* node.provider.update({
+                    id,
+                    news,
+                    instanceId,
+                    bindings: bindingOutputs,
+                    session: scopedSession,
+                    olds: node.state.status === "created" ||
+                        node.state.status === "updated" ||
+                        node.state.status === "replaced"
+                        ? node.state.props
+                        : node.state.old.props,
+                    output: node.state.attr,
+                });
+                yield* checkpoint(attr);
+                yield* report("post-attach");
+                bindingOutputs = yield* postAttachBindings({
+                    resource,
+                    bindings: node.bindings,
+                    bindingOutputs,
+                    target: {
+                        id,
+                        props: news,
+                        attr,
+                    },
+                });
+                if (node.state.status === "replaced") {
+                    yield* commit({
+                        ...node.state,
+                        attr,
+                        props: news,
+                    });
+                }
+                else {
+                    yield* commit({
+                        status: "updated",
+                        logicalId: id,
+                        instanceId,
+                        resourceType: node.resource.type,
+                        props: news,
+                        attr,
+                        bindings: node.bindings.map((binding, i) => ({
+                            ...binding,
+                            attr: bindingOutputs[i],
+                        })),
+                        providerVersion: node.provider.version ?? 0,
+                        downstream: node.downstream,
+                    });
+                }
+                yield* report("updated");
+                return attr;
             }
-            else if (node.action === "create") {
+            else if (node.action === "replace") {
+                if (node.state.status === "replaced") {
+                    // we've already created the replacement resource, return the output
+                    return node.state.attr;
+                }
+                let state;
+                if (node.state.status !== "replacing") {
+                    yield* commit((state = {
+                        status: "replacing",
+                        logicalId: id,
+                        instanceId,
+                        resourceType: node.resource.type,
+                        props: node.props,
+                        attr: node.state.attr,
+                        providerVersion: node.provider.version ?? 0,
+                        deleteFirst: node.deleteFirst,
+                        old: node.state,
+                        downstream: node.downstream,
+                    }));
+                }
+                else {
+                    state = node.state;
+                }
+                const upstream = Object.fromEntries(yield* Effect.all(Object.entries(Output.resolveUpstream(node.props)).map(([id]) => resolveUpstream(id).pipe(Effect.map(({ upstreamAttr }) => [id, upstreamAttr])))));
+                const news = (yield* Output.evaluate(node.props, upstream));
+                const checkpoint = ({ status, attr, bindings, }) => commit({
+                    status,
+                    logicalId: id,
+                    instanceId,
+                    resourceType: node.resource.type,
+                    props: news,
+                    attr,
+                    providerVersion: node.provider.version ?? 0,
+                    bindings: bindings ?? node.bindings,
+                    downstream: node.downstream,
+                    old: state.old,
+                    deleteFirst: node.deleteFirst,
+                });
                 let attr;
-                if (node.provider.precreate) {
-                    yield* Effect.logDebug("precreate", id);
+                if (node.provider.precreate &&
+                    // pre-create is only designed to ensure the resource exists, if we have state.attr, then it already exists and should be skipped
+                    node.state?.attr === undefined) {
+                    yield* report("pre-creating");
                     // stub the resource prior to resolving upstream resources or bindings if a stub is available
                     attr = yield* node.provider.precreate({
                         id,
-                        news: node.news,
+                        news: node.props,
                         session: scopedSession,
+                        instanceId,
+                    });
+                    yield* checkpoint({
+                        status: "replacing",
+                        attr,
                     });
                 }
-                yield* Effect.logDebug("create", id);
-                return yield* createOrUpdate({
-                    node,
-                    attr,
-                    phase: "create",
-                });
-            }
-            else if (node.action === "update") {
-                yield* Effect.logDebug("update", id);
-                return yield* createOrUpdate({
-                    node,
-                    attr: node.attributes,
-                    phase: "update",
+                yield* report("attaching");
+                let bindingOutputs = yield* attachBindings({
+                    resource,
+                    bindings: node.bindings,
+                    target: {
+                        id,
+                        props: news,
+                        attr,
+                    },
                 });
-            }
-            else if (node.action === "delete") {
-                yield* Effect.logDebug("delete", id);
-                yield* Effect.all(node.downstream.map((dep) => dep in plan.resources
-                    ? apply(plan.resources[dep])
-                    : Effect.void));
-                yield* report("deleting");
-                return yield* node.provider
-                    .delete({
+                yield* report("creating replacement");
+                attr = yield* node.provider.create({
                     id,
-                    olds: node.olds,
-                    output: node.output,
+                    news,
+                    instanceId,
+                    bindings: bindingOutputs,
                     session: scopedSession,
-                    bindings: [],
-                })
-                    .pipe(Effect.flatMap(() => state.delete(id)), Effect.tap(() => report("deleted")));
-            }
-            else if (node.action === "replace") {
-                const destroy = Effect.gen(function* () {
-                    yield* report("deleting");
-                    return yield* node.provider.delete({
-                        id,
-                        olds: node.olds,
-                        output: node.output,
-                        session: scopedSession,
-                        bindings: [],
-                    });
                 });
-                const create = Effect.gen(function* () {
-                    yield* report("creating");
-                    return yield* (node.provider
-                        .create({
+                yield* checkpoint({
+                    status: "replacing",
+                    attr,
+                });
+                yield* report("post-attach");
+                bindingOutputs = yield* postAttachBindings({
+                    resource,
+                    bindings: node.bindings,
+                    bindingOutputs,
+                    target: {
                         id,
-                        news: node.news,
-                        // TODO(sam): these need to only include attach actions
-                        bindings: yield* attachBindings({
-                            resource,
-                            bindings: node.bindings,
-                            target: {
-                                id,
-                                props: node.news,
-                                attr: node.attributes,
-                            },
-                        }),
-                        session: scopedSession,
-                    })
-                        // TODO(sam): delete and create will conflict here, we need to extend the state store for replace
-                        .pipe(checkpoint, Effect.tap(() => report("created"))));
+                        props: news,
+                        attr,
+                    },
                 });
-                if (!node.deleteFirst) {
-                    yield* destroy;
-                    return outputs;
-                }
-                else {
-                    yield* destroy;
-                    return yield* create;
-                }
+                yield* checkpoint({
+                    status: "replaced",
+                    attr,
+                    bindings: node.bindings.map((binding, i) => ({
+                        ...binding,
+                        attr: bindingOutputs[i],
+                    })),
+                });
+                yield* report("created");
+                return attr;
             }
+            // @ts-expect-error
+            return yield* Effect.dieMessage(`Unknown action: ${node.action}`);
         })));
     });
-    const nodes = [
-        ...Object.entries(plan.resources),
-        ...Object.entries(plan.deletions),
-    ];
-    const resources = Object.fromEntries(yield* Effect.all(nodes.map(Effect.fn(function* ([id, node]) {
+    return Object.fromEntries(yield* Effect.all(Object.entries(plan.resources).map(Effect.fn(function* ([id, node]) {
         return [id, yield* apply(node)];
     }))));
-    yield* done();
-    if (Object.keys(plan.resources).length === 0) {
-        // all resources are deleted, return undefined
-        return undefined;
-    }
-    return resources;
-})));
+});
+const collectGarbage = Effect.fnUntraced(function* (plan, session) {
+    const state = yield* State;
+    const app = yield* App;
+    const deletions = {};
+    // delete all replaced resources
+    const replacedResources = yield* state.getReplacedResources({
+        stack: app.name,
+        stage: app.stage,
+    });
+    const deletionGraph = {
+        ...plan.deletions,
+        ...Object.fromEntries(replacedResources.map((replaced) => [replaced.logicalId, replaced])),
+    };
+    const deleteResource = Effect.fnUntraced(function* (node) {
+        const isDeleteNode = (node) => "action" in node;
+        const { logicalId, resourceType, instanceId, downstream, props, attr, provider, } = isDeleteNode(node)
+            ? {
+                logicalId: node.resource.id,
+                resourceType: node.resource.type,
+                instanceId: node.state.instanceId,
+                downstream: node.downstream,
+                props: node.state.props,
+                attr: node.state.attr,
+                provider: node.provider,
+            }
+            : {
+                logicalId: node.logicalId,
+                resourceType: node.old.resourceType,
+                instanceId: node.old.instanceId,
+                downstream: node.old.downstream,
+                props: node.old.props,
+                attr: node.old.attr,
+                provider: yield* getProviderByType(node.old.resourceType),
+            };
+        const commit = (value) => state.set({
+            stack: app.name,
+            stage: app.stage,
+            resourceId: logicalId,
+            value,
+        });
+        const report = (status) => session.emit({
+            kind: "status-change",
+            id: logicalId,
+            type: resourceType,
+            status,
+        });
+        const scopedSession = {
+            ...session,
+            note: (note) => session.emit({
+                id: logicalId,
+                kind: "annotate",
+                message: note,
+            }),
+        };
+        return yield* (deletions[logicalId] ??= yield* Effect.cached(Effect.gen(function* () {
+            yield* Effect.all(downstream.map((dep) => dep in deletionGraph
+                ? deleteResource(deletionGraph[dep])
+                : Effect.void));
+            yield* report("deleting");
+            if (isDeleteNode(node)) {
+                yield* commit({
+                    status: "deleting",
+                    logicalId,
+                    instanceId,
+                    resourceType,
+                    props,
+                    attr,
+                    downstream,
+                    providerVersion: provider.version ?? 0,
+                    bindings: node.bindings,
+                });
+            }
+            yield* provider.delete({
+                id: logicalId,
+                instanceId,
+                olds: props,
+                output: attr,
+                session: scopedSession,
+                bindings: [],
+            });
+            if (isDeleteNode(node)) {
+                // TODO(sam): should we commit a tombstone instead? and then clean up tombstones after all deletions are complete?
+                yield* state.delete({
+                    stack: app.name,
+                    stage: app.stage,
+                    resourceId: logicalId,
+                });
+                yield* report("deleted");
+            }
+            else {
+                yield* commit({
+                    status: "created",
+                    logicalId,
+                    instanceId,
+                    resourceType,
+                    props: node.props,
+                    attr: node.attr,
+                    providerVersion: provider.version ?? 0,
+                    downstream: node.downstream,
+                    bindings: node.bindings,
+                });
+                yield* report("replaced");
+            }
+        })));
+    });
+    yield* Effect.all(Object.values(deletionGraph)
+        .filter((node) => node !== undefined)
+        .map(deleteResource));
+});
 //# sourceMappingURL=apply.js.map