alchemy-effect 0.3.0 → 0.5.0

package/lib/apply.js

@@ -1,20 +1,36 @@
 import * as Context from "effect/Context";
 import * as Effect from "effect/Effect";
+import { App } from "./app.js";
+import { CLI, } from "./cli/service.js";
+import { generateInstanceId, InstanceId } from "./instance-id.js";
 import * as Output from "./output.js";
 import { plan, } from "./plan.js";
-import { State } from "./state.js";
-import { App } from "./app.js";
+import { State, StateStoreError, } from "./state.js";
 import { asEffect } from "./util.js";
-import { CLI } from "./cli/service.js";
-export const apply = (...resources) => plan(...resources).pipe(Effect.flatMap(applyPlan));
+import { getProviderByType } from "./provider.js";
+import { Layer } from "effect";
+export const apply = (...resources) => plan(...resources).pipe(Effect.flatMap((p) => applyPlan(p)));
 export const applyPlan = (plan) => Effect.gen(function* () {
+    const cli = yield* CLI;
+    const session = yield* cli.startApplySession(plan);
+    // 1. expand the graph (create new resources, update existing and create replacements)
+    const resources = yield* expandAndPivot(plan, session);
+    // TODO(sam): support roll back to previous state if errors occur during expansion
+    // -> RISK: some UPDATEs may not be reverisble (i.e. trigger replacements)
+    // TODO(sam): should pivot be done separately? E.g shift traffic?
+    // 2. delete orphans and replaced resources
+    yield* collectGarbage(plan, session);
+    yield* session.done();
+    if (Object.keys(plan.resources).length === 0) {
+        // all resources are deleted, return undefined
+        return undefined;
+    }
+    return resources;
+});
+const expandAndPivot = Effect.fnUntraced(function* (plan, session) {
     const state = yield* State;
-    // TODO(sam): rename terminology to Stack
     const app = yield* App;
     const outputs = {};
-    const cli = yield* CLI;
-    const session = yield* cli.startApplySession(plan);
-    const { emit, done } = session;
     const resolveUpstream = Effect.fn(function* (resourceId) {
         const upstreamNode = plan.resources[resourceId];
         const upstreamAttr = upstreamNode
@@ -88,21 +104,12 @@ export const applyPlan = (plan) => Effect.gen(function* () {
         return oldBindingOutput;
     })));
     const apply = (node) => Effect.gen(function* () {
-        const saveState = ({ output, bindings = node.bindings, news, }) => state
-            .set({
+        const commit = (value) => state.set({
             stack: app.name,
             stage: app.stage,
             resourceId: node.resource.id,
-            value: {
-                id: node.resource.id,
-                type: node.resource.type,
-                status: node.action === "create" ? "created" : "updated",
-                props: news,
-                output,
-                bindings,
-            },
-        })
-            .pipe(Effect.map(() => output));
+            value,
+        });
         const id = node.resource.id;
         const resource = node.resource;
         const scopedSession = {
@@ -114,169 +121,447 @@ export const applyPlan = (plan) => Effect.gen(function* () {
             }),
         };
         return yield* (outputs[id] ??= yield* Effect.cached(Effect.gen(function* () {
-            const report = (status) => emit({
+            const report = (status) => session.emit({
                 kind: "status-change",
                 id,
                 type: node.resource.type,
                 status,
             });
-            const createOrUpdate = Effect.fn(function* ({ node, attr, phase, }) {
-                const upstream = Object.fromEntries(yield* Effect.all(Object.entries(Output.resolveUpstream(node.news)).map(([id]) => resolveUpstream(id).pipe(Effect.map(({ upstreamAttr }) => [id, upstreamAttr])))));
-                const news = yield* Output.evaluate(node.news, upstream);
-                yield* report(phase === "create" ? "creating" : "updating");
-                let bindingOutputs = yield* attachBindings({
-                    resource,
-                    bindings: node.bindings,
-                    target: {
+            if (node.action === "noop") {
+                return node.state.attr;
+            }
+            // resolve upstream dependencies before committing any changes to state
+            const upstream = Object.fromEntries(yield* Effect.all(Object.entries(Output.resolveUpstream(node.props)).map(([id]) => resolveUpstream(id).pipe(Effect.map(({ upstreamAttr }) => [id, upstreamAttr])))));
+            const instanceId = yield* Effect.gen(function* () {
+                if (node.action === "create" && !node.state?.instanceId) {
+                    const instanceId = yield* generateInstanceId();
+                    yield* commit({
+                        status: "creating",
+                        instanceId,
+                        logicalId: id,
+                        downstream: node.downstream,
+                        props: node.props,
+                        providerVersion: node.provider.version ?? 0,
+                        resourceType: node.resource.type,
+                        bindings: node.bindings,
+                    });
+                    return instanceId;
+                }
+                else if (node.action === "replace") {
+                    if (node.state.status === "replaced" ||
+                        node.state.status === "replacing") {
+                        // replace has already begun and we have the new instanceId, do not re-create it
+                        return node.state.instanceId;
+                    }
+                    const instanceId = yield* generateInstanceId();
+                    yield* commit({
+                        status: "replacing",
+                        instanceId,
+                        logicalId: id,
+                        downstream: node.downstream,
+                        props: node.props,
+                        providerVersion: node.provider.version ?? 0,
+                        resourceType: node.resource.type,
+                        bindings: node.bindings,
+                        old: node.state,
+                        deleteFirst: node.deleteFirst,
+                    });
+                    return instanceId;
+                }
+                else if (node.state?.instanceId) {
+                    // we're in a create, update or delete state with a stable instanceId, use it
+                    return node.state.instanceId;
+                }
+                // this should never happen
+                return yield* Effect.dieMessage(`Instance ID not found for resource '${id}' and action is '${node.action}'`);
+            });
+            const apply = Effect.gen(function* () {
+                if (node.action === "create") {
+                    const news = (yield* Output.evaluate(node.props, upstream));
+                    const checkpoint = (attr) => commit({
+                        status: "creating",
+                        logicalId: id,
+                        instanceId,
+                        resourceType: node.resource.type,
+                        props: news,
+                        attr,
+                        providerVersion: node.provider.version ?? 0,
+                        bindings: node.bindings,
+                        downstream: node.downstream,
+                    });
+                    if (!node.state) {
+                        yield* checkpoint(undefined);
+                    }
+                    let attr;
+                    if (node.action === "create" &&
+                        node.provider.precreate &&
+                        // pre-create is only designed to ensure the resource exists, if we have state.attr, then it already exists and should be skipped
+                        node.state?.attr === undefined) {
+                        yield* report("pre-creating");
+                        // stub the resource prior to resolving upstream resources or bindings if a stub is available
+                        attr = yield* node.provider.precreate({
+                            id,
+                            news: node.props,
+                            session: scopedSession,
+                            instanceId,
+                        });
+                        yield* checkpoint(attr);
+                    }
+                    yield* report("attaching");
+                    let bindingOutputs = yield* attachBindings({
+                        resource,
+                        bindings: node.bindings,
+                        target: {
+                            id,
+                            props: news,
+                            attr,
+                        },
+                    });
+                    yield* report("creating");
+                    attr = yield* node.provider.create({
                         id,
+                        news,
+                        instanceId,
+                        bindings: bindingOutputs,
+                        session: scopedSession,
+                    });
+                    yield* checkpoint(attr);
+                    yield* report("post-attach");
+                    bindingOutputs = yield* postAttachBindings({
+                        resource,
+                        bindings: node.bindings,
+                        bindingOutputs,
+                        target: {
+                            id,
+                            props: news,
+                            attr,
+                        },
+                    });
+                    yield* commit({
+                        status: "created",
+                        logicalId: id,
+                        instanceId,
+                        resourceType: node.resource.type,
                         props: news,
                         attr,
-                    },
-                });
-                const output = yield* (phase === "create" ? node.provider.create : node.provider.update)({
-                    id,
-                    news,
-                    bindings: bindingOutputs,
-                    session: scopedSession,
-                    ...(node.action === "update"
-                        ? {
-                            output: node.output,
-                            olds: node.olds,
+                        bindings: node.bindings.map((binding, i) => ({
+                            ...binding,
+                            attr: bindingOutputs[i],
+                        })),
+                        providerVersion: node.provider.version ?? 0,
+                        downstream: node.downstream,
+                    });
+                    yield* report("created");
+                    return attr;
+                }
+                else if (node.action === "update") {
+                    const upstream = Object.fromEntries(yield* Effect.all(Object.entries(Output.resolveUpstream(node.props)).map(([id]) => resolveUpstream(id).pipe(Effect.map(({ upstreamAttr }) => [id, upstreamAttr])))));
+                    const news = (yield* Output.evaluate(node.props, upstream));
+                    const checkpoint = (attr) => {
+                        if (node.state.status === "replaced") {
+                            return commit({
+                                ...node.state,
+                                attr,
+                                props: news,
+                            });
                         }
-                        : {}),
-                }).pipe(
-                // TODO(sam): partial checkpoints
-                // checkpoint,
-                Effect.tap(() => report(phase === "create" ? "created" : "updated")));
-                bindingOutputs = yield* postAttachBindings({
-                    resource,
-                    bindings: node.bindings,
-                    bindingOutputs,
-                    target: {
+                        else {
+                            return commit({
+                                status: "updating",
+                                logicalId: id,
+                                instanceId,
+                                resourceType: node.resource.type,
+                                props: news,
+                                attr,
+                                providerVersion: node.provider.version ?? 0,
+                                bindings: node.bindings,
+                                downstream: node.downstream,
+                                old: node.state.status === "updating"
+                                    ? node.state.old
+                                    : node.state,
+                            });
+                        }
+                    };
+                    yield* checkpoint(node.state.attr);
+                    yield* report("attaching");
+                    let bindingOutputs = yield* attachBindings({
+                        resource,
+                        bindings: node.bindings,
+                        target: {
+                            id,
+                            props: news,
+                            attr: node.state.attr,
+                        },
+                    });
+                    yield* report("updating");
+                    const attr = yield* node.provider.update({
                         id,
+                        news,
+                        instanceId,
+                        bindings: bindingOutputs,
+                        session: scopedSession,
+                        olds: node.state.status === "created" ||
+                            node.state.status === "updated" ||
+                            node.state.status === "replaced"
+                            ? node.state.props
+                            : node.state.old.props,
+                        output: node.state.attr,
+                    });
+                    yield* checkpoint(attr);
+                    yield* report("post-attach");
+                    bindingOutputs = yield* postAttachBindings({
+                        resource,
+                        bindings: node.bindings,
+                        bindingOutputs,
+                        target: {
+                            id,
+                            props: news,
+                            attr,
+                        },
+                    });
+                    if (node.state.status === "replaced") {
+                        yield* commit({
+                            ...node.state,
+                            attr,
+                            props: news,
+                        });
+                    }
+                    else {
+                        yield* commit({
+                            status: "updated",
+                            logicalId: id,
+                            instanceId,
+                            resourceType: node.resource.type,
+                            props: news,
+                            attr,
+                            bindings: node.bindings.map((binding, i) => ({
+                                ...binding,
+                                attr: bindingOutputs[i],
+                            })),
+                            providerVersion: node.provider.version ?? 0,
+                            downstream: node.downstream,
+                        });
+                    }
+                    yield* report("updated");
+                    return attr;
+                }
+                else if (node.action === "replace") {
+                    if (node.state.status === "replaced") {
+                        // we've already created the replacement resource, return the output
+                        return node.state.attr;
+                    }
+                    let state;
+                    if (node.state.status !== "replacing") {
+                        yield* commit((state = {
+                            status: "replacing",
+                            logicalId: id,
+                            instanceId,
+                            resourceType: node.resource.type,
+                            props: node.props,
+                            attr: node.state.attr,
+                            providerVersion: node.provider.version ?? 0,
+                            deleteFirst: node.deleteFirst,
+                            old: node.state,
+                            downstream: node.downstream,
+                        }));
+                    }
+                    else {
+                        state = node.state;
+                    }
+                    const upstream = Object.fromEntries(yield* Effect.all(Object.entries(Output.resolveUpstream(node.props)).map(([id]) => resolveUpstream(id).pipe(Effect.map(({ upstreamAttr }) => [id, upstreamAttr])))));
+                    const news = (yield* Output.evaluate(node.props, upstream));
+                    const checkpoint = ({ status, attr, bindings, }) => commit({
+                        status,
+                        logicalId: id,
+                        instanceId,
+                        resourceType: node.resource.type,
                         props: news,
                         attr,
-                    },
-                });
-                yield* saveState({
-                    news,
-                    output,
-                    bindings: node.bindings.map((binding, i) => ({
-                        ...binding,
-                        attr: bindingOutputs[i],
-                    })),
-                });
-                return output;
-            });
-            if (node.action === "noop") {
-                return (yield* state.get({
-                    stack: app.name,
-                    stage: app.stage,
-                    resourceId: id,
-                }))?.output;
-            }
-            else if (node.action === "create") {
-                let attr;
-                if (node.provider.precreate) {
-                    yield* Effect.logDebug("precreate", id);
-                    // stub the resource prior to resolving upstream resources or bindings if a stub is available
-                    attr = yield* node.provider.precreate({
+                        providerVersion: node.provider.version ?? 0,
+                        bindings: bindings ?? node.bindings,
+                        downstream: node.downstream,
+                        old: state.old,
+                        deleteFirst: node.deleteFirst,
+                    });
+                    let attr;
+                    if (node.provider.precreate &&
+                        // pre-create is only designed to ensure the resource exists, if we have state.attr, then it already exists and should be skipped
+                        node.state?.attr === undefined) {
+                        yield* report("pre-creating");
+                        // stub the resource prior to resolving upstream resources or bindings if a stub is available
+                        attr = yield* node.provider.precreate({
+                            id,
+                            news: node.props,
+                            session: scopedSession,
+                            instanceId,
+                        });
+                        yield* checkpoint({
+                            status: "replacing",
+                            attr,
+                        });
+                    }
+                    yield* report("attaching");
+                    let bindingOutputs = yield* attachBindings({
+                        resource,
+                        bindings: node.bindings,
+                        target: {
+                            id,
+                            props: news,
+                            attr,
+                        },
+                    });
+                    yield* report("creating replacement");
+                    attr = yield* node.provider.create({
                         id,
-                        news: node.news,
+                        news,
+                        instanceId,
+                        bindings: bindingOutputs,
                         session: scopedSession,
                     });
+                    yield* checkpoint({
+                        status: "replacing",
+                        attr,
+                    });
+                    yield* report("post-attach");
+                    bindingOutputs = yield* postAttachBindings({
+                        resource,
+                        bindings: node.bindings,
+                        bindingOutputs,
+                        target: {
+                            id,
+                            props: news,
+                            attr,
+                        },
+                    });
+                    yield* checkpoint({
+                        status: "replaced",
+                        attr,
+                        bindings: node.bindings.map((binding, i) => ({
+                            ...binding,
+                            attr: bindingOutputs[i],
+                        })),
+                    });
+                    yield* report("created");
+                    return attr;
                 }
-                yield* Effect.logDebug("create", id);
-                return yield* createOrUpdate({
-                    node,
-                    attr,
-                    phase: "create",
-                });
+                // @ts-expect-error
+                return yield* Effect.dieMessage(`Unknown action: ${node.action}`);
+            });
+            // provide the resource-specific context (InstanceId, etc.)
+            return yield* apply.pipe(Effect.provide(Layer.succeed(InstanceId, instanceId)));
+        })));
+    });
+    return Object.fromEntries(yield* Effect.all(Object.entries(plan.resources).map(Effect.fn(function* ([id, node]) {
+        return [id, yield* apply(node)];
+    }))));
+});
+const collectGarbage = Effect.fnUntraced(function* (plan, session) {
+    const state = yield* State;
+    const app = yield* App;
+    const deletions = {};
+    // delete all replaced resources
+    const replacedResources = yield* state.getReplacedResources({
+        stack: app.name,
+        stage: app.stage,
+    });
+    const deletionGraph = {
+        ...plan.deletions,
+        ...Object.fromEntries(replacedResources.map((replaced) => [replaced.logicalId, replaced])),
+    };
+    const deleteResource = Effect.fnUntraced(function* (node) {
+        const isDeleteNode = (node) => "action" in node;
+        const { logicalId, resourceType, instanceId, downstream, props, attr, provider, } = isDeleteNode(node)
+            ? {
+                logicalId: node.resource.id,
+                resourceType: node.resource.type,
+                instanceId: node.state.instanceId,
+                downstream: node.downstream,
+                props: node.state.props,
+                attr: node.state.attr,
+                provider: node.provider,
             }
-            else if (node.action === "update") {
-                yield* Effect.logDebug("update", id);
-                return yield* createOrUpdate({
-                    node,
-                    attr: node.attributes,
-                    phase: "update",
+            : {
+                logicalId: node.logicalId,
+                resourceType: node.old.resourceType,
+                instanceId: node.old.instanceId,
+                downstream: node.old.downstream,
+                props: node.old.props,
+                attr: node.old.attr,
+                provider: yield* getProviderByType(node.old.resourceType),
+            };
+        const commit = (value) => state.set({
+            stack: app.name,
+            stage: app.stage,
+            resourceId: logicalId,
+            value,
+        });
+        const report = (status) => session.emit({
+            kind: "status-change",
+            id: logicalId,
+            type: resourceType,
+            status,
+        });
+        const scopedSession = {
+            ...session,
+            note: (note) => session.emit({
+                id: logicalId,
+                kind: "annotate",
+                message: note,
+            }),
+        };
+        return yield* (deletions[logicalId] ??= yield* Effect.cached(Effect.gen(function* () {
+            yield* Effect.all(downstream.map((dep) => dep in deletionGraph
+                ? deleteResource(deletionGraph[dep])
+                : Effect.void));
+            yield* report("deleting");
+            if (isDeleteNode(node)) {
+                yield* commit({
+                    status: "deleting",
+                    logicalId,
+                    instanceId,
+                    resourceType,
+                    props,
+                    attr,
+                    downstream,
+                    providerVersion: provider.version ?? 0,
+                    bindings: node.bindings,
                 });
             }
-            else if (node.action === "delete") {
-                yield* Effect.logDebug("delete", id);
-                yield* Effect.all(node.downstream.map((dep) => dep in plan.resources
-                    ? apply(plan.resources[dep])
-                    : Effect.void));
-                yield* report("deleting");
-                return yield* node.provider
-                    .delete({
-                    id,
-                    olds: node.olds,
-                    output: node.output,
-                    session: scopedSession,
-                    bindings: [],
-                })
-                    .pipe(Effect.flatMap(() => state.delete({
+            yield* provider.delete({
+                id: logicalId,
+                instanceId,
+                olds: props,
+                output: attr,
+                session: scopedSession,
+                bindings: [],
+            });
+            if (isDeleteNode(node)) {
+                // TODO(sam): should we commit a tombstone instead? and then clean up tombstones after all deletions are complete?
+                yield* state.delete({
                     stack: app.name,
                     stage: app.stage,
-                    resourceId: id,
-                })), Effect.tap(() => report("deleted")));
-            }
-            else if (node.action === "replace") {
-                const destroy = Effect.gen(function* () {
-                    yield* report("deleting");
-                    return yield* node.provider.delete({
-                        id,
-                        olds: node.olds,
-                        output: node.output,
-                        session: scopedSession,
-                        bindings: [],
-                    });
+                    resourceId: logicalId,
                 });
-                const create = Effect.gen(function* () {
-                    yield* report("creating");
-                    // TODO(sam): delete and create will conflict here, we need to extend the state store for replace
-                    return yield* node.provider
-                        .create({
-                        id,
-                        news: node.news,
-                        // TODO(sam): these need to only include attach actions
-                        bindings: yield* attachBindings({
-                            resource,
-                            bindings: node.bindings,
-                            target: {
-                                id,
-                                // TODO(sam): resolve the news
-                                props: node.news,
-                                attr: node.attributes,
-                            },
-                        }),
-                        session: scopedSession,
-                    })
-                        .pipe(Effect.tap((output) => saveState({ news: node.news, output })));
+                yield* report("deleted");
+            }
+            else {
+                yield* commit({
+                    status: "created",
+                    logicalId,
+                    instanceId,
+                    resourceType,
+                    props: node.props,
+                    attr: node.attr,
+                    providerVersion: provider.version ?? 0,
+                    downstream: node.downstream,
+                    bindings: node.bindings,
                 });
-                if (!node.deleteFirst) {
-                    yield* destroy;
-                    return outputs;
-                }
-                else {
-                    yield* destroy;
-                    return yield* create;
-                }
+                yield* report("replaced");
             }
-        })));
+        }).pipe(Effect.provide(Layer.succeed(InstanceId, instanceId)))));
     });
-    const nodes = [
-        ...Object.entries(plan.resources),
-        ...Object.entries(plan.deletions),
-    ];
-    const resources = Object.fromEntries(yield* Effect.all(nodes.map(Effect.fn(function* ([id, node]) {
-        return [id, yield* apply(node)];
-    }))));
-    yield* done();
-    if (Object.keys(plan.resources).length === 0) {
-        // all resources are deleted, return undefined
-        return undefined;
-    }
-    return resources;
+    yield* Effect.all(Object.values(deletionGraph)
+        .filter((node) => node !== undefined)
+        .map(deleteResource));
 });
 //# sourceMappingURL=apply.js.map