alchemy 2.0.0-beta.23 → 2.0.0-beta.25

package/src/Cloudflare/StateStore/State.ts

@@ -7,6 +7,7 @@ import * as Redacted from "effect/Redacted";
 import * as HttpClient from "effect/unstable/http/HttpClient";
 
 import * as Config from "effect/Config";
+import { adopt } from "../../AdoptPolicy.ts";
 import { AuthError } from "../../Auth/AuthProvider.ts";
 import { readCredentials, writeCredentials } from "../../Auth/Credentials.ts";
 import { ALCHEMY_PROFILE } from "../../Auth/Profile.ts";
@@ -23,7 +24,7 @@ import * as Credentials from "../Credentials.ts";
 import { AlchemyContext } from "../../AlchemyContext.ts";
 import { makeLocalState } from "../../State/LocalState.ts";
 import { State, type StateService } from "../../State/State.ts";
-import { syncState } from "../../State/Sync.ts";
+import { recordStateStoreOp } from "../../Telemetry/Metrics.ts";
 import * as Clank from "../../Util/Clank.ts";
 import { CloudflareAuth } from "../Auth/AuthProvider.ts";
 import { EdgeSessionError, createEdgeSession } from "../EdgeSession.ts";
@@ -33,6 +34,105 @@ import { AuthTokenSecretName, TokenValue } from "./Token.ts";
 /** Filename used for stored credentials under the profile directory. */
 const CREDENTIALS_FILE = "cloudflare-state-store";
 
+export interface BootstrapOptions {
+  /**
+   * The name of the script to use for the state store.
+   * @default "alchemy-state-store"
+   */
+  workerName?: string;
+  /**
+   * Re-run the deploy even if the worker already exists in Cloudflare.
+   *
+   * Without this flag, an already-deployed worker is **adopted**: the
+   * auth-token secret is re-fetched via an edge probe and the local
+   * credentials file is refreshed, but no new code is uploaded. With
+   * `force: true` the full deploy runs again, redeploying the worker
+   * code and reconciling every resource in place.
+   */
+  force?: boolean;
+}
+
+/**
+ * Manually bootstrap (or repair) the Cloudflare-hosted HTTP State
+ * Store. Used by `alchemy bootstrap cloudflare`.
+ *
+ * The flow mirrors what {@link state} does on first use, but is
+ * exposed as a standalone effect so users can trigger it (or recover
+ * from a partially-failed previous deploy) explicitly:
+ *
+ *   1. Resume any in-progress local `CloudflareStateStore` stack.
+ *   2. If the worker already exists in Cloudflare and `force` is not
+ *      set, adopt it: read the auth token via an edge-preview probe
+ *      and persist credentials.
+ *   3. Otherwise run the full deploy with adoption enabled, sync local
+ *      state into the deployed store, persist credentials, and delete
+ *      the local stack.
+ */
+export const bootstrap = (options: BootstrapOptions = {}) =>
+  Effect.gen(function* () {
+    const isCI = yield* Config.boolean("CI").pipe(Config.withDefault(false));
+    const profileName = yield* ALCHEMY_PROFILE;
+    const scriptName = options.workerName ?? STATE_STORE_SCRIPT_NAME;
+    const force = options.force ?? false;
+
+    const localState = yield* makeLocalState();
+    const hasLocalStack = yield* Effect.map(
+      localState.listStages("CloudflareStateStore"),
+      (stages) => stages.includes(scriptName),
+    );
+
+    if (hasLocalStack) {
+      yield* Clank.info(
+        `Resuming Cloudflare State Store '${scriptName}' deployment...`,
+      );
+      yield* finishBootstrap({ scriptName, profileName, localState, isCI });
+      yield* Clank.success(
+        `Cloudflare State Store '${scriptName}' is ready.`,
+      );
+      return;
+    }
+
+    const { accountId } = yield* CloudflareEnvironment.CloudflareEnvironment;
+    const workerExists = yield* workers
+      .getScriptSetting({ accountId, scriptName })
+      .pipe(
+        Effect.map((setting) => setting !== undefined),
+        Effect.catchTag("WorkerNotFound", () => Effect.succeed(false)),
+      );
+
+    if (workerExists && !force) {
+      yield* Clank.info(
+        `Worker '${scriptName}' already exists; adopting and refreshing credentials. ` +
+          `Use --force to redeploy.`,
+      );
+      const credentials = yield* loginWithCloudflare();
+      if (!isCI) {
+        yield* writeCredentials<HttpStateStoreProps>(
+          profileName,
+          CREDENTIALS_FILE,
+          credentials,
+        );
+      }
+      yield* Clank.success(
+        `Cloudflare State Store '${scriptName}' is ready.`,
+      );
+      return;
+    }
+
+    if (workerExists) {
+      yield* Clank.info(
+        `Forcing redeploy of Cloudflare State Store '${scriptName}'...`,
+      );
+    } else {
+      yield* Clank.info(
+        `Deploying Cloudflare State Store '${scriptName}'...`,
+      );
+    }
+
+    yield* finishBootstrap({ scriptName, profileName, localState, isCI });
+    yield* Clank.success(`Cloudflare State Store '${scriptName}' is ready.`);
+  });
+
 export const state = (props?: {
   /**
    * The name of the script to use for the state store.
@@ -53,6 +153,38 @@ export const state = (props?: {
 
       const scriptName = props?.workerName ?? STATE_STORE_SCRIPT_NAME;
 
+      // The bootstrap of the Cloudflare State Store is only considered
+      // successful once two invariants hold:
+      //   1. the worker has been deployed and the credentials persisted
+      //   2. the local CloudflareStateStore stack has been synced into
+      //      the HTTP store and removed from disk
+      //
+      // If a previous run failed anywhere between deploying the worker
+      // and deleting the local stack (e.g. the worker got created but
+      // sync/credential-write/delete never ran) we will detect the
+      // leftover local stack here and resume the bootstrap. This loop
+      // re-runs deploy (idempotent over already-applied resources) and
+      // finishes the sync/credential/cleanup steps until both
+      // invariants are satisfied.
+      const localState = yield* makeLocalState();
+
+      const hasLocalStack = yield* Effect.map(
+        localState.listStages("CloudflareStateStore"),
+        (stages) => stages.includes(scriptName),
+      );
+
+      if (hasLocalStack) {
+        yield* Clank.info(
+          `Resuming Cloudflare State Store '${scriptName}' deployment...`,
+        );
+        return yield* finishBootstrap({
+          scriptName,
+          profileName,
+          localState,
+          isCI,
+        });
+      }
+
       // TODO(sam): support upgrading state store, right now we only deploy once
       if (credentials) {
         // it's in the profile, let's go
@@ -63,79 +195,63 @@ export const state = (props?: {
         }
 
         return httpState;
-      } else {
-        // our profile does not contain a reference to the state store, let's try and resolve it
-        const { accountId } =
-          yield* CloudflareEnvironment.CloudflareEnvironment;
-        const workerExists = yield* workers
-          .getScriptSetting({
-            accountId,
-            scriptName,
-          })
-          .pipe(
-            Effect.map((setting) => setting !== undefined),
-            Effect.catchTag("WorkerNotFound", () => Effect.succeed(false)),
-          );
+      }
+
+      // our profile does not contain a reference to the state store, let's try and resolve it
+      const { accountId } = yield* CloudflareEnvironment.CloudflareEnvironment;
+      const workerExists = yield* workers
+        .getScriptSetting({
+          accountId,
+          scriptName,
+        })
+        .pipe(
+          Effect.map((setting) => setting !== undefined),
+          Effect.catchTag("WorkerNotFound", () => Effect.succeed(false)),
+        );
 
-        if (workerExists) {
-          // it exists, so fetch the secret token
-          const credentials = yield* loginWithCloudflare();
-          if (!isCI) {
-            yield* writeCredentials<HttpStateStoreProps>(
-              profileName,
-              CREDENTIALS_FILE,
-              credentials,
-            );
-          }
-          const httpState = yield* makeHttpStateStore(credentials);
-
-          if (alchemyContext.updateStateStore) {
-            yield* deployStateStore(scriptName, httpState);
-          }
-
-          return httpState;
-        } else if (isCI) {
-          // TODO(sam): do we want to support bootstrapping the state store from CI?
-          // for now - just die here
-          return yield* Effect.die(
-            new AuthError({
-              message: `State store not found for script ${scriptName}. Deploy the state store first.`,
-            }),
+      if (workerExists) {
+        // it exists, so fetch the secret token
+        const credentials = yield* loginWithCloudflare();
+        if (!isCI) {
+          yield* writeCredentials<HttpStateStoreProps>(
+            profileName,
+            CREDENTIALS_FILE,
+            credentials,
           );
         }
+        const httpState = yield* makeHttpStateStore(credentials);
 
-        const shouldDeploy = yield* Clank.confirm({
-          message:
-            "Cloudflare State Store not found. Do you want to deploy it?",
-        });
-
-        if (!shouldDeploy) {
-          return yield* Effect.die(new Clank.PromptCancelled());
+        if (alchemyContext.updateStateStore) {
+          yield* deployStateStore(scriptName, httpState);
         }
 
-        const { url, authToken, localState } =
-          yield* deployStateStore(scriptName);
-
-        const httpState = yield* makeHttpStateStore({ url, authToken });
-
-        yield* syncState(localState, httpState);
+        return httpState;
+      }
 
-        yield* writeCredentials<HttpStateStoreProps>(
-          profileName,
-          CREDENTIALS_FILE,
-          {
-            url,
-            authToken,
-          },
+      if (isCI) {
+        // TODO(sam): do we want to support bootstrapping the state store from CI?
+        // for now - just die here
+        return yield* Effect.die(
+          new AuthError({
+            message: `State store not found for script ${scriptName}. Deploy the state store first.`,
+          }),
         );
+      }
 
-        yield* localState.deleteStack({
-          stack: "CloudflareStateStore",
-          stage: scriptName,
-        });
+      const shouldDeploy = yield* Clank.confirm({
+        message: "Cloudflare State Store not found. Do you want to deploy it?",
+      });
 
-        return httpState;
+      if (!shouldDeploy) {
+        return yield* Effect.die(new Clank.PromptCancelled());
       }
+
+      return yield* finishBootstrap({
+        scriptName,
+        profileName,
+        localState,
+        isCI,
+      });
     }).pipe(Effect.orDie),
   ).pipe(
     Layer.provideMerge(Credentials.fromAuthProvider()),
@@ -144,6 +260,93 @@ export const state = (props?: {
     Layer.orDie,
   );
 
+/**
+ * Non-destructively copy every resource in the
+ * `CloudflareStateStore/<scriptName>` stack from `source` into
+ * `destination`, leaving every other stack in `destination` untouched.
+ *
+ * This intentionally does not delete anything from `destination`: at
+ * bootstrap time the destination is the user's live remote state
+ * store, and removing entries that happen to be missing locally would
+ * be catastrophic.
+ */
+const hoistBootstrapStack = Effect.fnUntraced(function* (
+  source: StateService,
+  destination: StateService,
+  scriptName: string,
+) {
+  const stack = "CloudflareStateStore";
+  const stage = scriptName;
+  const fqns = yield* source.list({ stack, stage });
+  yield* Effect.forEach(
+    fqns,
+    Effect.fnUntraced(function* (fqn) {
+      const value = yield* source.get({ stack, stage, fqn });
+      if (value) {
+        yield* destination.set({ stack, stage, fqn, value });
+      }
+    }),
+    { concurrency: "unbounded" },
+  );
+});
+
+/**
+ * Finish (or resume) the bootstrap of the Cloudflare State Store using
+ * the provided local state as the source of truth. This is idempotent
+ * and safe to re-run: any resources already applied during a previous
+ * partial run will be reconciled, the local stack will be synced into
+ * the deployed HTTP store, credentials persisted, and finally the
+ * local stack removed - which is what marks the bootstrap as complete.
+ */
+const finishBootstrap = ({
+  scriptName,
+  profileName,
+  localState,
+  isCI,
+}: {
+  scriptName: string;
+  profileName: string;
+  localState: StateService;
+  isCI: boolean;
+}) =>
+  Effect.gen(function* () {
+    yield* deployStateStore(scriptName, localState);
+
+    // Don't trust the `authToken` returned by `deploy(...)`: when
+    // adoption kicks in (the Secrets Store secret already existed),
+    // the locally-generated `Random` value won't match the value
+    // actually persisted in Cloudflare, and any HTTP call to the
+    // worker would 401. Re-read the live token from the deployed
+    // worker via the same edge-preview probe `loginWithCloudflare`
+    // uses, so the credentials we persist always reflect what is
+    // actually deployed. `loginWithCloudflare` also persists the
+    // credentials file (skipping the write in CI), so we don't need
+    // to do that explicitly here.
+    const { url, authToken } = yield* loginWithCloudflare();
+    const httpState = yield* makeHttpStateStore({ url, authToken });
+    // `profileName` is intentionally unused here — `loginWithCloudflare`
+    // resolves it itself. Reference it to keep the surrounding API
+    // explicit and to avoid an unused-parameter lint.
+    void profileName;
+    void isCI;
+
+    // Copy the freshly-deployed CloudflareStateStore stack into the
+    // remote store. We deliberately do NOT call `syncState` here:
+    // syncState is a *mirror* that deletes everything in the
+    // destination that is not in the source, which would wipe every
+    // user stack in the remote store on any subsequent bootstrap
+    // (resume / --force / partial-failure recovery), since by that
+    // point local state only contains the bootstrap stack itself.
+    yield* hoistBootstrapStack(localState, httpState, scriptName);
+
+    yield* localState.deleteStack({
+      stack: "CloudflareStateStore",
+      stage: scriptName,
+    });
+
+    return httpState;
+  });
+
 const deployStateStore = (scriptName: string, state?: StateService) =>
   Effect.gen(function* () {
     const localState = state ?? (yield* makeLocalState());
@@ -172,11 +375,25 @@ const deployStateStore = (scriptName: string, state?: StateService) =>
         }),
       ),
     }).pipe(
+      // The Cloudflare State Store is account-level infrastructure that
+      // outlives any single deploy: its underlying Secrets Store and
+      // auth-token secret may already exist from a previous (possibly
+      // partially-failed) bootstrap. Opt in to adoption so the
+      // resources reconcile in place instead of failing on conflict.
+      adopt(true),
       // TODO(sam): we should not need to do this, but types do complain. fix deploy
       Effect.provide(stateLayer),
     );
     return { url, authToken, localState };
-  });
+  }).pipe(
+    Effect.withSpan("state_store.deploy", {
+      attributes: {
+        "alchemy.state_store.script_name": scriptName,
+        "alchemy.state_store.op": "deploy",
+      },
+    }),
+    recordStateStoreOp("deploy"),
+  );
 
 /**
  * Log in to a Cloudflare-deployed HTTP state-store.

package/src/Cloudflare/StateStore/index.ts

@@ -1 +1 @@
-export { state } from "./State.ts";
+export { bootstrap, state, type BootstrapOptions } from "./State.ts";

package/src/Cloudflare/Workers/Compatibility.ts

@@ -0,0 +1,12 @@
+import type { WorkerProps } from "./Worker.ts";
+
+// TODO: figure out why the later one from workerd breaks
+const DEFAULT_COMPATIBILITY_DATE = "2026-03-17";
+
+export const getCompatibility = (props: WorkerProps) => ({
+  date: props.compatibility?.date ?? DEFAULT_COMPATIBILITY_DATE,
+  flags: [
+    ...(props.compatibility?.flags ?? []),
+    ...(props.isExternal ? [] : ["nodejs_compat"]),
+  ].filter((value, index, self) => self.indexOf(value) === index),
+});

package/src/Cloudflare/Workers/LocalWorkerProvider.ts

@@ -0,0 +1,93 @@
+import * as Effect from "effect/Effect";
+import * as Redacted from "effect/Redacted";
+import * as Provider from "../../Provider.ts";
+import type { ResourceBinding } from "../../Resource.ts";
+import { Stack } from "../../Stack.ts";
+import { CloudflareEnvironment } from "../CloudflareEnvironment.ts";
+import { Sidecar } from "../Local/Sidecar.ts";
+import { getCompatibility } from "./Compatibility.ts";
+import { Worker, type WorkerBinding, type WorkerProps } from "./Worker.ts";
+import { createWorkerName } from "./WorkerName.ts";
+
+export const LocalWorkerProvider = () =>
+  Provider.effect(
+    Worker,
+    Effect.gen(function* () {
+      const { accountId } = yield* CloudflareEnvironment;
+      const stack = yield* Stack;
+      const sidecar = yield* Sidecar;
+
+      const run = Effect.fn(function* (
+        id: string,
+        props: WorkerProps,
+        bindings: ResourceBinding<Worker["Binding"]>[],
+      ) {
+        const name = yield* createWorkerName(id, props.name);
+        const workerBindings: WorkerBinding[] = [];
+        const durableObjectNamespaces: Record<string, string> = {};
+        for (const { sid, data } of bindings) {
+          for (const binding of data.bindings ?? []) {
+            workerBindings.push(binding);
+            if (binding.type === "durable_object_namespace") {
+              durableObjectNamespaces[binding.name] = sid;
+            }
+          }
+        }
+        for (const [key, value] of Object.entries(props.env ?? {})) {
+          if (Redacted.isRedacted(value)) {
+            workerBindings.push({
+              type: "secret_text",
+              name: key,
+              text: Redacted.value(value),
+            });
+          } else {
+            workerBindings.push({
+              type: "plain_text",
+              name: key,
+              text: value,
+            });
+          }
+        }
+        const result = yield* sidecar.serve({
+          id,
+          name,
+          main: props.main,
+          compatibility: getCompatibility(props),
+          entry: props.isExternal
+            ? {
+                kind: "external",
+              }
+            : {
+                kind: "effect",
+                exports: (props.exports ?? {}) as any,
+              },
+          stack: { name: stack.name, stage: stack.stage },
+          bindings: workerBindings,
+          durableObjectNamespaces: Object.entries(durableObjectNamespaces).map(
+            ([className, namespaceId]) => ({
+              className,
+              uniqueKey: namespaceId,
+              sql: true,
+            }),
+          ),
+        });
+        return {
+          workerId: name,
+          workerName: name,
+          logpush: undefined,
+          url: result.address,
+          tags: [],
+          durableObjectNamespaces,
+          domains: [],
+          accountId,
+        } satisfies Worker["Attributes"];
+      });
+
+      return {
+        diff: () => Effect.succeed({ action: "update" }),
+        create: ({ id, news, bindings }) => run(id, news, bindings),
+        update: ({ id, news, bindings }) => run(id, news, bindings),
+        delete: ({ output }) => sidecar.stop(output.workerName),
+      };
+    }),
+  );