alchemy 2.0.0-beta.52 → 2.0.0-beta.54

package/src/Cloudflare/Local.ts

@@ -3,10 +3,10 @@ import * as RpcServer from "../Local/RpcServer.ts";
 import { CloudflareAuth } from "./Auth/AuthProvider.ts";
 import * as CloudflareEnvironment from "./CloudflareEnvironment.ts";
 import * as Credentials from "./Credentials.ts";
-import {
-  LocalWorkerProvider,
-  localRuntimeServices,
-} from "./Workers/LocalWorkerProvider.ts";
+import { localRuntimeServices } from "./LocalRuntime.ts";
+import { QueueProviderLocal } from "./Queue/Queue.ts";
+import { QueueConsumerProviderLocal } from "./Queue/QueueConsumer.ts";
+import { LocalWorkerProvider } from "./Workers/LocalWorkerProvider.ts";
 
 const cloudflareServices = Layer.provide(
   Layer.merge(
@@ -16,7 +16,11 @@ const cloudflareServices = Layer.provide(
   CloudflareAuth,
 );
 
-LocalWorkerProvider().pipe(
+Layer.mergeAll(
+  LocalWorkerProvider(),
+  QueueProviderLocal(),
+  QueueConsumerProviderLocal(),
+).pipe(
   Layer.provide(localRuntimeServices()),
   Layer.provide(cloudflareServices),
   RpcServer.launch,

package/src/Cloudflare/LocalRuntime.ts

@@ -0,0 +1,70 @@
+import { layerRuntime } from "@distilled.cloud/cloudflare-runtime";
+import * as Context from "effect/Context";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as MutableHashMap from "effect/MutableHashMap";
+import * as Path from "effect/Path";
+import { AlchemyContext } from "../AlchemyContext.ts";
+import * as RpcProvider from "../Local/RpcProvider.ts";
+import { CloudflareEnvironment } from "./CloudflareEnvironment.ts";
+import type { Queue } from "./Queue/Queue.ts";
+import type { QueueConsumer } from "./Queue/QueueConsumer.ts";
+
+export const LOCAL_ENTRY_URL = import.meta.resolve(
+  // `import.meta.resolve(<string>)` is a runtime API — TypeScript's
+  // `rewriteRelativeImportExtensions` does NOT touch the string literal, so
+  // we have to pick the right extension ourselves. `import.meta.url` reflects
+  // the actual on-disk extension of *this* file (`.ts` when loaded from
+  // `src/` under Bun or vitest, `.js` when loaded from the compiled `lib/`
+  // under Node), which is exactly the signal we need.
+  import.meta.url.endsWith(".ts") ? "./Local.ts" : "./Local.js",
+  import.meta.url,
+);
+
+export class LocalRuntimeState extends Context.Service<
+  LocalRuntimeState,
+  {
+    readonly queues: MutableHashMap.MutableHashMap<
+      Queue["Attributes"]["queueId"],
+      Queue["Attributes"]
+    >;
+    readonly queueConsumers: MutableHashMap.MutableHashMap<
+      QueueConsumer["Attributes"]["consumerId"],
+      QueueConsumer["Attributes"]
+    >;
+  }
+>()("alchemy/cloudflare/LocalRuntimeState") {}
+
+const LocalRuntimeStateLive = Layer.succeed(
+  LocalRuntimeState,
+  LocalRuntimeState.of({
+    queues: MutableHashMap.empty(),
+    queueConsumers: MutableHashMap.empty(),
+  }),
+);
+
+export const localRuntimeServices = () =>
+  RpcProvider.providerServicesEffect(
+    Effect.gen(function* () {
+      const getEnv = yield* CloudflareEnvironment;
+      const { dotAlchemy } = yield* AlchemyContext;
+      const path = yield* Path.Path;
+      return Layer.merge(
+        LocalRuntimeStateLive,
+        layerRuntime({
+          api: {
+            accountId: getEnv.pipe(Effect.map((env) => env.accountId)),
+          },
+          storage: {
+            directory: path.join(dotAlchemy, "local"),
+          },
+        }),
+      );
+    }),
+  );
+
+export const isLocalId = (id: string | undefined): id is string =>
+  typeof id === "string" && id.startsWith("dev:");
+export const isLiveId = (id: string | undefined): id is string =>
+  typeof id === "string" && !id.startsWith("dev:");
+export const generateLocalId = (): string => `dev:${crypto.randomUUID()}`;

package/src/Cloudflare/Providers.ts

@@ -29,6 +29,7 @@ import * as Email from "./Email/index.ts";
 import * as Hyperdrive from "./Hyperdrive/index.ts";
 import * as Images from "./Images/index.ts";
 import * as KV from "./KV/index.ts";
+import { localRuntimeServices } from "./LocalRuntime.ts";
 import * as Queue from "./Queue/index.ts";
 import * as R2 from "./R2/index.ts";
 import * as RateLimit from "./RateLimit/index.ts";
@@ -173,6 +174,7 @@ export const providers = () =>
         RandomProvider(),
       ),
     ),
+    Layer.provideMerge(localRuntimeServices()),
     Layer.provideMerge(Credentials.fromAuthProvider()),
     Layer.provideMerge(CloudflareEnvironment.fromProfile()),
     Layer.provideMerge(CloudflareAuth),

package/src/Cloudflare/Queue/Queue.ts

@@ -1,12 +1,21 @@
 import * as queues from "@distilled.cloud/cloudflare/queues";
 import * as Effect from "effect/Effect";
+import * as MutableHashMap from "effect/MutableHashMap";
 import * as Option from "effect/Option";
 import * as Stream from "effect/Stream";
 import { isResolved } from "../../Diff.ts";
+import * as ProviderLayer from "../../Local/ProviderLayer.ts";
+import * as RpcProvider from "../../Local/RpcProvider.ts";
 import { createPhysicalName } from "../../PhysicalName.ts";
 import * as Provider from "../../Provider.ts";
 import { Resource } from "../../Resource.ts";
 import { CloudflareEnvironment } from "../CloudflareEnvironment.ts";
+import {
+  generateLocalId,
+  isLiveId,
+  LOCAL_ENTRY_URL,
+  LocalRuntimeState,
+} from "../LocalRuntime.ts";
 import type { Providers } from "../Providers.ts";
 import { QueueBinding } from "./QueueBinding.ts";
 
@@ -96,132 +105,197 @@ export const Queue = Resource<Queue>("Cloudflare.Queue")({
   bind: QueueBinding.bind,
 });
 
-export const QueueProvider = () =>
-  Provider.effect(
-    Queue,
-    Effect.gen(function* () {
-      const { accountId } = yield* CloudflareEnvironment;
-      const createQueue = yield* queues.createQueue;
-      const getQueue = yield* queues.getQueue;
-      const deleteQueue = yield* queues.deleteQueue;
+export const QueueProviderLive = () =>
+  Provider.succeed(Queue, {
+    // The `queueId` is not marked as stable because if you start in dev mode, the ID will change on first deploy.
+    stables: ["accountId"],
+    diff: Effect.fn(function* ({ id, olds = {}, news = {}, output }) {
+      const { accountId } = yield* yield* CloudflareEnvironment;
+      if (!isResolved(news)) return undefined;
+      // If the queueId is a `dev:` ID, we need to update to a live one.
+      // The live resource doesn't exist yet, so there's no need to replace even if the name or accountId changed.
+      if (!isLiveId(output?.queueId)) {
+        return { action: "update" };
+      }
+      if ((output?.accountId ?? accountId) !== accountId) {
+        return { action: "replace" } as const;
+      }
+      const name = yield* createQueueName(id, news.name);
+      const oldName = output?.queueName
+        ? output.queueName
+        : yield* createQueueName(id, olds.name);
+      if (name !== oldName) {
+        return { action: "replace" } as const;
+      }
+    }),
+    reconcile: Effect.fn(function* ({ id, news = {}, output }) {
+      const { accountId } = yield* yield* CloudflareEnvironment;
+      const queueName = yield* createQueueName(id, news.name);
+      const acct = output?.accountId ?? accountId;
 
-      const createQueueName = (id: string, name: string | undefined) =>
-        Effect.gen(function* () {
-          if (name) return name;
-          return (yield* createPhysicalName({
-            id,
-            maxLength: 63,
-          })).toLowerCase();
-        });
+      // Observe — re-fetch the cached queue; fall back to a name scan
+      // when the cached id is gone (out-of-band delete or partial
+      // state-persistence failure).
+      let observed:
+        | { queueId?: string | null; queueName?: string | null }
+        | undefined;
+      if (output?.queueId) {
+        observed = yield* queues
+          .getQueue({
+            accountId: acct,
+            queueId: output.queueId,
+          })
+          .pipe(Effect.catch(() => Effect.succeed(undefined)));
+      }
+      if (!observed) {
+        observed = yield* findQueueByName(queueName);
+      }
 
-      // Cloudflare's `listQueues` accepts no name/prefix filter, so
-      // adoption-by-name has to scan every page. Use the paginated
-      // `.items` stream off the un-yielded operation method.
-      const findQueueByName = (queueName: string) =>
-        queues.listQueues.items({ accountId }).pipe(
-          Stream.filter((q) => q.queueName === queueName),
-          Stream.runHead,
-          Effect.map(Option.getOrUndefined),
-        );
+      // Ensure — create if missing. Cloudflare returns a generic
+      // failure when the queue name is taken; tolerate by adopting
+      // the queue with the same name so reconciles converge after a
+      // crashed peer.
+      if (!observed) {
+        observed = yield* queues
+          .createQueue({
+            accountId: acct,
+            queueName,
+          })
+          .pipe(
+            Effect.catch(() =>
+              Effect.gen(function* () {
+                const match = yield* findQueueByName(queueName);
+                if (match && match.queueId && match.queueName) {
+                  return match;
+                }
+                return yield* Effect.die(
+                  `Queue "${queueName}" already exists but could not be found`,
+                );
+              }),
+            ),
+          );
+      }
 
+      // Sync — Cloudflare Queues have no mutable per-queue settings
+      // here (the queue name itself is treated as a replace by diff),
+      // so observed state is the answer.
       return {
-        stables: ["queueId", "accountId"],
+        queueId: observed.queueId!,
+        queueName: observed.queueName!,
+        accountId: acct,
+      };
+    }),
+    delete: Effect.fn(function* ({ output }) {
+      // If the queueId is a `dev:` ID, the resource only exists locally, so we don't need to delete it from Cloudflare.
+      if (!isLiveId(output.queueId)) return;
+      yield* queues
+        .deleteQueue({
+          accountId: output.accountId,
+          queueId: output.queueId,
+        })
+        .pipe(Effect.catch(() => Effect.void));
+    }),
+    read: Effect.fn(function* ({ id, output, olds }) {
+      const { accountId } = yield* yield* CloudflareEnvironment;
+      if (output?.queueId) {
+        return yield* queues
+          .getQueue({
+            accountId: output.accountId,
+            queueId: output.queueId,
+          })
+          .pipe(
+            Effect.map((queue) => ({
+              queueId: queue.queueId!,
+              queueName: queue.queueName!,
+              accountId: output.accountId,
+            })),
+            Effect.catch(() => Effect.succeed(undefined)),
+          );
+      }
+      const queueName = yield* createQueueName(id, olds?.name);
+      const match = yield* findQueueByName(queueName);
+      if (match && match.queueId && match.queueName) {
+        return {
+          queueId: match.queueId,
+          queueName: match.queueName,
+          accountId,
+        };
+      }
+      return undefined;
+    }),
+  });
+
+const createQueueName = (id: string, name: string | undefined) =>
+  Effect.gen(function* () {
+    if (name) return name;
+    return (yield* createPhysicalName({
+      id,
+      maxLength: 63,
+    })).toLowerCase();
+  });
+
+// Cloudflare's `listQueues` accepts no name/prefix filter, so
+// adoption-by-name has to scan every page. Use the paginated
+// `.items` stream off the un-yielded operation method.
+const findQueueByName = Effect.fnUntraced(function* (queueName: string) {
+  const { accountId } = yield* yield* CloudflareEnvironment;
+  return yield* queues.listQueues.items({ accountId }).pipe(
+    Stream.filter((q) => q.queueName === queueName),
+    Stream.runHead,
+    Effect.map(Option.getOrUndefined),
+  );
+});
+
+export const QueueProviderLocal = () =>
+  RpcProvider.effect(
+    Queue,
+    LOCAL_ENTRY_URL,
+    Effect.gen(function* () {
+      const localRuntimeState = yield* LocalRuntimeState;
+      return {
+        stables: ["accountId"],
         diff: Effect.fn(function* ({ id, olds = {}, news = {}, output }) {
+          const { accountId } = yield* yield* CloudflareEnvironment;
+          if (!output?.queueId) return { action: "update" };
           if (!isResolved(news)) return undefined;
-          if ((output?.accountId ?? accountId) !== accountId) {
-            return { action: "replace" } as const;
-          }
           const name = yield* createQueueName(id, news.name);
           const oldName = output?.queueName
-            ? output.queueName
+            ? yield* createQueueName(id, olds.name)
             : yield* createQueueName(id, olds.name);
-          if (name !== oldName) {
-            return { action: "replace" } as const;
+          if (name !== oldName || output.accountId !== accountId) {
+            return { action: "replace" };
           }
+          // If the resource is a noop, add it to the local runtime state so it's available downstream.
+          // We do it here instead of in the reconcile function so it doesn't appear as an update.
+          MutableHashMap.set(localRuntimeState.queues, output.queueId, output);
+          return { action: "noop" };
+        }),
+        read: Effect.fn(function* ({ output }) {
+          if (!output?.queueId) return undefined;
+          return MutableHashMap.get(
+            localRuntimeState.queues,
+            output.queueId,
+          ).pipe(Option.getOrUndefined);
         }),
         reconcile: Effect.fn(function* ({ id, news = {}, output }) {
-          const queueName = yield* createQueueName(id, news.name);
-          const acct = output?.accountId ?? accountId;
-
-          // Observe — re-fetch the cached queue; fall back to a name scan
-          // when the cached id is gone (out-of-band delete or partial
-          // state-persistence failure).
-          let observed:
-            | { queueId?: string | null; queueName?: string | null }
-            | undefined;
-          if (output?.queueId) {
-            observed = yield* getQueue({
-              accountId: acct,
-              queueId: output.queueId,
-            }).pipe(Effect.catch(() => Effect.succeed(undefined)));
-          }
-          if (!observed) {
-            observed = yield* findQueueByName(queueName);
-          }
-
-          // Ensure — create if missing. Cloudflare returns a generic
-          // failure when the queue name is taken; tolerate by adopting
-          // the queue with the same name so reconciles converge after a
-          // crashed peer.
-          if (!observed) {
-            observed = yield* createQueue({
-              accountId: acct,
-              queueName,
-            }).pipe(
-              Effect.catch(() =>
-                Effect.gen(function* () {
-                  const match = yield* findQueueByName(queueName);
-                  if (match && match.queueId && match.queueName) {
-                    return match;
-                  }
-                  return yield* Effect.die(
-                    `Queue "${queueName}" already exists but could not be found`,
-                  );
-                }),
-              ),
-            );
-          }
-
-          // Sync — Cloudflare Queues have no mutable per-queue settings
-          // here (the queue name itself is treated as a replace by diff),
-          // so observed state is the answer.
-          return {
-            queueId: observed.queueId!,
-            queueName: observed.queueName!,
-            accountId: acct,
+          const { accountId } = yield* yield* CloudflareEnvironment;
+          const queue: Queue["Attributes"] = {
+            queueId: output?.queueId ?? generateLocalId(),
+            queueName: yield* createQueueName(id, news.name),
+            accountId: output?.accountId ?? accountId,
           };
+          MutableHashMap.set(localRuntimeState.queues, queue.queueId, queue);
+          return queue;
         }),
         delete: Effect.fn(function* ({ output }) {
-          yield* deleteQueue({
-            accountId: output.accountId,
-            queueId: output.queueId,
-          }).pipe(Effect.catch(() => Effect.void));
-        }),
-        read: Effect.fn(function* ({ id, output, olds }) {
-          if (output?.queueId) {
-            return yield* getQueue({
-              accountId: output.accountId,
-              queueId: output.queueId,
-            }).pipe(
-              Effect.map((queue) => ({
-                queueId: queue.queueId!,
-                queueName: queue.queueName!,
-                accountId: output.accountId,
-              })),
-              Effect.catch(() => Effect.succeed(undefined)),
-            );
-          }
-          const queueName = yield* createQueueName(id, olds?.name);
-          const match = yield* findQueueByName(queueName);
-          if (match && match.queueId && match.queueName) {
-            return {
-              queueId: match.queueId,
-              queueName: match.queueName,
-              accountId,
-            };
-          }
-          return undefined;
+          MutableHashMap.remove(localRuntimeState.queues, output.queueId);
         }),
       };
     }),
   );
+
+export const QueueProvider = () =>
+  ProviderLayer.select({
+    local: () => QueueProviderLocal(),
+    live: () => QueueProviderLive(),
+  });