alchemy 0.24.0 → 0.25.0

package/src/cloudflare/do-state-store/types.ts

@@ -0,0 +1,74 @@
+import type { State } from "../../state.ts";
+
+export namespace DOStateStoreAPI {
+  export type API = {
+    validate: {
+      method: "validate";
+      params: null;
+      result: null;
+    };
+    get: {
+      method: "get";
+      params: { key: string };
+      result: string | undefined;
+    };
+    getBatch: {
+      method: "getBatch";
+      params: { keys: string[] };
+      result: Record<string, string>;
+    };
+    all: {
+      method: "all";
+      params: { prefix: string };
+      result: Record<string, string>;
+    };
+    set: {
+      method: "set";
+      params: { key: string; value: State };
+      result: null;
+    };
+    delete: {
+      method: "delete";
+      params: { key: string };
+      result: null;
+    };
+    list: {
+      method: "list";
+      params: { prefix: string };
+      result: string[];
+    };
+    count: {
+      method: "count";
+      params: { prefix: string };
+      result: number;
+    };
+  };
+
+  export interface Context {
+    app: string;
+    stage: string;
+  }
+
+  export type Request = {
+    [K in keyof API]: {
+      method: K;
+      params: API[K]["params"];
+    };
+  }[keyof API];
+
+  export interface SuccessResponse<T extends keyof API = keyof API> {
+    success: true;
+    status: number;
+    result: API[T]["result"];
+  }
+
+  export interface ErrorResponse {
+    success: false;
+    status: number;
+    error: string;
+  }
+
+  export type Response<T extends keyof API = keyof API> =
+    | SuccessResponse<T>
+    | ErrorResponse;
+}

package/src/cloudflare/do-state-store/worker.ts

@@ -0,0 +1,199 @@
+import { DurableObject, WorkerEntrypoint } from "cloudflare:workers";
+import { Fs } from "dofs";
+import { timingSafeEqual } from "node:crypto";
+import type { State } from "../../state.ts";
+import type { DOStateStoreAPI } from "./types.ts";
+
+interface Env {
+  DOFS_STATE_STORE: DurableObjectNamespace<DOFSStateStore>;
+  DOFS_TOKEN: string;
+}
+
+export default class extends WorkerEntrypoint<Env> {
+  override async fetch(request: Request) {
+    try {
+      const result = await this.handle(request);
+      const body: DOStateStoreAPI.Response = {
+        success: true,
+        status: 200,
+        result: result ?? undefined,
+      };
+      return Response.json(body, { status: body.status });
+    } catch (error) {
+      return APIError.fromUnknown(error).toResponse();
+    }
+  }
+
+  private async handle(request: Request) {
+    if (!this.authorize(request)) {
+      throw new APIError("Unauthorized", 401);
+    }
+    const stub = this.getStub(request);
+    const body = await this.parseBody(request);
+    switch (body.method) {
+      case "validate":
+        return null;
+      case "all":
+        return await stub.all(body.params.prefix);
+      case "count":
+        return await stub.count(body.params.prefix);
+      case "delete":
+        return await stub.delete(body.params.key);
+      case "get":
+        return await stub.get(body.params.key);
+      case "getBatch":
+        return await stub.getBatch(body.params.keys);
+      case "list":
+        return await stub.list(body.params.prefix);
+      case "set":
+        return await stub.set(body.params.key, body.params.value);
+      default: {
+        const _: never = body;
+        throw new APIError("Method not found", 404);
+      }
+    }
+  }
+
+  private authorize(request: Request): boolean {
+    const token = request.headers.get("Authorization")?.split(" ")[1];
+    if (!token) {
+      return false;
+    }
+    return timingSafeEqual(
+      new TextEncoder().encode(token),
+      new TextEncoder().encode(this.env.DOFS_TOKEN),
+    );
+  }
+
+  private getStub(request: Request): DurableObjectStub<DOFSStateStore> {
+    const url = new URL(request.url);
+    const app = url.searchParams.get("app");
+    const stage = url.searchParams.get("stage");
+    if (!app || !stage) {
+      throw new APIError("Missing app or stage", 400);
+    }
+    return this.env.DOFS_STATE_STORE.get(
+      this.env.DOFS_STATE_STORE.idFromName(`${app}/${stage}`),
+    );
+  }
+
+  private async parseBody(request: Request): Promise<DOStateStoreAPI.Request> {
+    try {
+      return await request.json<DOStateStoreAPI.Request>();
+    } catch {
+      throw new APIError("Invalid JSON", 400);
+    }
+  }
+}
+
+class APIError extends Error {
+  constructor(
+    message: string,
+    public readonly status: number,
+  ) {
+    super(message);
+  }
+
+  toResponse(): Response {
+    return Response.json(
+      {
+        success: false,
+        status: this.status,
+        error: this.message,
+      },
+      { status: this.status },
+    );
+  }
+
+  static fromUnknown(error: unknown): APIError {
+    if (error instanceof APIError) {
+      return error;
+    }
+    if (error instanceof Error) {
+      return new APIError(error.message, 500);
+    }
+    return new APIError("An unknown error occurred.", 500);
+  }
+}
+
+export class DOFSStateStore extends DurableObject<Env> {
+  fs = new Fs(this.ctx, this.env, {
+    chunkSize: 256 * 1024,
+  });
+
+  async all(prefix: string): Promise<Record<string, string>> {
+    const keys = this.list(prefix);
+    return await this.getBatch(keys);
+  }
+
+  count(prefix: string): number {
+    return this.list(prefix).length;
+  }
+
+  delete(key: string): void {
+    try {
+      this.fs.unlink(key);
+    } catch (error) {
+      if (isErrorCode(error, "ENOENT")) {
+        return;
+      }
+      throw error;
+    }
+  }
+
+  async get(key: string): Promise<string | undefined> {
+    try {
+      const file = this.fs.readFile(key);
+      return new Response(file).text();
+    } catch (error) {
+      if (isErrorCode(error, "ENOENT")) {
+        return undefined;
+      }
+      throw error;
+    }
+  }
+
+  async getBatch(keys: string[]): Promise<Record<string, string>> {
+    const result: Record<string, string> = {};
+    await Promise.all(
+      keys.map(async (key) => {
+        const value = await this.get(key);
+        if (value) {
+          result[key] = value;
+        }
+      }),
+    );
+    return result;
+  }
+
+  list(prefix: string): string[] {
+    try {
+      return this.fs
+        .listDir(prefix, { recursive: true })
+        .filter((item) => item !== "." && item !== "..");
+    } catch (error) {
+      if (isErrorCode(error, "ENOENT")) {
+        return [];
+      }
+      throw error;
+    }
+  }
+
+  async set(key: string, value: State): Promise<void> {
+    this.ensureDir(key);
+    await this.fs.writeFile(key, JSON.stringify(value));
+  }
+
+  private ensureDir(path: string): void {
+    const dir = path.split("/").slice(0, -1).join("/");
+    try {
+      this.fs.mkdir(dir, { recursive: true });
+    } catch {
+      // directory already exists, ignore
+    }
+  }
+}
+
+const isErrorCode = (error: unknown, code: string) => {
+  return error instanceof Error && "message" in error && error.message === code;
+};

package/src/cloudflare/index.ts

@@ -18,6 +18,7 @@ export * from "./d1-database.ts";
 export * from "./d1-export.ts";
 export * from "./d1-import.ts";
 export * from "./dns-records.ts";
+export * from "./do-state-store/index.ts";
 export * from "./durable-object-namespace.ts";
 export * from "./hyperdrive.ts";
 export * from "./kv-namespace.ts";

package/src/cloudflare/pipeline.ts

@@ -364,7 +364,7 @@ const PipelineResource = Resource("cloudflare::Pipeline", async function <
     // Update operation
     if (this.output?.id) {
       // Check if name is being changed, which is not allowed
-      if (props.name !== this.output.name) {
+      if (pipelineName !== this.output.name) {
         throw new Error(
           "Cannot update Pipeline name after creation. Pipeline name is immutable.",
         );

package/src/cloudflare/queue-consumer.ts

@@ -6,7 +6,7 @@ import {
   type CloudflareApi,
   type CloudflareApiOptions,
 } from "./api.ts";
-import type { QueueResource, Queue } from "./queue.ts";
+import type { Queue, QueueResource } from "./queue.ts";
 
 /**
  * Settings for configuring a Queue Consumer
@@ -450,3 +450,61 @@ export async function listQueueConsumers(
       : undefined,
   }));
 }
+
+export async function listQueueConsumersForWorker(
+  api: CloudflareApi,
+  workerName: string,
+) {
+  const response = await api.get(
+    `/accounts/${api.accountId}/workers/scripts/${workerName}/queue-consumers?perPage=100`,
+  );
+
+  if (response.status === 404) {
+    return [];
+  }
+
+  if (!response.ok) {
+    return await handleApiError(
+      response,
+      "list",
+      "QueueConsumer",
+      `for worker ${workerName}`,
+    );
+  }
+
+  const data = (await response.json()) as {
+    result: Array<{
+      script: string;
+      settings?: {
+        batch_size?: number;
+        max_retries?: number;
+        max_wait_time_ms?: number;
+        retry_delay?: number;
+        dead_letter_queue?: string;
+      };
+      type: string;
+      queue_name: string;
+      queue_id: string;
+      consumer_id: string;
+      created_on: string;
+    }>;
+    success: boolean;
+    errors: any[] | null;
+    messages: any[] | null;
+    result_info: {
+      page: number;
+      per_page: number;
+      count: number;
+      total_count: number;
+      total_pages: number;
+    };
+  };
+
+  return data.result.map((consumer) => ({
+    queueName: consumer.queue_name,
+    queueId: consumer.queue_id,
+    consumerId: consumer.consumer_id,
+    createdOn: consumer.created_on,
+    settings: consumer.settings,
+  }));
+}

package/src/cloudflare/website.ts

@@ -15,8 +15,37 @@ export interface WebsiteProps<B extends Bindings>
   extends Omit<WorkerProps<B>, "name" | "assets" | "entrypoint"> {
   /**
    * The command to run to build the site
+   *
+   * If one is not provided, the build is assumed to have already happened.
+   */
+  command?: string;
+
+  /**
+   * Whether to memoize the command (only re-run if the command changes)
+   *
+   * When set to `true`, the command will only be re-executed if the command string changes.
+   *
+   * When set to an object with `patterns`, the command will be re-executed if either:
+   * 1. The command string changes, or
+   * 2. The contents of any files matching the glob patterns change
+   *
+   * ⚠️ **Important Note**: When using memoization with build commands, the build outputs
+   * will not be produced if the command is memoized. This is because the command is not
+   * actually executed when memoized. Consider disabling memoization in CI environments:
+   *
+   * @example
+   * // Disable memoization in CI to ensure build outputs are always produced
+   * await Website("my-website", {
+   *   command: "vite build",
+   *   memoize: process.env.CI ? false : {
+   *     patterns: ["./src/**"]
+   *   }
+   * });
+   *
+   * @default false
    */
-  command: string;
+  memoize?: boolean | { patterns: string[] };
+
   /**
    * The name of the worker
    *
@@ -128,11 +157,14 @@ export default {
       });
     }
 
-    await Exec("build", {
-      cwd,
-      command: props.command,
-      env: props.env,
-    });
+    if (props.command) {
+      await Exec("build", {
+        cwd,
+        command: props.command,
+        env: props.env,
+        memoize: props.memoize,
+      });
+    }
 
     return (await Worker("worker", {
       ...workerProps,

package/src/cloudflare/worker-assets.ts

@@ -109,8 +109,8 @@ export async function uploadAssets(
   for (const bucket of buckets) {
     const formData = new FormData();
 
-    // Add each file in the bucket to the form
-    for (const fileHash of bucket) {
+    // Prepare file reading operations for parallel execution
+    const fileReadPromises = bucket.map(async (fileHash) => {
       // Find the file with this hash
       const file = assets.files.find((f) => {
         const filePath = f.path.startsWith("/") ? f.path : `/${f.path}`;
@@ -127,10 +127,22 @@ export async function uploadAssets(
       // Convert to base64 as required by the API when using base64=true
       const base64Content = fileContent.toString("base64");
 
+      return {
+        fileHash,
+        base64Content,
+        filePath: file.filePath,
+      };
+    });
+
+    // Read all files in parallel
+    const fileResults = await Promise.all(fileReadPromises);
+
+    // Add each file to the form
+    for (const { fileHash, base64Content, filePath } of fileResults) {
       // Add the file to the form with the hash as the key and set the correct content type
       const blob = new Blob([base64Content], {
         // if you set application/octet-stream, you get weird `-1` errors with a message to contact support
-        type: getContentType(file.filePath) ?? "application/null",
+        type: getContentType(filePath) ?? "application/null",
       });
       formData.append(fileHash, blob, fileHash);
     }

package/src/cloudflare/worker-metadata.ts

@@ -361,7 +361,7 @@ export async function prepareWorkerMetadata<B extends Bindings>(
       meta.bindings.push({
         type: "service",
         name: bindingName,
-        service: binding.name,
+        service: "service" in binding ? binding.service : binding.name,
       });
     } else if (binding.type === "durable_object_namespace") {
       meta.bindings.push({

package/src/cloudflare/worker.ts

@@ -23,6 +23,7 @@ import {
   type Binding,
   type Bindings,
   Json,
+  type WorkerBindingService,
   type WorkerBindingSpec,
 } from "./bindings.ts";
 import type { Bound } from "./bound.ts";
@@ -46,7 +47,7 @@ import { isPipeline } from "./pipeline.ts";
 import {
   QueueConsumer,
   deleteQueueConsumer,
-  listQueueConsumers,
+  listQueueConsumersForWorker,
 } from "./queue-consumer.ts";
 import { type QueueResource, isQueue } from "./queue.ts";
 import { isVectorizeIndex } from "./vectorize-index.ts";
@@ -384,6 +385,56 @@ export type Worker<
     compatibilityFlags: string[];
   };
 
+/**
+ * Represents a reference to a Cloudflare Worker service.
+ *
+ * @template RPC - The type of the worker's RPC entrypoint, defaults to Rpc.WorkerEntrypointBranded.
+ *
+ * This interface extends all properties of WorkerBindingService except for "name".
+ * It also includes an optional __rpc__ property for type branding.
+ */
+export type WorkerRef<
+  RPC extends Rpc.WorkerEntrypointBranded = Rpc.WorkerEntrypointBranded,
+> = Omit<WorkerBindingService, "name"> & {
+  /**
+   * Optional type branding for the worker's RPC entrypoint.
+   */
+  __rpc__?: RPC;
+};
+
+/**
+ * Creates a reference to a Cloudflare Worker service.
+ *
+ * @example
+ * // Create a reference to a Cloudflare Worker service:
+ * const ref = WorkerRef({
+ *   service: "my-worker",
+ *   environment: "production",
+ *   namespace: "main"
+ * });
+ *
+ * // Optionally, you can specify only the service:
+ * const ref2 = WorkerRef({ service: "my-worker" });
+ *
+ * // You can also specify the RPC type for stronger typing:
+ * interface MyWorkerRPC extends Rpc.WorkerEntrypointBranded {
+ *   myMethod(arg: string): Promise<number>;
+ * }
+ * const typedRef = WorkerRef<MyWorkerRPC>({ service: "my-worker" });
+ */
+export function WorkerRef<
+  RPC extends Rpc.WorkerEntrypointBranded = Rpc.WorkerEntrypointBranded,
+>(options?: {
+  service: string;
+  environment?: string;
+  namespace?: string;
+}): WorkerRef<RPC> {
+  return {
+    ...options,
+    type: "service",
+  } as WorkerRef<RPC>;
+}
+
 /**
  * A Cloudflare Worker is a serverless function that can be deployed to the Cloudflare network.
  *
@@ -869,7 +920,7 @@ export const _Worker = Resource(
 
     if (this.phase === "delete") {
       // Delete any queue consumers attached to this worker first
-      await deleteQueueConsumers(this, api, workerName);
+      await deleteQueueConsumers(api, workerName);
 
       // @ts-ignore
       await uploadWorkerScript({
@@ -879,7 +930,7 @@ export const _Worker = Resource(
         script:
           props.format === "cjs"
             ? "addEventListener('fetch', event => { event.respondWith(new Response('hello world')); });"
-            : "export default { fetch(request) { return new Response('hello world'); } }",
+            : "export default { fetch(request) { return new Response('hello world'); }, queue: () => {} }",
         bindings: {} as B,
         // we are writing a stub worker (to remove binding/event source dependencies)
         // queue consumers will no longer exist by this point
@@ -1074,7 +1125,14 @@ export async function putWorker(
 
       return formData;
     },
-    (err) => err.status === 404 || err.status === 500 || err.status === 503,
+    (err) =>
+      err.status === 404 ||
+      err.status === 500 ||
+      err.status === 503 ||
+      // this is a tranient error that cloudflare throws randomly
+      (err instanceof CloudflareApiError &&
+        err.status === 400 &&
+        err.message.match(/binding.*failed to generate/)),
     10,
     100,
   );
@@ -1248,53 +1306,15 @@ async function getWorkerBindings(api: CloudflareApi, workerName: string) {
  * @param api CloudflareApi instance
  * @param workerName Name of the worker script
  */
-async function deleteQueueConsumers<B extends Bindings>(
-  ctx: Context<Worker<B>>,
+async function deleteQueueConsumers(
   api: CloudflareApi,
   workerName: string,
 ): Promise<void> {
-  const eventSources = ctx.output?.eventSources || [];
-
-  // Extract queue IDs from event sources
-  const queueIds = eventSources.flatMap((eventSource) => {
-    if (isQueue(eventSource)) {
-      return [eventSource.id];
-    }
-    if (isQueueEventSource(eventSource)) {
-      return [eventSource.queue.id];
-    }
-    return [];
-  });
+  const consumers = await listQueueConsumersForWorker(api, workerName);
 
-  // Process each queue associated with this worker
   await Promise.all(
-    queueIds.map(async (queueId) => {
-      try {
-        // List all consumers for this queue
-        const consumers = await listQueueConsumers(api, queueId);
-
-        // Filter consumers by worker name
-        const workerConsumers = consumers.filter(
-          (consumer) => consumer.scriptName === workerName,
-        );
-
-        // Delete all consumers for this worker in parallel
-        await Promise.all(
-          workerConsumers.map(async (consumer) => {
-            console.log(
-              `Deleting queue consumer ${consumer.id} for worker ${workerName}`,
-            );
-            // Use the deleteQueueConsumer function from queue-consumer.ts
-            await deleteQueueConsumer(api, consumer.queueId, consumer.id);
-          }),
-        );
-      } catch (err) {
-        if (err instanceof CloudflareApiError && err.status === 404) {
-          // this is OK
-        } else {
-          throw err;
-        }
-      }
+    consumers.map(async (consumer) => {
+      await deleteQueueConsumer(api, consumer.queueId, consumer.consumerId);
     }),
   );
 }

package/src/cloudflare/wrangler.json.ts

@@ -447,7 +447,7 @@ function processBindings(
       // Service binding
       services.push({
         binding: bindingName,
-        service: binding.name,
+        service: "name" in binding ? binding.name : binding.service,
       });
     } else if (binding.type === "kv_namespace") {
       // KV Namespace binding