alchemy 0.51.0 → 0.51.2

package/src/cloudflare/miniflare/miniflare-worker-proxy.ts

@@ -1,97 +1,168 @@
-import type http from "node:http";
+import * as miniflare from "miniflare";
+import { once } from "node:events";
+import http from "node:http";
+import type Stream from "node:stream";
+import { Readable } from "node:stream";
 import { WebSocket, WebSocketServer } from "ws";
-import { HTTPServer } from "../../util/http.ts";
+import { logger } from "../../util/logger.ts";
 
-export interface MiniflareWorkerProxyOptions {
-  /**
-   * Used to proxy websocket connections to the worker.
-   * (This is a hack; used because trying to interact with `response.webSocket` doesn't work on Bun.)
-   */
-  getDirectURL: () => Promise<URL>;
-  /** Used to proxy HTTP requests to the worker. */
-  fetch: (request: Request) => Promise<Response>;
+interface MiniflareWorkerProxyOptions {
+  name: string;
+  port: number;
+  miniflare: miniflare.Miniflare;
 }
 
-export class MiniflareWorkerProxy extends HTTPServer {
-  wsServer: WebSocketServer;
-  wsReconnectAttempts = new Map<string, number>();
+export class MiniflareWorkerProxy {
+  private server = http.createServer();
+  private wss = new WebSocketServer({ noServer: true });
 
   constructor(private readonly options: MiniflareWorkerProxyOptions) {
-    super({
-      fetch: options.fetch,
+    this.server.on("upgrade", async (req, socket, head) => {
+      await this.handleUpgrade(req, socket, head);
     });
-    this.wsServer = new WebSocketServer({ noServer: true });
-    this.httpServer.on("upgrade", async (req, socket, head) => {
-      this.wsServer.handleUpgrade(req, socket, head, async (client) => {
-        const id = crypto.randomUUID();
-        await this.handleUpgrade(id, client, req);
-      });
+    this.server.on("request", async (req, res) => {
+      await this.handleRequest(req, res);
     });
+    this.server.listen(this.options.port);
+  }
+
+  get ready() {
+    if (!this.server.listening) {
+      return once(this.server, "listening");
+    }
+    return Promise.resolve();
+  }
+
+  get url() {
+    return `http://localhost:${this.options.port}`;
+  }
+
+  async close() {
+    await Promise.all([
+      new Promise((resolve) => this.wss.close(resolve)),
+      new Promise((resolve) => this.server.close(resolve)),
+    ]);
   }
 
   private async handleUpgrade(
-    id: string,
-    client: WebSocket,
-    request: http.IncomingMessage,
+    req: http.IncomingMessage,
+    socket: Stream.Duplex,
+    head: Buffer,
   ) {
-    const attempt = this.wsReconnectAttempts.get(id) ?? 0;
-    this.wsReconnectAttempts.set(id, attempt + 1);
-    if (attempt > 3) {
-      client.close(1006, "Too many reconnect attempts");
+    const server = await this.createServerWebSocket(req);
+    if (!server) {
+      socket.destroy();
       return;
     }
-    if (attempt > 0) {
-      const delay = attempt * 1000;
-      await new Promise((resolve) => setTimeout(resolve, delay));
-    }
-
-    const target = await this.options.getDirectURL();
-    const url = new URL(request.url ?? "/", target);
-    const headers = { ...request.headers };
-    // All of these headers are set by the WebSocket constructor,
-    // so if we don't delete them, the request will fail.
-    delete headers.host;
-    delete headers.connection;
-    delete headers.upgrade;
-    delete headers["sec-websocket-version"];
-    delete headers["sec-websocket-key"];
-    delete headers["sec-websocket-protocol"];
-    delete headers["sec-websocket-extensions"];
-    delete headers["sec-websocket-accept"];
-    const server = new WebSocket(url.toString(), {
-      protocol: request.headers["sec-websocket-protocol"],
-      key: request.headers["sec-websocket-key"],
-      headers,
-    });
-    server.on("open", () => {
-      // Reset the reconnect attempts when the connection is established successfully.
-      this.wsReconnectAttempts.set(id, 0);
-    });
-    server.on("message", (data, binary) => {
-      client.send(data, { binary });
-    });
-    server.on("close", async (code, reason) => {
-      if (code === 1006) {
-        // When the worker hot reloads, the websocket connection is closed with this code.
-        // Reconnecting allows the client to maintain the same connection.
-        await this.handleUpgrade(id, client, request);
-        return;
-      }
-      client.close(code, reason);
-    });
-    client.on("message", (data, binary) => {
-      server.send(data, { binary });
-    });
-    client.on("close", (code, reason) => {
-      this.wsReconnectAttempts.delete(id);
-      if (server.readyState === WebSocket.OPEN) {
+    this.wss.handleUpgrade(req, socket, head, (client) => {
+      client.on("message", (event, binary) => {
+        server.send(event, { binary });
+      });
+      client.on("close", (code, reason) => {
         server.close(code, reason);
-      }
+      });
+      server.on("message", (event, binary) => {
+        client.send(event, { binary });
+      });
+      server.on("close", (code, reason) => {
+        client.close(code, reason);
+      });
+      this.wss.emit("connection", client, req);
     });
   }
 
-  async close() {
-    this.wsServer.close();
-    await super.close();
+  private async createServerWebSocket(req: http.IncomingMessage) {
+    const target = await this.options.miniflare.unsafeGetDirectURL(
+      this.options.name,
+    );
+    if (!target) {
+      logger.error(
+        `[Alchemy] Websocket connection failed: The worker "${this.options.name}" is not running.`,
+      );
+      return null;
+    }
+    const url = new URL(req.url ?? "/", target);
+    url.protocol = url.protocol.replace("http", "ws");
+    const protocols = req.headers["sec-websocket-protocol"]
+      ?.split(",")
+      .map((p) => p.trim());
+    const server = new WebSocket(url, protocols);
+    const controller = new AbortController();
+    return await Promise.race([
+      once(server, "open", { signal: controller.signal }).then(() => server),
+      once(server, "close", { signal: controller.signal }).then((args) => {
+        const [code, reason] = args as [number, string];
+        logger.error(
+          `[Alchemy] Websocket connection failed for worker "${this.options.name}": ${code} ${reason}`,
+        );
+        return null;
+      }),
+    ]).finally(() => controller.abort());
+  }
+
+  private async handleRequest(
+    req: http.IncomingMessage,
+    res: http.ServerResponse,
+  ) {
+    const worker = await this.options.miniflare.getWorker(this.options.name);
+    if (!worker) {
+      res.statusCode = 503;
+      res.end(`[Alchemy] The worker "${this.options.name}" is not running.`);
+      return;
+    }
+    try {
+      const response = await worker.fetch(toMiniflareRequest(req));
+      writeMiniflareResponseToNode(response, res);
+    } catch (rawError) {
+      const message =
+        rawError instanceof Error
+          ? (rawError.stack ?? rawError.message)
+          : String(rawError);
+      res.statusCode = 500;
+      res.end(
+        `[Alchemy] The worker "${this.options.name}" threw an error:\n\n${message}`,
+      );
+    }
   }
 }
+
+const toMiniflareRequest = (req: http.IncomingMessage) => {
+  const method = req.method ?? "GET";
+  const url = new URL(req.url ?? "/", `http://${req.headers.host}`);
+  const headers = new miniflare.Headers();
+  for (const [key, value] of Object.entries(req.headers)) {
+    if (Array.isArray(value)) {
+      for (const v of value) {
+        headers.append(key, v);
+      }
+    } else if (value) {
+      headers.set(key, value);
+    }
+  }
+  const body =
+    ["GET", "HEAD", "OPTIONS"].includes(method) || !req.readable
+      ? undefined
+      : Readable.toWeb(req);
+  return new miniflare.Request(url, {
+    method,
+    headers,
+    body,
+    redirect: "manual",
+    duplex: body ? "half" : undefined,
+  });
+};
+
+const writeMiniflareResponseToNode = (
+  res: miniflare.Response,
+  out: http.ServerResponse,
+) => {
+  out.statusCode = res.status;
+  res.headers.forEach((value, key) => {
+    out.setHeader(key, value);
+  });
+  if (res.body) {
+    Readable.fromWeb(res.body).pipe(out, { end: true });
+  } else {
+    out.end();
+  }
+};

package/src/cloudflare/miniflare/miniflare.ts

@@ -1,13 +1,6 @@
-import {
-  MiniflareCoreError,
-  type Miniflare,
-  type MiniflareOptions,
-  type RemoteProxyConnectionString,
-  type WorkerOptions,
-} from "miniflare";
+import * as miniflare from "miniflare";
 import path from "node:path";
 import { findOpenPort } from "../../util/find-open-port.ts";
-import { logger } from "../../util/logger.ts";
 import {
   promiseWithResolvers,
   type PromiseWithResolvers,
@@ -24,14 +17,14 @@ import {
 } from "./remote-binding-proxy.ts";
 
 class MiniflareServer {
-  miniflare?: Miniflare;
-  workers = new Map<string, WorkerOptions>();
+  miniflare?: miniflare.Miniflare;
+  workers = new Map<string, miniflare.WorkerOptions>();
   workerProxies = new Map<string, MiniflareWorkerProxy>();
   remoteBindingProxies = new Map<string, RemoteBindingProxy>();
 
   stream = new WritableStream<{
     worker: MiniflareWorkerOptions;
-    promise: PromiseWithResolvers<MiniflareWorkerProxy>;
+    promise: PromiseWithResolvers<{ url: string }>;
   }>({
     write: async ({ worker, promise }) => {
       try {
@@ -48,7 +41,7 @@ class MiniflareServer {
   writer = this.stream.getWriter();
 
   async push(worker: MiniflareWorkerOptions) {
-    const promise = promiseWithResolvers<MiniflareWorkerProxy>();
+    const promise = promiseWithResolvers<{ url: string }>();
     const [, server] = await Promise.all([
       this.writer.write({ worker, promise }),
       promise.promise,
@@ -73,12 +66,6 @@ class MiniflareServer {
         this.miniflare.setOptions(await this.miniflareOptions()),
       );
     } else {
-      const { Miniflare } = await import("miniflare").catch(() => {
-        throw new Error(
-          "Miniflare is not installed, but is required in local mode for Workers. Please run `npm install miniflare`.",
-        );
-      });
-
       // Miniflare intercepts SIGINT and exits with 130, which is not a failure.
       // No one likes to see a non-zero exit code when they Ctrl+C, so here's our workaround.
       process.on("exit", (code) => {
@@ -86,26 +73,21 @@ class MiniflareServer {
           process.exit(0);
         }
       });
-      this.miniflare = new Miniflare(await this.miniflareOptions());
+      this.miniflare = new miniflare.Miniflare(await this.miniflareOptions());
       await withErrorRewrite(this.miniflare.ready);
     }
-    const existing = this.workerProxies.get(worker.name);
-    if (existing) {
-      return existing;
+    const existingProxy = this.workerProxies.get(worker.name);
+    if (existingProxy) {
+      return existingProxy;
     }
-    const server = new MiniflareWorkerProxy({
-      getDirectURL: async () => {
-        const url = await this.miniflare?.unsafeGetDirectURL(worker.name);
-        if (!url) {
-          throw new Error(`Worker "${worker.name}" is not running`);
-        }
-        return url;
-      },
-      fetch: this.createRequestHandler(worker.name),
+    const newProxy = new MiniflareWorkerProxy({
+      name: worker.name,
+      port: worker.port ?? (await findOpenPort()),
+      miniflare: this.miniflare,
     });
-    this.workerProxies.set(worker.name, server);
-    await server.listen(worker.port ?? (await findOpenPort()));
-    return server;
+    this.workerProxies.set(worker.name, newProxy);
+    await newProxy.ready;
+    return newProxy;
   }
 
   private async dispose() {
@@ -125,7 +107,7 @@ class MiniflareServer {
 
   private async maybeCreateRemoteProxy(
     worker: MiniflareWorkerOptions,
-  ): Promise<RemoteProxyConnectionString | undefined> {
+  ): Promise<miniflare.RemoteProxyConnectionString | undefined> {
     const bindings = buildRemoteBindings(worker);
     if (bindings.length === 0) {
       return undefined;
@@ -148,52 +130,11 @@ class MiniflareServer {
     return proxy.connectionString;
   }
 
-  private createRequestHandler(name: string) {
-    return async (req: Request) => {
-      try {
-        if (!this.miniflare) {
-          return new Response(
-            "[Alchemy] Miniflare is not initialized. Please try again.",
-            {
-              status: 503,
-            },
-          );
-        }
-        const worker = await this.miniflare?.getWorker(name);
-        if (!worker) {
-          return new Response(
-            `[Alchemy] Cannot find worker "${name}". Please try again.`,
-            {
-              status: 503,
-            },
-          );
-        }
-        const res = await worker.fetch(req.url, {
-          method: req.method,
-          headers: req.headers as any,
-          body: req.body as any,
-          duplex: "half",
-          redirect: "manual",
-        });
-        return res as unknown as Response;
-      } catch (error) {
-        logger.error(error);
-        return new Response(
-          `[Alchemy] Internal server error: ${String(error)}`,
-          {
-            status: 500,
-          },
-        );
-      }
-    };
-  }
-
-  private async miniflareOptions(): Promise<MiniflareOptions> {
-    const { getDefaultDevRegistryPath } = await import("miniflare");
+  private async miniflareOptions(): Promise<miniflare.MiniflareOptions> {
     return {
       workers: Array.from(this.workers.values()),
       defaultPersistRoot: path.join(process.cwd(), ".alchemy/miniflare"),
-      unsafeDevRegistryPath: getDefaultDevRegistryPath(),
+      unsafeDevRegistryPath: miniflare.getDefaultDevRegistryPath(),
       analyticsEngineDatasetsPersist: true,
       cachePersist: true,
       d1Persist: true,
@@ -202,6 +143,9 @@ class MiniflareServer {
       r2Persist: true,
       secretsStorePersist: true,
       workflowsPersist: true,
+      log: process.env.DEBUG
+        ? new miniflare.Log(miniflare.LogLevel.DEBUG)
+        : undefined,
     };
   }
 }
@@ -219,7 +163,7 @@ async function withErrorRewrite<T>(promise: Promise<T>) {
     return await promise;
   } catch (error) {
     if (
-      error instanceof MiniflareCoreError &&
+      error instanceof miniflare.MiniflareCoreError &&
       error.code === "ERR_MODULE_STRING_SCRIPT"
     ) {
       throw new ExternalDependencyError();

package/src/cloudflare/zone.ts

@@ -626,23 +626,46 @@ export async function findZoneForHostname(
   // Remove wildcard prefix if present
   const cleanHostname = hostname.replace(/^\*\./, "");
 
-  // Get all zones and find the best match
-  const response = await api.get("/zones");
+  // Helper to fetch a page of zones
+  const fetchZonePage = async (pageNum: number) => {
+    const response = await api.get(`/zones?per_page=50&page=${pageNum}`);
+    if (!response.ok) {
+      throw new Error(
+        `Failed to list zones (page ${pageNum}): ${response.statusText}`,
+      );
+    }
+    return response.json() as Promise<{
+      result: Array<{ id: string; name: string }>;
+      result_info?: {
+        count?: number;
+        page?: number;
+        per_page?: number;
+        total_count?: number;
+        total_pages?: number;
+      };
+    }>;
+  };
 
-  if (!response.ok) {
-    throw new Error(`Failed to list zones: ${response.statusText}`);
-  }
+  // Fetch the first page to get total_pages
+  const firstPageData = await fetchZonePage(1);
+  const totalPages = firstPageData.result_info?.total_pages ?? 1;
 
-  const zonesData = (await response.json()) as {
-    result: Array<{ id: string; name: string }>;
-  };
+  // Fetch remaining pages concurrently if needed
+  const allZones = totalPages > 1
+    ? await Promise.all([
+        Promise.resolve(firstPageData.result),
+        ...Array.from({ length: totalPages - 1 }, (_, i) =>
+          fetchZonePage(i + 2).then(data => data.result)
+        ),
+      ]).then(results => results.flat())
+    : firstPageData.result;
 
   // Find the zone that best matches the hostname
   // We look for the longest matching zone name (most specific)
   let bestMatch: { zoneId: string; zoneName: string } | null = null;
   let longestMatch = 0;
 
-  for (const zone of zonesData.result) {
+  for (const zone of allZones) {
     if (
       cleanHostname === zone.name ||
       cleanHostname.endsWith(`.${zone.name}`)
@@ -656,7 +679,7 @@ export async function findZoneForHostname(
 
   if (!bestMatch) {
     throw new Error(
-      `Could not find zone for hostname '${hostname}'. Available zones: ${zonesData.result.map((z) => z.name).join(", ")}`,
+      `Could not find zone for hostname '${hostname}'. Available zones: ${allZones.map((z) => z.name).join(", ")}`,
     );
   }
 

package/src/scope.ts

@@ -15,6 +15,7 @@ import {
 import type { State, StateStore, StateStoreType } from "./state.ts";
 import { D1StateStore } from "./state/d1-state-store.ts";
 import { FileSystemStateStore } from "./state/file-system-state-store.ts";
+import { InstrumentedStateStore } from "./state/instrumented-state-store.ts";
 import {
   createDummyLogger,
   createLoggerInstance,
@@ -201,12 +202,15 @@ export class Scope {
 
     this.stateStore =
       options.stateStore ?? this.parent?.stateStore ?? defaultStateStore;
-    this.state = this.stateStore(this);
+    this.telemetryClient =
+      options.telemetryClient ?? this.parent?.telemetryClient!;
+    this.state = new InstrumentedStateStore(
+      this.stateStore(this),
+      this.telemetryClient,
+    );
     if (!options.telemetryClient && !this.parent?.telemetryClient) {
       throw new Error("Telemetry client is required");
     }
-    this.telemetryClient =
-      options.telemetryClient ?? this.parent?.telemetryClient!;
     this.dataMutex = new AsyncMutex();
   }
 

package/src/state/instrumented-state-store.ts

@@ -0,0 +1,97 @@
+import type { State, StateStore } from "../state.ts";
+import type { ITelemetryClient } from "../util/telemetry/client.ts";
+import type { Telemetry } from "../util/telemetry/types.ts";
+
+//todo(michael): we should also handle serde here
+export class InstrumentedStateStore<T extends StateStore>
+  implements StateStore
+{
+  private readonly stateStore: StateStore;
+  private readonly telemetryClient: ITelemetryClient;
+  private readonly stateStoreClass: string;
+
+  constructor(stateStore: StateStore, telemetryClient: ITelemetryClient) {
+    this.stateStore = stateStore;
+    this.telemetryClient = telemetryClient;
+    this.stateStoreClass = stateStore.constructor.name;
+  }
+
+  private async callWithTelemetry<T>(
+    event: Telemetry.StateStoreEvent["event"],
+    fn: () => Promise<T>,
+  ): Promise<T> {
+    const start = performance.now();
+    let error: unknown;
+    return await fn()
+      .catch((err) => (error = err))
+      .finally(() => {
+        this.telemetryClient.record({
+          event,
+          stateStoreClass: this.stateStoreClass,
+          elapsed: performance.now() - start,
+          error: error instanceof Error ? error : new Error(String(error)),
+        });
+      });
+  }
+
+  async init() {
+    if (this.stateStore.init == null) {
+      return;
+    }
+    await this.callWithTelemetry(
+      "stateStore.init",
+      this.stateStore.init.bind(this.stateStore),
+    );
+  }
+  async deinit() {
+    if (this.stateStore.deinit == null) {
+      return;
+    }
+    await this.callWithTelemetry(
+      "stateStore.deinit",
+      this.stateStore.deinit.bind(this.stateStore),
+    );
+  }
+  async list() {
+    return await this.callWithTelemetry(
+      "stateStore.list",
+      this.stateStore.list.bind(this.stateStore),
+    );
+  }
+  async count() {
+    return await this.callWithTelemetry(
+      "stateStore.count",
+      this.stateStore.count.bind(this.stateStore),
+    );
+  }
+  async get(key: string) {
+    return await this.callWithTelemetry(
+      "stateStore.get",
+      this.stateStore.get.bind(this.stateStore, key),
+    );
+  }
+  async getBatch(ids: string[]) {
+    return await this.callWithTelemetry(
+      "stateStore.getBatch",
+      this.stateStore.getBatch.bind(this.stateStore, ids),
+    );
+  }
+  async all() {
+    return await this.callWithTelemetry(
+      "stateStore.all",
+      this.stateStore.all.bind(this.stateStore),
+    );
+  }
+  async set(key: string, value: State) {
+    await this.callWithTelemetry(
+      "stateStore.set",
+      this.stateStore.set.bind(this.stateStore, key, value),
+    );
+  }
+  async delete(key: string) {
+    await this.callWithTelemetry(
+      "stateStore.delete",
+      this.stateStore.delete.bind(this.stateStore, key),
+    );
+  }
+}

package/src/util/telemetry/client.ts

@@ -54,13 +54,9 @@ export class TelemetryClient implements ITelemetryClient {
   }
 
   record(event: Telemetry.EventInput, timestamp = Date.now()) {
-    if (!this.context) {
-      return;
-    }
     const payload = {
       ...event,
       error: this.serializeError(event.error),
-      context: this.context,
       timestamp,
     } as Telemetry.Event;
     this.events.push(payload);
@@ -93,9 +89,10 @@ export class TelemetryClient implements ITelemetryClient {
   }
 
   private async send(events: Telemetry.Event[]) {
-    if (events.length === 0) {
+    if (events.length === 0 || this.context) {
       return;
     }
+    const { userId, ...data } = this.context!;
     const response = await fetch(`${POSTHOG_CLIENT_API_HOST}/batch`, {
       method: "POST",
       headers: {
@@ -104,13 +101,18 @@ export class TelemetryClient implements ITelemetryClient {
       body: JSON.stringify({
         api_key: POSTHOG_PROJECT_ID,
         historical_migration: false,
-        batch: events.map((e) => ({
-          event: e.event,
-          properties: {
-            distinct_id: this.context?.sessionId,
-          },
-          timestamp: new Date(e.timestamp).toISOString(),
-        })),
+        batch: events.map((e) => {
+          const { event, ...eventData } = e;
+          return {
+            event: event,
+            properties: {
+              distinct_id: userId,
+              ...data,
+              ...eventData,
+            },
+            timestamp: new Date(e.timestamp).toISOString(),
+          };
+        }),
       }),
     });
     if (!response.ok) {