alchemy 0.55.3 → 0.57.0

package/src/cloudflare/miniflare/build-worker-options.ts

@@ -0,0 +1,454 @@
+import * as miniflare from "miniflare";
+import { assertNever } from "../../util/assert-never.ts";
+import type { HTTPServer } from "../../util/http.ts";
+import {
+  Self,
+  type Binding,
+  type Bindings,
+  type WorkerBindingService,
+  type WorkerBindingSpec,
+} from "../bindings.ts";
+import { isQueueEventSource, type EventSource } from "../event-source.ts";
+import type { WorkerBundle, WorkerBundleSource } from "../worker-bundle.ts";
+import type { AssetsConfig } from "../worker.ts";
+import { createRemoteProxyWorker } from "./remote-binding-proxy.ts";
+
+export interface MiniflareWorkerInput {
+  id: string;
+  name: string;
+  compatibilityDate: string | undefined;
+  compatibilityFlags: string[] | undefined;
+  bindings: Bindings | undefined;
+  eventSources: EventSource[] | undefined;
+  assets: AssetsConfig | undefined;
+  bundle: WorkerBundleSource;
+  port: number | undefined;
+}
+
+type RemoteOnlyBindingType =
+  | "ai"
+  | "ai_gateway"
+  | "browser"
+  | "dispatch_namespace"
+  | "mtls_certificate"
+  | "vectorize";
+type RemoteOptionalBindingType =
+  | "d1"
+  | "images"
+  | "kv_namespace"
+  | "queue"
+  | "r2_bucket";
+
+type RemoteBinding =
+  | (Extract<
+      WorkerBindingSpec,
+      {
+        type: RemoteOnlyBindingType | RemoteOptionalBindingType;
+      }
+    > & {
+      raw: true;
+    })
+  | WorkerBindingService;
+
+type BaseWorkerOptions = {
+  [K in keyof miniflare.WorkerOptions]: K extends "compatibilityFlags"
+    ? miniflare.WorkerOptions[K]
+    : Exclude<miniflare.WorkerOptions[K], string[]>;
+};
+
+export const buildWorkerOptions = async (
+  input: MiniflareWorkerInput,
+): Promise<{
+  watch: (signal: AbortSignal) => AsyncGenerator<miniflare.WorkerOptions>;
+  remoteProxy: HTTPServer | undefined;
+}> => {
+  const remoteBindings: RemoteBinding[] = [];
+  const options: Partial<BaseWorkerOptions> = {
+    name: input.name,
+    compatibilityDate: input.compatibilityDate,
+    compatibilityFlags: input.compatibilityFlags,
+    // TODO: Setting `proxy: true` here causes the following error when connecting via a websocket:
+    // workerd/io/worker.c++:2164: info: uncaught exception; source = Uncaught (in promise); stack = TypeError: Invalid URL string.
+    unsafeDirectSockets: [{ proxy: false }],
+    containerEngine: {
+      localDocker: {
+        socketPath:
+          process.platform === "win32"
+            ? "//./pipe/docker_engine"
+            : "unix:///var/run/docker.sock",
+      },
+    },
+  };
+  for (const [key, binding] of Object.entries(input.bindings ?? {})) {
+    if (typeof binding === "string") {
+      (options.bindings ??= {})[key] = binding;
+      continue;
+    }
+    if (binding === Self) {
+      (options.serviceBindings ??= {})[key] = miniflare.kCurrentWorker;
+      continue;
+    }
+    switch (binding.type) {
+      case "ai":
+      case "ai_gateway": {
+        remoteBindings.push({
+          type: "ai",
+          name: key,
+          raw: true,
+        });
+        break;
+      }
+      case "analytics_engine": {
+        (options.analyticsEngineDatasets ??= {})[key] = {
+          dataset: binding.dataset,
+        };
+        break;
+      }
+      case "assets": {
+        options.assets = {
+          binding: key,
+          directory: binding.path,
+          assetConfig: {
+            html_handling: input.assets?.html_handling,
+            not_found_handling: input.assets?.not_found_handling,
+          },
+        };
+        break;
+      }
+      case "browser": {
+        remoteBindings.push({
+          type: "browser",
+          name: key,
+          raw: true,
+        });
+        break;
+      }
+      case "container": {
+        (options.durableObjects ??= {})[key] = {
+          className: binding.className,
+          scriptName: binding.scriptName,
+          useSQLite: binding.sqlite,
+          container: {
+            imageName: binding.image.imageRef,
+          },
+        };
+        break;
+      }
+      case "d1": {
+        if (isRemoteBinding(binding)) {
+          remoteBindings.push({
+            type: "d1",
+            name: key,
+            id: binding.id,
+            raw: true,
+          });
+        } else {
+          (options.d1Databases ??= {})[key] = binding.id;
+        }
+        break;
+      }
+      case "dispatch_namespace": {
+        remoteBindings.push({
+          type: "dispatch_namespace",
+          name: key,
+          namespace: binding.namespace,
+          raw: true,
+        });
+        break;
+      }
+      case "durable_object_namespace": {
+        (options.durableObjects ??= {})[key] = {
+          className: binding.className,
+          scriptName: binding.scriptName,
+          useSQLite: binding.sqlite,
+        };
+        break;
+      }
+      case "hyperdrive": {
+        throw new Error("Hyperdrive bindings are not supported in local mode");
+      }
+      case "images": {
+        if (isRemoteBinding(binding)) {
+          remoteBindings.push({
+            type: "images",
+            name: key,
+            raw: true,
+          });
+        } else {
+          options.images = {
+            binding: key,
+          };
+        }
+        break;
+      }
+      case "kv_namespace": {
+        const namespaceId =
+          "namespaceId" in binding ? binding.namespaceId : binding.id;
+        if (isRemoteBinding(binding)) {
+          remoteBindings.push({
+            type: "kv_namespace",
+            name: key,
+            namespace_id: namespaceId,
+            raw: true,
+          });
+        } else {
+          (options.kvNamespaces ??= {})[key] = namespaceId;
+        }
+        break;
+      }
+      case "json": {
+        (options.bindings ??= {})[key] = binding.json;
+        break;
+      }
+      case "queue": {
+        if (isRemoteBinding(binding)) {
+          remoteBindings.push({
+            type: "queue",
+            name: key,
+            queue_name: binding.name,
+            raw: true,
+          });
+        } else {
+          (options.queueProducers ??= {})[key] = {
+            queueName: binding.name,
+            deliveryDelay: binding.settings?.deliveryDelay,
+          };
+        }
+        break;
+      }
+      case "pipeline": {
+        (options.pipelines ??= {})[key] = binding.id;
+        break;
+      }
+      case "ratelimit": {
+        (options.ratelimits ??= {})[key] = {
+          simple: binding.simple,
+        };
+        break;
+      }
+      case "secret": {
+        (options.bindings ??= {})[key] = binding.unencrypted;
+        break;
+      }
+      case "secret_key": {
+        throw new Error("Secret keys are not supported in local mode");
+      }
+      case "secrets_store_secret": {
+        (options.secretsStoreSecrets ??= {})[key] = {
+          store_id: binding.storeId,
+          secret_name: binding.name,
+        };
+        break;
+      }
+      case "r2_bucket": {
+        if (isRemoteBinding(binding)) {
+          remoteBindings.push({
+            type: "r2_bucket",
+            name: key,
+            bucket_name: binding.name,
+            raw: true,
+          });
+        } else {
+          (options.r2Buckets ??= {})[key] = binding.name;
+        }
+        break;
+      }
+      case "service": {
+        const service = "service" in binding ? binding.service : binding.name;
+        const normalized: WorkerBindingService = {
+          type: "service",
+          name: key,
+          service,
+          environment:
+            "environment" in binding ? binding.environment : undefined,
+        };
+        if (isRemoteBinding(binding)) {
+          remoteBindings.push(normalized);
+        } else {
+          (options.serviceBindings ??= {})[key] = service;
+        }
+        break;
+      }
+      case "vectorize": {
+        remoteBindings.push({
+          type: "vectorize",
+          name: key,
+          index_name: binding.name,
+          raw: true,
+        });
+        break;
+      }
+      case "version_metadata": {
+        (options.bindings ??= {})[key] = {
+          id: crypto.randomUUID(),
+          tag: "",
+          timestamp: "0",
+        };
+        break;
+      }
+      case "workflow": {
+        (options.workflows ??= {})[key] = {
+          name: binding.workflowName,
+          className: binding.className,
+          scriptName: binding.scriptName,
+        };
+        break;
+      }
+      default: {
+        assertNever(binding);
+      }
+    }
+  }
+  for (const eventSource of input.eventSources ?? []) {
+    const queue = "queue" in eventSource ? eventSource.queue : eventSource;
+    if (queue.dev?.remote) {
+      throw new Error(
+        `Locally emulated workers cannot consume remote queues. Worker "${input.name}" is locally emulated but is consuming remote queue "${queue.name}".`,
+      );
+    }
+    if (isQueueEventSource(eventSource)) {
+      (options.queueConsumers ??= {})[queue.name] = {
+        maxBatchSize: eventSource.settings?.batchSize,
+        maxBatchTimeout: eventSource.settings?.maxWaitTimeMs,
+        maxRetries: eventSource.settings?.maxRetries,
+        retryDelay: eventSource.settings?.retryDelay,
+      };
+    } else {
+      (options.queueConsumers ??= {})[eventSource.name] = {};
+    }
+  }
+  async function* watch(signal: AbortSignal) {
+    for await (const bundle of input.bundle.watch(signal)) {
+      const { modules, rootPath } = normalizeBundle(bundle);
+      yield {
+        ...options,
+        modules,
+        rootPath,
+      };
+    }
+  }
+  if (remoteBindings.length > 0) {
+    const remoteProxy = await createRemoteProxyWorker({
+      name: input.name,
+      bindings: remoteBindings,
+    });
+    for (const binding of remoteBindings) {
+      switch (binding.type) {
+        case "ai":
+          options.ai = {
+            binding: binding.name,
+            remoteProxyConnectionString: remoteProxy.connectionString,
+          };
+          break;
+        case "browser":
+          options.browserRendering = {
+            binding: binding.name,
+            remoteProxyConnectionString: remoteProxy.connectionString,
+          };
+          break;
+        case "d1":
+          (options.d1Databases ??= {})[binding.name] = {
+            id: binding.id,
+            remoteProxyConnectionString: remoteProxy.connectionString,
+          };
+          break;
+        case "dispatch_namespace":
+          (options.dispatchNamespaces ??= {})[binding.name] = {
+            namespace: binding.namespace,
+            remoteProxyConnectionString: remoteProxy.connectionString,
+          };
+          break;
+        case "kv_namespace":
+          (options.kvNamespaces ??= {})[binding.name] = {
+            id: binding.namespace_id,
+            remoteProxyConnectionString: remoteProxy.connectionString,
+          };
+          break;
+        case "images":
+          options.images = {
+            binding: binding.name,
+            remoteProxyConnectionString: remoteProxy.connectionString,
+          };
+          break;
+        case "mtls_certificate":
+          (options.mtlsCertificates ??= {})[binding.name] = {
+            certificate_id: binding.certificate_id,
+            remoteProxyConnectionString: remoteProxy.connectionString,
+          };
+          break;
+        case "queue":
+          (options.queueProducers ??= {})[binding.name] = {
+            queueName: binding.queue_name,
+            remoteProxyConnectionString: remoteProxy.connectionString,
+          };
+          break;
+        case "r2_bucket":
+          (options.r2Buckets ??= {})[binding.name] = {
+            id: binding.bucket_name,
+            remoteProxyConnectionString: remoteProxy.connectionString,
+          };
+          break;
+        case "service":
+          (options.serviceBindings ??= {})[binding.name] = {
+            name: binding.name,
+            remoteProxyConnectionString: remoteProxy.connectionString,
+          };
+          break;
+        case "vectorize":
+          (options.vectorize ??= {})[binding.name] = {
+            index_name: binding.index_name,
+            remoteProxyConnectionString: remoteProxy.connectionString,
+          };
+          break;
+        default: {
+          assertNever(binding);
+        }
+      }
+    }
+    return {
+      watch,
+      remoteProxy: remoteProxy.server,
+    };
+  }
+  return {
+    watch,
+    remoteProxy: undefined,
+  };
+};
+
+const moduleTypes = {
+  esm: "ESModule",
+  cjs: "CommonJS",
+  text: "Text",
+  data: "Data",
+  wasm: "CompiledWasm",
+  sourcemap: "Text",
+} as const;
+
+const normalizeBundle = (bundle: WorkerBundle) => {
+  const modules = bundle.modules.map((module) => ({
+    type: moduleTypes[module.type],
+    path: module.path,
+    contents: module.content,
+  }));
+  const entry = modules.find((module) => module.path === bundle.entrypoint);
+  if (!entry) {
+    throw new Error(`Entrypoint "${bundle.entrypoint}" not found in bundle.`);
+  }
+  return {
+    modules: [entry, ...modules.filter((module) => module.path !== entry.path)],
+    rootPath: bundle.root,
+  };
+};
+
+const isRemoteBinding = (binding: Binding) => {
+  if (typeof binding === "string" || binding === Self) {
+    return false;
+  }
+  return (
+    "dev" in binding &&
+    typeof binding.dev === "object" &&
+    "remote" in binding.dev &&
+    !!binding.dev.remote
+  );
+};

package/src/cloudflare/miniflare/miniflare-controller.ts

@@ -0,0 +1,129 @@
+import * as miniflare from "miniflare";
+import assert from "node:assert";
+import path from "node:path";
+import { findOpenPort } from "../../util/find-open-port.ts";
+import type { HTTPServer } from "../../util/http.ts";
+import { logger } from "../../util/logger.ts";
+import { AsyncMutex } from "../../util/mutex.ts";
+import {
+  buildWorkerOptions,
+  type MiniflareWorkerInput,
+} from "./build-worker-options.ts";
+import { MiniflareWorkerProxy } from "./miniflare-worker-proxy.ts";
+import { DEFAULT_PERSIST_PATH } from "./paths.ts";
+
+declare global {
+  var ALCHEMY_MINIFLARE_CONTROLLER: MiniflareController | undefined;
+}
+
+export class MiniflareController {
+  abort = new AbortController();
+  miniflare: miniflare.Miniflare | undefined;
+  options = new Map<string, miniflare.WorkerOptions>();
+  localProxies = new Map<string, MiniflareWorkerProxy>();
+  remoteProxies = new Map<string, HTTPServer>();
+  mutex = new AsyncMutex();
+
+  static get singleton() {
+    globalThis.ALCHEMY_MINIFLARE_CONTROLLER ??= new MiniflareController();
+    return globalThis.ALCHEMY_MINIFLARE_CONTROLLER;
+  }
+
+  async add(input: MiniflareWorkerInput) {
+    const { watch, remoteProxy } = await buildWorkerOptions(input);
+    if (remoteProxy) {
+      this.remoteProxies.set(input.name, remoteProxy);
+    }
+    const watcher = watch(this.abort.signal);
+    const first = await watcher.next();
+    assert(first.value, "First value is undefined");
+    this.options.set(input.name, first.value);
+    const miniflare = await this.update();
+    const proxy = new MiniflareWorkerProxy({
+      name: input.name,
+      port: input.port ?? (await findOpenPort()),
+      miniflare,
+    });
+    this.localProxies.set(input.name, proxy);
+    void this.watch(input.id, watcher);
+    logger.task(input.id, {
+      message: `Ready at ${proxy.url}`,
+      status: "success",
+      resource: input.id,
+      prefix: "dev",
+      prefixColor: "cyanBright",
+    });
+    return proxy.url;
+  }
+
+  private async watch(
+    id: string,
+    watcher: AsyncGenerator<miniflare.WorkerOptions>,
+  ) {
+    for await (const options of watcher) {
+      this.options.set(options.name!, options);
+      await this.update();
+      logger.task(id, {
+        message: "Updated",
+        status: "success",
+        resource: id,
+        prefix: "dev",
+        prefixColor: "cyanBright",
+      });
+    }
+  }
+
+  private async update() {
+    return await this.mutex.lock(async () => {
+      const options: miniflare.MiniflareOptions = {
+        workers: Array.from(this.options.values()),
+        defaultPersistRoot: path.resolve(DEFAULT_PERSIST_PATH),
+        unsafeDevRegistryPath: miniflare.getDefaultDevRegistryPath(),
+        analyticsEngineDatasetsPersist: true,
+        cachePersist: true,
+        d1Persist: true,
+        durableObjectsPersist: true,
+        kvPersist: true,
+        r2Persist: true,
+        secretsStorePersist: true,
+        workflowsPersist: true,
+        log: process.env.DEBUG
+          ? new miniflare.Log(miniflare.LogLevel.DEBUG)
+          : undefined,
+      };
+      return await this.setMiniflareOptions(options);
+    });
+  }
+
+  private async setMiniflareOptions(options: miniflare.MiniflareOptions) {
+    try {
+      if (this.miniflare) {
+        await this.miniflare.setOptions(options);
+      } else {
+        this.miniflare = new miniflare.Miniflare(options);
+        await this.miniflare.ready;
+      }
+      return this.miniflare;
+    } catch (error) {
+      if (
+        error instanceof miniflare.MiniflareCoreError &&
+        error.code === "ERR_MODULE_STRING_SCRIPT"
+      ) {
+        throw new Error(
+          'Miniflare detected an external dependency that could not be resolved. This typically occurs when the "nodejs_compat" or "nodejs_als" compatibility flag is not enabled.',
+        );
+      } else {
+        throw error;
+      }
+    }
+  }
+
+  async dispose() {
+    this.abort.abort();
+    await Promise.all([
+      this.miniflare?.dispose(),
+      ...this.localProxies.values().map((proxy) => proxy.close()),
+      ...this.remoteProxies.values().map((proxy) => proxy.close()),
+    ]);
+  }
+}

package/src/cloudflare/miniflare/miniflare-worker-proxy.ts

@@ -55,18 +55,10 @@ export class MiniflareWorkerProxy {
       return;
     }
     this.wss.handleUpgrade(req, socket, head, (client) => {
-      client.on("message", (event, binary) => {
-        server.send(event, { binary });
-      });
-      client.on("close", (code, reason) => {
-        server.close(code, reason);
-      });
-      server.on("message", (event, binary) => {
-        client.send(event, { binary });
-      });
-      server.on("close", (code, reason) => {
-        client.close(code, reason);
-      });
+      client.on("message", (event, binary) => server.send(event, { binary }));
+      client.on("close", (code, reason) => server.close(code, reason));
+      server.on("message", (event, binary) => client.send(event, { binary }));
+      server.on("close", (code, reason) => client.close(code, reason));
       this.wss.emit("connection", client, req);
     });
   }
@@ -157,9 +149,8 @@ const writeMiniflareResponseToNode = (
   out: http.ServerResponse,
 ) => {
   out.statusCode = res.status;
-  res.headers.forEach((value, key) => {
-    out.setHeader(key, value);
-  });
+  res.headers.forEach((value, key) => out.setHeader(key, value));
+
   if (res.body) {
     Readable.fromWeb(res.body).pipe(out, { end: true });
   } else {

package/src/cloudflare/miniflare/paths.ts

@@ -0,0 +1,52 @@
+import { existsSync, readlinkSync, statSync } from "node:fs";
+
+export const DEFAULT_CONFIG_PATH = ".alchemy/local/wrangler.jsonc";
+export const DEFAULT_PERSIST_PATH = ".alchemy/miniflare/v3";
+
+export const validateConfigPath = (
+  path = DEFAULT_CONFIG_PATH,
+  throws = true,
+) => {
+  if (!existsSync(path)) {
+    warnOrThrow(
+      `The Wrangler config path, "${path}", could not be found. Please run \`alchemy dev\` or \`alchemy deploy\` to create it.`,
+      throws,
+    );
+  }
+  return path;
+};
+
+const DEFAULT_VALIDATE_PERSIST =
+  process.env.NODE_ENV === "development" ||
+  !process.argv.some((arg) => ["build", "prepare", "typegen"].includes(arg));
+
+export const validatePersistPath = (
+  path = DEFAULT_PERSIST_PATH,
+  throws = DEFAULT_VALIDATE_PERSIST,
+) => {
+  try {
+    const stat = statSync(path);
+    if (stat.isSymbolicLink()) {
+      return readlinkSync(path);
+    }
+    return path;
+  } catch {
+    warnOrThrow(
+      `The Alchemy data path, "${path}", could not be resolved. This is required for Cloudflare bindings during development. Please run \`alchemy dev\` to create it.`,
+      throws,
+    );
+    return path;
+  }
+};
+
+const warned = new Set<string>();
+
+const warnOrThrow = (message: string, throws: boolean) => {
+  if (throws) {
+    throw new Error(message);
+  }
+  if (!warned.has(message)) {
+    console.warn(message);
+    warned.add(message);
+  }
+};