alchemy 2.0.0-beta.23 → 2.0.0-beta.24

package/src/Sidecar/RpcClient.ts

@@ -0,0 +1,141 @@
+import { newWebSocketRpcSession } from "capnweb";
+import * as Context from "effect/Context";
+import * as Effect from "effect/Effect";
+import * as Fiber from "effect/Fiber";
+import * as FileSystem from "effect/FileSystem";
+import * as Layer from "effect/Layer";
+import * as Schedule from "effect/Schedule";
+import * as Schema from "effect/Schema";
+import * as ChildProcess from "effect/unstable/process/ChildProcess";
+import { fileURLToPath } from "node:url";
+import * as Lock from "./Lock.ts";
+import {
+  deserializeRpcHandlers,
+  type RpcHandlerDecoders,
+  type RpcHandlers,
+  type SerializedRpcHandlers,
+} from "./RpcHandler.ts";
+import * as RpcPaths from "./RpcPaths.ts";
+
+export const RpcClientService = <Self, T extends RpcHandlers>() =>
+  Context.Service<Self, T>();
+
+class RpcClientError extends Schema.TaggedErrorClass<RpcClientError>()(
+  "RpcClientError",
+  {
+    reason: Schema.Literals(["InvalidURL", "WebSocketError"]),
+    message: Schema.String,
+    cause: Schema.optional(Schema.DefectWithStack),
+  },
+) {}
+
+export const layer = <Self, T extends RpcHandlers>(
+  tag: Context.ServiceClass<Self, any, T>,
+  options: {
+    main: string;
+    schema: RpcHandlerDecoders<T>;
+  },
+) =>
+  Layer.effect(
+    tag,
+    Effect.gen(function* () {
+      const fiber = yield* maybeStartRpcServer(
+        fileURLToPath(options.main),
+      ).pipe(
+        Effect.flatMap(() => RpcSession),
+        Effect.map((session) =>
+          deserializeRpcHandlers(
+            session as SerializedRpcHandlers<T>,
+            options.schema,
+          ),
+        ),
+        Effect.retry(Schedule.exponential("100 millis")),
+        Effect.provide(
+          Layer.provideMerge(Lock.LockLive, RpcPaths.layer(options.main)),
+        ),
+        Effect.forkScoped,
+      );
+      return new Proxy({} as T, {
+        get(target, prop) {
+          return (...args: any[]) =>
+            Fiber.join(fiber).pipe(
+              Effect.flatMap((session) => session[prop as never](...args)),
+            );
+        },
+      });
+    }),
+  );
+
+const maybeStartRpcServer = Effect.fn(function* (main: string) {
+  const lock = yield* Lock.Lock;
+  if (!(yield* lock.check)) {
+    yield* Effect.logDebug("[RpcClient] Starting RPC server", main);
+    yield* ChildProcess.make("bun", ["run", main], {
+      stdout: "inherit",
+      stderr: "inherit",
+      detached: true,
+    });
+  } else {
+    yield* Effect.logDebug("[RpcClient] RPC server already running", main);
+  }
+});
+
+const RpcSession = Effect.gen(function* () {
+  const paths = yield* RpcPaths.RpcPaths;
+  const fs = yield* FileSystem.FileSystem;
+  const ws = yield* fs.readFileString(paths.url).pipe(
+    Effect.flatMap((url) =>
+      Effect.try({
+        try: () => new URL(url),
+        catch: (e) =>
+          new RpcClientError({
+            reason: "InvalidURL",
+            message: `"${url}" is not a valid URL`,
+            cause: e,
+          }),
+      }),
+    ),
+    Effect.flatMap((url) =>
+      Effect.callback<WebSocket, RpcClientError>((resume) => {
+        const ws = new WebSocket(url);
+        ws.onopen = () => resume(Effect.succeed(ws));
+        ws.onerror = (e) =>
+          resume(
+            Effect.fail(
+              new RpcClientError({
+                reason: "WebSocketError",
+                message: "WebSocket connection failed",
+                cause: e,
+              }),
+            ),
+          );
+        return Effect.sync(() => ws.close());
+      }),
+    ),
+    Effect.retry({
+      while: (e) => e.reason !== "WebSocketError",
+      schedule: Schedule.spaced("50 millis"),
+      times: 100,
+    }),
+    Effect.retry({
+      while: (e) => e.reason === "WebSocketError",
+      schedule: Schedule.exponential("100 millis"),
+      times: 3,
+    }),
+  );
+  const session = yield* Effect.acquireRelease(
+    Effect.sync(() =>
+      newWebSocketRpcSession<{
+        heartbeat: () => Promise<void>;
+        shutdown: () => Promise<void>;
+      }>(ws),
+    ),
+    (session) => Effect.sync(() => session[Symbol.dispose]()),
+  );
+  yield* Effect.promise(() => session.heartbeat()).pipe(
+    Effect.repeat(Schedule.spaced("1 second")),
+    Effect.ensuring(Effect.promise(() => session.shutdown())),
+    Effect.forkScoped,
+  );
+  return session;
+});

package/src/Sidecar/RpcHandler.ts

@@ -0,0 +1,169 @@
+import type { RpcCompatible } from "capnweb";
+import * as Cause from "effect/Cause";
+import * as Effect from "effect/Effect";
+import * as Exit from "effect/Exit";
+import * as Schema from "effect/Schema";
+
+type RpcHandler<Args extends Array<any>, Success, Error> = (
+  ...args: Args
+) => Effect.Effect<Success, Error>;
+
+type SerializedRpcHandler<Args extends Array<any>, Success, Error> = (
+  ...args: Args
+) => Promise<SerializedExit<RpcCompatible<Success>, RpcCompatible<Error>>>;
+
+type SerializedCause<Error> =
+  | { _tag: "Fail"; error: Error }
+  | { _tag: "Die"; defect: unknown }
+  | { _tag: "Interrupt"; fiberId: number | undefined };
+
+type SerializedExit<Success, Error> =
+  | { _tag: "Success"; value: Success }
+  | {
+      _tag: "Failure";
+      cause: Array<SerializedCause<Error>>;
+    };
+
+export type RpcHandlers = Record<string, RpcHandler<any, any, any>>;
+
+export type SerializedRpcHandlers<T extends RpcHandlers> = {
+  [K in keyof T]: T[K] extends RpcHandler<
+    infer Args,
+    infer Success,
+    infer Error
+  >
+    ? SerializedRpcHandler<Args, Success, Error>
+    : never;
+} extends infer O extends object & RpcCompatible<O>
+  ? O
+  : never;
+
+type RpcHandlerEncoder<Success, Error> = {
+  success: Schema.Encoder<Success>;
+  error: Schema.Encoder<Error>;
+};
+
+export type RpcHandlerEncoders<T extends RpcHandlers> = {
+  [K in keyof T]: T[K] extends RpcHandler<any, infer Success, infer Error>
+    ? RpcHandlerEncoder<Success, Error>
+    : never;
+};
+
+type RpcHandlerDecoder<Success, Error> = {
+  success: Schema.Decoder<Success>;
+  error: Schema.Decoder<Error>;
+};
+
+export type RpcHandlerDecoders<T extends RpcHandlers> = {
+  [K in keyof T]: T[K] extends RpcHandler<any, infer Success, infer Error>
+    ? RpcHandlerDecoder<Success, Error>
+    : never;
+};
+
+const serializeRpcHandler =
+  <Args extends Array<any>, Success, Error>(
+    handler: RpcHandler<Args, Success, Error>,
+    schema: RpcHandlerEncoder<Success, Error>,
+  ): SerializedRpcHandler<Args, Success, Error> =>
+  (...args) =>
+    handler(...args).pipe(
+      Effect.exit,
+      Effect.map((exit): SerializedExit<Success, Error> => {
+        if (exit._tag === "Success") {
+          return {
+            _tag: "Success",
+            value: Schema.encodeSync(schema.success)(exit.value),
+          } as const;
+        }
+        return {
+          _tag: "Failure",
+          cause: exit.cause.reasons.map((reason): SerializedCause<Error> => {
+            switch (reason._tag) {
+              case "Fail":
+                return {
+                  _tag: "Fail",
+                  error: Schema.encodeSync(schema.error)(reason.error),
+                };
+              case "Die":
+                return {
+                  _tag: "Die",
+                  defect: Schema.encodeSync(Schema.DefectWithStack)(
+                    reason.defect,
+                  ),
+                };
+              case "Interrupt":
+                return { _tag: "Interrupt", fiberId: reason.fiberId };
+            }
+          }),
+        } as const;
+      }),
+      Effect.runPromise,
+    );
+
+export const serializeRpcHandlers = <T extends RpcHandlers>(
+  handlers: T,
+  schema: RpcHandlerEncoders<T>,
+): SerializedRpcHandlers<T> =>
+  Object.fromEntries(
+    Object.entries(schema).map(([key, schema]) => [
+      key,
+      serializeRpcHandler(handlers[key], schema),
+    ]),
+  ) as SerializedRpcHandlers<T>;
+
+const deserializeRpcHandler =
+  <Args extends Array<any>, Success, Error>(
+    handler: SerializedRpcHandler<Args, Success, Error>,
+    schema: RpcHandlerDecoder<Success, Error>,
+  ): RpcHandler<Args, Success, Error> =>
+  (...args) =>
+    Effect.promise(async () => {
+      try {
+        return await handler(...args);
+      } catch (error) {
+        console.error("Error calling handler", error);
+        throw error;
+      }
+    }).pipe(
+      Effect.map((exit): Exit.Exit<Success, Error> => {
+        if (exit._tag === "Success") {
+          return Exit.succeed(Schema.decodeSync(schema.success)(exit.value));
+        }
+        return Exit.failCause(
+          Cause.fromReasons(
+            exit.cause.map((reason): Cause.Reason<Error> => {
+              switch (reason._tag) {
+                case "Fail":
+                  return Cause.makeFailReason(
+                    Schema.decodeSync(schema.error)(reason.error),
+                  );
+                case "Die":
+                  return Cause.makeDieReason(
+                    Schema.decodeSync(Schema.DefectWithStack)(reason.defect),
+                  );
+                case "Interrupt":
+                  return Cause.makeInterruptReason(reason.fiberId);
+              }
+            }),
+          ),
+        );
+      }),
+      Effect.flatMap((exit) => exit),
+    );
+
+export const deserializeRpcHandlers = <T extends RpcHandlers>(
+  handlers: SerializedRpcHandlers<T>,
+  schema: RpcHandlerDecoders<T>,
+): T =>
+  Object.fromEntries(
+    Object.entries(schema).map(([key, schema]) => [
+      key,
+      deserializeRpcHandler(handlers[key as never], schema),
+    ]),
+  ) as T;
+
+export const defineSchema = <T extends RpcHandlers>(schema: {
+  [K in keyof T]: T[K] extends RpcHandler<any, infer Success, infer Error>
+    ? { success: Schema.Codec<Success, any>; error: Schema.Codec<Error, any> }
+    : never;
+}) => schema as RpcHandlerEncoders<T> & RpcHandlerDecoders<T>;

package/src/Sidecar/RpcPaths.ts

@@ -0,0 +1,29 @@
+import * as Context from "effect/Context";
+import * as Effect from "effect/Effect";
+import * as FileSystem from "effect/FileSystem";
+import * as Hash from "effect/Hash";
+import * as Layer from "effect/Layer";
+import * as Path from "effect/Path";
+import * as AlchemyContext from "../AlchemyContext.ts";
+
+export class RpcPaths extends Context.Service<
+  RpcPaths,
+  { readonly lock: string; readonly url: string }
+>()("RpcPaths") {}
+
+export const layer = (main: string) =>
+  Layer.effect(
+    RpcPaths,
+    Effect.gen(function* () {
+      const { dotAlchemy } = yield* AlchemyContext.AlchemyContext;
+      const path = yield* Path.Path;
+      const fs = yield* FileSystem.FileSystem;
+      const id = Math.abs(Hash.string(main));
+      const dir = path.resolve(dotAlchemy, "local");
+      yield* fs.makeDirectory(dir, { recursive: true });
+      return RpcPaths.of({
+        lock: path.join(dir, `${id}.lock`),
+        url: path.join(dir, `${id}.url`),
+      });
+    }),
+  );

package/src/Sidecar/RpcServer.ts

@@ -0,0 +1,80 @@
+import * as Effect from "effect/Effect";
+import * as Fiber from "effect/Fiber";
+import * as FileSystem from "effect/FileSystem";
+import * as Layer from "effect/Layer";
+import * as Schedule from "effect/Schedule";
+import * as AlchemyContext from "../AlchemyContext.ts";
+import * as Lock from "./Lock.ts";
+import {
+  serializeRpcHandlers,
+  type RpcHandlerEncoders,
+  type RpcHandlers,
+} from "./RpcHandler.ts";
+import * as RpcPaths from "./RpcPaths.ts";
+import { makeBunWebSocketRpcServer } from "./RpcTransport.ts";
+
+export const layerServices = (main: string) =>
+  Layer.provideMerge(
+    Lock.LockLive,
+    Layer.provideMerge(RpcPaths.layer(main), AlchemyContext.AlchemyContextLive),
+  );
+
+export const makeRpcServer = Effect.fn(function* <T extends RpcHandlers, E, R>(
+  handlersEffect: Effect.Effect<T, E, R>,
+  schema: RpcHandlerEncoders<T>,
+) {
+  const lock = yield* Lock.Lock.use((lock) => lock.acquire);
+  const fs = yield* FileSystem.FileSystem;
+  const paths = yield* RpcPaths.RpcPaths;
+
+  const server = yield* Effect.gen(function* () {
+    const heartbeat = yield* Heartbeat;
+    const handlers = yield* handlersEffect;
+    const server = yield* Effect.acquireRelease(
+      Effect.sync(() =>
+        makeBunWebSocketRpcServer(() => {
+          const methods = Object.assign(
+            serializeRpcHandlers(handlers, schema),
+            {
+              heartbeat: () => Effect.runPromise(heartbeat.touch),
+              shutdown: () => Effect.runPromise(heartbeat.shutdown),
+            },
+          );
+          void methods.heartbeat();
+          return methods;
+        }),
+      ),
+      (server) => Effect.promise(() => server.stop(true)),
+    );
+    const address = `ws://${server.hostname}:${server.port}`;
+    yield* fs.writeFileString(paths.url, address);
+    yield* Effect.addFinalizer(() =>
+      fs.readFileString(paths.url).pipe(
+        Effect.flatMap((text) =>
+          text === address ? fs.remove(paths.url) : Effect.void,
+        ),
+        Effect.ignore,
+      ),
+    );
+    yield* heartbeat.await;
+  }).pipe(Effect.forkScoped);
+
+  yield* Fiber.joinAll([lock, server]);
+});
+
+const Heartbeat = Effect.gen(function* () {
+  let last = Date.now();
+  const fiber = yield* Effect.suspend(() => {
+    if (Date.now() - last > 10_000) {
+      return Effect.fail({ _tag: "Timeout" } as const);
+    }
+    return Effect.void;
+  }).pipe(Effect.repeat(Schedule.spaced("4 seconds")), Effect.forkScoped);
+  return {
+    touch: Effect.sync(() => {
+      last = Date.now();
+    }),
+    shutdown: Fiber.interrupt(fiber),
+    await: Fiber.join(fiber),
+  };
+});

package/src/Sidecar/RpcTransport.ts

@@ -0,0 +1,85 @@
+import type { ServerWebSocket } from "bun";
+import type { RpcCompatible, RpcTransport } from "capnweb";
+import { RpcSession } from "capnweb";
+
+export function makeBunWebSocketRpcServer<T extends RpcCompatible<T>>(
+  main: () => T,
+) {
+  return Bun.serve<{
+    transport: BunWebSocketRpcTransport;
+    session: RpcSession<T>;
+  }>({
+    port: 0,
+    fetch: (request, server) => {
+      if (server.upgrade(request, { data: undefined! })) {
+        return;
+      }
+      return new Response("Upgrade failed", { status: 400 });
+    },
+    websocket: {
+      open: (ws) => {
+        const transport = new BunWebSocketRpcTransport(ws);
+        const session = new RpcSession<T>(transport, main());
+        ws.data = { transport, session };
+      },
+      message: (ws, message) => {
+        ws.data.transport.dispatchMessage(message);
+      },
+      close: (ws, code, reason) => {
+        ws.data.transport.dispatchClose(code, reason);
+      },
+    },
+  });
+}
+
+class BunWebSocketRpcTransport implements RpcTransport {
+  private receiveQueue: Array<string> = [];
+  private receiveResolver?: (value: string) => void;
+  private receiveRejecter?: (reason: unknown) => void;
+  private error?: unknown;
+  constructor(private readonly ws: ServerWebSocket<any>) {}
+  async send(message: string): Promise<void> {
+    this.ws.send(message);
+  }
+  async receive(): Promise<string> {
+    const next = this.receiveQueue.shift();
+    if (next) {
+      return next;
+    } else if (this.error) {
+      throw this.error;
+    }
+    return new Promise((resolve, reject) => {
+      this.receiveResolver = resolve;
+      this.receiveRejecter = reject;
+    });
+  }
+  abort?(reason: any): void {
+    const message = reason instanceof Error ? reason.message : String(reason);
+    this.ws.close(3000, message);
+    this.error ??= reason;
+  }
+  dispatchMessage(data: string | Buffer<ArrayBuffer>): void {
+    if (this.error) {
+      return;
+    }
+    data = typeof data === "string" ? data : data.toString("utf-8");
+
+    if (this.receiveResolver) {
+      this.receiveResolver(data);
+      this.receiveResolver = undefined;
+      this.receiveRejecter = undefined;
+    } else {
+      this.receiveQueue.push(data);
+    }
+  }
+  dispatchClose(code: number, reason: string): void {
+    if (!this.error) {
+      this.error = new Error(`WebSocket closed with code ${code}: ${reason}`);
+      if (this.receiveRejecter) {
+        this.receiveRejecter(this.error);
+        this.receiveRejecter = undefined;
+        this.receiveResolver = undefined;
+      }
+    }
+  }
+}

package/src/Telemetry/Metrics.ts

@@ -30,6 +30,24 @@ export const cliCounter = Metric.counter("alchemy.cli.invocations", {
   incremental: true,
 });
 
+/**
+ * Counter for Cloudflare State Store bootstrap/deploy operations.
+ * Tagged per call with `op` (e.g. `deploy`) and `status`
+ * (`success`/`error`) so that the deploy success rate of the
+ * state store itself can be tracked separately from regular
+ * resource lifecycle ops.
+ */
+export const stateStoreCounter = Metric.counter(
+  "alchemy.state_store.operations",
+  {
+    description:
+      "Number of Cloudflare State Store deploy/bootstrap operations dispatched by alchemy.",
+    incremental: true,
+  },
+);
+
+export type StateStoreOp = "deploy";
+
 /**
  * Wraps a resource lifecycle Effect to record a counter + timer entry,
  * tagged with `resource_type`, `op`, and `status` (`success`/`error`).
@@ -75,6 +93,26 @@ export const recordCli =
       ),
     );
 
+/**
+ * Wraps a Cloudflare State Store deploy/bootstrap Effect to bump
+ * {@link stateStoreCounter} with the outcome (`success`/`error`),
+ * tagged with the given `op`.
+ */
+export const recordStateStoreOp =
+  (op: StateStoreOp) =>
+  <A, E, R>(self: Effect.Effect<A, E, R>): Effect.Effect<A, E, R> =>
+    self.pipe(
+      Effect.onExit((exit) =>
+        Metric.update(
+          Metric.withAttributes(stateStoreCounter, {
+            op,
+            status: Exit.isSuccess(exit) ? "success" : "error",
+          }),
+          1,
+        ),
+      ),
+    );
+
 export type ResourceOp = "precreate" | "create" | "update" | "delete" | "read";
 
 const elapsed = (startNs: bigint): Duration.Duration =>

package/src/Util/PlatformServices.ts

@@ -5,6 +5,8 @@ import type { Path } from "effect/Path";
 import type { Teardown } from "effect/Runtime";
 import type { Stdio } from "effect/Stdio";
 import type { Terminal } from "effect/Terminal";
+import type { HttpServer } from "effect/unstable/http/HttpServer";
+import type { ServeError } from "effect/unstable/http/HttpServerError";
 import type { ChildProcessSpawner } from "effect/unstable/process/ChildProcessSpawner";
 import type { WebSocketConstructor } from "effect/unstable/socket/Socket";
 
@@ -60,3 +62,20 @@ export const runMain = <E, A>(
     );
   }
 };
+
+export const httpServer = (
+  port: number = 0,
+  host: string = "localhost",
+): Layer.Layer<HttpServer, ServeError> =>
+  Effect.promise(async () => {
+    if (isBun) {
+      const BunHttpServer = await import("@effect/platform-bun/BunHttpServer");
+      return BunHttpServer.layer({ hostname: host, port });
+    } else {
+      const [NodeHttpServer, Http] = await Promise.all([
+        import("@effect/platform-node/NodeHttpServer"),
+        import("node:http"),
+      ]);
+      return NodeHttpServer.layerServer(Http.createServer, { host, port });
+    }
+  }).pipe(Layer.unwrap);

package/src/Util/layer-scoped.ts

@@ -0,0 +1,25 @@
+import type * as Context from "effect/Context";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import type * as Scope from "effect/Scope";
+
+/**
+ * Builds a layer with the current scope.
+ * @param layer - The layer to build.
+ * @returns A scoped effect returning a Context with the provided layer.
+ */
+export const buildLayerScoped = <ROut, E, RIn>(
+  layer: Layer.Layer<ROut, E, RIn>,
+): Effect.Effect<Context.Context<ROut>, E, RIn | Scope.Scope> =>
+  Effect.flatMap(Effect.scope, (scope) => Layer.buildWithScope(layer, scope));
+
+/**
+ * Builds a layer with the current scope and provides it to the given effect.
+ * @param layer - The layer to build.
+ */
+export const provideLayerScoped =
+  <ROut, E1, RIn>(layer: Layer.Layer<ROut, E1, RIn>) =>
+  <A, E2, R>(effect: Effect.Effect<A, E2, R>) =>
+    Effect.flatMap(buildLayerScoped(layer), (context) =>
+      Effect.provideContext(effect, context),
+    );