alchemy 2.0.0-beta.23 → 2.0.0-beta.25

package/src/Sidecar/Lock.ts

@@ -0,0 +1,190 @@
+import * as Context from "effect/Context";
+import * as Duration from "effect/Duration";
+import * as Effect from "effect/Effect";
+import * as Fiber from "effect/Fiber";
+import * as FileSystem from "effect/FileSystem";
+import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
+import * as Schedule from "effect/Schedule";
+import * as Schema from "effect/Schema";
+import type * as Scope from "effect/Scope";
+import * as RpcPaths from "./RpcPaths.ts";
+
+export class LockError extends Schema.TaggedErrorClass<LockError>()(
+  "LockError",
+  {
+    reason: Schema.Literals([
+      "Cancelled",
+      "Conflict",
+      "Invalid",
+      "Timeout",
+      "PlatformError",
+    ]),
+    message: Schema.String,
+    cause: Schema.optional(Schema.DefectWithStack),
+  },
+) {}
+
+export class Lock extends Context.Service<
+  Lock,
+  {
+    readonly check: Effect.Effect<boolean>;
+    readonly acquire: Effect.Effect<
+      Fiber.Fiber<number, LockError>,
+      LockError,
+      Scope.Scope
+    >;
+  }
+>()("Lock") {}
+
+const LOCK_TTL = Duration.seconds(10);
+const TOUCH_INTERVAL = Duration.seconds(1);
+
+const make = Effect.gen(function* () {
+  const fs = yield* FileSystem.FileSystem;
+  const paths = yield* RpcPaths.RpcPaths;
+
+  const readLockFile = fs.readFileString(paths.lock).pipe(
+    Effect.mapError(
+      (e) =>
+        new LockError({
+          reason: "PlatformError",
+          message:
+            e.reason._tag === "NotFound"
+              ? "Lock file not found"
+              : "Failed to read lock file",
+          cause: e,
+        }),
+    ),
+    Effect.flatMap((text) => {
+      const pid = Number.parseInt(text);
+      return Number.isNaN(pid)
+        ? Effect.fail(
+            new LockError({
+              reason: "Invalid",
+              message: "Lock file is invalid",
+            }),
+          )
+        : Effect.succeed(pid);
+    }),
+  );
+
+  const isLockProcessAlive = readLockFile.pipe(
+    Effect.flatMap((pid) =>
+      Effect.sync(() => {
+        try {
+          process.kill(pid, 0);
+          return true;
+        } catch {
+          return false;
+        }
+      }),
+    ),
+    Effect.orElseSucceed(() => false),
+  );
+
+  const isLockFileStale = fs.stat(paths.lock).pipe(
+    Effect.map((stat) => {
+      const mtime = Option.getOrElse(stat.mtime, () => new Date(0));
+      return mtime.getTime() < Date.now() - Duration.toMillis(LOCK_TTL);
+    }),
+    Effect.orElseSucceed(() => true),
+  );
+
+  const isLockValid = Effect.zipWith(
+    isLockProcessAlive,
+    isLockFileStale,
+    (alive, stale) => alive && !stale,
+    {
+      concurrent: true,
+    },
+  );
+
+  const makeLockFile = fs
+    .writeFileString(paths.lock, process.pid.toString(), { flag: "wx" })
+    .pipe(
+      Effect.mapError((e) =>
+        e.reason._tag === "AlreadyExists"
+          ? new LockError({
+              reason: "Conflict",
+              message: "Lock already held by another process",
+            })
+          : new LockError({
+              reason: "PlatformError",
+              message: "Failed to create lock file",
+              cause: e,
+            }),
+      ),
+    );
+
+  const assertOwnLock = readLockFile.pipe(
+    Effect.flatMap((pid) =>
+      pid === process.pid
+        ? Effect.void
+        : Effect.fail(
+            new LockError({
+              reason: "Conflict",
+              message: "Lock not held by this process",
+            }),
+          ),
+    ),
+  );
+
+  const removeLock = fs.remove(paths.lock, { force: true }).pipe(
+    Effect.mapError(
+      (e) =>
+        new LockError({
+          reason: "PlatformError",
+          message: "Failed to remove lock file",
+          cause: e,
+        }),
+    ),
+  );
+
+  const releaseLock = assertOwnLock.pipe(Effect.andThen(() => removeLock));
+
+  const acquireLock = makeLockFile.pipe(
+    Effect.catchIf(
+      (e) => e.reason === "Conflict",
+      (e) =>
+        isLockValid.pipe(
+          Effect.flatMap((valid) =>
+            valid
+              ? Effect.fail(e)
+              : removeLock.pipe(Effect.andThen(() => makeLockFile)),
+          ),
+        ),
+    ),
+  );
+
+  const touchLock = assertOwnLock.pipe(
+    Effect.flatMap(() => {
+      const now = Date.now();
+      return fs.utimes(paths.lock, now, now).pipe(
+        Effect.mapError(
+          (e) =>
+            new LockError({
+              reason: "PlatformError",
+              message: "Failed to update lock file",
+              cause: e,
+            }),
+        ),
+      );
+    }),
+  );
+
+  return Lock.of({
+    check: isLockValid,
+    acquire: acquireLock.pipe(
+      Effect.flatMap(() =>
+        touchLock.pipe(
+          Effect.repeat(Schedule.spaced(TOUCH_INTERVAL)),
+          Effect.ensuring(releaseLock.pipe(Effect.ignore)),
+          Effect.forkScoped,
+        ),
+      ),
+    ),
+  });
+});
+
+export const LockLive = Layer.effect(Lock, make);

package/src/Sidecar/RpcClient.ts

@@ -0,0 +1,141 @@
+import { newWebSocketRpcSession } from "capnweb";
+import * as Context from "effect/Context";
+import * as Effect from "effect/Effect";
+import * as Fiber from "effect/Fiber";
+import * as FileSystem from "effect/FileSystem";
+import * as Layer from "effect/Layer";
+import * as Schedule from "effect/Schedule";
+import * as Schema from "effect/Schema";
+import * as ChildProcess from "effect/unstable/process/ChildProcess";
+import { fileURLToPath } from "node:url";
+import * as Lock from "./Lock.ts";
+import {
+  deserializeRpcHandlers,
+  type RpcHandlerDecoders,
+  type RpcHandlers,
+  type SerializedRpcHandlers,
+} from "./RpcHandler.ts";
+import * as RpcPaths from "./RpcPaths.ts";
+
+export const RpcClientService = <Self, T extends RpcHandlers>() =>
+  Context.Service<Self, T>();
+
+class RpcClientError extends Schema.TaggedErrorClass<RpcClientError>()(
+  "RpcClientError",
+  {
+    reason: Schema.Literals(["InvalidURL", "WebSocketError"]),
+    message: Schema.String,
+    cause: Schema.optional(Schema.DefectWithStack),
+  },
+) {}
+
+export const layer = <Self, T extends RpcHandlers>(
+  tag: Context.ServiceClass<Self, any, T>,
+  options: {
+    main: string;
+    schema: RpcHandlerDecoders<T>;
+  },
+) =>
+  Layer.effect(
+    tag,
+    Effect.gen(function* () {
+      const fiber = yield* maybeStartRpcServer(
+        fileURLToPath(options.main),
+      ).pipe(
+        Effect.flatMap(() => RpcSession),
+        Effect.map((session) =>
+          deserializeRpcHandlers(
+            session as SerializedRpcHandlers<T>,
+            options.schema,
+          ),
+        ),
+        Effect.retry(Schedule.exponential("100 millis")),
+        Effect.provide(
+          Layer.provideMerge(Lock.LockLive, RpcPaths.layer(options.main)),
+        ),
+        Effect.forkScoped,
+      );
+      return new Proxy({} as T, {
+        get(target, prop) {
+          return (...args: any[]) =>
+            Fiber.join(fiber).pipe(
+              Effect.flatMap((session) => session[prop as never](...args)),
+            );
+        },
+      });
+    }),
+  );
+
+const maybeStartRpcServer = Effect.fn(function* (main: string) {
+  const lock = yield* Lock.Lock;
+  if (!(yield* lock.check)) {
+    yield* Effect.logDebug("[RpcClient] Starting RPC server", main);
+    yield* ChildProcess.make("bun", ["run", main], {
+      stdout: "inherit",
+      stderr: "inherit",
+      detached: true,
+    });
+  } else {
+    yield* Effect.logDebug("[RpcClient] RPC server already running", main);
+  }
+});
+
+const RpcSession = Effect.gen(function* () {
+  const paths = yield* RpcPaths.RpcPaths;
+  const fs = yield* FileSystem.FileSystem;
+  const ws = yield* fs.readFileString(paths.url).pipe(
+    Effect.flatMap((url) =>
+      Effect.try({
+        try: () => new URL(url),
+        catch: (e) =>
+          new RpcClientError({
+            reason: "InvalidURL",
+            message: `"${url}" is not a valid URL`,
+            cause: e,
+          }),
+      }),
+    ),
+    Effect.flatMap((url) =>
+      Effect.callback<WebSocket, RpcClientError>((resume) => {
+        const ws = new WebSocket(url);
+        ws.onopen = () => resume(Effect.succeed(ws));
+        ws.onerror = (e) =>
+          resume(
+            Effect.fail(
+              new RpcClientError({
+                reason: "WebSocketError",
+                message: "WebSocket connection failed",
+                cause: e,
+              }),
+            ),
+          );
+        return Effect.sync(() => ws.close());
+      }),
+    ),
+    Effect.retry({
+      while: (e) => e.reason !== "WebSocketError",
+      schedule: Schedule.spaced("50 millis"),
+      times: 100,
+    }),
+    Effect.retry({
+      while: (e) => e.reason === "WebSocketError",
+      schedule: Schedule.exponential("100 millis"),
+      times: 3,
+    }),
+  );
+  const session = yield* Effect.acquireRelease(
+    Effect.sync(() =>
+      newWebSocketRpcSession<{
+        heartbeat: () => Promise<void>;
+        shutdown: () => Promise<void>;
+      }>(ws),
+    ),
+    (session) => Effect.sync(() => session[Symbol.dispose]()),
+  );
+  yield* Effect.promise(() => session.heartbeat()).pipe(
+    Effect.repeat(Schedule.spaced("1 second")),
+    Effect.ensuring(Effect.promise(() => session.shutdown())),
+    Effect.forkScoped,
+  );
+  return session;
+});

package/src/Sidecar/RpcHandler.ts

@@ -0,0 +1,169 @@
+import type { RpcCompatible } from "capnweb";
+import * as Cause from "effect/Cause";
+import * as Effect from "effect/Effect";
+import * as Exit from "effect/Exit";
+import * as Schema from "effect/Schema";
+
+type RpcHandler<Args extends Array<any>, Success, Error> = (
+  ...args: Args
+) => Effect.Effect<Success, Error>;
+
+type SerializedRpcHandler<Args extends Array<any>, Success, Error> = (
+  ...args: Args
+) => Promise<SerializedExit<RpcCompatible<Success>, RpcCompatible<Error>>>;
+
+type SerializedCause<Error> =
+  | { _tag: "Fail"; error: Error }
+  | { _tag: "Die"; defect: unknown }
+  | { _tag: "Interrupt"; fiberId: number | undefined };
+
+type SerializedExit<Success, Error> =
+  | { _tag: "Success"; value: Success }
+  | {
+      _tag: "Failure";
+      cause: Array<SerializedCause<Error>>;
+    };
+
+export type RpcHandlers = Record<string, RpcHandler<any, any, any>>;
+
+export type SerializedRpcHandlers<T extends RpcHandlers> = {
+  [K in keyof T]: T[K] extends RpcHandler<
+    infer Args,
+    infer Success,
+    infer Error
+  >
+    ? SerializedRpcHandler<Args, Success, Error>
+    : never;
+} extends infer O extends object & RpcCompatible<O>
+  ? O
+  : never;
+
+type RpcHandlerEncoder<Success, Error> = {
+  success: Schema.Encoder<Success>;
+  error: Schema.Encoder<Error>;
+};
+
+export type RpcHandlerEncoders<T extends RpcHandlers> = {
+  [K in keyof T]: T[K] extends RpcHandler<any, infer Success, infer Error>
+    ? RpcHandlerEncoder<Success, Error>
+    : never;
+};
+
+type RpcHandlerDecoder<Success, Error> = {
+  success: Schema.Decoder<Success>;
+  error: Schema.Decoder<Error>;
+};
+
+export type RpcHandlerDecoders<T extends RpcHandlers> = {
+  [K in keyof T]: T[K] extends RpcHandler<any, infer Success, infer Error>
+    ? RpcHandlerDecoder<Success, Error>
+    : never;
+};
+
+const serializeRpcHandler =
+  <Args extends Array<any>, Success, Error>(
+    handler: RpcHandler<Args, Success, Error>,
+    schema: RpcHandlerEncoder<Success, Error>,
+  ): SerializedRpcHandler<Args, Success, Error> =>
+  (...args) =>
+    handler(...args).pipe(
+      Effect.exit,
+      Effect.map((exit): SerializedExit<Success, Error> => {
+        if (exit._tag === "Success") {
+          return {
+            _tag: "Success",
+            value: Schema.encodeSync(schema.success)(exit.value),
+          } as const;
+        }
+        return {
+          _tag: "Failure",
+          cause: exit.cause.reasons.map((reason): SerializedCause<Error> => {
+            switch (reason._tag) {
+              case "Fail":
+                return {
+                  _tag: "Fail",
+                  error: Schema.encodeSync(schema.error)(reason.error),
+                };
+              case "Die":
+                return {
+                  _tag: "Die",
+                  defect: Schema.encodeSync(Schema.DefectWithStack)(
+                    reason.defect,
+                  ),
+                };
+              case "Interrupt":
+                return { _tag: "Interrupt", fiberId: reason.fiberId };
+            }
+          }),
+        } as const;
+      }),
+      Effect.runPromise,
+    );
+
+export const serializeRpcHandlers = <T extends RpcHandlers>(
+  handlers: T,
+  schema: RpcHandlerEncoders<T>,
+): SerializedRpcHandlers<T> =>
+  Object.fromEntries(
+    Object.entries(schema).map(([key, schema]) => [
+      key,
+      serializeRpcHandler(handlers[key], schema),
+    ]),
+  ) as SerializedRpcHandlers<T>;
+
+const deserializeRpcHandler =
+  <Args extends Array<any>, Success, Error>(
+    handler: SerializedRpcHandler<Args, Success, Error>,
+    schema: RpcHandlerDecoder<Success, Error>,
+  ): RpcHandler<Args, Success, Error> =>
+  (...args) =>
+    Effect.promise(async () => {
+      try {
+        return await handler(...args);
+      } catch (error) {
+        console.error("Error calling handler", error);
+        throw error;
+      }
+    }).pipe(
+      Effect.map((exit): Exit.Exit<Success, Error> => {
+        if (exit._tag === "Success") {
+          return Exit.succeed(Schema.decodeSync(schema.success)(exit.value));
+        }
+        return Exit.failCause(
+          Cause.fromReasons(
+            exit.cause.map((reason): Cause.Reason<Error> => {
+              switch (reason._tag) {
+                case "Fail":
+                  return Cause.makeFailReason(
+                    Schema.decodeSync(schema.error)(reason.error),
+                  );
+                case "Die":
+                  return Cause.makeDieReason(
+                    Schema.decodeSync(Schema.DefectWithStack)(reason.defect),
+                  );
+                case "Interrupt":
+                  return Cause.makeInterruptReason(reason.fiberId);
+              }
+            }),
+          ),
+        );
+      }),
+      Effect.flatMap((exit) => exit),
+    );
+
+export const deserializeRpcHandlers = <T extends RpcHandlers>(
+  handlers: SerializedRpcHandlers<T>,
+  schema: RpcHandlerDecoders<T>,
+): T =>
+  Object.fromEntries(
+    Object.entries(schema).map(([key, schema]) => [
+      key,
+      deserializeRpcHandler(handlers[key as never], schema),
+    ]),
+  ) as T;
+
+export const defineSchema = <T extends RpcHandlers>(schema: {
+  [K in keyof T]: T[K] extends RpcHandler<any, infer Success, infer Error>
+    ? { success: Schema.Codec<Success, any>; error: Schema.Codec<Error, any> }
+    : never;
+}) => schema as RpcHandlerEncoders<T> & RpcHandlerDecoders<T>;

package/src/Sidecar/RpcPaths.ts

@@ -0,0 +1,29 @@
+import * as Context from "effect/Context";
+import * as Effect from "effect/Effect";
+import * as FileSystem from "effect/FileSystem";
+import * as Hash from "effect/Hash";
+import * as Layer from "effect/Layer";
+import * as Path from "effect/Path";
+import * as AlchemyContext from "../AlchemyContext.ts";
+
+export class RpcPaths extends Context.Service<
+  RpcPaths,
+  { readonly lock: string; readonly url: string }
+>()("RpcPaths") {}
+
+export const layer = (main: string) =>
+  Layer.effect(
+    RpcPaths,
+    Effect.gen(function* () {
+      const { dotAlchemy } = yield* AlchemyContext.AlchemyContext;
+      const path = yield* Path.Path;
+      const fs = yield* FileSystem.FileSystem;
+      const id = Math.abs(Hash.string(main));
+      const dir = path.resolve(dotAlchemy, "local");
+      yield* fs.makeDirectory(dir, { recursive: true });
+      return RpcPaths.of({
+        lock: path.join(dir, `${id}.lock`),
+        url: path.join(dir, `${id}.url`),
+      });
+    }),
+  );

package/src/Sidecar/RpcServer.ts

@@ -0,0 +1,80 @@
+import * as Effect from "effect/Effect";
+import * as Fiber from "effect/Fiber";
+import * as FileSystem from "effect/FileSystem";
+import * as Layer from "effect/Layer";
+import * as Schedule from "effect/Schedule";
+import * as AlchemyContext from "../AlchemyContext.ts";
+import * as Lock from "./Lock.ts";
+import {
+  serializeRpcHandlers,
+  type RpcHandlerEncoders,
+  type RpcHandlers,
+} from "./RpcHandler.ts";
+import * as RpcPaths from "./RpcPaths.ts";
+import { makeBunWebSocketRpcServer } from "./RpcTransport.ts";
+
+export const layerServices = (main: string) =>
+  Layer.provideMerge(
+    Lock.LockLive,
+    Layer.provideMerge(RpcPaths.layer(main), AlchemyContext.AlchemyContextLive),
+  );
+
+export const makeRpcServer = Effect.fn(function* <T extends RpcHandlers, E, R>(
+  handlersEffect: Effect.Effect<T, E, R>,
+  schema: RpcHandlerEncoders<T>,
+) {
+  const lock = yield* Lock.Lock.use((lock) => lock.acquire);
+  const fs = yield* FileSystem.FileSystem;
+  const paths = yield* RpcPaths.RpcPaths;
+
+  const server = yield* Effect.gen(function* () {
+    const heartbeat = yield* Heartbeat;
+    const handlers = yield* handlersEffect;
+    const server = yield* Effect.acquireRelease(
+      Effect.sync(() =>
+        makeBunWebSocketRpcServer(() => {
+          const methods = Object.assign(
+            serializeRpcHandlers(handlers, schema),
+            {
+              heartbeat: () => Effect.runPromise(heartbeat.touch),
+              shutdown: () => Effect.runPromise(heartbeat.shutdown),
+            },
+          );
+          void methods.heartbeat();
+          return methods;
+        }),
+      ),
+      (server) => Effect.promise(() => server.stop(true)),
+    );
+    const address = `ws://${server.hostname}:${server.port}`;
+    yield* fs.writeFileString(paths.url, address);
+    yield* Effect.addFinalizer(() =>
+      fs.readFileString(paths.url).pipe(
+        Effect.flatMap((text) =>
+          text === address ? fs.remove(paths.url) : Effect.void,
+        ),
+        Effect.ignore,
+      ),
+    );
+    yield* heartbeat.await;
+  }).pipe(Effect.forkScoped);
+
+  yield* Fiber.joinAll([lock, server]);
+});
+
+const Heartbeat = Effect.gen(function* () {
+  let last = Date.now();
+  const fiber = yield* Effect.suspend(() => {
+    if (Date.now() - last > 10_000) {
+      return Effect.fail({ _tag: "Timeout" } as const);
+    }
+    return Effect.void;
+  }).pipe(Effect.repeat(Schedule.spaced("4 seconds")), Effect.forkScoped);
+  return {
+    touch: Effect.sync(() => {
+      last = Date.now();
+    }),
+    shutdown: Fiber.interrupt(fiber),
+    await: Fiber.join(fiber),
+  };
+});

package/src/Sidecar/RpcTransport.ts

@@ -0,0 +1,85 @@
+import type { ServerWebSocket } from "bun";
+import type { RpcCompatible, RpcTransport } from "capnweb";
+import { RpcSession } from "capnweb";
+
+export function makeBunWebSocketRpcServer<T extends RpcCompatible<T>>(
+  main: () => T,
+) {
+  return Bun.serve<{
+    transport: BunWebSocketRpcTransport;
+    session: RpcSession<T>;
+  }>({
+    port: 0,
+    fetch: (request, server) => {
+      if (server.upgrade(request, { data: undefined! })) {
+        return;
+      }
+      return new Response("Upgrade failed", { status: 400 });
+    },
+    websocket: {
+      open: (ws) => {
+        const transport = new BunWebSocketRpcTransport(ws);
+        const session = new RpcSession<T>(transport, main());
+        ws.data = { transport, session };
+      },
+      message: (ws, message) => {
+        ws.data.transport.dispatchMessage(message);
+      },
+      close: (ws, code, reason) => {
+        ws.data.transport.dispatchClose(code, reason);
+      },
+    },
+  });
+}
+
+class BunWebSocketRpcTransport implements RpcTransport {
+  private receiveQueue: Array<string> = [];
+  private receiveResolver?: (value: string) => void;
+  private receiveRejecter?: (reason: unknown) => void;
+  private error?: unknown;
+  constructor(private readonly ws: ServerWebSocket<any>) {}
+  async send(message: string): Promise<void> {
+    this.ws.send(message);
+  }
+  async receive(): Promise<string> {
+    const next = this.receiveQueue.shift();
+    if (next) {
+      return next;
+    } else if (this.error) {
+      throw this.error;
+    }
+    return new Promise((resolve, reject) => {
+      this.receiveResolver = resolve;
+      this.receiveRejecter = reject;
+    });
+  }
+  abort?(reason: any): void {
+    const message = reason instanceof Error ? reason.message : String(reason);
+    this.ws.close(3000, message);
+    this.error ??= reason;
+  }
+  dispatchMessage(data: string | Buffer<ArrayBuffer>): void {
+    if (this.error) {
+      return;
+    }
+    data = typeof data === "string" ? data : data.toString("utf-8");
+
+    if (this.receiveResolver) {
+      this.receiveResolver(data);
+      this.receiveResolver = undefined;
+      this.receiveRejecter = undefined;
+    } else {
+      this.receiveQueue.push(data);
+    }
+  }
+  dispatchClose(code: number, reason: string): void {
+    if (!this.error) {
+      this.error = new Error(`WebSocket closed with code ${code}: ${reason}`);
+      if (this.receiveRejecter) {
+        this.receiveRejecter(this.error);
+        this.receiveRejecter = undefined;
+        this.receiveResolver = undefined;
+      }
+    }
+  }
+}