alchemy 2.0.0-beta.42 → 2.0.0-beta.44

package/src/Local/RpcServer.ts

@@ -0,0 +1,138 @@
+import * as Console from "effect/Console";
+import * as Context from "effect/Context";
+import * as Deferred from "effect/Deferred";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import type * as Scope from "effect/Scope";
+import * as FetchHttpClient from "effect/unstable/http/FetchHttpClient";
+import type { HttpClient } from "effect/unstable/http/HttpClient";
+import type { ProviderService } from "../Provider.ts";
+import type { ResourceLike } from "../Resource.ts";
+import {
+  platformLayer,
+  PlatformServices,
+  runMain,
+} from "../Util/PlatformServices.ts";
+import * as RpcSerialization from "./RpcSerialization.ts";
+import * as RpcServerEnvironment from "./RpcServerEnvironment.ts";
+import {
+  makeServerRpcSession,
+  type ServerRpcSession,
+  type ServerWebSocketLike,
+} from "./RpcServerSession.ts";
+
+/**
+ * A service that exposes one or more resource providers over RPC.
+ * This returns `never` because it is meant to be used with `Layer.launch` (see {@link launch}).
+ */
+export class RpcServer extends Context.Service<RpcServer, never>()(
+  "alchemy/Local/RpcServer",
+) {}
+
+/**
+ * The RPC API that is implemented by the server and consumed by {@link RpcProviderProxy}.
+ */
+export interface RpcProxyApi {
+  /**
+   * Retrieves a provider from the RPC server context.
+   * The consumer must unwrap the provider using {@link RpcSerialization.unwrapRpcHandlers} before using it.
+   */
+  readonly getProvider: <R extends ResourceLike>(
+    type: R["Type"],
+  ) => Promise<RpcSerialization.RpcWrapped<ProviderService<R>>>;
+}
+
+const serverPlatformLayer = platformLayer({
+  bun: async () => {
+    const { RpcServerBun } = await import("./RpcServerBun.ts");
+    return RpcServerBun;
+  },
+  node: async () => {
+    const { RpcServerNode } = await import("./RpcServerNode.ts");
+    return RpcServerNode;
+  },
+});
+
+/**
+ * Launches an RPC server that serves the given providers.
+ * Alchemy globals such as `AlchemyContext`, `Profile`, and `Stack` are inherited from the parent via {@link RpcServerEnvironment.fromEnv} and should not be provided manually.
+ * `PlatformServices` and `HttpClient` are also included.
+ *
+ * @example
+ * ```ts
+ * RpcServer.launch(
+ *   Layer.merge(
+ *     FunctionProvider,
+ *     QueueProvider,
+ *   ),
+ * );
+ * ```
+ *
+ * @param providers - A layer containing the providers to serve.
+ */
+export const launch = <ROut, E>(
+  providers: Layer.Layer<
+    ROut,
+    E,
+    | Scope.Scope
+    | RpcServerEnvironment.RpcEnvironmentServices
+    | PlatformServices
+    | HttpClient
+  >,
+) =>
+  serverPlatformLayer.pipe(
+    Layer.provide(providers),
+    Layer.provide(RpcServerEnvironment.fromEnv()),
+    Layer.provide(Layer.merge(PlatformServices, FetchHttpClient.layer)),
+    Layer.launch,
+    Effect.scoped,
+    runMain,
+  );
+
+/**
+ * Constructs an `RpcServer` layer using the given server implementation.
+ * @param serve - A function that spawns a websocket server and returns its URL.
+ * @returns An `RpcServer` layer.
+ */
+export const layerServer = (
+  serve: (handlers: {
+    /** Creates a new RPC session over the given websocket. */
+    createRpcSession: (
+      ws: ServerWebSocketLike,
+    ) => ServerRpcSession<RpcProxyApi>;
+    /** Called when the parent connection, indicated by the `/parent` path, is established. */
+    parentConnected: () => void;
+    /** Called when the parent disconnects. The server will shut down when this is called. */
+    parentDisconnected: () => void;
+  }) => Effect.Effect<{ readonly url: string }, never, Scope.Scope>,
+) =>
+  Layer.effect(
+    RpcServer,
+    Effect.gen(function* () {
+      const context = yield* Effect.context();
+      const connected = yield* Deferred.make<void>();
+      const disconnected = yield* Deferred.make<void>();
+      const { url } = yield* serve({
+        createRpcSession: (ws) =>
+          makeServerRpcSession<RpcProxyApi>(ws, {
+            getProvider: async <R extends ResourceLike>(type: R["Type"]) => {
+              const provider = context.mapUnsafe.get(type);
+              if (!provider) {
+                throw new Error(`Provider "${type}" not found`);
+              }
+              return RpcSerialization.wrapRpcHandlers(
+                provider as ProviderService<R>,
+                ["tail"],
+              );
+            },
+          }),
+        parentConnected: () => Deferred.doneUnsafe(connected, Effect.void),
+        parentDisconnected: () =>
+          Deferred.doneUnsafe(disconnected, Effect.void),
+      });
+      yield* Console.log(`<ALCHEMY_RPC_ADDRESS>${url}</ALCHEMY_RPC_ADDRESS>`);
+      yield* Deferred.await(connected).pipe(Effect.timeout("10 seconds")); // TODO(john): should the timeout be shorter?
+      yield* Deferred.await(disconnected);
+      return yield* Effect.interrupt;
+    }),
+  );

package/src/Local/RpcServerBun.ts

@@ -0,0 +1,59 @@
+import * as Effect from "effect/Effect";
+import * as RpcServer from "./RpcServer.ts";
+import type { ServerRpcSession } from "./RpcServerSession.ts";
+
+export const RpcServerBun = RpcServer.layerServer(
+  Effect.fnUntraced(function* ({
+    parentConnected,
+    parentDisconnected,
+    createRpcSession,
+  }) {
+    const server = yield* Effect.sync(() =>
+      Bun.serve<
+        { type: "session"; session: ServerRpcSession<any> } | { type: "parent" }
+      >({
+        port: 0,
+        fetch: (request, server) => {
+          const url = new URL(request.url);
+          if (
+            server.upgrade(request, {
+              data:
+                url.pathname === "/parent" ? { type: "parent" } : undefined!,
+            })
+          ) {
+            return;
+          }
+          return new Response("Upgrade failed", { status: 400 });
+        },
+        websocket: {
+          open: (ws) => {
+            if (ws.data && ws.data.type === "parent") {
+              parentConnected();
+            } else {
+              ws.data = {
+                type: "session",
+                session: createRpcSession(ws),
+              };
+            }
+          },
+          message: (ws, message) => {
+            if (ws.data.type === "session") {
+              ws.data.session.dispatch.message(message);
+            }
+          },
+          close: (ws, code, reason) => {
+            if (ws.data.type === "session") {
+              ws.data.session.dispatch.close(code, reason);
+            } else if (ws.data.type === "parent") {
+              parentDisconnected();
+            }
+          },
+        },
+      }),
+    );
+    yield* Effect.addFinalizer(() => Effect.sync(() => server.stop(true)));
+    return {
+      url: `ws://${server.hostname}:${server.port}`,
+    };
+  }),
+);

package/src/Local/RpcServerEnvironment.ts

@@ -0,0 +1,55 @@
+import * as Config from "effect/Config";
+import * as ConfigProvider from "effect/ConfigProvider";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
+import { AlchemyContext } from "../AlchemyContext.ts";
+import { AuthProviders } from "../Auth/AuthProvider.ts";
+import { CredentialsStoreLive } from "../Auth/Credentials.ts";
+import { ProfileLive, withProfileOverride } from "../Auth/Profile.ts";
+import { Stack } from "../Stack.ts";
+import { Stage } from "../Stage.ts";
+import { loadConfigProvider } from "../Util/ConfigProvider.ts";
+
+export interface RpcServerEnvironment {
+  alchemyContext: AlchemyContext["Service"];
+  profile: string | undefined;
+  envFile: string | undefined;
+  stack: {
+    name: string;
+    stage: string;
+  };
+}
+
+export type RpcEnvironmentServices = Layer.Success<ReturnType<typeof layer>>;
+
+export const layer = (environment: RpcServerEnvironment) =>
+  Layer.mergeAll(
+    ProfileLive,
+    CredentialsStoreLive,
+    Layer.succeed(AuthProviders, {}),
+    ConfigProvider.layer(
+      loadConfigProvider(Option.fromUndefinedOr(environment.envFile)).pipe(
+        Effect.map((base) => withProfileOverride(base, environment.profile)),
+      ),
+    ),
+    Layer.succeed(AlchemyContext, environment.alchemyContext),
+    Layer.succeed(Stack, {
+      name: environment.stack.name,
+      stage: environment.stack.stage,
+      resources: {},
+      bindings: {},
+      actions: {},
+    }),
+    Layer.succeed(Stage, environment.stack.stage),
+  );
+
+export const RPC_SERVER_ENVIRONMENT_KEY =
+  "ALCHEMY_RPC_SERVER_ENVIRONMENT" as const;
+
+export const fromEnv = () =>
+  Config.string(RPC_SERVER_ENVIRONMENT_KEY).pipe(
+    Config.map(JSON.parse),
+    Effect.map((environment) => layer(environment)),
+    Layer.unwrap,
+  );

package/src/Local/RpcServerNode.ts

@@ -0,0 +1,58 @@
+import * as Effect from "effect/Effect";
+import { WebSocketServer, type Server } from "ws";
+import * as RpcServer from "./RpcServer.ts";
+
+export const RpcServerNode = RpcServer.layerServer(
+  Effect.fnUntraced(function* ({
+    parentConnected,
+    parentDisconnected,
+    createRpcSession,
+  }) {
+    const server = new WebSocketServer({ host: "127.0.0.1", port: 0 });
+    const url = yield* Effect.callback<string>((resume) => {
+      server.on("connection", (ws, req) => {
+        if (req.url?.startsWith("/parent")) {
+          parentConnected();
+          ws.on("close", () => {
+            parentDisconnected();
+          });
+          return;
+        }
+        const session = createRpcSession(ws);
+        ws.on("message", (data) => {
+          session.dispatch.message(data.toString());
+        });
+        ws.on("close", (code, reason) => {
+          session.dispatch.close(code, reason.toString());
+        });
+      });
+      server.on("error", (error) => {
+        resume(Effect.die(error));
+      });
+      server.on("listening", () => {
+        resume(getServerAddress(server));
+      });
+    });
+    yield* Effect.addFinalizer(() => Effect.sync(() => server.close()));
+    return { url };
+  }),
+);
+
+function getServerAddress(server: Server): Effect.Effect<string> {
+  const address = server.address();
+  if (
+    typeof address === "object" &&
+    address !== null &&
+    "address" in address &&
+    "port" in address
+  ) {
+    return Effect.succeed(
+      `ws://${address.address === "::" ? "localhost" : address.address}:${address.port}`,
+    );
+  }
+  return Effect.die(
+    new Error(
+      `Server address is not an object with address and port properties: ${JSON.stringify(address)}`,
+    ),
+  );
+}

package/src/Local/RpcServerSession.ts

@@ -0,0 +1,87 @@
+import { RpcSession, type RpcCompatible, type RpcTransport } from "capnweb";
+
+/**
+ * A platform-agnostic interface for a server-side WebSocket.
+ */
+export interface ServerWebSocketLike {
+  send: (message: string) => any | Promise<any>;
+  close: (code?: number, reason?: string) => void;
+}
+
+/**
+ * Represents a websocket RPC session with capnweb.
+ * Use the `dispatch` object to handle messages that are received from the client.
+ */
+export type ServerRpcSession<T extends RpcCompatible<T>> = ReturnType<
+  typeof makeServerRpcSession<T>
+>;
+
+/**
+ * Constructs a ServerRpcSession using capnweb.
+ * @param ws - The WebSocket to use for the session.
+ * @param main - The main object to use for the session.
+ * @returns A ServerRpcSession.
+ */
+export function makeServerRpcSession<T extends RpcCompatible<T>>(
+  ws: ServerWebSocketLike,
+  main: T,
+) {
+  const { transport, dispatch } = makeWebSocketRpcTransport(ws);
+  const session = new RpcSession(transport, main);
+  return { session, dispatch };
+}
+
+function makeWebSocketRpcTransport(ws: ServerWebSocketLike) {
+  let receiveQueue: Array<string> = [];
+  let receiveResolver: ((value: string) => void) | undefined;
+  let receiveRejecter: ((reason: unknown) => void) | undefined;
+  let error: unknown | undefined;
+  return {
+    transport: {
+      send: async (message: string) => await ws.send(message),
+      receive: async () => {
+        const next = receiveQueue.shift();
+        if (next) {
+          return next;
+        } else if (error) {
+          throw error;
+        }
+        return new Promise<string>((resolve, reject) => {
+          receiveResolver = resolve;
+          receiveRejecter = reject;
+        });
+      },
+      abort: (reason: unknown) => {
+        const message =
+          reason instanceof Error ? reason.message : String(reason);
+        ws.close(3000, message);
+        error ??= reason;
+      },
+    } satisfies RpcTransport,
+    dispatch: {
+      message: (data: string | Buffer<ArrayBuffer>) => {
+        if (error) {
+          return;
+        }
+        data = typeof data === "string" ? data : data.toString("utf-8");
+        if (receiveResolver) {
+          receiveResolver(data);
+          receiveResolver = undefined;
+          receiveRejecter = undefined;
+        } else {
+          receiveQueue.push(data);
+        }
+      },
+      close: (code: number, reason: string) => {
+        if (!error) {
+          error = new Error(`WebSocket closed with code ${code}: ${reason}`);
+          if (receiveRejecter) {
+            receiveRejecter(error);
+            receiveRejecter = undefined;
+            receiveResolver = undefined;
+          }
+        }
+      },
+    },
+  };
+}

package/src/Local/RpcSpawner.ts

@@ -0,0 +1,196 @@
+import { exitHook } from "@alchemy.run/node-utils/exit-hook";
+import * as Cache from "effect/Cache";
+import * as Context from "effect/Context";
+import * as Deferred from "effect/Deferred";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import type { PlatformError } from "effect/PlatformError";
+import * as Scope from "effect/Scope";
+import * as Stream from "effect/Stream";
+import * as HttpServer from "effect/unstable/http/HttpServer";
+import { HttpServerRequest } from "effect/unstable/http/HttpServerRequest";
+import * as HttpServerResponse from "effect/unstable/http/HttpServerResponse";
+import * as ChildProcess from "effect/unstable/process/ChildProcess";
+import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner";
+import * as NodeChildProcess from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { httpServer } from "../Util/PlatformServices.ts";
+import {
+  RPC_SERVER_ENVIRONMENT_KEY,
+  type RpcServerEnvironment,
+} from "./RpcServerEnvironment.ts";
+
+export class RpcSpawner extends Context.Service<
+  RpcSpawner,
+  {
+    readonly url: string;
+  }
+>()("alchemy/Local/RpcSpawner") {}
+
+export interface RpcSpawnPayload extends Pick<
+  RpcServerEnvironment,
+  "alchemyContext" | "stack"
+> {
+  serverEntryUrl: string;
+}
+
+export const make = Effect.fnUntraced(function* ({
+  profile,
+  envFile,
+}: Pick<RpcServerEnvironment, "profile" | "envFile">) {
+  const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
+  const scope = yield* Effect.scope;
+  const cache = yield* Cache.make({
+    lookup: (payload: RpcSpawnPayload) =>
+      spawn(payload).pipe(Scope.provide(scope)),
+    capacity: Infinity,
+  });
+
+  const spawn = Effect.fnUntraced(function* ({
+    serverEntryUrl,
+    alchemyContext,
+    stack,
+  }: RpcSpawnPayload) {
+    const bin = typeof globalThis.Bun !== "undefined" ? "bun" : "node";
+    const main = fileURLToPath(serverEntryUrl);
+    const environment: RpcServerEnvironment = {
+      profile,
+      envFile,
+      alchemyContext,
+      stack,
+    };
+    const command = ChildProcess.make(
+      bin,
+      {
+        bun: ["run", main],
+        // Under Node, transparently strip TypeScript types so that `.ts`
+        // entry points work the same way they do under Bun. Mirrors what
+        // `dev.ts` already does for the outer process, so the dev experience
+        // is symmetric on both runtimes whether the entry came from `src/`
+        // (dev/tests) or `lib/` (published packages).
+        node: main.endsWith(".ts")
+          ? [
+              "--experimental-transform-types",
+              "--no-warnings=ExperimentalWarning",
+              main,
+            ]
+          : [main],
+      }[bin],
+      {
+        stdout: "pipe",
+        stderr: "inherit",
+        detached: false,
+        env: {
+          [RPC_SERVER_ENVIRONMENT_KEY]: JSON.stringify(environment),
+        },
+        extendEnv: true,
+      },
+    );
+    const handle = yield* spawner.spawn(command);
+    const unregister = exitHook(() => {
+      killProcessGroup(handle.pid, "SIGKILL");
+    });
+    const kill = handle
+      .kill({ forceKillAfter: "500 millis" })
+      .pipe(Effect.tap(() => Effect.sync(unregister)));
+    yield* Effect.addFinalizer(() => kill.pipe(Effect.ignore));
+    const url = yield* getRpcAddress(handle.stdout);
+    const ws = yield* Effect.acquireRelease(
+      Effect.sync(() => new WebSocket(new URL("/parent", url))),
+      (ws) => Effect.sync(() => ws.close()),
+    );
+    return {
+      url,
+      isRunning: Effect.zipWith(
+        handle.isRunning,
+        Effect.sync(
+          () =>
+            ws.readyState === WebSocket.CONNECTING ||
+            ws.readyState === WebSocket.OPEN,
+        ),
+        (a, b) => a && b,
+        { concurrent: true },
+      ),
+      kill,
+    };
+  });
+
+  const register = Effect.fnUntraced(function* (
+    payload: RpcSpawnPayload,
+    attempt = 0,
+  ): Effect.fn.Return<string, PlatformError> {
+    const child = yield* Cache.get(cache, payload);
+    if (yield* child.isRunning) {
+      return child.url;
+    }
+    if (attempt > 3) {
+      return yield* Effect.die(
+        new Error(
+          `Failed to spawn RPC server for "${payload.serverEntryUrl}" after ${attempt} attempts.`,
+        ),
+      );
+    }
+    yield* child.kill;
+    yield* Cache.invalidate(cache, payload);
+    return yield* register(payload, attempt + 1);
+  });
+
+  const server = yield* HttpServer.HttpServer;
+
+  yield* server.serve(
+    Effect.gen(function* () {
+      const request = yield* HttpServerRequest;
+      const payload = (yield* request.json) as unknown as RpcSpawnPayload;
+      const url = yield* register(payload);
+      return HttpServerResponse.text(url);
+    }),
+  );
+
+  return RpcSpawner.of({
+    url: HttpServer.formatAddress(server.address),
+  });
+});
+
+export const layerServer = (
+  environment: Pick<RpcServerEnvironment, "profile" | "envFile">,
+) =>
+  Layer.effect(RpcSpawner, make(environment)).pipe(Layer.provide(httpServer()));
+
+const RPC_ADDRESS_REGEX =
+  /(<ALCHEMY_RPC_ADDRESS>)(.+)(<\/ALCHEMY_RPC_ADDRESS>)/;
+
+const getRpcAddress = (stdout: Stream.Stream<Uint8Array, PlatformError>) =>
+  Effect.gen(function* () {
+    const address = yield* Deferred.make<string>();
+    let done = false;
+    yield* stdout.pipe(
+      Stream.decodeText,
+      Stream.splitLines,
+      Stream.runForEach((line) => {
+        if (done) {
+          console.log(line);
+        } else {
+          const match = line.match(RPC_ADDRESS_REGEX);
+          if (match) {
+            done = true;
+            return Deferred.succeed(address, match[2]);
+          }
+        }
+        return Effect.void;
+      }),
+      Effect.forkScoped,
+    );
+    return yield* Deferred.await(address);
+  });
+
+const killProcessGroup = (pid: number, signal: NodeJS.Signals) => {
+  try {
+    if (process.platform === "win32") {
+      NodeChildProcess.execSync(`taskkill /pid ${pid} /T /F`);
+    } else {
+      process.kill(-pid, signal);
+    }
+  } catch {
+    // ignore errors during best-effort cleanup
+  }
+};

package/src/Neon/Branch.ts

@@ -18,7 +18,7 @@ import { applyMigrations, runSql } from "./Migrations.ts";
 import { parsePostgresOrigin, type PostgresOrigin } from "./PostgresOrigin.ts";
 import { type Project, waitForOperations } from "./Project.ts";
 import type { Providers } from "./Providers.ts";
-import { listSqlFiles, readSqlFile } from "./SqlFile.ts";
+import { listSqlFiles, readSqlFile } from "../Sql/SqlFile.ts";
 
 const findBranchByName = (projectId: string, name: string) =>
   Effect.gen(function* () {

package/src/Neon/Migrations.ts

@@ -2,7 +2,7 @@ import * as Data from "effect/Data";
 import * as Effect from "effect/Effect";
 import * as Redacted from "effect/Redacted";
 import { Client } from "pg";
-import type { SqlFile } from "./SqlFile.ts";
+import type { SqlFile } from "../Sql/SqlFile.ts";
 
 export class PgError extends Data.TaggedError("PgError")<{
   message: string;

package/src/Neon/Project.ts

@@ -24,7 +24,12 @@ import { Resource } from "../Resource.ts";
 import { applyMigrations, runSql } from "./Migrations.ts";
 import { parsePostgresOrigin, type PostgresOrigin } from "./PostgresOrigin.ts";
 import type { Providers } from "./Providers.ts";
-import { listSqlFiles, readSqlFile } from "./SqlFile.ts";
+import {
+  listSqlFiles,
+  readSqlFile,
+  hashMigrations,
+  hashImports,
+} from "../Sql/SqlFile.ts";
 
 type NeonOperationStatus =
   | "scheduling"
@@ -638,25 +643,6 @@ const runImports = (
     return hashes;
   });
 
-const hashMigrations = (migrationsDir: string) =>
-  listSqlFiles(migrationsDir).pipe(
-    Effect.map((files) => {
-      const hashes: Record<string, string> = {};
-      for (const file of files) hashes[file.id] = file.hash;
-      return hashes;
-    }),
-  );
-
-const hashImports = (importFiles: ReadonlyArray<string>, rootDir: string) =>
-  Effect.gen(function* () {
-    const hashes: Record<string, string> = {};
-    for (const filePath of importFiles) {
-      const file = yield* readSqlFile(rootDir, filePath);
-      hashes[filePath] = file.hash;
-    }
-    return hashes;
-  });
-
 const recordsEqual = (
   a: Record<string, string>,
   b: Record<string, string>,

package/src/Phase.ts

@@ -1,5 +1,7 @@
 import * as Config from "effect/Config";
+import * as Effect from "effect/Effect";
 
 export const ALCHEMY_PHASE = Config.string("ALCHEMY_PHASE").pipe(
   Config.withDefault("plan"),
+  Effect.orDie,
 );