alchemy 2.0.0-beta.43 → 2.0.0-beta.45

package/src/Local/RpcServer.ts

@@ -0,0 +1,138 @@
+import * as Console from "effect/Console";
+import * as Context from "effect/Context";
+import * as Deferred from "effect/Deferred";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import type * as Scope from "effect/Scope";
+import * as FetchHttpClient from "effect/unstable/http/FetchHttpClient";
+import type { HttpClient } from "effect/unstable/http/HttpClient";
+import type { ProviderService } from "../Provider.ts";
+import type { ResourceLike } from "../Resource.ts";
+import {
+  platformLayer,
+  PlatformServices,
+  runMain,
+} from "../Util/PlatformServices.ts";
+import * as RpcSerialization from "./RpcSerialization.ts";
+import * as RpcServerEnvironment from "./RpcServerEnvironment.ts";
+import {
+  makeServerRpcSession,
+  type ServerRpcSession,
+  type ServerWebSocketLike,
+} from "./RpcServerSession.ts";
+
+/**
+ * A service that exposes one or more resource providers over RPC.
+ * This returns `never` because it is meant to be used with `Layer.launch` (see {@link launch}).
+ */
+export class RpcServer extends Context.Service<RpcServer, never>()(
+  "alchemy/Local/RpcServer",
+) {}
+
+/**
+ * The RPC API that is implemented by the server and consumed by {@link RpcProviderProxy}.
+ */
+export interface RpcProxyApi {
+  /**
+   * Retrieves a provider from the RPC server context.
+   * The consumer must unwrap the provider using {@link RpcSerialization.unwrapRpcHandlers} before using it.
+   */
+  readonly getProvider: <R extends ResourceLike>(
+    type: R["Type"],
+  ) => Promise<RpcSerialization.RpcWrapped<ProviderService<R>>>;
+}
+
+const serverPlatformLayer = platformLayer({
+  bun: async () => {
+    const { RpcServerBun } = await import("./RpcServerBun.ts");
+    return RpcServerBun;
+  },
+  node: async () => {
+    const { RpcServerNode } = await import("./RpcServerNode.ts");
+    return RpcServerNode;
+  },
+});
+
+/**
+ * Launches an RPC server that serves the given providers.
+ * Alchemy globals such as `AlchemyContext`, `Profile`, and `Stack` are inherited from the parent via {@link RpcServerEnvironment.fromEnv} and should not be provided manually.
+ * `PlatformServices` and `HttpClient` are also included.
+ *
+ * @example
+ * ```ts
+ * RpcServer.launch(
+ *   Layer.merge(
+ *     FunctionProvider,
+ *     QueueProvider,
+ *   ),
+ * );
+ * ```
+ *
+ * @param providers - A layer containing the providers to serve.
+ */
+export const launch = <ROut, E>(
+  providers: Layer.Layer<
+    ROut,
+    E,
+    | Scope.Scope
+    | RpcServerEnvironment.RpcEnvironmentServices
+    | PlatformServices
+    | HttpClient
+  >,
+) =>
+  serverPlatformLayer.pipe(
+    Layer.provide(providers),
+    Layer.provide(RpcServerEnvironment.fromEnv()),
+    Layer.provide(Layer.merge(PlatformServices, FetchHttpClient.layer)),
+    Layer.launch,
+    Effect.scoped,
+    runMain,
+  );
+
+/**
+ * Constructs an `RpcServer` layer using the given server implementation.
+ * @param serve - A function that spawns a websocket server and returns its URL.
+ * @returns An `RpcServer` layer.
+ */
+export const layerServer = (
+  serve: (handlers: {
+    /** Creates a new RPC session over the given websocket. */
+    createRpcSession: (
+      ws: ServerWebSocketLike,
+    ) => ServerRpcSession<RpcProxyApi>;
+    /** Called when the parent connection, indicated by the `/parent` path, is established. */
+    parentConnected: () => void;
+    /** Called when the parent disconnects. The server will shut down when this is called. */
+    parentDisconnected: () => void;
+  }) => Effect.Effect<{ readonly url: string }, never, Scope.Scope>,
+) =>
+  Layer.effect(
+    RpcServer,
+    Effect.gen(function* () {
+      const context = yield* Effect.context();
+      const connected = yield* Deferred.make<void>();
+      const disconnected = yield* Deferred.make<void>();
+      const { url } = yield* serve({
+        createRpcSession: (ws) =>
+          makeServerRpcSession<RpcProxyApi>(ws, {
+            getProvider: async <R extends ResourceLike>(type: R["Type"]) => {
+              const provider = context.mapUnsafe.get(type);
+              if (!provider) {
+                throw new Error(`Provider "${type}" not found`);
+              }
+              return RpcSerialization.wrapRpcHandlers(
+                provider as ProviderService<R>,
+                ["tail"],
+              );
+            },
+          }),
+        parentConnected: () => Deferred.doneUnsafe(connected, Effect.void),
+        parentDisconnected: () =>
+          Deferred.doneUnsafe(disconnected, Effect.void),
+      });
+      yield* Console.log(`<ALCHEMY_RPC_ADDRESS>${url}</ALCHEMY_RPC_ADDRESS>`);
+      yield* Deferred.await(connected).pipe(Effect.timeout("10 seconds")); // TODO(john): should the timeout be shorter?
+      yield* Deferred.await(disconnected);
+      return yield* Effect.interrupt;
+    }),
+  );

package/src/Local/RpcServerBun.ts

@@ -0,0 +1,59 @@
+import * as Effect from "effect/Effect";
+import * as RpcServer from "./RpcServer.ts";
+import type { ServerRpcSession } from "./RpcServerSession.ts";
+
+export const RpcServerBun = RpcServer.layerServer(
+  Effect.fnUntraced(function* ({
+    parentConnected,
+    parentDisconnected,
+    createRpcSession,
+  }) {
+    const server = yield* Effect.sync(() =>
+      Bun.serve<
+        { type: "session"; session: ServerRpcSession<any> } | { type: "parent" }
+      >({
+        port: 0,
+        fetch: (request, server) => {
+          const url = new URL(request.url);
+          if (
+            server.upgrade(request, {
+              data:
+                url.pathname === "/parent" ? { type: "parent" } : undefined!,
+            })
+          ) {
+            return;
+          }
+          return new Response("Upgrade failed", { status: 400 });
+        },
+        websocket: {
+          open: (ws) => {
+            if (ws.data && ws.data.type === "parent") {
+              parentConnected();
+            } else {
+              ws.data = {
+                type: "session",
+                session: createRpcSession(ws),
+              };
+            }
+          },
+          message: (ws, message) => {
+            if (ws.data.type === "session") {
+              ws.data.session.dispatch.message(message);
+            }
+          },
+          close: (ws, code, reason) => {
+            if (ws.data.type === "session") {
+              ws.data.session.dispatch.close(code, reason);
+            } else if (ws.data.type === "parent") {
+              parentDisconnected();
+            }
+          },
+        },
+      }),
+    );
+    yield* Effect.addFinalizer(() => Effect.sync(() => server.stop(true)));
+    return {
+      url: `ws://${server.hostname}:${server.port}`,
+    };
+  }),
+);

package/src/Local/RpcServerEnvironment.ts

@@ -0,0 +1,55 @@
+import * as Config from "effect/Config";
+import * as ConfigProvider from "effect/ConfigProvider";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
+import { AlchemyContext } from "../AlchemyContext.ts";
+import { AuthProviders } from "../Auth/AuthProvider.ts";
+import { CredentialsStoreLive } from "../Auth/Credentials.ts";
+import { ProfileLive, withProfileOverride } from "../Auth/Profile.ts";
+import { Stack } from "../Stack.ts";
+import { Stage } from "../Stage.ts";
+import { loadConfigProvider } from "../Util/ConfigProvider.ts";
+
+export interface RpcServerEnvironment {
+  alchemyContext: AlchemyContext["Service"];
+  profile: string | undefined;
+  envFile: string | undefined;
+  stack: {
+    name: string;
+    stage: string;
+  };
+}
+
+export type RpcEnvironmentServices = Layer.Success<ReturnType<typeof layer>>;
+
+export const layer = (environment: RpcServerEnvironment) =>
+  Layer.mergeAll(
+    ProfileLive,
+    CredentialsStoreLive,
+    Layer.succeed(AuthProviders, {}),
+    ConfigProvider.layer(
+      loadConfigProvider(Option.fromNullishOr(environment.envFile)).pipe(
+        Effect.map((base) => withProfileOverride(base, environment.profile)),
+      ),
+    ),
+    Layer.succeed(AlchemyContext, environment.alchemyContext),
+    Layer.succeed(Stack, {
+      name: environment.stack.name,
+      stage: environment.stack.stage,
+      resources: {},
+      bindings: {},
+      actions: {},
+    }),
+    Layer.succeed(Stage, environment.stack.stage),
+  );
+
+export const RPC_SERVER_ENVIRONMENT_KEY =
+  "ALCHEMY_RPC_SERVER_ENVIRONMENT" as const;
+
+export const fromEnv = () =>
+  Config.string(RPC_SERVER_ENVIRONMENT_KEY).pipe(
+    Config.map(JSON.parse),
+    Effect.map((environment) => layer(environment)),
+    Layer.unwrap,
+  );

package/src/Local/RpcServerNode.ts

@@ -0,0 +1,58 @@
+import * as Effect from "effect/Effect";
+import { WebSocketServer, type Server } from "ws";
+import * as RpcServer from "./RpcServer.ts";
+
+export const RpcServerNode = RpcServer.layerServer(
+  Effect.fnUntraced(function* ({
+    parentConnected,
+    parentDisconnected,
+    createRpcSession,
+  }) {
+    const server = new WebSocketServer({ host: "127.0.0.1", port: 0 });
+    const url = yield* Effect.callback<string>((resume) => {
+      server.on("connection", (ws, req) => {
+        if (req.url?.startsWith("/parent")) {
+          parentConnected();
+          ws.on("close", () => {
+            parentDisconnected();
+          });
+          return;
+        }
+        const session = createRpcSession(ws);
+        ws.on("message", (data) => {
+          session.dispatch.message(data.toString());
+        });
+        ws.on("close", (code, reason) => {
+          session.dispatch.close(code, reason.toString());
+        });
+      });
+      server.on("error", (error) => {
+        resume(Effect.die(error));
+      });
+      server.on("listening", () => {
+        resume(getServerAddress(server));
+      });
+    });
+    yield* Effect.addFinalizer(() => Effect.sync(() => server.close()));
+    return { url };
+  }),
+);
+
+function getServerAddress(server: Server): Effect.Effect<string> {
+  const address = server.address();
+  if (
+    typeof address === "object" &&
+    address !== null &&
+    "address" in address &&
+    "port" in address
+  ) {
+    return Effect.succeed(
+      `ws://${address.address === "::" ? "localhost" : address.address}:${address.port}`,
+    );
+  }
+  return Effect.die(
+    new Error(
+      `Server address is not an object with address and port properties: ${JSON.stringify(address)}`,
+    ),
+  );
+}

package/src/Local/RpcServerSession.ts

@@ -0,0 +1,87 @@
+import { RpcSession, type RpcCompatible, type RpcTransport } from "capnweb";
+
+/**
+ * A platform-agnostic interface for a server-side WebSocket.
+ */
+export interface ServerWebSocketLike {
+  send: (message: string) => any | Promise<any>;
+  close: (code?: number, reason?: string) => void;
+}
+
+/**
+ * Represents a websocket RPC session with capnweb.
+ * Use the `dispatch` object to handle messages that are received from the client.
+ */
+export type ServerRpcSession<T extends RpcCompatible<T>> = ReturnType<
+  typeof makeServerRpcSession<T>
+>;
+
+/**
+ * Constructs a ServerRpcSession using capnweb.
+ * @param ws - The WebSocket to use for the session.
+ * @param main - The main object to use for the session.
+ * @returns A ServerRpcSession.
+ */
+export function makeServerRpcSession<T extends RpcCompatible<T>>(
+  ws: ServerWebSocketLike,
+  main: T,
+) {
+  const { transport, dispatch } = makeWebSocketRpcTransport(ws);
+  const session = new RpcSession(transport, main);
+  return { session, dispatch };
+}
+
+function makeWebSocketRpcTransport(ws: ServerWebSocketLike) {
+  let receiveQueue: Array<string> = [];
+  let receiveResolver: ((value: string) => void) | undefined;
+  let receiveRejecter: ((reason: unknown) => void) | undefined;
+  let error: unknown | undefined;
+  return {
+    transport: {
+      send: async (message: string) => await ws.send(message),
+      receive: async () => {
+        const next = receiveQueue.shift();
+        if (next) {
+          return next;
+        } else if (error) {
+          throw error;
+        }
+        return new Promise<string>((resolve, reject) => {
+          receiveResolver = resolve;
+          receiveRejecter = reject;
+        });
+      },
+      abort: (reason: unknown) => {
+        const message =
+          reason instanceof Error ? reason.message : String(reason);
+        ws.close(3000, message);
+        error ??= reason;
+      },
+    } satisfies RpcTransport,
+    dispatch: {
+      message: (data: string | Buffer<ArrayBuffer>) => {
+        if (error) {
+          return;
+        }
+        data = typeof data === "string" ? data : data.toString("utf-8");
+        if (receiveResolver) {
+          receiveResolver(data);
+          receiveResolver = undefined;
+          receiveRejecter = undefined;
+        } else {
+          receiveQueue.push(data);
+        }
+      },
+      close: (code: number, reason: string) => {
+        if (!error) {
+          error = new Error(`WebSocket closed with code ${code}: ${reason}`);
+          if (receiveRejecter) {
+            receiveRejecter(error);
+            receiveRejecter = undefined;
+            receiveResolver = undefined;
+          }
+        }
+      },
+    },
+  };
+}

package/src/Local/RpcSpawner.ts

@@ -0,0 +1,196 @@
+import { exitHook } from "@alchemy.run/node-utils/exit-hook";
+import * as Cache from "effect/Cache";
+import * as Context from "effect/Context";
+import * as Deferred from "effect/Deferred";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import type { PlatformError } from "effect/PlatformError";
+import * as Scope from "effect/Scope";
+import * as Stream from "effect/Stream";
+import * as HttpServer from "effect/unstable/http/HttpServer";
+import { HttpServerRequest } from "effect/unstable/http/HttpServerRequest";
+import * as HttpServerResponse from "effect/unstable/http/HttpServerResponse";
+import * as ChildProcess from "effect/unstable/process/ChildProcess";
+import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner";
+import * as NodeChildProcess from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { httpServer } from "../Util/PlatformServices.ts";
+import {
+  RPC_SERVER_ENVIRONMENT_KEY,
+  type RpcServerEnvironment,
+} from "./RpcServerEnvironment.ts";
+
+export class RpcSpawner extends Context.Service<
+  RpcSpawner,
+  {
+    readonly url: string;
+  }
+>()("alchemy/Local/RpcSpawner") {}
+
+export interface RpcSpawnPayload extends Pick<
+  RpcServerEnvironment,
+  "alchemyContext" | "stack"
+> {
+  serverEntryUrl: string;
+}
+
+export const make = Effect.fnUntraced(function* ({
+  profile,
+  envFile,
+}: Pick<RpcServerEnvironment, "profile" | "envFile">) {
+  const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
+  const scope = yield* Effect.scope;
+  const cache = yield* Cache.make({
+    lookup: (payload: RpcSpawnPayload) =>
+      spawn(payload).pipe(Scope.provide(scope)),
+    capacity: Infinity,
+  });
+
+  const spawn = Effect.fnUntraced(function* ({
+    serverEntryUrl,
+    alchemyContext,
+    stack,
+  }: RpcSpawnPayload) {
+    const bin = typeof globalThis.Bun !== "undefined" ? "bun" : "node";
+    const main = fileURLToPath(serverEntryUrl);
+    const environment: RpcServerEnvironment = {
+      profile,
+      envFile,
+      alchemyContext,
+      stack,
+    };
+    const command = ChildProcess.make(
+      bin,
+      {
+        bun: ["run", main],
+        // Under Node, transparently strip TypeScript types so that `.ts`
+        // entry points work the same way they do under Bun. Mirrors what
+        // `dev.ts` already does for the outer process, so the dev experience
+        // is symmetric on both runtimes whether the entry came from `src/`
+        // (dev/tests) or `lib/` (published packages).
+        node: main.endsWith(".ts")
+          ? [
+              "--experimental-transform-types",
+              "--no-warnings=ExperimentalWarning",
+              main,
+            ]
+          : [main],
+      }[bin],
+      {
+        stdout: "pipe",
+        stderr: "inherit",
+        detached: false,
+        env: {
+          [RPC_SERVER_ENVIRONMENT_KEY]: JSON.stringify(environment),
+        },
+        extendEnv: true,
+      },
+    );
+    const handle = yield* spawner.spawn(command);
+    const unregister = exitHook(() => {
+      killProcessGroup(handle.pid, "SIGKILL");
+    });
+    const kill = handle
+      .kill({ forceKillAfter: "500 millis" })
+      .pipe(Effect.tap(() => Effect.sync(unregister)));
+    yield* Effect.addFinalizer(() => kill.pipe(Effect.ignore));
+    const url = yield* getRpcAddress(handle.stdout);
+    const ws = yield* Effect.acquireRelease(
+      Effect.sync(() => new WebSocket(new URL("/parent", url))),
+      (ws) => Effect.sync(() => ws.close()),
+    );
+    return {
+      url,
+      isRunning: Effect.zipWith(
+        handle.isRunning,
+        Effect.sync(
+          () =>
+            ws.readyState === WebSocket.CONNECTING ||
+            ws.readyState === WebSocket.OPEN,
+        ),
+        (a, b) => a && b,
+        { concurrent: true },
+      ),
+      kill,
+    };
+  });
+
+  const register = Effect.fnUntraced(function* (
+    payload: RpcSpawnPayload,
+    attempt = 0,
+  ): Effect.fn.Return<string, PlatformError> {
+    const child = yield* Cache.get(cache, payload);
+    if (yield* child.isRunning) {
+      return child.url;
+    }
+    if (attempt > 3) {
+      return yield* Effect.die(
+        new Error(
+          `Failed to spawn RPC server for "${payload.serverEntryUrl}" after ${attempt} attempts.`,
+        ),
+      );
+    }
+    yield* child.kill;
+    yield* Cache.invalidate(cache, payload);
+    return yield* register(payload, attempt + 1);
+  });
+
+  const server = yield* HttpServer.HttpServer;
+
+  yield* server.serve(
+    Effect.gen(function* () {
+      const request = yield* HttpServerRequest;
+      const payload = (yield* request.json) as unknown as RpcSpawnPayload;
+      const url = yield* register(payload);
+      return HttpServerResponse.text(url);
+    }),
+  );
+
+  return RpcSpawner.of({
+    url: HttpServer.formatAddress(server.address),
+  });
+});
+
+export const layerServer = (
+  environment: Pick<RpcServerEnvironment, "profile" | "envFile">,
+) =>
+  Layer.effect(RpcSpawner, make(environment)).pipe(Layer.provide(httpServer()));
+
+const RPC_ADDRESS_REGEX =
+  /(<ALCHEMY_RPC_ADDRESS>)(.+)(<\/ALCHEMY_RPC_ADDRESS>)/;
+
+const getRpcAddress = (stdout: Stream.Stream<Uint8Array, PlatformError>) =>
+  Effect.gen(function* () {
+    const address = yield* Deferred.make<string>();
+    let done = false;
+    yield* stdout.pipe(
+      Stream.decodeText,
+      Stream.splitLines,
+      Stream.runForEach((line) => {
+        if (done) {
+          console.log(line);
+        } else {
+          const match = line.match(RPC_ADDRESS_REGEX);
+          if (match) {
+            done = true;
+            return Deferred.succeed(address, match[2]);
+          }
+        }
+        return Effect.void;
+      }),
+      Effect.forkScoped,
+    );
+    return yield* Deferred.await(address);
+  });
+
+const killProcessGroup = (pid: number, signal: NodeJS.Signals) => {
+  try {
+    if (process.platform === "win32") {
+      NodeChildProcess.execSync(`taskkill /pid ${pid} /T /F`);
+    } else {
+      process.kill(-pid, signal);
+    }
+  } catch {
+    // ignore errors during best-effort cleanup
+  }
+};

package/src/Neon/Project.ts

@@ -136,7 +136,7 @@ const findProjectByName = (name: string) =>
   Effect.gen(function* () {
     const matches: ListProjectsOutput["projects"][number][] = [];
     let cursor: string | undefined;
-    do {
+    while (true) {
       const page = yield* listProjects({
         search: name,
         ...(cursor !== undefined ? { cursor } : {}),
@@ -144,8 +144,21 @@ const findProjectByName = (name: string) =>
       for (const p of page.projects) {
         if (p.name === name) matches.push(p);
       }
-      cursor = page.pagination?.cursor;
-    } while (cursor);
+      const nextCursor = page.pagination?.cursor;
+      // Neon returns a `pagination.cursor` on every response — it's the
+      // `created_at` of the last row, not a "has next page" flag — so we
+      // can't loop on cursor presence alone or we spin forever re-fetching
+      // empty/identical pages. Stop once a page comes back empty or the
+      // cursor stops advancing.
+      if (
+        page.projects.length === 0 ||
+        nextCursor === undefined ||
+        nextCursor === cursor
+      ) {
+        break;
+      }
+      cursor = nextCursor;
+    }
     return matches;
   });
 

package/src/Output.ts

@@ -1,4 +1,5 @@
 import * as Data from "effect/Data";
+import * as Duration from "effect/Duration";
 import * as Effect from "effect/Effect";
 import { pipe } from "effect/Function";
 import type { Pipeable } from "effect/Pipeable";
@@ -6,7 +7,7 @@ import * as Redacted from "effect/Redacted";
 import { SingleShotGen } from "effect/Utils";
 import { getRefMetadata, isRef, type Ref } from "./Ref.ts";
 import { isResource, type Resource, type ResourceLike } from "./Resource.ts";
-import { RuntimeContext } from "./RuntimeContext.ts";
+import { RuntimeContext, sanitizeKey } from "./RuntimeContext.ts";
 import { Stack } from "./Stack.ts";
 import { Stage } from "./Stage.ts";
 import * as State from "./State/State.ts";
@@ -112,9 +113,11 @@ export abstract class BaseExpr<A = any, Req = any> implements Output<A, Req> {
   }
 
   public bind(id: string): any {
+    // `set`/`get` store keys verbatim, so canonicalize here (the caller's job).
+    const key = sanitizeKey(id);
     return RuntimeContext.pipe(
       Effect.flatMap((ctx) =>
-        Effect.map(ctx.set(id, this), (key) => ctx.get<A>(key)),
+        Effect.map(ctx.set(key, this), (k) => ctx.get<A>(k)),
       ),
     );
   }
@@ -590,6 +593,9 @@ export const evaluate: <A, Req = never>(
       return yield* Effect.all(expr.map((item) => evaluate(item, upstream)));
     } else if (Redacted.isRedacted(expr)) {
       return expr;
+    } else if (Duration.isDuration(expr)) {
+      // Opaque value — see resolveInput in Plan.ts for rationale.
+      return expr;
     } else if (typeof expr === "object" && expr !== null) {
       return Object.fromEntries(
         yield* Effect.all(

package/src/Phase.ts

@@ -1,7 +1,15 @@
 import * as Config from "effect/Config";
 import * as Effect from "effect/Effect";
 
+export type AlchemyPhase = "plan" | "runtime";
+
 export const ALCHEMY_PHASE = Config.string("ALCHEMY_PHASE").pipe(
   Config.withDefault("plan"),
+  Config.mapOrFail((value) => {
+    if (value !== "plan" && value !== "runtime") {
+      return Effect.die(new Error(`Invalid ALCHEMY_PHASE: ${value}`));
+    }
+    return Effect.succeed(value as AlchemyPhase);
+  }),
   Effect.orDie,
 );