alchemy 2.0.0-beta.36 → 2.0.0-beta.38

package/src/Neon/Credentials.ts

@@ -1,5 +1,5 @@
-import { Credentials } from "@distilled.cloud/neon/Credentials";
 import { ConfigError } from "@distilled.cloud/core/errors";
+import { Credentials } from "@distilled.cloud/neon";
 import * as Config from "effect/Config";
 import * as Effect from "effect/Effect";
 import * as Layer from "effect/Layer";
@@ -11,7 +11,7 @@ import {
   type NeonResolvedCredentials,
 } from "./AuthProvider.ts";
 
-export { Credentials } from "@distilled.cloud/neon/Credentials";
+export { Credentials } from "@distilled.cloud/neon";
 
 const DEFAULT_BASE_URL = "https://console.neon.tech/api/v2";
 

package/src/Neon/Project.ts

@@ -1,15 +1,148 @@
+import {
+  type CreateProjectOutput,
+  deleteProject,
+  getConnectionURI,
+  getProject,
+  getProjectOperation,
+  listProjectBranchDatabases,
+  listProjectBranches,
+  type ListProjectsOutput,
+  listProjects,
+  createProject as sdkCreateProject,
+  updateProject,
+} from "@distilled.cloud/neon";
+import * as Data from "effect/Data";
+import * as Duration from "effect/Duration";
 import * as Effect from "effect/Effect";
 import * as Redacted from "effect/Redacted";
+import * as Schedule from "effect/Schedule";
 import { isResolved } from "../Diff.ts";
 import { createPhysicalName } from "../PhysicalName.ts";
 import * as Provider from "../Provider.ts";
 import { Resource } from "../Resource.ts";
-import * as api from "./api.ts";
 import { applyMigrations, runSql } from "./Migrations.ts";
 import { parsePostgresOrigin, type PostgresOrigin } from "./PostgresOrigin.ts";
 import type { Providers } from "./Providers.ts";
 import { listSqlFiles, readSqlFile } from "./SqlFile.ts";
 
+type NeonOperationStatus =
+  | "scheduling"
+  | "running"
+  | "finished"
+  | "failed"
+  | "error"
+  | "cancelling"
+  | "cancelled"
+  | "skipped";
+
+export class OperationFailed extends Data.TaggedError("OperationFailed")<{
+  operationId: string;
+  action: string;
+  status: NeonOperationStatus;
+  error?: string;
+}> {}
+
+class OperationPending extends Data.TaggedError("OperationPending")<{
+  operationId: string;
+}> {}
+
+const isOperationComplete = (status: NeonOperationStatus): boolean =>
+  status === "finished" ||
+  status === "failed" ||
+  status === "error" ||
+  status === "cancelled" ||
+  status === "skipped";
+
+/**
+ * Wait for the given operations to reach a terminal state. Polls every
+ * 500ms with exponential backoff up to ~30s per operation.
+ */
+export const waitForOperations = (
+  operations: ReadonlyArray<{
+    readonly id: string;
+    readonly project_id: string;
+    readonly action: string;
+    readonly status: NeonOperationStatus;
+    readonly error?: string;
+  }>,
+) =>
+  Effect.gen(function* () {
+    for (const op of operations) {
+      if (isOperationComplete(op.status)) {
+        if (op.status === "failed" || op.status === "error") {
+          return yield* new OperationFailed({
+            operationId: op.id,
+            action: op.action,
+            status: op.status,
+            error: op.error,
+          });
+        }
+        continue;
+      }
+      yield* getProjectOperation({
+        project_id: op.project_id,
+        operation_id: op.id,
+      }).pipe(
+        Effect.flatMap(
+          ({
+            operation,
+          }): Effect.Effect<void, OperationFailed | OperationPending> => {
+            const status = operation.status as NeonOperationStatus;
+            if (status === "failed" || status === "error") {
+              return Effect.fail(
+                new OperationFailed({
+                  operationId: operation.id,
+                  action: operation.action,
+                  status,
+                  error: operation.error,
+                }),
+              );
+            }
+            if (!isOperationComplete(status)) {
+              return Effect.fail(new OperationPending({ operationId: op.id }));
+            }
+            return Effect.void;
+          },
+        ),
+        Effect.retry({
+          while: (e: unknown) => {
+            const tag = (e as { _tag?: string })._tag;
+            return (
+              tag === "OperationPending" ||
+              tag === "TooManyRequests" ||
+              tag === "ServiceUnavailable" ||
+              tag === "InternalServerError" ||
+              tag === "BadGateway" ||
+              tag === "GatewayTimeout"
+            );
+          },
+          schedule: Schedule.both(
+            Schedule.exponential(Duration.millis(500), 1.5),
+            Schedule.recurs(60),
+          ),
+        }),
+        Effect.catchTag("OperationPending", () => Effect.void),
+      );
+    }
+  });
+
+const findProjectByName = (name: string) =>
+  Effect.gen(function* () {
+    const matches: ListProjectsOutput["projects"][number][] = [];
+    let cursor: string | undefined;
+    do {
+      const page = yield* listProjects({
+        search: name,
+        ...(cursor !== undefined ? { cursor } : {}),
+      });
+      for (const p of page.projects) {
+        if (p.name === name) matches.push(p);
+      }
+      cursor = page.pagination?.cursor;
+    } while (cursor);
+    return matches;
+  });
+
 export type NeonRegion =
   | "aws-us-east-1"
   | "aws-us-east-2"
@@ -184,22 +317,12 @@ export type Project = Resource<
  */
 export const Project = Resource<Project>("Neon.Project");
 
-const getRoleName = (creation: api.CreateProjectOutput) =>
+const getRoleName = (creation: CreateProjectOutput) =>
   creation.roles.find((r) => !r.protected)?.name ?? creation.roles[0]?.name;
 
-const getDatabaseName = (creation: api.CreateProjectOutput) =>
+const getDatabaseName = (creation: CreateProjectOutput) =>
   creation.databases[0]?.name ?? "neondb";
 
-const buildConnectionUri = (
-  uri: string,
-  pooledHost: string,
-): { uri: string; pooled: string } => {
-  const url = new URL(uri);
-  const pooledUrl = new URL(uri);
-  pooledUrl.host = pooledHost;
-  return { uri: url.toString(), pooled: pooledUrl.toString() };
-};
-
 const resolveConnection = (
   projectId: string,
   branchId: string,
@@ -207,13 +330,15 @@ const resolveConnection = (
   roleName: string,
 ) =>
   Effect.gen(function* () {
-    const direct = yield* api.getConnectionUri(projectId, {
+    const direct = yield* getConnectionURI({
+      project_id: projectId,
       branch_id: branchId,
       database_name: databaseName,
       role_name: roleName,
       pooled: false,
     });
-    const pooled = yield* api.getConnectionUri(projectId, {
+    const pooled = yield* getConnectionURI({
+      project_id: projectId,
       branch_id: branchId,
       database_name: databaseName,
       role_name: roleName,
@@ -286,7 +411,7 @@ export const ProjectProvider = () =>
         }),
         read: Effect.fn(function* ({ id, output, olds }) {
           if (output?.projectId) {
-            return yield* api.getProject(output.projectId).pipe(
+            return yield* getProject({ project_id: output.projectId }).pipe(
               Effect.map(({ project }) => ({
                 ...output,
                 projectName: project.name,
@@ -294,25 +419,24 @@ export const ProjectProvider = () =>
                 enableLogicalReplication:
                   project.settings?.enable_logical_replication === true,
               })),
-              Effect.catchTag("ProjectNotFound", () =>
-                Effect.succeed(undefined),
-              ),
+              Effect.catchTag("NotFound", () => Effect.succeed(undefined)),
             );
           }
           const name = yield* createProjectName(id, olds?.name);
-          const matches = yield* api.findProjectByName(name);
+          const matches = yield* findProjectByName(name);
           const match = matches[0];
           if (!match) return undefined;
-          const branches = yield* api.listProjectBranches(match.id, {
+          const branches = yield* listProjectBranches({
+            project_id: match.id,
             search: olds?.defaultBranchName ?? "main",
           });
           const defaultBranch =
             branches.branches.find((b) => b.default) ?? branches.branches[0];
           if (!defaultBranch) return undefined;
-          const databases = yield* api.listProjectBranchDatabases(
-            match.id,
-            defaultBranch.id,
-          );
+          const databases = yield* listProjectBranchDatabases({
+            project_id: match.id,
+            branch_id: defaultBranch.id,
+          });
           const db = databases.databases[0];
           if (!db) return undefined;
           const conn = yield* resolveConnection(
@@ -333,7 +457,7 @@ export const ProjectProvider = () =>
             connectionUri: conn.uri,
             pooledConnectionUri: conn.pooled,
             origin: parsePostgresOrigin(conn.uri),
-            historyRetentionSeconds: match.history_retention_seconds,
+            historyRetentionSeconds: match.history_retention_seconds ?? 86400,
             enableLogicalReplication:
               match.settings?.enable_logical_replication === true,
             migrationsDir: olds?.migrationsDir,
@@ -347,8 +471,9 @@ export const ProjectProvider = () =>
           // (and let `read` upstream decide adoption); otherwise update
           // the mutable scalar fields on the existing project.
           const projectInfo = output
-            ? yield* api
-                .updateProject(output.projectId, {
+            ? yield* updateProject({
+                project_id: output.projectId,
+                project: {
                   name: news.name,
                   history_retention_seconds: news.historyRetentionSeconds,
                   settings:
@@ -359,45 +484,47 @@ export const ProjectProvider = () =>
                             news.enableLogicalReplication ?? false,
                         }
                       : undefined,
-                })
-                .pipe(
-                  Effect.map((r) => ({
-                    projectId: output.projectId,
-                    projectName: r.project.name,
-                    region: output.region,
-                    pgVersion: output.pgVersion,
-                    defaultBranchId: output.defaultBranchId,
-                    defaultBranchName: output.defaultBranchName,
-                    databaseName: output.databaseName,
-                    roleName: output.roleName,
-                    connectionUri: output.connectionUri,
-                    pooledConnectionUri: output.pooledConnectionUri,
-                    origin: output.origin,
-                    historyRetentionSeconds:
-                      r.project.history_retention_seconds ??
-                      output.historyRetentionSeconds,
-                    enableLogicalReplication:
-                      r.project.settings?.enable_logical_replication === true,
-                  })),
-                )
+                },
+              }).pipe(
+                Effect.map((r) => ({
+                  projectId: output.projectId,
+                  projectName: r.project.name,
+                  region: output.region,
+                  pgVersion: output.pgVersion,
+                  defaultBranchId: output.defaultBranchId,
+                  defaultBranchName: output.defaultBranchName,
+                  databaseName: output.databaseName,
+                  roleName: output.roleName,
+                  connectionUri: output.connectionUri,
+                  pooledConnectionUri: output.pooledConnectionUri,
+                  origin: output.origin,
+                  historyRetentionSeconds:
+                    r.project.history_retention_seconds ??
+                    output.historyRetentionSeconds,
+                  enableLogicalReplication:
+                    r.project.settings?.enable_logical_replication === true,
+                })),
+              )
             : yield* Effect.gen(function* () {
                 const name = yield* createProjectName(id, news.name);
-                const created = yield* api.createProject({
-                  name,
-                  region_id: news.region,
-                  pg_version: news.pgVersion,
-                  default_branch: {
-                    name: news.defaultBranchName,
-                    role_name: news.roleName,
-                    database_name: news.databaseName,
+                const created = yield* sdkCreateProject({
+                  project: {
+                    name,
+                    region_id: news.region,
+                    pg_version: news.pgVersion,
+                    branch: {
+                      name: news.defaultBranchName,
+                      role_name: news.roleName,
+                      database_name: news.databaseName,
+                    },
+                    history_retention_seconds: news.historyRetentionSeconds,
+                    org_id: news.orgId,
+                    settings: news.enableLogicalReplication
+                      ? { enable_logical_replication: true }
+                      : undefined,
                   },
-                  history_retention_seconds: news.historyRetentionSeconds,
-                  org_id: news.orgId,
-                  settings: news.enableLogicalReplication
-                    ? { enable_logical_replication: true }
-                    : undefined,
                 });
-                yield* api.waitForOperations(created.operations);
+                yield* waitForOperations(created.operations);
 
                 const branchId = created.branch.id;
                 const databaseName = getDatabaseName(created);
@@ -458,7 +585,9 @@ export const ProjectProvider = () =>
           };
         }),
         delete: Effect.fn(function* ({ output }) {
-          yield* api.deleteProject(output.projectId);
+          yield* deleteProject({ project_id: output.projectId }).pipe(
+            Effect.catchTag("NotFound", () => Effect.void),
+          );
         }),
       };
     }),

package/src/Output.ts

@@ -1,11 +1,10 @@
 import * as Data from "effect/Data";
-import type { Yieldable } from "effect/Effect";
 import * as Effect from "effect/Effect";
 import { pipe } from "effect/Function";
 import type { Pipeable } from "effect/Pipeable";
 import * as Redacted from "effect/Redacted";
 import { SingleShotGen } from "effect/Utils";
-import { getRefMetadata, isRef, ref as stageRef, type Ref } from "./Ref.ts";
+import { getRefMetadata, isRef, type Ref } from "./Ref.ts";
 import { isResource, type Resource, type ResourceLike } from "./Resource.ts";
 import { RuntimeContext } from "./RuntimeContext.ts";
 import { Stack } from "./Stack.ts";
@@ -48,7 +47,7 @@ export interface Output<A = any, Req = any> extends Pipeable {
   readonly req: Req;
   /** @internal phantom */
   [Symbol.iterator](): Iterator<
-    Yieldable<any, void, never, Req>,
+    Effect.Effect<void, never, Req>,
     Accessor<A>,
     void
   >;
@@ -82,9 +81,11 @@ export type Expr<A = any, Req = any> =
   | ApplyExpr<any, A, Req>
   | EffectExpr<any, A, Req>
   | LiteralExpr<A>
+  | NamedExpr<A, Req>
   | PropExpr<A, keyof A, Req>
   | ResourceExpr<A, Req>
-  | RefExpr<A>;
+  | RefExpr<A>
+  | StackRefExpr<A>;
 
 export abstract class BaseExpr<A = any, Req = any> implements Output<A, Req> {
   declare readonly kind: any;
@@ -98,12 +99,12 @@ export abstract class BaseExpr<A = any, Req = any> implements Output<A, Req> {
   }
 
   [Symbol.iterator](): Iterator<
-    Yieldable<any, void, never, Req>,
+    Effect.Effect<void, never, Req>,
     Accessor<A>,
     void
   > {
     // @ts-expect-error - TODO(sam): fix this (works at runtime, but maybe indicates a bad assumption)
-    return new SingleShotGen(this);
+    return new SingleShotGen(this.asEffect());
   }
 
   asEffect(): any {
@@ -111,7 +112,7 @@ export abstract class BaseExpr<A = any, Req = any> implements Output<A, Req> {
   }
 
   public bind(id: string): any {
-    return RuntimeContext.asEffect().pipe(
+    return RuntimeContext.pipe(
       Effect.flatMap((ctx) =>
         Effect.map(ctx.set(id, this), (key) => ctx.get<A>(key)),
       ),
@@ -255,6 +256,36 @@ export class EffectExpr<A, B, Req = never, Req2 = never> extends BaseExpr<
   }
 }
 
+export const isNamedExpr = <A = any, Req = any>(
+  node: any,
+): node is NamedExpr<A, Req> => node?.kind === "NamedExpr";
+
+/**
+ * Wraps another `Expr` and overrides its `toString()` / inspect output.
+ *
+ * `BaseExpr` derives the binding id from `this.toString()`, so
+ * wrapping an expression in `NamedExpr` makes that derived id stable and
+ * caller-controlled (e.g. an env var name like `"API_KEY"`).
+ */
+export class NamedExpr<A, Req = never> extends BaseExpr<A, Req> {
+  readonly kind = "NamedExpr";
+  constructor(
+    public readonly expr: Expr<A, Req>,
+    public readonly bindingName: string,
+  ) {
+    super();
+    return proxy(this);
+  }
+  [inspect](): string {
+    return this.bindingName;
+  }
+}
+
+export const named = <A, Req>(
+  expr: Output<A, Req>,
+  name: string,
+): Output<A, Req> => new NamedExpr(expr as Expr<A, Req>, name) as any;
+
 export const all = <Outs extends (Output | Expr)[]>(...outs: Outs) =>
   new AllExpr(outs as any) as unknown as All<Outs>;
 
@@ -309,6 +340,48 @@ export class RefExpr<A> extends BaseExpr<A, never> {
   }
 }
 
+export const isStackRefExpr = <A = any>(node: any): node is StackRefExpr<A> =>
+  node?.kind === "StackRefExpr";
+
+/**
+ * A reference to the persisted output of a Stack at `(stack, stage)`.
+ *
+ * Resolved at evaluation time by reading `state.getOutput({ stack,
+ * stage })`. Distinct from {@link RefExpr}, which references a
+ * single resource's attributes within a stack/stage. `stage` may be
+ * `undefined`, in which case it falls back to the current stage.
+ */
+export class StackRefExpr<A> extends BaseExpr<A, never> {
+  readonly kind = "StackRefExpr";
+  constructor(
+    public readonly stack: string,
+    public readonly stage: string | undefined,
+  ) {
+    super();
+    return proxy(this);
+  }
+  [inspect](): string {
+    return `stackRef(${this.stack}${
+      this.stage ? `, { stage: ${this.stage} }` : ""
+    })`;
+  }
+}
+
+/**
+ * Build an `Output<A>` referencing the persisted output of another
+ * Stack. The returned Effect resolves to a lazy `Output<A>` whose
+ * value is read from the state store at plan/apply time.
+ *
+ * Returns `Effect<Output<A>>` (not `Output<A>` directly) so that
+ * `yield* Output.stackRef(...)` reads ergonomically inside an Effect
+ * generator and lines up with `Resource.ref` and `Stack.stage.<name>`.
+ */
+export const stackRef = <A>(
+  stack: string,
+  options: { stage?: string } = {},
+): Effect.Effect<Output<A, never>> =>
+  Effect.succeed(new StackRefExpr<A>(stack, options.stage) as any);
+
 export const filter = <Outs extends any[]>(...outs: Outs) =>
   outs.filter(isOutput) as unknown as Filter<Outs>;
 
@@ -353,7 +426,32 @@ function proxy(self: any): any {
       prop === ExprSymbol || prop === inspect ? true : prop in self,
     get: (target, prop) =>
       prop === Symbol.toPrimitive
-        ? (hint: string) => (hint === "number" ? Number.NaN : self.toString())
+        ? (hint: string) => {
+            // Any JS-level coercion of an unresolved Output produces a
+            // placeholder that *looks* like a real value but isn't:
+            //
+            //   - `string` / `default` hints (`${output}`, `output + ""`,
+            //     `==` against a primitive) previously fell through to
+            //     `self.toString()` and returned the inspect form
+            //     (e.g. "tunnel.tunnelId"). The bogus string flowed
+            //     into resource props and into the cloud — only
+            //     surfacing as an opaque downstream error (see PR
+            //     description for a real Cloudflare DNS landing).
+            //
+            //   - `number` hint (`+output`, `output * 2`,
+            //     `Math.max(0, output)`) previously returned NaN, which
+            //     propagates silently through arithmetic and lands as
+            //     "the API rejected a NaN field" much later.
+            //
+            // All three hints fail loud at the coercion site with a
+            // pointer to the right composition API.
+            throw new Error(
+              `Cannot coerce Output<${self[inspect]()}> to a ` +
+                `${hint === "number" ? "number" : "string"} via JS coercion. ` +
+                `Use Output.interpolate\`...\` or Output.map(output, fn) ` +
+                `to compose Outputs — the value isn't known until deploy time.`,
+            );
+          }
         : prop === ExprSymbol
           ? self
           : prop === inspect
@@ -445,6 +543,8 @@ export const evaluate: <A, Req = never>(
         );
       } else if (isPropExpr(expr)) {
         return (yield* evaluate(expr.expr, upstream))?.[expr.identifier];
+      } else if (isNamedExpr(expr)) {
+        return yield* evaluate(expr.expr, upstream);
       } else if (isRefExpr(expr)) {
         const state = yield* State.State;
         const stack = expr.stack ?? (yield* Stack).name;
@@ -464,7 +564,26 @@ export const evaluate: <A, Req = never>(
             }),
           );
         }
-        return resource.attr;
+        // RefExpr targets persisted resources; tasks aren't cross-stack
+        // referenceable. Return the resource's output attrs, otherwise the
+        // task's output value, otherwise undefined.
+        return (resource as any).attr ?? (resource as any).output;
+      } else if (isStackRefExpr(expr)) {
+        const state = yield* State.State;
+        const stack = expr.stack;
+        const stage = expr.stage ?? (yield* Stage);
+        const output = yield* state.getOutput({ stack, stage });
+        if (output == null) {
+          return yield* Effect.fail(
+            new InvalidReferenceError({
+              message: `Reference to stack '${stack}' at stage '${stage}' not found. Have you deployed stage '${stage}' of '${stack}'?`,
+              stack,
+              stage,
+              resourceId: stack,
+            }),
+          );
+        }
+        return output;
       }
     }
     if (Array.isArray(expr)) {
@@ -483,14 +602,6 @@ export const evaluate: <A, Req = never>(
     return expr;
   }) as Effect.Effect<any>;
 
-export const ref = <R extends ResourceLike>(
-  resourceId: R["LogicalId"],
-  options?: {
-    stage?: string;
-    stack?: string;
-  },
-) => of(stageRef({ id: resourceId, ...options }));
-
 export const hasOutputs = (value: any): value is Output<any, any> =>
   Object.keys(upstreamAny(value)).length > 0;
 
@@ -531,7 +642,7 @@ export const upstream = <E extends Output<any, any>>(expr: E): any => {
     return upstream(expr.expr);
   } else if (isAllExpr(expr)) {
     return Object.assign({}, ...expr.outs.map((out) => upstream(out)));
-  } else if (isEffectExpr(expr) || isApplyExpr(expr)) {
+  } else if (isEffectExpr(expr) || isApplyExpr(expr) || isNamedExpr(expr)) {
     return upstream(expr.expr);
   } else if (Array.isArray(expr)) {
     return expr.map(upstream).reduce(toObject, {});