alchemy 2.0.0-beta.65 → 2.0.0-beta.67

package/src/Cloudflare/D1/ApplyMigrations.ts

@@ -1,9 +1,29 @@
-import type { Credentials } from "@distilled.cloud/cloudflare/Credentials";
 import * as d1 from "@distilled.cloud/cloudflare/d1";
 import * as Effect from "effect/Effect";
-import type * as HttpClient from "effect/unstable/http/HttpClient";
 import type { SqlFile } from "../../SQL/SqlFile.ts";
 
+/**
+ * The minimal query surface the migration flow needs: run (possibly
+ * multi-statement) SQL and return the D1 HTTP API's result envelope. Two
+ * implementations exist:
+ *
+ * - the cloud executor ({@link applyMigrations}), backed by distilled's
+ *   `d1.queryDatabase`;
+ * - the local executor (`LocalD1Gateway.ts`), which tunnels the same
+ *   protocol into the local workerd D1 simulator.
+ */
+export interface D1QueryResult {
+  result: Array<{
+    results?: unknown;
+    success?: boolean | null;
+    meta?: unknown;
+  }>;
+}
+
+export type D1SqlExecutor<E = unknown, R = never> = (
+  sql: string,
+) => Effect.Effect<D1QueryResult, E, R>;
+
 export interface ApplyMigrationsOptions {
   accountId: string;
   databaseId: string;
@@ -12,19 +32,38 @@ export interface ApplyMigrationsOptions {
 }
 
 /**
- * Apply pending D1 migrations in order. Uses the wrangler-compatible
- * 3-column schema `(id TEXT PK, name TEXT, applied_at TEXT)`.
+ * Apply pending D1 migrations in order against the live cloud API. Uses the
+ * wrangler-compatible 3-column schema `(id TEXT PK, name TEXT, applied_at
+ * TEXT)`.
  */
 export const applyMigrations = (options: ApplyMigrationsOptions) =>
   Effect.gen(function* () {
-    const { accountId, databaseId, migrationsTable, migrationsFiles } = options;
-    yield* ensureMigrationsTable(accountId, databaseId, migrationsTable);
-    const applied = yield* getAppliedMigrations(
-      accountId,
-      databaseId,
-      migrationsTable,
-    );
-    let nextSeq = yield* getNextSeq(accountId, databaseId, migrationsTable);
+    const queryDb = yield* d1.queryDatabase;
+    const executor: D1SqlExecutor<d1.QueryDatabaseError> = (sql) =>
+      queryDb({
+        accountId: options.accountId,
+        databaseId: options.databaseId,
+        sql,
+      });
+    yield* applyMigrationsWith(executor, options);
+  });
+
+/**
+ * Apply pending D1 migrations in order through the given SQL executor —
+ * the executor-agnostic core shared by the cloud and local flows.
+ */
+export const applyMigrationsWith = <E, R>(
+  executor: D1SqlExecutor<E, R>,
+  options: {
+    migrationsTable: string;
+    migrationsFiles: ReadonlyArray<SqlFile>;
+  },
+): Effect.Effect<void, E, R> =>
+  Effect.gen(function* () {
+    const { migrationsTable, migrationsFiles } = options;
+    yield* ensureMigrationsTable(executor, migrationsTable);
+    const applied = yield* getAppliedMigrations(executor, migrationsTable);
+    let nextSeq = yield* getNextSeq(executor, migrationsTable);
 
     for (const migration of migrationsFiles) {
       if (applied.has(migration.id)) continue;
@@ -32,9 +71,7 @@ export const applyMigrations = (options: ApplyMigrationsOptions) =>
       nextSeq += 1;
       // D1 over HTTP doesn't support transactions; run migration + record
       // in a single batched query.
-      yield* executeSQL(
-        accountId,
-        databaseId,
+      yield* executor(
         [
           migration.sql,
           `INSERT INTO ${migrationsTable} (id, name, applied_at) VALUES ('${migrationId}', '${migration.id}', datetime('now'));`,
@@ -57,29 +94,12 @@ interface SchemaInfo {
   columns: TableColumn[];
 }
 
-const executeSQL = (
-  accountId: string,
-  databaseId: string,
-  sql: string,
-): Effect.Effect<
-  d1.QueryDatabaseResponse,
-  d1.QueryDatabaseError,
-  Credentials | HttpClient.HttpClient
-> =>
-  Effect.gen(function* () {
-    const queryDb = yield* d1.queryDatabase;
-    return yield* queryDb({ accountId, databaseId, sql });
-  });
-
-const detectSchema = (
-  accountId: string,
-  databaseId: string,
+const detectSchema = <E, R>(
+  executor: D1SqlExecutor<E, R>,
   migrationsTable: string,
 ) =>
   Effect.gen(function* () {
-    const result = yield* executeSQL(
-      accountId,
-      databaseId,
+    const result = yield* executor(
       `PRAGMA table_info(${migrationsTable});`,
     ).pipe(Effect.option);
 
@@ -120,9 +140,8 @@ const detectSchema = (
     } satisfies SchemaInfo;
   });
 
-const migrateLegacySchema = (
-  accountId: string,
-  databaseId: string,
+const migrateLegacySchema = <E, R>(
+  executor: D1SqlExecutor<E, R>,
   migrationsTable: string,
   schema: SchemaInfo,
 ) =>
@@ -135,18 +154,14 @@ const migrateLegacySchema = (
       );
     }
     const tempTable = `${migrationsTable}_temp_migration`;
-    yield* executeSQL(
-      accountId,
-      databaseId,
+    yield* executor(
       `CREATE TABLE ${tempTable} (
         id TEXT PRIMARY KEY,
         name TEXT NOT NULL,
         applied_at TEXT NOT NULL
       );`,
     );
-    yield* executeSQL(
-      accountId,
-      databaseId,
+    yield* executor(
       `INSERT INTO ${tempTable} (id, name, applied_at)
        SELECT
          printf('%05d', row_number() OVER (ORDER BY applied_at)) as id,
@@ -155,25 +170,18 @@ const migrateLegacySchema = (
        FROM ${migrationsTable}
        ORDER BY applied_at;`,
     );
-    yield* executeSQL(accountId, databaseId, `DROP TABLE ${migrationsTable};`);
-    yield* executeSQL(
-      accountId,
-      databaseId,
-      `ALTER TABLE ${tempTable} RENAME TO ${migrationsTable};`,
-    );
+    yield* executor(`DROP TABLE ${migrationsTable};`);
+    yield* executor(`ALTER TABLE ${tempTable} RENAME TO ${migrationsTable};`);
   });
 
-const ensureMigrationsTable = (
-  accountId: string,
-  databaseId: string,
+const ensureMigrationsTable = <E, R>(
+  executor: D1SqlExecutor<E, R>,
   migrationsTable: string,
 ) =>
   Effect.gen(function* () {
-    const schema = yield* detectSchema(accountId, databaseId, migrationsTable);
+    const schema = yield* detectSchema(executor, migrationsTable);
     if (!schema.exists) {
-      yield* executeSQL(
-        accountId,
-        databaseId,
+      yield* executor(
         `CREATE TABLE ${migrationsTable} (
           id TEXT PRIMARY KEY,
           name TEXT NOT NULL,
@@ -183,39 +191,26 @@ const ensureMigrationsTable = (
       return;
     }
     if (schema.isLegacySchema || !schema.hasIdColumn || !schema.hasNameColumn) {
-      yield* migrateLegacySchema(
-        accountId,
-        databaseId,
-        migrationsTable,
-        schema,
-      );
+      yield* migrateLegacySchema(executor, migrationsTable, schema);
     }
   });
 
-const getAppliedMigrations = (
-  accountId: string,
-  databaseId: string,
+const getAppliedMigrations = <E, R>(
+  executor: D1SqlExecutor<E, R>,
   migrationsTable: string,
 ) =>
   Effect.gen(function* () {
-    const result = yield* executeSQL(
-      accountId,
-      databaseId,
-      `SELECT name FROM ${migrationsTable};`,
-    );
+    const result = yield* executor(`SELECT name FROM ${migrationsTable};`);
     const rows = (result.result[0]?.results ?? []) as Array<{ name: string }>;
     return new Set(rows.map((r) => r.name));
   });
 
-const getNextSeq = (
-  accountId: string,
-  databaseId: string,
+const getNextSeq = <E, R>(
+  executor: D1SqlExecutor<E, R>,
   migrationsTable: string,
 ) =>
   Effect.gen(function* () {
-    const result = yield* executeSQL(
-      accountId,
-      databaseId,
+    const result = yield* executor(
       `SELECT id FROM ${migrationsTable} ORDER BY id;`,
     );
     const rows = (result.result[0]?.results ?? []) as Array<{ id: string }>;

package/src/Cloudflare/D1/Database.ts

@@ -1,19 +1,28 @@
+import type { RuntimeServices } from "@distilled.cloud/cloudflare-runtime";
 import * as d1 from "@distilled.cloud/cloudflare/d1";
 import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
 import * as Option from "effect/Option";
 import * as Stream from "effect/Stream";
+import * as HttpClient from "effect/unstable/http/HttpClient";
 
 import { isResolved } from "../../Diff.ts";
+import * as ProviderLayer from "../../Local/ProviderLayer.ts";
 import { createPhysicalName } from "../../PhysicalName.ts";
 import * as Provider from "../../Provider.ts";
 import { isResourceOfType, Resource } from "../../Resource.ts";
 import { listSqlFiles, readSqlFile } from "../../SQL/SqlFile.ts";
 import { recordsEqual } from "../../Util/equal.ts";
 import { CloudflareEnvironment } from "../CloudflareEnvironment.ts";
+import { generateLocalId, localRuntimeServices } from "../LocalRuntime.ts";
 import type { Providers } from "../Providers.ts";
-import { applyMigrations } from "./ApplyMigrations.ts";
+import { applyMigrations, applyMigrationsWith } from "./ApplyMigrations.ts";
 import { cloneDatabase } from "./CloneDatabase.ts";
 import { importD1Database } from "./ImportDatabase.ts";
+import {
+  localD1GatewayRuntime,
+  withLocalD1Executor,
+} from "./LocalD1Gateway.ts";
 
 export const isDatabase = (value: unknown): value is Database =>
   isResourceOfType(value, "Cloudflare.D1Database");
@@ -248,7 +257,7 @@ export type Database = Resource<
  */
 export const Database = Resource<Database>("Cloudflare.D1Database");
 
-export const DatabaseProvider = () =>
+export const ProviderLive = () =>
   Provider.succeed(Database, {
     stables: ["databaseId", "accountId"],
     diff: Effect.fn(function* ({ id, olds = {}, news = {}, output }) {
@@ -552,6 +561,164 @@ export const DatabaseProvider = () =>
     }),
   });
 
+/**
+ * Local (dev) provider — the database is purely virtual: a `dev:` id keyed
+ * into the local workerd D1 simulator (DO SQLite under `.alchemy/local/d1`).
+ * `toRuntimeBinding` lowers a `d1` binding whose id is `dev:`-prefixed onto
+ * the local D1 service.
+ *
+ * Migrations ARE applied locally: reconcile boots an ephemeral gateway
+ * workerd (see `LocalD1Gateway.ts`) and drives the same executor-agnostic
+ * migration flow the live provider uses, against the simulator's storage.
+ * Imports (`importFiles`) are still cloud-only — the import flow is a
+ * multi-step HTTP upload with no local counterpart.
+ */
+export const ProviderLocal = () =>
+  Provider.effect(
+    Database,
+    Effect.gen(function* () {
+      // The local runtime services (workerd `Runtime`, binding plugins) and
+      // the HTTP client are resolved once at layer build and closed over —
+      // lifecycle effects run with the engine's call-time context, which
+      // doesn't include them.
+      const runtimeContext = yield* Effect.context<
+        RuntimeServices | HttpClient.HttpClient
+      >();
+
+      return {
+        stables: ["accountId"],
+        diff: Effect.fn(function* ({ news = {}, output }) {
+          const { accountId } = yield* yield* CloudflareEnvironment;
+          if (!output?.databaseId) return { action: "update" } as const;
+          if (!isResolved(news)) return undefined;
+          if (output.accountId !== accountId) {
+            return { action: "replace" } as const;
+          }
+          // Detect migration/import file drift — same rules as the live
+          // provider.
+          if (news.migrationsDir) {
+            const newHashes = yield* hashMigrations(news.migrationsDir);
+            if (!recordsEqual(newHashes, output.migrationsHashes ?? {})) {
+              return { action: "update" } as const;
+            }
+            if (
+              (news.migrationsTable ?? DEFAULT_MIGRATIONS_TABLE) !==
+              (output.migrationsTable ?? DEFAULT_MIGRATIONS_TABLE)
+            ) {
+              return { action: "update" } as const;
+            }
+          }
+          if (news.importFiles?.length) {
+            const newHashes = yield* hashImports(
+              news.importFiles,
+              yield* rootDir,
+            );
+            if (!recordsEqual(newHashes, output.importHashes ?? {})) {
+              return { action: "update" } as const;
+            }
+          }
+          // Fall through to the engine's default prop diff.
+        }),
+        read: Effect.fn(function* ({ output }) {
+          // Purely virtual — the persisted state row is the source of truth.
+          return output ?? undefined;
+        }),
+        reconcile: Effect.fn(function* ({ id, news = {}, output }) {
+          const { accountId } = yield* yield* CloudflareEnvironment;
+          const databaseId = output?.databaseId ?? generateLocalId();
+
+          // Sync migrations — the shared flow is idempotent (skips applied
+          // entries), driven through the ephemeral local gateway.
+          const migrationsTable =
+            news.migrationsTable ??
+            output?.migrationsTable ??
+            DEFAULT_MIGRATIONS_TABLE;
+          let migrationsHashes: Record<string, string> = {};
+          if (news.migrationsDir) {
+            const files = yield* listSqlFiles(news.migrationsDir);
+            if (files.length > 0) {
+              yield* withLocalD1Executor(databaseId, (executor) =>
+                applyMigrationsWith(executor, {
+                  migrationsTable,
+                  migrationsFiles: files,
+                }),
+              ).pipe(
+                Effect.provide(
+                  localD1GatewayRuntime.pipe(
+                    Layer.provideMerge(Layer.succeedContext(runtimeContext)),
+                  ),
+                ),
+              );
+            }
+            for (const file of files) migrationsHashes[file.id] = file.hash;
+          } else {
+            migrationsHashes = output?.migrationsHashes ?? {};
+          }
+
+          // Sync imports — locally an import file is just multi-statement
+          // SQL, executed through the same gateway. Files whose hash matches
+          // previously-imported state are skipped (mirroring `runImports`).
+          const importHashes: Record<string, string> = {
+            ...(output?.importHashes ?? {}),
+          };
+          if (news.importFiles?.length) {
+            const importRootDir = yield* rootDir;
+            const pending: Array<{ path: string; sql: string; hash: string }> =
+              [];
+            for (const filePath of news.importFiles) {
+              const file = yield* readSqlFile(importRootDir, filePath);
+              if (importHashes[filePath] === file.hash) continue;
+              pending.push({ path: filePath, sql: file.sql, hash: file.hash });
+            }
+            if (pending.length > 0) {
+              yield* withLocalD1Executor(databaseId, (executor) =>
+                Effect.forEach(pending, (file) => executor(file.sql), {
+                  discard: true,
+                }),
+              ).pipe(
+                Effect.provide(
+                  localD1GatewayRuntime.pipe(
+                    Layer.provideMerge(Layer.succeedContext(runtimeContext)),
+                  ),
+                ),
+              );
+              for (const file of pending) {
+                importHashes[file.path] = file.hash;
+              }
+            }
+          }
+
+          return {
+            databaseId,
+            databaseName: yield* createDatabaseName(id, news.name),
+            jurisdiction: (news.jurisdiction ?? "default") as Jurisdiction,
+            readReplication: news.readReplication,
+            accountId: output?.accountId ?? accountId,
+            migrationsDir: news.migrationsDir,
+            migrationsTable: news.migrationsDir ? migrationsTable : undefined,
+            migrationsHashes,
+            importHashes,
+          };
+        }),
+        delete: Effect.fn(function* () {
+          // The simulator's on-disk data is keyed by the dev id; dropping
+          // the state row is enough — orphaned data is reclaimed with
+          // `.alchemy`.
+        }),
+      };
+    }),
+  );
+
+export const DatabaseProvider = () =>
+  ProviderLayer.dual(Database, {
+    // The local provider's reconcile boots an ephemeral workerd gateway to
+    // apply migrations, so it needs the shared local runtime layer (the
+    // module-memoized reference — one runtime instance per stack build,
+    // shared with the Worker/Queue/Container local providers).
+    local: () => ProviderLocal().pipe(Layer.provide(localRuntimeServices())),
+    live: () => ProviderLive(),
+  });
+
 const createDatabaseName = (id: string, name: string | undefined) =>
   Effect.gen(function* () {
     return name ?? (yield* createPhysicalName({ id }));

package/src/Cloudflare/D1/LocalD1Gateway.ts

@@ -0,0 +1,150 @@
+/**
+ * Node-side query path into the local workerd D1 simulator, built on the
+ * runtime's platform proxy (`PlatformProxy.open` — our `getPlatformProxy`).
+ *
+ * The simulator (a `d1` service routing to a per-database
+ * `D1DatabaseObject` Durable Object over DO SQLite) is only reachable from
+ * inside workerd — but alchemy's migration runner and the
+ * `QueryDatabaseLocal` capability run in Node. The platform proxy bridges
+ * the two: it hosts the binding in a scoped workerd instance and exposes it
+ * to Node. We bind the raw `d1` service (the same designator the
+ * `cloudflare-internal:d1-api` wrapped binding targets) so Node speaks the
+ * full D1 HTTP protocol (`POST /query`, multi-statement SQL) rather than
+ * the line-based `exec()` surface of the wrapped binding:
+ *
+ *   Node ── proxy.env.D1_RAW.fetch({sql}) ──▶ `d1` service ──▶ D1DatabaseObject
+ *
+ * Data lands in the same `{storage}/d1` directory every local worker
+ * binding reads.
+ *
+ * NOT exported from `index.ts` — provider-internal scaffolding.
+ */
+import { open } from "@distilled.cloud/cloudflare-runtime/platform-proxy";
+import { D1 } from "@distilled.cloud/cloudflare-runtime/bindings";
+import { SERVICE_D1 } from "@distilled.cloud/cloudflare-runtime/bindings/d1/D1Options";
+import type { BindingHook } from "@distilled.cloud/cloudflare-runtime/PluginContext";
+import * as Data from "effect/Data";
+import * as Effect from "effect/Effect";
+import { gatewayName, localGatewayRuntime } from "../LocalGateway.ts";
+import type { D1QueryResult, D1SqlExecutor } from "./ApplyMigrations.ts";
+
+export { localGatewayRuntime as localD1GatewayRuntime };
+
+export class LocalD1QueryError extends Data.TaggedError("LocalD1QueryError")<{
+  message: string;
+  cause?: unknown;
+}> {}
+
+/**
+ * A single query or batch in the shape the cloud `d1.queryDatabase` op
+ * accepts — the local gateway speaks the same surface so client code can
+ * swap transports without translation.
+ */
+export type D1QueryBody =
+  | { sql: string; params?: unknown[] }
+  | { batch: Array<{ sql: string; params?: unknown[] }> };
+
+interface D1StatementResponse {
+  success: boolean;
+  results?: unknown;
+  meta?: unknown;
+  error?: string;
+}
+
+/**
+ * A plain service binding onto the raw `d1` service for `databaseId` —
+ * bypassing the `cloudflare-internal:d1-api` wrapper so the proxy's fetch
+ * passthrough can POST the full D1 HTTP protocol.
+ */
+const rawD1Binding = (databaseId: string): BindingHook =>
+  Effect.succeed({
+    name: "D1_RAW",
+    service: {
+      name: SERVICE_D1,
+      props: { json: JSON.stringify({ databaseId }) },
+    },
+  });
+
+/**
+ * Boot a scoped platform proxy for `databaseId`, hand `use` a query
+ * function that tunnels {@link D1QueryBody} requests into the local
+ * simulator, and tear the instance down when `use` completes.
+ */
+export const withLocalD1Query = <A, E, R>(
+  databaseId: string,
+  use: (
+    query: (
+      body: D1QueryBody,
+    ) => Effect.Effect<D1QueryResult, LocalD1QueryError>,
+  ) => Effect.Effect<A, E, R>,
+) =>
+  Effect.scoped(
+    Effect.gen(function* () {
+      const proxy = yield* open({
+        name: gatewayName("alchemy-d1-gateway", databaseId),
+        bindings: [
+          // The wrapped binding is unused by the gateway itself, but its
+          // hook registers the database with the D1 plugin so the `d1`
+          // service is emitted into this workerd config.
+          D1.local({ binding: "DB", id: databaseId }),
+          rawD1Binding(databaseId),
+        ],
+      });
+      const raw = (proxy.env as Record<string, unknown>).D1_RAW as {
+        fetch: (input: string, init?: RequestInit) => Promise<Response>;
+      };
+
+      const query = (body: D1QueryBody) =>
+        Effect.tryPromise({
+          try: async () => {
+            const response = await raw.fetch("http://d1/query", {
+              method: "POST",
+              headers: { "content-type": "application/json" },
+              // The DO accepts a single D1Query or an array (one
+              // transaction) — a batch maps to the array form.
+              body: JSON.stringify("batch" in body ? body.batch : body),
+            });
+            return (await response.json()) as unknown;
+          },
+          catch: (cause) =>
+            new LocalD1QueryError({
+              message: "Failed to reach the local D1 gateway",
+              cause,
+            }),
+        }).pipe(
+          Effect.flatMap((responseBody) => {
+            // Protocol errors come back `{ success: false, error }` (a
+            // single object); successes are one envelope per statement.
+            const responses = (
+              Array.isArray(responseBody) ? responseBody : [responseBody]
+            ) as D1StatementResponse[];
+            const failed = responses.find((r) => !r.success);
+            if (failed) {
+              return Effect.fail(
+                new LocalD1QueryError({
+                  message: failed.error ?? "Local D1 query failed",
+                }),
+              );
+            }
+            return Effect.succeed({
+              result: responses.map((r) => ({
+                results: r.results,
+                success: true,
+                meta: r.meta,
+              })),
+            } as D1QueryResult);
+          }),
+        );
+
+      return yield* use(query);
+    }),
+  );
+
+/**
+ * SQL-only view of {@link withLocalD1Query} matching the migration flow's
+ * {@link D1SqlExecutor} contract.
+ */
+export const withLocalD1Executor = <A, E, R>(
+  databaseId: string,
+  use: (executor: D1SqlExecutor<LocalD1QueryError>) => Effect.Effect<A, E, R>,
+) => withLocalD1Query(databaseId, (query) => use((sql) => query({ sql })));