alchemy 2.0.0-beta.21 → 2.0.0-beta.22

package/src/Cloudflare/D1/D1Import.ts

@@ -0,0 +1,156 @@
+import * as d1 from "@distilled.cloud/cloudflare/d1";
+import {
+  Credentials,
+  formatHeaders,
+} from "@distilled.cloud/cloudflare/Credentials";
+import crypto from "node:crypto";
+import * as Effect from "effect/Effect";
+import * as HttpClient from "effect/unstable/http/HttpClient";
+import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest";
+
+export interface ImportD1DatabaseOptions {
+  accountId: string;
+  databaseId: string;
+  sqlData: string | Uint8Array;
+  filename?: string;
+}
+
+export interface ImportD1DatabaseResult {
+  filename: string;
+  numQueries: number;
+}
+
+interface ImportPollingResponse {
+  at_bookmark?: string | null;
+  error?: string | null;
+  filename?: string | null;
+  messages?: string[] | null;
+  result?: {
+    final_bookmark?: string | null;
+    num_queries?: number | null;
+  } | null;
+  status?: "complete" | "error" | null;
+  success?: boolean | null;
+  upload_url?: string | null;
+}
+
+const md5 = (data: string | Uint8Array): string =>
+  crypto.createHash("md5").update(data).digest("hex");
+
+const importEndpoint = (
+  apiBaseUrl: string,
+  accountId: string,
+  databaseId: string,
+) => `${apiBaseUrl}/accounts/${accountId}/d1/database/${databaseId}/import`;
+
+/**
+ * Import SQL into a D1 database via the multi-step Cloudflare flow:
+ *   1. POST `action: "init"` -> returns presigned `upload_url`
+ *   2. PUT raw SQL to `upload_url`
+ *   3. POST `action: "ingest"` -> returns `at_bookmark`
+ *   4. POST `action: "poll"` until `status === "complete"`
+ */
+export const importD1Database = (options: ImportD1DatabaseOptions) =>
+  Effect.gen(function* () {
+    const credentialsEff = yield* Credentials;
+    const credentials = yield* credentialsEff;
+    const authHeaders = formatHeaders(credentials);
+    const url = importEndpoint(
+      credentials.apiBaseUrl,
+      options.accountId,
+      options.databaseId,
+    );
+    const client = yield* HttpClient.HttpClient;
+
+    const postJson = (
+      body: unknown,
+    ): Effect.Effect<ImportPollingResponse, never, never> =>
+      Effect.gen(function* () {
+        const req = HttpClientRequest.post(url).pipe(
+          HttpClientRequest.setHeaders(authHeaders),
+          HttpClientRequest.bodyJsonUnsafe(body),
+        );
+        const res = yield* client.execute(req).pipe(Effect.orDie);
+        if (res.status < 200 || res.status >= 300) {
+          const text = yield* res.text.pipe(Effect.orElseSucceed(() => ""));
+          return yield* Effect.die(
+            `D1 import request failed (${res.status}): ${text}`,
+          );
+        }
+        const text = yield* res.text.pipe(Effect.orDie);
+        const json = JSON.parse(text) as { result: ImportPollingResponse };
+        return json.result;
+      });
+
+    const etag = md5(options.sqlData);
+
+    // Step 1: init via the typed d1 client
+    const initDb = yield* d1.importDatabase;
+    const init = yield* initDb({
+      accountId: options.accountId,
+      databaseId: options.databaseId,
+      action: "init",
+      etag,
+    });
+
+    if (!init.uploadUrl) {
+      return yield* Effect.die(
+        init.error ?? "Failed to get upload URL for D1 import",
+      );
+    }
+    const uploadFilename = options.filename ?? init.filename ?? "import.sql";
+
+    // Step 2: PUT to the presigned upload URL
+    const bytes =
+      typeof options.sqlData === "string"
+        ? new TextEncoder().encode(options.sqlData)
+        : options.sqlData;
+    const putReq = HttpClientRequest.put(init.uploadUrl).pipe(
+      HttpClientRequest.bodyUint8Array(bytes, "application/sql"),
+    );
+    const putRes = yield* client.execute(putReq).pipe(Effect.orDie);
+    if (putRes.status < 200 || putRes.status >= 300) {
+      const text = yield* putRes.text.pipe(Effect.orElseSucceed(() => ""));
+      return yield* Effect.die(
+        `Failed to upload SQL file to D1 (${putRes.status}): ${text}`,
+      );
+    }
+
+    // Step 3: ingest
+    const ingest = yield* postJson({
+      action: "ingest",
+      etag,
+      filename: init.filename,
+    });
+    if (!ingest.at_bookmark) {
+      return yield* Effect.die(
+        ingest.error ?? "Ingest response missing bookmark",
+      );
+    }
+
+    // Step 4: poll until complete
+    const poll = (
+      bookmark: string,
+    ): Effect.Effect<ImportD1DatabaseResult, never, never> =>
+      Effect.gen(function* () {
+        const data = yield* postJson({
+          action: "poll",
+          current_bookmark: bookmark,
+        });
+        if (data.status === "complete" && data.result) {
+          return {
+            filename: data.filename ?? uploadFilename,
+            numQueries: data.result.num_queries ?? 0,
+          };
+        }
+        if (data.status === "error") {
+          return yield* Effect.die(data.error ?? "Error during D1 import");
+        }
+        if (!data.at_bookmark) {
+          return yield* Effect.die("D1 import poll missing bookmark");
+        }
+        return yield* poll(data.at_bookmark);
+      });
+
+    return yield* poll(ingest.at_bookmark);
+  });

package/src/Cloudflare/D1/D1Migrations.ts

@@ -0,0 +1,229 @@
+import * as d1 from "@distilled.cloud/cloudflare/d1";
+import * as Effect from "effect/Effect";
+import type * as HttpClient from "effect/unstable/http/HttpClient";
+import type { Credentials } from "@distilled.cloud/cloudflare/Credentials";
+import type { D1SqlFile } from "./D1SqlFile.ts";
+
+export interface ApplyMigrationsOptions {
+  accountId: string;
+  databaseId: string;
+  migrationsTable: string;
+  migrationsFiles: ReadonlyArray<D1SqlFile>;
+}
+
+interface TableColumn {
+  name: string;
+  type: string;
+  pk: number;
+}
+
+interface SchemaInfo {
+  exists: boolean;
+  hasIdColumn: boolean;
+  hasNameColumn: boolean;
+  isLegacySchema: boolean;
+  columns: TableColumn[];
+}
+
+const executeSQL = (
+  accountId: string,
+  databaseId: string,
+  sql: string,
+): Effect.Effect<
+  d1.QueryDatabaseResponse,
+  d1.QueryDatabaseError,
+  Credentials | HttpClient.HttpClient
+> =>
+  Effect.gen(function* () {
+    const queryDb = yield* d1.queryDatabase;
+    return yield* queryDb({ accountId, databaseId, sql });
+  });
+
+const detectSchema = (
+  accountId: string,
+  databaseId: string,
+  migrationsTable: string,
+) =>
+  Effect.gen(function* () {
+    const result = yield* executeSQL(
+      accountId,
+      databaseId,
+      `PRAGMA table_info(${migrationsTable});`,
+    ).pipe(Effect.option);
+
+    const columns: TableColumn[] = [];
+    if (result._tag === "Some") {
+      const rows = (result.value.result[0]?.results ?? []) as Array<{
+        name: string;
+        type: string;
+        pk: number;
+      }>;
+      for (const row of rows) {
+        columns.push({ name: row.name, type: row.type, pk: row.pk });
+      }
+    }
+
+    if (columns.length === 0) {
+      return {
+        exists: false,
+        hasIdColumn: false,
+        hasNameColumn: false,
+        isLegacySchema: false,
+        columns,
+      } satisfies SchemaInfo;
+    }
+
+    const names = columns.map((c) => c.name);
+    const hasIdColumn = names.includes("id");
+    const hasNameColumn = names.includes("name");
+    const isLegacySchema =
+      columns.length === 2 && !(hasIdColumn && hasNameColumn);
+
+    return {
+      exists: true,
+      hasIdColumn,
+      hasNameColumn,
+      isLegacySchema,
+      columns,
+    } satisfies SchemaInfo;
+  });
+
+const migrateLegacySchema = (
+  accountId: string,
+  databaseId: string,
+  migrationsTable: string,
+  schema: SchemaInfo,
+) =>
+  Effect.gen(function* () {
+    const primaryColumn =
+      schema.columns.find((c) => c.pk === 1)?.name ?? schema.columns[0]?.name;
+    if (!primaryColumn) {
+      return yield* Effect.die(
+        "Cannot migrate legacy migration table: no columns found",
+      );
+    }
+    const tempTable = `${migrationsTable}_temp_migration`;
+    yield* executeSQL(
+      accountId,
+      databaseId,
+      `CREATE TABLE ${tempTable} (
+        id TEXT PRIMARY KEY,
+        name TEXT NOT NULL,
+        applied_at TEXT NOT NULL
+      );`,
+    );
+    yield* executeSQL(
+      accountId,
+      databaseId,
+      `INSERT INTO ${tempTable} (id, name, applied_at)
+       SELECT
+         printf('%05d', row_number() OVER (ORDER BY applied_at)) as id,
+         ${primaryColumn} as name,
+         applied_at
+       FROM ${migrationsTable}
+       ORDER BY applied_at;`,
+    );
+    yield* executeSQL(accountId, databaseId, `DROP TABLE ${migrationsTable};`);
+    yield* executeSQL(
+      accountId,
+      databaseId,
+      `ALTER TABLE ${tempTable} RENAME TO ${migrationsTable};`,
+    );
+  });
+
+const ensureMigrationsTable = (
+  accountId: string,
+  databaseId: string,
+  migrationsTable: string,
+) =>
+  Effect.gen(function* () {
+    const schema = yield* detectSchema(accountId, databaseId, migrationsTable);
+    if (!schema.exists) {
+      yield* executeSQL(
+        accountId,
+        databaseId,
+        `CREATE TABLE ${migrationsTable} (
+          id TEXT PRIMARY KEY,
+          name TEXT NOT NULL,
+          applied_at TEXT NOT NULL
+        );`,
+      );
+      return;
+    }
+    if (schema.isLegacySchema || !schema.hasIdColumn || !schema.hasNameColumn) {
+      yield* migrateLegacySchema(
+        accountId,
+        databaseId,
+        migrationsTable,
+        schema,
+      );
+    }
+  });
+
+const getAppliedMigrations = (
+  accountId: string,
+  databaseId: string,
+  migrationsTable: string,
+) =>
+  Effect.gen(function* () {
+    const result = yield* executeSQL(
+      accountId,
+      databaseId,
+      `SELECT name FROM ${migrationsTable};`,
+    );
+    const rows = (result.result[0]?.results ?? []) as Array<{ name: string }>;
+    return new Set(rows.map((r) => r.name));
+  });
+
+const getNextSeq = (
+  accountId: string,
+  databaseId: string,
+  migrationsTable: string,
+) =>
+  Effect.gen(function* () {
+    const result = yield* executeSQL(
+      accountId,
+      databaseId,
+      `SELECT id FROM ${migrationsTable} ORDER BY id;`,
+    );
+    const rows = (result.result[0]?.results ?? []) as Array<{ id: string }>;
+    let max = 0;
+    for (const { id } of rows) {
+      if (/^\d+$/.test(id)) {
+        max = Math.max(max, Number.parseInt(id, 10));
+      }
+    }
+    return max + 1;
+  });
+
+/**
+ * Apply pending D1 migrations in order. Uses the wrangler-compatible
+ * 3-column schema `(id TEXT PK, name TEXT, applied_at TEXT)`.
+ */
+export const applyMigrations = (options: ApplyMigrationsOptions) =>
+  Effect.gen(function* () {
+    const { accountId, databaseId, migrationsTable, migrationsFiles } = options;
+    yield* ensureMigrationsTable(accountId, databaseId, migrationsTable);
+    const applied = yield* getAppliedMigrations(
+      accountId,
+      databaseId,
+      migrationsTable,
+    );
+    let nextSeq = yield* getNextSeq(accountId, databaseId, migrationsTable);
+
+    for (const migration of migrationsFiles) {
+      if (applied.has(migration.id)) continue;
+      const migrationId = nextSeq.toString().padStart(5, "0");
+      nextSeq += 1;
+      // D1 over HTTP doesn't support transactions; run migration + record
+      // in a single batched query.
+      yield* executeSQL(
+        accountId,
+        databaseId,
+        [
+          migration.sql,
+          `INSERT INTO ${migrationsTable} (id, name, applied_at) VALUES ('${migrationId}', '${migration.id}', datetime('now'));`,
+        ].join("\n"),
+      );
+    }
+  });

package/src/Cloudflare/D1/D1SqlFile.ts

@@ -0,0 +1,57 @@
+import crypto from "node:crypto";
+import * as Effect from "effect/Effect";
+import * as FileSystem from "effect/FileSystem";
+import * as Path from "effect/Path";
+
+export interface D1SqlFile {
+  id: string;
+  sql: string;
+  hash: string;
+}
+
+/**
+ * Recursively list `.sql` files under `directory`, sorted by their numeric
+ * prefix (e.g. `0001_init.sql`) and then by name.
+ */
+export const listSqlFiles = (directory: string) =>
+  Effect.gen(function* () {
+    const fs = yield* FileSystem.FileSystem;
+    const entries = yield* fs.readDirectory(directory, { recursive: true });
+
+    const sqlFiles = entries
+      .filter((name) => name.endsWith(".sql"))
+      .sort((a, b) => {
+        const aNum = getPrefix(a);
+        const bNum = getPrefix(b);
+        if (aNum !== null && bNum !== null) return aNum - bNum;
+        if (aNum !== null) return -1;
+        if (bNum !== null) return 1;
+        return a.localeCompare(b);
+      });
+
+    return yield* Effect.all(
+      sqlFiles.map((id) => readSqlFile(directory, id)),
+    );
+  });
+
+/**
+ * Read a single `.sql` file relative to `directory` and compute its content
+ * hash. The `sql` field is marked non-enumerable so it isn't serialized into
+ * resource state.
+ */
+export const readSqlFile = (directory: string, name: string) =>
+  Effect.gen(function* () {
+    const fs = yield* FileSystem.FileSystem;
+    const path = yield* Path.Path;
+    const sql = yield* fs.readFileString(path.resolve(directory, name));
+    const hash = crypto.createHash("sha256").update(sql).digest("hex");
+    const file: D1SqlFile = { id: name, sql, hash };
+    Object.defineProperty(file, "sql", { enumerable: false });
+    return file;
+  });
+
+const getPrefix = (name: string): number | null => {
+  const prefix = name.split("_")[0];
+  const num = Number.parseInt(prefix, 10);
+  return Number.isNaN(num) ? null : num;
+};