alchemy 0.7.1 → 0.7.3

package/lib/cloudflare/pipeline.js

@@ -0,0 +1,250 @@
+import { Resource } from "../resource";
+import { CloudflareApi, createCloudflareApi, } from "./api";
+import { CloudflareApiError, handleApiError } from "./api-error";
+/**
+ * Creates and manages Cloudflare Pipelines.
+ *
+ * Pipelines provide a managed data pipeline service that lets you collect, transform,
+ * and route data to various destinations like R2 buckets.
+ *
+ * @example
+ * // Create a basic pipeline with an R2 bucket destination
+ * const bucket = await R2Bucket("logs-bucket", {
+ *   name: "logs-bucket"
+ * });
+ *
+ * const accessKey = alchemy.secret(process.env.R2_ACCESS_KEY_ID!);
+ * const secretKey = alchemy.secret(process.env.R2_SECRET_ACCESS_KEY!);
+ *
+ * const pipeline = await Pipeline("logs-pipeline", {
+ *   name: "logs-pipeline",
+ *   destination: {
+ *     type: "r2",
+ *     format: "json",
+ *     path: {
+ *       bucket: bucket.name,
+ *       prefix: "app-logs",
+ *     },
+ *     credentials: {
+ *       accessKeyId: accessKey,
+ *       secretAccessKey: secretKey
+ *     }
+ *   },
+ *   batch: {
+ *     maxMb: 50,
+ *     maxSeconds: 60
+ *   }
+ * });
+ *
+ * @example
+ * // Create a pipeline with custom source configuration
+ * const customPipeline = await Pipeline("custom-pipeline", {
+ *   name: "custom-pipeline",
+ *   source: [{
+ *     type: "http",
+ *     format: "json",
+ *     authentication: true,
+ *     cors: {
+ *       origins: ["https://example.com"]
+ *     }
+ *   }],
+ *   destination: {
+ *     type: "r2",
+ *     format: "json",
+ *     path: {
+ *       bucket: "my-bucket",
+ *       prefix: "data"
+ *     },
+ *     credentials: {
+ *       accessKeyId: alchemy.secret(process.env.R2_ACCESS_KEY_ID!),
+ *       secretAccessKey: alchemy.secret(process.env.R2_SECRET_ACCESS_KEY!)
+ *     },
+ *     compression: {
+ *       type: "gzip"
+ *     }
+ *   }
+ * });
+ *
+ * @see https://developers.cloudflare.com/pipelines/
+ */
+export const Pipeline = Resource("cloudflare::Pipeline", async function (id, props) {
+    const api = await createCloudflareApi(props);
+    const pipelineName = props.name || id;
+    if (this.phase === "delete") {
+        console.log("Deleting Cloudflare Pipeline:", pipelineName);
+        if (props.delete !== false) {
+            // Delete Pipeline
+            await deletePipeline(api, pipelineName);
+        }
+        // Return void (a deleted pipeline has no content)
+        return this.destroy();
+    }
+    else {
+        let pipelineData;
+        if (this.phase === "create") {
+            console.log("Creating Cloudflare Pipeline:", pipelineName);
+            pipelineData = await createPipeline(api, pipelineName, props);
+        }
+        else {
+            // Update operation
+            if (this.output?.id) {
+                console.log("Updating Cloudflare Pipeline:", pipelineName);
+                // Check if name is being changed, which is not allowed
+                if (props.name !== this.output.name) {
+                    throw new Error("Cannot update Pipeline name after creation. Pipeline name is immutable.");
+                }
+                // Update the pipeline with new settings
+                pipelineData = await updatePipeline(api, pipelineName, props);
+            }
+            else {
+                // If no ID exists, fall back to creating a new pipeline
+                console.log("No existing Pipeline ID found, creating new Cloudflare Pipeline:", pipelineName);
+                pipelineData = await createPipeline(api, pipelineName, props);
+            }
+        }
+        return this({
+            type: "pipeline",
+            id: pipelineData.result.id,
+            name: pipelineName,
+            endpoint: pipelineData.result.endpoint,
+            version: pipelineData.result.version,
+            source: pipelineData.result.source.map((s) => ({
+                type: s.type,
+                format: s.format,
+                authentication: s.authentication,
+                cors: s.cors,
+            })),
+            destination: props.destination, // Use the input destination, not the API response
+            compression: props.compression,
+            accountId: api.accountId,
+        });
+    }
+});
+/**
+ * Get a pipeline
+ */
+export async function getPipeline(api, pipelineName) {
+    const response = await api.get(`/accounts/${api.accountId}/pipelines/${pipelineName}`);
+    if (!response.ok) {
+        return await handleApiError(response, "getting", "Pipeline", pipelineName);
+    }
+    return (await response.json());
+}
+/**
+ * Delete a pipeline
+ */
+export async function deletePipeline(api, pipelineName) {
+    // Delete Pipeline
+    const deleteResponse = await api.delete(`/accounts/${api.accountId}/pipelines/${pipelineName}`);
+    if (!deleteResponse.ok && deleteResponse.status !== 404) {
+        const errorData = await deleteResponse.json().catch(() => ({
+            errors: [{ message: deleteResponse.statusText }],
+        }));
+        throw new CloudflareApiError(`Error deleting Cloudflare Pipeline '${pipelineName}': ${errorData.errors?.[0]?.message || deleteResponse.statusText}`, deleteResponse);
+    }
+}
+/**
+ * Create a new pipeline
+ */
+export async function createPipeline(api, pipelineName, props) {
+    // Prepare the create payload
+    const createPayload = preparePipelinePayload(api, pipelineName, props);
+    const createResponse = await api.post(`/accounts/${api.accountId}/pipelines`, createPayload);
+    if (!createResponse.ok) {
+        return await handleApiError(createResponse, "creating", "Pipeline", pipelineName);
+    }
+    return (await createResponse.json());
+}
+/**
+ * Update a pipeline
+ */
+export async function updatePipeline(api, pipelineName, props) {
+    // Get current pipeline to build update payload
+    const currentPipeline = await getPipeline(api, pipelineName);
+    // Prepare the update payload
+    const updatePayload = preparePipelinePayload(api, pipelineName, props, currentPipeline);
+    const updateResponse = await api.put(`/accounts/${api.accountId}/pipelines/${pipelineName}`, updatePayload);
+    if (!updateResponse.ok) {
+        return await handleApiError(updateResponse, "updating", "Pipeline", pipelineName);
+    }
+    return (await updateResponse.json());
+}
+/**
+ * Helper function to prepare pipeline payload for create/update operations
+ */
+function preparePipelinePayload(api, pipelineName, props, currentPipeline) {
+    // Prepare the payload with name and source
+    const payload = {
+        name: pipelineName,
+        source: props.source ||
+            currentPipeline?.result.source || [
+            {
+                type: "http",
+                format: "json",
+                authentication: true,
+                cors: { origins: ["*"] },
+            },
+        ],
+    };
+    // Handle destination
+    if (props.destination) {
+        payload.destination = { ...props.destination };
+        // Handle special formatting for R2 destination
+        const r2Dest = props.destination;
+        // Format credentials for API
+        if (payload.destination.credentials) {
+            payload.destination.credentials = {
+                access_key_id: r2Dest.credentials.accessKeyId.unencrypted,
+                secret_access_key: r2Dest.credentials.secretAccessKey.unencrypted,
+                endpoint: r2Dest.credentials.endpoint ??
+                    `https://${api.accountId}.r2.cloudflarestorage.com`,
+            };
+        }
+        // Format batch settings
+        payload.destination.batch = convertBatchSettings(payload.destination.batch);
+    }
+    else if (currentPipeline?.result.destination) {
+        payload.destination = currentPipeline.result.destination;
+    }
+    else if (!props.destination && !currentPipeline) {
+        throw new Error("An R2 destination is required for creating/updating a pipeline");
+    }
+    // Add compression if not specified
+    if (!payload.destination.compression) {
+        payload.destination.compression = { type: "gzip" };
+    }
+    return payload;
+}
+/**
+ * List all pipelines in an account
+ */
+export async function listPipelines(api) {
+    const response = await api.get(`/accounts/${api.accountId}/pipelines`);
+    if (!response.ok) {
+        throw new CloudflareApiError(`Failed to list pipelines: ${response.statusText}`, response);
+    }
+    const data = (await response.json());
+    if (!data.success) {
+        const errorMessage = data.errors?.[0]?.message || "Unknown error";
+        throw new Error(`Failed to list pipelines: ${errorMessage}`);
+    }
+    // Transform API response
+    return (data.results || []).map((pipeline) => ({
+        name: pipeline.name,
+        id: pipeline.id,
+    }));
+}
+function convertBatchSettings(batch) {
+    const result = {};
+    if (batch?.maxMb !== undefined) {
+        // Convert MB to bytes
+        result.max_bytes = batch.maxMb * 1024 * 1024;
+    }
+    if (batch?.maxRows !== undefined) {
+        result.max_rows = batch.maxRows;
+    }
+    if (batch?.maxSeconds !== undefined) {
+        result.max_duration_s = batch.maxSeconds;
+    }
+    return result;
+}

package/lib/cloudflare/r2-rest-state-store.js

@@ -1,6 +1,7 @@
 import { deserialize, serialize } from "../serde";
 import { withExponentialBackoff } from "../util/retry";
 import { createCloudflareApi, } from "./api";
+import { CloudflareApiError, handleApiError } from "./api-error";
 /**
  * State store implementation using Cloudflare R2 API
  * Uses R2 for immediate consistency compared to KV's eventual consistency
@@ -69,17 +70,12 @@ export class R2RestStateStore {
             const response = await withExponentialBackoff(async () => {
                 const response = await this.api.get(listPath);
                 if (!response.ok) {
-                    const errorData = await response.json().catch(() => ({
-                        errors: [{ message: response.statusText }],
-                    }));
-                    throw new Error(`Error listing R2 objects: ${errorData.errors?.[0]?.message || response.statusText}`);
+                    await handleApiError(response, "list", "bucket", this.bucketName);
                 }
                 return response;
             }, 
             // Retry on transient errors
-            (error) => error.message?.includes("502") ||
-                error.message?.includes("503") ||
-                error.message?.includes("timeout"), 5, // 5 retry attempts
+            isRetryableError, 5, // 5 retry attempts
             1000 // Start with 1 second delay
             );
             const data = (await response.json());
@@ -117,21 +113,13 @@ export class R2RestStateStore {
         try {
             const response = await withExponentialBackoff(async () => {
                 const response = await this.api.get(`/accounts/${this.api.accountId}/r2/buckets/${this.bucketName}/objects/${this.getObjectKey(key)}`);
-                if (!response.ok) {
-                    if (response.status === 404) {
-                        return response;
-                    }
-                    const errorData = await response.json().catch(() => ({
-                        errors: [{ message: response.statusText }],
-                    }));
-                    throw new Error(`Error getting R2 object: ${errorData.errors?.[0]?.message || response.statusText}`);
+                if (!response.ok && response.status !== 404) {
+                    await handleApiError(response, "get", "object", key);
                 }
                 return response;
             }, 
             // Retry on transient errors
-            (error) => error.message?.includes("502") ||
-                error.message?.includes("503") ||
-                error.message?.includes("timeout"), 5, // 5 retry attempts
+            isRetryableError, 5, // 5 retry attempts
             1000 // Start with 1 second delay
             );
             if (response.status === 404) {
@@ -202,15 +190,12 @@ export class R2RestStateStore {
                 },
             });
             if (!response.ok) {
-                const errorData = await response.json().catch(() => ({
-                    errors: [{ message: response.statusText }],
-                }));
-                throw new Error(`Error writing to R2: ${errorData.errors?.[0]?.message || response.statusText}`);
+                await handleApiError(response, "put", "object", objectKey);
             }
             return response;
         }, 
         // Retry on transient errors
-        (error) => error.message?.includes("503") || error.message?.includes("timeout"), 5, // 5 retry attempts
+        isRetryableError, 5, // 5 retry attempts
         1000 // Start with 1 second delay
         );
     }
@@ -224,17 +209,10 @@ export class R2RestStateStore {
         await withExponentialBackoff(async () => {
             const response = await this.api.delete(`/accounts/${this.api.accountId}/r2/buckets/${this.bucketName}/objects/${this.getObjectKey(key)}`);
             if (!response.ok && response.status !== 404) {
-                const errorData = await response.json().catch(() => ({
-                    errors: [{ message: response.statusText }],
-                }));
-                throw new Error(`Error deleting from R2: ${errorData.errors?.[0]?.message || response.statusText}`);
+                await handleApiError(response, "delete", "object", key);
             }
             return response;
-        }, 
-        // Retry on transient errors
-        (error) => error.message?.includes("502") ||
-            error.message?.includes("503") ||
-            error.message?.includes("timeout"), 5, // 5 retry attempts
+        }, isRetryableError, 5, // 5 retry attempts
         1000 // Start with 1 second delay
         );
     }
@@ -275,3 +253,13 @@ export class R2RestStateStore {
         }
     }
 }
+function isRetryableError(error) {
+    if (error instanceof CloudflareApiError) {
+        return (error.status === 500 ||
+            error.status === 502 ||
+            error.status === 503 ||
+            error.message.includes("timeout") ||
+            error.message.includes("internal error"));
+    }
+    return false;
+}

package/lib/cloudflare/worker.d.ts

@@ -74,6 +74,15 @@ export interface WorkerProps<B extends Bindings = Bindings> extends CloudflareAp
      * Whether to adopt the Worker if it already exists when creating
      */
     adopt?: boolean;
+    /**
+     * The compatibility date for the worker
+     * @default "2024-09-09"
+     */
+    compatibilityDate?: string;
+    /**
+     * The compatibility flags for the worker
+     */
+    compatibilityFlags?: string[];
 }
 /**
  * Output returned after Worker creation/update

package/lib/cloudflare/worker.js

@@ -239,6 +239,8 @@ class NotFoundError extends Error {
 async function prepareWorkerMetadata(ctx, oldBindings, props, assetUploadResult) {
     // Prepare metadata with bindings
     const meta = {
+        compatibility_date: props.compatibilityDate ?? "2024-09-09",
+        compatibility_flags: props.compatibilityFlags,
         bindings: [],
         observability: {
             enabled: props.observability?.enabled !== false,
@@ -344,6 +346,13 @@ async function prepareWorkerMetadata(ctx, oldBindings, props, assetUploadResult)
                 queue_name: binding.name,
             });
         }
+        else if (binding.type === "pipeline") {
+            meta.bindings.push({
+                type: "pipelines",
+                name: bindingName,
+                pipeline: binding.name,
+            });
+        }
         else {
             // @ts-expect-error - we should never reach here
             throw new Error(`Unsupported binding type: ${binding.type}`);
@@ -422,6 +431,10 @@ async function bundleWorkerScript(props) {
         minify: true,
         options: {
             keepNames: true, // Important for Durable Object classes
+            loader: {
+                ".sql": "text",
+                ".json": "json",
+            },
         },
     };
     // Merge with user-provided options

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "alchemy",
-  "version": "0.7.1",
+  "version": "0.7.3",
   "type": "module",
   "module": "./lib/index.js",
   "scripts": {

package/src/cloudflare/bindings.ts

@@ -9,6 +9,7 @@ import type { R2Bucket } from "./bucket";
 import type { D1Database } from "./d1-database";
 import type { DurableObjectNamespace } from "./durable-object-namespace";
 import type { KVNamespace } from "./kv-namespace";
+import type { Pipeline } from "./pipeline";
 import type { Queue } from "./queue";
 import type { Worker } from "./worker";
 import type { Workflow } from "./workflow";
@@ -25,6 +26,7 @@ export type Binding =
   | D1Database
   | DurableObjectNamespace
   | KVNamespace
+  | Pipeline
   | Queue
   | R2Bucket
   | Secret
@@ -47,6 +49,7 @@ export type WorkerBindingSpec =
   | WorkerBindingJson
   | WorkerBindingKVNamespace
   | WorkerBindingMTLSCertificate
+  | WorkerBindingPipeline
   | WorkerBindingPlainText
   | WorkerBindingQueue
   | WorkerBindingR2Bucket
@@ -335,3 +338,15 @@ export interface WorkerBindingWorkflow {
    */
   script_name?: string;
 }
+
+/**
+ * Pipeline binding type
+ */
+export interface WorkerBindingPipeline {
+  /** The name of the binding */
+  name: string;
+  /** Type identifier for Pipeline binding */
+  type: "pipelines";
+  /** Pipeline name */
+  pipeline: string;
+}

package/src/cloudflare/bound.ts

@@ -1,3 +1,4 @@
+import type { Pipeline } from "cloudflare:pipelines";
 import type { Secret } from "../secret";
 import type { Assets } from "./assets";
 import type { Binding } from "./bindings";
@@ -5,6 +6,7 @@ import type { R2Bucket as _R2Bucket } from "./bucket";
 import type { D1Database as _D1Database } from "./d1-database";
 import type { DurableObjectNamespace as _DurableObjectNamespace } from "./durable-object-namespace";
 import type { KVNamespace as _KVNamespace } from "./kv-namespace";
+import type { Pipeline as _Pipeline } from "./pipeline";
 import type { Queue as _Queue } from "./queue";
 import type { Worker as _Worker } from "./worker";
 import type { Workflow as _Workflow } from "./workflow";
@@ -27,4 +29,6 @@ export type Bound<T extends Binding> = T extends _DurableObjectNamespace
                 ? D1Database
                 : T extends _Queue
                   ? Queue
-                  : Service;
+                  : T extends _Pipeline<infer R>
+                    ? Pipeline<R>
+                    : Service;

package/src/cloudflare/bucket.ts

@@ -415,6 +415,7 @@ export async function emptyBucket(
 
   try {
     do {
+      console.log(`Listing objects in bucket ${bucketName}`);
       // List objects in the bucket
       const { objects, continuationToken: nextToken } = await listObjects(
         r2,

package/src/cloudflare/d1-database.ts

@@ -49,6 +49,14 @@ export interface D1DatabaseProps extends CloudflareApiOptions {
    * @default true
    */
   delete?: boolean;
+
+  /**
+   * Whether to adopt an existing database with the same name if it exists
+   * If true and a database with the same name exists, it will be adopted rather than creating a new one
+   *
+   * @default false
+   */
+  adopt?: boolean;
 }
 
 /**
@@ -108,6 +116,16 @@ export interface D1Database
  *   primaryLocationHint: "wnam"
  * });
  *
+ * @example
+ * // Adopt an existing database if it already exists instead of failing
+ * const existingDb = await D1Database("existing-db", {
+ *   name: "existing-db",
+ *   adopt: true,
+ *   readReplication: {
+ *     mode: "auto"
+ *   }
+ * });
+ *
  * @see https://developers.cloudflare.com/d1/
  */
 export const D1Database = Resource(
@@ -135,7 +153,41 @@ export const D1Database = Resource(
 
       if (this.phase === "create") {
         console.log("Creating D1 database:", databaseName);
-        dbData = await createDatabase(api, databaseName, props);
+        try {
+          dbData = await createDatabase(api, databaseName, props);
+        } catch (error) {
+          // Check if this is a "database already exists" error and adopt is enabled
+          if (
+            props.adopt &&
+            error instanceof CloudflareApiError &&
+            error.message.includes("already exists")
+          ) {
+            console.log(`Database ${databaseName} already exists, adopting it`);
+            // Find the existing database by name
+            const databases = await listDatabases(api, databaseName);
+            const existingDb = databases.find((db) => db.name === databaseName);
+
+            if (!existingDb) {
+              throw new Error(
+                `Failed to find existing database '${databaseName}' for adoption`
+              );
+            }
+
+            // Get the database details using its ID
+            dbData = await getDatabase(api, existingDb.id);
+
+            // Update the database with the provided properties
+            if (props.readReplication) {
+              console.log(
+                `Updating adopted database ${databaseName} with new properties`
+              );
+              dbData = await updateDatabase(api, existingDb.id, props);
+            }
+          } else {
+            // Re-throw the error if adopt is false or it's not a "database already exists" error
+            throw error;
+          }
+        }
       } else {
         // Update operation
         if (this.output?.id) {
@@ -272,9 +324,15 @@ export async function deleteDatabase(
  * List all D1 databases in an account
  */
 export async function listDatabases(
-  api: CloudflareApi
+  api: CloudflareApi,
+  name?: string
 ): Promise<{ name: string; id: string }[]> {
-  const response = await api.get(`/accounts/${api.accountId}/d1/database`);
+  // Construct query string if name is provided
+  const queryParams = name ? `?name=${encodeURIComponent(name)}` : "";
+
+  const response = await api.get(
+    `/accounts/${api.accountId}/d1/database${queryParams}`
+  );
 
   if (!response.ok) {
     throw new CloudflareApiError(

package/src/cloudflare/index.ts

@@ -10,6 +10,7 @@ export * from "./dns-records";
 export * from "./durable-object-namespace";
 export * from "./kv-namespace";
 export * from "./permission-groups";
+export * from "./pipeline";
 export * from "./queue";
 export * from "./r2-rest-state-store";
 export * from "./worker";