alchemy 0.7.0 → 0.7.2

package/lib/cloudflare/pipeline.d.ts

@@ -0,0 +1,335 @@
+import type { Context } from "../context";
+import { Resource } from "../resource";
+import type { Secret } from "../secret";
+import { CloudflareApi, type CloudflareApiOptions } from "./api";
+/**
+ * Settings for compression of pipeline output
+ */
+export interface CompressionSettings {
+    /**
+     * Type of compression to use for pipeline output
+     * @default "gzip"
+     */
+    type: "gzip" | "none";
+}
+/**
+ * Settings for batching behavior of pipeline output
+ */
+export interface BatchSettings {
+    /**
+     * Maximum size of batch in megabytes before delivery (1-100 MB)
+     * @default 100
+     */
+    maxMb?: number;
+    /**
+     * Maximum number of rows in a batch before delivery (1-10,000,000 rows)
+     * @default 10000000
+     */
+    maxRows?: number;
+    /**
+     * Maximum duration of a batch in seconds before delivery (1-300 seconds)
+     * @default 300
+     */
+    maxSeconds?: number;
+}
+/**
+ * Configuration for a pipeline HTTP source
+ */
+export interface HttpSource {
+    /**
+     * Format of the source data
+     * @default "json"
+     */
+    format: "json";
+    /**
+     * Type of source
+     */
+    type: "http";
+    /**
+     * Whether authentication is required
+     * @default true
+     */
+    authentication?: boolean;
+    /**
+     * CORS configuration for HTTP endpoint source
+     */
+    cors?: {
+        /**
+         * Allowed origins for CORS requests
+         * @default ["*"]
+         */
+        origins: string[];
+    };
+}
+/**
+ * Configuration for a pipeline binding source
+ */
+export interface BindingSource {
+    /**
+     * Format of the source data
+     * @default "json"
+     */
+    format: "json";
+    /**
+     * Type of source
+     */
+    type: "binding";
+}
+/**
+ * Configuration for a pipeline source
+ */
+export type PipelineSource = HttpSource | BindingSource;
+/**
+ * Configuration for an R2 destination
+ */
+export interface R2DestinationConfig {
+    /**
+     * Type of destination (R2)
+     */
+    type: "r2";
+    /**
+     * Format of the output data
+     * @default "json"
+     */
+    format: "json" | "ndjson";
+    /**
+     * Path configuration for the R2 destination
+     */
+    path: {
+        /**
+         * R2 bucket name
+         */
+        bucket: string;
+        /**
+         * Optional prefix for files in the bucket
+         */
+        prefix?: string;
+        /**
+         * Optional filename pattern
+         * @default "${slug}${extension}"
+         */
+        filename?: string;
+        /**
+         * Optional filepath pattern
+         * @default "${date}/${hour}"
+         */
+        filepath?: string;
+    };
+    /**
+     * Compression settings
+     */
+    compression?: CompressionSettings;
+    /**
+     * Batch settings
+     */
+    batch?: BatchSettings;
+    /**
+     * Credentials for the R2 bucket
+     * Required for R2 destinations
+     */
+    credentials: {
+        /**
+         * Access key ID for the R2 bucket
+         */
+        accessKeyId: Secret;
+        /**
+         * Secret access key for the R2 bucket
+         */
+        secretAccessKey: Secret;
+        /**
+         * Endpoint for the R2 bucket
+         */
+        endpoint?: string;
+    };
+}
+/**
+ * Allowed destination types
+ */
+export type PipelineDestination = R2DestinationConfig;
+/**
+ * Properties for creating or updating a Pipeline
+ */
+export interface PipelineProps extends CloudflareApiOptions {
+    /**
+     * Name of the pipeline
+     */
+    name: string;
+    /**
+     * Source configuration for the pipeline
+     */
+    source: PipelineSource[];
+    /**
+     * Destination configuration for the pipeline
+     */
+    destination: PipelineDestination;
+    /**
+     * Compression settings for the pipeline
+     * @default { type: "gzip" }
+     */
+    compression?: CompressionSettings;
+    /**
+     * Whether to delete the pipeline.
+     * If set to false, the pipeline will remain but the resource will be removed from state
+     *
+     * @default true
+     */
+    delete?: boolean;
+}
+/**
+ * Base type for pipeline records
+ */
+export interface PipelineRecord {
+    [key: string]: any;
+}
+/**
+ * Output returned after Pipeline creation/update
+ */
+export interface Pipeline<T extends PipelineRecord = PipelineRecord> extends Resource<"cloudflare::Pipeline">, PipelineProps {
+    /**
+     * Type identifier for the Pipeline resource
+     */
+    type: "pipeline";
+    /**
+     * The unique ID of the pipeline
+     */
+    id: string;
+    /**
+     * HTTP endpoint URL for the pipeline
+     */
+    endpoint: string;
+    /**
+     * Version of the pipeline
+     */
+    version: number;
+}
+/**
+ * Creates and manages Cloudflare Pipelines.
+ *
+ * Pipelines provide a managed data pipeline service that lets you collect, transform,
+ * and route data to various destinations like R2 buckets.
+ *
+ * @example
+ * // Create a basic pipeline with an R2 bucket destination
+ * const bucket = await R2Bucket("logs-bucket", {
+ *   name: "logs-bucket"
+ * });
+ *
+ * const accessKey = alchemy.secret(process.env.R2_ACCESS_KEY_ID!);
+ * const secretKey = alchemy.secret(process.env.R2_SECRET_ACCESS_KEY!);
+ *
+ * const pipeline = await Pipeline("logs-pipeline", {
+ *   name: "logs-pipeline",
+ *   destination: {
+ *     type: "r2",
+ *     format: "json",
+ *     path: {
+ *       bucket: bucket.name,
+ *       prefix: "app-logs",
+ *     },
+ *     credentials: {
+ *       accessKeyId: accessKey,
+ *       secretAccessKey: secretKey
+ *     }
+ *   },
+ *   batch: {
+ *     maxMb: 50,
+ *     maxSeconds: 60
+ *   }
+ * });
+ *
+ * @example
+ * // Create a pipeline with custom source configuration
+ * const customPipeline = await Pipeline("custom-pipeline", {
+ *   name: "custom-pipeline",
+ *   source: [{
+ *     type: "http",
+ *     format: "json",
+ *     authentication: true,
+ *     cors: {
+ *       origins: ["https://example.com"]
+ *     }
+ *   }],
+ *   destination: {
+ *     type: "r2",
+ *     format: "json",
+ *     path: {
+ *       bucket: "my-bucket",
+ *       prefix: "data"
+ *     },
+ *     credentials: {
+ *       accessKeyId: alchemy.secret(process.env.R2_ACCESS_KEY_ID!),
+ *       secretAccessKey: alchemy.secret(process.env.R2_SECRET_ACCESS_KEY!)
+ *     },
+ *     compression: {
+ *       type: "gzip"
+ *     }
+ *   }
+ * });
+ *
+ * @see https://developers.cloudflare.com/pipelines/
+ */
+export declare const Pipeline: (((this: any, id: string, props?: {}) => never) & (new (_: never) => never)) | (<T extends PipelineRecord = PipelineRecord>(this: Context<Pipeline<T>>, id: string, props: PipelineProps) => Promise<Pipeline<T>>);
+interface CloudflarePipelineResponse {
+    result: {
+        id: string;
+        name: string;
+        endpoint: string;
+        version: number;
+        source: Array<{
+            type: "http" | "binding";
+            format: string;
+            authentication?: boolean;
+            cors?: {
+                origins: string[];
+            };
+        }>;
+        destination: {
+            type: string;
+            format: string;
+            path?: {
+                bucket: string;
+                prefix?: string;
+                filename?: string;
+                filepath?: string;
+            };
+            compression?: {
+                type: string;
+            };
+            batch: {
+                max_bytes?: number;
+                max_rows?: number;
+                max_duration_s?: number;
+            };
+        };
+    };
+    success: boolean;
+    errors: Array<{
+        code: number;
+        message: string;
+    }>;
+    messages: string[];
+}
+/**
+ * Get a pipeline
+ */
+export declare function getPipeline(api: CloudflareApi, pipelineName: string): Promise<CloudflarePipelineResponse>;
+/**
+ * Delete a pipeline
+ */
+export declare function deletePipeline(api: CloudflareApi, pipelineName: string): Promise<void>;
+/**
+ * Create a new pipeline
+ */
+export declare function createPipeline(api: CloudflareApi, pipelineName: string, props: PipelineProps): Promise<CloudflarePipelineResponse>;
+/**
+ * Update a pipeline
+ */
+export declare function updatePipeline(api: CloudflareApi, pipelineName: string, props: PipelineProps): Promise<CloudflarePipelineResponse>;
+/**
+ * List all pipelines in an account
+ */
+export declare function listPipelines(api: CloudflareApi): Promise<{
+    name: string;
+    id: string;
+}[]>;
+export {};

package/lib/cloudflare/pipeline.js

@@ -0,0 +1,250 @@
+import { Resource } from "../resource";
+import { CloudflareApi, createCloudflareApi, } from "./api";
+import { CloudflareApiError, handleApiError } from "./api-error";
+/**
+ * Creates and manages Cloudflare Pipelines.
+ *
+ * Pipelines provide a managed data pipeline service that lets you collect, transform,
+ * and route data to various destinations like R2 buckets.
+ *
+ * @example
+ * // Create a basic pipeline with an R2 bucket destination
+ * const bucket = await R2Bucket("logs-bucket", {
+ *   name: "logs-bucket"
+ * });
+ *
+ * const accessKey = alchemy.secret(process.env.R2_ACCESS_KEY_ID!);
+ * const secretKey = alchemy.secret(process.env.R2_SECRET_ACCESS_KEY!);
+ *
+ * const pipeline = await Pipeline("logs-pipeline", {
+ *   name: "logs-pipeline",
+ *   destination: {
+ *     type: "r2",
+ *     format: "json",
+ *     path: {
+ *       bucket: bucket.name,
+ *       prefix: "app-logs",
+ *     },
+ *     credentials: {
+ *       accessKeyId: accessKey,
+ *       secretAccessKey: secretKey
+ *     }
+ *   },
+ *   batch: {
+ *     maxMb: 50,
+ *     maxSeconds: 60
+ *   }
+ * });
+ *
+ * @example
+ * // Create a pipeline with custom source configuration
+ * const customPipeline = await Pipeline("custom-pipeline", {
+ *   name: "custom-pipeline",
+ *   source: [{
+ *     type: "http",
+ *     format: "json",
+ *     authentication: true,
+ *     cors: {
+ *       origins: ["https://example.com"]
+ *     }
+ *   }],
+ *   destination: {
+ *     type: "r2",
+ *     format: "json",
+ *     path: {
+ *       bucket: "my-bucket",
+ *       prefix: "data"
+ *     },
+ *     credentials: {
+ *       accessKeyId: alchemy.secret(process.env.R2_ACCESS_KEY_ID!),
+ *       secretAccessKey: alchemy.secret(process.env.R2_SECRET_ACCESS_KEY!)
+ *     },
+ *     compression: {
+ *       type: "gzip"
+ *     }
+ *   }
+ * });
+ *
+ * @see https://developers.cloudflare.com/pipelines/
+ */
+export const Pipeline = Resource("cloudflare::Pipeline", async function (id, props) {
+    const api = await createCloudflareApi(props);
+    const pipelineName = props.name || id;
+    if (this.phase === "delete") {
+        console.log("Deleting Cloudflare Pipeline:", pipelineName);
+        if (props.delete !== false) {
+            // Delete Pipeline
+            await deletePipeline(api, pipelineName);
+        }
+        // Return void (a deleted pipeline has no content)
+        return this.destroy();
+    }
+    else {
+        let pipelineData;
+        if (this.phase === "create") {
+            console.log("Creating Cloudflare Pipeline:", pipelineName);
+            pipelineData = await createPipeline(api, pipelineName, props);
+        }
+        else {
+            // Update operation
+            if (this.output?.id) {
+                console.log("Updating Cloudflare Pipeline:", pipelineName);
+                // Check if name is being changed, which is not allowed
+                if (props.name !== this.output.name) {
+                    throw new Error("Cannot update Pipeline name after creation. Pipeline name is immutable.");
+                }
+                // Update the pipeline with new settings
+                pipelineData = await updatePipeline(api, pipelineName, props);
+            }
+            else {
+                // If no ID exists, fall back to creating a new pipeline
+                console.log("No existing Pipeline ID found, creating new Cloudflare Pipeline:", pipelineName);
+                pipelineData = await createPipeline(api, pipelineName, props);
+            }
+        }
+        return this({
+            type: "pipeline",
+            id: pipelineData.result.id,
+            name: pipelineName,
+            endpoint: pipelineData.result.endpoint,
+            version: pipelineData.result.version,
+            source: pipelineData.result.source.map((s) => ({
+                type: s.type,
+                format: s.format,
+                authentication: s.authentication,
+                cors: s.cors,
+            })),
+            destination: props.destination, // Use the input destination, not the API response
+            compression: props.compression,
+            accountId: api.accountId,
+        });
+    }
+});
+/**
+ * Get a pipeline
+ */
+export async function getPipeline(api, pipelineName) {
+    const response = await api.get(`/accounts/${api.accountId}/pipelines/${pipelineName}`);
+    if (!response.ok) {
+        return await handleApiError(response, "getting", "Pipeline", pipelineName);
+    }
+    return (await response.json());
+}
+/**
+ * Delete a pipeline
+ */
+export async function deletePipeline(api, pipelineName) {
+    // Delete Pipeline
+    const deleteResponse = await api.delete(`/accounts/${api.accountId}/pipelines/${pipelineName}`);
+    if (!deleteResponse.ok && deleteResponse.status !== 404) {
+        const errorData = await deleteResponse.json().catch(() => ({
+            errors: [{ message: deleteResponse.statusText }],
+        }));
+        throw new CloudflareApiError(`Error deleting Cloudflare Pipeline '${pipelineName}': ${errorData.errors?.[0]?.message || deleteResponse.statusText}`, deleteResponse);
+    }
+}
+/**
+ * Create a new pipeline
+ */
+export async function createPipeline(api, pipelineName, props) {
+    // Prepare the create payload
+    const createPayload = preparePipelinePayload(api, pipelineName, props);
+    const createResponse = await api.post(`/accounts/${api.accountId}/pipelines`, createPayload);
+    if (!createResponse.ok) {
+        return await handleApiError(createResponse, "creating", "Pipeline", pipelineName);
+    }
+    return (await createResponse.json());
+}
+/**
+ * Update a pipeline
+ */
+export async function updatePipeline(api, pipelineName, props) {
+    // Get current pipeline to build update payload
+    const currentPipeline = await getPipeline(api, pipelineName);
+    // Prepare the update payload
+    const updatePayload = preparePipelinePayload(api, pipelineName, props, currentPipeline);
+    const updateResponse = await api.put(`/accounts/${api.accountId}/pipelines/${pipelineName}`, updatePayload);
+    if (!updateResponse.ok) {
+        return await handleApiError(updateResponse, "updating", "Pipeline", pipelineName);
+    }
+    return (await updateResponse.json());
+}
+/**
+ * Helper function to prepare pipeline payload for create/update operations
+ */
+function preparePipelinePayload(api, pipelineName, props, currentPipeline) {
+    // Prepare the payload with name and source
+    const payload = {
+        name: pipelineName,
+        source: props.source ||
+            currentPipeline?.result.source || [
+            {
+                type: "http",
+                format: "json",
+                authentication: true,
+                cors: { origins: ["*"] },
+            },
+        ],
+    };
+    // Handle destination
+    if (props.destination) {
+        payload.destination = { ...props.destination };
+        // Handle special formatting for R2 destination
+        const r2Dest = props.destination;
+        // Format credentials for API
+        if (payload.destination.credentials) {
+            payload.destination.credentials = {
+                access_key_id: r2Dest.credentials.accessKeyId.unencrypted,
+                secret_access_key: r2Dest.credentials.secretAccessKey.unencrypted,
+                endpoint: r2Dest.credentials.endpoint ??
+                    `https://${api.accountId}.r2.cloudflarestorage.com`,
+            };
+        }
+        // Format batch settings
+        payload.destination.batch = convertBatchSettings(payload.destination.batch);
+    }
+    else if (currentPipeline?.result.destination) {
+        payload.destination = currentPipeline.result.destination;
+    }
+    else if (!props.destination && !currentPipeline) {
+        throw new Error("An R2 destination is required for creating/updating a pipeline");
+    }
+    // Add compression if not specified
+    if (!payload.destination.compression) {
+        payload.destination.compression = { type: "gzip" };
+    }
+    return payload;
+}
+/**
+ * List all pipelines in an account
+ */
+export async function listPipelines(api) {
+    const response = await api.get(`/accounts/${api.accountId}/pipelines`);
+    if (!response.ok) {
+        throw new CloudflareApiError(`Failed to list pipelines: ${response.statusText}`, response);
+    }
+    const data = (await response.json());
+    if (!data.success) {
+        const errorMessage = data.errors?.[0]?.message || "Unknown error";
+        throw new Error(`Failed to list pipelines: ${errorMessage}`);
+    }
+    // Transform API response
+    return (data.results || []).map((pipeline) => ({
+        name: pipeline.name,
+        id: pipeline.id,
+    }));
+}
+function convertBatchSettings(batch) {
+    const result = {};
+    if (batch?.maxMb !== undefined) {
+        // Convert MB to bytes
+        result.max_bytes = batch.maxMb * 1024 * 1024;
+    }
+    if (batch?.maxRows !== undefined) {
+        result.max_rows = batch.maxRows;
+    }
+    if (batch?.maxSeconds !== undefined) {
+        result.max_duration_s = batch.maxSeconds;
+    }
+    return result;
+}