@cloudflare/sandbox 0.2.1 → 0.2.2

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @cloudflare/sandbox
 
+## 0.2.2
+
+### Patch Changes
+
+- [#51](https://github.com/cloudflare/sandbox-sdk/pull/51) [`4aceb32`](https://github.com/cloudflare/sandbox-sdk/commit/4aceb3215c836f59afcb88b2b325016b3f623f46) Thanks [@ghostwriternr](https://github.com/ghostwriternr)! - Handle intermittent interpreter failures and decouple jupyter startup
+
 ## 0.2.1
 
 ### Patch Changes

package/README.md

@@ -72,7 +72,7 @@ npm install @cloudflare/sandbox
 1. **Create a Dockerfile** (temporary requirement, will be removed in future releases):
 
 ```dockerfile
-FROM docker.io/cloudflare/sandbox:0.2.1
+FROM docker.io/cloudflare/sandbox:0.2.2
 
 # Expose the ports you want to expose
 EXPOSE 3000

package/container_src/circuit-breaker.ts

@@ -0,0 +1,121 @@
+/**
+ * Circuit Breaker implementation to prevent cascading failures
+ */
+export class CircuitBreaker {
+  private failures = 0;
+  private lastFailure: number = 0;
+  private successCount = 0;
+  private state: "closed" | "open" | "half-open" = "closed";
+
+  // Configuration
+  private readonly threshold: number;
+  private readonly timeout: number;
+  private readonly halfOpenSuccessThreshold: number;
+  private readonly name: string;
+
+  constructor(options: {
+    name: string;
+    threshold?: number;
+    timeout?: number;
+    halfOpenSuccessThreshold?: number;
+  }) {
+    this.name = options.name;
+    this.threshold = options.threshold || 5;
+    this.timeout = options.timeout || 30000; // 30 seconds
+    this.halfOpenSuccessThreshold = options.halfOpenSuccessThreshold || 3;
+  }
+
+  /**
+   * Execute an operation with circuit breaker protection
+   */
+  async execute<T>(operation: () => Promise<T>): Promise<T> {
+    // Check circuit state
+    if (this.state === "open") {
+      if (Date.now() - this.lastFailure > this.timeout) {
+        console.log(
+          `[CircuitBreaker ${this.name}] Transitioning from open to half-open`
+        );
+        this.state = "half-open";
+        this.successCount = 0;
+      } else {
+        throw new Error(
+          `Circuit breaker is open for ${this.name}. Retry after ${
+            this.timeout - (Date.now() - this.lastFailure)
+          }ms`
+        );
+      }
+    }
+
+    try {
+      const result = await operation();
+
+      // Record success
+      if (this.state === "half-open") {
+        this.successCount++;
+        if (this.successCount >= this.halfOpenSuccessThreshold) {
+          console.log(
+            `[CircuitBreaker ${this.name}] Transitioning from half-open to closed`
+          );
+          this.state = "closed";
+          this.failures = 0;
+        }
+      } else if (this.state === "closed") {
+        // Reset failure count on success
+        this.failures = 0;
+      }
+
+      return result;
+    } catch (error) {
+      this.recordFailure();
+      throw error;
+    }
+  }
+
+  /**
+   * Record a failure and update circuit state
+   */
+  private recordFailure() {
+    this.failures++;
+    this.lastFailure = Date.now();
+
+    if (this.state === "half-open") {
+      console.log(
+        `[CircuitBreaker ${this.name}] Failure in half-open state, transitioning to open`
+      );
+      this.state = "open";
+    } else if (this.failures >= this.threshold) {
+      console.log(
+        `[CircuitBreaker ${this.name}] Threshold reached (${this.failures}/${this.threshold}), transitioning to open`
+      );
+      this.state = "open";
+    }
+  }
+
+  /**
+   * Get current circuit breaker state
+   */
+  getState(): {
+    state: string;
+    failures: number;
+    lastFailure: number;
+    isOpen: boolean;
+  } {
+    return {
+      state: this.state,
+      failures: this.failures,
+      lastFailure: this.lastFailure,
+      isOpen: this.state === "open",
+    };
+  }
+
+  /**
+   * Reset the circuit breaker
+   */
+  reset() {
+    this.state = "closed";
+    this.failures = 0;
+    this.successCount = 0;
+    this.lastFailure = 0;
+    console.log(`[CircuitBreaker ${this.name}] Reset to closed state`);
+  }
+}

package/container_src/index.ts

@@ -1,6 +1,9 @@
 import { randomBytes } from "node:crypto";
 import { serve } from "bun";
-import { handleExecuteRequest, handleStreamingExecuteRequest } from "./handler/exec";
+import {
+  handleExecuteRequest,
+  handleStreamingExecuteRequest,
+} from "./handler/exec";
 import {
   handleDeleteFileRequest,
   handleMkdirRequest,
@@ -25,7 +28,8 @@ import {
   handleStartProcessRequest,
   handleStreamProcessLogsRequest,
 } from "./handler/process";
-import { type CreateContextRequest, JupyterServer } from "./jupyter-server";
+import type { CreateContextRequest } from "./jupyter-server";
+import { JupyterNotReadyError, JupyterService } from "./jupyter-service";
 import type { ProcessRecord, SessionData } from "./types";
 
 // In-memory session storage (in production, you'd want to use a proper database)
@@ -39,7 +43,7 @@ const processes = new Map<string, ProcessRecord>();
 
 // Generate a unique session ID using cryptographically secure randomness
 function generateSessionId(): string {
-  return `session_${Date.now()}_${randomBytes(6).toString('hex')}`;
+  return `session_${Date.now()}_${randomBytes(6).toString("hex")}`;
 }
 
 // Clean up old sessions (older than 1 hour)
@@ -56,25 +60,28 @@ function cleanupOldSessions() {
 // Run cleanup every 10 minutes
 setInterval(cleanupOldSessions, 10 * 60 * 1000);
 
-// Initialize Jupyter server
-const jupyterServer = new JupyterServer();
-let jupyterInitialized = false;
-
-// Initialize Jupyter immediately since startup.sh ensures it's ready
-(async () => {
-  try {
-    await jupyterServer.initialize();
-    jupyterInitialized = true;
-    console.log("[Container] Jupyter integration initialized successfully");
-  } catch (error) {
-    console.error("[Container] Failed to initialize Jupyter:", error);
-    // Log more details to help debug
-    if (error instanceof Error) {
-      console.error("[Container] Error details:", error.message);
-      console.error("[Container] Stack trace:", error.stack);
-    }
-  }
-})();
+// Initialize Jupyter service with graceful degradation
+const jupyterService = new JupyterService();
+
+// Start Jupyter initialization in background (non-blocking)
+console.log("[Container] Starting Jupyter initialization in background...");
+console.log(
+  "[Container] API endpoints are available immediately. Jupyter-dependent features will be available shortly."
+);
+
+jupyterService
+  .initialize()
+  .then(() => {
+    console.log(
+      "[Container] Jupyter fully initialized - all features available"
+    );
+  })
+  .catch((error) => {
+    console.error("[Container] Jupyter initialization failed:", error.message);
+    console.error(
+      "[Container] The API will continue in degraded mode without code execution capabilities"
+    );
+  });
 
 const server = serve({
   async fetch(req: Request) {
@@ -176,11 +183,17 @@ const server = serve({
 
         case "/api/ping":
           if (req.method === "GET") {
+            const health = await jupyterService.getHealthStatus();
             return new Response(
               JSON.stringify({
                 message: "pong",
                 timestamp: new Date().toISOString(),
-                jupyter: jupyterInitialized ? "ready" : "not ready",
+                jupyter: health.ready
+                  ? "ready"
+                  : health.initializing
+                  ? "initializing"
+                  : "not ready",
+                jupyterHealth: health,
               }),
               {
                 headers: {
@@ -305,30 +318,16 @@ const server = serve({
         // Code interpreter endpoints
         case "/api/contexts":
           if (req.method === "POST") {
-            if (!jupyterInitialized) {
-              return new Response(
-                JSON.stringify({
-                  error: "Jupyter server is not ready. Please try again in a moment."
-                }),
-                {
-                  status: 503,
-                  headers: {
-                    "Content-Type": "application/json",
-                    ...corsHeaders,
-                  },
-                }
-              );
-            }
             try {
-              const body = await req.json() as CreateContextRequest;
-              const context = await jupyterServer.createContext(body);
+              const body = (await req.json()) as CreateContextRequest;
+              const context = await jupyterService.createContext(body);
               return new Response(
                 JSON.stringify({
                   id: context.id,
                   language: context.language,
                   cwd: context.cwd,
                   createdAt: context.createdAt,
-                  lastUsed: context.lastUsed
+                  lastUsed: context.lastUsed,
                 }),
                 {
                   headers: {
@@ -338,10 +337,63 @@ const server = serve({
                 }
               );
             } catch (error) {
+              if (error instanceof JupyterNotReadyError) {
+                // This happens when request times out waiting for Jupyter
+                console.log(
+                  `[Container] Request timed out waiting for Jupyter (${error.progress}% complete)`
+                );
+                return new Response(
+                  JSON.stringify({
+                    error: error.message,
+                    status: "initializing",
+                    progress: error.progress,
+                  }),
+                  {
+                    status: 503,
+                    headers: {
+                      "Content-Type": "application/json",
+                      "Retry-After": String(error.retryAfter),
+                      ...corsHeaders,
+                    },
+                  }
+                );
+              }
+
+              // Check if it's a circuit breaker error
+              if (
+                error instanceof Error &&
+                error.message.includes("Circuit breaker is open")
+              ) {
+                console.log(
+                  "[Container] Circuit breaker is open:",
+                  error.message
+                );
+                return new Response(
+                  JSON.stringify({
+                    error:
+                      "Service temporarily unavailable due to high error rate. Please try again later.",
+                    status: "circuit_open",
+                    details: error.message,
+                  }),
+                  {
+                    status: 503,
+                    headers: {
+                      "Content-Type": "application/json",
+                      "Retry-After": "60",
+                      ...corsHeaders,
+                    },
+                  }
+                );
+              }
+
+              // Only log actual errors with stack traces
               console.error("[Container] Error creating context:", error);
               return new Response(
                 JSON.stringify({
-                  error: error instanceof Error ? error.message : "Failed to create context"
+                  error:
+                    error instanceof Error
+                      ? error.message
+                      : "Failed to create context",
                 }),
                 {
                   status: 500,
@@ -353,54 +405,75 @@ const server = serve({
               );
             }
           } else if (req.method === "GET") {
-            if (!jupyterInitialized) {
-              return new Response(
-                JSON.stringify({ contexts: [] }),
-                {
-                  headers: {
-                    "Content-Type": "application/json",
-                    ...corsHeaders,
-                  },
-                }
-              );
-            }
-            const contexts = await jupyterServer.listContexts();
-            return new Response(
-              JSON.stringify({ contexts }),
-              {
-                headers: {
-                  "Content-Type": "application/json",
-                  ...corsHeaders,
-                },
-              }
-            );
+            const contexts = await jupyterService.listContexts();
+            return new Response(JSON.stringify({ contexts }), {
+              headers: {
+                "Content-Type": "application/json",
+                ...corsHeaders,
+              },
+            });
           }
           break;
 
         case "/api/execute/code":
           if (req.method === "POST") {
-            if (!jupyterInitialized) {
-              return new Response(
-                JSON.stringify({
-                  error: "Jupyter server is not ready. Please try again in a moment."
-                }),
-                {
-                  status: 503,
-                  headers: {
-                    "Content-Type": "application/json",
-                    ...corsHeaders,
-                  },
-                }
-              );
-            }
             try {
-              const body = await req.json() as { context_id: string; code: string; language?: string };
-              return await jupyterServer.executeCode(body.context_id, body.code, body.language);
+              const body = (await req.json()) as {
+                context_id: string;
+                code: string;
+                language?: string;
+              };
+              return await jupyterService.executeCode(
+                body.context_id,
+                body.code,
+                body.language
+              );
             } catch (error) {
-              console.error("[Container] Error executing code:", error);
+              // Check if it's a circuit breaker error
+              if (
+                error instanceof Error &&
+                error.message.includes("Circuit breaker is open")
+              ) {
+                console.log(
+                  "[Container] Circuit breaker is open for code execution:",
+                  error.message
+                );
+                return new Response(
+                  JSON.stringify({
+                    error:
+                      "Service temporarily unavailable due to high error rate. Please try again later.",
+                    status: "circuit_open",
+                    details: error.message,
+                  }),
+                  {
+                    status: 503,
+                    headers: {
+                      "Content-Type": "application/json",
+                      "Retry-After": "30",
+                      ...corsHeaders,
+                    },
+                  }
+                );
+              }
+
+              // Don't log stack traces for expected initialization state
+              if (
+                error instanceof Error &&
+                error.message.includes("initializing")
+              ) {
+                console.log(
+                  "[Container] Code execution deferred - Jupyter still initializing"
+                );
+              } else {
+                console.error("[Container] Error executing code:", error);
+              }
+              // Error response is already handled by jupyterService.executeCode for not ready state
               return new Response(
                 JSON.stringify({
-                  error: error instanceof Error ? error.message : "Failed to execute code"
+                  error:
+                    error instanceof Error
+                      ? error.message
+                      : "Failed to execute code",
                 }),
                 {
                   status: 500,
@@ -416,27 +489,54 @@ const server = serve({
 
         default:
           // Handle dynamic routes for contexts
-          if (pathname.startsWith("/api/contexts/") && pathname.split('/').length === 4) {
-            const contextId = pathname.split('/')[3];
+          if (
+            pathname.startsWith("/api/contexts/") &&
+            pathname.split("/").length === 4
+          ) {
+            const contextId = pathname.split("/")[3];
             if (req.method === "DELETE") {
               try {
-                await jupyterServer.deleteContext(contextId);
-                return new Response(
-                  JSON.stringify({ success: true }),
-                  {
-                    headers: {
-                      "Content-Type": "application/json",
-                      ...corsHeaders,
-                    },
-                  }
-                );
+                await jupyterService.deleteContext(contextId);
+                return new Response(JSON.stringify({ success: true }), {
+                  headers: {
+                    "Content-Type": "application/json",
+                    ...corsHeaders,
+                  },
+                });
               } catch (error) {
+                if (error instanceof JupyterNotReadyError) {
+                  console.log(
+                    `[Container] Request timed out waiting for Jupyter (${error.progress}% complete)`
+                  );
+                  return new Response(
+                    JSON.stringify({
+                      error: error.message,
+                      status: "initializing",
+                      progress: error.progress,
+                    }),
+                    {
+                      status: 503,
+                      headers: {
+                        "Content-Type": "application/json",
+                        "Retry-After": "5",
+                        ...corsHeaders,
+                      },
+                    }
+                  );
+                }
                 return new Response(
                   JSON.stringify({
-                    error: error instanceof Error ? error.message : "Failed to delete context"
+                    error:
+                      error instanceof Error
+                        ? error.message
+                        : "Failed to delete context",
                   }),
                   {
-                    status: error instanceof Error && error.message.includes("not found") ? 404 : 500,
+                    status:
+                      error instanceof Error &&
+                      error.message.includes("not found")
+                        ? 404
+                        : 500,
                     headers: {
                       "Content-Type": "application/json",
                       ...corsHeaders,
@@ -446,22 +546,42 @@ const server = serve({
               }
             }
           }
-          
+
           // Handle dynamic routes for individual processes
           if (pathname.startsWith("/api/process/")) {
-            const segments = pathname.split('/');
+            const segments = pathname.split("/");
             if (segments.length >= 4) {
               const processId = segments[3];
               const action = segments[4]; // Optional: logs, stream, etc.
 
               if (!action && req.method === "GET") {
-                return handleGetProcessRequest(processes, req, corsHeaders, processId);
+                return handleGetProcessRequest(
+                  processes,
+                  req,
+                  corsHeaders,
+                  processId
+                );
               } else if (!action && req.method === "DELETE") {
-                return handleKillProcessRequest(processes, req, corsHeaders, processId);
+                return handleKillProcessRequest(
+                  processes,
+                  req,
+                  corsHeaders,
+                  processId
+                );
               } else if (action === "logs" && req.method === "GET") {
-                return handleGetProcessLogsRequest(processes, req, corsHeaders, processId);
+                return handleGetProcessLogsRequest(
+                  processes,
+                  req,
+                  corsHeaders,
+                  processId
+                );
               } else if (action === "stream" && req.method === "GET") {
-                return handleStreamProcessLogsRequest(processes, req, corsHeaders, processId);
+                return handleStreamProcessLogsRequest(
+                  processes,
+                  req,
+                  corsHeaders,
+                  processId
+                );
               }
             }
           }
@@ -477,7 +597,10 @@ const server = serve({
           });
       }
     } catch (error) {
-      console.error(`[Container] Error handling ${req.method} ${pathname}:`, error);
+      console.error(
+        `[Container] Error handling ${req.method} ${pathname}:`,
+        error
+      );
       return new Response(
         JSON.stringify({
           error: "Internal server error",
@@ -496,7 +619,7 @@ const server = serve({
   hostname: "0.0.0.0",
   port: 3000,
   // We don't need this, but typescript complains
-  websocket: { async message() { } },
+  websocket: { async message() {} },
 });
 
 console.log(`🚀 Bun server running on http://0.0.0.0:${server.port}`);
@@ -526,6 +649,8 @@ console.log(`   GET  /proxy/{port}/* - Proxy requests to exposed ports`);
 console.log(`   POST /api/contexts - Create a code execution context`);
 console.log(`   GET  /api/contexts - List all contexts`);
 console.log(`   DELETE /api/contexts/{id} - Delete a context`);
-console.log(`   POST /api/execute/code - Execute code in a context (streaming)`);
+console.log(
+  `   POST /api/execute/code - Execute code in a context (streaming)`
+);
 console.log(`   GET  /api/ping - Health check`);
 console.log(`   GET  /api/commands - List available commands`);