@donkeylabs/server 0.5.0 → 0.6.3

package/examples/external-jobs/shell/donkeylabs-job.sh

@@ -0,0 +1,264 @@
+#!/bin/bash
+#
+# Donkeylabs External Job Shell Wrapper
+#
+# This script provides functions for shell scripts to communicate
+# with the Donkeylabs job system via Unix sockets or TCP.
+#
+# Usage:
+#   #!/bin/bash
+#   source /path/to/donkeylabs-job.sh
+#
+#   # Initialize the job (reads from stdin)
+#   job_init
+#
+#   # Report progress
+#   job_progress 50 "Halfway done"
+#
+#   # Log messages
+#   job_log info "Processing data..."
+#
+#   # Complete the job
+#   job_complete '{"result": "success"}'
+#
+#   # Or fail the job
+#   job_fail "Something went wrong"
+#
+
+# Global variables (set by job_init)
+DONKEYLABS_JOB_ID=""
+DONKEYLABS_JOB_NAME=""
+DONKEYLABS_JOB_DATA=""
+DONKEYLABS_SOCKET_PATH=""
+DONKEYLABS_HEARTBEAT_PID=""
+
+# Get current timestamp in milliseconds
+_job_timestamp() {
+    # Try to use date with milliseconds, fall back to seconds * 1000
+    if date '+%s%3N' >/dev/null 2>&1; then
+        date '+%s%3N'
+    else
+        echo "$(($(date '+%s') * 1000))"
+    fi
+}
+
+# Send a message to the socket
+_job_send() {
+    local message="$1"
+
+    if [[ "$DONKEYLABS_SOCKET_PATH" == tcp://* ]]; then
+        # TCP connection
+        local addr="${DONKEYLABS_SOCKET_PATH#tcp://}"
+        local host="${addr%:*}"
+        local port="${addr##*:}"
+
+        # Use bash's /dev/tcp or nc
+        if [[ -e /dev/tcp ]]; then
+            echo "$message" > /dev/tcp/"$host"/"$port" 2>/dev/null
+        else
+            echo "$message" | nc -q0 "$host" "$port" 2>/dev/null || \
+            echo "$message" | nc -w0 "$host" "$port" 2>/dev/null
+        fi
+    else
+        # Unix socket
+        if command -v socat >/dev/null 2>&1; then
+            echo "$message" | socat - UNIX-CONNECT:"$DONKEYLABS_SOCKET_PATH" 2>/dev/null
+        elif command -v nc >/dev/null 2>&1; then
+            echo "$message" | nc -U "$DONKEYLABS_SOCKET_PATH" 2>/dev/null
+        else
+            echo "Error: Neither socat nor nc (netcat) found. Cannot send messages." >&2
+            return 1
+        fi
+    fi
+}
+
+# Build a JSON message
+_job_build_message() {
+    local type="$1"
+    local extra="$2"
+
+    local timestamp
+    timestamp=$(_job_timestamp)
+
+    local message="{\"type\":\"$type\",\"jobId\":\"$DONKEYLABS_JOB_ID\",\"timestamp\":$timestamp"
+
+    if [[ -n "$extra" ]]; then
+        message="$message,$extra"
+    fi
+
+    message="$message}"
+
+    echo "$message"
+}
+
+# Start the heartbeat background process
+_job_start_heartbeat() {
+    local interval="${1:-5}"
+
+    (
+        while true; do
+            sleep "$interval"
+            _job_send "$(_job_build_message "heartbeat")"
+        done
+    ) &
+
+    DONKEYLABS_HEARTBEAT_PID=$!
+}
+
+# Stop the heartbeat background process
+_job_stop_heartbeat() {
+    if [[ -n "$DONKEYLABS_HEARTBEAT_PID" ]]; then
+        kill "$DONKEYLABS_HEARTBEAT_PID" 2>/dev/null
+        wait "$DONKEYLABS_HEARTBEAT_PID" 2>/dev/null
+        DONKEYLABS_HEARTBEAT_PID=""
+    fi
+}
+
+# Initialize the job by reading payload from stdin
+job_init() {
+    local heartbeat_interval="${1:-5}"
+
+    # Read payload from stdin
+    local payload
+    read -r payload
+
+    if [[ -z "$payload" ]]; then
+        echo "Error: No payload received on stdin" >&2
+        exit 1
+    fi
+
+    # Parse JSON payload using jq if available, otherwise use basic grep/sed
+    if command -v jq >/dev/null 2>&1; then
+        DONKEYLABS_JOB_ID=$(echo "$payload" | jq -r '.jobId // empty')
+        DONKEYLABS_JOB_NAME=$(echo "$payload" | jq -r '.name // empty')
+        DONKEYLABS_JOB_DATA=$(echo "$payload" | jq -c '.data // {}')
+        DONKEYLABS_SOCKET_PATH=$(echo "$payload" | jq -r '.socketPath // empty')
+    else
+        # Basic parsing (less robust)
+        DONKEYLABS_JOB_ID=$(echo "$payload" | grep -o '"jobId":"[^"]*"' | cut -d'"' -f4)
+        DONKEYLABS_JOB_NAME=$(echo "$payload" | grep -o '"name":"[^"]*"' | cut -d'"' -f4)
+        DONKEYLABS_SOCKET_PATH=$(echo "$payload" | grep -o '"socketPath":"[^"]*"' | cut -d'"' -f4)
+        DONKEYLABS_JOB_DATA="{}"
+    fi
+
+    # Fall back to environment variables
+    DONKEYLABS_JOB_ID="${DONKEYLABS_JOB_ID:-$DONKEYLABS_JOB_ID}"
+    DONKEYLABS_SOCKET_PATH="${DONKEYLABS_SOCKET_PATH:-$DONKEYLABS_SOCKET_PATH}"
+
+    # If TCP port is set but not socket path, construct TCP URL
+    if [[ -z "$DONKEYLABS_SOCKET_PATH" && -n "$DONKEYLABS_TCP_PORT" ]]; then
+        DONKEYLABS_SOCKET_PATH="tcp://127.0.0.1:$DONKEYLABS_TCP_PORT"
+    fi
+
+    if [[ -z "$DONKEYLABS_JOB_ID" || -z "$DONKEYLABS_SOCKET_PATH" ]]; then
+        echo "Error: Missing jobId or socketPath" >&2
+        exit 1
+    fi
+
+    # Start heartbeat in background
+    _job_start_heartbeat "$heartbeat_interval"
+
+    # Send started message
+    _job_send "$(_job_build_message "started")"
+
+    # Set up cleanup trap
+    trap '_job_stop_heartbeat' EXIT
+}
+
+# Report progress
+# Usage: job_progress <percent> [message]
+job_progress() {
+    local percent="$1"
+    local message="${2:-}"
+
+    local extra="\"percent\":$percent"
+
+    if [[ -n "$message" ]]; then
+        # Escape message for JSON
+        message="${message//\\/\\\\}"
+        message="${message//\"/\\\"}"
+        message="${message//$'\n'/\\n}"
+        extra="$extra,\"message\":\"$message\""
+    fi
+
+    _job_send "$(_job_build_message "progress" "$extra")"
+}
+
+# Send a log message
+# Usage: job_log <level> <message>
+job_log() {
+    local level="$1"
+    local message="$2"
+
+    # Escape message for JSON
+    message="${message//\\/\\\\}"
+    message="${message//\"/\\\"}"
+    message="${message//$'\n'/\\n}"
+
+    _job_send "$(_job_build_message "log" "\"level\":\"$level\",\"message\":\"$message\"")"
+}
+
+# Convenience log functions
+job_debug() { job_log "debug" "$1"; }
+job_info() { job_log "info" "$1"; }
+job_warn() { job_log "warn" "$1"; }
+job_error() { job_log "error" "$1"; }
+
+# Complete the job
+# Usage: job_complete [result_json]
+job_complete() {
+    local result="${1:-null}"
+
+    _job_stop_heartbeat
+
+    if [[ "$result" == "null" || -z "$result" ]]; then
+        _job_send "$(_job_build_message "completed")"
+    else
+        _job_send "$(_job_build_message "completed" "\"result\":$result")"
+    fi
+}
+
+# Fail the job
+# Usage: job_fail <error_message>
+job_fail() {
+    local error="$1"
+
+    _job_stop_heartbeat
+
+    # Escape error for JSON
+    error="${error//\\/\\\\}"
+    error="${error//\"/\\\"}"
+    error="${error//$'\n'/\\n}"
+
+    _job_send "$(_job_build_message "failed" "\"error\":\"$error\"")"
+}
+
+# Get a value from job data (requires jq)
+# Usage: job_data_get <path>
+job_data_get() {
+    local path="$1"
+
+    if command -v jq >/dev/null 2>&1; then
+        echo "$DONKEYLABS_JOB_DATA" | jq -r "$path"
+    else
+        echo "Error: jq is required to parse job data" >&2
+        return 1
+    fi
+}
+
+# Example usage (only runs if script is executed directly)
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    echo "Donkeylabs Job Shell Wrapper"
+    echo ""
+    echo "Usage: source this file in your shell script"
+    echo ""
+    echo "Example:"
+    echo "  #!/bin/bash"
+    echo "  source donkeylabs-job.sh"
+    echo ""
+    echo "  job_init"
+    echo "  job_progress 0 \"Starting...\""
+    echo "  # Do work..."
+    echo "  job_progress 100 \"Done!\""
+    echo "  job_complete '{\"result\": \"success\"}'"
+fi

package/examples/external-jobs/shell/example-job.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+#
+# Example External Job Script
+#
+# This script demonstrates how to use the donkeylabs-job.sh wrapper
+# to create an external job that can be executed by the Donkeylabs server.
+#
+
+# Get the directory of this script
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Source the job wrapper
+source "$SCRIPT_DIR/donkeylabs-job.sh"
+
+# Initialize the job (reads payload from stdin, starts heartbeat)
+job_init 5  # 5 second heartbeat interval
+
+# Log that we're starting
+job_info "Starting example job"
+job_info "Job ID: $DONKEYLABS_JOB_ID"
+job_info "Job Name: $DONKEYLABS_JOB_NAME"
+
+# Get configuration from job data
+STEPS=$(job_data_get '.steps // 5')
+DELAY=$(job_data_get '.delay // 1')
+
+job_info "Processing $STEPS steps with ${DELAY}s delay"
+
+# Process each step
+for i in $(seq 1 "$STEPS"); do
+    # Calculate progress
+    PROGRESS=$(( (i - 1) * 100 / STEPS ))
+
+    # Report progress
+    job_progress "$PROGRESS" "Processing step $i of $STEPS"
+
+    # Simulate work
+    sleep "$DELAY"
+
+    job_debug "Completed step $i"
+done
+
+# Final progress
+job_progress 100 "All steps completed"
+
+# Complete the job with result
+job_complete "{\"processed\": true, \"steps\": $STEPS}"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@donkeylabs/server",
-  "version": "0.5.0",
+  "version": "0.6.3",
   "type": "module",
   "description": "Type-safe plugin system for building RPC-style APIs with Bun",
   "main": "./src/index.ts",
@@ -30,6 +30,7 @@
   "files": [
     "src",
     "docs",
+    "examples",
     "CLAUDE.md",
     "context.d.ts",
     "registry.d.ts",
@@ -74,7 +75,7 @@
   ],
   "repository": {
     "type": "git",
-    "url": "https://github.com/donkeylabs/server"
+    "url": "https://github.com/donkeylabs-io/donkeylabs"
   },
   "license": "MIT"
 }

package/src/client/base.ts

@@ -248,19 +248,21 @@ export class ApiClientBase<TEvents extends Record<string, any> = Record<string,
   }
 
   /**
-   * Make a raw request (for non-JSON endpoints)
+   * Make a raw request (for non-JSON endpoints like streaming)
    */
   protected async rawRequest(
     route: string,
-    init: RequestInit = {}
+    init?: RequestInit
   ): Promise<Response> {
     const fetchFn = this.options.fetch || fetch;
+    const requestInit = init ?? {};
 
     return fetchFn(`${this.baseUrl}/${route}`, {
-      ...init,
+      method: "POST",
+      ...requestInit,
       headers: {
         ...this.options.headers,
-        ...init.headers,
+        ...requestInit.headers,
       },
       credentials: this.options.credentials,
     });

package/src/core/external-job-socket.ts

@@ -34,12 +34,18 @@ export interface SocketServerOptions {
 export interface ExternalJobSocketServer {
   /** Create a new socket for a job (returns socket path or TCP port) */
   createSocket(jobId: string): Promise<{ socketPath?: string; tcpPort?: number }>;
-  /** Close a specific job's socket */
+  /** Close a specific job's socket and release reservations */
   closeSocket(jobId: string): Promise<void>;
   /** Get all active job connections */
   getActiveConnections(): string[];
   /** Attempt to reconnect to an existing socket */
   reconnect(jobId: string, socketPath?: string, tcpPort?: number): Promise<boolean>;
+  /** Reserve a socket path/port for an orphaned job (prevents reuse until released) */
+  reserve(jobId: string, socketPath?: string, tcpPort?: number): void;
+  /** Release reservation for a job (called when job is cleaned up) */
+  release(jobId: string): void;
+  /** Check if a socket path or port is reserved */
+  isReserved(socketPath?: string, tcpPort?: number): boolean;
   /** Shutdown all sockets and cleanup */
   shutdown(): Promise<void>;
   /** Clean orphaned socket files from a previous run */
@@ -68,6 +74,14 @@ export class ExternalJobSocketServerImpl implements ExternalJobSocketServer {
   private tcpPorts = new Map<string, number>();
   // Track used TCP ports
   private usedPorts = new Set<number>();
+  // Track reserved socket paths (for jobs that might reconnect)
+  private reservedSocketPaths = new Set<string>();
+  // Track reserved TCP ports (for jobs that might reconnect)
+  private reservedTcpPorts = new Set<number>();
+  // Map jobId -> reserved socket path (for release by jobId)
+  private jobReservedSocketPath = new Map<string, string>();
+  // Map jobId -> reserved TCP port (for release by jobId)
+  private jobReservedTcpPort = new Map<string, number>();
 
   private isWindows = process.platform === "win32";
 
@@ -96,6 +110,11 @@ export class ExternalJobSocketServerImpl implements ExternalJobSocketServer {
   private async createUnixServer(jobId: string): Promise<{ socketPath: string }> {
     const socketPath = join(this.socketDir, `job_${jobId}.sock`);
 
+    // Check if this socket path is reserved by another job
+    if (this.reservedSocketPaths.has(socketPath) && !this.jobReservedSocketPath.has(jobId)) {
+      throw new Error(`Socket path ${socketPath} is reserved by another job`);
+    }
+
     // Remove existing socket file if it exists
     if (existsSync(socketPath)) {
       await unlink(socketPath);
@@ -148,12 +167,14 @@ export class ExternalJobSocketServerImpl implements ExternalJobSocketServer {
     // Try random ports within range
     for (let i = 0; i < 100; i++) {
       const port = minPort + Math.floor(Math.random() * (maxPort - minPort));
-      if (!this.usedPorts.has(port)) {
-        // Check if port is actually available
-        const isAvailable = await this.checkPortAvailable(port);
-        if (isAvailable) {
-          return port;
-        }
+      // Skip if port is already in use or reserved by another job
+      if (this.usedPorts.has(port) || this.reservedTcpPorts.has(port)) {
+        continue;
+      }
+      // Check if port is actually available
+      const isAvailable = await this.checkPortAvailable(port);
+      if (isAvailable) {
+        return port;
       }
     }
 
@@ -241,12 +262,62 @@ export class ExternalJobSocketServerImpl implements ExternalJobSocketServer {
       this.usedPorts.delete(port);
       this.tcpPorts.delete(jobId);
     }
+
+    // Release any reservations for this job
+    this.release(jobId);
   }
 
   getActiveConnections(): string[] {
     return Array.from(this.clientSockets.keys());
   }
 
+  reserve(jobId: string, socketPath?: string, tcpPort?: number): void {
+    if (socketPath) {
+      this.reservedSocketPaths.add(socketPath);
+      this.jobReservedSocketPath.set(jobId, socketPath);
+    }
+    if (tcpPort) {
+      this.reservedTcpPorts.add(tcpPort);
+      this.jobReservedTcpPort.set(jobId, tcpPort);
+    }
+  }
+
+  release(jobId: string): void {
+    // Release socket path reservation
+    const socketPath = this.jobReservedSocketPath.get(jobId);
+    if (socketPath) {
+      this.reservedSocketPaths.delete(socketPath);
+      this.jobReservedSocketPath.delete(jobId);
+    }
+    // Also check socketPaths map (for active jobs)
+    const activeSocketPath = this.socketPaths.get(jobId);
+    if (activeSocketPath) {
+      this.reservedSocketPaths.delete(activeSocketPath);
+    }
+
+    // Release TCP port reservation
+    const tcpPort = this.jobReservedTcpPort.get(jobId);
+    if (tcpPort) {
+      this.reservedTcpPorts.delete(tcpPort);
+      this.jobReservedTcpPort.delete(jobId);
+    }
+    // Also check tcpPorts map (for active jobs)
+    const activeTcpPort = this.tcpPorts.get(jobId);
+    if (activeTcpPort) {
+      this.reservedTcpPorts.delete(activeTcpPort);
+    }
+  }
+
+  isReserved(socketPath?: string, tcpPort?: number): boolean {
+    if (socketPath && this.reservedSocketPaths.has(socketPath)) {
+      return true;
+    }
+    if (tcpPort && this.reservedTcpPorts.has(tcpPort)) {
+      return true;
+    }
+    return false;
+  }
+
   async reconnect(
     jobId: string,
     socketPath?: string,
@@ -257,19 +328,67 @@ export class ExternalJobSocketServerImpl implements ExternalJobSocketServer {
       return true;
     }
 
-    // For Unix sockets, check if the socket file exists
-    if (socketPath && existsSync(socketPath)) {
-      // The external process should still be connected to this socket
-      // We need to create a server to listen for the existing connection
-      // However, this is tricky because the original server is gone
-      // The reconnection strategy depends on the external process behavior
-
-      // For now, we'll just check if the socket exists
-      // The external process should attempt to reconnect if configured
-      return false;
+    // For Unix sockets, recreate the server on the same path
+    // The external process should be retrying to connect
+    if (socketPath && !this.isWindows) {
+      try {
+        // Remove old socket file if it exists
+        if (existsSync(socketPath)) {
+          await unlink(socketPath);
+        }
+
+        // Create new server on the same path
+        return new Promise((resolve) => {
+          const server = createNetServer((socket) => {
+            this.handleConnection(jobId, socket);
+          });
+
+          server.on("error", (err) => {
+            this.onError?.(err, jobId);
+            resolve(false);
+          });
+
+          server.listen(socketPath, () => {
+            this.servers.set(jobId, server);
+            this.socketPaths.set(jobId, socketPath);
+            console.log(`[SocketServer] Recreated socket for job ${jobId} at ${socketPath}`);
+            // Return true - the server is ready, external process should reconnect
+            resolve(true);
+          });
+        });
+      } catch (err) {
+        this.onError?.(err as Error, jobId);
+        return false;
+      }
+    }
+
+    // For TCP, recreate the server on the same port
+    if (tcpPort && this.isWindows) {
+      try {
+        return new Promise((resolve) => {
+          const server = createNetServer((socket) => {
+            this.handleConnection(jobId, socket);
+          });
+
+          server.on("error", (err) => {
+            this.onError?.(err, jobId);
+            resolve(false);
+          });
+
+          server.listen(tcpPort, "127.0.0.1", () => {
+            this.servers.set(jobId, server);
+            this.tcpPorts.set(jobId, tcpPort);
+            this.usedPorts.add(tcpPort);
+            console.log(`[SocketServer] Recreated TCP server for job ${jobId} on port ${tcpPort}`);
+            resolve(true);
+          });
+        });
+      } catch (err) {
+        this.onError?.(err as Error, jobId);
+        return false;
+      }
     }
 
-    // For TCP, we can't easily reconnect without the process knowing
     return false;
   }
 
@@ -319,9 +438,11 @@ export class ExternalJobSocketServerImpl implements ExternalJobSocketServer {
         const match = file.match(/^job_(.+)\.sock$/);
         if (match) {
           const jobId = match[1]!;
-          if (!activeJobIds.has(jobId)) {
-            // This socket file doesn't correspond to any active job
-            const socketPath = join(this.socketDir, file);
+          const socketPath = join(this.socketDir, file);
+
+          // Don't clean if job is active or socket path is reserved
+          if (!activeJobIds.has(jobId) && !this.reservedSocketPaths.has(socketPath)) {
+            // This socket file doesn't correspond to any active job and isn't reserved
             await unlink(socketPath).catch(() => {});
           }
         }

package/src/core/index.ts

@@ -47,6 +47,11 @@ export {
   createJobs,
 } from "./jobs";
 
+export {
+  SqliteJobAdapter,
+  type SqliteJobAdapterConfig,
+} from "./job-adapter-sqlite";
+
 export {
   type ExternalJobConfig,
   type ExternalJob,
@@ -141,3 +146,27 @@ export {
   workflow,
   createWorkflows,
 } from "./workflows";
+
+export {
+  type Processes,
+  type ProcessesConfig,
+  type ProcessStatus,
+  type ProcessConfig,
+  type ProcessDefinition,
+  type ManagedProcess,
+  type SpawnOptions,
+  createProcesses,
+} from "./processes";
+
+export {
+  SqliteProcessAdapter,
+  type SqliteProcessAdapterConfig,
+  type ProcessAdapter,
+} from "./process-adapter-sqlite";
+
+export {
+  type ProcessSocketServer,
+  type ProcessMessage,
+  type ProcessSocketConfig,
+  createProcessSocketServer,
+} from "./process-socket";