4runr-os 2.9.32 → 2.9.34

package/apps/gateway/.dockerignore

@@ -0,0 +1,11 @@
+# Exclude client-side packages that aren't needed on server
+packages/os-cli
+packages/cli
+packages/cli-tool
+packages/cli-tool-standalone
+
+# Exclude other client-side files
+4runr-os-enhanced.ts
+4runr-os.ts
+run-os.ps1
+

package/apps/gateway/.eslintrc.json

@@ -0,0 +1,28 @@
+{
+  "root": true,
+  "parser": "@typescript-eslint/parser",
+  "parserOptions": {
+    "ecmaVersion": 2022,
+    "sourceType": "module"
+  },
+  "plugins": ["@typescript-eslint"],
+  "extends": [
+    "eslint:recommended",
+    "plugin:@typescript-eslint/recommended"
+  ],
+  "rules": {
+    "@typescript-eslint/no-explicit-any": "off",
+    "@typescript-eslint/no-unused-vars": "warn",
+    "@typescript-eslint/no-non-null-assertion": "off"
+  },
+  "env": {
+    "node": true,
+    "es2022": true
+  },
+  "ignorePatterns": [
+    "**/__tests__/**",
+    "**/*.test.ts",
+    "dist/**"
+  ]
+}
+

package/apps/gateway/Dockerfile

@@ -0,0 +1,122 @@
+FROM node:20-slim AS base
+WORKDIR /workspace
+RUN apt-get update && apt-get install -y ca-certificates curl && rm -rf /var/lib/apt/lists/*
+
+FROM base AS deps
+RUN apt-get update && apt-get install -y python3 make g++ openssl libssl-dev && rm -rf /var/lib/apt/lists/*
+COPY pnpm-workspace.yaml package.json tsconfig.json ./
+# Copy .npmrc if it exists (optional - create empty if missing)
+COPY apps/gateway ./apps/gateway
+# Copy packages directory but we'll remove client-side ones immediately
+COPY packages ./packages
+# Remove client-side packages BEFORE pnpm reads the workspace (os-cli depends on unpublished @4runr/devkit)
+RUN rm -rf packages/os-cli packages/cli packages/cli-tool packages/cli-tool-standalone packages/devkit || true
+COPY prisma ./prisma
+COPY src ./src
+# Create .npmrc if it doesn't exist (pnpm will work without it)
+RUN touch .npmrc || true
+RUN corepack enable && corepack prepare pnpm@9.12.1 --activate
+# Set node-linker to hoisted for flat node_modules structure
+RUN pnpm config set node-linker hoisted
+# Skip Prisma generation during install (we'll generate it in build if needed)
+ENV PRISMA_SKIP_POSTINSTALL_GENERATE=1
+RUN pnpm install --filter @4runr/shared --filter @4runr/sentinel --filter @4runr/gateway --workspace-root
+
+FROM deps AS build
+# Clean shared and sentinel package dists to ensure fresh build with updated types
+RUN rm -rf packages/shared/dist packages/shared/.tsbuildinfo packages/sentinel/dist packages/sentinel/.tsbuildinfo
+RUN pnpm --filter @4runr/shared run build
+RUN pnpm --filter @4runr/sentinel run build
+# Generate Prisma client before building gateway
+# Use dummy DATABASE_URL for generation (just needs schema, not actual connection)
+# Prisma schema is at /workspace/prisma/schema.prisma
+RUN cd /workspace/apps/gateway && \
+    export DATABASE_URL="postgresql://dummy:dummy@localhost:5432/dummy" && \
+    pnpm db:generate && \
+    (test -d node_modules/.prisma/client || test -d ../../node_modules/.prisma/client) && \
+    echo "✓ Prisma client generated" || echo "✗ Prisma client generation failed"
+# Compile root src/crypto folder (used by gateway routes)
+# Create a temporary tsconfig for compiling just the crypto folder
+RUN mkdir -p dist/src/crypto && \
+    echo '{"compilerOptions":{"target":"ES2022","module":"ES2022","moduleResolution":"bundler","esModuleInterop":true,"skipLibCheck":true,"outDir":"dist/src/crypto","rootDir":"src/crypto"},"include":["src/crypto/envelope.ts"]}' > /tmp/crypto-tsconfig.json && \
+    npx tsc --project /tmp/crypto-tsconfig.json 2>&1 | head -10 && \
+    test -f dist/src/crypto/envelope.js && echo "✓ Compiled envelope.js" || echo "⚠ envelope.js compilation may have failed"
+
+RUN rm -rf apps/gateway/dist && \
+    pnpm --filter @4runr/gateway run build && \
+    (test -f apps/gateway/dist/index.js || test -f apps/gateway/dist/apps/gateway/src/index.js) && \
+    echo "✓ Gateway build completed successfully" || \
+    (echo "✗ Build failed: index.js not found" && exit 1)
+
+FROM base AS runner
+WORKDIR /workspace
+RUN apt-get update && apt-get install -y openssl ca-certificates curl && rm -rf /var/lib/apt/lists/*
+# Copy package files, config, and built artifacts
+COPY --from=build /workspace/package.json ./package.json
+COPY --from=build /workspace/pnpm-workspace.yaml ./pnpm-workspace.yaml
+# .npmrc is optional - create empty if it doesn't exist
+RUN touch .npmrc || true
+COPY --from=build /workspace/packages ./packages
+COPY --from=build /workspace/apps/gateway/package.json ./apps/gateway/package.json
+COPY --from=build /workspace/apps/gateway/dist ./apps/gateway/dist
+# Copy public directory (static files like dashboard)
+COPY --from=deps /workspace/apps/gateway/public ./apps/gateway/public
+# Copy node_modules from build stage (hoisted node-linker creates flat structure)
+# Use --chown to set ownership during copy to avoid breaking symlinks
+COPY --from=build --chown=node:node /workspace/node_modules ./node_modules
+# Copy Prisma schema (needed for runtime client generation if needed)
+COPY --from=build /workspace/prisma ./prisma
+# Copy src directory (contains crypto and memory-db for AI providers)
+COPY --from=build /workspace/src ./src
+# Copy compiled crypto (if it was compiled)
+RUN mkdir -p dist/src && (cp -r /workspace/dist/src/* dist/src/ 2>/dev/null || echo "Note: No compiled src found")
+# Verify Prisma client was copied (could be in root node_modules with hoisted linker)
+RUN (test -d node_modules/.prisma/client || test -d apps/gateway/node_modules/.prisma/client) && \
+    echo "✓ Prisma client found" || \
+    echo "⚠ Prisma client not found - checking..." && \
+    find node_modules -name ".prisma" -type d 2>/dev/null | head -5 || \
+    echo "⚠ Will generate Prisma client at runtime if needed"
+# Verify shared package is accessible
+RUN test -d packages/shared/dist && echo "✓ shared/dist exists" || echo "✗ shared/dist missing"
+RUN test -f packages/shared/dist/index.js && echo "✓ shared/dist/index.js exists" || echo "✗ shared/dist/index.js missing"
+# Check if @4runr/shared exists and if it's a symlink
+# If not, create it (symlinks don't need special ownership)
+RUN if [ ! -L node_modules/@4runr/shared ] && [ ! -d node_modules/@4runr/shared ]; then \
+        echo "✗ @4runr/shared NOT found in node_modules, creating symlink..."; \
+        mkdir -p node_modules/@4runr && \
+        ln -s ../../packages/shared node_modules/@4runr/shared && \
+        echo "✓ Created symlink: node_modules/@4runr/shared -> ../../packages/shared"; \
+    fi
+# Verify the shared package's package.json exists (for ESM resolution)
+RUN test -f node_modules/@4runr/shared/package.json && echo "✓ @4runr/shared/package.json exists" || echo "✗ @4runr/shared/package.json missing"
+# Verify the package.json has the correct exports field for ESM resolution
+RUN node -e "try { const pkg = require('./node_modules/@4runr/shared/package.json'); if (pkg.exports && pkg.exports['.']) { console.log('✓ package.json exports field configured correctly'); } else { console.error('✗ package.json missing exports field'); process.exit(1); } } catch(e) { console.error('✗ Error reading shared package.json:', e.message); process.exit(1); }"
+# Verify Sentinel package is accessible
+RUN test -d packages/sentinel/dist && echo "✓ sentinel/dist exists" || echo "✗ sentinel/dist missing"
+RUN test -f packages/sentinel/dist/index.js && echo "✓ sentinel/dist/index.js exists" || echo "✗ sentinel/dist/index.js missing"
+# Check if @4runr/sentinel exists and if it's a symlink
+# If not, create it (symlinks don't need special ownership)
+RUN if [ ! -L node_modules/@4runr/sentinel ] && [ ! -d node_modules/@4runr/sentinel ]; then \
+        echo "✗ @4runr/sentinel NOT found in node_modules, creating symlink..."; \
+        mkdir -p node_modules/@4runr && \
+        ln -s ../../packages/sentinel node_modules/@4runr/sentinel && \
+        echo "✓ Created symlink: node_modules/@4runr/sentinel -> ../../packages/sentinel"; \
+    fi
+# Verify the Sentinel package's package.json exists (for ESM resolution)
+RUN test -f node_modules/@4runr/sentinel/package.json && echo "✓ @4runr/sentinel/package.json exists" || echo "✗ @4runr/sentinel/package.json missing"
+# Verify the package.json has the correct exports field for ESM resolution
+RUN node -e "try { const pkg = require('./node_modules/@4runr/sentinel/package.json'); if (pkg.exports && pkg.exports['.']) { console.log('✓ Sentinel package.json exports field configured correctly'); } else { console.error('✗ Sentinel package.json missing exports field'); process.exit(1); } } catch(e) { console.error('✗ Error reading Sentinel package.json:', e.message); process.exit(1); }"
+# Verify fastify exists (should be directly in node_modules with hoisted node-linker)
+RUN test -d node_modules/fastify && echo "✓ fastify found in node_modules" || \
+    (echo "✗ fastify NOT found!" && ls -la node_modules/ | head -40)
+RUN test -f node_modules/fastify/package.json && echo "✓ fastify package.json exists" || echo "✗ fastify package.json missing"
+# Test Node.js ESM resolution from the gateway dist directory
+RUN cd apps/gateway/dist && \
+    node --input-type=module -e "import('fastify').then(() => console.log('✓ Node.js can resolve fastify from dist')).catch(e => console.error('✗ Cannot resolve:', e.message))" || \
+    echo "Testing from workspace root:" && \
+    cd /workspace && \
+    node --input-type=module -e "import('fastify').then(() => console.log('✓ Node.js can resolve fastify from root')).catch(e => console.error('✗ Cannot resolve from root:', e.message))"
+USER node
+WORKDIR /workspace
+# Run from workspace root - Node.js ESM should find node_modules at /workspace/node_modules
+CMD ["node", "apps/gateway/dist/apps/gateway/src/index.js"]

package/apps/gateway/Dockerfile.bak

@@ -0,0 +1,41 @@
+FROM node:18-alpine AS base
+
+# Install dependencies only when needed
+FROM base AS deps
+RUN apk add --no-cache libc6-compat
+WORKDIR /app
+
+# Install dependencies based on the preferred package manager
+COPY package.json ./
+RUN npm install
+
+# Rebuild the source code only when needed
+FROM base AS builder
+WORKDIR /app
+COPY --from=deps /app/node_modules ./node_modules
+COPY . .
+
+# Build the application
+RUN npm run build
+
+# Production image, copy all the files and run the app
+FROM base AS runner
+WORKDIR /app
+
+ENV NODE_ENV=production
+
+RUN addgroup --system --gid 1001 nodejs
+RUN adduser --system --uid 1001 4runr
+
+COPY --from=builder /app/dist ./dist
+COPY --from=builder /app/node_modules ./node_modules
+COPY --from=builder /app/package.json ./package.json
+
+USER 4runr
+
+EXPOSE 3000
+
+ENV PORT=3000
+ENV HOST=0.0.0.0
+
+CMD ["node", "dist/index.js"]

package/apps/gateway/create-test-script.sh

@@ -0,0 +1,386 @@
+#!/bin/bash
+# Script to create the clean test-all-phases.sh file on the server
+
+cat > /opt/4runr/gateway/test-all-phases.sh << 'EOFTEST'
+#!/bin/bash
+# Comprehensive test suite for all phases
+# Tests Phase 1-6 features end-to-end
+
+set +e  # Don't exit on errors, we want to see all test results
+
+GATEWAY_URL="${TEST_SERVER_URL:-http://localhost:3000}"
+API_KEY="${TEST_API_KEY:-test-key-ce67dabb0d4e27e3d72877c921b89cae}"
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+# Test counters
+TOTAL_TESTS=0
+PASSED_TESTS=0
+FAILED_TESTS=0
+
+# Test result tracking
+FAILED_TEST_NAMES=()
+
+# Generate a valid UUID v4
+generate_uuid() {
+    if command -v uuidgen > /dev/null 2>&1; then
+        uuidgen
+    else
+        cat /proc/sys/kernel/random/uuid 2>/dev/null || \
+        echo "$(od -x /dev/urandom | head -1 | awk '{OFS="-"; print $2$3,$4,$5,$6,$7$8$9}' | tr '[:upper:]' '[:lower:]')"
+    fi
+}
+
+# Helper function to run a test
+run_test() {
+    local test_name="$1"
+    local test_command="$2"
+    
+    TOTAL_TESTS=$((TOTAL_TESTS + 1))
+    echo -n "Testing: $test_name... "
+    
+    if eval "$test_command" > /tmp/test_output_$$.log 2>&1; then
+        echo -e "${GREEN}✓ PASSED${NC}"
+        PASSED_TESTS=$((PASSED_TESTS + 1))
+        return 0
+    else
+        echo -e "${RED}✗ FAILED${NC}"
+        FAILED_TESTS=$((FAILED_TESTS + 1))
+        FAILED_TEST_NAMES+=("$test_name")
+        echo "  Error output:"
+        cat /tmp/test_output_$$.log | sed 's/^/    /'
+        return 1
+    fi
+}
+
+echo "========================================="
+echo "4Runr Gateway - Comprehensive Phase Test"
+echo "========================================="
+echo ""
+echo "Gateway URL: $GATEWAY_URL"
+echo "API Key: ${API_KEY:0:20}..."
+echo ""
+
+# Wait for server to be ready
+echo "Waiting for server to be ready..."
+for i in {1..30}; do
+    if curl -sf "$GATEWAY_URL/health" > /dev/null 2>&1; then
+        echo "✓ Server is ready"
+        break
+    fi
+    if [ $i -eq 30 ]; then
+        echo "❌ Server is not ready after 30 seconds"
+        exit 1
+    fi
+    sleep 1
+done
+
+echo ""
+echo "========================================="
+echo "Phase 1-3: Core Features"
+echo "========================================="
+echo ""
+
+# Phase 1: Idempotency
+echo "--- Phase 1: Idempotency ---"
+IDEMPOTENCY_KEY=$(generate_uuid)
+IDEMPOTENT_RESPONSE=$(curl -sf --max-time 5 -X POST "$GATEWAY_URL/api/runs" \
+    -H "x-api-key: $API_KEY" \
+    -H "Idempotency-Key: $IDEMPOTENCY_KEY" \
+    -H "Content-Type: application/json" \
+    -d '{"name":"Idempotency Test","input":{"test":"idempotency"}}' 2>/dev/null)
+
+run_test "Idempotency: Create run with idempotency key" \
+    "echo \"$IDEMPOTENT_RESPONSE\" | grep -q '\"run\"\|\"success\"'"
+
+FIRST_RUN_ID=$(echo "$IDEMPOTENT_RESPONSE" | grep -o '"run":{[^}]*"id":"[^"]*' | grep -o '"id":"[^"]*' | cut -d'"' -f4 | head -1)
+if [ -z "$FIRST_RUN_ID" ]; then
+    FIRST_RUN_ID=$(echo "$IDEMPOTENT_RESPONSE" | grep -o '"id":"[^"]*' | cut -d'"' -f4 | head -1)
+fi
+
+if [ ! -z "$FIRST_RUN_ID" ]; then
+    SECOND_RESPONSE=$(curl -sf --max-time 5 -X POST "$GATEWAY_URL/api/runs" \
+        -H "x-api-key: $API_KEY" \
+        -H "Idempotency-Key: $IDEMPOTENCY_KEY" \
+        -H "Content-Type: application/json" \
+        -d '{"name":"Idempotency Test","input":{"test":"idempotency"}}' 2>/dev/null)
+    
+    SECOND_RUN_ID=$(echo "$SECOND_RESPONSE" | grep -o '"run":{[^}]*"id":"[^"]*' | grep -o '"id":"[^"]*' | cut -d'"' -f4 | head -1)
+    if [ -z "$SECOND_RUN_ID" ]; then
+        SECOND_RUN_ID=$(echo "$SECOND_RESPONSE" | grep -o '"id":"[^"]*' | cut -d'"' -f4 | head -1)
+    fi
+    
+    run_test "Idempotency: Duplicate request returns same run" \
+        "[ ! -z \"$SECOND_RUN_ID\" ] && [ \"$FIRST_RUN_ID\" = \"$SECOND_RUN_ID\" ]"
+fi
+
+# Phase 2: Structured Logging & Metrics
+echo ""
+echo "--- Phase 2: Logging & Metrics ---"
+run_test "Metrics: Prometheus metrics endpoint accessible" \
+    "curl -sf '$GATEWAY_URL/metrics' | grep -q 'http_request_total'"
+
+run_test "Metrics: Metrics include run metrics" \
+    "curl -sf '$GATEWAY_URL/metrics' | grep -q 'runs_created_total'"
+
+# Phase 3: Queue System
+echo ""
+echo "--- Phase 3: Queue System ---"
+QUEUE_RESPONSE=$(curl -sf --max-time 5 -X POST "$GATEWAY_URL/api/runs" \
+    -H "x-api-key: $API_KEY" \
+    -H "Content-Type: application/json" \
+    -d '{"name":"Queue Test","input":{"test":"queue"}}' 2>/dev/null)
+
+QUEUE_RUN_ID=$(echo "$QUEUE_RESPONSE" | grep -o '"run":{[^}]*"id":"[^"]*' | grep -o '"id":"[^"]*' | cut -d'"' -f4 | head -1)
+if [ -z "$QUEUE_RUN_ID" ]; then
+    QUEUE_RUN_ID=$(echo "$QUEUE_RESPONSE" | grep -o '"id":"[^"]*' | cut -d'"' -f4 | head -1)
+fi
+
+run_test "Queue: Create run successfully" \
+    "[ ! -z \"$QUEUE_RUN_ID\" ]"
+
+if [ ! -z "$QUEUE_RUN_ID" ]; then
+    START_RESPONSE=$(curl -sf --max-time 5 -X POST "$GATEWAY_URL/api/runs/$QUEUE_RUN_ID/start" \
+        -H "x-api-key: $API_KEY" \
+        -H "Content-Type: application/json" \
+        -d '{"priority":"high"}' 2>/dev/null)
+    
+    run_test "Queue: Start run (should queue)" \
+        "echo \"$START_RESPONSE\" | grep -q 'queued\|success'"
+    
+    sleep 2
+    
+    STATUS_RESPONSE=$(curl -sf --max-time 5 "$GATEWAY_URL/api/runs/$QUEUE_RUN_ID" \
+        -H "x-api-key: $API_KEY" 2>/dev/null)
+    
+    run_test "Queue: Run status updated" \
+        "echo \"$STATUS_RESPONSE\" | grep -q '\"status\"'"
+fi
+
+echo ""
+echo "========================================="
+echo "Phase 4: Production Hardening"
+echo "========================================="
+echo ""
+
+# Phase 4.1: Security Headers
+echo "--- Phase 4.1: Security Headers ---"
+run_test "Security: Content-Security-Policy header present" \
+    "curl -sf -I '$GATEWAY_URL/health' | grep -qi 'content-security-policy'"
+
+run_test "Security: X-Frame-Options header present" \
+    "curl -sf -I '$GATEWAY_URL/health' | grep -qi 'x-frame-options'"
+
+run_test "Security: X-Content-Type-Options header present" \
+    "curl -sf -I '$GATEWAY_URL/health' | grep -qi 'x-content-type-options'"
+
+run_test "Security: Server header removed" \
+    "! curl -sf -I '$GATEWAY_URL/health' | grep -qi '^server:'"
+
+# Phase 4.2: Error Handling
+echo ""
+echo "--- Phase 4.2: Error Handling ---"
+run_test "Error Handling: 404 error includes correlation ID" \
+    "curl -s '$GATEWAY_URL/api/runs/00000000-0000-0000-0000-000000000000' \
+    -H 'x-api-key: $API_KEY' | grep -q 'error'"
+
+run_test "Error Handling: Validation error includes details" \
+    "curl -s -X POST '$GATEWAY_URL/api/runs' \
+    -H 'x-api-key: $API_KEY' \
+    -H 'Content-Type: application/json' \
+    -d '{\"input\":{}}' | grep -q 'details'"
+
+run_test "Error Handling: Error response includes timestamp" \
+    "curl -s '$GATEWAY_URL/api/runs/invalid-id' \
+    -H 'x-api-key: $API_KEY' | grep -q 'timestamp'"
+
+# Phase 4.3: Health Checks
+echo ""
+echo "--- Phase 4.3: Health Checks ---"
+run_test "Health: Liveness endpoint returns 200" \
+    "curl -sf '$GATEWAY_URL/health' | grep -q '\"status\"'"
+
+run_test "Health: Readiness endpoint returns 200" \
+    "curl -sf '$GATEWAY_URL/ready' | grep -q '\"ready\"'"
+
+run_test "Health: Readiness includes dependency checks" \
+    "curl -sf '$GATEWAY_URL/ready' | grep -q '\"checks\"'"
+
+run_test "Health: Startup endpoint accessible" \
+    "curl -sf '$GATEWAY_URL/startup' | grep -q '\"started\"'"
+
+echo ""
+echo "========================================="
+echo "Phase 5: Testing & Validation"
+echo "========================================="
+echo ""
+
+# Phase 5: Integration Tests
+echo "--- Phase 5: Integration Tests ---"
+run_test "Integration: Can run integration tests" \
+    "cd /opt/4runr/gateway && pnpm test:integration -- simple.test.ts 2>&1 | grep -q 'PASS'"
+
+echo ""
+echo "========================================="
+echo "End-to-End Workflow Tests"
+echo "========================================="
+echo ""
+
+# Complete Workflow
+echo "--- Complete Run Workflow ---"
+WORKFLOW_RESPONSE=$(curl -sf -X POST "$GATEWAY_URL/api/runs" \
+    -H "x-api-key: $API_KEY" \
+    -H "Content-Type: application/json" \
+    -d '{"name":"E2E Workflow Test","input":{"test":"workflow"}}' 2>/dev/null)
+
+# Extract run ID - try multiple patterns
+WORKFLOW_RUN_ID=$(echo "$WORKFLOW_RESPONSE" | grep -o '"run"[^}]*"id":"[^"]*' | grep -o '"id":"[^"]*' | cut -d'"' -f4 | head -1)
+if [ -z "$WORKFLOW_RUN_ID" ]; then
+    # Try simpler pattern
+    WORKFLOW_RUN_ID=$(echo "$WORKFLOW_RESPONSE" | grep -o '"id":"[a-f0-9-]\{36\}' | cut -d'"' -f4 | head -1)
+fi
+
+run_test "E2E: Create run" \
+    "[ ! -z \"$WORKFLOW_RUN_ID\" ]"
+
+if [ ! -z "$WORKFLOW_RUN_ID" ]; then
+    run_test "E2E: Start run" \
+        "curl -sf -X POST '$GATEWAY_URL/api/runs/$WORKFLOW_RUN_ID/start' \
+        -H 'x-api-key: $API_KEY' \
+        -H 'Content-Type: application/json' \
+        -d '{\"priority\":\"normal\"}' | grep -q '\"queued\"'"
+    
+    run_test "E2E: Get run status" \
+        "curl -sf '$GATEWAY_URL/api/runs/$WORKFLOW_RUN_ID' \
+        -H 'x-api-key: $API_KEY' | grep -q '\"status\"'"
+    
+    run_test "E2E: List runs" \
+        "curl -sf '$GATEWAY_URL/api/runs' \
+        -H 'x-api-key: $API_KEY' | grep -q '\"runs\"'"
+fi
+
+# Authentication Tests
+echo ""
+echo "--- Authentication Tests ---"
+run_test "Auth: API key authentication required" \
+    "curl -sf '$GATEWAY_URL/api/runs' | grep -q '401\|403\|Unauthorized'"
+
+run_test "Auth: Valid API key works" \
+    "curl -sf '$GATEWAY_URL/api/runs' \
+    -H 'x-api-key: $API_KEY' | grep -q '\"runs\"'"
+
+run_test "Auth: Health endpoint doesn't require auth" \
+    "curl -sf '$GATEWAY_URL/health' | grep -q '\"status\"'"
+
+# Rate Limiting Tests
+echo ""
+echo "--- Rate Limiting Tests ---"
+run_test "Rate Limit: Health endpoint bypasses rate limiting" \
+    "for i in {1..20}; do curl -sf '$GATEWAY_URL/health' > /dev/null; done && true"
+
+# Concurrent Operations
+echo ""
+echo "--- Concurrent Operations ---"
+CONCURRENT_SUCCESS=0
+for i in {1..5}; do
+    RESPONSE=$(curl -s --max-time 5 -X POST "$GATEWAY_URL/api/runs" \
+        -H "x-api-key: $API_KEY" \
+        -H "Content-Type: application/json" \
+        -d "{\"name\":\"Concurrent Test $i\",\"input\":{\"index\":$i}}" 2>/dev/null)
+    if [ ! -z "$RESPONSE" ] && (echo "$RESPONSE" | grep -q '"run"\|"success"'); then
+        CONCURRENT_SUCCESS=$((CONCURRENT_SUCCESS + 1))
+    fi
+done
+
+run_test "Concurrency: Multiple runs can be created concurrently" \
+    "[ $CONCURRENT_SUCCESS -ge 3 ]"
+
+# Priority Queue
+echo ""
+echo "--- Priority Queue ---"
+PRIORITY_RUN1_RESPONSE=$(curl -sf -X POST "$GATEWAY_URL/api/runs" \
+    -H "x-api-key: $API_KEY" \
+    -H "Content-Type: application/json" \
+    -d '{"name":"Normal Priority","input":{"priority":"normal"}}' 2>/dev/null)
+
+PRIORITY_RUN1=$(echo "$PRIORITY_RUN1_RESPONSE" | grep -o '"run":{[^}]*"id":"[^"]*' | grep -o '"id":"[^"]*' | cut -d'"' -f4 | head -1)
+if [ -z "$PRIORITY_RUN1" ]; then
+    PRIORITY_RUN1=$(echo "$PRIORITY_RUN1_RESPONSE" | grep -o '"id":"[^"]*' | cut -d'"' -f4 | head -1)
+fi
+
+PRIORITY_RUN2_RESPONSE=$(curl -sf -X POST "$GATEWAY_URL/api/runs" \
+    -H "x-api-key: $API_KEY" \
+    -H "Content-Type: application/json" \
+    -d '{"name":"High Priority","input":{"priority":"high"}}' 2>/dev/null)
+
+PRIORITY_RUN2=$(echo "$PRIORITY_RUN2_RESPONSE" | grep -o '"run":{[^}]*"id":"[^"]*' | grep -o '"id":"[^"]*' | cut -d'"' -f4 | head -1)
+if [ -z "$PRIORITY_RUN2" ]; then
+    PRIORITY_RUN2=$(echo "$PRIORITY_RUN2_RESPONSE" | grep -o '"id":"[^"]*' | cut -d'"' -f4 | head -1)
+fi
+
+if [ ! -z "$PRIORITY_RUN1" ] && [ ! -z "$PRIORITY_RUN2" ]; then
+    curl -sf -X POST "$GATEWAY_URL/api/runs/$PRIORITY_RUN1/start" \
+        -H "x-api-key: $API_KEY" \
+        -H "Content-Type: application/json" \
+        -d '{"priority":"normal"}' > /dev/null
+    
+    curl -sf -X POST "$GATEWAY_URL/api/runs/$PRIORITY_RUN2/start" \
+        -H "x-api-key: $API_KEY" \
+        -H "Content-Type: application/json" \
+        -d '{"priority":"high"}' > /dev/null
+    
+    sleep 2
+    
+    run_test "Priority: Both runs started successfully" \
+        "curl -sf '$GATEWAY_URL/api/runs/$PRIORITY_RUN1' -H 'x-api-key: $API_KEY' | grep -q '\"status\"' && \
+         curl -sf '$GATEWAY_URL/api/runs/$PRIORITY_RUN2' -H 'x-api-key: $API_KEY' | grep -q '\"status\"'"
+fi
+
+# Error Scenarios
+echo ""
+echo "--- Error Scenarios ---"
+run_test "Errors: Invalid run ID returns 404" \
+    "curl -s '$GATEWAY_URL/api/runs/00000000-0000-0000-0000-000000000000' \
+    -H 'x-api-key: $API_KEY' | grep -q 'error'"
+
+run_test "Errors: Invalid UUID format returns 400" \
+    "curl -s '$GATEWAY_URL/api/runs/invalid-id' \
+    -H 'x-api-key: $API_KEY' | grep -q 'error'"
+
+run_test "Errors: Missing required field returns 400" \
+    "curl -s -X POST '$GATEWAY_URL/api/runs' \
+    -H 'x-api-key: $API_KEY' \
+    -H 'Content-Type: application/json' \
+    -d '{\"input\":{}}' | grep -q 'error'"
+
+echo ""
+echo "========================================="
+echo "Test Summary"
+echo "========================================="
+echo ""
+echo "Total Tests: $TOTAL_TESTS"
+echo -e "${GREEN}Passed: $PASSED_TESTS${NC}"
+echo -e "${RED}Failed: $FAILED_TESTS${NC}"
+echo ""
+
+if [ $FAILED_TESTS -gt 0 ]; then
+    echo -e "${RED}Failed Tests:${NC}"
+    for test_name in "${FAILED_TEST_NAMES[@]}"; do
+        echo "  - $test_name"
+    done
+    echo ""
+    echo -e "${RED}❌ Some tests failed. Please review the errors above.${NC}"
+    exit 1
+else
+    echo -e "${GREEN}✅ All tests passed!${NC}"
+    exit 0
+fi
+EOFTEST
+
+chmod +x /opt/4runr/gateway/test-all-phases.sh
+echo "✓ Clean test-all-phases.sh created successfully"
+