elasticdash-sdk 0.2.7-beta → 0.2.8

package/src/execution/tool-runner.ts

@@ -132,13 +132,48 @@ export function runToolInSubprocess(
     const runtimeArgs = denoProject ? ['run', '--allow-all', workerScript] : [workerScript]
     debugLog(`[elasticdash portal] Runtime for tool subprocess: ${runtime} ${runtimeArgs.join(' ')}`, { isDenoProject: denoProject })
 
+    // detached:true on POSIX puts the child in its OWN process group, so
+    // process.kill(-pid, sig) reaches the whole subtree (worker + any
+    // grandchildren the tool spawned — HTTP keepalive sockets, etc.).
+    // We still own stdio pipes so the IPC protocol is unchanged. On
+    // Windows we fall back to plain child.kill — process-group semantics
+    // differ and detached:true there would let the child outlive us.
+    const useProcessGroup = process.platform !== 'win32'
     const child = spawn(runtime, runtimeArgs, {
       env: childEnv,
       cwd: projectDir,
       stdio: ['pipe', 'pipe', 'pipe'],
+      ...(useProcessGroup ? { detached: true } : {}),
     })
     const pid = child.pid ?? -1
-    debugLog(`[elasticdash portal] tool subprocess stage=spawned pid=${pid} elapsedMs=${elapsed()} tool=${toolName}`)
+    debugLog(`[elasticdash portal] tool subprocess stage=spawned pid=${pid} elapsedMs=${elapsed()} tool=${toolName} pgroup=${useProcessGroup}`)
+
+    // Tree-kill helper: signal the process group on POSIX so the worker
+    // AND any subprocess the tool spawned die together. Falls back to
+    // direct child.kill on Windows / when the group send fails (e.g.
+    // child already exited).
+    const killTree = (signal: NodeJS.Signals) => {
+      if (useProcessGroup && pid > 0) {
+        try { process.kill(-pid, signal); return } catch { /* fall through */ }
+      }
+      try { child.kill(signal) } catch { /* already gone */ }
+    }
+
+    // Forward SIGTERM/SIGINT/SIGHUP received by THIS process to the child
+    // group, so when the MCP / npm shim / agent kills us, the worker
+    // tree dies too instead of orphaning. Without this, the inner node
+    // process can sit alive after the shim has exited.
+    const forwarded: NodeJS.Signals[] = ['SIGTERM', 'SIGINT', 'SIGHUP']
+    const forwardHandlers: Partial<Record<NodeJS.Signals, () => void>> = {}
+    for (const sig of forwarded) {
+      const h = () => {
+        debugLog(`[elasticdash portal] parent received ${sig} — forwarding to child group pid=${pid}`)
+        killTree(sig)
+        setTimeout(() => killTree('SIGKILL'), 2000)
+      }
+      forwardHandlers[sig] = h
+      process.on(sig, h)
+    }
 
     // Heartbeat: prove the parent is still waiting on a live child. 0 disables.
     const heartbeatMs = Number(process.env.ELASTICDASH_HEARTBEAT_MS ?? 5000)
@@ -148,23 +183,32 @@ export function runToolInSubprocess(
         }, heartbeatMs)
       : null
 
-    // Optional kill switch. Default unset = no timeout (preserves prior behavior).
+    // Kill switch. Default 300_000ms (5 min) — covers most legitimate tool
+    // runs while still preventing the "child sits alive forever" hang the
+    // research_bot_js agent ran into. Set ELASTICDASH_TOOL_TIMEOUT_MS=0 to
+    // restore the old behavior, or any positive number to tune it.
     let timedOut = false
-    const timeoutMs = Number(process.env.ELASTICDASH_TOOL_TIMEOUT_MS ?? 0)
+    const timeoutMsRaw = process.env.ELASTICDASH_TOOL_TIMEOUT_MS
+    const timeoutMs = timeoutMsRaw === undefined ? 300_000 : Number(timeoutMsRaw)
     const timeout = timeoutMs > 0
       ? setTimeout(() => {
           timedOut = true
-          debugLog(`[elasticdash portal] tool subprocess TIMEOUT pid=${pid} after ${timeoutMs}ms — sending SIGTERM`)
-          try { child.kill('SIGTERM') } catch { /* already dead */ }
-          setTimeout(() => {
-            try { child.kill('SIGKILL') } catch { /* already dead */ }
-          }, 2000)
+          debugLog(`[elasticdash portal] tool subprocess TIMEOUT pid=${pid} after ${timeoutMs}ms — sending SIGTERM to group`)
+          killTree('SIGTERM')
+          setTimeout(() => killTree('SIGKILL'), 2000)
         }, timeoutMs)
       : null
 
     const cleanup = () => {
       if (heartbeat) clearInterval(heartbeat)
       if (timeout) clearTimeout(timeout)
+      // Remove the parent-side signal forwarders we registered above so
+      // when the parent process eventually receives its own SIGTERM after
+      // legit exit, it doesn't double-handle.
+      for (const sig of forwarded) {
+        const h = forwardHandlers[sig]
+        if (h) process.removeListener(sig, h)
+      }
     }
 
     const RESULT_PREFIX = '__ELASTICDASH_RESULT__:'

package/src/tool-runner-worker.ts

@@ -164,8 +164,116 @@ function installFrozenFetchFallback(frozenEvents: FrozenEvent[]): void {
   }
 }
 
+// ─── Signal-driven shutdown + diagnostic dump ─────────────────────────
+//
+// Pre-fix behavior: this worker had ZERO signal handlers. When the parent
+// CLI / MCP / agent sent SIGTERM (e.g. on timeout), the inner tool's
+// blocking call (typically a hung HTTP fetch) just kept running. The
+// process never exited and no diagnostic landed anywhere.
+//
+// Post-fix:
+//   1. Install SIGTERM / SIGINT / SIGHUP handlers that flip a shared
+//      AbortSignal and start a 2s grace timer before forced exit.
+//   2. Before forced exit, dump a structured snapshot to stderr listing
+//      the current tool name, elapsed time, time since last heartbeat,
+//      and process.getActiveResourcesInfo() — tells the user "stuck on
+//      TCPWRAP / TLSWRAP / Timeout" without needing a real stack trace.
+//   3. While a tool is running, emit a 10s heartbeat (default — tunable
+//      via ELASTICDASH_WORKER_HEARTBEAT_MS=0 to disable) so a watchdog
+//      upstream sees continuous output instead of declaring the worker
+//      stalled.
+const workerAbortController = new AbortController()
+let currentToolName: string | undefined
+let toolStartedAtMs: number | undefined
+let lastHeartbeatAtMs: number = Date.now()
+let shuttingDown = false
+
+function activeResourcesSnapshot(): string[] {
+  // process.getActiveResourcesInfo() is stable in Node ≥17.3. It returns
+  // a list of resource names (TCPWRAP, TLSWRAP, Timeout, etc.) keeping
+  // the event loop alive. That alone tells a user whether the hang is
+  // network-bound, timer-bound, or CPU-bound (empty array = likely a
+  // sync infinite loop, which getActiveResourcesInfo wouldn't catch).
+  try {
+    const getActiveResourcesInfo = (process as unknown as {
+      getActiveResourcesInfo?: () => string[]
+    }).getActiveResourcesInfo
+    return typeof getActiveResourcesInfo === 'function' ? getActiveResourcesInfo() : []
+  } catch {
+    return []
+  }
+}
+
+function diagnosticHint(resources: string[]): string {
+  const set = new Set(resources)
+  if (set.has('TCPWRAP') || set.has('TLSWRAP')) {
+    return 'TCPWRAP/TLSWRAP present — likely a hung HTTP / network request. Add per-request timeouts (e.g. AbortSignal.timeout(...)) inside the tool implementation.'
+  }
+  if (set.has('Timeout')) {
+    return 'Timeout resources only — the tool may be waiting on a setTimeout / sleep. Check for missing clear or long backoff.'
+  }
+  if (resources.length === 0) {
+    return 'No active async resources — likely a sync infinite loop or sync I/O. Inspect tight loops in the tool.'
+  }
+  return `Active: ${resources.join(', ')}`
+}
+
+function dumpDiagnostic(reason: string): void {
+  const elapsedMs = toolStartedAtMs ? Date.now() - toolStartedAtMs : 0
+  const sinceHeartbeatMs = Date.now() - lastHeartbeatAtMs
+  const resources = activeResourcesSnapshot()
+  const lines = [
+    `[elasticdash-worker] ${reason} — diagnostic snapshot:`,
+    `  tool: ${currentToolName ?? '<no tool running>'}`,
+    `  elapsed_ms: ${elapsedMs}`,
+    `  since_last_heartbeat_ms: ${sinceHeartbeatMs}`,
+    `  active_async_resources: ${JSON.stringify(resources)}`,
+    `  hint: ${diagnosticHint(resources)}`,
+  ]
+  process.stderr.write(lines.join('\n') + '\n')
+}
+
+function installSignalHandlers(): void {
+  const handle = (sig: NodeJS.Signals) => {
+    if (shuttingDown) return
+    shuttingDown = true
+    process.stderr.write(`[elasticdash-worker] received ${sig} — aborting in-flight work\n`)
+    dumpDiagnostic(`${sig} received`)
+    try { workerAbortController.abort(new Error(`worker received ${sig}`)) } catch { /* may not be wired */ }
+    // 2s grace for in-flight async cleanup (flush stdout, close sockets);
+    // then force exit. .unref() so the timer itself doesn't keep us alive
+    // past natural completion if the abort actually lets cleanup happen.
+    setTimeout(() => {
+      process.stderr.write(`[elasticdash-worker] grace expired — force exit\n`)
+      process.exit(1)
+    }, 2_000).unref()
+  }
+  process.on('SIGTERM', () => handle('SIGTERM'))
+  process.on('SIGINT', () => handle('SIGINT'))
+  process.on('SIGHUP', () => handle('SIGHUP'))
+}
+
+function startToolHeartbeat(toolName: string): () => void {
+  currentToolName = toolName
+  toolStartedAtMs = Date.now()
+  lastHeartbeatAtMs = Date.now()
+  const heartbeatMs = Number(process.env.ELASTICDASH_WORKER_HEARTBEAT_MS ?? 10_000)
+  if (!(heartbeatMs > 0)) return () => { currentToolName = undefined; toolStartedAtMs = undefined }
+  const interval = setInterval(() => {
+    lastHeartbeatAtMs = Date.now()
+    const elapsedSec = Math.round((Date.now() - (toolStartedAtMs ?? Date.now())) / 1000)
+    process.stderr.write(`[elasticdash-worker] alive tool=${toolName} elapsed=${elapsedSec}s\n`)
+  }, heartbeatMs)
+  return () => {
+    clearInterval(interval)
+    currentToolName = undefined
+    toolStartedAtMs = undefined
+  }
+}
+
 async function main() {
   stage('boot')
+  installSignalHandlers()
   const originalExit = process.exit.bind(process)
 
   // Prevent the SDK's tryAutoInitHttpContext from triggering full observability
@@ -244,7 +352,13 @@ async function main() {
     stage('tool-resolved', { tool: toolName, via: resolvedVia })
 
     stage('tool-call-start', { tool: toolName })
-    const currentOutput = await fn(...args)
+    const stopHeartbeat = startToolHeartbeat(toolName)
+    let currentOutput: unknown
+    try {
+      currentOutput = await fn(...args)
+    } finally {
+      stopHeartbeat()
+    }
     stage('tool-call-end', { tool: toolName })
     await writeResult({ ok: true, currentOutput })
     stage('result-written')

package/src/workflow-runner-worker.ts

@@ -221,8 +221,93 @@ async function loadAndWrapTools(
   }
 }
 
+// ─── Signal-driven shutdown + diagnostic dump ─────────────────────────
+// See tool-runner-worker.ts for the rationale — same pattern, workflow-
+// scoped. When SIGTERM lands here, we dump a snapshot to stderr so the
+// user can see exactly what the workflow was stuck on (tool name,
+// elapsed, active resources) before the 2s grace and forced exit.
+let currentWorkflowName: string | undefined
+let workflowStartedAtMs: number | undefined
+let lastWfHeartbeatAtMs: number = Date.now()
+let workflowShuttingDown = false
+
+function activeResourcesSnapshotWf(): string[] {
+  try {
+    const getActiveResourcesInfo = (process as unknown as {
+      getActiveResourcesInfo?: () => string[]
+    }).getActiveResourcesInfo
+    return typeof getActiveResourcesInfo === 'function' ? getActiveResourcesInfo() : []
+  } catch {
+    return []
+  }
+}
+
+function workflowDiagnosticHint(resources: string[]): string {
+  const set = new Set(resources)
+  if (set.has('TCPWRAP') || set.has('TLSWRAP')) {
+    return 'TCPWRAP/TLSWRAP present — likely a hung HTTP / AI provider request. Check per-request timeouts in workflow tools.'
+  }
+  if (set.has('Timeout')) {
+    return 'Timeout resources only — workflow may be waiting on a sleep / backoff.'
+  }
+  if (resources.length === 0) {
+    return 'No active async resources — likely a sync infinite loop or sync I/O in user code.'
+  }
+  return `Active: ${resources.join(', ')}`
+}
+
+function dumpWorkflowDiagnostic(reason: string): void {
+  const elapsedMs = workflowStartedAtMs ? Date.now() - workflowStartedAtMs : 0
+  const sinceHeartbeatMs = Date.now() - lastWfHeartbeatAtMs
+  const resources = activeResourcesSnapshotWf()
+  const lines = [
+    `[elasticdash-worker workflow] ${reason} — diagnostic snapshot:`,
+    `  workflow: ${currentWorkflowName ?? '<not started>'}`,
+    `  elapsed_ms: ${elapsedMs}`,
+    `  since_last_heartbeat_ms: ${sinceHeartbeatMs}`,
+    `  active_async_resources: ${JSON.stringify(resources)}`,
+    `  hint: ${workflowDiagnosticHint(resources)}`,
+  ]
+  process.stderr.write(lines.join('\n') + '\n')
+}
+
+function installWorkflowSignalHandlers(): void {
+  const handle = (sig: NodeJS.Signals) => {
+    if (workflowShuttingDown) return
+    workflowShuttingDown = true
+    process.stderr.write(`[elasticdash-worker workflow] received ${sig} — aborting in-flight work\n`)
+    dumpWorkflowDiagnostic(`${sig} received`)
+    setTimeout(() => {
+      process.stderr.write(`[elasticdash-worker workflow] grace expired — force exit\n`)
+      process.exit(1)
+    }, 2_000).unref()
+  }
+  process.on('SIGTERM', () => handle('SIGTERM'))
+  process.on('SIGINT', () => handle('SIGINT'))
+  process.on('SIGHUP', () => handle('SIGHUP'))
+}
+
+function startWorkflowHeartbeat(workflowName: string): () => void {
+  currentWorkflowName = workflowName
+  workflowStartedAtMs = Date.now()
+  lastWfHeartbeatAtMs = Date.now()
+  const heartbeatMs = Number(process.env.ELASTICDASH_WORKER_HEARTBEAT_MS ?? 10_000)
+  if (!(heartbeatMs > 0)) return () => { currentWorkflowName = undefined; workflowStartedAtMs = undefined }
+  const interval = setInterval(() => {
+    lastWfHeartbeatAtMs = Date.now()
+    const elapsedSec = Math.round((Date.now() - (workflowStartedAtMs ?? Date.now())) / 1000)
+    process.stderr.write(`[elasticdash-worker workflow] alive workflow=${workflowName} elapsed=${elapsedSec}s\n`)
+  }, heartbeatMs)
+  return () => {
+    clearInterval(interval)
+    currentWorkflowName = undefined
+    workflowStartedAtMs = undefined
+  }
+}
+
 async function main() {
   stage('boot')
+  installWorkflowSignalHandlers()
   // Keep a reference to the real process.exit so we can call it after flushing stdout.
   const originalExit = process.exit.bind(process)
 
@@ -347,7 +432,12 @@ async function main() {
         // Standardize workflow argument resolution: always pass [input] if args is empty
         const callArgs = args.length ? args : [input]
         stage('workflow-call-start', { workflow: workflowName })
-        currentOutput = await (workflowFn as (...a: unknown[]) => unknown)(...callArgs)
+        const stopWfHeartbeat = startWorkflowHeartbeat(workflowName)
+        try {
+          currentOutput = await (workflowFn as (...a: unknown[]) => unknown)(...callArgs)
+        } finally {
+          stopWfHeartbeat()
+        }
         stage('workflow-call-end', { workflow: workflowName })
         console.error('[worker] workflowFn resolved, currentOutput:', currentOutput)  // stderr so it's visible
       }