elasticdash-sdk 0.2.7-beta-2 → 0.2.8

package/src/tool-runner-worker.ts

@@ -164,8 +164,116 @@ function installFrozenFetchFallback(frozenEvents: FrozenEvent[]): void {
   }
 }
 
+// ─── Signal-driven shutdown + diagnostic dump ─────────────────────────
+//
+// Pre-fix behavior: this worker had ZERO signal handlers. When the parent
+// CLI / MCP / agent sent SIGTERM (e.g. on timeout), the inner tool's
+// blocking call (typically a hung HTTP fetch) just kept running. The
+// process never exited and no diagnostic landed anywhere.
+//
+// Post-fix:
+//   1. Install SIGTERM / SIGINT / SIGHUP handlers that flip a shared
+//      AbortSignal and start a 2s grace timer before forced exit.
+//   2. Before forced exit, dump a structured snapshot to stderr listing
+//      the current tool name, elapsed time, time since last heartbeat,
+//      and process.getActiveResourcesInfo() — tells the user "stuck on
+//      TCPWRAP / TLSWRAP / Timeout" without needing a real stack trace.
+//   3. While a tool is running, emit a 10s heartbeat (default — tunable
+//      via ELASTICDASH_WORKER_HEARTBEAT_MS=0 to disable) so a watchdog
+//      upstream sees continuous output instead of declaring the worker
+//      stalled.
+const workerAbortController = new AbortController()
+let currentToolName: string | undefined
+let toolStartedAtMs: number | undefined
+let lastHeartbeatAtMs: number = Date.now()
+let shuttingDown = false
+
+function activeResourcesSnapshot(): string[] {
+  // process.getActiveResourcesInfo() is stable in Node ≥17.3. It returns
+  // a list of resource names (TCPWRAP, TLSWRAP, Timeout, etc.) keeping
+  // the event loop alive. That alone tells a user whether the hang is
+  // network-bound, timer-bound, or CPU-bound (empty array = likely a
+  // sync infinite loop, which getActiveResourcesInfo wouldn't catch).
+  try {
+    const getActiveResourcesInfo = (process as unknown as {
+      getActiveResourcesInfo?: () => string[]
+    }).getActiveResourcesInfo
+    return typeof getActiveResourcesInfo === 'function' ? getActiveResourcesInfo() : []
+  } catch {
+    return []
+  }
+}
+
+function diagnosticHint(resources: string[]): string {
+  const set = new Set(resources)
+  if (set.has('TCPWRAP') || set.has('TLSWRAP')) {
+    return 'TCPWRAP/TLSWRAP present — likely a hung HTTP / network request. Add per-request timeouts (e.g. AbortSignal.timeout(...)) inside the tool implementation.'
+  }
+  if (set.has('Timeout')) {
+    return 'Timeout resources only — the tool may be waiting on a setTimeout / sleep. Check for missing clear or long backoff.'
+  }
+  if (resources.length === 0) {
+    return 'No active async resources — likely a sync infinite loop or sync I/O. Inspect tight loops in the tool.'
+  }
+  return `Active: ${resources.join(', ')}`
+}
+
+function dumpDiagnostic(reason: string): void {
+  const elapsedMs = toolStartedAtMs ? Date.now() - toolStartedAtMs : 0
+  const sinceHeartbeatMs = Date.now() - lastHeartbeatAtMs
+  const resources = activeResourcesSnapshot()
+  const lines = [
+    `[elasticdash-worker] ${reason} — diagnostic snapshot:`,
+    `  tool: ${currentToolName ?? '<no tool running>'}`,
+    `  elapsed_ms: ${elapsedMs}`,
+    `  since_last_heartbeat_ms: ${sinceHeartbeatMs}`,
+    `  active_async_resources: ${JSON.stringify(resources)}`,
+    `  hint: ${diagnosticHint(resources)}`,
+  ]
+  process.stderr.write(lines.join('\n') + '\n')
+}
+
+function installSignalHandlers(): void {
+  const handle = (sig: NodeJS.Signals) => {
+    if (shuttingDown) return
+    shuttingDown = true
+    process.stderr.write(`[elasticdash-worker] received ${sig} — aborting in-flight work\n`)
+    dumpDiagnostic(`${sig} received`)
+    try { workerAbortController.abort(new Error(`worker received ${sig}`)) } catch { /* may not be wired */ }
+    // 2s grace for in-flight async cleanup (flush stdout, close sockets);
+    // then force exit. .unref() so the timer itself doesn't keep us alive
+    // past natural completion if the abort actually lets cleanup happen.
+    setTimeout(() => {
+      process.stderr.write(`[elasticdash-worker] grace expired — force exit\n`)
+      process.exit(1)
+    }, 2_000).unref()
+  }
+  process.on('SIGTERM', () => handle('SIGTERM'))
+  process.on('SIGINT', () => handle('SIGINT'))
+  process.on('SIGHUP', () => handle('SIGHUP'))
+}
+
+function startToolHeartbeat(toolName: string): () => void {
+  currentToolName = toolName
+  toolStartedAtMs = Date.now()
+  lastHeartbeatAtMs = Date.now()
+  const heartbeatMs = Number(process.env.ELASTICDASH_WORKER_HEARTBEAT_MS ?? 10_000)
+  if (!(heartbeatMs > 0)) return () => { currentToolName = undefined; toolStartedAtMs = undefined }
+  const interval = setInterval(() => {
+    lastHeartbeatAtMs = Date.now()
+    const elapsedSec = Math.round((Date.now() - (toolStartedAtMs ?? Date.now())) / 1000)
+    process.stderr.write(`[elasticdash-worker] alive tool=${toolName} elapsed=${elapsedSec}s\n`)
+  }, heartbeatMs)
+  return () => {
+    clearInterval(interval)
+    currentToolName = undefined
+    toolStartedAtMs = undefined
+  }
+}
+
 async function main() {
   stage('boot')
+  installSignalHandlers()
   const originalExit = process.exit.bind(process)
 
   // Prevent the SDK's tryAutoInitHttpContext from triggering full observability
@@ -244,7 +352,13 @@ async function main() {
     stage('tool-resolved', { tool: toolName, via: resolvedVia })
 
     stage('tool-call-start', { tool: toolName })
-    const currentOutput = await fn(...args)
+    const stopHeartbeat = startToolHeartbeat(toolName)
+    let currentOutput: unknown
+    try {
+      currentOutput = await fn(...args)
+    } finally {
+      stopHeartbeat()
+    }
     stage('tool-call-end', { tool: toolName })
     await writeResult({ ok: true, currentOutput })
     stage('result-written')

package/src/workflow-runner-worker.ts

@@ -221,8 +221,93 @@ async function loadAndWrapTools(
   }
 }
 
+// ─── Signal-driven shutdown + diagnostic dump ─────────────────────────
+// See tool-runner-worker.ts for the rationale — same pattern, workflow-
+// scoped. When SIGTERM lands here, we dump a snapshot to stderr so the
+// user can see exactly what the workflow was stuck on (tool name,
+// elapsed, active resources) before the 2s grace and forced exit.
+let currentWorkflowName: string | undefined
+let workflowStartedAtMs: number | undefined
+let lastWfHeartbeatAtMs: number = Date.now()
+let workflowShuttingDown = false
+
+function activeResourcesSnapshotWf(): string[] {
+  try {
+    const getActiveResourcesInfo = (process as unknown as {
+      getActiveResourcesInfo?: () => string[]
+    }).getActiveResourcesInfo
+    return typeof getActiveResourcesInfo === 'function' ? getActiveResourcesInfo() : []
+  } catch {
+    return []
+  }
+}
+
+function workflowDiagnosticHint(resources: string[]): string {
+  const set = new Set(resources)
+  if (set.has('TCPWRAP') || set.has('TLSWRAP')) {
+    return 'TCPWRAP/TLSWRAP present — likely a hung HTTP / AI provider request. Check per-request timeouts in workflow tools.'
+  }
+  if (set.has('Timeout')) {
+    return 'Timeout resources only — workflow may be waiting on a sleep / backoff.'
+  }
+  if (resources.length === 0) {
+    return 'No active async resources — likely a sync infinite loop or sync I/O in user code.'
+  }
+  return `Active: ${resources.join(', ')}`
+}
+
+function dumpWorkflowDiagnostic(reason: string): void {
+  const elapsedMs = workflowStartedAtMs ? Date.now() - workflowStartedAtMs : 0
+  const sinceHeartbeatMs = Date.now() - lastWfHeartbeatAtMs
+  const resources = activeResourcesSnapshotWf()
+  const lines = [
+    `[elasticdash-worker workflow] ${reason} — diagnostic snapshot:`,
+    `  workflow: ${currentWorkflowName ?? '<not started>'}`,
+    `  elapsed_ms: ${elapsedMs}`,
+    `  since_last_heartbeat_ms: ${sinceHeartbeatMs}`,
+    `  active_async_resources: ${JSON.stringify(resources)}`,
+    `  hint: ${workflowDiagnosticHint(resources)}`,
+  ]
+  process.stderr.write(lines.join('\n') + '\n')
+}
+
+function installWorkflowSignalHandlers(): void {
+  const handle = (sig: NodeJS.Signals) => {
+    if (workflowShuttingDown) return
+    workflowShuttingDown = true
+    process.stderr.write(`[elasticdash-worker workflow] received ${sig} — aborting in-flight work\n`)
+    dumpWorkflowDiagnostic(`${sig} received`)
+    setTimeout(() => {
+      process.stderr.write(`[elasticdash-worker workflow] grace expired — force exit\n`)
+      process.exit(1)
+    }, 2_000).unref()
+  }
+  process.on('SIGTERM', () => handle('SIGTERM'))
+  process.on('SIGINT', () => handle('SIGINT'))
+  process.on('SIGHUP', () => handle('SIGHUP'))
+}
+
+function startWorkflowHeartbeat(workflowName: string): () => void {
+  currentWorkflowName = workflowName
+  workflowStartedAtMs = Date.now()
+  lastWfHeartbeatAtMs = Date.now()
+  const heartbeatMs = Number(process.env.ELASTICDASH_WORKER_HEARTBEAT_MS ?? 10_000)
+  if (!(heartbeatMs > 0)) return () => { currentWorkflowName = undefined; workflowStartedAtMs = undefined }
+  const interval = setInterval(() => {
+    lastWfHeartbeatAtMs = Date.now()
+    const elapsedSec = Math.round((Date.now() - (workflowStartedAtMs ?? Date.now())) / 1000)
+    process.stderr.write(`[elasticdash-worker workflow] alive workflow=${workflowName} elapsed=${elapsedSec}s\n`)
+  }, heartbeatMs)
+  return () => {
+    clearInterval(interval)
+    currentWorkflowName = undefined
+    workflowStartedAtMs = undefined
+  }
+}
+
 async function main() {
   stage('boot')
+  installWorkflowSignalHandlers()
   // Keep a reference to the real process.exit so we can call it after flushing stdout.
   const originalExit = process.exit.bind(process)
 
@@ -347,7 +432,12 @@ async function main() {
         // Standardize workflow argument resolution: always pass [input] if args is empty
         const callArgs = args.length ? args : [input]
         stage('workflow-call-start', { workflow: workflowName })
-        currentOutput = await (workflowFn as (...a: unknown[]) => unknown)(...callArgs)
+        const stopWfHeartbeat = startWorkflowHeartbeat(workflowName)
+        try {
+          currentOutput = await (workflowFn as (...a: unknown[]) => unknown)(...callArgs)
+        } finally {
+          stopWfHeartbeat()
+        }
         stage('workflow-call-end', { workflow: workflowName })
         console.error('[worker] workflowFn resolved, currentOutput:', currentOutput)  // stderr so it's visible
       }