elasticdash-sdk 0.2.7-beta-2 → 0.2.9

package/src/interceptors/http.ts

@@ -12,6 +12,7 @@ const AI_URL_PATTERNS = [
   /https?:\/\/generativelanguage\.googleapis\.com\/.*\/models\/[^/:]+:(generateContent|streamGenerateContent)/,
   /https?:\/\/api\.x\.ai\/v1\/(chat\/)?completions/,
   /https?:\/\/api\.moonshot\.ai\/v1\/(chat\/)?completions/,
+  /https?:\/\/bedrock-runtime\.[^./]+\.amazonaws\.com\/model\/[^/]+\/(invoke|invoke-with-response-stream|converse|converse-stream)/,
 ]
 
 function isAIProviderUrl(url: string): boolean {
@@ -336,6 +337,22 @@ export function interceptFetch(): void {
     const httpCtx = getHttpRunContext()
     const obsCtx = getObservabilityContext()
 
+    const url =
+      typeof input === 'string'
+        ? input
+        : input instanceof URL
+          ? input.href
+          : (input as Request).url
+
+    // Let ai-interceptor handle AI provider URLs — it assigns its own event IDs.
+    // Short-circuit BEFORE attaching the mock-config header: AI provider servers
+    // don't run the SDK so they can't honor the header, and on Bedrock the
+    // gzipped header can exceed the edge proxy's header-size cap (yields a
+    // generic 400 "Request Header Or Cookie Too Large" page).
+    if (isAIProviderUrl(url)) {
+      return originalFetch!(input, init)
+    }
+
     // Option-A passthrough: even when there is no SDK context active, if the
     // CLI seeded mock globals we still attach the header so downstream Node
     // processes (e.g. a Next.js dev server hit by an HTTP-mode workflow) can
@@ -348,21 +365,10 @@ export function interceptFetch(): void {
     }
     if (mockHeader) init = attachMockConfigHeader(init, mockHeader)
 
-    const url =
-      typeof input === 'string'
-        ? input
-        : input instanceof URL
-          ? input.href
-          : (input as Request).url
     const method = (init?.method ?? (input instanceof Request ? input.method : 'GET')).toUpperCase()
     const rawHeaders = init?.headers ?? (input instanceof Request ? input.headers : undefined)
     const rawBody = init?.body ?? (input instanceof Request ? input.body : undefined)
 
-    // Let ai-interceptor handle AI provider URLs — it assigns its own event IDs
-    if (isAIProviderUrl(url)) {
-      return originalFetch!(input, init)
-    }
-
 
     // --- Observability-only mode: record and push, no mocks/replay ---
     if (!ctx && !httpCtx && obsCtx) {

package/src/interceptors/workflow-ai.ts

@@ -96,7 +96,11 @@ function enrichFromLLMCapture(
     return { input: enrichedInput, usage }
   }
 
-  // Interceptor capture failed (e.g. OpenAI SDK uses native fetch not globalThis.fetch).
+  // Interceptor capture failed (e.g. OpenAI SDK uses native fetch not globalThis.fetch;
+  // @aws-sdk/client-bedrock-runtime uses its own HTTP signer/handler on Node and likewise
+  // bypasses globalThis.fetch — when wrapping a Bedrock call, pass
+  // wrapAI(..., { provider: 'bedrock', model: '<modelId>' }) so this fallback enrichment
+  // still tags the event so mocked rerun can match it).
   // Fall back to model/provider from wrapAI options or function arguments.
   const resolvedModel = fallbackModel
     || (input && typeof input === 'object' ? (input as Record<string, unknown>).model as string | undefined : undefined)

package/src/tool-runner-worker.ts

@@ -164,8 +164,116 @@ function installFrozenFetchFallback(frozenEvents: FrozenEvent[]): void {
   }
 }
 
+// ─── Signal-driven shutdown + diagnostic dump ─────────────────────────
+//
+// Pre-fix behavior: this worker had ZERO signal handlers. When the parent
+// CLI / MCP / agent sent SIGTERM (e.g. on timeout), the inner tool's
+// blocking call (typically a hung HTTP fetch) just kept running. The
+// process never exited and no diagnostic landed anywhere.
+//
+// Post-fix:
+//   1. Install SIGTERM / SIGINT / SIGHUP handlers that flip a shared
+//      AbortSignal and start a 2s grace timer before forced exit.
+//   2. Before forced exit, dump a structured snapshot to stderr listing
+//      the current tool name, elapsed time, time since last heartbeat,
+//      and process.getActiveResourcesInfo() — tells the user "stuck on
+//      TCPWRAP / TLSWRAP / Timeout" without needing a real stack trace.
+//   3. While a tool is running, emit a 10s heartbeat (default — tunable
+//      via ELASTICDASH_WORKER_HEARTBEAT_MS=0 to disable) so a watchdog
+//      upstream sees continuous output instead of declaring the worker
+//      stalled.
+const workerAbortController = new AbortController()
+let currentToolName: string | undefined
+let toolStartedAtMs: number | undefined
+let lastHeartbeatAtMs: number = Date.now()
+let shuttingDown = false
+
+function activeResourcesSnapshot(): string[] {
+  // process.getActiveResourcesInfo() is stable in Node ≥17.3. It returns
+  // a list of resource names (TCPWRAP, TLSWRAP, Timeout, etc.) keeping
+  // the event loop alive. That alone tells a user whether the hang is
+  // network-bound, timer-bound, or CPU-bound (empty array = likely a
+  // sync infinite loop, which getActiveResourcesInfo wouldn't catch).
+  try {
+    const getActiveResourcesInfo = (process as unknown as {
+      getActiveResourcesInfo?: () => string[]
+    }).getActiveResourcesInfo
+    return typeof getActiveResourcesInfo === 'function' ? getActiveResourcesInfo() : []
+  } catch {
+    return []
+  }
+}
+
+function diagnosticHint(resources: string[]): string {
+  const set = new Set(resources)
+  if (set.has('TCPWRAP') || set.has('TLSWRAP')) {
+    return 'TCPWRAP/TLSWRAP present — likely a hung HTTP / network request. Add per-request timeouts (e.g. AbortSignal.timeout(...)) inside the tool implementation.'
+  }
+  if (set.has('Timeout')) {
+    return 'Timeout resources only — the tool may be waiting on a setTimeout / sleep. Check for missing clear or long backoff.'
+  }
+  if (resources.length === 0) {
+    return 'No active async resources — likely a sync infinite loop or sync I/O. Inspect tight loops in the tool.'
+  }
+  return `Active: ${resources.join(', ')}`
+}
+
+function dumpDiagnostic(reason: string): void {
+  const elapsedMs = toolStartedAtMs ? Date.now() - toolStartedAtMs : 0
+  const sinceHeartbeatMs = Date.now() - lastHeartbeatAtMs
+  const resources = activeResourcesSnapshot()
+  const lines = [
+    `[elasticdash-worker] ${reason} — diagnostic snapshot:`,
+    `  tool: ${currentToolName ?? '<no tool running>'}`,
+    `  elapsed_ms: ${elapsedMs}`,
+    `  since_last_heartbeat_ms: ${sinceHeartbeatMs}`,
+    `  active_async_resources: ${JSON.stringify(resources)}`,
+    `  hint: ${diagnosticHint(resources)}`,
+  ]
+  process.stderr.write(lines.join('\n') + '\n')
+}
+
+function installSignalHandlers(): void {
+  const handle = (sig: NodeJS.Signals) => {
+    if (shuttingDown) return
+    shuttingDown = true
+    process.stderr.write(`[elasticdash-worker] received ${sig} — aborting in-flight work\n`)
+    dumpDiagnostic(`${sig} received`)
+    try { workerAbortController.abort(new Error(`worker received ${sig}`)) } catch { /* may not be wired */ }
+    // 2s grace for in-flight async cleanup (flush stdout, close sockets);
+    // then force exit. .unref() so the timer itself doesn't keep us alive
+    // past natural completion if the abort actually lets cleanup happen.
+    setTimeout(() => {
+      process.stderr.write(`[elasticdash-worker] grace expired — force exit\n`)
+      process.exit(1)
+    }, 2_000).unref()
+  }
+  process.on('SIGTERM', () => handle('SIGTERM'))
+  process.on('SIGINT', () => handle('SIGINT'))
+  process.on('SIGHUP', () => handle('SIGHUP'))
+}
+
+function startToolHeartbeat(toolName: string): () => void {
+  currentToolName = toolName
+  toolStartedAtMs = Date.now()
+  lastHeartbeatAtMs = Date.now()
+  const heartbeatMs = Number(process.env.ELASTICDASH_WORKER_HEARTBEAT_MS ?? 10_000)
+  if (!(heartbeatMs > 0)) return () => { currentToolName = undefined; toolStartedAtMs = undefined }
+  const interval = setInterval(() => {
+    lastHeartbeatAtMs = Date.now()
+    const elapsedSec = Math.round((Date.now() - (toolStartedAtMs ?? Date.now())) / 1000)
+    process.stderr.write(`[elasticdash-worker] alive tool=${toolName} elapsed=${elapsedSec}s\n`)
+  }, heartbeatMs)
+  return () => {
+    clearInterval(interval)
+    currentToolName = undefined
+    toolStartedAtMs = undefined
+  }
+}
+
 async function main() {
   stage('boot')
+  installSignalHandlers()
   const originalExit = process.exit.bind(process)
 
   // Prevent the SDK's tryAutoInitHttpContext from triggering full observability
@@ -244,7 +352,13 @@ async function main() {
     stage('tool-resolved', { tool: toolName, via: resolvedVia })
 
     stage('tool-call-start', { tool: toolName })
-    const currentOutput = await fn(...args)
+    const stopHeartbeat = startToolHeartbeat(toolName)
+    let currentOutput: unknown
+    try {
+      currentOutput = await fn(...args)
+    } finally {
+      stopHeartbeat()
+    }
     stage('tool-call-end', { tool: toolName })
     await writeResult({ ok: true, currentOutput })
     stage('result-written')

package/src/workflow-runner-worker.ts

@@ -221,8 +221,93 @@ async function loadAndWrapTools(
   }
 }
 
+// ─── Signal-driven shutdown + diagnostic dump ─────────────────────────
+// See tool-runner-worker.ts for the rationale — same pattern, workflow-
+// scoped. When SIGTERM lands here, we dump a snapshot to stderr so the
+// user can see exactly what the workflow was stuck on (tool name,
+// elapsed, active resources) before the 2s grace and forced exit.
+let currentWorkflowName: string | undefined
+let workflowStartedAtMs: number | undefined
+let lastWfHeartbeatAtMs: number = Date.now()
+let workflowShuttingDown = false
+
+function activeResourcesSnapshotWf(): string[] {
+  try {
+    const getActiveResourcesInfo = (process as unknown as {
+      getActiveResourcesInfo?: () => string[]
+    }).getActiveResourcesInfo
+    return typeof getActiveResourcesInfo === 'function' ? getActiveResourcesInfo() : []
+  } catch {
+    return []
+  }
+}
+
+function workflowDiagnosticHint(resources: string[]): string {
+  const set = new Set(resources)
+  if (set.has('TCPWRAP') || set.has('TLSWRAP')) {
+    return 'TCPWRAP/TLSWRAP present — likely a hung HTTP / AI provider request. Check per-request timeouts in workflow tools.'
+  }
+  if (set.has('Timeout')) {
+    return 'Timeout resources only — workflow may be waiting on a sleep / backoff.'
+  }
+  if (resources.length === 0) {
+    return 'No active async resources — likely a sync infinite loop or sync I/O in user code.'
+  }
+  return `Active: ${resources.join(', ')}`
+}
+
+function dumpWorkflowDiagnostic(reason: string): void {
+  const elapsedMs = workflowStartedAtMs ? Date.now() - workflowStartedAtMs : 0
+  const sinceHeartbeatMs = Date.now() - lastWfHeartbeatAtMs
+  const resources = activeResourcesSnapshotWf()
+  const lines = [
+    `[elasticdash-worker workflow] ${reason} — diagnostic snapshot:`,
+    `  workflow: ${currentWorkflowName ?? '<not started>'}`,
+    `  elapsed_ms: ${elapsedMs}`,
+    `  since_last_heartbeat_ms: ${sinceHeartbeatMs}`,
+    `  active_async_resources: ${JSON.stringify(resources)}`,
+    `  hint: ${workflowDiagnosticHint(resources)}`,
+  ]
+  process.stderr.write(lines.join('\n') + '\n')
+}
+
+function installWorkflowSignalHandlers(): void {
+  const handle = (sig: NodeJS.Signals) => {
+    if (workflowShuttingDown) return
+    workflowShuttingDown = true
+    process.stderr.write(`[elasticdash-worker workflow] received ${sig} — aborting in-flight work\n`)
+    dumpWorkflowDiagnostic(`${sig} received`)
+    setTimeout(() => {
+      process.stderr.write(`[elasticdash-worker workflow] grace expired — force exit\n`)
+      process.exit(1)
+    }, 2_000).unref()
+  }
+  process.on('SIGTERM', () => handle('SIGTERM'))
+  process.on('SIGINT', () => handle('SIGINT'))
+  process.on('SIGHUP', () => handle('SIGHUP'))
+}
+
+function startWorkflowHeartbeat(workflowName: string): () => void {
+  currentWorkflowName = workflowName
+  workflowStartedAtMs = Date.now()
+  lastWfHeartbeatAtMs = Date.now()
+  const heartbeatMs = Number(process.env.ELASTICDASH_WORKER_HEARTBEAT_MS ?? 10_000)
+  if (!(heartbeatMs > 0)) return () => { currentWorkflowName = undefined; workflowStartedAtMs = undefined }
+  const interval = setInterval(() => {
+    lastWfHeartbeatAtMs = Date.now()
+    const elapsedSec = Math.round((Date.now() - (workflowStartedAtMs ?? Date.now())) / 1000)
+    process.stderr.write(`[elasticdash-worker workflow] alive workflow=${workflowName} elapsed=${elapsedSec}s\n`)
+  }, heartbeatMs)
+  return () => {
+    clearInterval(interval)
+    currentWorkflowName = undefined
+    workflowStartedAtMs = undefined
+  }
+}
+
 async function main() {
   stage('boot')
+  installWorkflowSignalHandlers()
   // Keep a reference to the real process.exit so we can call it after flushing stdout.
   const originalExit = process.exit.bind(process)
 
@@ -347,7 +432,12 @@ async function main() {
         // Standardize workflow argument resolution: always pass [input] if args is empty
         const callArgs = args.length ? args : [input]
         stage('workflow-call-start', { workflow: workflowName })
-        currentOutput = await (workflowFn as (...a: unknown[]) => unknown)(...callArgs)
+        const stopWfHeartbeat = startWorkflowHeartbeat(workflowName)
+        try {
+          currentOutput = await (workflowFn as (...a: unknown[]) => unknown)(...callArgs)
+        } finally {
+          stopWfHeartbeat()
+        }
         stage('workflow-call-end', { workflow: workflowName })
         console.error('[worker] workflowFn resolved, currentOutput:', currentOutput)  // stderr so it's visible
       }