@hotmeshio/long-tail 0.1.11 → 0.1.12

package/build/api/escalations.js

@@ -730,7 +730,12 @@ async function resolveEscalation(input, auth) {
                 const handle = await client.workflow.getHandle(signalRouting.taskQueue, signalRouting.workflowType, signalRouting.workflowId);
                 await handle.signal(signalRouting.signalId, signalPayload);
             }
-            await escalationService.resolveEscalation(escalation.id, resolverPayload);
+            // For YAML workflows, the resolve worker inside the workflow calls
+            // claim_and_resolve to close the escalation transactionally. Only resolve
+            // here for Durable workflows that lack an in-workflow resolve step.
+            if (signalRouting.engine !== 'yaml') {
+                await escalationService.resolveEscalation(escalation.id, resolverPayload);
+            }
             (0, publish_1.publishEscalationEvent)({
                 type: 'escalation.resolved',
                 source: 'api',
@@ -740,7 +745,7 @@ async function resolveEscalation(input, auth) {
                 taskId: escalation.task_id,
                 escalationId: escalation.id,
                 originId: escalation.origin_id ?? undefined,
-                status: 'resolved',
+                status: signalRouting.engine === 'yaml' ? 'signaled' : 'resolved',
             });
             return {
                 status: 200,

package/build/services/mcp/client/connection.d.ts

@@ -5,6 +5,19 @@ import type { LTMcpServerRecord, LTMcpToolManifest } from '../../../types';
  * when callServerTool is invoked with its name.
  */
 export declare function registerBuiltinServer(name: string, factory: () => Promise<any>): void;
+/**
+ * Dispatch a tool call directly to a built-in server's handler,
+ * bypassing MCP Client/Transport entirely. Returns null if the server
+ * or tool is not a built-in — caller should fall through to MCP transport.
+ *
+ * Each built-in server is lazily instantiated once and cached. Tool handlers
+ * are called via server._registeredTools[toolName].handler(args). This
+ * eliminates the InMemoryTransport bottleneck under concurrent load.
+ */
+export declare function dispatchBuiltinTool(serverId: string, toolName: string, args: Record<string, any>): Promise<{
+    dispatched: true;
+    result: any;
+} | null>;
 /**
  * Connect to a registered MCP server.
  * Creates the appropriate transport based on transport_type,

package/build/services/mcp/client/connection.js

@@ -34,6 +34,7 @@ var __importStar = (this && this.__importStar) || (function () {
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.registerBuiltinServer = registerBuiltinServer;
+exports.dispatchBuiltinTool = dispatchBuiltinTool;
 exports.connectToServer = connectToServer;
 exports.disconnectFromServer = disconnectFromServer;
 exports.resolveClient = resolveClient;
@@ -58,6 +59,12 @@ const clients = new Map();
  * rather than external stdio/SSE connections.
  */
 const builtinFactories = new Map();
+/**
+ * Cached built-in McpServer instances -- keyed by canonical server name.
+ * Used by dispatchBuiltinTool() to call tool handlers directly without
+ * going through MCP Client/Transport. One instance per server.
+ */
+const builtinServers = new Map();
 /**
  * Register a built-in server factory so it can be auto-connected
  * when callServerTool is invoked with its name.
@@ -65,6 +72,60 @@ const builtinFactories = new Map();
 function registerBuiltinServer(name, factory) {
     builtinFactories.set(name, factory);
 }
+/**
+ * Dispatch a tool call directly to a built-in server's handler,
+ * bypassing MCP Client/Transport entirely. Returns null if the server
+ * or tool is not a built-in — caller should fall through to MCP transport.
+ *
+ * Each built-in server is lazily instantiated once and cached. Tool handlers
+ * are called via server._registeredTools[toolName].handler(args). This
+ * eliminates the InMemoryTransport bottleneck under concurrent load.
+ */
+async function dispatchBuiltinTool(serverId, toolName, args) {
+    // Normalize and match against builtin factories
+    const norm = (s) => s.replace(/[^a-zA-Z0-9]/g, '').toLowerCase();
+    const normId = norm(serverId);
+    let matchedName = null;
+    for (const [name] of builtinFactories) {
+        const normName = norm(name);
+        if (normName === normId || normName.includes(normId) || normId.includes(normName)) {
+            matchedName = name;
+            break;
+        }
+    }
+    if (!matchedName)
+        return null;
+    // Lazily create and cache the server instance
+    if (!builtinServers.has(matchedName)) {
+        const factory = builtinFactories.get(matchedName);
+        const server = await factory();
+        builtinServers.set(matchedName, server);
+        logger_1.loggerRegistry.info(`[lt-mcp:builtin] ${matchedName} ready (direct dispatch)`);
+    }
+    const server = builtinServers.get(matchedName);
+    const tool = server._registeredTools?.[toolName];
+    if (!tool?.handler)
+        return null;
+    // Call the handler directly — no transport, no JSON-RPC.
+    // Tool handlers return MCP-shaped responses: { content: [{ type: 'text', text: '...' }] }
+    // Parse the text content the same way callServerTool does.
+    const mcpResponse = await tool.handler(args);
+    let parsed = mcpResponse;
+    if (mcpResponse && Array.isArray(mcpResponse.content)) {
+        const textContent = mcpResponse.content.find((c) => c.type === 'text');
+        if (textContent && 'text' in textContent) {
+            try {
+                parsed = JSON.parse(textContent.text);
+            }
+            catch {
+                parsed = mcpResponse.isError ? { error: textContent.text } : textContent.text;
+            }
+        }
+    }
+    const isError = parsed && typeof parsed === 'object' && 'error' in parsed;
+    logger_1.loggerRegistry.debug(`[lt-mcp:builtin] ${matchedName}/${toolName} ok=${!isError} resultKeys=[${typeof parsed === 'object' && parsed ? Object.keys(parsed).join(',') : 'raw'}]`);
+    return { dispatched: true, result: parsed };
+}
 /**
  * Connect to a registered MCP server.
  * Creates the appropriate transport based on transport_type,
@@ -330,4 +391,5 @@ async function testConnection(transportType, transportConfig) {
  */
 function clear() {
     clients.clear();
+    builtinServers.clear();
 }

package/build/services/mcp/client/tools.js

@@ -59,13 +59,8 @@ function deriveAuthFromToolContext() {
  */
 async function callServerTool(serverId, toolName, args, authContext) {
     logger_1.loggerRegistry.debug(`[lt-mcp:call] entering ${serverId}/${toolName} argKeys=[${Object.keys(args).join(',')}]`);
-    const client = await (0, connection_1.resolveClient)(serverId);
-    if (!client) {
-        throw new Error(`MCP server ${serverId} is not connected`);
-    }
-    // Resolve auth: explicit authContext > ambient ToolContext > none
+    // Resolve auth context before dispatch — both paths need it
     const resolvedAuth = authContext ?? deriveAuthFromToolContext();
-    // Inject auth context as a hidden _auth argument when available
     const toolArgs = resolvedAuth?.userId || resolvedAuth?.delegationToken
         ? { ...args, _auth: { userId: resolvedAuth.userId, token: resolvedAuth.delegationToken } }
         : args;
@@ -74,7 +69,25 @@ async function callServerTool(serverId, toolName, args, authContext) {
     if (ctx?.principal.id) {
         logger_1.loggerRegistry.debug(`[lt-mcp:audit] ${toolName} on ${serverId} by ${ctx.principal.type}:${ctx.principal.id}`);
     }
-    const result = await client.callTool({ name: toolName, arguments: toolArgs }, undefined, { timeout: defaults_1.MCP_TOOL_TIMEOUT_MS });
+    // Direct dispatch for built-in servers — bypasses MCP Client/Transport.
+    // Each built-in server is a cached singleton; tool handlers are called
+    // as plain functions. No transport contention under concurrent load.
+    const builtin = await (0, connection_1.dispatchBuiltinTool)(serverId, toolName, toolArgs);
+    if (builtin) {
+        logger_1.loggerRegistry.debug(`[lt-mcp:call] leaving ${serverId}/${toolName} (builtin) resultKeys=[${typeof builtin.result === 'object' && builtin.result ? Object.keys(builtin.result).join(',') : 'raw'}]`);
+        return builtin.result;
+    }
+    // External servers — use MCP Client/Transport with timeout guard
+    const client = await (0, connection_1.resolveClient)(serverId);
+    if (!client) {
+        throw new Error(`MCP server ${serverId} is not connected`);
+    }
+    // Guard against hung transports: the MCP SDK timeout relies on the transport
+    // to respond, which fails when InMemoryTransport is saturated under concurrency.
+    // Promise.race ensures we throw on timeout regardless of transport state.
+    const callPromise = client.callTool({ name: toolName, arguments: toolArgs }, undefined, { timeout: defaults_1.MCP_TOOL_TIMEOUT_MS });
+    const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error(`MCP tool ${serverId}/${toolName} timed out after ${defaults_1.MCP_TOOL_TIMEOUT_MS}ms`)), defaults_1.MCP_TOOL_TIMEOUT_MS));
+    const result = await Promise.race([callPromise, timeoutPromise]);
     // Extract text content from MCP response
     if (Array.isArray(result.content)) {
         const textContent = result.content.find((c) => c.type === 'text');

package/build/services/yaml-workflow/workers/register.js

@@ -40,6 +40,7 @@ const db_1 = require("../../../lib/db");
 const logger_1 = require("../../../lib/logger");
 const ephemeral_1 = require("../../iam/ephemeral");
 const mcpClient = __importStar(require("../../mcp/client"));
+const connection_1 = require("../../mcp/client/connection");
 const yamlDb = __importStar(require("../db"));
 const scope_1 = require("./scope");
 const callbacks_1 = require("./callbacks");
@@ -167,14 +168,18 @@ async function registerWorkersForWorkflow(workflow) {
             if (!serverId)
                 continue;
             const storedArgs = activity.tool_arguments;
-            const yamlHookTopic = hookTopicByEscalationTool.get(activity.activity_id);
+            // For escalate_and_wait, resolve hookTopic at runtime from the activity
+            // context — multiple escalation workers share a single registered callback,
+            // so we look up by activity_id from the incoming metadata, not the static
+            // activity captured at registration time.
+            const staticHookTopic = hookTopicByEscalationTool.get(activity.activity_id);
             // Identify keys that are wired via input_mappings. When a wired key
             // resolves to nothing (upstream step failed/returned null), we must
             // NOT fall back to stored tool_arguments — that would leak hardcoded
             // values from the original execution trace.
             const wiredKeys = new Set(Object.keys(activity.input_mappings || {}).filter(k => k !== '_scope' && k !== 'workflowName'));
             if (toolName === 'escalate_and_wait') {
-                logger_1.loggerRegistry.info(`[yaml-workflow] escalate_and_wait worker: activityId=${activity.activity_id}, hookTopic=${yamlHookTopic || 'NONE'}, mapKeys=[${[...hookTopicByEscalationTool.keys()].join(',')}]`);
+                logger_1.loggerRegistry.info(`[yaml-workflow] escalate_and_wait worker: activityId=${activity.activity_id}, hookTopic=${staticHookTopic || 'NONE'}, mapKeys=[${[...hookTopicByEscalationTool.keys()].join(',')}]`);
             }
             workerConfigs.push({
                 topic: activity.topic,
@@ -201,7 +206,13 @@ async function registerWorkersForWorkflow(workflow) {
                     }
                     logger_1.loggerRegistry.debug(`[yaml-workflow:worker] merged mcp/${toolName} wf=${wfName} mergedKeys=[${Object.keys(mergedArgs).join(',')}]`);
                     // For escalate_and_wait: inject YAML signal routing so the MCP tool
-                    // stores engine:'yaml' + hookTopic + jobId in the escalation metadata
+                    // stores engine:'yaml' + hookTopic + jobId in the escalation metadata.
+                    // Resolve hookTopic at runtime — multiple escalation workers share
+                    // this callback, so we look up by the current activity_id from metadata.
+                    const runtimeActivityId = data.metadata?.aid;
+                    const yamlHookTopic = runtimeActivityId
+                        ? hookTopicByEscalationTool.get(runtimeActivityId) || staticHookTopic
+                        : staticHookTopic;
                     if (yamlHookTopic) {
                         const jid = data.metadata?.jid;
                         mergedArgs._yaml_signal_routing = {
@@ -216,7 +227,16 @@ async function registerWorkersForWorkflow(workflow) {
                     }
                     const exchangedArgs = await (0, ephemeral_1.exchangeTokensInArgs)(mergedArgs);
                     const coercedArgs = coerceNumericObjects(exchangedArgs);
-                    const result = await mcpClient.callServerTool(serverId, toolName, coercedArgs);
+                    // Try direct dispatch for built-in servers (bypasses MCP transport).
+                    // Falls through to mcpClient.callServerTool() for external servers.
+                    let result;
+                    const builtin = await (0, connection_1.dispatchBuiltinTool)(serverId, toolName, coercedArgs);
+                    if (builtin) {
+                        result = builtin.result;
+                    }
+                    else {
+                        result = await mcpClient.callServerTool(serverId, toolName, coercedArgs);
+                    }
                     if (result && typeof result === 'object' && 'error' in result) {
                         logger_1.loggerRegistry.error(`[yaml-workflow:worker] ${toolName} error: ${JSON.stringify(result).slice(0, 200)}`);
                     }

package/docs/cloud.md

@@ -270,3 +270,126 @@ services:
 ```
 
 The combined `index.js` entry point (used in development and the demo) calls `start()` with both server and workers enabled. In production, split them into `api.js` and `worker.js` with different `start()` configs.
+
+## PostgreSQL Performance Tuning
+
+HotMesh's durable execution model is write-heavy. Every workflow creates a `jobs` row, and every field mutation creates rows in `jobs_attributes`. A simple 3-step workflow generates ~100 attribute rows per execution. At 1,000 concurrent workflows, that's 300K+ inserts in seconds.
+
+The default Postgres configuration is tuned for mixed workloads on modest hardware. For Long Tail, the write-heavy profile needs specific adjustments.
+
+### Determining Your Profile
+
+Run a baseline throughput test to understand your bottleneck:
+
+```bash
+# Submit 100 minimal workflows, measure submit rate
+time for i in $(seq 1 100); do
+  curl -s -X POST http://localhost:3000/api/workflows/basicEcho/invoke \
+    -H "Authorization: Bearer $TOKEN" \
+    -H 'Content-Type: application/json' \
+    -d '{"data":{"message":"test","sleepSeconds":0}}' > /dev/null
+done
+```
+
+Then check where Postgres is spending time:
+
+```sql
+-- Check for write pressure (high buffers_checkpoint = WAL bottleneck)
+SELECT * FROM pg_stat_bgwriter;
+
+-- Check for connection saturation
+SELECT count(*) as active, max_conn
+FROM pg_stat_activity, (SELECT setting::int as max_conn FROM pg_settings WHERE name = 'max_connections') mc
+WHERE state = 'active'
+GROUP BY max_conn;
+
+-- Check table bloat after burst writes
+SELECT relname, n_live_tup, n_dead_tup,
+       round(100.0 * n_dead_tup / nullif(n_live_tup + n_dead_tup, 0), 1) as dead_pct
+FROM pg_stat_user_tables
+WHERE n_live_tup > 1000
+ORDER BY n_dead_tup DESC LIMIT 10;
+```
+
+### Recommended Settings
+
+| Parameter | Default | Recommended | Why |
+|-----------|---------|-------------|-----|
+| `shared_buffers` | 128MB | 25% of RAM (256MB–1GB) | Cache hot pages — `jobs_attributes` partitions are read/written constantly |
+| `work_mem` | 4MB | 16MB | Workflow queries join across partitions; larger sort memory avoids disk spills |
+| `maintenance_work_mem` | 64MB | 128MB–256MB | Speeds VACUUM on large `jobs_attributes` tables after burst writes |
+| `wal_buffers` | -1 (auto) | 16MB | Write-heavy workloads saturate the default 8MB WAL buffer |
+| `max_wal_size` | 1GB | 1GB–2GB | Prevents excessive checkpointing during sustained write bursts |
+| `checkpoint_completion_target` | 0.9 | 0.9 | Spread checkpoint I/O over time — already optimal |
+| `effective_cache_size` | 4GB | 50–75% of RAM | Query planner hint — tells Postgres how much OS cache to expect |
+| `synchronous_commit` | on | off (dev/staging) | Trades durability for 2–5x write throughput. WAL is still written; only fsync is deferred. Acceptable for dev and staging. **Keep `on` in production** unless you understand the trade-off. |
+| `max_connections` | 100 | 200 | HotMesh uses connection-per-worker; concurrent workflows can exhaust 100 connections |
+
+### Docker Compose Configuration
+
+```yaml
+postgres:
+  image: postgres:16
+  command:
+    - postgres
+    - -c
+    - shared_buffers=256MB
+    - -c
+    - work_mem=16MB
+    - -c
+    - maintenance_work_mem=128MB
+    - -c
+    - wal_buffers=16MB
+    - -c
+    - max_wal_size=1GB
+    - -c
+    - checkpoint_completion_target=0.9
+    - -c
+    - effective_cache_size=512MB
+    - -c
+    - synchronous_commit=off
+    - -c
+    - max_connections=200
+  shm_size: 512m    # Required: shared_buffers > 128MB needs larger /dev/shm
+```
+
+The `shm_size` setting is critical — Docker defaults to 64MB for `/dev/shm`, but `shared_buffers=256MB` requires at least that much shared memory. Without it, Postgres will fail to start or silently fall back to smaller buffers.
+
+### Production (RDS / Cloud SQL)
+
+For managed databases, apply the same parameters through parameter groups:
+
+**AWS RDS:**
+```
+# Custom parameter group
+shared_buffers = {DBInstanceClassMemory/4}
+work_mem = 16384          # 16MB in KB
+maintenance_work_mem = 262144
+wal_buffers = 16384
+max_wal_size = 2048       # 2GB in MB
+synchronous_commit = on   # Keep on for production
+max_connections = 200
+```
+
+**GCP Cloud SQL:**
+```
+# Database flags
+shared_buffers: 25% of instance RAM (auto-tuned by Cloud SQL)
+work_mem: 16MB
+maintenance_work_mem: 256MB
+max_wal_size: 2GB
+synchronous_commit: on
+max_connections: 200
+```
+
+### Maintenance
+
+After burst workloads, dead tuples accumulate in `jobs_attributes`. Autovacuum handles this, but for large bursts (10K+ workflows), consider:
+
+```sql
+-- Manual VACUUM after a load test or batch run
+VACUUM ANALYZE durable.jobs_attributes;
+VACUUM ANALYZE durable.engine_streams;
+```
+
+Long Tail includes a built-in maintenance cron that prunes completed workflow data. Configure it via the dashboard or API to keep table sizes manageable.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/long-tail",
-  "version": "0.1.11",
+  "version": "0.1.12",
   "description": "Long Tail Workflows — Durable AI workflows with human-in-the-loop escalation. Powered by PostgreSQL.",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",
@@ -59,7 +59,7 @@
   "author": "luke.birdeau@gmail.com",
   "license": "SEE LICENSE IN LICENSE",
   "dependencies": {
-    "@anthropic-ai/sdk": "^0.82.0",
+    "@anthropic-ai/sdk": "^0.92.0",
     "@aws-sdk/client-s3": "^3.1017.0",
     "@hotmeshio/hotmesh": "^0.14.4",
     "@modelcontextprotocol/sdk": "^1.27.1",