@hotmeshio/long-tail 0.1.11 → 0.1.13

package/build/lib/db/schemas/002_seed.sql

@@ -1,6 +1,7 @@
--- Seed data: workflow configs, MCP servers, escalation chains.
+-- System seed data: built-in MCP server, system workflow configs, escalation chains.
+-- Example workflow configs are seeded at runtime when examples: true.
 
--- ─── MCP servers ────────────────────────────────────────────────────────────
+-- ─── Built-in MCP server ───────────────────────────────────────────────────
 
 INSERT INTO lt_mcp_servers (name, description, transport_type, transport_config, auto_connect, status)
 VALUES (
@@ -24,52 +25,68 @@ INSERT INTO lt_config_role_escalations (source_role, target_role) VALUES
   ('admin',     'superadmin')
 ON CONFLICT DO NOTHING;
 
--- ─── Example workflows (all directly invocable) ────────────────────────────
+-- ─── System workflow configs ────────────────────────────────────────────────
 
 INSERT INTO lt_config_workflows
-  (workflow_type, task_queue, default_role, invocable, description, tool_tags, envelope_schema, resolver_schema)
+  (workflow_type, task_queue, default_role, invocable, description, tool_tags)
 VALUES
-  -- Review content
-  ('reviewContent', 'long-tail-examples', 'reviewer', true,
-   'Content review — AI-powered moderation with human escalation for low-confidence results',
-   ARRAY['document-processing', 'vision', 'ocr', 'translation'],
-   '{"data": {"contentId": "article-001", "content": "Content to review...", "contentType": "article"}, "metadata": {"source": "dashboard"}}'::jsonb,
-   '{"approved": true, "analysis": {"confidence": 0.95, "flags": [], "summary": "Manually reviewed and approved."}}'::jsonb),
+  ('mcpQuery', 'long-tail-system', 'engineer', false,
+   'Dynamic MCP tool orchestration — LLM agentic loop with raw MCP tools',
+   '{}'),
+  ('mcpTriage', 'long-tail-system', 'engineer', false,
+   'Dynamic MCP triage — LLM agentic loop for escalation remediation',
+   '{}'),
+  ('mcpWorkflowBuilder', 'long-tail-system', 'engineer', false,
+   'Direct pipeline builder — LLM constructs DAG from tool schemas',
+   '{}'),
+  ('mcpWorkflowPlanner', 'long-tail-system', 'engineer', false,
+   'Plan mode — decomposes specifications into multi-workflow sets',
+   '{}')
+ON CONFLICT (workflow_type) DO NOTHING;
 
-  -- Verify document
-  ('verifyDocument', 'long-tail-examples', 'reviewer', true,
-   'Document verification — AI Vision analyzes identity documents',
-   ARRAY['document-processing', 'vision', 'ocr', 'translation'],
-   '{"data": {"documentId": "doc-001", "documentUrl": "https://example.com/doc.jpg", "documentType": "drivers_license", "memberId": "member-12345"}, "metadata": {"source": "dashboard"}}'::jsonb,
-   '{"memberId": "", "extractedInfo": {}, "validationResult": "match", "confidence": 1.0}'::jsonb),
+-- Query router (orchestrator entry point)
+INSERT INTO lt_config_workflows
+  (workflow_type, task_queue, default_role, invocable, description, tool_tags, envelope_schema)
+VALUES
+  ('mcpQueryRouter', 'long-tail-system', 'engineer', true,
+   'Do anything with tools — browser automation, file operations, HTTP requests, database queries, document processing, and more',
+   '{}',
+   '{"data": {"prompt": "Describe what you want to accomplish using available tools..."}, "metadata": {"source": "dashboard"}}'::jsonb)
+ON CONFLICT (workflow_type) DO NOTHING;
 
-  -- Process claim
-  ('processClaim', 'long-tail-examples', 'reviewer', true,
-   'Insurance claim processing — document analysis, validation, and human review',
-   ARRAY['document-processing', 'vision', 'database', 'query'],
-   '{"data": {"claimId": "CLM-2024-001", "claimantId": "POL-5551234", "claimType": "auto_collision", "amount": 12500, "documents": ["incident_report.pdf", "photo_evidence.jpg"]}, "metadata": {"source": "dashboard"}}'::jsonb,
-   '{"approved": true, "analysis": {"confidence": 0.92, "flags": [], "summary": "Documents reviewed and verified."}, "status": "resolved"}'::jsonb),
+-- Deterministic execution (compiled YAML workflows)
+INSERT INTO lt_config_workflows
+  (workflow_type, task_queue, default_role, invocable, description, tool_tags)
+VALUES
+  ('mcpDeterministic', 'long-tail-system', 'engineer', false,
+   'Deterministic execution — invokes matched compiled YAML workflows with extracted inputs',
+   '{}')
+ON CONFLICT (workflow_type) DO NOTHING;
 
-  -- Kitchen sink
-  ('kitchenSink', 'long-tail-examples', 'reviewer', true,
-   'Kitchen sink — demonstrates sleep, signals, parallel activities, escalation, and every durable primitive',
-   '{}',
-   '{"data": {"name": "World", "mode": "full"}, "metadata": {"source": "dashboard"}}'::jsonb,
-   NULL),
+-- Triage router
+INSERT INTO lt_config_workflows
+  (workflow_type, task_queue, default_role, invocable, description, tool_tags)
+VALUES
+  ('mcpTriageRouter', 'long-tail-system', 'engineer', false,
+   'Triage router — discovers compiled workflows for remediation, routes to deterministic or dynamic triage',
+   '{}')
+ON CONFLICT (workflow_type) DO NOTHING;
 
-  -- Basic signal (configured, NOT certified — no escalation roles)
-  ('basicSignal', 'long-tail-examples', 'reviewer', true,
-   'Signal-based escalation — workflow stays running while waiting for human input via conditionLT',
-   '{}',
-   '{"data": {"message": "Deployment approval needed for v2.1.0", "role": "reviewer"}, "metadata": {"certified": false, "source": "dashboard"}}'::jsonb,
-   '{"properties": {"approved": {"type": "boolean", "default": false, "description": "Approve this deployment?"}, "notes": {"type": "string", "default": "", "description": "Reviewer notes — visible to the workflow author"}}}'::jsonb)
+-- Triage deterministic
+INSERT INTO lt_config_workflows
+  (workflow_type, task_queue, default_role, invocable, description, tool_tags)
+VALUES
+  ('mcpTriageDeterministic', 'long-tail-system', 'engineer', false,
+   'Deterministic triage — invokes matched compiled workflows for escalation remediation',
+   '{}')
 ON CONFLICT (workflow_type) DO NOTHING;
 
--- ─── Assign roles to all workflows ──────────────────────────────────────────
+-- ─── Assign roles to all system workflows ──────────────────────────────────
 
 INSERT INTO lt_config_roles (workflow_type, role)
-SELECT workflow_type, unnest(ARRAY['reviewer', 'engineer', 'admin'])
-FROM lt_config_workflows
-WHERE workflow_type != 'basicSignal'
+SELECT wt, unnest(ARRAY['reviewer', 'engineer', 'admin'])
+FROM unnest(ARRAY[
+  'mcpQuery', 'mcpTriage', 'mcpWorkflowBuilder', 'mcpWorkflowPlanner',
+  'mcpQueryRouter', 'mcpDeterministic', 'mcpTriageRouter', 'mcpTriageDeterministic'
+]) AS wt
 ON CONFLICT (workflow_type, role) DO NOTHING;
-

package/build/services/mcp/client/connection.d.ts

@@ -5,6 +5,19 @@ import type { LTMcpServerRecord, LTMcpToolManifest } from '../../../types';
  * when callServerTool is invoked with its name.
  */
 export declare function registerBuiltinServer(name: string, factory: () => Promise<any>): void;
+/**
+ * Dispatch a tool call directly to a built-in server's handler,
+ * bypassing MCP Client/Transport entirely. Returns null if the server
+ * or tool is not a built-in — caller should fall through to MCP transport.
+ *
+ * Each built-in server is lazily instantiated once and cached. Tool handlers
+ * are called via server._registeredTools[toolName].handler(args). This
+ * eliminates the InMemoryTransport bottleneck under concurrent load.
+ */
+export declare function dispatchBuiltinTool(serverId: string, toolName: string, args: Record<string, any>): Promise<{
+    dispatched: true;
+    result: any;
+} | null>;
 /**
  * Connect to a registered MCP server.
  * Creates the appropriate transport based on transport_type,

package/build/services/mcp/client/connection.js

@@ -34,6 +34,7 @@ var __importStar = (this && this.__importStar) || (function () {
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.registerBuiltinServer = registerBuiltinServer;
+exports.dispatchBuiltinTool = dispatchBuiltinTool;
 exports.connectToServer = connectToServer;
 exports.disconnectFromServer = disconnectFromServer;
 exports.resolveClient = resolveClient;
@@ -58,6 +59,12 @@ const clients = new Map();
  * rather than external stdio/SSE connections.
  */
 const builtinFactories = new Map();
+/**
+ * Cached built-in McpServer instances -- keyed by canonical server name.
+ * Used by dispatchBuiltinTool() to call tool handlers directly without
+ * going through MCP Client/Transport. One instance per server.
+ */
+const builtinServers = new Map();
 /**
  * Register a built-in server factory so it can be auto-connected
  * when callServerTool is invoked with its name.
@@ -65,6 +72,60 @@ const builtinFactories = new Map();
 function registerBuiltinServer(name, factory) {
     builtinFactories.set(name, factory);
 }
+/**
+ * Dispatch a tool call directly to a built-in server's handler,
+ * bypassing MCP Client/Transport entirely. Returns null if the server
+ * or tool is not a built-in — caller should fall through to MCP transport.
+ *
+ * Each built-in server is lazily instantiated once and cached. Tool handlers
+ * are called via server._registeredTools[toolName].handler(args). This
+ * eliminates the InMemoryTransport bottleneck under concurrent load.
+ */
+async function dispatchBuiltinTool(serverId, toolName, args) {
+    // Normalize and match against builtin factories
+    const norm = (s) => s.replace(/[^a-zA-Z0-9]/g, '').toLowerCase();
+    const normId = norm(serverId);
+    let matchedName = null;
+    for (const [name] of builtinFactories) {
+        const normName = norm(name);
+        if (normName === normId || normName.includes(normId) || normId.includes(normName)) {
+            matchedName = name;
+            break;
+        }
+    }
+    if (!matchedName)
+        return null;
+    // Lazily create and cache the server instance
+    if (!builtinServers.has(matchedName)) {
+        const factory = builtinFactories.get(matchedName);
+        const server = await factory();
+        builtinServers.set(matchedName, server);
+        logger_1.loggerRegistry.info(`[lt-mcp:builtin] ${matchedName} ready (direct dispatch)`);
+    }
+    const server = builtinServers.get(matchedName);
+    const tool = server._registeredTools?.[toolName];
+    if (!tool?.handler)
+        return null;
+    // Call the handler directly — no transport, no JSON-RPC.
+    // Tool handlers return MCP-shaped responses: { content: [{ type: 'text', text: '...' }] }
+    // Parse the text content the same way callServerTool does.
+    const mcpResponse = await tool.handler(args);
+    let parsed = mcpResponse;
+    if (mcpResponse && Array.isArray(mcpResponse.content)) {
+        const textContent = mcpResponse.content.find((c) => c.type === 'text');
+        if (textContent && 'text' in textContent) {
+            try {
+                parsed = JSON.parse(textContent.text);
+            }
+            catch {
+                parsed = mcpResponse.isError ? { error: textContent.text } : textContent.text;
+            }
+        }
+    }
+    const isError = parsed && typeof parsed === 'object' && 'error' in parsed;
+    logger_1.loggerRegistry.debug(`[lt-mcp:builtin] ${matchedName}/${toolName} ok=${!isError} resultKeys=[${typeof parsed === 'object' && parsed ? Object.keys(parsed).join(',') : 'raw'}]`);
+    return { dispatched: true, result: parsed };
+}
 /**
  * Connect to a registered MCP server.
  * Creates the appropriate transport based on transport_type,
@@ -330,4 +391,5 @@ async function testConnection(transportType, transportConfig) {
  */
 function clear() {
     clients.clear();
+    builtinServers.clear();
 }

package/build/services/mcp/client/tools.js

@@ -59,13 +59,8 @@ function deriveAuthFromToolContext() {
  */
 async function callServerTool(serverId, toolName, args, authContext) {
     logger_1.loggerRegistry.debug(`[lt-mcp:call] entering ${serverId}/${toolName} argKeys=[${Object.keys(args).join(',')}]`);
-    const client = await (0, connection_1.resolveClient)(serverId);
-    if (!client) {
-        throw new Error(`MCP server ${serverId} is not connected`);
-    }
-    // Resolve auth: explicit authContext > ambient ToolContext > none
+    // Resolve auth context before dispatch — both paths need it
     const resolvedAuth = authContext ?? deriveAuthFromToolContext();
-    // Inject auth context as a hidden _auth argument when available
     const toolArgs = resolvedAuth?.userId || resolvedAuth?.delegationToken
         ? { ...args, _auth: { userId: resolvedAuth.userId, token: resolvedAuth.delegationToken } }
         : args;
@@ -74,7 +69,25 @@ async function callServerTool(serverId, toolName, args, authContext) {
     if (ctx?.principal.id) {
         logger_1.loggerRegistry.debug(`[lt-mcp:audit] ${toolName} on ${serverId} by ${ctx.principal.type}:${ctx.principal.id}`);
     }
-    const result = await client.callTool({ name: toolName, arguments: toolArgs }, undefined, { timeout: defaults_1.MCP_TOOL_TIMEOUT_MS });
+    // Direct dispatch for built-in servers — bypasses MCP Client/Transport.
+    // Each built-in server is a cached singleton; tool handlers are called
+    // as plain functions. No transport contention under concurrent load.
+    const builtin = await (0, connection_1.dispatchBuiltinTool)(serverId, toolName, toolArgs);
+    if (builtin) {
+        logger_1.loggerRegistry.debug(`[lt-mcp:call] leaving ${serverId}/${toolName} (builtin) resultKeys=[${typeof builtin.result === 'object' && builtin.result ? Object.keys(builtin.result).join(',') : 'raw'}]`);
+        return builtin.result;
+    }
+    // External servers — use MCP Client/Transport with timeout guard
+    const client = await (0, connection_1.resolveClient)(serverId);
+    if (!client) {
+        throw new Error(`MCP server ${serverId} is not connected`);
+    }
+    // Guard against hung transports: the MCP SDK timeout relies on the transport
+    // to respond, which fails when InMemoryTransport is saturated under concurrency.
+    // Promise.race ensures we throw on timeout regardless of transport state.
+    const callPromise = client.callTool({ name: toolName, arguments: toolArgs }, undefined, { timeout: defaults_1.MCP_TOOL_TIMEOUT_MS });
+    const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error(`MCP tool ${serverId}/${toolName} timed out after ${defaults_1.MCP_TOOL_TIMEOUT_MS}ms`)), defaults_1.MCP_TOOL_TIMEOUT_MS));
+    const result = await Promise.race([callPromise, timeoutPromise]);
     // Extract text content from MCP response
     if (Array.isArray(result.content)) {
         const textContent = result.content.find((c) => c.type === 'text');

package/build/services/mcp/server.js

@@ -64,6 +64,10 @@ const claimAndResolveSchema = zod_1.z.object({
     resolver_id: zod_1.z.string().describe('Identifier for who/what is resolving'),
     payload: zod_1.z.record(zod_1.z.any()).describe('Resolution payload data'),
 });
+const resolveEscalationSchema = zod_1.z.object({
+    escalation_id: zod_1.z.string().describe('The escalation ID to resolve'),
+    payload: zod_1.z.record(zod_1.z.any()).describe('Resolution payload data'),
+});
 const escalateAndWaitSchema = zod_1.z.object({
     role: zod_1.z.string().describe('Target role for the escalation (e.g., "reviewer")'),
     message: zod_1.z.string().describe('Description of what input is needed from the human'),
@@ -218,6 +222,33 @@ async function createHumanQueueServer(options) {
                 }],
         };
     });
+    // ── resolve_escalation ──────────────────────────────────────────────
+    server.registerTool('resolve_escalation', {
+        title: 'Resolve Escalation',
+        description: 'Resolve an already-claimed escalation with a payload. Use when the claim happened externally (e.g. via API).',
+        inputSchema: resolveEscalationSchema,
+    }, async (args) => {
+        const resolved = await escalationService.resolveEscalation(args.escalation_id, args.payload);
+        if (!resolved) {
+            return {
+                content: [{
+                        type: 'text',
+                        text: JSON.stringify({ error: 'Failed to resolve escalation' }),
+                    }],
+                isError: true,
+            };
+        }
+        return {
+            content: [{
+                    type: 'text',
+                    text: JSON.stringify({
+                        escalation_id: resolved.id,
+                        status: resolved.status,
+                        resolved_at: resolved.resolved_at,
+                    }),
+                }],
+        };
+    });
     // ── escalate_and_wait ──────────────────────────────────────────────
     server.registerTool('escalate_and_wait', {
         title: 'Escalate and Wait',

package/build/services/yaml-workflow/workers/register.js

@@ -40,6 +40,7 @@ const db_1 = require("../../../lib/db");
 const logger_1 = require("../../../lib/logger");
 const ephemeral_1 = require("../../iam/ephemeral");
 const mcpClient = __importStar(require("../../mcp/client"));
+const connection_1 = require("../../mcp/client/connection");
 const yamlDb = __importStar(require("../db"));
 const scope_1 = require("./scope");
 const callbacks_1 = require("./callbacks");
@@ -167,14 +168,18 @@ async function registerWorkersForWorkflow(workflow) {
             if (!serverId)
                 continue;
             const storedArgs = activity.tool_arguments;
-            const yamlHookTopic = hookTopicByEscalationTool.get(activity.activity_id);
+            // For escalate_and_wait, resolve hookTopic at runtime from the activity
+            // context — multiple escalation workers share a single registered callback,
+            // so we look up by activity_id from the incoming metadata, not the static
+            // activity captured at registration time.
+            const staticHookTopic = hookTopicByEscalationTool.get(activity.activity_id);
             // Identify keys that are wired via input_mappings. When a wired key
             // resolves to nothing (upstream step failed/returned null), we must
             // NOT fall back to stored tool_arguments — that would leak hardcoded
             // values from the original execution trace.
             const wiredKeys = new Set(Object.keys(activity.input_mappings || {}).filter(k => k !== '_scope' && k !== 'workflowName'));
             if (toolName === 'escalate_and_wait') {
-                logger_1.loggerRegistry.info(`[yaml-workflow] escalate_and_wait worker: activityId=${activity.activity_id}, hookTopic=${yamlHookTopic || 'NONE'}, mapKeys=[${[...hookTopicByEscalationTool.keys()].join(',')}]`);
+                logger_1.loggerRegistry.info(`[yaml-workflow] escalate_and_wait worker: activityId=${activity.activity_id}, hookTopic=${staticHookTopic || 'NONE'}, mapKeys=[${[...hookTopicByEscalationTool.keys()].join(',')}]`);
             }
             workerConfigs.push({
                 topic: activity.topic,
@@ -201,7 +206,13 @@ async function registerWorkersForWorkflow(workflow) {
                     }
                     logger_1.loggerRegistry.debug(`[yaml-workflow:worker] merged mcp/${toolName} wf=${wfName} mergedKeys=[${Object.keys(mergedArgs).join(',')}]`);
                     // For escalate_and_wait: inject YAML signal routing so the MCP tool
-                    // stores engine:'yaml' + hookTopic + jobId in the escalation metadata
+                    // stores engine:'yaml' + hookTopic + jobId in the escalation metadata.
+                    // Resolve hookTopic at runtime — multiple escalation workers share
+                    // this callback, so we look up by the current activity_id from metadata.
+                    const runtimeActivityId = data.metadata?.aid;
+                    const yamlHookTopic = runtimeActivityId
+                        ? hookTopicByEscalationTool.get(runtimeActivityId) || staticHookTopic
+                        : staticHookTopic;
                     if (yamlHookTopic) {
                         const jid = data.metadata?.jid;
                         mergedArgs._yaml_signal_routing = {
@@ -216,7 +227,16 @@ async function registerWorkersForWorkflow(workflow) {
                     }
                     const exchangedArgs = await (0, ephemeral_1.exchangeTokensInArgs)(mergedArgs);
                     const coercedArgs = coerceNumericObjects(exchangedArgs);
-                    const result = await mcpClient.callServerTool(serverId, toolName, coercedArgs);
+                    // Try direct dispatch for built-in servers (bypasses MCP transport).
+                    // Falls through to mcpClient.callServerTool() for external servers.
+                    let result;
+                    const builtin = await (0, connection_1.dispatchBuiltinTool)(serverId, toolName, coercedArgs);
+                    if (builtin) {
+                        result = builtin.result;
+                    }
+                    else {
+                        result = await mcpClient.callServerTool(serverId, toolName, coercedArgs);
+                    }
                     if (result && typeof result === 'object' && 'error' in result) {
                         logger_1.loggerRegistry.error(`[yaml-workflow:worker] ${toolName} error: ${JSON.stringify(result).slice(0, 200)}`);
                     }

package/build/system/mcp-servers/human-queue.js

@@ -64,6 +64,10 @@ const claimAndResolveSchema = zod_1.z.object({
     resolver_id: zod_1.z.string().describe('Identifier for who/what is resolving'),
     payload: zod_1.z.record(zod_1.z.any()).describe('Resolution payload data'),
 });
+const resolveEscalationSchema = zod_1.z.object({
+    escalation_id: zod_1.z.string().describe('The escalation ID to resolve'),
+    payload: zod_1.z.record(zod_1.z.any()).describe('Resolution payload data'),
+});
 const escalateAndWaitSchema = zod_1.z.object({
     role: zod_1.z.string().describe('Target role for the escalation (e.g., "reviewer")'),
     message: zod_1.z.string().describe('Description of what input is needed from the human'),
@@ -218,6 +222,33 @@ async function createHumanQueueServer(options) {
                 }],
         };
     });
+    // ── resolve_escalation ──────────────────────────────────────────────
+    server.registerTool('resolve_escalation', {
+        title: 'Resolve Escalation',
+        description: 'Resolve an already-claimed escalation with a payload. Use when the claim happened externally (e.g. via API).',
+        inputSchema: resolveEscalationSchema,
+    }, async (args) => {
+        const resolved = await escalationService.resolveEscalation(args.escalation_id, args.payload);
+        if (!resolved) {
+            return {
+                content: [{
+                        type: 'text',
+                        text: JSON.stringify({ error: 'Failed to resolve escalation' }),
+                    }],
+                isError: true,
+            };
+        }
+        return {
+            content: [{
+                    type: 'text',
+                    text: JSON.stringify({
+                        escalation_id: resolved.id,
+                        status: resolved.status,
+                        resolved_at: resolved.resolved_at,
+                    }),
+                }],
+        };
+    });
     // ── escalate_and_wait ──────────────────────────────────────────────
     server.registerTool('escalate_and_wait', {
         title: 'Escalate and Wait',

package/docs/cloud.md

@@ -270,3 +270,126 @@ services:
 ```
 
 The combined `index.js` entry point (used in development and the demo) calls `start()` with both server and workers enabled. In production, split them into `api.js` and `worker.js` with different `start()` configs.
+
+## PostgreSQL Performance Tuning
+
+HotMesh's durable execution model is write-heavy. Every workflow creates a `jobs` row, and every field mutation creates rows in `jobs_attributes`. A simple 3-step workflow generates ~100 attribute rows per execution. At 1,000 concurrent workflows, that's 300K+ inserts in seconds.
+
+The default Postgres configuration is tuned for mixed workloads on modest hardware. For Long Tail, the write-heavy profile needs specific adjustments.
+
+### Determining Your Profile
+
+Run a baseline throughput test to understand your bottleneck:
+
+```bash
+# Submit 100 minimal workflows, measure submit rate
+time for i in $(seq 1 100); do
+  curl -s -X POST http://localhost:3000/api/workflows/basicEcho/invoke \
+    -H "Authorization: Bearer $TOKEN" \
+    -H 'Content-Type: application/json' \
+    -d '{"data":{"message":"test","sleepSeconds":0}}' > /dev/null
+done
+```
+
+Then check where Postgres is spending time:
+
+```sql
+-- Check for write pressure (high buffers_checkpoint = WAL bottleneck)
+SELECT * FROM pg_stat_bgwriter;
+
+-- Check for connection saturation
+SELECT count(*) as active, max_conn
+FROM pg_stat_activity, (SELECT setting::int as max_conn FROM pg_settings WHERE name = 'max_connections') mc
+WHERE state = 'active'
+GROUP BY max_conn;
+
+-- Check table bloat after burst writes
+SELECT relname, n_live_tup, n_dead_tup,
+       round(100.0 * n_dead_tup / nullif(n_live_tup + n_dead_tup, 0), 1) as dead_pct
+FROM pg_stat_user_tables
+WHERE n_live_tup > 1000
+ORDER BY n_dead_tup DESC LIMIT 10;
+```
+
+### Recommended Settings
+
+| Parameter | Default | Recommended | Why |
+|-----------|---------|-------------|-----|
+| `shared_buffers` | 128MB | 25% of RAM (256MB–1GB) | Cache hot pages — `jobs_attributes` partitions are read/written constantly |
+| `work_mem` | 4MB | 16MB | Workflow queries join across partitions; larger sort memory avoids disk spills |
+| `maintenance_work_mem` | 64MB | 128MB–256MB | Speeds VACUUM on large `jobs_attributes` tables after burst writes |
+| `wal_buffers` | -1 (auto) | 16MB | Write-heavy workloads saturate the default 8MB WAL buffer |
+| `max_wal_size` | 1GB | 1GB–2GB | Prevents excessive checkpointing during sustained write bursts |
+| `checkpoint_completion_target` | 0.9 | 0.9 | Spread checkpoint I/O over time — already optimal |
+| `effective_cache_size` | 4GB | 50–75% of RAM | Query planner hint — tells Postgres how much OS cache to expect |
+| `synchronous_commit` | on | off (dev/staging) | Trades durability for 2–5x write throughput. WAL is still written; only fsync is deferred. Acceptable for dev and staging. **Keep `on` in production** unless you understand the trade-off. |
+| `max_connections` | 100 | 200 | HotMesh uses connection-per-worker; concurrent workflows can exhaust 100 connections |
+
+### Docker Compose Configuration
+
+```yaml
+postgres:
+  image: postgres:16
+  command:
+    - postgres
+    - -c
+    - shared_buffers=256MB
+    - -c
+    - work_mem=16MB
+    - -c
+    - maintenance_work_mem=128MB
+    - -c
+    - wal_buffers=16MB
+    - -c
+    - max_wal_size=1GB
+    - -c
+    - checkpoint_completion_target=0.9
+    - -c
+    - effective_cache_size=512MB
+    - -c
+    - synchronous_commit=off
+    - -c
+    - max_connections=200
+  shm_size: 512m    # Required: shared_buffers > 128MB needs larger /dev/shm
+```
+
+The `shm_size` setting is critical — Docker defaults to 64MB for `/dev/shm`, but `shared_buffers=256MB` requires at least that much shared memory. Without it, Postgres will fail to start or silently fall back to smaller buffers.
+
+### Production (RDS / Cloud SQL)
+
+For managed databases, apply the same parameters through parameter groups:
+
+**AWS RDS:**
+```
+# Custom parameter group
+shared_buffers = {DBInstanceClassMemory/4}
+work_mem = 16384          # 16MB in KB
+maintenance_work_mem = 262144
+wal_buffers = 16384
+max_wal_size = 2048       # 2GB in MB
+synchronous_commit = on   # Keep on for production
+max_connections = 200
+```
+
+**GCP Cloud SQL:**
+```
+# Database flags
+shared_buffers: 25% of instance RAM (auto-tuned by Cloud SQL)
+work_mem: 16MB
+maintenance_work_mem: 256MB
+max_wal_size: 2GB
+synchronous_commit: on
+max_connections: 200
+```
+
+### Maintenance
+
+After burst workloads, dead tuples accumulate in `jobs_attributes`. Autovacuum handles this, but for large bursts (10K+ workflows), consider:
+
+```sql
+-- Manual VACUUM after a load test or batch run
+VACUUM ANALYZE durable.jobs_attributes;
+VACUUM ANALYZE durable.engine_streams;
+```
+
+Long Tail includes a built-in maintenance cron that prunes completed workflow data. Configure it via the dashboard or API to keep table sizes manageable.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/long-tail",
-  "version": "0.1.11",
+  "version": "0.1.13",
   "description": "Long Tail Workflows — Durable AI workflows with human-in-the-loop escalation. Powered by PostgreSQL.",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",
@@ -59,9 +59,9 @@
   "author": "luke.birdeau@gmail.com",
   "license": "SEE LICENSE IN LICENSE",
   "dependencies": {
-    "@anthropic-ai/sdk": "^0.82.0",
+    "@anthropic-ai/sdk": "^0.92.0",
     "@aws-sdk/client-s3": "^3.1017.0",
-    "@hotmeshio/hotmesh": "^0.14.4",
+    "@hotmeshio/hotmesh": "^0.14.7",
     "@modelcontextprotocol/sdk": "^1.27.1",
     "@opentelemetry/exporter-trace-otlp-proto": "^0.215.0",
     "@opentelemetry/resources": "^2.5.1",

package/build/lib/db/schemas/003_workflow_discovery.sql

@@ -1,39 +0,0 @@
--- Workflow discovery: full-text search, original prompt, and category.
-
--- Original prompt that spawned this workflow (richest semantic signal)
-ALTER TABLE lt_yaml_workflows ADD COLUMN IF NOT EXISTS original_prompt TEXT;
-
--- Capability category derived from tool usage patterns
-ALTER TABLE lt_yaml_workflows ADD COLUMN IF NOT EXISTS category TEXT;
-
--- Full-text search vector, auto-maintained by trigger
-ALTER TABLE lt_yaml_workflows ADD COLUMN IF NOT EXISTS search_vector TSVECTOR;
-
--- GIN index on search_vector for fast full-text search
-CREATE INDEX IF NOT EXISTS idx_lt_yaml_workflows_search
-  ON lt_yaml_workflows USING GIN (search_vector);
-
--- Index on category for filtered queries
-CREATE INDEX IF NOT EXISTS idx_lt_yaml_workflows_category
-  ON lt_yaml_workflows (category) WHERE category IS NOT NULL;
-
--- Trigger: rebuild search_vector from name, description, tags, original_prompt, category
-CREATE OR REPLACE FUNCTION lt_yaml_workflows_search_vector_update()
-RETURNS TRIGGER AS $$
-BEGIN
-  NEW.search_vector :=
-    setweight(to_tsvector('english', coalesce(NEW.name, '')), 'A') ||
-    setweight(to_tsvector('english', coalesce(NEW.original_prompt, '')), 'A') ||
-    setweight(to_tsvector('english', coalesce(NEW.description, '')), 'B') ||
-    setweight(to_tsvector('english', coalesce(NEW.category, '')), 'C') ||
-    setweight(to_tsvector('english', coalesce(array_to_string(NEW.tags, ' '), '')), 'C');
-  RETURN NEW;
-END;
-$$ LANGUAGE plpgsql;
-
-CREATE OR REPLACE TRIGGER trg_lt_yaml_workflows_search_vector
-  BEFORE INSERT OR UPDATE ON lt_yaml_workflows
-  FOR EACH ROW EXECUTE FUNCTION lt_yaml_workflows_search_vector_update();
-
--- Backfill search_vector for existing rows
-UPDATE lt_yaml_workflows SET updated_at = NOW();

package/build/lib/db/schemas/004_query_router.sql

@@ -1,38 +0,0 @@
--- Split mcpQuery into router + dynamic + deterministic workflows.
--- mcpQueryRouter is the new entry point (orchestrator).
--- mcpQuery becomes dynamic-only (leaf).
--- mcpDeterministic invokes compiled YAML workflows (leaf).
-
--- Update existing mcpQuery: no longer directly invocable (called via router)
-UPDATE lt_config_workflows
-SET invocable = false,
-    description = 'Dynamic MCP tool orchestration — LLM agentic loop with raw MCP tools'
-WHERE workflow_type = 'mcpQuery';
-
--- Add mcpQueryRouter (orchestrator — the new entry point)
-INSERT INTO lt_config_workflows
-  (workflow_type, task_queue, default_role, invocable, description, tool_tags, envelope_schema)
-VALUES
-  ('mcpQueryRouter', 'long-tail-system', 'engineer', true,
-   'Do anything with tools — browser automation, file operations, HTTP requests, database queries, document processing, and more',
-   '{}',
-   '{"data": {"prompt": "Describe what you want to accomplish using available tools..."}, "metadata": {"source": "dashboard"}}'::jsonb)
-ON CONFLICT (workflow_type) DO NOTHING;
-
--- Add mcpDeterministic (leaf — invokes compiled YAML workflows)
-INSERT INTO lt_config_workflows
-  (workflow_type, task_queue, default_role, invocable, description, tool_tags)
-VALUES
-  ('mcpDeterministic', 'long-tail-system', 'engineer', false,
-   'Deterministic execution — invokes matched compiled YAML workflows with extracted inputs',
-   '{}')
-ON CONFLICT (workflow_type) DO NOTHING;
-
--- Assign roles
-INSERT INTO lt_config_roles (workflow_type, role)
-SELECT 'mcpQueryRouter', unnest(ARRAY['reviewer', 'engineer', 'admin'])
-ON CONFLICT (workflow_type, role) DO NOTHING;
-
-INSERT INTO lt_config_roles (workflow_type, role)
-SELECT 'mcpDeterministic', unnest(ARRAY['reviewer', 'engineer', 'admin'])
-ON CONFLICT (workflow_type, role) DO NOTHING;