@hotmeshio/long-tail 0.1.10 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/api/escalations.js +7 -2
- package/build/services/mcp/client/connection.d.ts +13 -0
- package/build/services/mcp/client/connection.js +68 -1
- package/build/services/mcp/client/tools.js +20 -7
- package/build/services/yaml-workflow/workers/register.js +46 -4
- package/build/system/workflows/mcp-workflow-builder/prompts.js +10 -9
- package/docs/cloud.md +123 -0
- package/package.json +3 -3
package/build/api/escalations.js
CHANGED
|
@@ -730,7 +730,12 @@ async function resolveEscalation(input, auth) {
|
|
|
730
730
|
const handle = await client.workflow.getHandle(signalRouting.taskQueue, signalRouting.workflowType, signalRouting.workflowId);
|
|
731
731
|
await handle.signal(signalRouting.signalId, signalPayload);
|
|
732
732
|
}
|
|
733
|
-
|
|
733
|
+
// For YAML workflows, the resolve worker inside the workflow calls
|
|
734
|
+
// claim_and_resolve to close the escalation transactionally. Only resolve
|
|
735
|
+
// here for Durable workflows that lack an in-workflow resolve step.
|
|
736
|
+
if (signalRouting.engine !== 'yaml') {
|
|
737
|
+
await escalationService.resolveEscalation(escalation.id, resolverPayload);
|
|
738
|
+
}
|
|
734
739
|
(0, publish_1.publishEscalationEvent)({
|
|
735
740
|
type: 'escalation.resolved',
|
|
736
741
|
source: 'api',
|
|
@@ -740,7 +745,7 @@ async function resolveEscalation(input, auth) {
|
|
|
740
745
|
taskId: escalation.task_id,
|
|
741
746
|
escalationId: escalation.id,
|
|
742
747
|
originId: escalation.origin_id ?? undefined,
|
|
743
|
-
status: 'resolved',
|
|
748
|
+
status: signalRouting.engine === 'yaml' ? 'signaled' : 'resolved',
|
|
744
749
|
});
|
|
745
750
|
return {
|
|
746
751
|
status: 200,
|
|
@@ -5,6 +5,19 @@ import type { LTMcpServerRecord, LTMcpToolManifest } from '../../../types';
|
|
|
5
5
|
* when callServerTool is invoked with its name.
|
|
6
6
|
*/
|
|
7
7
|
export declare function registerBuiltinServer(name: string, factory: () => Promise<any>): void;
|
|
8
|
+
/**
|
|
9
|
+
* Dispatch a tool call directly to a built-in server's handler,
|
|
10
|
+
* bypassing MCP Client/Transport entirely. Returns null if the server
|
|
11
|
+
* or tool is not a built-in — caller should fall through to MCP transport.
|
|
12
|
+
*
|
|
13
|
+
* Each built-in server is lazily instantiated once and cached. Tool handlers
|
|
14
|
+
* are called via server._registeredTools[toolName].handler(args). This
|
|
15
|
+
* eliminates the InMemoryTransport bottleneck under concurrent load.
|
|
16
|
+
*/
|
|
17
|
+
export declare function dispatchBuiltinTool(serverId: string, toolName: string, args: Record<string, any>): Promise<{
|
|
18
|
+
dispatched: true;
|
|
19
|
+
result: any;
|
|
20
|
+
} | null>;
|
|
8
21
|
/**
|
|
9
22
|
* Connect to a registered MCP server.
|
|
10
23
|
* Creates the appropriate transport based on transport_type,
|
|
@@ -34,6 +34,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
36
|
exports.registerBuiltinServer = registerBuiltinServer;
|
|
37
|
+
exports.dispatchBuiltinTool = dispatchBuiltinTool;
|
|
37
38
|
exports.connectToServer = connectToServer;
|
|
38
39
|
exports.disconnectFromServer = disconnectFromServer;
|
|
39
40
|
exports.resolveClient = resolveClient;
|
|
@@ -58,6 +59,12 @@ const clients = new Map();
|
|
|
58
59
|
* rather than external stdio/SSE connections.
|
|
59
60
|
*/
|
|
60
61
|
const builtinFactories = new Map();
|
|
62
|
+
/**
|
|
63
|
+
* Cached built-in McpServer instances -- keyed by canonical server name.
|
|
64
|
+
* Used by dispatchBuiltinTool() to call tool handlers directly without
|
|
65
|
+
* going through MCP Client/Transport. One instance per server.
|
|
66
|
+
*/
|
|
67
|
+
const builtinServers = new Map();
|
|
61
68
|
/**
|
|
62
69
|
* Register a built-in server factory so it can be auto-connected
|
|
63
70
|
* when callServerTool is invoked with its name.
|
|
@@ -65,6 +72,60 @@ const builtinFactories = new Map();
|
|
|
65
72
|
function registerBuiltinServer(name, factory) {
|
|
66
73
|
builtinFactories.set(name, factory);
|
|
67
74
|
}
|
|
75
|
+
/**
|
|
76
|
+
* Dispatch a tool call directly to a built-in server's handler,
|
|
77
|
+
* bypassing MCP Client/Transport entirely. Returns null if the server
|
|
78
|
+
* or tool is not a built-in — caller should fall through to MCP transport.
|
|
79
|
+
*
|
|
80
|
+
* Each built-in server is lazily instantiated once and cached. Tool handlers
|
|
81
|
+
* are called via server._registeredTools[toolName].handler(args). This
|
|
82
|
+
* eliminates the InMemoryTransport bottleneck under concurrent load.
|
|
83
|
+
*/
|
|
84
|
+
async function dispatchBuiltinTool(serverId, toolName, args) {
|
|
85
|
+
// Normalize and match against builtin factories
|
|
86
|
+
const norm = (s) => s.replace(/[^a-zA-Z0-9]/g, '').toLowerCase();
|
|
87
|
+
const normId = norm(serverId);
|
|
88
|
+
let matchedName = null;
|
|
89
|
+
for (const [name] of builtinFactories) {
|
|
90
|
+
const normName = norm(name);
|
|
91
|
+
if (normName === normId || normName.includes(normId) || normId.includes(normName)) {
|
|
92
|
+
matchedName = name;
|
|
93
|
+
break;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
if (!matchedName)
|
|
97
|
+
return null;
|
|
98
|
+
// Lazily create and cache the server instance
|
|
99
|
+
if (!builtinServers.has(matchedName)) {
|
|
100
|
+
const factory = builtinFactories.get(matchedName);
|
|
101
|
+
const server = await factory();
|
|
102
|
+
builtinServers.set(matchedName, server);
|
|
103
|
+
logger_1.loggerRegistry.info(`[lt-mcp:builtin] ${matchedName} ready (direct dispatch)`);
|
|
104
|
+
}
|
|
105
|
+
const server = builtinServers.get(matchedName);
|
|
106
|
+
const tool = server._registeredTools?.[toolName];
|
|
107
|
+
if (!tool?.handler)
|
|
108
|
+
return null;
|
|
109
|
+
// Call the handler directly — no transport, no JSON-RPC.
|
|
110
|
+
// Tool handlers return MCP-shaped responses: { content: [{ type: 'text', text: '...' }] }
|
|
111
|
+
// Parse the text content the same way callServerTool does.
|
|
112
|
+
const mcpResponse = await tool.handler(args);
|
|
113
|
+
let parsed = mcpResponse;
|
|
114
|
+
if (mcpResponse && Array.isArray(mcpResponse.content)) {
|
|
115
|
+
const textContent = mcpResponse.content.find((c) => c.type === 'text');
|
|
116
|
+
if (textContent && 'text' in textContent) {
|
|
117
|
+
try {
|
|
118
|
+
parsed = JSON.parse(textContent.text);
|
|
119
|
+
}
|
|
120
|
+
catch {
|
|
121
|
+
parsed = mcpResponse.isError ? { error: textContent.text } : textContent.text;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
const isError = parsed && typeof parsed === 'object' && 'error' in parsed;
|
|
126
|
+
logger_1.loggerRegistry.debug(`[lt-mcp:builtin] ${matchedName}/${toolName} ok=${!isError} resultKeys=[${typeof parsed === 'object' && parsed ? Object.keys(parsed).join(',') : 'raw'}]`);
|
|
127
|
+
return { dispatched: true, result: parsed };
|
|
128
|
+
}
|
|
68
129
|
/**
|
|
69
130
|
* Connect to a registered MCP server.
|
|
70
131
|
* Creates the appropriate transport based on transport_type,
|
|
@@ -160,13 +221,18 @@ async function resolveClient(serverId) {
|
|
|
160
221
|
if (clients.has(serverId))
|
|
161
222
|
return clients.get(serverId);
|
|
162
223
|
// 2. Check built-in server factories -- exact match first, then fuzzy
|
|
224
|
+
// Normalize strips non-alphanumeric chars so hyphens and underscores
|
|
225
|
+
// match (e.g., "long_tail_vision" matches "long-tail-vision").
|
|
226
|
+
const norm = (s) => s.replace(/[^a-zA-Z0-9]/g, '').toLowerCase();
|
|
163
227
|
let matchedName = null;
|
|
164
228
|
if (builtinFactories.has(serverId)) {
|
|
165
229
|
matchedName = serverId;
|
|
166
230
|
}
|
|
167
231
|
else {
|
|
232
|
+
const normId = norm(serverId);
|
|
168
233
|
for (const [name] of builtinFactories) {
|
|
169
|
-
|
|
234
|
+
const normName = norm(name);
|
|
235
|
+
if (normName.includes(normId) || normId.includes(normName)) {
|
|
170
236
|
matchedName = name;
|
|
171
237
|
break;
|
|
172
238
|
}
|
|
@@ -325,4 +391,5 @@ async function testConnection(transportType, transportConfig) {
|
|
|
325
391
|
*/
|
|
326
392
|
function clear() {
|
|
327
393
|
clients.clear();
|
|
394
|
+
builtinServers.clear();
|
|
328
395
|
}
|
|
@@ -59,13 +59,8 @@ function deriveAuthFromToolContext() {
|
|
|
59
59
|
*/
|
|
60
60
|
async function callServerTool(serverId, toolName, args, authContext) {
|
|
61
61
|
logger_1.loggerRegistry.debug(`[lt-mcp:call] entering ${serverId}/${toolName} argKeys=[${Object.keys(args).join(',')}]`);
|
|
62
|
-
|
|
63
|
-
if (!client) {
|
|
64
|
-
throw new Error(`MCP server ${serverId} is not connected`);
|
|
65
|
-
}
|
|
66
|
-
// Resolve auth: explicit authContext > ambient ToolContext > none
|
|
62
|
+
// Resolve auth context before dispatch — both paths need it
|
|
67
63
|
const resolvedAuth = authContext ?? deriveAuthFromToolContext();
|
|
68
|
-
// Inject auth context as a hidden _auth argument when available
|
|
69
64
|
const toolArgs = resolvedAuth?.userId || resolvedAuth?.delegationToken
|
|
70
65
|
? { ...args, _auth: { userId: resolvedAuth.userId, token: resolvedAuth.delegationToken } }
|
|
71
66
|
: args;
|
|
@@ -74,7 +69,25 @@ async function callServerTool(serverId, toolName, args, authContext) {
|
|
|
74
69
|
if (ctx?.principal.id) {
|
|
75
70
|
logger_1.loggerRegistry.debug(`[lt-mcp:audit] ${toolName} on ${serverId} by ${ctx.principal.type}:${ctx.principal.id}`);
|
|
76
71
|
}
|
|
77
|
-
|
|
72
|
+
// Direct dispatch for built-in servers — bypasses MCP Client/Transport.
|
|
73
|
+
// Each built-in server is a cached singleton; tool handlers are called
|
|
74
|
+
// as plain functions. No transport contention under concurrent load.
|
|
75
|
+
const builtin = await (0, connection_1.dispatchBuiltinTool)(serverId, toolName, toolArgs);
|
|
76
|
+
if (builtin) {
|
|
77
|
+
logger_1.loggerRegistry.debug(`[lt-mcp:call] leaving ${serverId}/${toolName} (builtin) resultKeys=[${typeof builtin.result === 'object' && builtin.result ? Object.keys(builtin.result).join(',') : 'raw'}]`);
|
|
78
|
+
return builtin.result;
|
|
79
|
+
}
|
|
80
|
+
// External servers — use MCP Client/Transport with timeout guard
|
|
81
|
+
const client = await (0, connection_1.resolveClient)(serverId);
|
|
82
|
+
if (!client) {
|
|
83
|
+
throw new Error(`MCP server ${serverId} is not connected`);
|
|
84
|
+
}
|
|
85
|
+
// Guard against hung transports: the MCP SDK timeout relies on the transport
|
|
86
|
+
// to respond, which fails when InMemoryTransport is saturated under concurrency.
|
|
87
|
+
// Promise.race ensures we throw on timeout regardless of transport state.
|
|
88
|
+
const callPromise = client.callTool({ name: toolName, arguments: toolArgs }, undefined, { timeout: defaults_1.MCP_TOOL_TIMEOUT_MS });
|
|
89
|
+
const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error(`MCP tool ${serverId}/${toolName} timed out after ${defaults_1.MCP_TOOL_TIMEOUT_MS}ms`)), defaults_1.MCP_TOOL_TIMEOUT_MS));
|
|
90
|
+
const result = await Promise.race([callPromise, timeoutPromise]);
|
|
78
91
|
// Extract text content from MCP response
|
|
79
92
|
if (Array.isArray(result.content)) {
|
|
80
93
|
const textContent = result.content.find((c) => c.type === 'text');
|
|
@@ -40,10 +40,32 @@ const db_1 = require("../../../lib/db");
|
|
|
40
40
|
const logger_1 = require("../../../lib/logger");
|
|
41
41
|
const ephemeral_1 = require("../../iam/ephemeral");
|
|
42
42
|
const mcpClient = __importStar(require("../../mcp/client"));
|
|
43
|
+
const connection_1 = require("../../mcp/client/connection");
|
|
43
44
|
const yamlDb = __importStar(require("../db"));
|
|
44
45
|
const scope_1 = require("./scope");
|
|
45
46
|
const callbacks_1 = require("./callbacks");
|
|
46
47
|
const events_1 = require("./events");
|
|
48
|
+
/**
|
|
49
|
+
* HotMesh YAML maps serialize arrays as objects with numeric keys
|
|
50
|
+
* (e.g., {0: "a", 1: "b"}). Recursively coerce these back to arrays
|
|
51
|
+
* before passing args to MCP tools that expect real arrays.
|
|
52
|
+
*/
|
|
53
|
+
function coerceNumericObjects(obj) {
|
|
54
|
+
if (Array.isArray(obj))
|
|
55
|
+
return obj.map(coerceNumericObjects);
|
|
56
|
+
if (obj === null || typeof obj !== 'object')
|
|
57
|
+
return obj;
|
|
58
|
+
const record = obj;
|
|
59
|
+
const keys = Object.keys(record);
|
|
60
|
+
if (keys.length > 0 && keys.every((k, i) => k === String(i))) {
|
|
61
|
+
return keys.map((k) => coerceNumericObjects(record[k]));
|
|
62
|
+
}
|
|
63
|
+
const result = {};
|
|
64
|
+
for (const [k, v] of Object.entries(record)) {
|
|
65
|
+
result[k] = coerceNumericObjects(v);
|
|
66
|
+
}
|
|
67
|
+
return result;
|
|
68
|
+
}
|
|
47
69
|
/** Track which topics already have registered workers */
|
|
48
70
|
const registeredTopics = new Set();
|
|
49
71
|
/**
|
|
@@ -146,14 +168,18 @@ async function registerWorkersForWorkflow(workflow) {
|
|
|
146
168
|
if (!serverId)
|
|
147
169
|
continue;
|
|
148
170
|
const storedArgs = activity.tool_arguments;
|
|
149
|
-
|
|
171
|
+
// For escalate_and_wait, resolve hookTopic at runtime from the activity
|
|
172
|
+
// context — multiple escalation workers share a single registered callback,
|
|
173
|
+
// so we look up by activity_id from the incoming metadata, not the static
|
|
174
|
+
// activity captured at registration time.
|
|
175
|
+
const staticHookTopic = hookTopicByEscalationTool.get(activity.activity_id);
|
|
150
176
|
// Identify keys that are wired via input_mappings. When a wired key
|
|
151
177
|
// resolves to nothing (upstream step failed/returned null), we must
|
|
152
178
|
// NOT fall back to stored tool_arguments — that would leak hardcoded
|
|
153
179
|
// values from the original execution trace.
|
|
154
180
|
const wiredKeys = new Set(Object.keys(activity.input_mappings || {}).filter(k => k !== '_scope' && k !== 'workflowName'));
|
|
155
181
|
if (toolName === 'escalate_and_wait') {
|
|
156
|
-
logger_1.loggerRegistry.info(`[yaml-workflow] escalate_and_wait worker: activityId=${activity.activity_id}, hookTopic=${
|
|
182
|
+
logger_1.loggerRegistry.info(`[yaml-workflow] escalate_and_wait worker: activityId=${activity.activity_id}, hookTopic=${staticHookTopic || 'NONE'}, mapKeys=[${[...hookTopicByEscalationTool.keys()].join(',')}]`);
|
|
157
183
|
}
|
|
158
184
|
workerConfigs.push({
|
|
159
185
|
topic: activity.topic,
|
|
@@ -180,7 +206,13 @@ async function registerWorkersForWorkflow(workflow) {
|
|
|
180
206
|
}
|
|
181
207
|
logger_1.loggerRegistry.debug(`[yaml-workflow:worker] merged mcp/${toolName} wf=${wfName} mergedKeys=[${Object.keys(mergedArgs).join(',')}]`);
|
|
182
208
|
// For escalate_and_wait: inject YAML signal routing so the MCP tool
|
|
183
|
-
// stores engine:'yaml' + hookTopic + jobId in the escalation metadata
|
|
209
|
+
// stores engine:'yaml' + hookTopic + jobId in the escalation metadata.
|
|
210
|
+
// Resolve hookTopic at runtime — multiple escalation workers share
|
|
211
|
+
// this callback, so we look up by the current activity_id from metadata.
|
|
212
|
+
const runtimeActivityId = data.metadata?.aid;
|
|
213
|
+
const yamlHookTopic = runtimeActivityId
|
|
214
|
+
? hookTopicByEscalationTool.get(runtimeActivityId) || staticHookTopic
|
|
215
|
+
: staticHookTopic;
|
|
184
216
|
if (yamlHookTopic) {
|
|
185
217
|
const jid = data.metadata?.jid;
|
|
186
218
|
mergedArgs._yaml_signal_routing = {
|
|
@@ -194,7 +226,17 @@ async function registerWorkersForWorkflow(workflow) {
|
|
|
194
226
|
};
|
|
195
227
|
}
|
|
196
228
|
const exchangedArgs = await (0, ephemeral_1.exchangeTokensInArgs)(mergedArgs);
|
|
197
|
-
const
|
|
229
|
+
const coercedArgs = coerceNumericObjects(exchangedArgs);
|
|
230
|
+
// Try direct dispatch for built-in servers (bypasses MCP transport).
|
|
231
|
+
// Falls through to mcpClient.callServerTool() for external servers.
|
|
232
|
+
let result;
|
|
233
|
+
const builtin = await (0, connection_1.dispatchBuiltinTool)(serverId, toolName, coercedArgs);
|
|
234
|
+
if (builtin) {
|
|
235
|
+
result = builtin.result;
|
|
236
|
+
}
|
|
237
|
+
else {
|
|
238
|
+
result = await mcpClient.callServerTool(serverId, toolName, coercedArgs);
|
|
239
|
+
}
|
|
198
240
|
if (result && typeof result === 'object' && 'error' in result) {
|
|
199
241
|
logger_1.loggerRegistry.error(`[yaml-workflow:worker] ${toolName} error: ${JSON.stringify(result).slice(0, 200)}`);
|
|
200
242
|
}
|
|
@@ -205,14 +205,15 @@ IMPORTANT: A bare array like \`['.png']\` as a row after sub-pipes will CRASH
|
|
|
205
205
|
2. **Worker per tool**: Each MCP tool call is a worker activity
|
|
206
206
|
3. **Collision-proof activity IDs**: Multiple workflows share the same app namespace. Activity IDs MUST be globally unique within the app. Use a descriptive name with a shared 4-char random suffix appended to every activity in the flow: \`trigger_x8kf\`, \`capture_x8kf\`, \`analyze_x8kf\`, \`store_x8kf\`. The suffix is the same for all activities in one workflow but unique across workflows. NEVER use bare names like \`trigger\`, \`capture\`, \`analyze\` — they WILL collide with other workflows in the same app.
|
|
207
207
|
4. **workflowName**: Every worker MUST have \`workflowName: '<tool_name>'\` in its input.maps — this routes to the correct MCP tool handler
|
|
208
|
-
5. **
|
|
209
|
-
6. **
|
|
210
|
-
7. **
|
|
211
|
-
8. **
|
|
212
|
-
9. **
|
|
213
|
-
10. **
|
|
214
|
-
11. **
|
|
215
|
-
12. **
|
|
208
|
+
5. **mcp_server_id**: In the activity_manifest, use the exact hyphenated server name from the inventory (e.g., "long-tail-vision"), NOT the underscored tool prefix (e.g., "long_tail_vision")
|
|
209
|
+
6. **_scope threading**: Every worker MUST have \`_scope: '{trigger_x8kf.output.data._scope}'\` (using YOUR trigger's ID) for IAM context
|
|
210
|
+
7. **Wire outputs forward**: Use \`{prevActivity.output.data.fieldName}\` to pass data between steps
|
|
211
|
+
8. **Use @pipe for transforms**: When a value needs runtime computation (date stamp, string concat, slugify), use @pipe — never hardcode computed values
|
|
212
|
+
9. **Simple fields stay simple**: If a field just passes a trigger value through (domain, key, url), use a plain reference like \`'{trigger_x8kf.output.data.domain}'\` — NEVER wrap it in @pipe. Only use @pipe when actual transformation is needed.
|
|
213
|
+
10. **File extensions**: Screenshot paths MUST include .png extension. Use @pipe concat if deriving from a slug
|
|
214
|
+
11. **job.maps on last activity**: The final activity should have job.maps to promote output fields to the workflow result
|
|
215
|
+
12. **Linear transitions**: Chain activities with transitions unless branching or iteration is needed
|
|
216
|
+
13. **Conditional transitions**: For branching, use multi-target transitions with conditions:
|
|
216
217
|
\`\`\`yaml
|
|
217
218
|
transitions:
|
|
218
219
|
check_x8kf:
|
|
@@ -222,7 +223,7 @@ transitions:
|
|
|
222
223
|
- to: proceed_x8kf
|
|
223
224
|
\`\`\`
|
|
224
225
|
Conditions can match on \`code\` (HTTP status) or \`match\` (field comparisons). The first matching condition wins; the last entry (no conditions) is the default.
|
|
225
|
-
|
|
226
|
+
14. **Trigger stats for idempotency**: Use \`stats.id\` and \`stats.key\` on the trigger when the workflow needs custom job IDs or indexed lookups:
|
|
226
227
|
\`\`\`yaml
|
|
227
228
|
trigger_x8kf:
|
|
228
229
|
type: trigger
|
package/docs/cloud.md
CHANGED
|
@@ -270,3 +270,126 @@ services:
|
|
|
270
270
|
```
|
|
271
271
|
|
|
272
272
|
The combined `index.js` entry point (used in development and the demo) calls `start()` with both server and workers enabled. In production, split them into `api.js` and `worker.js` with different `start()` configs.
|
|
273
|
+
|
|
274
|
+
## PostgreSQL Performance Tuning
|
|
275
|
+
|
|
276
|
+
HotMesh's durable execution model is write-heavy. Every workflow creates a `jobs` row, and every field mutation creates rows in `jobs_attributes`. A simple 3-step workflow generates ~100 attribute rows per execution. At 1,000 concurrent workflows, that's 300K+ inserts in seconds.
|
|
277
|
+
|
|
278
|
+
The default Postgres configuration is tuned for mixed workloads on modest hardware. For Long Tail, the write-heavy profile needs specific adjustments.
|
|
279
|
+
|
|
280
|
+
### Determining Your Profile
|
|
281
|
+
|
|
282
|
+
Run a baseline throughput test to understand your bottleneck:
|
|
283
|
+
|
|
284
|
+
```bash
|
|
285
|
+
# Submit 100 minimal workflows, measure submit rate
|
|
286
|
+
time for i in $(seq 1 100); do
|
|
287
|
+
curl -s -X POST http://localhost:3000/api/workflows/basicEcho/invoke \
|
|
288
|
+
-H "Authorization: Bearer $TOKEN" \
|
|
289
|
+
-H 'Content-Type: application/json' \
|
|
290
|
+
-d '{"data":{"message":"test","sleepSeconds":0}}' > /dev/null
|
|
291
|
+
done
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
Then check where Postgres is spending time:
|
|
295
|
+
|
|
296
|
+
```sql
|
|
297
|
+
-- Check for write pressure (high buffers_checkpoint = WAL bottleneck)
|
|
298
|
+
SELECT * FROM pg_stat_bgwriter;
|
|
299
|
+
|
|
300
|
+
-- Check for connection saturation
|
|
301
|
+
SELECT count(*) as active, max_conn
|
|
302
|
+
FROM pg_stat_activity, (SELECT setting::int as max_conn FROM pg_settings WHERE name = 'max_connections') mc
|
|
303
|
+
WHERE state = 'active'
|
|
304
|
+
GROUP BY max_conn;
|
|
305
|
+
|
|
306
|
+
-- Check table bloat after burst writes
|
|
307
|
+
SELECT relname, n_live_tup, n_dead_tup,
|
|
308
|
+
round(100.0 * n_dead_tup / nullif(n_live_tup + n_dead_tup, 0), 1) as dead_pct
|
|
309
|
+
FROM pg_stat_user_tables
|
|
310
|
+
WHERE n_live_tup > 1000
|
|
311
|
+
ORDER BY n_dead_tup DESC LIMIT 10;
|
|
312
|
+
```
|
|
313
|
+
|
|
314
|
+
### Recommended Settings
|
|
315
|
+
|
|
316
|
+
| Parameter | Default | Recommended | Why |
|
|
317
|
+
|-----------|---------|-------------|-----|
|
|
318
|
+
| `shared_buffers` | 128MB | 25% of RAM (256MB–1GB) | Cache hot pages — `jobs_attributes` partitions are read/written constantly |
|
|
319
|
+
| `work_mem` | 4MB | 16MB | Workflow queries join across partitions; larger sort memory avoids disk spills |
|
|
320
|
+
| `maintenance_work_mem` | 64MB | 128MB–256MB | Speeds VACUUM on large `jobs_attributes` tables after burst writes |
|
|
321
|
+
| `wal_buffers` | -1 (auto) | 16MB | Write-heavy workloads saturate the default 8MB WAL buffer |
|
|
322
|
+
| `max_wal_size` | 1GB | 1GB–2GB | Prevents excessive checkpointing during sustained write bursts |
|
|
323
|
+
| `checkpoint_completion_target` | 0.9 | 0.9 | Spread checkpoint I/O over time — already optimal |
|
|
324
|
+
| `effective_cache_size` | 4GB | 50–75% of RAM | Query planner hint — tells Postgres how much OS cache to expect |
|
|
325
|
+
| `synchronous_commit` | on | off (dev/staging) | Trades durability for 2–5x write throughput. WAL is still written; only fsync is deferred. Acceptable for dev and staging. **Keep `on` in production** unless you understand the trade-off. |
|
|
326
|
+
| `max_connections` | 100 | 200 | HotMesh uses connection-per-worker; concurrent workflows can exhaust 100 connections |
|
|
327
|
+
|
|
328
|
+
### Docker Compose Configuration
|
|
329
|
+
|
|
330
|
+
```yaml
|
|
331
|
+
postgres:
|
|
332
|
+
image: postgres:16
|
|
333
|
+
command:
|
|
334
|
+
- postgres
|
|
335
|
+
- -c
|
|
336
|
+
- shared_buffers=256MB
|
|
337
|
+
- -c
|
|
338
|
+
- work_mem=16MB
|
|
339
|
+
- -c
|
|
340
|
+
- maintenance_work_mem=128MB
|
|
341
|
+
- -c
|
|
342
|
+
- wal_buffers=16MB
|
|
343
|
+
- -c
|
|
344
|
+
- max_wal_size=1GB
|
|
345
|
+
- -c
|
|
346
|
+
- checkpoint_completion_target=0.9
|
|
347
|
+
- -c
|
|
348
|
+
- effective_cache_size=512MB
|
|
349
|
+
- -c
|
|
350
|
+
- synchronous_commit=off
|
|
351
|
+
- -c
|
|
352
|
+
- max_connections=200
|
|
353
|
+
shm_size: 512m # Required: shared_buffers > 128MB needs larger /dev/shm
|
|
354
|
+
```
|
|
355
|
+
|
|
356
|
+
The `shm_size` setting is critical — Docker defaults to 64MB for `/dev/shm`, but `shared_buffers=256MB` requires at least that much shared memory. Without it, Postgres will fail to start or silently fall back to smaller buffers.
|
|
357
|
+
|
|
358
|
+
### Production (RDS / Cloud SQL)
|
|
359
|
+
|
|
360
|
+
For managed databases, apply the same parameters through parameter groups:
|
|
361
|
+
|
|
362
|
+
**AWS RDS:**
|
|
363
|
+
```
|
|
364
|
+
# Custom parameter group
|
|
365
|
+
shared_buffers = {DBInstanceClassMemory/4}
|
|
366
|
+
work_mem = 16384 # 16MB in KB
|
|
367
|
+
maintenance_work_mem = 262144
|
|
368
|
+
wal_buffers = 16384
|
|
369
|
+
max_wal_size = 2048 # 2GB in MB
|
|
370
|
+
synchronous_commit = on # Keep on for production
|
|
371
|
+
max_connections = 200
|
|
372
|
+
```
|
|
373
|
+
|
|
374
|
+
**GCP Cloud SQL:**
|
|
375
|
+
```
|
|
376
|
+
# Database flags
|
|
377
|
+
shared_buffers: 25% of instance RAM (auto-tuned by Cloud SQL)
|
|
378
|
+
work_mem: 16MB
|
|
379
|
+
maintenance_work_mem: 256MB
|
|
380
|
+
max_wal_size: 2GB
|
|
381
|
+
synchronous_commit: on
|
|
382
|
+
max_connections: 200
|
|
383
|
+
```
|
|
384
|
+
|
|
385
|
+
### Maintenance
|
|
386
|
+
|
|
387
|
+
After burst workloads, dead tuples accumulate in `jobs_attributes`. Autovacuum handles this, but for large bursts (10K+ workflows), consider:
|
|
388
|
+
|
|
389
|
+
```sql
|
|
390
|
+
-- Manual VACUUM after a load test or batch run
|
|
391
|
+
VACUUM ANALYZE durable.jobs_attributes;
|
|
392
|
+
VACUUM ANALYZE durable.engine_streams;
|
|
393
|
+
```
|
|
394
|
+
|
|
395
|
+
Long Tail includes a built-in maintenance cron that prunes completed workflow data. Configure it via the dashboard or API to keep table sizes manageable.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hotmeshio/long-tail",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.12",
|
|
4
4
|
"description": "Long Tail Workflows — Durable AI workflows with human-in-the-loop escalation. Powered by PostgreSQL.",
|
|
5
5
|
"main": "./build/index.js",
|
|
6
6
|
"types": "./build/index.d.ts",
|
|
@@ -59,9 +59,9 @@
|
|
|
59
59
|
"author": "luke.birdeau@gmail.com",
|
|
60
60
|
"license": "SEE LICENSE IN LICENSE",
|
|
61
61
|
"dependencies": {
|
|
62
|
-
"@anthropic-ai/sdk": "^0.
|
|
62
|
+
"@anthropic-ai/sdk": "^0.92.0",
|
|
63
63
|
"@aws-sdk/client-s3": "^3.1017.0",
|
|
64
|
-
"@hotmeshio/hotmesh": "^0.14.
|
|
64
|
+
"@hotmeshio/hotmesh": "^0.14.4",
|
|
65
65
|
"@modelcontextprotocol/sdk": "^1.27.1",
|
|
66
66
|
"@opentelemetry/exporter-trace-otlp-proto": "^0.215.0",
|
|
67
67
|
"@opentelemetry/resources": "^2.5.1",
|