@hotmeshio/long-tail 0.1.0

package/build/system/mcp-servers/workflow-compiler.js

@@ -0,0 +1,230 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createWorkflowCompilerServer = createWorkflowCompilerServer;
+exports.getWorkflowCompilerServer = getWorkflowCompilerServer;
+exports.stopWorkflowCompilerServer = stopWorkflowCompilerServer;
+const mcp_js_1 = require("@modelcontextprotocol/sdk/server/mcp.js");
+const zod_1 = require("zod");
+const logger_1 = require("../../services/logger");
+const yamlGenerator = __importStar(require("../../services/yaml-workflow/generator"));
+const yamlDb = __importStar(require("../../services/yaml-workflow/db"));
+const yamlDeployer = __importStar(require("../../services/yaml-workflow/deployer"));
+const yamlWorkers = __importStar(require("../../services/yaml-workflow/workers"));
+let server = null;
+// ── Schemas (extracted to break TS2589 deep-instantiation in registerTool generics) ──
+const convertSchema = zod_1.z.object({
+    workflow_id: zod_1.z.string().describe('The workflow execution ID to analyze'),
+    task_queue: zod_1.z.string().describe('HotMesh task queue (e.g., "long-tail-examples")'),
+    workflow_name: zod_1.z.string().describe('Workflow name (e.g., "reviewContent")'),
+    yaml_name: zod_1.z.string().describe('Name for the generated YAML workflow'),
+    description: zod_1.z.string().optional().describe('Optional description'),
+});
+const deploySchema = zod_1.z.object({
+    yaml_workflow_id: zod_1.z.string().describe('UUID of the stored YAML workflow'),
+    activate: zod_1.z.boolean().optional().default(false)
+        .describe('Whether to activate immediately after deployment'),
+});
+const listSchema = zod_1.z.object({
+    status: zod_1.z.enum(['draft', 'deployed', 'active', 'archived']).optional()
+        .describe('Filter by status'),
+    limit: zod_1.z.number().int().min(1).max(100).optional().default(25)
+        .describe('Maximum number of results'),
+    offset: zod_1.z.number().int().min(0).optional().default(0)
+        .describe('Pagination offset'),
+});
+/**
+ * Create the Workflow Compiler MCP server.
+ *
+ * Provides tools for converting MCP tool call sequences into
+ * deterministic HotMesh YAML workflows:
+ * - convert_execution_to_yaml — analyze execution and generate YAML
+ * - deploy_yaml_workflow — deploy and optionally activate
+ * - list_yaml_workflows — list stored YAML workflows
+ */
+async function createWorkflowCompilerServer(options) {
+    if (server)
+        return server;
+    const name = options?.name || 'workflow-compiler';
+    server = new mcp_js_1.McpServer({ name, version: '1.0.0' });
+    // ── convert_execution_to_yaml ─────────────────────────────────────
+    server.registerTool('convert_execution_to_yaml', {
+        title: 'Convert Execution to YAML',
+        description: 'Analyze a completed workflow execution and convert its tool call sequence ' +
+            'into a deterministic HotMesh YAML workflow. Extracts callLLM→callDbTool pairs ' +
+            'and mcp_* activities, replacing LLM reasoning with direct tool-to-tool data piping. ' +
+            'The generated YAML is stored as a draft that can be deployed and activated.',
+        inputSchema: convertSchema,
+    }, async (args) => {
+        const result = await yamlGenerator.generateYamlFromExecution({
+            workflowId: args.workflow_id,
+            taskQueue: args.task_queue,
+            workflowName: args.workflow_name,
+            name: args.yaml_name,
+            description: args.description,
+        });
+        const record = await yamlDb.createYamlWorkflow({
+            name: args.yaml_name,
+            description: args.description,
+            app_id: result.appId,
+            yaml_content: result.yaml,
+            graph_topic: result.graphTopic,
+            input_schema: result.inputSchema,
+            output_schema: result.outputSchema,
+            activity_manifest: result.activityManifest,
+            input_field_meta: result.inputFieldMeta,
+            original_prompt: result.originalPrompt,
+            category: result.category,
+            tags: result.tags,
+            source_workflow_id: args.workflow_id,
+            source_workflow_type: args.workflow_name,
+        });
+        return {
+            content: [{
+                    type: 'text',
+                    text: JSON.stringify({
+                        yaml_workflow_id: record.id,
+                        app_id: record.app_id,
+                        activity_count: result.activityManifest.filter((a) => a.type === 'worker').length,
+                        input_schema: result.inputSchema,
+                        input_field_meta: result.inputFieldMeta,
+                        yaml_preview: result.yaml.slice(0, 500),
+                        ...(result.compilationPlan ? {
+                            compilation: {
+                                intent: result.compilationPlan.intent,
+                                core_steps: result.compilationPlan.coreStepIndices.length,
+                                has_iteration: result.compilationPlan.hasIteration,
+                                llm_refined: true,
+                            },
+                        } : { compilation: { llm_refined: false } }),
+                    }),
+                }],
+        };
+    });
+    // ── deploy_yaml_workflow ──────────────────────────────────────────
+    server.registerTool('deploy_yaml_workflow', {
+        title: 'Deploy YAML Workflow',
+        description: 'Deploy a stored YAML workflow to HotMesh. Optionally activate it immediately ' +
+            'and register workers so it can receive invocations.',
+        inputSchema: deploySchema,
+    }, async (args) => {
+        const wf = await yamlDb.getYamlWorkflow(args.yaml_workflow_id);
+        if (!wf)
+            throw new Error('YAML workflow not found');
+        try {
+            await yamlDeployer.deployYamlWorkflow(wf.app_id, wf.yaml_content);
+            await yamlDb.updateYamlWorkflowStatus(wf.id, 'deployed');
+            let status = 'deployed';
+            if (args.activate) {
+                await yamlDeployer.activateYamlWorkflow(wf.app_id, wf.app_version);
+                await yamlWorkers.registerWorkersForWorkflow(wf);
+                await yamlDb.updateYamlWorkflowStatus(wf.id, 'active');
+                status = 'active';
+            }
+            return {
+                content: [{
+                        type: 'text',
+                        text: JSON.stringify({
+                            status,
+                            app_id: wf.app_id,
+                            graph_topic: wf.graph_topic,
+                        }),
+                    }],
+            };
+        }
+        catch (err) {
+            // Mark workflow as failed so it doesn't block future deploys
+            await yamlDb.updateYamlWorkflowStatus(wf.id, 'error');
+            return {
+                content: [{
+                        type: 'text',
+                        text: JSON.stringify({
+                            error: err.message,
+                            status: 'error',
+                            app_id: wf.app_id,
+                            graph_topic: wf.graph_topic,
+                            hint: err.message.includes('Duplicate activity id')
+                                ? 'Two workflows have colliding activity IDs. Archive or rename one before redeploying.'
+                                : 'Check server logs for the full YAML that failed validation.',
+                        }),
+                    }],
+                isError: true,
+            };
+        }
+    });
+    // ── list_yaml_workflows ───────────────────────────────────────────
+    server.registerTool('list_yaml_workflows', {
+        title: 'List YAML Workflows',
+        description: 'List stored YAML workflows with optional status filter. ' +
+            'Returns workflow metadata including name, status, source, and activity count.',
+        inputSchema: listSchema,
+    }, async (args) => {
+        const { workflows, total } = await yamlDb.listYamlWorkflows({
+            status: args.status,
+            limit: args.limit,
+            offset: args.offset,
+        });
+        return {
+            content: [{
+                    type: 'text',
+                    text: JSON.stringify({
+                        total,
+                        count: workflows.length,
+                        workflows: workflows.map((wf) => ({
+                            id: wf.id,
+                            name: wf.name,
+                            app_id: wf.app_id,
+                            status: wf.status,
+                            source_workflow_type: wf.source_workflow_type,
+                            activity_count: wf.activity_manifest.filter((a) => a.type === 'worker').length,
+                            graph_topic: wf.graph_topic,
+                            created_at: wf.created_at,
+                        })),
+                    }),
+                }],
+        };
+    });
+    logger_1.loggerRegistry.info(`[lt-mcp] workflow compiler server created: ${name}`);
+    return server;
+}
+function getWorkflowCompilerServer() {
+    return server;
+}
+async function stopWorkflowCompilerServer() {
+    if (server) {
+        await server.close?.();
+        server = null;
+    }
+}

package/build/system/mcp-servers/workflow.d.ts

@@ -0,0 +1,24 @@
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+/**
+ * Create the Long Tail MCP Workflows server.
+ *
+ * Exposes compiled YAML workflows — hardened, deterministic pipelines
+ * that were originally discovered through dynamic MCP triage — as
+ * invocable MCP tools.
+ *
+ * Tools:
+ * - list_workflows — discover available compiled workflows
+ * - get_workflow — inspect a workflow's schema and activity manifest
+ * - invoke_workflow — run a workflow (sync or async)
+ */
+export declare function createWorkflowServer(options?: {
+    name?: string;
+}): Promise<McpServer>;
+/**
+ * Get the current workflow MCP server instance.
+ */
+export declare function getWorkflowServer(): McpServer | null;
+/**
+ * Stop the workflow MCP server.
+ */
+export declare function stopWorkflowServer(): Promise<void>;

package/build/system/mcp-servers/workflow.js

@@ -0,0 +1,231 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createWorkflowServer = createWorkflowServer;
+exports.getWorkflowServer = getWorkflowServer;
+exports.stopWorkflowServer = stopWorkflowServer;
+const mcp_js_1 = require("@modelcontextprotocol/sdk/server/mcp.js");
+const zod_1 = require("zod");
+const defaults_1 = require("../../modules/defaults");
+const logger_1 = require("../../services/logger");
+const yamlDb = __importStar(require("../../services/yaml-workflow/db"));
+const yamlDeployer = __importStar(require("../../services/yaml-workflow/deployer"));
+let server = null;
+// ── Schemas ─────────────────────────────────────────────────────────────────
+const listWorkflowsSchema = zod_1.z.object({
+    status: zod_1.z.enum(['active', 'deployed', 'draft', 'archived']).optional()
+        .describe('Filter by lifecycle status. Defaults to "active" (ready to invoke).'),
+});
+const invokeWorkflowSchema = zod_1.z.object({
+    workflow_name: zod_1.z.string()
+        .describe('Name of the compiled workflow to invoke (from list_workflows)'),
+    input: zod_1.z.record(zod_1.z.any()).optional().default({})
+        .describe('Input data matching the workflow input schema'),
+    async: zod_1.z.boolean().optional().default(false)
+        .describe('If true, fire-and-forget (returns job ID). If false, wait for result.'),
+    timeout: zod_1.z.number().optional()
+        .describe('Max milliseconds to wait for result (sync mode only). Default: 120000.'),
+});
+const getWorkflowSchema = zod_1.z.object({
+    workflow_name: zod_1.z.string()
+        .describe('Name of the workflow to inspect'),
+});
+/**
+ * Create the Long Tail MCP Workflows server.
+ *
+ * Exposes compiled YAML workflows — hardened, deterministic pipelines
+ * that were originally discovered through dynamic MCP triage — as
+ * invocable MCP tools.
+ *
+ * Tools:
+ * - list_workflows — discover available compiled workflows
+ * - get_workflow — inspect a workflow's schema and activity manifest
+ * - invoke_workflow — run a workflow (sync or async)
+ */
+async function createWorkflowServer(options) {
+    if (server)
+        return server;
+    const name = options?.name || 'mcp-workflows-longtail';
+    server = new mcp_js_1.McpServer({ name, version: '1.0.0' });
+    // ── list_workflows ──────────────────────────────────────────────────
+    server.registerTool('list_workflows', {
+        title: 'List Compiled Workflows',
+        description: 'List available compiled YAML workflows. These are deterministic pipelines ' +
+            'converted from successful MCP triage executions. Each workflow represents ' +
+            'a proven solution to a specific edge case (e.g., rotate-and-extract for ' +
+            'upside-down documents). Defaults to listing active (invocable) workflows.',
+        inputSchema: listWorkflowsSchema,
+    }, async (args) => {
+        const { workflows } = await yamlDb.listYamlWorkflows({
+            status: args.status || 'active',
+            limit: defaults_1.QUERY_LIMIT_MAX,
+        });
+        const items = workflows.map((wf) => ({
+            name: wf.name,
+            description: wf.description,
+            tags: wf.tags,
+            graph_topic: wf.graph_topic,
+            source_workflow_type: wf.source_workflow_type,
+            input_schema: wf.input_schema,
+            output_schema: wf.output_schema,
+            activity_count: wf.activity_manifest.filter((a) => a.type === 'worker').length,
+            status: wf.status,
+            activated_at: wf.activated_at,
+        }));
+        return {
+            content: [{
+                    type: 'text',
+                    text: JSON.stringify({ count: items.length, workflows: items }),
+                }],
+        };
+    });
+    // ── get_workflow ─────────────────────────────────────────────────────
+    server.registerTool('get_workflow', {
+        title: 'Get Workflow Details',
+        description: 'Inspect a compiled workflow by name. Returns the full activity manifest, ' +
+            'input/output schemas, and provenance (which execution it was compiled from).',
+        inputSchema: getWorkflowSchema,
+    }, async (args) => {
+        const wf = await yamlDb.getYamlWorkflowByName(args.workflow_name);
+        if (!wf) {
+            return {
+                content: [{
+                        type: 'text',
+                        text: JSON.stringify({ error: `Workflow "${args.workflow_name}" not found` }),
+                    }],
+                isError: true,
+            };
+        }
+        return {
+            content: [{
+                    type: 'text',
+                    text: JSON.stringify({
+                        name: wf.name,
+                        description: wf.description,
+                        app_id: wf.app_id,
+                        graph_topic: wf.graph_topic,
+                        status: wf.status,
+                        source_workflow_id: wf.source_workflow_id,
+                        source_workflow_type: wf.source_workflow_type,
+                        input_schema: wf.input_schema,
+                        output_schema: wf.output_schema,
+                        activities: wf.activity_manifest.filter((a) => a.type === 'worker').map((a) => ({
+                            title: a.title,
+                            tool_source: a.tool_source,
+                            mcp_server_id: a.mcp_server_id,
+                            mcp_tool_name: a.mcp_tool_name,
+                        })),
+                        activated_at: wf.activated_at,
+                        created_at: wf.created_at,
+                    }),
+                }],
+        };
+    });
+    // ── invoke_workflow ──────────────────────────────────────────────────
+    server.registerTool('invoke_workflow', {
+        title: 'Invoke Compiled Workflow',
+        description: 'Run a compiled YAML workflow by name. These are deterministic pipelines — ' +
+            'no LLM reasoning, just direct tool-to-tool data piping. Use list_workflows ' +
+            'to discover available workflows and their input schemas. ' +
+            'Set async=true for fire-and-forget (returns job ID), or leave false to wait for the result.',
+        inputSchema: invokeWorkflowSchema,
+    }, async (args) => {
+        const wf = await yamlDb.getYamlWorkflowByName(args.workflow_name);
+        if (!wf) {
+            return {
+                content: [{
+                        type: 'text',
+                        text: JSON.stringify({ error: `Workflow "${args.workflow_name}" not found` }),
+                    }],
+                isError: true,
+            };
+        }
+        if (wf.status !== 'active') {
+            return {
+                content: [{
+                        type: 'text',
+                        text: JSON.stringify({
+                            error: `Workflow "${args.workflow_name}" is not active (status: ${wf.status})`,
+                        }),
+                    }],
+                isError: true,
+            };
+        }
+        if (args.async) {
+            const jobId = await yamlDeployer.invokeYamlWorkflow(wf.app_id, wf.graph_topic, args.input || {}, wf.graph_topic);
+            return {
+                content: [{
+                        type: 'text',
+                        text: JSON.stringify({
+                            job_id: jobId,
+                            workflow: wf.name,
+                            status: 'started',
+                        }),
+                    }],
+            };
+        }
+        const { job_id, result } = await yamlDeployer.invokeYamlWorkflowSync(wf.app_id, wf.graph_topic, args.input || {}, args.timeout, wf.graph_topic);
+        return {
+            content: [{
+                    type: 'text',
+                    text: JSON.stringify({
+                        job_id,
+                        namespace: wf.app_id,
+                        workflow: wf.name,
+                        status: 'completed',
+                        result,
+                    }),
+                }],
+        };
+    });
+    const toolCount = 3;
+    logger_1.loggerRegistry.info(`[lt-mcp:server] ${name} ready (${toolCount} tools registered)`);
+    return server;
+}
+/**
+ * Get the current workflow MCP server instance.
+ */
+function getWorkflowServer() {
+    return server;
+}
+/**
+ * Stop the workflow MCP server.
+ */
+async function stopWorkflowServer() {
+    if (server) {
+        await server.close?.();
+        server = null;
+    }
+}

package/build/system/seed/index.d.ts

@@ -0,0 +1,7 @@
+import { SEED_MCP_SERVERS } from './server-definitions';
+export { SEED_MCP_SERVERS };
+/**
+ * Seed system MCP servers into lt_mcp_servers.
+ * Upserts each built-in server with its tool manifest, metadata, and tags.
+ */
+export declare function seedSystemMcpServers(): Promise<void>;

package/build/system/seed/index.js

@@ -0,0 +1,36 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SEED_MCP_SERVERS = void 0;
+exports.seedSystemMcpServers = seedSystemMcpServers;
+const db_1 = require("../../services/db");
+const logger_1 = require("../../services/logger");
+const sql_1 = require("../../services/mcp/sql");
+const server_definitions_1 = require("./server-definitions");
+Object.defineProperty(exports, "SEED_MCP_SERVERS", { enumerable: true, get: function () { return server_definitions_1.SEED_MCP_SERVERS; } });
+/**
+ * Seed system MCP servers into lt_mcp_servers.
+ * Upserts each built-in server with its tool manifest, metadata, and tags.
+ */
+async function seedSystemMcpServers() {
+    const pool = (0, db_1.getPool)();
+    for (const srv of server_definitions_1.SEED_MCP_SERVERS) {
+        try {
+            await pool.query(sql_1.SEED_MCP_SERVER, [
+                srv.name,
+                srv.description,
+                srv.transport_type,
+                JSON.stringify(srv.transport_config),
+                JSON.stringify(srv.tool_manifest),
+                JSON.stringify(srv.metadata),
+                srv.tags,
+                srv.compile_hints,
+                srv.credential_providers || [],
+            ]);
+        }
+        catch (err) {
+            logger_1.loggerRegistry.warn(`[system] failed to seed MCP server ${srv.name}: ${err.message}`);
+        }
+    }
+    const totalTools = server_definitions_1.SEED_MCP_SERVERS.reduce((sum, s) => sum + s.tool_manifest.length, 0);
+    logger_1.loggerRegistry.info(`[system] MCP servers seeded (${server_definitions_1.SEED_MCP_SERVERS.length} servers, ${totalTools} tools)`);
+}