@hotmeshio/long-tail 0.1.5 → 0.1.6

package/README.md

@@ -82,9 +82,39 @@ curl -X PUT http://localhost:3000/api/workflows/reviewContent/config \
 
 De-certifying removes the interceptor. The workflow continues as a standard durable workflow — same code, different guarantees.
 
-## Register an MCP Server
+## Register MCP Servers
 
-Write an MCP server to expose your own tools. The `registerMcpTool` helper handles the SDK's type complexity.
+Long Tail connects to any MCP server — an npm package, a remote service, or one you write yourself. Registered tools become durable activities and are available to the Pipeline Designer.
+
+**Use an existing package** — no code, just register:
+
+```bash
+curl -X POST http://localhost:3000/api/mcp/servers \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "name": "filesystem",
+    "transport_type": "stdio",
+    "transport_config": { "command": "npx", "args": ["-y", "@modelcontextprotocol/server-filesystem", "/data"] },
+    "tags": ["files", "storage"],
+    "auto_connect": true
+  }'
+```
+
+**Connect a remote server** — point at a URL:
+
+```bash
+curl -X POST http://localhost:3000/api/mcp/servers \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "name": "my-python-server",
+    "transport_type": "sse",
+    "transport_config": { "url": "http://python-service:8000/mcp" },
+    "tags": ["ml", "classification"],
+    "compile_hints": "Returns confidence scores. Use threshold 0.85 for auto-approve."
+  }'
+```
+
+**Write your own** and register it in-process:
 
 ```typescript
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
@@ -106,8 +136,6 @@ export function createImageToolsServer(): McpServer {
 }
 ```
 
-Register it at startup and it appears in the dashboard:
-
 ```typescript
 const lt = await start({
   // ...
@@ -117,6 +145,8 @@ const lt = await start({
 });
 ```
 
+All three paths produce the same outcome: tools callable as durable activities. Tags enable discovery. Compile hints guide the compiler when tools are compiled into deterministic pipelines. See the [MCP guide](docs/mcp.md) for the full registration lifecycle.
+
 ## Ask It Anything
 
 Once your tools are registered, the Pipeline Designer orchestrates them. Describe what you need in plain language:

package/build/lib/db/migrate.js

@@ -38,12 +38,9 @@ const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const index_1 = require("./index");
 const logger_1 = require("../logger");
-// In dev: __dirname = lib/db → schemas is ./schemas
-// In built/published: __dirname = build/lib/db → schemas is ../../../lib/db/schemas
-const devPath = path.join(__dirname, 'schemas');
-const SCHEMAS_DIR = fs.existsSync(devPath)
-    ? devPath
-    : path.join(__dirname, '..', '..', '..', 'services', 'db', 'schemas');
+// Both dev (lib/db/) and build (build/lib/db/) have schemas/ as a sibling.
+// In dev it exists naturally; in build it's copied by the build script.
+const SCHEMAS_DIR = path.join(__dirname, 'schemas');
 async function migrate() {
     const pool = (0, index_1.getPool)();
     // ensure migration tracking table

package/{lib → build/lib}/db/schemas/001_schema.sql

@@ -258,6 +258,9 @@ CREATE TABLE IF NOT EXISTS lt_yaml_workflows (
   tags                     TEXT[] NOT NULL DEFAULT '{}',
   input_field_meta         JSONB NOT NULL DEFAULT '[]'::JSONB,
   metadata                 JSONB,
+  cron_schedule            TEXT,
+  cron_envelope            JSONB,
+  execute_as               TEXT,
   created_at               TIMESTAMPTZ NOT NULL DEFAULT NOW(),
   updated_at               TIMESTAMPTZ NOT NULL DEFAULT NOW()
 );

package/{lib → build/lib}/db/schemas/011_system_workflow_configs.sql

@@ -10,6 +10,9 @@ VALUES
    '{}'),
   ('mcpTriage', 'long-tail-system', 'engineer', false,
    'Dynamic MCP triage — LLM agentic loop for escalation remediation',
+   '{}'),
+  ('mcpWorkflowBuilder', 'long-tail-system', 'engineer', false,
+   'Direct pipeline builder — LLM constructs DAG from tool schemas',
    '{}')
 ON CONFLICT (workflow_type) DO NOTHING;
 
@@ -21,3 +24,7 @@ ON CONFLICT (workflow_type, role) DO NOTHING;
 INSERT INTO lt_config_roles (workflow_type, role)
 SELECT 'mcpTriage', unnest(ARRAY['reviewer', 'engineer', 'admin'])
 ON CONFLICT (workflow_type, role) DO NOTHING;
+
+INSERT INTO lt_config_roles (workflow_type, role)
+SELECT 'mcpWorkflowBuilder', unnest(ARRAY['reviewer', 'engineer', 'admin'])
+ON CONFLICT (workflow_type, role) DO NOTHING;

package/build/lib/db/schemas/016_streamable_http.sql

@@ -0,0 +1,7 @@
+-- Allow 'streamable-http' as a transport type for MCP servers
+ALTER TABLE lt_mcp_servers
+  DROP CONSTRAINT IF EXISTS lt_mcp_servers_transport_type_check;
+
+ALTER TABLE lt_mcp_servers
+  ADD CONSTRAINT lt_mcp_servers_transport_type_check
+  CHECK (transport_type IN ('stdio', 'sse', 'streamable-http'));

package/build/routes/docs.js

@@ -37,13 +37,13 @@ const express_1 = require("express");
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const router = (0, express_1.Router)();
-function docsDir() {
+function projectRoot() {
     const candidates = [
-        path.join(__dirname, '..', 'docs'),
-        path.join(__dirname, '..', '..', 'docs'),
+        path.join(__dirname, '..'),
+        path.join(__dirname, '..', '..'),
     ];
     for (const dir of candidates) {
-        if (fs.existsSync(dir))
+        if (fs.existsSync(path.join(dir, 'docs')))
             return dir;
     }
     return candidates[0];
@@ -66,8 +66,26 @@ function listDocs(dir, prefix = '') {
     }
     return results;
 }
+function resolveDocPath(docPath) {
+    const root = projectRoot();
+    // README.md lives at project root, everything else under docs/
+    if (docPath === 'README.md') {
+        const fp = path.join(root, 'README.md');
+        return fs.existsSync(fp) ? fp : null;
+    }
+    const fp = path.join(root, 'docs', docPath);
+    return fs.existsSync(fp) ? fp : null;
+}
 router.get('/', (_req, res) => {
-    res.json({ docs: listDocs(docsDir()) });
+    const root = projectRoot();
+    const docs = [];
+    // README first
+    const readmePath = path.join(root, 'README.md');
+    if (fs.existsSync(readmePath)) {
+        docs.push({ path: 'README.md', title: 'Long Tail' });
+    }
+    docs.push(...listDocs(path.join(root, 'docs')));
+    res.json({ docs });
 });
 // Use query param for doc path to avoid Express wildcard issues
 router.get('/read', (req, res) => {
@@ -76,8 +94,8 @@ router.get('/read', (req, res) => {
         res.status(400).json({ error: 'Invalid path' });
         return;
     }
-    const filePath = path.join(docsDir(), docPath);
-    if (!fs.existsSync(filePath)) {
+    const filePath = resolveDocPath(docPath);
+    if (!filePath) {
         res.status(404).json({ error: 'Document not found' });
         return;
     }

package/build/routes/insight.js

@@ -37,6 +37,69 @@ router.post('/mcp-query', async (req, res) => {
         res.status(500).json({ error: err.message });
     }
 });
+/**
+ * POST /api/insight/build-workflow
+ * Build a HotMesh YAML DAG directly from a natural language description.
+ * The LLM reasons about tool schemas and constructs the YAML declaratively.
+ * Body: { prompt: string, tags?: string[], wait?: boolean, feedback?: string, prior_yaml?: string }
+ */
+router.post('/build-workflow', async (req, res) => {
+    try {
+        const { prompt, tags, wait, feedback, prior_yaml, answers, prior_questions } = req.body;
+        if (!prompt || typeof prompt !== 'string') {
+            res.status(400).json({ error: 'prompt is required' });
+            return;
+        }
+        if (!process.env.OPENAI_API_KEY && !process.env.ANTHROPIC_API_KEY) {
+            res.status(503).json({ error: 'Workflow builder requires an LLM API key (OPENAI_API_KEY or ANTHROPIC_API_KEY)' });
+            return;
+        }
+        const result = await (0, insight_1.startWorkflowBuilder)({
+            prompt,
+            tags,
+            wait,
+            feedback,
+            prior_yaml,
+            answers,
+            prior_questions,
+            userId: req.auth?.userId,
+        });
+        res.json(result);
+    }
+    catch (err) {
+        if (err.message?.includes('timeout') || err.message?.includes('TIMEOUT')) {
+            res.status(504).json({ error: 'Workflow builder timed out. Try a simpler description.' });
+            return;
+        }
+        res.status(500).json({ error: err.message });
+    }
+});
+/**
+ * POST /api/insight/build-workflow/refine
+ * Refine a previously built workflow using execution feedback.
+ * Body: { prompt: string, prior_yaml: string, feedback: string, tags?: string[], wait?: boolean }
+ */
+router.post('/build-workflow/refine', async (req, res) => {
+    try {
+        const { prompt, prior_yaml, feedback, tags, wait } = req.body;
+        if (!prompt || !prior_yaml || !feedback) {
+            res.status(400).json({ error: 'prompt, prior_yaml, and feedback are required' });
+            return;
+        }
+        const result = await (0, insight_1.startWorkflowBuilder)({
+            prompt,
+            tags,
+            wait,
+            feedback,
+            prior_yaml,
+            userId: req.auth?.userId,
+        });
+        res.json(result);
+    }
+    catch (err) {
+        res.status(500).json({ error: err.message });
+    }
+});
 /**
  * POST /api/insight/mcp-query/describe
  * Generate a workflow description and suggested tags from prompt and result.

package/build/routes/mcp.js

@@ -66,7 +66,7 @@ router.get('/servers', async (req, res) => {
  */
 router.post('/servers', async (req, res) => {
     try {
-        const { name, description, transport_type, transport_config, auto_connect, metadata } = req.body;
+        const { name, description, transport_type, transport_config, auto_connect, metadata, tags, compile_hints, credential_providers } = req.body;
         if (!name || !transport_type || !transport_config) {
             res.status(400).json({ error: 'name, transport_type, and transport_config are required' });
             return;
@@ -78,6 +78,9 @@ router.post('/servers', async (req, res) => {
             transport_config,
             auto_connect,
             metadata,
+            tags,
+            compile_hints,
+            credential_providers,
         });
         res.status(201).json(server);
     }
@@ -89,6 +92,25 @@ router.post('/servers', async (req, res) => {
         res.status(500).json({ error: err.message });
     }
 });
+/**
+ * POST /api/mcp/servers/test-connection
+ * Test connectivity to an MCP server without persisting it.
+ */
+router.post('/servers/test-connection', async (req, res) => {
+    try {
+        const { transport_type, transport_config } = req.body;
+        if (!transport_type || !transport_config) {
+            res.status(400).json({ error: 'transport_type and transport_config are required' });
+            return;
+        }
+        const { testConnection } = await Promise.resolve().then(() => __importStar(require('../services/mcp/client/connection')));
+        const result = await testConnection(transport_type, transport_config);
+        res.json(result);
+    }
+    catch (err) {
+        res.json({ success: false, error: err.message, tools: [] });
+    }
+});
 // ── Parameterized routes (must come after literal paths) ──────────────
 /**
  * GET /api/mcp/servers/:id

package/build/routes/yaml-workflows/cron.d.ts

@@ -0,0 +1,2 @@
+declare const router: import("express-serve-static-core").Router;
+export default router;

package/build/routes/yaml-workflows/cron.js

@@ -0,0 +1,117 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const express_1 = require("express");
+const yamlDb = __importStar(require("../../services/yaml-workflow/db"));
+const cron_1 = require("../../services/cron");
+const helpers_1 = require("./helpers");
+const router = (0, express_1.Router)();
+/**
+ * PUT /api/yaml-workflows/:id/cron
+ * Set or update cron schedule + envelope + execute_as.
+ */
+router.put('/:id/cron', async (req, res) => {
+    try {
+        const wf = await yamlDb.getYamlWorkflow(req.params.id);
+        if (!wf) {
+            res.status(404).json({ error: 'YAML workflow not found' });
+            return;
+        }
+        const { cron_schedule, cron_envelope, execute_as } = req.body;
+        if (!cron_schedule || typeof cron_schedule !== 'string') {
+            res.status(400).json({ error: 'cron_schedule is required' });
+            return;
+        }
+        const updated = await yamlDb.updateCronSchedule(wf.id, cron_schedule.trim(), cron_envelope ?? null, execute_as ?? null);
+        if (updated) {
+            await cron_1.cronRegistry.restartYamlCron(updated);
+        }
+        res.json(updated);
+    }
+    catch (err) {
+        if ((0, helpers_1.isNotFoundError)(err)) {
+            res.status(404).json({ error: 'YAML workflow not found' });
+            return;
+        }
+        res.status(500).json({ error: err.message });
+    }
+});
+/**
+ * DELETE /api/yaml-workflows/:id/cron
+ * Clear cron schedule.
+ */
+router.delete('/:id/cron', async (req, res) => {
+    try {
+        const wf = await yamlDb.getYamlWorkflow(req.params.id);
+        if (!wf) {
+            res.status(404).json({ error: 'YAML workflow not found' });
+            return;
+        }
+        await cron_1.cronRegistry.stopYamlCron(wf.id);
+        const updated = await yamlDb.clearCronSchedule(wf.id);
+        res.json(updated);
+    }
+    catch (err) {
+        if ((0, helpers_1.isNotFoundError)(err)) {
+            res.status(404).json({ error: 'YAML workflow not found' });
+            return;
+        }
+        res.status(500).json({ error: err.message });
+    }
+});
+/**
+ * GET /api/yaml-workflows/cron/status
+ * List all YAML workflows with active cron schedules.
+ */
+router.get('/cron/status', async (_req, res) => {
+    try {
+        const workflows = await yamlDb.getCronScheduledWorkflows();
+        const activeTypes = cron_1.cronRegistry.activeWorkflowTypes;
+        const schedules = workflows.map((wf) => ({
+            id: wf.id,
+            name: wf.name,
+            graph_topic: wf.graph_topic,
+            app_id: wf.app_id,
+            cron_schedule: wf.cron_schedule,
+            execute_as: wf.execute_as,
+            active: activeTypes.includes(`yaml:${wf.id}`),
+        }));
+        res.json({ schedules });
+    }
+    catch (err) {
+        res.status(500).json({ error: err.message });
+    }
+});
+exports.default = router;

package/build/routes/yaml-workflows/crud.js

@@ -147,6 +147,44 @@ router.get('/app-ids', async (_req, res) => {
         res.status(500).json({ error: err.message });
     }
 });
+/**
+ * POST /api/yaml-workflows/direct
+ * Create a YAML workflow from raw YAML content (workflow builder output).
+ * Unlike POST /, this does not require a source execution — the YAML is provided directly.
+ * Body: { name, description?, yaml_content, input_schema?, activity_manifest?, tags?, app_id? }
+ */
+router.post('/direct', async (req, res) => {
+    try {
+        const { name, description, yaml_content, input_schema, activity_manifest, tags, app_id, graph_topic } = req.body;
+        if (!name || !yaml_content) {
+            res.status(400).json({ error: 'name and yaml_content are required' });
+            return;
+        }
+        // Extract subscribes topic from YAML content, or fall back to name-derived slug
+        let graphTopic = graph_topic || name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
+        const subscribesMatch = yaml_content.match(/subscribes:\s*(.+)/);
+        if (subscribesMatch) {
+            graphTopic = subscribesMatch[1].trim().replace(/^['"]|['"]$/g, '');
+        }
+        const wf = await yamlDb.createYamlWorkflow({
+            name,
+            description,
+            app_id: app_id || 'longtail',
+            yaml_content,
+            graph_topic: graphTopic,
+            input_schema: input_schema || {},
+            output_schema: {},
+            activity_manifest: activity_manifest || [],
+            tags: tags || [],
+            original_prompt: description,
+            category: 'builder',
+        });
+        res.json(wf);
+    }
+    catch (err) {
+        res.status(500).json({ error: err.message });
+    }
+});
 // -- Parameterized routes --
 /**
  * GET /api/yaml-workflows/:id

package/build/routes/yaml-workflows/deployment.js

@@ -37,6 +37,7 @@ const express_1 = require("express");
 const yamlDb = __importStar(require("../../services/yaml-workflow/db"));
 const yamlDeployer = __importStar(require("../../services/yaml-workflow/deployer"));
 const yamlWorkers = __importStar(require("../../services/yaml-workflow/workers"));
+const invoke_1 = require("../../services/yaml-workflow/invoke");
 const helpers_1 = require("./helpers");
 const router = (0, express_1.Router)();
 /**
@@ -133,40 +134,14 @@ router.post('/:id/invoke', async (req, res) => {
             res.status(400).json({ error: 'Workflow must be active to invoke' });
             return;
         }
-        const data = req.body.data || {};
-        // Inject _scope so compiled workflow activities have identity context
-        const executeAs = req.body.execute_as;
-        const userId = req.auth?.userId;
-        if (!data._scope) {
-            const { resolvePrincipal } = await Promise.resolve().then(() => __importStar(require('../../services/iam/principal')));
-            if (executeAs) {
-                const [botPrincipal, invokerPrincipal] = await Promise.all([
-                    resolvePrincipal(executeAs),
-                    userId ? resolvePrincipal(userId) : Promise.resolve(null),
-                ]);
-                if (botPrincipal) {
-                    data._scope = {
-                        principal: botPrincipal,
-                        scopes: ['mcp:tool:call'],
-                        ...(invokerPrincipal ? { initiatedBy: userId, initiatingPrincipal: invokerPrincipal } : {}),
-                    };
-                }
-            }
-            else if (userId) {
-                const principal = await resolvePrincipal(userId);
-                if (principal) {
-                    data._scope = { principal, scopes: ['mcp:tool:call'] };
-                }
-            }
-        }
-        if (req.body.sync) {
-            const { job_id, result } = await yamlDeployer.invokeYamlWorkflowSync(wf.app_id, wf.graph_topic, data, req.body.timeout, wf.graph_topic);
-            res.json({ job_id, result });
-        }
-        else {
-            const jobId = await yamlDeployer.invokeYamlWorkflow(wf.app_id, wf.graph_topic, data, wf.graph_topic);
-            res.json({ job_id: jobId });
-        }
+        const result = await (0, invoke_1.invokeYamlWorkflow)(wf, {
+            data: req.body.data,
+            sync: req.body.sync,
+            timeout: req.body.timeout,
+            execute_as: req.body.execute_as,
+            userId: req.auth?.userId,
+        });
+        res.json(result);
     }
     catch (err) {
         if ((0, helpers_1.isNotFoundError)(err)) {

package/build/routes/yaml-workflows/index.js

@@ -4,11 +4,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const express_1 = require("express");
+const cron_1 = __importDefault(require("./cron"));
 const crud_1 = __importDefault(require("./crud"));
 const deployment_1 = __importDefault(require("./deployment"));
 const versions_1 = __importDefault(require("./versions"));
 const router = (0, express_1.Router)();
-// Mount in the same order as the original monolithic file.
+// cron.ts handles: GET /cron/status, PUT /:id/cron, DELETE /:id/cron
+// Mount before crud so /cron/status doesn't match /:id
+router.use(cron_1.default);
 // crud.ts handles: GET /, POST /, GET /app-ids, GET /:id, PUT /:id,
 //                  POST /:id/regenerate, DELETE /:id
 router.use(crud_1.default);

package/build/services/cron/index.d.ts

@@ -1,4 +1,5 @@
 import type { LTWorkflowConfig } from '../../types';
+import type { LTYamlWorkflowRecord } from '../../types/yaml-workflow';
 /**
  * Singleton registry for workflow cron schedules.
  *
@@ -30,6 +31,22 @@ declare class LTCronRegistry {
      * Stops the old cron (if any) and starts a new one if cron_schedule is set.
      */
     restartCron(config: LTWorkflowConfig): Promise<void>;
+    /**
+     * Load all YAML workflows with a cron_schedule and start Virtual.cron for each.
+     */
+    connectYamlCrons(): Promise<void>;
+    /**
+     * Start a single Virtual.cron for a YAML workflow.
+     */
+    startYamlCron(wf: LTYamlWorkflowRecord): Promise<void>;
+    /**
+     * Stop a single YAML workflow cron.
+     */
+    stopYamlCron(yamlWfId: string): Promise<void>;
+    /**
+     * Restart a YAML workflow cron after config change.
+     */
+    restartYamlCron(wf: LTYamlWorkflowRecord): Promise<void>;
     /**
      * Stop all active crons. Call during graceful shutdown.
      */