@hotmeshio/long-tail 0.1.13 → 0.1.15

package/build/routes/knowledge.js

@@ -0,0 +1,141 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const express_1 = require("express");
+const api = __importStar(require("../api/knowledge"));
+const router = (0, express_1.Router)();
+/**
+ * GET /api/knowledge/domains
+ * List all knowledge domains with entry counts.
+ */
+router.get('/domains', async (_req, res) => {
+    const result = await api.listDomains();
+    res.status(result.status).json(result.data ?? { error: result.error });
+});
+/**
+ * GET /api/knowledge/entries
+ * List entries within a domain.
+ * Query: ?domain=...&tags=a,b&limit=50&offset=0
+ */
+router.get('/entries', async (req, res) => {
+    const domain = req.query.domain;
+    if (!domain) {
+        res.status(400).json({ error: 'domain is required' });
+        return;
+    }
+    const tags = req.query.tags
+        ? req.query.tags.split(',').map((t) => t.trim()).filter(Boolean)
+        : undefined;
+    const search = req.query.search || undefined;
+    const limit = req.query.limit ? parseInt(req.query.limit, 10) : undefined;
+    const offset = req.query.offset ? parseInt(req.query.offset, 10) : undefined;
+    const result = await api.listEntries({ domain, tags, search, limit, offset });
+    res.status(result.status).json(result.data ?? { error: result.error });
+});
+/**
+ * GET /api/knowledge/entry
+ * Get a single knowledge entry.
+ * Query: ?domain=...&key=...
+ */
+router.get('/entry', async (req, res) => {
+    const domain = req.query.domain;
+    const key = req.query.key;
+    if (!domain || !key) {
+        res.status(400).json({ error: 'domain and key are required' });
+        return;
+    }
+    const result = await api.getEntry({ domain, key });
+    res.status(result.status).json(result.data ?? { error: result.error });
+});
+/**
+ * POST /api/knowledge/entry
+ * Create or update a knowledge entry.
+ * Body: { domain, key, data, tags? }
+ */
+router.post('/entry', async (req, res) => {
+    const { domain, key, data, tags, replace } = req.body;
+    if (!domain || !key || !data) {
+        res.status(400).json({ error: 'domain, key, and data are required' });
+        return;
+    }
+    const result = await api.storeEntry({ domain, key, data, tags, replace: !!replace });
+    res.status(result.status).json(result.data ?? { error: result.error });
+});
+/**
+ * DELETE /api/knowledge/entry
+ * Delete a knowledge entry.
+ * Query: ?domain=...&key=...
+ */
+router.delete('/entry', async (req, res) => {
+    const domain = req.query.domain;
+    const key = req.query.key;
+    if (!domain || !key) {
+        res.status(400).json({ error: 'domain and key are required' });
+        return;
+    }
+    const result = await api.deleteEntry({ domain, key });
+    res.status(result.status).json(result.data ?? { error: result.error });
+});
+/**
+ * PUT /api/knowledge/field
+ * Set a value at a specific JSONB path without clobbering siblings.
+ * Body: { domain, key, path, value, tags? }
+ */
+router.put('/field', async (req, res) => {
+    const { domain, key, path, value, tags } = req.body;
+    if (!domain || !key || !path || value === undefined) {
+        res.status(400).json({ error: 'domain, key, path, and value are required' });
+        return;
+    }
+    const result = await api.setField({ domain, key, path, value, tags });
+    res.status(result.status).json(result.data ?? { error: result.error });
+});
+/**
+ * DELETE /api/knowledge/field
+ * Remove a specific field at a JSONB path.
+ * Query: ?domain=...&key=...&path=...
+ */
+router.delete('/field', async (req, res) => {
+    const domain = req.query.domain;
+    const key = req.query.key;
+    const path = req.query.path;
+    if (!domain || !key || !path) {
+        res.status(400).json({ error: 'domain, key, and path are required' });
+        return;
+    }
+    const result = await api.removeField({ domain, key, path });
+    res.status(result.status).json(result.data ?? { error: result.error });
+});
+exports.default = router;

package/build/routes/mcp-runs.js

@@ -60,6 +60,8 @@ router.get('/', async (req, res) => {
         entity: req.query.entity || undefined,
         search: req.query.search || undefined,
         status: req.query.status || undefined,
+        sort_by: req.query.sort_by || undefined,
+        order: req.query.order || undefined,
     });
     res.status(result.status).json(result.data ?? { error: result.error });
 });

package/build/routes/workflow-sets.js

@@ -79,6 +79,15 @@ router.put('/:id/plan', async (req, res) => {
     });
     res.status(result.status).json(result.data ?? { error: result.error });
 });
+/**
+ * POST /api/workflow-sets/:id/add
+ * Add additional workflows to an existing set from a new specification.
+ */
+router.post('/:id/add', async (req, res) => {
+    const { specification } = req.body;
+    const result = await api.addToWorkflowSet({ id: req.params.id, specification }, req.auth ? { userId: req.auth.userId } : undefined);
+    res.status(result.status).json(result.data ?? { error: result.error });
+});
 /**
  * POST /api/workflow-sets/:id/build
  * Trigger the build phase (resume the planner workflow).

package/build/routes/yaml-workflows/crud.js

@@ -80,6 +80,15 @@ router.post('/direct', async (req, res) => {
     const result = await api.createYamlWorkflowDirect(req.body);
     res.status(result.status).json(result.data ?? { error: result.error });
 });
+/**
+ * POST /api/yaml-workflows/from-durable
+ * Compile a durable TypeScript workflow into a YAML DAG.
+ * Body: { source, is_file_path?, workflow_name, name, description?, app_id?, subscribes?, tags? }
+ */
+router.post('/from-durable', async (req, res) => {
+    const result = await api.createYamlWorkflowFromDurable(req.body);
+    res.status(result.status).json(result.data ?? { error: result.error });
+});
 // -- Parameterized routes --
 /**
  * GET /api/yaml-workflows/:id

package/build/routes/yaml-workflows/deployment.js

@@ -76,4 +76,12 @@ router.post('/:id/archive', async (req, res) => {
     const result = await api.archiveYamlWorkflow({ id: req.params.id });
     res.status(result.status).json(result.data ?? { error: result.error });
 });
+/**
+ * POST /api/yaml-workflows/:id/restore
+ * Restore an archived YAML workflow back to draft status for redeployment.
+ */
+router.post('/:id/restore', async (req, res) => {
+    const result = await api.restoreYamlWorkflow({ id: req.params.id });
+    res.status(result.status).json(result.data ?? { error: result.error });
+});
 exports.default = router;

package/build/sdk/index.d.ts

@@ -16,6 +16,8 @@ import * as workflowSetsApi from '../api/workflow-sets';
 import * as dbaApi from '../api/dba';
 import * as namespacesApi from '../api/namespaces';
 import * as maintenanceApi from '../api/maintenance';
+import * as filesApi from '../api/files';
+import * as knowledgeApi from '../api/knowledge';
 import type { LTApiAuth, LTApiResult } from '../types/sdk';
 import type { LTEvent, LTEventType } from '../types/events';
 /**
@@ -330,6 +332,21 @@ export declare function createClient(options?: LTClientOptions): {
         getConfig: typeof maintenanceApi.getMaintenanceConfig;
         updateConfig: typeof maintenanceApi.updateMaintenanceConfig;
     };
+    files: {
+        browse: typeof filesApi.browseFiles;
+        getMetadata: typeof filesApi.getFileMetadata;
+        delete: typeof filesApi.deleteFile;
+        generateSignedUrl: typeof filesApi.generateSignedUrl;
+    };
+    knowledge: {
+        listDomains: typeof knowledgeApi.listDomains;
+        listEntries: typeof knowledgeApi.listEntries;
+        getEntry: typeof knowledgeApi.getEntry;
+        storeEntry: typeof knowledgeApi.storeEntry;
+        setField: typeof knowledgeApi.setField;
+        removeField: typeof knowledgeApi.removeField;
+        deleteEntry: typeof knowledgeApi.deleteEntry;
+    };
     events: {
         /**
          * Subscribe to Long Tail events.

package/build/sdk/index.js

@@ -52,6 +52,8 @@ const workflowSetsApi = __importStar(require("../api/workflow-sets"));
 const dbaApi = __importStar(require("../api/dba"));
 const namespacesApi = __importStar(require("../api/namespaces"));
 const maintenanceApi = __importStar(require("../api/maintenance"));
+const filesApi = __importStar(require("../api/files"));
+const knowledgeApi = __importStar(require("../api/knowledge"));
 const events_1 = require("../lib/events");
 const callback_1 = require("../lib/events/callback");
 /** Bind auth context to a function that expects it as the last argument. */
@@ -275,6 +277,23 @@ function createClient(options = {}) {
             getConfig: maintenanceApi.getMaintenanceConfig,
             updateConfig: maintenanceApi.updateMaintenanceConfig,
         },
+        // ── Files ──────────────────────────────────────────────────────────────
+        files: {
+            browse: filesApi.browseFiles,
+            getMetadata: filesApi.getFileMetadata,
+            delete: filesApi.deleteFile,
+            generateSignedUrl: filesApi.generateSignedUrl,
+        },
+        // ── Knowledge ─────────────────────────────────────────────────────────
+        knowledge: {
+            listDomains: knowledgeApi.listDomains,
+            listEntries: knowledgeApi.listEntries,
+            getEntry: knowledgeApi.getEntry,
+            storeEntry: knowledgeApi.storeEntry,
+            setField: knowledgeApi.setField,
+            removeField: knowledgeApi.removeField,
+            deleteEntry: knowledgeApi.deleteEntry,
+        },
         // ── Events ─────────────────────────────────────────────────────────────
         events: {
             /**

package/build/services/insight/index.d.ts

@@ -44,6 +44,19 @@ export interface WorkflowPlannerInput {
     setId: string;
     wait?: boolean;
     userId?: string;
+    /** When adding to an existing set, pass the current plan items so the planner avoids duplicates. */
+    existingPlan?: Array<{
+        name: string;
+        description: string;
+        namespace: string;
+    }>;
+    /** Sibling schemas from already-built workflows for composition context. */
+    existingSchemas?: Array<{
+        name: string;
+        input_schema: Record<string, unknown>;
+        output_schema: Record<string, unknown>;
+        graph_topic: string;
+    }>;
 }
 export declare function startWorkflowPlanner(input: WorkflowPlannerInput): Promise<McpQueryResult>;
 export declare function describeWorkflow(input: DescribeInput): Promise<DescribeResult>;

package/build/services/insight/index.js

@@ -11,6 +11,7 @@ exports.describeWorkflow = describeWorkflow;
 const hotmesh_1 = require("@hotmeshio/hotmesh");
 const db_1 = require("../../lib/db");
 const defaults_1 = require("../../modules/defaults");
+const utils_1 = require("../../modules/utils");
 const llm_1 = require("../llm");
 const prompts_1 = require("./prompts");
 // ── MCP query invocation ─────────────────────────────────────────────────────
@@ -76,13 +77,13 @@ async function startWorkflowBuilder(input) {
     };
 }
 async function startWorkflowPlanner(input) {
-    const { specification, setId, wait = true, userId } = input;
+    const { specification, setId, wait = true, userId, existingPlan, existingSchemas } = input;
     const startTime = Date.now();
     const client = new hotmesh_1.Durable.Client({ connection: (0, db_1.getConnection)() });
     const workflowId = `wf-planner-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
     const handle = await client.workflow.start({
         args: [{
-                data: { specification, setId },
+                data: { specification, setId, existingPlan, existingSchemas },
                 metadata: { source: 'dashboard' },
                 lt: { userId },
             }],
@@ -126,11 +127,7 @@ async function describeWorkflow(input) {
     const raw = response.content || '{}';
     const cleaned = raw.replace(/^```(?:json)?\s*/m, '').replace(/\s*```$/m, '').trim();
     const parsed = JSON.parse(cleaned);
-    const toolName = (parsed.tool_name || '')
-        .toLowerCase()
-        .replace(/[^a-z0-9]+/g, '-')
-        .replace(/^-|-$/g, '')
-        .slice(0, 60);
+    const toolName = (0, utils_1.sanitizeToolName)(parsed.tool_name || '').slice(0, 60);
     return {
         tool_name: toolName || undefined,
         description: parsed.description || prompt,

package/build/services/insight/prompts.d.ts

@@ -1,4 +1,4 @@
 /**
  * Externalized LLM prompt constants for the insight service.
  */
-export declare const DESCRIBE_WORKFLOW_SYSTEM_PROMPT = "You generate concise workflow descriptions, tool names, and discovery tags.\n\nGiven a user's original query and the execution result, produce:\n1. A short, descriptive tool name as a lowercase kebab-case slug (e.g. \"screenshot-all-nav-pages\", \"fetch-order-status\", \"translate-content\"). The name should describe what the workflow does, not be generic like \"query-complete\".\n2. A clear, reusable description of what this workflow does (not what the user asked, but what the workflow accomplishes as a reusable tool). Write it as if describing a tool in a catalog. 2-3 sentences max.\n3. Discovery tags \u2014 lowercase keywords that help find this workflow when similar future queries are made.\n\nReturn ONLY a JSON object:\n{ \"tool_name\": \"...\", \"description\": \"...\", \"tags\": [\"tag1\", \"tag2\", ...] }";
+export declare const DESCRIBE_WORKFLOW_SYSTEM_PROMPT = "You generate concise workflow descriptions, tool names, and discovery tags.\n\nGiven a user's original query and the execution result, produce:\n1. A short, descriptive tool name in snake_case (e.g. \"screenshot_all_nav_pages\", \"fetch_order_status\", \"translate_content\"). Only lowercase letters, digits, and underscores. The name should describe what the workflow does, not be generic like \"query_complete\".\n2. A clear, reusable description of what this workflow does (not what the user asked, but what the workflow accomplishes as a reusable tool). Write it as if describing a tool in a catalog. 2-3 sentences max.\n3. Discovery tags \u2014 lowercase keywords that help find this workflow when similar future queries are made.\n\nReturn ONLY a JSON object:\n{ \"tool_name\": \"...\", \"description\": \"...\", \"tags\": [\"tag1\", \"tag2\", ...] }";

package/build/services/insight/prompts.js

@@ -7,7 +7,7 @@ exports.DESCRIBE_WORKFLOW_SYSTEM_PROMPT = void 0;
 exports.DESCRIBE_WORKFLOW_SYSTEM_PROMPT = `You generate concise workflow descriptions, tool names, and discovery tags.
 
 Given a user's original query and the execution result, produce:
-1. A short, descriptive tool name as a lowercase kebab-case slug (e.g. "screenshot-all-nav-pages", "fetch-order-status", "translate-content"). The name should describe what the workflow does, not be generic like "query-complete".
+1. A short, descriptive tool name in snake_case (e.g. "screenshot_all_nav_pages", "fetch_order_status", "translate_content"). Only lowercase letters, digits, and underscores. The name should describe what the workflow does, not be generic like "query_complete".
 2. A clear, reusable description of what this workflow does (not what the user asked, but what the workflow accomplishes as a reusable tool). Write it as if describing a tool in a catalog. 2-3 sentences max.
 3. Discovery tags — lowercase keywords that help find this workflow when similar future queries are made.
 

package/build/services/mcp/client/connection-dispatch.d.ts

@@ -0,0 +1,18 @@
+import type { LTMcpToolManifest } from '../../../types';
+/**
+ * Dispatch a tool call directly to a built-in server's handler,
+ * bypassing MCP Client/Transport entirely. Returns null if the server
+ * or tool is not a built-in — caller should fall through to MCP transport.
+ *
+ * Each built-in server is lazily instantiated once and cached. Tool handlers
+ * are called via server._registeredTools[toolName].handler(args). This
+ * eliminates the InMemoryTransport bottleneck under concurrent load.
+ */
+export declare function dispatchBuiltinTool(serverId: string, toolName: string, args: Record<string, any>): Promise<{
+    dispatched: true;
+    result: any;
+} | null>;
+/**
+ * List tools from a connected server.
+ */
+export declare function listServerTools(serverId: string): Promise<LTMcpToolManifest[]>;

package/build/services/mcp/client/connection-dispatch.js

@@ -0,0 +1,78 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.dispatchBuiltinTool = dispatchBuiltinTool;
+exports.listServerTools = listServerTools;
+const logger_1 = require("../../../lib/logger");
+const connection_lifecycle_1 = require("./connection-lifecycle");
+/**
+ * Dispatch a tool call directly to a built-in server's handler,
+ * bypassing MCP Client/Transport entirely. Returns null if the server
+ * or tool is not a built-in — caller should fall through to MCP transport.
+ *
+ * Each built-in server is lazily instantiated once and cached. Tool handlers
+ * are called via server._registeredTools[toolName].handler(args). This
+ * eliminates the InMemoryTransport bottleneck under concurrent load.
+ */
+async function dispatchBuiltinTool(serverId, toolName, args) {
+    const builtinFactories = (0, connection_lifecycle_1.getBuiltinFactories)();
+    const builtinServers = (0, connection_lifecycle_1.getBuiltinServers)();
+    // Normalize and match against builtin factories
+    const norm = (s) => s.replace(/[^a-zA-Z0-9]/g, '').toLowerCase();
+    const normId = norm(serverId);
+    let matchedName = null;
+    for (const [name] of builtinFactories) {
+        const normName = norm(name);
+        if (normName === normId || normName.includes(normId) || normId.includes(normName)) {
+            matchedName = name;
+            break;
+        }
+    }
+    if (!matchedName)
+        return null;
+    // Lazily create and cache the server instance
+    if (!builtinServers.has(matchedName)) {
+        const factory = builtinFactories.get(matchedName);
+        const server = await factory();
+        builtinServers.set(matchedName, server);
+        logger_1.loggerRegistry.info(`[lt-mcp:builtin] ${matchedName} ready (direct dispatch)`);
+    }
+    const server = builtinServers.get(matchedName);
+    const tool = server._registeredTools?.[toolName];
+    if (!tool?.handler)
+        return null;
+    // Call the handler directly — no transport, no JSON-RPC.
+    // Tool handlers return MCP-shaped responses: { content: [{ type: 'text', text: '...' }] }
+    // Parse the text content the same way callServerTool does.
+    const mcpResponse = await tool.handler(args);
+    let parsed = mcpResponse;
+    if (mcpResponse && Array.isArray(mcpResponse.content)) {
+        const textContent = mcpResponse.content.find((c) => c.type === 'text');
+        if (textContent && 'text' in textContent) {
+            try {
+                parsed = JSON.parse(textContent.text);
+            }
+            catch {
+                parsed = mcpResponse.isError ? { error: textContent.text } : textContent.text;
+            }
+        }
+    }
+    const isError = parsed && typeof parsed === 'object' && 'error' in parsed;
+    logger_1.loggerRegistry.debug(`[lt-mcp:builtin] ${matchedName}/${toolName} ok=${!isError} resultKeys=[${typeof parsed === 'object' && parsed ? Object.keys(parsed).join(',') : 'raw'}]`);
+    return { dispatched: true, result: parsed };
+}
+/**
+ * List tools from a connected server.
+ */
+async function listServerTools(serverId) {
+    const clients = (0, connection_lifecycle_1.getClients)();
+    const client = clients.get(serverId);
+    if (!client) {
+        throw new Error(`MCP server ${serverId} is not connected`);
+    }
+    const { tools } = await client.listTools();
+    return tools.map((t) => ({
+        name: t.name,
+        description: t.description || '',
+        inputSchema: t.inputSchema || {},
+    }));
+}

package/build/services/mcp/client/connection-lifecycle.d.ts

@@ -0,0 +1,48 @@
+import { Client } from '@modelcontextprotocol/sdk/client/index.js';
+import type { LTMcpServerRecord } from '../../../types';
+/** Read-only access to the clients map for sibling modules. */
+export declare function getClients(): Map<string, Client>;
+/** Read-only access to the builtin factories map for sibling modules. */
+export declare function getBuiltinFactories(): Map<string, () => Promise<any>>;
+/** Read-only access to the builtin servers map for sibling modules. */
+export declare function getBuiltinServers(): Map<string, any>;
+/**
+ * Register a built-in server factory so it can be auto-connected
+ * when callServerTool is invoked with its name.
+ */
+export declare function registerBuiltinServer(name: string, factory: () => Promise<any>): void;
+/**
+ * Connect to a registered MCP server.
+ * Creates the appropriate transport based on transport_type,
+ * connects, and caches tool manifest in DB.
+ */
+export declare function connectToServer(server: LTMcpServerRecord): Promise<Client>;
+/**
+ * Disconnect from a specific server.
+ */
+export declare function disconnectFromServer(serverId: string): Promise<void>;
+/**
+ * Resolve a server by ID or name, auto-connecting built-in servers if needed.
+ * Returns the client or null if not found.
+ *
+ * Built-in servers are connected once under their canonical factory name.
+ * Alias lookups (e.g. 'translation' matching 'long-tail-translation') reuse
+ * the same client instance to avoid double-connecting the singleton server.
+ */
+export declare function resolveClient(serverId: string): Promise<Client | null>;
+/**
+ * Connect to all auto-connect servers.
+ */
+export declare function connectAutoServers(): Promise<void>;
+/**
+ * Disconnect all clients.
+ */
+export declare function disconnectAll(): Promise<void>;
+/**
+ * Check if a server is connected.
+ */
+export declare function isConnected(serverId: string): boolean;
+/**
+ * Clear all state. Used in tests.
+ */
+export declare function clear(): void;