@hotmeshio/long-tail 0.1.16 → 0.1.17

package/build/services/mcp/db.js

@@ -9,7 +9,10 @@ exports.updateMcpServerStatus = updateMcpServerStatus;
 exports.listMcpServers = listMcpServers;
 exports.getAutoConnectServers = getAutoConnectServers;
 exports.findServersByTags = findServersByTags;
+exports.seedMcpServer = seedMcpServer;
+exports.cleanStaleBuiltinServers = cleanStaleBuiltinServers;
 const db_1 = require("../../lib/db");
+const logger_1 = require("../../lib/logger");
 const sql_1 = require("./sql");
 async function createMcpServer(input) {
     const pool = (0, db_1.getPool)();
@@ -149,3 +152,50 @@ async function findServersByTags(tags, match = 'any') {
     const { rows } = await pool.query(`SELECT * FROM lt_mcp_servers WHERE tags ${operator} $1::text[] ORDER BY name`, [tags]);
     return rows;
 }
+/**
+ * Seed an MCP server at startup (insert-if-absent).
+ * DB is the source of truth — if the row already exists, log drift warnings
+ * but do not overwrite. Returns true if inserted, false if already existed.
+ */
+async function seedMcpServer(input) {
+    const pool = (0, db_1.getPool)();
+    const { rowCount } = await pool.query(sql_1.SEED_MCP_SERVER, [
+        input.name,
+        input.description || null,
+        'stdio',
+        JSON.stringify({ builtin: true, process: 'in-memory' }),
+        JSON.stringify(input.toolManifest || []),
+        JSON.stringify({ builtin: true }),
+        input.tags || [],
+        input.compileHints || null,
+        input.credentialProviders || [],
+    ]);
+    const inserted = (rowCount ?? 0) > 0;
+    if (!inserted) {
+        // Drift detection
+        const existing = await getMcpServerByName(input.name);
+        if (existing) {
+            const drifts = [];
+            if (input.description && existing.description !== input.description)
+                drifts.push('description');
+            if (input.compileHints && existing.compile_hints !== input.compileHints)
+                drifts.push('compile_hints');
+            if (JSON.stringify(input.tags || []) !== JSON.stringify(existing.tags || []))
+                drifts.push('tags');
+            if (drifts.length) {
+                logger_1.loggerRegistry.warn(`[long-tail] MCP server drift: ${input.name} — ${drifts.join(', ')} differ between code and DB`);
+            }
+        }
+    }
+    return inserted;
+}
+/**
+ * Remove builtin MCP servers that are no longer declared in factory config.
+ */
+async function cleanStaleBuiltinServers(activeNames) {
+    const pool = (0, db_1.getPool)();
+    const { rows } = await pool.query(sql_1.DELETE_STALE_BUILTIN_SERVERS, [activeNames]);
+    for (const row of rows) {
+        logger_1.loggerRegistry.info(`[long-tail] removed stale builtin MCP server: ${row.name}`);
+    }
+}

package/build/services/mcp/sql.d.ts

@@ -15,4 +15,7 @@ export declare const HEALTH_COMPILED_WORKFLOWS = "\n  SELECT\n    COUNT(*)::int
 export declare const HEALTH_WORKFLOW_CONFIGS = "\n  SELECT workflow_type, task_queue, description\n  FROM lt_config_workflows ORDER BY workflow_type";
 export declare const HEALTH_DURABLE_WORKFLOWS = "\n  SELECT\n    COALESCE(entity, '_internal') AS entity,\n    COUNT(*)::int AS total,\n    COUNT(*) FILTER (WHERE status = 0)::int AS active,\n    COUNT(*) FILTER (WHERE status = 1)::int AS completed\n  FROM durable.jobs\n  WHERE entity IS NOT NULL AND entity != ''\n  GROUP BY entity\n  ORDER BY entity";
 export declare const DELETE_STALE_BUILTIN_SERVERS = "\n  DELETE FROM lt_mcp_servers\n  WHERE (metadata->>'builtin')::boolean = true\n    AND name != ALL($1)\n  RETURNING name";
-export declare const SEED_MCP_SERVER = "\n  INSERT INTO lt_mcp_servers\n    (name, description, transport_type, transport_config, auto_connect, status, tool_manifest, metadata, tags, compile_hints, credential_providers, last_connected_at)\n  VALUES ($1, $2, $3, $4, true, 'connected', $5, $6, $7, $8, $9, NOW())\n  ON CONFLICT (name) DO UPDATE SET\n    tool_manifest = EXCLUDED.tool_manifest,\n    metadata = EXCLUDED.metadata,\n    description = EXCLUDED.description,\n    tags = EXCLUDED.tags,\n    compile_hints = EXCLUDED.compile_hints,\n    credential_providers = EXCLUDED.credential_providers,\n    status = 'connected',\n    last_connected_at = NOW()";
+/** Upsert — used by dashboard/API writes (overwrites intentionally). */
+export declare const UPSERT_MCP_SERVER = "\n  INSERT INTO lt_mcp_servers\n    (name, description, transport_type, transport_config, auto_connect, status, tool_manifest, metadata, tags, compile_hints, credential_providers, last_connected_at)\n  VALUES ($1, $2, $3, $4, true, 'connected', $5, $6, $7, $8, $9, NOW())\n  ON CONFLICT (name) DO UPDATE SET\n    tool_manifest = EXCLUDED.tool_manifest,\n    metadata = EXCLUDED.metadata,\n    description = EXCLUDED.description,\n    tags = EXCLUDED.tags,\n    compile_hints = EXCLUDED.compile_hints,\n    credential_providers = EXCLUDED.credential_providers,\n    status = 'connected',\n    last_connected_at = NOW()";
+/** Seed — insert-if-absent. DB is source of truth after first boot. */
+export declare const SEED_MCP_SERVER = "\n  INSERT INTO lt_mcp_servers\n    (name, description, transport_type, transport_config, auto_connect, status, tool_manifest, metadata, tags, compile_hints, credential_providers, last_connected_at)\n  VALUES ($1, $2, $3, $4, true, 'connected', $5, $6, $7, $8, $9, NOW())\n  ON CONFLICT (name) DO NOTHING";

package/build/services/mcp/sql.js

@@ -1,7 +1,7 @@
 "use strict";
 // ─── MCP server CRUD ────────────────────────────────────────────────────────
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SEED_MCP_SERVER = exports.DELETE_STALE_BUILTIN_SERVERS = exports.HEALTH_DURABLE_WORKFLOWS = exports.HEALTH_WORKFLOW_CONFIGS = exports.HEALTH_COMPILED_WORKFLOWS = exports.HEALTH_MCP_SERVER_LIST = exports.HEALTH_MCP_SERVERS = exports.HEALTH_RECENT_ACTIVITY = exports.HEALTH_ACTIVE_WORKFLOW_TYPES = exports.HEALTH_ESCALATION_COUNTS = exports.HEALTH_TASK_COUNTS = exports.GET_AUTO_CONNECT_SERVERS = exports.UPDATE_STATUS = exports.UPDATE_STATUS_CONNECTED = exports.DELETE_MCP_SERVER = exports.GET_MCP_SERVER_BY_NAME = exports.GET_MCP_SERVER = exports.CREATE_MCP_SERVER = void 0;
+exports.SEED_MCP_SERVER = exports.UPSERT_MCP_SERVER = exports.DELETE_STALE_BUILTIN_SERVERS = exports.HEALTH_DURABLE_WORKFLOWS = exports.HEALTH_WORKFLOW_CONFIGS = exports.HEALTH_COMPILED_WORKFLOWS = exports.HEALTH_MCP_SERVER_LIST = exports.HEALTH_MCP_SERVERS = exports.HEALTH_RECENT_ACTIVITY = exports.HEALTH_ACTIVE_WORKFLOW_TYPES = exports.HEALTH_ESCALATION_COUNTS = exports.HEALTH_TASK_COUNTS = exports.GET_AUTO_CONNECT_SERVERS = exports.UPDATE_STATUS = exports.UPDATE_STATUS_CONNECTED = exports.DELETE_MCP_SERVER = exports.GET_MCP_SERVER_BY_NAME = exports.GET_MCP_SERVER = exports.CREATE_MCP_SERVER = void 0;
 exports.CREATE_MCP_SERVER = `
   INSERT INTO lt_mcp_servers
     (name, description, transport_type, transport_config, auto_connect, metadata, tags, compile_hints, credential_providers)
@@ -77,7 +77,8 @@ exports.DELETE_STALE_BUILTIN_SERVERS = `
   WHERE (metadata->>'builtin')::boolean = true
     AND name != ALL($1)
   RETURNING name`;
-exports.SEED_MCP_SERVER = `
+/** Upsert — used by dashboard/API writes (overwrites intentionally). */
+exports.UPSERT_MCP_SERVER = `
   INSERT INTO lt_mcp_servers
     (name, description, transport_type, transport_config, auto_connect, status, tool_manifest, metadata, tags, compile_hints, credential_providers, last_connected_at)
   VALUES ($1, $2, $3, $4, true, 'connected', $5, $6, $7, $8, $9, NOW())
@@ -90,3 +91,9 @@ exports.SEED_MCP_SERVER = `
     credential_providers = EXCLUDED.credential_providers,
     status = 'connected',
     last_connected_at = NOW()`;
+/** Seed — insert-if-absent. DB is source of truth after first boot. */
+exports.SEED_MCP_SERVER = `
+  INSERT INTO lt_mcp_servers
+    (name, description, transport_type, transport_config, auto_connect, status, tool_manifest, metadata, tags, compile_hints, credential_providers, last_connected_at)
+  VALUES ($1, $2, $3, $4, true, 'connected', $5, $6, $7, $8, $9, NOW())
+  ON CONFLICT (name) DO NOTHING`;

package/build/services/oauth/index.js

@@ -57,11 +57,14 @@ function initializeOAuth(config) {
         }
     }
     // Auto-detect providers from environment variables
+    const port = process.env.PORT || '3000';
+    const baseUrl = process.env.BASE_URL || `http://localhost:${port}`;
     for (const { provider, idVar, secretVar } of ENV_PROVIDERS) {
         const clientId = process.env[idVar];
         const clientSecret = process.env[secretVar];
         if (clientId && clientSecret && !(0, providers_1.getProvider)(provider)) {
-            (0, providers_1.registerProvider)({ provider, clientId, clientSecret, scopes: [] });
+            const redirectUri = `${baseUrl}/api/auth/oauth/${provider}/callback`;
+            (0, providers_1.registerProvider)({ provider, clientId, clientSecret, scopes: [], redirectUri });
         }
     }
     // Register built-in credential providers (API key paste flow, no real OAuth)

package/build/services/oauth/providers/google.js

@@ -48,8 +48,13 @@ function createGoogleHandler(cfg) {
         config: cfg,
         async createAuthorizationURL(state, codeVerifier) {
             const google = await getClient();
-            const scopes = cfg.scopes.length > 0 ? cfg.scopes : ['openid', 'email', 'profile'];
-            return google.createAuthorizationURL(state, codeVerifier, scopes);
+            const scopes = cfg.scopes.length > 0
+                ? cfg.scopes
+                : ['openid', 'email', 'profile', 'https://www.googleapis.com/auth/gmail.readonly'];
+            const url = google.createAuthorizationURL(state, codeVerifier, scopes);
+            url.searchParams.set('access_type', 'offline');
+            url.searchParams.set('prompt', 'consent');
+            return url;
         },
         async validateAuthorizationCode(code, codeVerifier) {
             const google = await getClient();

package/build/services/orchestrator/index.js

@@ -131,6 +131,10 @@ async function executeLT(options) {
     if (envelope) {
         const userId = envelope.lt?.userId || orchCtx?.userId;
         envelope.lt = { ...envelope.lt, taskId, originId, parentId, userId };
+        // Propagate certified flag from parent to child
+        if (envelope0?.metadata?.certified === true && envelope.metadata) {
+            envelope.metadata.certified = true;
+        }
     }
     // 5. Start child workflow (fire-and-forget — only the start is awaited)
     await hotmesh_1.Durable.workflow.startChild({
@@ -140,6 +144,7 @@ async function executeLT(options) {
         workflowId: childWorkflowId,
         expire: expire || defaults_1.JOB_EXPIRE_SECS,
         entity: workflowName,
+        signalIn: false,
     });
     // 6. Wait for the child's interceptor to signal back with the result
     const result = await hotmesh_1.Durable.workflow.condition(signalId);

package/build/services/task/crud.js

@@ -28,6 +28,7 @@ async function createTask(input) {
         input.initiated_by || null,
         input.principal_type || null,
         input.executing_as || null,
+        input.status || null,
     ]);
     return rows[0];
 }

package/build/services/task/sql.d.ts

@@ -1,4 +1,4 @@
-export declare const CREATE_TASK = "\n  INSERT INTO lt_tasks\n    (workflow_id, workflow_type, lt_type, task_queue, signal_id,\n     parent_workflow_id, origin_id, parent_id, envelope, metadata, priority,\n     trace_id, span_id, initiated_by, principal_type, executing_as)\n  VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)\n  RETURNING *";
+export declare const CREATE_TASK = "\n  INSERT INTO lt_tasks\n    (workflow_id, workflow_type, lt_type, task_queue, signal_id,\n     parent_workflow_id, origin_id, parent_id, envelope, metadata, priority,\n     trace_id, span_id, initiated_by, principal_type, executing_as, status)\n  VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, COALESCE($17, 'pending'))\n  RETURNING *";
 export declare const APPEND_MILESTONES = "\n  UPDATE lt_tasks\n  SET milestones = milestones || $1::jsonb\n  WHERE id = $2\n  RETURNING *";
 export declare const GET_TASK = "SELECT * FROM lt_tasks WHERE id = $1";
 export declare const GET_TASK_BY_SIGNAL_ID = "SELECT * FROM lt_tasks WHERE signal_id = $1";

package/build/services/task/sql.js

@@ -11,8 +11,8 @@ exports.CREATE_TASK = `
   INSERT INTO lt_tasks
     (workflow_id, workflow_type, lt_type, task_queue, signal_id,
      parent_workflow_id, origin_id, parent_id, envelope, metadata, priority,
-     trace_id, span_id, initiated_by, principal_type, executing_as)
-  VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)
+     trace_id, span_id, initiated_by, principal_type, executing_as, status)
+  VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, COALESCE($17, 'pending'))
   RETURNING *`;
 exports.APPEND_MILESTONES = `
   UPDATE lt_tasks

package/build/services/task/types.d.ts

@@ -23,6 +23,8 @@ export interface CreateTaskInput {
     principal_type?: string;
     /** Executing principal (bot external_id) when proxy invocation is used. */
     executing_as?: string;
+    /** Initial status. Default: 'pending'. */
+    status?: string;
 }
 export interface UpdateTaskInput {
     status?: LTTaskStatus;

package/build/services/workers/registry.js

@@ -17,10 +17,12 @@ function getRegisteredWorkers() {
 /** System workflows excluded from the overview by default. */
 exports.SYSTEM_WORKFLOWS = new Set([
     'mcpQuery',
-    'mcpDeterministic',
     'mcpQueryRouter',
+    'mcpDeterministic',
     'mcpTriage',
     'mcpTriageRouter',
     'mcpTriageDeterministic',
+    'mcpWorkflowBuilder',
+    'mcpWorkflowPlanner',
     'insightQuery',
 ]);

package/build/services/workflow-invocation.js

@@ -91,7 +91,13 @@ async function invokeWorkflow(input) {
     }
     // 4. Resolve principals and build envelope
     const wfConfig = await configService.getWorkflowConfig(workflowType);
-    const envelope = await buildEnvelope(data, metadata ?? {}, auth.userId, executeAsOverride, wfConfig?.execute_as ?? undefined);
+    const isCertified = !!(wfConfig &&
+        ((wfConfig.roles?.length ?? 0) > 0 || (wfConfig.consumes?.length ?? 0) > 0));
+    const resolvedMetadata = { ...(metadata ?? {}) };
+    if (isCertified && resolvedMetadata.certified !== false) {
+        resolvedMetadata.certified = true;
+    }
+    const envelope = await buildEnvelope(data, resolvedMetadata, auth.userId, executeAsOverride, wfConfig?.execute_as ?? undefined);
     // 5. Start workflow — caller options pass through to Durable client
     const client = (0, workers_1.createClient)();
     const callerOpts = input.options ?? {};
@@ -104,6 +110,7 @@ async function invokeWorkflow(input) {
         workflowId,
         expire: callerOpts.expire ?? defaults_1.JOB_EXPIRE_SECS,
         entity: callerOpts.entity ?? workflowType,
+        signalIn: false,
     });
     return { workflowId };
 }

package/build/start/workers.d.ts

@@ -1,4 +1,4 @@
-import type { LTStartConfig } from '../types/startup';
+import type { LTStartConfig, LTWorkerConfig } from '../types/startup';
 type WorkerEntry = {
     taskQueue: string;
     workflow: (...args: any[]) => any;
@@ -6,6 +6,7 @@ type WorkerEntry = {
         readonly?: boolean;
         retry?: Record<string, unknown>;
     };
+    config?: LTWorkerConfig;
 };
 /**
  * Build the connection descriptor used by HotMesh / Durable.

package/build/start/workers.js

@@ -132,6 +132,39 @@ async function startWorkers(startConfig, workers, builtinMcpServerFactories) {
             (0, registry_1.registerWorker)(w.workflow.name, w.taskQueue);
         }
         logger_1.loggerRegistry.info(`[long-tail] workers started on queues: ${workers.map((w) => w.taskQueue).join(', ')}`);
+        // Seed workflow configs (insert-if-absent — DB is source of truth)
+        const workersWithConfig = workers.filter((w) => w.config);
+        if (workersWithConfig.length) {
+            const { seedWorkflowConfig } = await Promise.resolve().then(() => __importStar(require('../services/config/write')));
+            const { ltConfig } = await Promise.resolve().then(() => __importStar(require('../modules/ltconfig')));
+            for (const w of workersWithConfig) {
+                const workflowType = w.workflow.name;
+                const c = w.config;
+                try {
+                    const inserted = await seedWorkflowConfig({
+                        workflow_type: workflowType,
+                        task_queue: w.taskQueue,
+                        invocable: c.invocable ?? false,
+                        default_role: c.defaultRole ?? 'reviewer',
+                        description: c.description ?? null,
+                        roles: c.roles ?? [],
+                        invocation_roles: c.invocationRoles ?? [],
+                        consumes: c.consumes ?? [],
+                        tool_tags: c.toolTags ?? [],
+                        envelope_schema: c.envelopeSchema ?? null,
+                        resolver_schema: c.resolverSchema ?? null,
+                        cron_schedule: c.cronSchedule ?? null,
+                        execute_as: c.executeAs ?? null,
+                    });
+                    if (inserted)
+                        logger_1.loggerRegistry.info(`[long-tail] config seeded: ${workflowType}`);
+                }
+                catch (err) {
+                    logger_1.loggerRegistry.warn(`[long-tail] config seed failed for ${workflowType}: ${err.message}`);
+                }
+            }
+            ltConfig.invalidate();
+        }
         // Start maintenance cron
         if (maintenance_1.maintenanceRegistry.hasConfig) {
             await maintenance_1.maintenanceRegistry.connect();
@@ -145,15 +178,45 @@ async function startWorkers(startConfig, workers, builtinMcpServerFactories) {
             logger_1.loggerRegistry.info('[long-tail] MCP adapter connected');
         }
         // Register MCP server factories: built-in (from system/) + user-provided
+        // Both system and user factories can carry inline config for DB seeding.
         const { registerBuiltinServer } = await Promise.resolve().then(() => __importStar(require('../services/mcp/client')));
+        const { seedMcpServer, cleanStaleBuiltinServers } = await Promise.resolve().then(() => __importStar(require('../services/mcp/db')));
+        const userFactories = startConfig.mcp?.serverFactories ?? {};
+        // Resolve user factories — plain function or { factory, config }
+        const resolvedUserFactories = {};
+        for (const [name, entry] of Object.entries(userFactories)) {
+            if (typeof entry === 'function') {
+                resolvedUserFactories[name] = { factory: entry };
+            }
+            else {
+                resolvedUserFactories[name] = entry;
+            }
+        }
+        // Merge system (always have config) + user factories
         const allFactories = {
             ...builtinMcpServerFactories,
-            ...(startConfig.mcp?.serverFactories ?? {}),
+            ...resolvedUserFactories,
         };
-        for (const [name, factory] of Object.entries(allFactories)) {
-            registerBuiltinServer(name, factory);
+        // 1. Register all factories (runtime — always applied)
+        for (const [name, entry] of Object.entries(allFactories)) {
+            registerBuiltinServer(name, entry.factory);
         }
         logger_1.loggerRegistry.info(`[long-tail] ${Object.keys(allFactories).length} MCP server factories registered`);
+        // 2. Seed MCP server configs (insert-if-absent + drift log)
+        for (const [name, entry] of Object.entries(allFactories)) {
+            if (entry.config) {
+                try {
+                    const inserted = await seedMcpServer({ name, ...entry.config });
+                    if (inserted)
+                        logger_1.loggerRegistry.info(`[long-tail] MCP server seeded: ${name}`);
+                }
+                catch (err) {
+                    logger_1.loggerRegistry.warn(`[long-tail] MCP server seed failed for ${name}: ${err.message}`);
+                }
+            }
+        }
+        // 3. Clean stale builtin servers no longer in factory list
+        await cleanStaleBuiltinServers(Object.keys(allFactories));
         // Register workers for active YAML (deterministic) workflows
         await yamlWorkflowWorkers.registerAllActiveWorkers();
     }
@@ -163,9 +226,6 @@ async function startWorkers(startConfig, workers, builtinMcpServerFactories) {
         await events_1.eventRegistry.connect();
         logger_1.loggerRegistry.info('[long-tail] event adapters connected');
     }
-    // Seed system MCP servers (always)
-    const { seedSystemMcpServers } = await Promise.resolve().then(() => __importStar(require('../system/seed')));
-    await seedSystemMcpServers();
     // Ensure system bot account exists for cron/system-initiated workflows
     const { ensureSystemBot } = await Promise.resolve().then(() => __importStar(require('../services/iam/bots')));
     await ensureSystemBot().catch((err) => logger_1.loggerRegistry.warn(`[long-tail] system bot seed error: ${err.message}`));

package/build/system/index.d.ts

@@ -1,16 +1,18 @@
 /**
  * System workers and built-in MCP server factories.
  *
- * All MCP-powered workflows (triage, query, their routers and deterministic
- * counterparts) require an LLM API key. They load conditionally.
+ * All declarations carry inline `config` that seeds the DB on first boot.
+ * DB is the source of truth after seeding — startup never overwrites.
  */
-export declare function getSystemWorkers(): Array<{
+import type { LTWorkerConfig, LTMcpServerConfig } from '../types/startup';
+export type SystemWorkerEntry = {
     taskQueue: string;
     workflow: (...args: any[]) => any;
-}>;
-/**
- * Built-in MCP server factories.
- * Each entry maps a server name to a lazy factory that creates the McpServer instance.
- * Registered via `registerBuiltinServer()` in `services/mcp/client.ts`.
- */
-export declare const builtinMcpServerFactories: Record<string, () => Promise<any>>;
+    config?: LTWorkerConfig;
+};
+export declare function getSystemWorkers(): SystemWorkerEntry[];
+export type McpServerFactoryEntry = {
+    factory: () => Promise<any>;
+    config: LTMcpServerConfig;
+};
+export declare const builtinMcpServerFactories: Record<string, McpServerFactoryEntry>;