@hotmeshio/long-tail 0.1.5 → 0.1.7

package/README.md

@@ -82,9 +82,39 @@ curl -X PUT http://localhost:3000/api/workflows/reviewContent/config \
 
 De-certifying removes the interceptor. The workflow continues as a standard durable workflow — same code, different guarantees.
 
-## Register an MCP Server
+## Register MCP Servers
 
-Write an MCP server to expose your own tools. The `registerMcpTool` helper handles the SDK's type complexity.
+Long Tail connects to any MCP server — an npm package, a remote service, or one you write yourself. Registered tools become durable activities and are available to the Pipeline Designer.
+
+**Use an existing package** — no code, just register:
+
+```bash
+curl -X POST http://localhost:3000/api/mcp/servers \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "name": "filesystem",
+    "transport_type": "stdio",
+    "transport_config": { "command": "npx", "args": ["-y", "@modelcontextprotocol/server-filesystem", "/data"] },
+    "tags": ["files", "storage"],
+    "auto_connect": true
+  }'
+```
+
+**Connect a remote server** — point at a URL:
+
+```bash
+curl -X POST http://localhost:3000/api/mcp/servers \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "name": "my-python-server",
+    "transport_type": "sse",
+    "transport_config": { "url": "http://python-service:8000/mcp" },
+    "tags": ["ml", "classification"],
+    "compile_hints": "Returns confidence scores. Use threshold 0.85 for auto-approve."
+  }'
+```
+
+**Write your own** and register it in-process:
 
 ```typescript
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
@@ -106,8 +136,6 @@ export function createImageToolsServer(): McpServer {
 }
 ```
 
-Register it at startup and it appears in the dashboard:
-
 ```typescript
 const lt = await start({
   // ...
@@ -117,6 +145,8 @@ const lt = await start({
 });
 ```
 
+All three paths produce the same outcome: tools callable as durable activities. Tags enable discovery. Compile hints guide the compiler when tools are compiled into deterministic pipelines. See the [MCP guide](docs/mcp.md) for the full registration lifecycle.
+
 ## Ask It Anything
 
 Once your tools are registered, the Pipeline Designer orchestrates them. Describe what you need in plain language:
@@ -147,34 +177,74 @@ const lt = await start({
 });
 ```
 
+## Use as a Package (No HTTP Server)
+
+Long Tail can run as an embedded package inside your existing application — no Express server, no socket.io, no extra ports. The same API surface is available as direct function calls.
+
+```typescript
+import { start, createClient } from '@hotmeshio/long-tail';
+
+await start({
+  database: { connectionString: process.env.DATABASE_URL },
+  server: { enabled: false },
+  workers: [{ taskQueue: 'default', workflow: reviewContent.reviewContent }],
+});
+
+const lt = createClient({ auth: { userId: 'system' } });
+
+// Same operations as the REST API — no HTTP overhead
+const tasks = await lt.tasks.list({ status: 'completed', limit: 10 });
+const result = await lt.escalations.claim({ id: 'esc_123', durationMinutes: 30 });
+```
+
+Subscribe to events with callbacks instead of socket.io:
+
+```typescript
+lt.events.on('task.completed', (event) => {
+  console.log('done:', event.workflowId);
+});
+
+lt.events.on('escalation.*', (event) => {
+  notifyTeam(event);
+});
+```
+
+Every SDK call returns an `LTApiResult` — same status codes, same validation, same RBAC. The transport is the only thing that changes. See the [SDK guide](docs/sdk.md) for the full API reference.
+
 ## Deployment
 
-Two container types from the same codebase:
+Three modes from the same codebase:
 
 ```typescript
-// api.ts — dashboard + REST API
+// 1. Standalone — dashboard + REST API + workers
 await start({ database: { connectionString: process.env.DATABASE_URL } });
 
-// worker.ts — workflow execution, no HTTP server
+// 2. Worker-only — workflow execution, no HTTP server
 await start({
   database: { connectionString: process.env.DATABASE_URL },
   server: { enabled: false },
   workers: [{ taskQueue: 'default', workflow: reviewContent.reviewContent }],
 });
+
+// 3. Embedded — inside your NestJS/Next.js/Express app, SDK calls only
+await start({ database: { connectionString: process.env.DATABASE_URL }, server: { enabled: false } });
+const lt = createClient({ auth: { userId: 'service' } });
 ```
 
-Both share PostgreSQL and scale independently. See [Cloud Deployment](docs/cloud.md).
+All modes share PostgreSQL and scale independently. See [Cloud Deployment](docs/cloud.md).
 
 ## Docs
 
 | Guide | What it covers |
 |-------|---------------|
+| [The Long Tail Story](docs/story.md) | Why this exists, what accumulates over time, what you own |
 | [Workflows](docs/workflows.md) | Activities, interceptor, escalation lifecycle, composition |
 | [IAM](docs/iam.md) | Identity propagation, service accounts, credential exchange |
 | [Dashboard](docs/dashboard.md) | Navigation, key pages, event feed |
 | [MCP](docs/mcp.md) | Server registration, tool calls, human queue |
 | [Compilation](docs/compilation.md) | Dynamic → deterministic pipeline wizard |
 | [Escalation Strategies](docs/escalation-strategies.md) | Default, MCP triage, custom handlers |
+| [SDK](docs/sdk.md) | Embedded usage, `createClient`, event subscriptions |
 | [Architecture](docs/architecture.md) | Project structure, conventions, discovery |
 | [Cloud](docs/cloud.md) | AWS ECS, GCP Cloud Run, Docker |
 | [Data Model](docs/data.md) | Database schema |

package/build/api/auth.d.ts

@@ -0,0 +1,5 @@
+import type { LTApiResult } from '../types/sdk';
+export declare function login(input: {
+    username: string;
+    password: string;
+}): Promise<LTApiResult>;

package/build/api/auth.js

@@ -0,0 +1,42 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.login = login;
+const user_1 = require("../services/user");
+const auth_1 = require("../modules/auth");
+async function login(input) {
+    try {
+        const { username, password } = input;
+        if (!username || !password) {
+            return { status: 400, error: 'username and password are required' };
+        }
+        const user = await (0, user_1.verifyPassword)(username, password);
+        if (!user) {
+            return { status: 401, error: 'Invalid credentials' };
+        }
+        const highestType = user.roles.some((r) => r.type === 'superadmin')
+            ? 'superadmin'
+            : user.roles.some((r) => r.type === 'admin')
+                ? 'admin'
+                : 'member';
+        const token = (0, auth_1.signToken)({
+            userId: user.id,
+            role: highestType,
+            roles: user.roles.map((r) => ({ role: r.role, type: r.type })),
+        }, '24h');
+        return {
+            status: 200,
+            data: {
+                token,
+                user: {
+                    id: user.id,
+                    external_id: user.external_id,
+                    display_name: user.display_name,
+                    roles: user.roles,
+                },
+            },
+        };
+    }
+    catch (err) {
+        return { status: 500, error: err.message };
+    }
+}

package/build/api/bot-accounts.d.ts

@@ -0,0 +1,50 @@
+import type { LTApiResult, LTApiAuth } from '../types/sdk';
+export declare function listBots(input: {
+    limit?: number;
+    offset?: number;
+}): Promise<LTApiResult>;
+export declare function getBot(input: {
+    id: string;
+}): Promise<LTApiResult>;
+export declare function createBot(input: {
+    name: string;
+    description?: string;
+    display_name?: string;
+    roles?: {
+        role: string;
+        type: string;
+    }[];
+}, auth?: LTApiAuth): Promise<LTApiResult>;
+export declare function updateBot(input: {
+    id: string;
+    display_name?: string;
+    description?: string;
+    status?: string;
+}): Promise<LTApiResult>;
+export declare function deleteBot(input: {
+    id: string;
+}): Promise<LTApiResult>;
+export declare function getBotRoles(input: {
+    id: string;
+}): Promise<LTApiResult>;
+export declare function addBotRole(input: {
+    id: string;
+    role: string;
+    type: string;
+}): Promise<LTApiResult>;
+export declare function removeBotRole(input: {
+    id: string;
+    role: string;
+}): Promise<LTApiResult>;
+export declare function listBotKeys(input: {
+    id: string;
+}): Promise<LTApiResult>;
+export declare function createBotKey(input: {
+    id: string;
+    name: string;
+    scopes?: string[];
+    expires_at?: string;
+}): Promise<LTApiResult>;
+export declare function revokeBotKey(input: {
+    keyId: string;
+}): Promise<LTApiResult>;

package/build/api/bot-accounts.js

@@ -0,0 +1,215 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.listBots = listBots;
+exports.getBot = getBot;
+exports.createBot = createBot;
+exports.updateBot = updateBot;
+exports.deleteBot = deleteBot;
+exports.getBotRoles = getBotRoles;
+exports.addBotRole = addBotRole;
+exports.removeBotRole = removeBotRole;
+exports.listBotKeys = listBotKeys;
+exports.createBotKey = createBotKey;
+exports.revokeBotKey = revokeBotKey;
+const iam = __importStar(require("../services/iam"));
+const user_1 = require("../services/user");
+async function listBots(input) {
+    try {
+        const limit = input.limit ?? 50;
+        const offset = input.offset ?? 0;
+        const result = await iam.listBots(limit, offset);
+        return { status: 200, data: result };
+    }
+    catch (err) {
+        return { status: 500, error: err.message };
+    }
+}
+async function getBot(input) {
+    try {
+        const bot = await iam.getBot(input.id);
+        if (!bot) {
+            return { status: 404, error: 'Bot not found' };
+        }
+        return { status: 200, data: bot };
+    }
+    catch (err) {
+        return { status: 500, error: err.message };
+    }
+}
+async function createBot(input, auth) {
+    try {
+        if (!input.name) {
+            return { status: 400, error: 'name is required' };
+        }
+        if (input.roles) {
+            for (const r of input.roles) {
+                if (!r.role || !r.type || !(0, user_1.isValidRoleType)(r.type)) {
+                    return {
+                        status: 400,
+                        error: 'Each role must have a role name and type (superadmin, admin, member)',
+                    };
+                }
+            }
+        }
+        const bot = await iam.createBot({
+            name: input.name,
+            description: input.description,
+            display_name: input.display_name,
+            roles: input.roles,
+            created_by: auth?.userId,
+        });
+        return { status: 201, data: bot };
+    }
+    catch (err) {
+        if (err.code === '23505') {
+            return { status: 409, error: 'Bot with this name already exists' };
+        }
+        return { status: 500, error: err.message };
+    }
+}
+async function updateBot(input) {
+    try {
+        const { id, ...fields } = input;
+        const bot = await iam.updateBot(id, fields);
+        if (!bot) {
+            return { status: 404, error: 'Bot not found' };
+        }
+        return { status: 200, data: bot };
+    }
+    catch (err) {
+        return { status: 500, error: err.message };
+    }
+}
+async function deleteBot(input) {
+    try {
+        const deleted = await iam.deleteBot(input.id);
+        if (!deleted) {
+            return { status: 404, error: 'Bot not found' };
+        }
+        return { status: 200, data: { deleted: true } };
+    }
+    catch (err) {
+        return { status: 500, error: err.message };
+    }
+}
+async function getBotRoles(input) {
+    try {
+        const bot = await iam.getBot(input.id);
+        if (!bot) {
+            return { status: 404, error: 'Bot not found' };
+        }
+        const roles = await iam.getBotRoles(input.id);
+        return { status: 200, data: { roles } };
+    }
+    catch (err) {
+        return { status: 500, error: err.message };
+    }
+}
+async function addBotRole(input) {
+    try {
+        if (!input.role || !input.type) {
+            return { status: 400, error: 'role and type are required' };
+        }
+        if (!(0, user_1.isValidRoleType)(input.type)) {
+            return { status: 400, error: 'type must be superadmin, admin, or member' };
+        }
+        const bot = await iam.getBot(input.id);
+        if (!bot) {
+            return { status: 404, error: 'Bot not found' };
+        }
+        const result = await iam.addBotRole(input.id, input.role, input.type);
+        return { status: 201, data: result };
+    }
+    catch (err) {
+        return { status: 500, error: err.message };
+    }
+}
+async function removeBotRole(input) {
+    try {
+        const removed = await iam.removeBotRole(input.id, input.role);
+        if (!removed) {
+            return { status: 404, error: 'Role not found' };
+        }
+        return { status: 200, data: { removed: true } };
+    }
+    catch (err) {
+        return { status: 500, error: err.message };
+    }
+}
+async function listBotKeys(input) {
+    try {
+        const bot = await iam.getBot(input.id);
+        if (!bot) {
+            return { status: 404, error: 'Bot not found' };
+        }
+        const keys = await iam.listBotKeys(input.id);
+        return { status: 200, data: { keys } };
+    }
+    catch (err) {
+        return { status: 500, error: err.message };
+    }
+}
+async function createBotKey(input) {
+    try {
+        if (!input.name) {
+            return { status: 400, error: 'name is required' };
+        }
+        const expiresAt = input.expires_at ? new Date(input.expires_at) : undefined;
+        const result = await iam.createBotKey(input.id, input.name, input.scopes || [], expiresAt);
+        return { status: 201, data: result };
+    }
+    catch (err) {
+        if (err.message === 'Bot not found') {
+            return { status: 404, error: err.message };
+        }
+        if (err.code === '23505') {
+            return { status: 409, error: 'API key with this name already exists for this bot' };
+        }
+        return { status: 500, error: err.message };
+    }
+}
+async function revokeBotKey(input) {
+    try {
+        const revoked = await iam.revokeBotKey(input.keyId);
+        if (!revoked) {
+            return { status: 404, error: 'API key not found' };
+        }
+        return { status: 200, data: { revoked: true } };
+    }
+    catch (err) {
+        return { status: 500, error: err.message };
+    }
+}

package/build/api/controlplane.d.ts

@@ -0,0 +1,20 @@
+import type { LTApiResult } from '../types/sdk';
+export declare function listApps(): Promise<LTApiResult>;
+export declare function rollCall(input: {
+    appId?: string;
+    delay?: number;
+}): Promise<LTApiResult>;
+export declare function applyThrottle(input: {
+    appId?: string;
+    throttle: number;
+    topic?: string;
+    guid?: string;
+}): Promise<LTApiResult>;
+export declare function getStreamStats(input: {
+    app_id?: string;
+    duration?: string;
+    stream?: string;
+}): Promise<LTApiResult>;
+export declare function subscribeMesh(input: {
+    appId?: string;
+}): Promise<LTApiResult>;

package/build/api/controlplane.js

@@ -0,0 +1,110 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.listApps = listApps;
+exports.rollCall = rollCall;
+exports.applyThrottle = applyThrottle;
+exports.getStreamStats = getStreamStats;
+exports.subscribeMesh = subscribeMesh;
+const controlplane = __importStar(require("../services/controlplane"));
+const events_1 = require("../lib/events");
+async function listApps() {
+    try {
+        const apps = await controlplane.listApps();
+        return { status: 200, data: { apps } };
+    }
+    catch (err) {
+        return { status: 500, error: err.message };
+    }
+}
+async function rollCall(input) {
+    try {
+        const appId = input.appId || 'durable';
+        const profiles = await controlplane.rollCall(appId, input.delay);
+        return { status: 200, data: { profiles } };
+    }
+    catch (err) {
+        return { status: 500, error: err.message };
+    }
+}
+async function applyThrottle(input) {
+    try {
+        const appId = input.appId || 'durable';
+        if (typeof input.throttle !== 'number') {
+            return { status: 400, error: 'throttle (number) is required' };
+        }
+        const result = await controlplane.applyThrottle(appId, {
+            throttle: input.throttle,
+            topic: input.topic,
+            guid: input.guid,
+        });
+        // Publish a synthetic event so the dashboard event stream sees it
+        const throttleEvent = {
+            type: 'mesh.throttle',
+            source: 'controlplane',
+            workflowId: input.guid || '',
+            workflowName: '',
+            taskQueue: appId,
+            data: { throttle: input.throttle, topic: input.topic, guid: input.guid, appId },
+            timestamp: new Date().toISOString(),
+        };
+        events_1.eventRegistry.publish(throttleEvent).catch(() => { });
+        return { status: 200, data: { success: result } };
+    }
+    catch (err) {
+        return { status: 500, error: err.message };
+    }
+}
+async function getStreamStats(input) {
+    try {
+        const schema = input.app_id || 'durable';
+        const duration = input.duration || '1h';
+        const stats = await controlplane.getStreamStats(schema, duration, input.stream);
+        return { status: 200, data: stats };
+    }
+    catch (err) {
+        return { status: 500, error: err.message };
+    }
+}
+async function subscribeMesh(input) {
+    try {
+        const appId = input.appId || 'durable';
+        await controlplane.subscribeMesh(appId);
+        return { status: 200, data: { subscribed: true, appId } };
+    }
+    catch (err) {
+        return { status: 500, error: err.message };
+    }
+}

package/build/api/dba.d.ts

@@ -0,0 +1,15 @@
+import type { LTApiResult } from '../types/sdk';
+export declare function prune(input: {
+    expire?: string;
+    jobs?: boolean;
+    streams?: boolean;
+    engineStreams?: boolean;
+    engineStreamsExpire?: string;
+    workerStreams?: boolean;
+    workerStreamsExpire?: string;
+    attributes?: boolean;
+    entities?: string[];
+    pruneTransient?: boolean;
+    keepHmark?: boolean;
+}): Promise<LTApiResult>;
+export declare function deploy(): Promise<LTApiResult>;

package/build/api/dba.js

@@ -0,0 +1,68 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.prune = prune;
+exports.deploy = deploy;
+const dbaService = __importStar(require("../services/dba"));
+async function prune(input) {
+    try {
+        const result = await dbaService.prune({
+            expire: input.expire,
+            jobs: input.jobs,
+            streams: input.streams,
+            engineStreams: input.engineStreams,
+            engineStreamsExpire: input.engineStreamsExpire,
+            workerStreams: input.workerStreams,
+            workerStreamsExpire: input.workerStreamsExpire,
+            attributes: input.attributes,
+            entities: input.entities,
+            pruneTransient: input.pruneTransient,
+            keepHmark: input.keepHmark,
+        });
+        return { status: 200, data: result };
+    }
+    catch (err) {
+        return { status: 500, error: err.message };
+    }
+}
+async function deploy() {
+    try {
+        await dbaService.deploy();
+        return { status: 200, data: { ok: true } };
+    }
+    catch (err) {
+        return { status: 500, error: err.message };
+    }
+}