@agenticmail/enterprise 0.5.199 → 0.5.201

package/src/engine/model-fallback.ts

@@ -0,0 +1,141 @@
+/**
+ * Model Fallback / Backup Provider System
+ *
+ * Provides a chain of fallback models when the primary model fails.
+ * Configurable per-org and per-agent. Tracks which models failed and
+ * which backup was ultimately used.
+ */
+
+export interface ModelFallbackConfig {
+  primary: string;               // e.g. 'openai/gpt-4o'
+  fallbacks: string[];           // ordered fallback chain
+  maxRetries: number;            // retries per model before moving to next
+  retryDelayMs: number;          // delay between retries
+  enabled: boolean;
+}
+
+export interface FallbackResult {
+  modelUsed: string;
+  wasFallback: boolean;
+  attemptedModels: string[];
+  failureReasons: Record<string, string>;
+}
+
+const DEFAULT_CONFIG: ModelFallbackConfig = {
+  primary: '',
+  fallbacks: [],
+  maxRetries: 2,
+  retryDelayMs: 1000,
+  enabled: true,
+};
+
+/**
+ * Build a model chain from agent/org config.
+ * Checks agent-level fallbacks first, then org-level defaults.
+ */
+export function buildModelChain(
+  agentConfig?: { model?: string; fallbackModels?: string[]; modelFallback?: ModelFallbackConfig },
+  orgConfig?: { defaultModel?: string; fallbackModels?: string[]; modelFallback?: ModelFallbackConfig }
+): ModelFallbackConfig {
+  // Agent-level explicit config
+  if (agentConfig?.modelFallback?.enabled !== false && agentConfig?.modelFallback) {
+    return { ...DEFAULT_CONFIG, ...agentConfig.modelFallback };
+  }
+
+  const primary = (typeof agentConfig?.model === 'string' ? agentConfig.model : '')
+    || orgConfig?.defaultModel || '';
+
+  const fallbacks = agentConfig?.fallbackModels
+    || orgConfig?.fallbackModels
+    || orgConfig?.modelFallback?.fallbacks
+    || [];
+
+  return {
+    ...DEFAULT_CONFIG,
+    primary,
+    fallbacks: fallbacks.filter(m => m !== primary),
+  };
+}
+
+/**
+ * Execute a function with model fallback.
+ * Tries primary model, then each fallback in order.
+ *
+ * @param chain - The model fallback configuration
+ * @param fn - Async function that takes a model name and executes
+ * @returns The result + metadata about which model was used
+ */
+export async function withModelFallback<T>(
+  chain: ModelFallbackConfig,
+  fn: (model: string) => Promise<T>
+): Promise<{ result: T } & FallbackResult> {
+  const models = [chain.primary, ...chain.fallbacks].filter(Boolean);
+  if (!models.length) throw new Error('No models configured');
+
+  const failureReasons: Record<string, string> = {};
+  const attemptedModels: string[] = [];
+
+  for (let mi = 0; mi < models.length; mi++) {
+    const model = models[mi];
+    attemptedModels.push(model);
+
+    for (let retry = 0; retry < chain.maxRetries; retry++) {
+      try {
+        const result = await fn(model);
+        return {
+          result,
+          modelUsed: model,
+          wasFallback: mi > 0,
+          attemptedModels,
+          failureReasons,
+        };
+      } catch (err: any) {
+        const reason = err?.message || String(err);
+
+        // Don't retry on auth errors or invalid model — move to next model immediately
+        if (/auth|unauthorized|forbidden|invalid.*model|not.*found|does.*not.*exist/i.test(reason)) {
+          failureReasons[model] = reason;
+          break;
+        }
+
+        // Rate limit / overloaded — retry after delay
+        if (/rate.?limit|overloaded|too.?many|429|503|capacity/i.test(reason)) {
+          failureReasons[model] = reason;
+          if (retry < chain.maxRetries - 1) {
+            await sleep(chain.retryDelayMs * (retry + 1));
+            continue;
+          }
+          break;
+        }
+
+        // Other errors — retry
+        failureReasons[model] = reason;
+        if (retry < chain.maxRetries - 1) {
+          await sleep(chain.retryDelayMs);
+        }
+      }
+    }
+  }
+
+  throw new ModelFallbackExhaustedError(
+    `All models failed: ${models.join(', ')}`,
+    attemptedModels,
+    failureReasons
+  );
+}
+
+export class ModelFallbackExhaustedError extends Error {
+  attemptedModels: string[];
+  failureReasons: Record<string, string>;
+
+  constructor(message: string, attemptedModels: string[], failureReasons: Record<string, string>) {
+    super(message);
+    this.name = 'ModelFallbackExhaustedError';
+    this.attemptedModels = attemptedModels;
+    this.failureReasons = failureReasons;
+  }
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}

package/src/engine/routes.ts

@@ -86,6 +86,8 @@ import { createChatWebhookRoutes } from './chat-webhook-routes.js';
 import { ChatPoller } from './chat-poller.js';
 import { EmailPoller } from './email-poller.js';
 import { MessagingPoller } from './messaging-poller.js';
+import { TaskQueueManager } from './task-queue.js';
+import { createTaskQueueRoutes } from './task-queue-routes.js';
 import type { DatabaseAdapter } from '../db/adapter.js';
 
 const engine = new Hono<AppEnv>();
@@ -128,6 +130,7 @@ const knowledgeContribution = new KnowledgeContributionManager({ memoryCallback:
 import { AgentHierarchyManager } from './agent-hierarchy.js';
 let hierarchyManager: AgentHierarchyManager | null = null;
 const knowledgeImport = new KnowledgeImportManager({ knowledgeContribution });
+const taskQueue = new TaskQueueManager();
 const skillUpdater = new SkillAutoUpdater({ registry: communityRegistry });
 
 // Wire onboarding into guardrails for onboarding gate checks
@@ -190,6 +193,7 @@ engine.route('/anomaly-rules', createAnomalyRoutes(guardrails));
 engine.route('/journal', createJournalRoutes(journal));
 engine.route('/messages', createCommunicationRoutes(commBus));
 engine.route('/tasks', createTaskRoutes(commBus));
+engine.route('/task-pipeline', createTaskQueueRoutes(taskQueue));
 engine.route('/compliance', createComplianceRoutes(compliance));
 
 engine.route('/', createCatalogRoutes({
@@ -486,6 +490,7 @@ export async function setEngineDb(
     vault.setDb(db),
     storageManager.setDb(db),
     policyImporter.setDb(db),
+    (async () => { (taskQueue as any).db = (db as any)?.db || db; await taskQueue.init(); })(),
   ]);
   // Initialize hierarchy manager + start background task monitor
   hierarchyManager = new AgentHierarchyManager(db);

package/src/engine/task-queue-after-spawn.ts

@@ -0,0 +1,66 @@
+/**
+ * Task Queue — After Spawn Hook
+ *
+ * Called AFTER an agent finishes a task (success, failure, or cancellation).
+ * Records the outcome, model used, tokens, cost, and any result metadata.
+ */
+
+import type { TaskQueueManager, TaskStatus } from './task-queue.js';
+
+export interface AfterSpawnContext {
+  taskId: string;
+  status: 'completed' | 'failed' | 'cancelled';
+  result?: Record<string, unknown>;
+  error?: string;
+  modelUsed?: string;
+  tokensUsed?: number;
+  costUsd?: number;
+  sessionId?: string;
+}
+
+/**
+ * Record task outcome AFTER the agent finishes.
+ */
+export async function afterSpawn(
+  taskQueue: TaskQueueManager,
+  ctx: AfterSpawnContext
+): Promise<void> {
+  const updates: Record<string, unknown> = {
+    status: ctx.status as TaskStatus,
+  };
+
+  if (ctx.result) updates.result = ctx.result;
+  if (ctx.error) updates.error = ctx.error;
+  if (ctx.modelUsed) updates.modelUsed = ctx.modelUsed;
+  if (ctx.tokensUsed !== undefined) updates.tokensUsed = ctx.tokensUsed;
+  if (ctx.costUsd !== undefined) updates.costUsd = ctx.costUsd;
+  if (ctx.sessionId) updates.sessionId = ctx.sessionId;
+
+  await taskQueue.updateTask(ctx.taskId, updates as any);
+}
+
+/**
+ * Mark a task as in_progress (call when agent session actually starts executing).
+ */
+export async function markInProgress(
+  taskQueue: TaskQueueManager,
+  taskId: string,
+  opts?: { sessionId?: string; modelUsed?: string }
+): Promise<void> {
+  await taskQueue.updateTask(taskId, {
+    status: 'in_progress',
+    ...(opts?.sessionId ? { sessionId: opts.sessionId } : {}),
+    ...(opts?.modelUsed ? { modelUsed: opts.modelUsed } : {}),
+  });
+}
+
+/**
+ * Update task progress (0-100) during execution.
+ */
+export async function updateProgress(
+  taskQueue: TaskQueueManager,
+  taskId: string,
+  progress: number
+): Promise<void> {
+  await taskQueue.updateTask(taskId, { progress: Math.min(100, Math.max(0, progress)) });
+}

package/src/engine/task-queue-before-spawn.ts

@@ -0,0 +1,109 @@
+/**
+ * Task Queue — Before Spawn Hook
+ *
+ * Called BEFORE an agent is spawned for a task. Records the task intent
+ * with smart metadata extraction from the request context.
+ *
+ * Usage:
+ *   const taskId = await beforeSpawn(taskQueue, { ... });
+ *   // ... spawn agent ...
+ *   await afterSpawn(taskQueue, taskId, { ... });
+ */
+
+import type { TaskQueueManager, TaskPriority } from './task-queue.js';
+
+export interface BeforeSpawnContext {
+  orgId: string;
+  agentId: string;
+  agentName: string;
+  createdBy?: string;        // who initiated — agent ID, user ID, or 'system'
+  createdByName?: string;
+  task: string;              // raw task description
+  model?: string;
+  fallbackModel?: string;
+  sessionId?: string;
+  parentTaskId?: string;
+  relatedAgentIds?: string[];
+  priority?: TaskPriority;
+  estimatedDurationMs?: number;
+}
+
+/**
+ * Extract smart metadata from a task description.
+ * Infers category, title, tags, and priority from natural language.
+ */
+function extractTaskMetadata(task: string): {
+  title: string;
+  category: string;
+  tags: string[];
+  priority: TaskPriority;
+} {
+  const lower = task.toLowerCase();
+
+  // Extract a short title (first sentence or first 80 chars)
+  let title = task.split(/[.\n!?]/)[0]?.trim() || task;
+  if (title.length > 80) title = title.slice(0, 77) + '...';
+
+  // Infer category
+  let category = 'custom';
+  if (/\b(email|inbox|reply|forward|send mail|compose)\b/.test(lower)) category = 'email';
+  else if (/\b(research|search|find|look up|investigate|analyze)\b/.test(lower)) category = 'research';
+  else if (/\b(meeting|calendar|schedule|call|agenda)\b/.test(lower)) category = 'meeting';
+  else if (/\b(workflow|pipeline|automat|process|batch)\b/.test(lower)) category = 'workflow';
+  else if (/\b(write|draft|document|report|summary|blog|article)\b/.test(lower)) category = 'writing';
+  else if (/\b(deploy|build|compile|publish|release|ship)\b/.test(lower)) category = 'deployment';
+  else if (/\b(review|approve|check|audit|verify)\b/.test(lower)) category = 'review';
+  else if (/\b(monitor|watch|track|alert|notify)\b/.test(lower)) category = 'monitoring';
+
+  // Extract tags from common patterns
+  const tags: string[] = [];
+  if (/\burgent\b/i.test(task)) tags.push('urgent');
+  if (/\basap\b/i.test(task)) tags.push('asap');
+  if (/\bfollow[- ]?up\b/i.test(task)) tags.push('follow-up');
+  if (/\bbug\b|error\b|fix\b/i.test(task)) tags.push('bug-fix');
+  if (/\bcustomer\b|client\b/i.test(task)) tags.push('customer');
+  if (/\binternal\b/i.test(task)) tags.push('internal');
+
+  // Infer priority
+  let priority: TaskPriority = 'normal';
+  if (/\b(urgent|critical|emergency|asap|immediately)\b/i.test(task)) priority = 'urgent';
+  else if (/\b(important|high.?priority|priority|rush)\b/i.test(task)) priority = 'high';
+  else if (/\b(low.?priority|when.?you.?can|no.?rush|whenever)\b/i.test(task)) priority = 'low';
+
+  return { title, category, tags, priority };
+}
+
+/**
+ * Record a task BEFORE spawning the agent.
+ * Returns the task ID for linking with afterSpawn.
+ */
+export async function beforeSpawn(
+  taskQueue: TaskQueueManager,
+  ctx: BeforeSpawnContext
+): Promise<string> {
+  const meta = extractTaskMetadata(ctx.task);
+
+  const task = await taskQueue.createTask({
+    orgId: ctx.orgId,
+    assignedTo: ctx.agentId,
+    assignedToName: ctx.agentName,
+    createdBy: ctx.createdBy || 'system',
+    createdByName: ctx.createdByName || 'System',
+    title: meta.title,
+    description: ctx.task,
+    category: meta.category,
+    tags: meta.tags,
+    priority: ctx.priority || meta.priority,
+    parentTaskId: ctx.parentTaskId,
+    relatedAgentIds: ctx.relatedAgentIds,
+    sessionId: ctx.sessionId,
+    model: ctx.model,
+    fallbackModel: ctx.fallbackModel,
+    estimatedDurationMs: ctx.estimatedDurationMs,
+  });
+
+  // Immediately mark as assigned
+  await taskQueue.updateTask(task.id, { status: 'assigned' });
+
+  return task.id;
+}

package/src/engine/task-queue-routes.ts

@@ -0,0 +1,133 @@
+/**
+ * Task Queue Routes
+ *
+ * REST + SSE endpoints for the centralized task pipeline.
+ * Dashboard subscribes to /task-pipeline/stream for real-time updates.
+ */
+
+import { Hono } from 'hono';
+import type { TaskQueueManager } from './task-queue.js';
+
+export function createTaskQueueRoutes(taskQueue: TaskQueueManager) {
+  const router = new Hono();
+
+  // GET /task-pipeline — all tasks (paginated)
+  router.get('/', async (c) => {
+    const orgId = c.req.query('orgId') || '';
+    const limit = parseInt(c.req.query('limit') || '100');
+    const offset = parseInt(c.req.query('offset') || '0');
+    const tasks = await taskQueue.getTaskHistory(orgId, limit, offset);
+    return c.json({ tasks });
+  });
+
+  // GET /task-pipeline/active — only active tasks
+  router.get('/active', (c) => {
+    const orgId = c.req.query('orgId') || '';
+    const tasks = taskQueue.getActiveTasks(orgId);
+    return c.json({ tasks });
+  });
+
+  // GET /task-pipeline/stats — pipeline statistics
+  router.get('/stats', (c) => {
+    const orgId = c.req.query('orgId') || '';
+    const stats = taskQueue.getPipelineStats(orgId);
+    return c.json(stats);
+  });
+
+  // GET /task-pipeline/agent/:agentId — tasks for specific agent
+  router.get('/agent/:agentId', (c) => {
+    const agentId = c.req.param('agentId');
+    const includeCompleted = c.req.query('completed') === 'true';
+    const tasks = taskQueue.getAgentTasks(agentId, includeCompleted);
+    return c.json({ tasks });
+  });
+
+  // GET /task-pipeline/:id — single task detail
+  router.get('/:id', (c) => {
+    const task = taskQueue.getTask(c.req.param('id'));
+    if (!task) return c.json({ error: 'Task not found' }, 404);
+    return c.json({ task });
+  });
+
+  // POST /task-pipeline — create task manually
+  router.post('/', async (c) => {
+    const body = await c.req.json();
+    const task = await taskQueue.createTask({
+      orgId: body.orgId || '',
+      assignedTo: body.assignedTo || '',
+      assignedToName: body.assignedToName || '',
+      createdBy: body.createdBy || 'dashboard',
+      createdByName: body.createdByName || 'Dashboard User',
+      title: body.title || 'Untitled Task',
+      description: body.description || '',
+      category: body.category,
+      tags: body.tags,
+      priority: body.priority,
+      parentTaskId: body.parentTaskId,
+      relatedAgentIds: body.relatedAgentIds,
+      model: body.model,
+      fallbackModel: body.fallbackModel,
+    });
+    return c.json({ task }, 201);
+  });
+
+  // PATCH /task-pipeline/:id — update task
+  router.patch('/:id', async (c) => {
+    const body = await c.req.json();
+    const task = await taskQueue.updateTask(c.req.param('id'), body);
+    if (!task) return c.json({ error: 'Task not found' }, 404);
+    return c.json({ task });
+  });
+
+  // POST /task-pipeline/:id/cancel — cancel a task
+  router.post('/:id/cancel', async (c) => {
+    const task = await taskQueue.updateTask(c.req.param('id'), { status: 'cancelled' });
+    if (!task) return c.json({ error: 'Task not found' }, 404);
+    return c.json({ task });
+  });
+
+  // ─── SSE Stream ────────────────────────────────────────
+  router.get('/stream', (c) => {
+    let alive = true;
+    const stream = new ReadableStream({
+      start(controller) {
+        const enc = new TextEncoder();
+        const send = (data: string) => {
+          if (!alive) return;
+          try { controller.enqueue(enc.encode(data)); } catch { alive = false; }
+        };
+
+        // Send initial state
+        const active = taskQueue.getActiveTasks();
+        const stats = taskQueue.getPipelineStats();
+        send(`data: ${JSON.stringify({ type: 'init', tasks: active, stats })}\n\n`);
+
+        // Subscribe to real-time events
+        const unsub = taskQueue.subscribe((event) => {
+          send(`data: ${JSON.stringify(event)}\n\n`);
+        });
+
+        // Heartbeat every 30s
+        const hb = setInterval(() => { send(': heartbeat\n\n'); }, 30_000);
+
+        // Cleanup on close
+        c.req.raw.signal?.addEventListener('abort', () => {
+          alive = false;
+          unsub();
+          clearInterval(hb);
+          try { controller.close(); } catch { /* ignore */ }
+        });
+      },
+    });
+
+    return new Response(stream, {
+      headers: {
+        'Content-Type': 'text/event-stream',
+        'Cache-Control': 'no-cache',
+        'Connection': 'keep-alive',
+      },
+    });
+  });
+
+  return router;
+}