agentgate 0.4.0 → 0.5.0

package/src/index.js

@@ -2,7 +2,7 @@ import express from 'express';
 import cookieParser from 'cookie-parser';
 import { fileURLToPath } from 'url';
 import { dirname, join } from 'path';
-import { validateApiKey, getAccountsByService, getCookieSecret, getMessagingMode } from './lib/db.js';
+import { validateApiKey, getAccountsByService, getCookieSecret, getMessagingMode, checkServiceAccess } from './lib/db.js';
 import { connectHsync } from './lib/hsyncManager.js';
 import { initSocket } from './lib/socketManager.js';
 import githubRoutes, { serviceInfo as githubInfo } from './routes/github.js';
@@ -16,7 +16,10 @@ import jiraRoutes, { serviceInfo as jiraInfo } from './routes/jira.js';
 import fitbitRoutes, { serviceInfo as fitbitInfo } from './routes/fitbit.js';
 import queueRoutes from './routes/queue.js';
 import agentsRoutes from './routes/agents.js';
-import uiRoutes from './routes/ui.js';
+import mementoRoutes from './routes/memento.js';
+import uiRoutes from './routes/ui/index.js';
+import webhooksRoutes from './routes/webhooks.js';
+import servicesRoutes from './routes/services.js';
 
 // Aggregate service metadata from all routes
 const SERVICE_REGISTRY = {
@@ -36,7 +39,15 @@ const app = express();
 const PORT = process.env.PORT || 3050;
 const BASE_URL = process.env.BASE_URL || `http://localhost:${PORT}`;
 
-app.use(express.json({ limit: '10mb' }));
+app.use(express.json({
+  limit: '10mb',
+  verify: (req, res, buf) => {
+    // Capture raw body for webhook signature verification
+    if (req.originalUrl.startsWith('/webhooks')) {
+      req.rawBody = buf;
+    }
+  }
+}));
 app.use(express.urlencoded({ extended: true }));
 app.use(cookieParser(getCookieSecret()));
 app.use('/public', express.static(join(__dirname, '../public')));
@@ -54,6 +65,11 @@ async function apiKeyAuth(req, res, next) {
     return res.status(401).json({ error: 'Invalid API key' });
   }
 
+  if (!valid.enabled) {
+    return res.status(403).json({ error: 'Agent is disabled' });
+  }
+
+
   req.apiKeyInfo = valid;
   next();
 }
@@ -66,17 +82,49 @@ function readOnlyEnforce(req, res, next) {
   next();
 }
 
-// API routes - require auth and read-only
+
+// Service access control middleware factory
+// Checks if the agent has access to the requested service/account
+function serviceAccessCheck(serviceName) {
+  return (req, res, next) => {
+    // Extract accountName from the URL path (first segment after the service mount point)
+    // e.g., /api/github/monteslu/repos -> req.path = /monteslu/repos -> accountName = monteslu
+    const pathSegments = req.path.split('/').filter(Boolean);
+    const accountName = pathSegments[0];
+    if (!accountName) {
+      return next(); // No account specified, let the route handle it
+    }
+
+    const agentName = req.apiKeyInfo?.name;
+    if (!agentName) {
+      return next(); // No agent info, let other middleware handle auth
+    }
+
+    const access = checkServiceAccess(serviceName, accountName, agentName);
+    if (!access.allowed) {
+      return res.status(403).json({
+        error: `Agent '${agentName}' does not have access to service '${serviceName}/${accountName}'`,
+        reason: access.reason
+      });
+    }
+    next();
+  };
+}
+
+// API routes - require auth, read-only, and service access check
 // Pattern: /api/{service}/{accountName}/...
-app.use('/api/github', apiKeyAuth, readOnlyEnforce, githubRoutes);
-app.use('/api/bluesky', apiKeyAuth, readOnlyEnforce, blueskyRoutes);
-app.use('/api/reddit', apiKeyAuth, readOnlyEnforce, redditRoutes);
-app.use('/api/calendar', apiKeyAuth, readOnlyEnforce, calendarRoutes);
-app.use('/api/mastodon', apiKeyAuth, readOnlyEnforce, mastodonRoutes);
-app.use('/api/linkedin', apiKeyAuth, readOnlyEnforce, linkedinRoutes);
-app.use('/api/youtube', apiKeyAuth, readOnlyEnforce, youtubeRoutes);
-app.use('/api/jira', apiKeyAuth, readOnlyEnforce, jiraRoutes);
-app.use('/api/fitbit', apiKeyAuth, readOnlyEnforce, fitbitRoutes);
+app.use('/api/github', apiKeyAuth, readOnlyEnforce, serviceAccessCheck('github'), githubRoutes);
+app.use('/api/bluesky', apiKeyAuth, readOnlyEnforce, serviceAccessCheck('bluesky'), blueskyRoutes);
+app.use('/api/reddit', apiKeyAuth, readOnlyEnforce, serviceAccessCheck('reddit'), redditRoutes);
+app.use('/api/calendar', apiKeyAuth, readOnlyEnforce, serviceAccessCheck('calendar'), calendarRoutes);
+app.use('/api/mastodon', apiKeyAuth, readOnlyEnforce, serviceAccessCheck('mastodon'), mastodonRoutes);
+app.use('/api/linkedin', apiKeyAuth, readOnlyEnforce, serviceAccessCheck('linkedin'), linkedinRoutes);
+app.use('/api/youtube', apiKeyAuth, readOnlyEnforce, serviceAccessCheck('youtube'), youtubeRoutes);
+app.use('/api/jira', apiKeyAuth, readOnlyEnforce, serviceAccessCheck('jira'), jiraRoutes);
+app.use('/api/fitbit', apiKeyAuth, readOnlyEnforce, serviceAccessCheck('fitbit'), fitbitRoutes);
+
+// Service access management - admin API (requires auth)
+app.use('/api/services', apiKeyAuth, servicesRoutes);
 
 // Queue routes - require auth but allow POST for submitting write requests
 // Pattern: /api/queue/{service}/{accountName}/submit
@@ -89,35 +137,59 @@ app.use('/api/agents', apiKeyAuth, (req, res, next) => {
   next();
 }, agentsRoutes);
 
+// Memento routes - require auth, allow POST for creating mementos
+app.use('/api/agents/memento', apiKeyAuth, (req, res, next) => {
+  req.apiKeyName = req.apiKeyInfo.name;
+  next();
+}, mementoRoutes);
+
 // UI routes - no API key needed (local admin access)
 app.use('/ui', uiRoutes);
 
+// Webhook routes - no API key needed (uses signature verification instead)
+app.use('/webhooks', webhooksRoutes);
+
 // Agent readme endpoint - requires auth
 app.get('/api/readme', apiKeyAuth, (req, res) => {
   const accountsByService = getAccountsByService();
+  const agentName = req.apiKeyInfo?.name;
 
-  // Build services object from registry
+  // Build services object from registry, filtering by agent access
   const services = {};
   for (const [key, info] of Object.entries(SERVICE_REGISTRY)) {
     const dbKey = info.dbKey || key;
-    services[key] = {
-      accounts: accountsByService[dbKey] || [],
-      authType: info.authType,
-      description: info.description
-    };
+    const allAccounts = accountsByService[dbKey] || [];
+    
+    // Filter accounts based on agent access
+    const accessibleAccounts = allAccounts.filter(accountName => {
+      const access = checkServiceAccess(key, accountName, agentName);
+      return access.allowed;
+    });
+    
+    // Only include service if agent has access to at least one account
+    if (accessibleAccounts.length > 0) {
+      services[key] = {
+        accounts: accessibleAccounts,
+        authType: info.authType,
+        description: info.description
+      };
+    }
   }
 
-  // Build endpoints object from registry
+  // Build endpoints object from registry (only for accessible services)
   const endpoints = {};
   for (const [key, info] of Object.entries(SERVICE_REGISTRY)) {
-    endpoints[key] = {
-      base: `/api/${key}/{accountName}`,
-      description: info.description,
-      docs: info.docs,
-      examples: info.examples
-    };
-    if (info.writeGuidelines) {
-      endpoints[key].writeGuidelines = info.writeGuidelines;
+    // Only include endpoint if agent has access to this service
+    if (services[key]) {
+      endpoints[key] = {
+        base: `/api/${key}/{accountName}`,
+        description: info.description,
+        docs: info.docs,
+        examples: info.examples
+      };
+      if (info.writeGuidelines) {
+        endpoints[key].writeGuidelines = info.writeGuidelines;
+      }
     }
   }
 
@@ -194,12 +266,15 @@ app.get('/api/readme', apiKeyAuth, (req, res) => {
         description: 'Withdraw your own pending submission (requires agent_withdraw_enabled setting)',
         method: 'DELETE',
         path: '/api/queue/{service}/{accountName}/status/{id}',
+        body: {
+          reason: 'Optional: Explain why you are withdrawing this request'
+        },
         constraints: [
           'Only the submitting agent can withdraw their own items',
           'Only works for "pending" status - cannot withdraw approved/completed/etc',
           'Requires admin to enable agent_withdraw_enabled setting'
         ],
-        response: '{ success: true, message: "Queue entry withdrawn", id }'
+        response: '{ success: true, message: "Queue entry withdrawn", id, reason }'
       },
       statuses: {
         pending: 'Waiting for human approval',
@@ -327,10 +402,17 @@ app.get('/api/readme', apiKeyAuth, (req, res) => {
             body: { message: 'Your broadcast message' },
             response: '{ delivered: ["Agent1", "Agent2"], failed: [{ name: "Agent3", error: "HTTP 500" }], total: 3 }',
             notes: [
-              'Broadcasts go directly to agent webhooks - not stored in messages table',
+              'Broadcasts are stored in the database and appear in message history',
               'Sender is automatically excluded from recipients',
-              'Requires messaging mode to be "supervised" or "open" (not "off")'
+              'Requires messaging mode to be "supervised" or "open" (not "off")',
+              'View broadcast history in Admin UI under Messages → Broadcast'
             ]
+          },
+          getBroadcasts: {
+            method: 'GET',
+            path: '/api/agents/messages',
+            description: 'Broadcasts appear in your regular message history with from="[BROADCAST]"',
+            note: 'No separate endpoint - broadcasts are included with regular messages'
           }
         },
         modes: {
@@ -344,7 +426,124 @@ app.get('/api/readme', apiKeyAuth, (req, res) => {
           'Maximum message length is 10KB'
         ]
       };
-    })()
+    })(),
+    memento: {
+      description: 'Durable memory storage for agents. Store and retrieve memory snapshots tagged with keywords.',
+      design: {
+        appendOnly: 'Mementos are immutable once stored. New memories can be added but not edited.',
+        keywordTagging: 'Each memento has 1-10 keywords. Keywords are normalized (lowercase) and stemmed (Porter stemmer).',
+        twoStepRetrieval: 'Search returns metadata only. Fetch specific IDs to get full content. This prevents context bloat.',
+        tokenBudget: 'Recommended ~1.5-2K tokens per memento. Hard cap: 12KB characters.'
+      },
+      endpoints: {
+        store: {
+          method: 'POST',
+          path: '/api/agents/memento',
+          body: {
+            content: 'Your memory content (required)',
+            keywords: ['keyword1', 'keyword2', '...'],
+            model: 'Model at time of storage (optional)',
+            role: 'Agent role/tier (optional)'
+          },
+          response: '{ id, agent_id, keywords, created_at }'
+        },
+        listKeywords: {
+          method: 'GET',
+          path: '/api/agents/memento/keywords',
+          description: 'List all keywords you have used (returned in stemmed form, e.g., "games" → "game")',
+          response: '{ keywords: [{ keyword, count }, ...] }'
+        },
+        search: {
+          method: 'GET',
+          path: '/api/agents/memento/search?keywords=game,project&limit=10',
+          description: 'Search mementos by keyword. Returns metadata only (preview, not full content).',
+          response: '{ matches: [{ id, keywords, created_at, preview, match_count }, ...] }'
+        },
+        recent: {
+          method: 'GET',
+          path: '/api/agents/memento/recent?limit=5',
+          description: 'Get most recent mementos (metadata only)',
+          response: '{ mementos: [{ id, keywords, created_at, preview }, ...] }'
+        },
+        fetch: {
+          method: 'GET',
+          path: '/api/agents/memento/42,38,15',
+          description: 'Fetch full content by IDs (comma-separated, max 20)',
+          response: '{ mementos: [{ id, agent_id, model, role, keywords, content, created_at }, ...] }'
+        }
+      },
+      retrievalHierarchy: [
+        '1. Check current context first — already in conversation?',
+        '2. Query Memento — if not in context, search by keyword',
+        '3. Web search — if no memento, fall back to internet'
+      ],
+      notes: [
+        'Each agent sees only their own mementos',
+        'Keywords are stemmed: "games" matches "game", "running" matches "run"',
+        'Maximum 10 keywords per memento',
+        'Maximum 12KB content per memento'
+      ]
+    },
+    serviceAccess: {
+      description: 'Service-level access control. Restrict which agents can access specific services.',
+      accessModes: {
+        all: 'All agents can access (default)',
+        allowlist: 'Only listed agents can access',
+        denylist: 'All agents EXCEPT listed ones can access'
+      },
+      endpoints: {
+        list: {
+          method: 'GET',
+          path: '/api/services',
+          description: 'List all services with their access configuration',
+          response: '{ services: [{ service, account_name, access_mode, agent_count }, ...] }'
+        },
+        getAccess: {
+          method: 'GET',
+          path: '/api/services/:service/:account/access',
+          description: 'Get access config for a specific service/account',
+          response: '{ service, account_name, access_mode, agents: [{ name, allowed }, ...] }'
+        }
+      },
+      errorResponse: {
+        status: 403,
+        body: '{ error: "Agent \'X\' does not have access to service \'Y/Z\'", reason: "not_in_allowlist" }'
+      },
+      notes: [
+        'Access checks apply to all service API calls',
+        'Default mode is "all" (backwards compatible)',
+        'Agent names are case-insensitive',
+        'Admin UI shows visual indicators for restricted services'
+      ]
+    },
+    bypassAuth: {
+      description: 'Trusted agents can bypass the write queue entirely, executing write operations immediately without human approval.',
+      warning: 'Use with extreme caution. Only enable for agents you completely trust with unsupervised write access.',
+      setup: {
+        description: 'Configure in Admin UI under API Keys → Configure → Auth Bypass',
+        note: 'This is a per-agent setting managed by the admin'
+      },
+      behavior: {
+        enabled: [
+          'All write operations (POST/PUT/DELETE) execute immediately',
+          'No queue entries are created',
+          'The agent is effectively operating unsupervised',
+          'Reads work the same as before'
+        ],
+        disabled: 'Default behavior - writes are queued for human approval'
+      },
+      checkStatus: {
+        method: 'GET',
+        path: '/api/agents/status',
+        description: 'Check if your agent has bypass_auth enabled',
+        response: '{ mode, enabled, unread_count, bypass_auth: true|false }'
+      },
+      notes: [
+        'Bypass applies to all services the agent has access to',
+        'Useful for automation agents that need to perform routine operations',
+        'Admin can revoke bypass at any time'
+      ]
+    }
   });
 });
 
@@ -504,3 +703,4 @@ server.on('error', (err) => {
   }
   process.exit(1);
 });
+

package/src/lib/agentNotifier.js

@@ -1,5 +1,5 @@
 // Agent notification delivery - sends webhooks to agent gateways
-import { getApiKeyByName, updateQueueNotification } from './db.js';
+import { getApiKeyByName, updateQueueNotification, getQueueWarnings } from './db.js';
 
 // Send a notification to an agent's webhook
 export async function notifyAgent(agentName, payload) {
@@ -83,9 +83,90 @@ export async function notifyAgentQueueStatus(entry) {
     updateQueueNotification(entry.id, false, result.error);
   }
 
+  // Also notify any agents who warned on this item
+  await notifyWarningAgentsOnResolution(entry);
+
   return result;
 }
 
+// Notify all agents who warned on a queue item when it's resolved
+async function notifyWarningAgentsOnResolution(entry) {
+  const warnings = getQueueWarnings(entry.id);
+  if (!warnings || warnings.length === 0) {
+    return;
+  }
+
+  // Get unique warning agent IDs (excluding the submitter - they already get notified)
+  const warningAgents = [...new Set(warnings.map(w => w.agent_id))]
+    .filter(agentId => agentId !== entry.submitted_by);
+
+  if (warningAgents.length === 0) {
+    return;
+  }
+
+  const statusEmoji = {
+    completed: '✅',
+    failed: '❌',
+    rejected: '🚫',
+    approved: '✅'
+  };
+
+  for (const agentId of warningAgents) {
+    const payload = {
+      type: 'queue_warning_resolved',
+      entry: {
+        id: entry.id,
+        service: entry.service,
+        account_name: entry.account_name,
+        status: entry.status,
+        comment: entry.comment,
+        rejection_reason: entry.rejection_reason,
+        submitted_by: entry.submitted_by
+      },
+      // Human-readable for Clawdbot-style gateways
+      text: `${statusEmoji[entry.status] || '📋'} [agentgate] Queue #${entry.id.substring(0, 8)} you warned on was ${entry.status}\n→ ${entry.service}/${entry.account_name}\nSubmitted by: ${entry.submitted_by}${entry.rejection_reason ? `\nReason: ${entry.rejection_reason}` : ''}`,
+      mode: 'now'
+    };
+
+    // Fire and forget - don't block on warning agent notifications
+    notifyAgent(agentId, payload).catch(err => {
+      console.error(`[agentNotifier] Failed to notify warning agent ${agentId}:`, err.message);
+    });
+  }
+}
+
+// Notify agent about a warning on their queue submission
+export async function notifyAgentQueueWarning(entry, warningAgent, warningMessage) {
+  const agentName = entry.submitted_by;
+  if (!agentName) {
+    return { success: false, error: 'No submitter on queue entry' };
+  }
+
+  const agent = getApiKeyByName(agentName);
+  if (!agent?.webhook_url) {
+    return { success: false, error: 'No webhook configured' };
+  }
+
+  const payload = {
+    type: 'queue_warning',
+    entry: {
+      id: entry.id,
+      service: entry.service,
+      account_name: entry.account_name,
+      status: entry.status,
+      comment: entry.comment
+    },
+    warning: {
+      from: warningAgent,
+      message: warningMessage
+    },
+    text: `⚠️ [agentgate] Warning on Queue #${entry.id.substring(0, 8)}\n→ ${entry.service}/${entry.account_name}\nFrom: ${warningAgent}\n"${warningMessage.substring(0, 200)}"`,
+    mode: 'now'
+  };
+
+  return await notifyAgent(agentName, payload);
+}
+
 // Notify agent about a new message (delivered to recipient)
 export async function notifyAgentMessage(message) {
   const agentName = message.to_agent;