@agenticmail/enterprise 0.5.83 → 0.5.85

package/src/agent-tools/tools/google/meetings.ts

@@ -0,0 +1,468 @@
+/**
+ * Google Meeting Tools
+ *
+ * Tools for detecting, managing, and joining meetings.
+ * Works with Google Calendar events + Gmail invites to automatically
+ * find meeting links and join via the browser.
+ */
+
+import type { AnyAgentTool, ToolCreationOptions } from '../../types.js';
+import { jsonResult, errorResult } from '../../common.js';
+import type { GoogleToolsConfig } from './index.js';
+
+const CALENDAR_BASE = 'https://www.googleapis.com/calendar/v3';
+
+async function calendarApi(token: string, path: string, opts?: { method?: string; body?: any; query?: Record<string, string> }): Promise<any> {
+  const url = new URL(CALENDAR_BASE + path);
+  if (opts?.query) for (const [k, v] of Object.entries(opts.query)) { if (v) url.searchParams.set(k, v); }
+  const res = await fetch(url.toString(), {
+    method: opts?.method || 'GET',
+    headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },
+    body: opts?.body ? JSON.stringify(opts.body) : undefined,
+  });
+  if (!res.ok) throw new Error(`Calendar API ${res.status}: ${await res.text()}`);
+  return res.json();
+}
+
+/** Extract meeting link from various sources */
+function extractMeetingLink(event: any): { platform: string; url: string } | null {
+  // Google Meet — direct from conferenceData
+  if (event.conferenceData?.entryPoints) {
+    for (const ep of event.conferenceData.entryPoints) {
+      if (ep.entryPointType === 'video' && ep.uri) {
+        return { platform: 'google_meet', url: ep.uri };
+      }
+    }
+  }
+
+  // Check hangoutLink
+  if (event.hangoutLink) {
+    return { platform: 'google_meet', url: event.hangoutLink };
+  }
+
+  // Parse description and location for meeting URLs
+  const text = [event.description || '', event.location || ''].join(' ');
+
+  // Zoom
+  const zoomMatch = text.match(/https:\/\/[\w.-]*zoom\.us\/[jw]\/[\d?=&\w]+/i);
+  if (zoomMatch) return { platform: 'zoom', url: zoomMatch[0] };
+
+  // Microsoft Teams
+  const teamsMatch = text.match(/https:\/\/teams\.microsoft\.com\/l\/meetup-join\/[^\s"<>]+/i);
+  if (teamsMatch) return { platform: 'teams', url: teamsMatch[0] };
+
+  // Webex
+  const webexMatch = text.match(/https:\/\/[\w.-]*webex\.com\/[\w.-]+\/j\.php[^\s"<>]*/i);
+  if (webexMatch) return { platform: 'webex', url: webexMatch[0] };
+
+  // Generic meet link in description
+  const genericMeet = text.match(/https:\/\/meet\.google\.com\/[a-z-]+/i);
+  if (genericMeet) return { platform: 'google_meet', url: genericMeet[0] };
+
+  return null;
+}
+
+function parseEventTime(event: any): { start: Date; end: Date } | null {
+  const startStr = event.start?.dateTime || event.start?.date;
+  const endStr = event.end?.dateTime || event.end?.date;
+  if (!startStr) return null;
+  return {
+    start: new Date(startStr),
+    end: endStr ? new Date(endStr) : new Date(new Date(startStr).getTime() + 3600000),
+  };
+}
+
+export function createMeetingTools(config: GoogleToolsConfig, _options?: ToolCreationOptions): AnyAgentTool[] {
+  const tp = config.tokenProvider;
+
+  return [
+    // ─── Upcoming Meetings ─────────────────────────────
+    {
+      name: 'meetings_upcoming',
+      description: 'List upcoming meetings with their join links, times, and attendees. Checks Google Calendar for events with video conference links (Google Meet, Zoom, Teams, Webex). Use this to know what meetings are coming up and when to join them.',
+      category: 'utility' as const,
+      parameters: {
+        type: 'object' as const,
+        properties: {
+          hours: { type: 'number', description: 'Look ahead this many hours (default: 24)' },
+          calendarId: { type: 'string', description: 'Calendar ID (default: "primary")' },
+          includeDeclined: { type: 'string', description: '"true" to include declined meetings' },
+        },
+        required: [],
+      },
+      async execute(_id: string, params: any) {
+        try {
+          const token = await tp.getAccessToken();
+          const hours = params.hours || 24;
+          const now = new Date();
+          const later = new Date(now.getTime() + hours * 3600000);
+          const calendarId = params.calendarId || 'primary';
+
+          const data = await calendarApi(token, `/calendars/${encodeURIComponent(calendarId)}/events`, {
+            query: {
+              timeMin: now.toISOString(),
+              timeMax: later.toISOString(),
+              singleEvents: 'true',
+              orderBy: 'startTime',
+              maxResults: '50',
+            },
+          });
+
+          const meetings = (data.items || [])
+            .map((event: any) => {
+              const times = parseEventTime(event);
+              const meetingLink = extractMeetingLink(event);
+              const myStatus = (event.attendees || []).find((a: any) => a.self)?.responseStatus;
+
+              if (params.includeDeclined !== 'true' && myStatus === 'declined') return null;
+
+              return {
+                id: event.id,
+                title: event.summary,
+                start: times?.start?.toISOString(),
+                end: times?.end?.toISOString(),
+                startsIn: times ? Math.round((times.start.getTime() - now.getTime()) / 60000) + ' minutes' : null,
+                isNow: times ? now >= times.start && now <= times.end : false,
+                meetingLink: meetingLink?.url || null,
+                platform: meetingLink?.platform || null,
+                organizer: event.organizer?.email,
+                attendees: (event.attendees || []).map((a: any) => ({
+                  email: a.email, name: a.displayName, status: a.responseStatus, self: a.self || false,
+                })),
+                myStatus,
+                description: event.description?.slice(0, 500),
+                location: event.location,
+                recurring: !!event.recurringEventId,
+              };
+            })
+            .filter(Boolean);
+
+          const withLinks = meetings.filter((m: any) => m.meetingLink);
+          const happening = meetings.filter((m: any) => m.isNow);
+
+          return jsonResult({
+            meetings,
+            total: meetings.length,
+            withMeetingLinks: withLinks.length,
+            happeningNow: happening.length,
+            nextMeeting: meetings[0] || null,
+          });
+        } catch (e: any) { return errorResult(e.message); }
+      },
+    },
+
+    // ─── Join Meeting ──────────────────────────────────
+    {
+      name: 'meeting_join',
+      description: 'Join a video meeting (Google Meet, Zoom, Teams) using the browser. The agent will navigate to the meeting URL, handle permission dialogs, click join buttons, and enter the meeting. Returns the meeting status. Requires the meeting browser to be running.',
+      category: 'utility' as const,
+      parameters: {
+        type: 'object' as const,
+        properties: {
+          url: { type: 'string', description: 'Meeting URL to join (Google Meet, Zoom, or Teams link)' },
+          eventId: { type: 'string', description: 'Google Calendar event ID — will auto-extract the meeting link' },
+          displayName: { type: 'string', description: 'Name to display in the meeting (default: agent name)' },
+          muteAudio: { type: 'string', description: '"true" to join muted (default: "true")' },
+          muteVideo: { type: 'string', description: '"true" to join with camera off (default: "true")' },
+        },
+        required: [],
+      },
+      async execute(_id: string, params: any) {
+        try {
+          const token = await tp.getAccessToken();
+          let meetingUrl = params.url;
+          let platform = '';
+
+          // If eventId provided, fetch meeting link from calendar
+          if (!meetingUrl && params.eventId) {
+            const event = await calendarApi(token, `/calendars/primary/events/${params.eventId}`);
+            const link = extractMeetingLink(event);
+            if (!link) return errorResult('No meeting link found in calendar event: ' + (event.summary || params.eventId));
+            meetingUrl = link.url;
+            platform = link.platform;
+          }
+
+          if (!meetingUrl) return errorResult('No meeting URL provided. Pass url or eventId.');
+
+          // Detect platform
+          if (!platform) {
+            if (meetingUrl.includes('meet.google.com')) platform = 'google_meet';
+            else if (meetingUrl.includes('zoom.us')) platform = 'zoom';
+            else if (meetingUrl.includes('teams.microsoft.com')) platform = 'teams';
+            else if (meetingUrl.includes('webex.com')) platform = 'webex';
+            else platform = 'unknown';
+          }
+
+          // Return instructions for the agent to execute via browser tool
+          // The agent will use browser actions (navigate, snapshot, act) to join
+          const joinInstructions: any = {
+            action: 'join_meeting',
+            platform,
+            url: meetingUrl,
+            muteAudio: params.muteAudio !== 'false',
+            muteVideo: params.muteVideo !== 'false',
+            displayName: params.displayName,
+          };
+
+          if (platform === 'google_meet') {
+            joinInstructions.steps = [
+              { step: 1, action: 'navigate', description: 'Navigate to the Meet URL', url: meetingUrl },
+              { step: 2, action: 'wait', description: 'Wait for the pre-join screen to load (2-5 seconds)' },
+              { step: 3, action: 'snapshot', description: 'Take a snapshot to see the pre-join UI' },
+              { step: 4, action: 'mute', description: 'If muteAudio=true, click the microphone toggle button to mute' },
+              { step: 5, action: 'camera_off', description: 'If muteVideo=true, click the camera toggle button to turn off camera' },
+              { step: 6, action: 'set_name', description: 'If not logged in, enter display name in the name field' },
+              { step: 7, action: 'join', description: 'Click "Join now" or "Ask to join" button' },
+              { step: 8, action: 'verify', description: 'Take a snapshot to confirm you are in the meeting' },
+            ];
+            joinInstructions.selectors = {
+              joinButton: '[data-is-muted] ~ div button, [jsname="Qx7uuf"], button[data-idom-class*="join"]',
+              muteAudioButton: '[data-is-muted][aria-label*="microphone"], [data-tooltip*="microphone"]',
+              muteVideoButton: '[data-is-muted][aria-label*="camera"], [data-tooltip*="camera"]',
+              nameInput: 'input[aria-label*="name"], input[placeholder*="name"]',
+              gotItButton: 'button[jsname="LgbsSe"]',
+            };
+          } else if (platform === 'zoom') {
+            joinInstructions.steps = [
+              { step: 1, action: 'navigate', description: 'Navigate to Zoom URL', url: meetingUrl },
+              { step: 2, action: 'snapshot', description: 'Look for "Join from Your Browser" or "Launch Meeting" link' },
+              { step: 3, action: 'click_browser_join', description: 'Click "Join from Your Browser" to avoid Zoom client' },
+              { step: 4, action: 'enter_name', description: 'Enter display name if prompted' },
+              { step: 5, action: 'join', description: 'Click Join button' },
+            ];
+          } else if (platform === 'teams') {
+            joinInstructions.steps = [
+              { step: 1, action: 'navigate', description: 'Navigate to Teams URL', url: meetingUrl },
+              { step: 2, action: 'snapshot', description: 'Look for "Continue on this browser" option' },
+              { step: 3, action: 'click_browser', description: 'Click "Continue on this browser" (not the app)' },
+              { step: 4, action: 'configure', description: 'Toggle mic/camera as needed on pre-join screen' },
+              { step: 5, action: 'join', description: 'Click "Join now"' },
+            ];
+          }
+
+          return jsonResult(joinInstructions);
+        } catch (e: any) { return errorResult(e.message); }
+      },
+    },
+
+    // ─── Meeting Actions ───────────────────────────────
+    {
+      name: 'meeting_action',
+      description: 'Perform actions during an active meeting: mute/unmute, toggle camera, send chat message, raise hand, leave meeting, take notes from what is visible on screen, read chat messages.',
+      category: 'utility' as const,
+      parameters: {
+        type: 'object' as const,
+        properties: {
+          action: { type: 'string', description: 'Action: mute, unmute, camera_on, camera_off, chat, raise_hand, lower_hand, leave, read_chat, screenshot, participants' },
+          message: { type: 'string', description: 'Chat message to send (for chat action)' },
+          platform: { type: 'string', description: 'Meeting platform: google_meet, zoom, teams (auto-detected from current page if omitted)' },
+        },
+        required: ['action'],
+      },
+      async execute(_id: string, params: any) {
+        try {
+          const action = params.action;
+          const platform = params.platform || 'google_meet';
+
+          // Return browser instructions for the agent to execute
+          const instructions: any = { action, platform };
+
+          const platformSelectors: Record<string, Record<string, any>> = {
+            google_meet: {
+              mute: { description: 'Click microphone button to mute', selector: '[aria-label*="microphone"][data-is-muted="false"]', key: 'Ctrl+d' },
+              unmute: { description: 'Click microphone button to unmute', selector: '[aria-label*="microphone"][data-is-muted="true"]', key: 'Ctrl+d' },
+              camera_on: { description: 'Click camera button to turn on', selector: '[aria-label*="camera"][data-is-muted="true"]', key: 'Ctrl+e' },
+              camera_off: { description: 'Click camera button to turn off', selector: '[aria-label*="camera"][data-is-muted="false"]', key: 'Ctrl+e' },
+              chat: { description: 'Open chat, type message, send', steps: ['Click chat icon', 'Type in chat input', 'Press Enter'] },
+              raise_hand: { description: 'Click raise hand button', key: 'Ctrl+Alt+h' },
+              lower_hand: { description: 'Click lower hand button', key: 'Ctrl+Alt+h' },
+              leave: { description: 'Click the red leave/hang up button', selector: '[aria-label*="Leave"], [aria-label*="leave call"]' },
+              read_chat: { description: 'Take a snapshot focused on the chat panel to read messages' },
+              screenshot: { description: 'Take a screenshot of the current meeting view' },
+              participants: { description: 'Open participants panel and take a snapshot to list attendees' },
+            },
+            zoom: {
+              mute: { description: 'Click Mute button', selector: '.join-audio-container button, [aria-label*="mute"]' },
+              unmute: { description: 'Click Unmute button', selector: '.join-audio-container button, [aria-label*="unmute"]' },
+              leave: { description: 'Click Leave button', selector: '[aria-label*="Leave"], .leave-meeting-btn' },
+              chat: { description: 'Open chat panel and type message' },
+            },
+            teams: {
+              mute: { description: 'Click microphone button', selector: '[aria-label*="Mute"], #microphone-button' },
+              unmute: { description: 'Click microphone button', selector: '[aria-label*="Unmute"], #microphone-button' },
+              leave: { description: 'Click Leave/Hang up', selector: '[aria-label*="Leave"], [aria-label*="Hang up"], #hangup-button' },
+              chat: { description: 'Open chat and type message' },
+            },
+          };
+
+          const platSel = platformSelectors[platform] || platformSelectors.google_meet;
+          const actionInfo = platSel[action];
+
+          if (actionInfo) {
+            instructions.browserAction = actionInfo;
+            instructions.hint = 'Use the browser tool to execute this action. Take a snapshot first to confirm the meeting state, then perform the action.';
+          } else {
+            instructions.hint = 'Unsupported action for this platform. Take a screenshot to see the current meeting UI and interact manually via browser tool.';
+          }
+
+          return jsonResult(instructions);
+        } catch (e: any) { return errorResult(e.message); }
+      },
+    },
+
+    // ─── Detect Meeting Invites in Email ───────────────
+    {
+      name: 'meetings_scan_inbox',
+      description: 'Scan recent emails for meeting invitations and extract meeting links, times, and details. Checks for Google Meet, Zoom, Teams, and Webex links in email bodies and calendar attachments (.ics files).',
+      category: 'utility' as const,
+      parameters: {
+        type: 'object' as const,
+        properties: {
+          hours: { type: 'number', description: 'Scan emails from last N hours (default: 24)' },
+          maxResults: { type: 'number', description: 'Max emails to scan (default: 30)' },
+        },
+        required: [],
+      },
+      async execute(_id: string, params: any) {
+        try {
+          const token = await tp.getAccessToken();
+          const hours = params.hours || 24;
+          const after = new Date(Date.now() - hours * 3600000);
+          const dateStr = `${after.getFullYear()}/${String(after.getMonth() + 1).padStart(2, '0')}/${String(after.getDate()).padStart(2, '0')}`;
+
+          // Search for emails with meeting-related content
+          const GMAIL_BASE = 'https://gmail.googleapis.com/gmail/v1/users/me';
+          const query = `after:${dateStr} (meet.google.com OR zoom.us OR teams.microsoft.com OR "meeting invitation" OR "calendar invitation" OR filename:ics)`;
+          const searchUrl = new URL(`${GMAIL_BASE}/messages`);
+          searchUrl.searchParams.set('q', query);
+          searchUrl.searchParams.set('maxResults', String(params.maxResults || 30));
+
+          const searchRes = await fetch(searchUrl.toString(), { headers: { Authorization: `Bearer ${token}` } });
+          if (!searchRes.ok) throw new Error(`Gmail search failed: ${searchRes.status}`);
+          const searchData = await searchRes.json() as any;
+
+          if (!searchData.messages?.length) {
+            return jsonResult({ meetings: [], count: 0, message: 'No meeting invites found in the last ' + hours + ' hours' });
+          }
+
+          // Fetch each message to extract meeting links
+          const meetings: any[] = [];
+          for (const msg of searchData.messages.slice(0, 20)) {
+            try {
+              const msgRes = await fetch(`${GMAIL_BASE}/messages/${msg.id}?format=full`, { headers: { Authorization: `Bearer ${token}` } });
+              if (!msgRes.ok) continue;
+              const msgData = await msgRes.json() as any;
+
+              const headers = msgData.payload?.headers || [];
+              const subject = headers.find((h: any) => h.name?.toLowerCase() === 'subject')?.value || '';
+              const from = headers.find((h: any) => h.name?.toLowerCase() === 'from')?.value || '';
+              const date = headers.find((h: any) => h.name?.toLowerCase() === 'date')?.value || '';
+
+              // Extract body text
+              let bodyText = '';
+              function walkParts(part: any) {
+                if (part.body?.data) {
+                  bodyText += Buffer.from(part.body.data.replace(/-/g, '+').replace(/_/g, '/'), 'base64').toString('utf-8') + ' ';
+                }
+                if (part.parts) part.parts.forEach(walkParts);
+              }
+              walkParts(msgData.payload);
+
+              // Find meeting links
+              const meetLinks: any[] = [];
+              const patterns = [
+                { regex: /https:\/\/meet\.google\.com\/[a-z-]+/gi, platform: 'google_meet' },
+                { regex: /https:\/\/[\w.-]*zoom\.us\/[jw]\/[\d?=&\w]+/gi, platform: 'zoom' },
+                { regex: /https:\/\/teams\.microsoft\.com\/l\/meetup-join\/[^\s"<>]+/gi, platform: 'teams' },
+                { regex: /https:\/\/[\w.-]*webex\.com\/[\w.-]+\/j\.php[^\s"<>]*/gi, platform: 'webex' },
+              ];
+
+              for (const p of patterns) {
+                const matches = bodyText.match(p.regex);
+                if (matches) {
+                  for (const url of [...new Set(matches)]) {
+                    meetLinks.push({ platform: p.platform, url });
+                  }
+                }
+              }
+
+              if (meetLinks.length > 0) {
+                meetings.push({
+                  messageId: msg.id,
+                  subject,
+                  from,
+                  date,
+                  meetingLinks: meetLinks,
+                  snippet: msgData.snippet?.slice(0, 200),
+                });
+              }
+            } catch { /* skip individual message errors */ }
+          }
+
+          return jsonResult({
+            meetings,
+            count: meetings.length,
+            scannedEmails: searchData.messages.length,
+            timeRange: hours + ' hours',
+          });
+        } catch (e: any) { return errorResult(e.message); }
+      },
+    },
+
+    // ─── RSVP to Meeting ───────────────────────────────
+    {
+      name: 'meeting_rsvp',
+      description: 'Accept or decline a Google Calendar meeting invitation.',
+      category: 'utility' as const,
+      parameters: {
+        type: 'object' as const,
+        properties: {
+          eventId: { type: 'string', description: 'Calendar event ID (required)' },
+          response: { type: 'string', description: '"accepted", "declined", or "tentative" (required)' },
+          calendarId: { type: 'string', description: 'Calendar ID (default: "primary")' },
+          comment: { type: 'string', description: 'Optional RSVP comment' },
+        },
+        required: ['eventId', 'response'],
+      },
+      async execute(_id: string, params: any) {
+        try {
+          const token = await tp.getAccessToken();
+          const calendarId = params.calendarId || 'primary';
+          const email = tp.getEmail();
+
+          // Get the event first
+          const event = await calendarApi(token, `/calendars/${encodeURIComponent(calendarId)}/events/${params.eventId}`);
+
+          // Update attendee status
+          const attendees = (event.attendees || []).map((a: any) => {
+            if (a.self || a.email === email) {
+              return { ...a, responseStatus: params.response, comment: params.comment };
+            }
+            return a;
+          });
+
+          // If agent isn't in attendees list, add self
+          if (!attendees.find((a: any) => a.self || a.email === email)) {
+            attendees.push({ email, responseStatus: params.response, comment: params.comment });
+          }
+
+          const updated = await calendarApi(token, `/calendars/${encodeURIComponent(calendarId)}/events/${params.eventId}`, {
+            method: 'PATCH',
+            query: { sendUpdates: 'all' },
+            body: { attendees },
+          });
+
+          const link = extractMeetingLink(updated);
+          return jsonResult({
+            rsvp: params.response,
+            eventId: params.eventId,
+            title: updated.summary,
+            start: updated.start?.dateTime || updated.start?.date,
+            meetingLink: link?.url,
+            platform: link?.platform,
+          });
+        } catch (e: any) { return errorResult(e.message); }
+      },
+    },
+  ];
+}