@mcinteerj/openclaw-gmail 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +117 -0
- package/index.ts +14 -0
- package/openclaw.plugin.json +11 -0
- package/package.json +54 -0
- package/src/accounts.ts +57 -0
- package/src/attachments.ts +29 -0
- package/src/channel.ts +360 -0
- package/src/config.ts +32 -0
- package/src/history-store.ts +42 -0
- package/src/html.ts +5 -0
- package/src/inbound.ts +183 -0
- package/src/monitor.ts +472 -0
- package/src/normalize.ts +42 -0
- package/src/onboarding.ts +214 -0
- package/src/outbound-check.test.ts +159 -0
- package/src/outbound-check.ts +233 -0
- package/src/outbound.ts +176 -0
- package/src/quoting.ts +230 -0
- package/src/runtime.ts +14 -0
- package/src/sanitize.test.ts +239 -0
- package/src/sanitize.ts +165 -0
- package/src/semaphore.ts +36 -0
- package/src/strip-quotes.ts +33 -0
- package/src/threading.ts +8 -0
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import fs from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import os from "node:os";
|
|
4
|
+
|
|
5
|
+
// Store historyIds in ~/.moltbot/state/gmail/history-{account}.json
|
|
6
|
+
const STORE_DIR = path.join(os.homedir(), ".moltbot", "state", "gmail");
|
|
7
|
+
|
|
8
|
+
interface HistoryState {
|
|
9
|
+
historyId: string;
|
|
10
|
+
lastSync: number;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
function getStorePath(account: string) {
|
|
14
|
+
// Sanitize email for filename
|
|
15
|
+
const safeAccount = account.replace(/[^a-z0-9@.-]/gi, "_");
|
|
16
|
+
return path.join(STORE_DIR, `history-${safeAccount}.json`);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export async function loadHistoryId(account: string): Promise<string | null> {
|
|
20
|
+
try {
|
|
21
|
+
const file = getStorePath(account);
|
|
22
|
+
const raw = await fs.readFile(file, "utf-8");
|
|
23
|
+
const data = JSON.parse(raw) as HistoryState;
|
|
24
|
+
return data.historyId;
|
|
25
|
+
} catch {
|
|
26
|
+
return null;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export async function saveHistoryId(account: string, historyId: string): Promise<void> {
|
|
31
|
+
const file = getStorePath(account);
|
|
32
|
+
const dir = path.dirname(file);
|
|
33
|
+
|
|
34
|
+
await fs.mkdir(dir, { recursive: true });
|
|
35
|
+
|
|
36
|
+
const data: HistoryState = {
|
|
37
|
+
historyId,
|
|
38
|
+
lastSync: Date.now(),
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
await fs.writeFile(file, JSON.stringify(data, null, 2), "utf-8");
|
|
42
|
+
}
|
package/src/html.ts
ADDED
package/src/inbound.ts
ADDED
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
import { type InboundMessage } from "openclaw/plugin-sdk";
|
|
2
|
+
import { extractTextBody } from "./strip-quotes.js";
|
|
3
|
+
import { extractAttachments } from "./attachments.js";
|
|
4
|
+
|
|
5
|
+
// Type for the payload from gog (simplified)
|
|
6
|
+
export interface GogPayload {
|
|
7
|
+
account: string;
|
|
8
|
+
id: string; // messageId
|
|
9
|
+
threadId: string;
|
|
10
|
+
historyId: string;
|
|
11
|
+
labelIds: string[];
|
|
12
|
+
snippet: string;
|
|
13
|
+
payload: GogMessagePart;
|
|
14
|
+
sizeEstimate: number;
|
|
15
|
+
internalDate: string;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface GogMessagePart {
|
|
19
|
+
partId: string;
|
|
20
|
+
mimeType: string;
|
|
21
|
+
filename: string;
|
|
22
|
+
headers: { name: string; value: string }[];
|
|
23
|
+
body?: { size: number; data?: string; attachmentId?: string };
|
|
24
|
+
parts?: GogMessagePart[];
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function parseInboundGmail(payload: GogPayload, accountId?: string): InboundMessage | null {
|
|
28
|
+
const headers = payload.payload.headers || [];
|
|
29
|
+
const getHeader = (name: string) => headers.find(h => h.name.toLowerCase() === name.toLowerCase())?.value;
|
|
30
|
+
|
|
31
|
+
const from = getHeader("From");
|
|
32
|
+
if (!from) return null;
|
|
33
|
+
|
|
34
|
+
// Normalize From: "Name <email>"
|
|
35
|
+
const nameMatch = from.match(/^(.*) <(.*)>$/);
|
|
36
|
+
const senderName = nameMatch ? nameMatch[1].replace(/^"|"$/g, "").trim() : undefined;
|
|
37
|
+
const senderId = nameMatch ? nameMatch[2].trim() : from.trim();
|
|
38
|
+
|
|
39
|
+
// Self-reply prevention - skip if sender is the account itself
|
|
40
|
+
if (senderId.toLowerCase() === payload.account.toLowerCase()) {
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const subject = getHeader("Subject") || "(no subject)";
|
|
45
|
+
|
|
46
|
+
// Extract both HTML and plain text parts
|
|
47
|
+
const { html, plain } = extractBody(payload.payload);
|
|
48
|
+
const cleanText = extractTextBody(html, plain);
|
|
49
|
+
|
|
50
|
+
// Only fall back to snippet if we have no body content at all
|
|
51
|
+
let finalText = cleanText;
|
|
52
|
+
if (!html && !plain) {
|
|
53
|
+
finalText = payload.snippet || "";
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// Extract and append attachment metadata
|
|
57
|
+
const attachments = extractAttachments(payload.payload);
|
|
58
|
+
let attachmentContext = "";
|
|
59
|
+
if (attachments.length > 0) {
|
|
60
|
+
const seenNames = new Map<string, number>();
|
|
61
|
+
attachmentContext = "\n\n### Attachments\n" + attachments.map(att => {
|
|
62
|
+
let displayPath = att.filename;
|
|
63
|
+
const count = seenNames.get(att.filename) || 0;
|
|
64
|
+
if (count > 0) {
|
|
65
|
+
// Handle duplicate filenames in the same message by appending short ID
|
|
66
|
+
const ext = att.filename.includes(".") ? att.filename.split(".").pop() : "";
|
|
67
|
+
const base = att.filename.includes(".") ? att.filename.split(".").slice(0, -1).join(".") : att.filename;
|
|
68
|
+
displayPath = `${base}_${att.attachmentId.substring(0, 6)}${ext ? "." + ext : ""}`;
|
|
69
|
+
}
|
|
70
|
+
seenNames.set(att.filename, count + 1);
|
|
71
|
+
|
|
72
|
+
return `- **${displayPath}** (Type: ${att.mimeType}, Size: ${formatBytes(att.size)}, ID: \`${att.attachmentId}\`)`;
|
|
73
|
+
}).join("\n");
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
const fullText = `[Thread Context: ID=${payload.threadId}, Subject="${subject}"]\n\n${finalText}${attachmentContext}`;
|
|
77
|
+
|
|
78
|
+
return {
|
|
79
|
+
channelId: "gmail",
|
|
80
|
+
accountId,
|
|
81
|
+
channelMessageId: payload.id,
|
|
82
|
+
threadId: payload.threadId,
|
|
83
|
+
text: fullText,
|
|
84
|
+
sender: {
|
|
85
|
+
id: senderId,
|
|
86
|
+
name: senderName,
|
|
87
|
+
isBot: false,
|
|
88
|
+
},
|
|
89
|
+
raw: payload,
|
|
90
|
+
isGroup: false, // Treat as direct by default for session consistency
|
|
91
|
+
replyTo: {
|
|
92
|
+
channelMessageId: payload.id,
|
|
93
|
+
},
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function formatBytes(bytes: number): string {
|
|
98
|
+
if (bytes === 0) return "0 B";
|
|
99
|
+
const k = 1024;
|
|
100
|
+
const sizes = ["B", "KB", "MB", "GB"];
|
|
101
|
+
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
|
102
|
+
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + " " + sizes[i];
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export interface GogSearchMessage {
|
|
106
|
+
id: string;
|
|
107
|
+
threadId: string;
|
|
108
|
+
date: string;
|
|
109
|
+
from: string;
|
|
110
|
+
subject: string;
|
|
111
|
+
body: string;
|
|
112
|
+
labels: string[];
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export function parseSearchGmail(msg: GogSearchMessage, accountId?: string, accountEmail?: string): InboundMessage | null {
|
|
116
|
+
const from = msg.from;
|
|
117
|
+
if (!from) return null;
|
|
118
|
+
|
|
119
|
+
// Normalize From: "Name <email>"
|
|
120
|
+
const nameMatch = from.match(/^(.*) <(.*)>$/);
|
|
121
|
+
const senderName = nameMatch ? nameMatch[1].replace(/^"|"$/g, "").trim() : undefined;
|
|
122
|
+
const senderId = nameMatch ? nameMatch[2].trim() : from.trim();
|
|
123
|
+
|
|
124
|
+
// Self-reply prevention
|
|
125
|
+
if (accountEmail && senderId.toLowerCase() === accountEmail.toLowerCase()) {
|
|
126
|
+
return null;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const subject = msg.subject || "(no subject)";
|
|
130
|
+
|
|
131
|
+
// The body from search --include-body is plain text (decoded)
|
|
132
|
+
// We still want to try stripping quotes if possible
|
|
133
|
+
const cleanText = extractTextBody(undefined, msg.body);
|
|
134
|
+
const finalText = cleanText || msg.body || "";
|
|
135
|
+
|
|
136
|
+
// For Search messages, gog doesn't give us the part structure needed for attachment ID extraction
|
|
137
|
+
// in the same sync tick. If user sees an empty body, they'll know something is there from the snippet.
|
|
138
|
+
const fullText = `[Thread Context: ID=${msg.threadId}, Subject="${subject}"]\n\n${finalText}`;
|
|
139
|
+
|
|
140
|
+
return {
|
|
141
|
+
channelId: "gmail",
|
|
142
|
+
accountId,
|
|
143
|
+
channelMessageId: msg.id,
|
|
144
|
+
threadId: msg.threadId,
|
|
145
|
+
text: fullText,
|
|
146
|
+
sender: {
|
|
147
|
+
id: senderId,
|
|
148
|
+
name: senderName,
|
|
149
|
+
isBot: false,
|
|
150
|
+
},
|
|
151
|
+
raw: msg,
|
|
152
|
+
isGroup: false,
|
|
153
|
+
replyTo: {
|
|
154
|
+
channelMessageId: msg.id,
|
|
155
|
+
},
|
|
156
|
+
timestamp: Date.parse(msg.date),
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function extractBody(part: GogMessagePart): { html?: string; plain?: string } {
|
|
161
|
+
let html: string | undefined;
|
|
162
|
+
let plain: string | undefined;
|
|
163
|
+
|
|
164
|
+
if (part.mimeType === "text/html" && part.body?.data) {
|
|
165
|
+
html = Buffer.from(part.body.data, "base64").toString("utf-8");
|
|
166
|
+
} else if (part.mimeType === "text/plain" && part.body?.data) {
|
|
167
|
+
plain = Buffer.from(part.body.data, "base64").toString("utf-8");
|
|
168
|
+
} else if (part.parts) {
|
|
169
|
+
if (part.mimeType === "multipart/alternative") {
|
|
170
|
+
const htmlPart = part.parts.find(p => p.mimeType === "text/html");
|
|
171
|
+
const plainPart = part.parts.find(p => p.mimeType === "text/plain");
|
|
172
|
+
if (htmlPart) html = extractBody(htmlPart).html;
|
|
173
|
+
if (plainPart) plain = extractBody(plainPart).plain;
|
|
174
|
+
} else {
|
|
175
|
+
const parts = part.parts.map(p => extractBody(p));
|
|
176
|
+
const htmlParts = parts.map(p => p.html).filter(Boolean);
|
|
177
|
+
const plainParts = parts.map(p => p.plain).filter(Boolean);
|
|
178
|
+
if (htmlParts.length > 0) html = htmlParts.join("\n");
|
|
179
|
+
if (plainParts.length > 0) plain = plainParts.join("\n");
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
return { html, plain };
|
|
183
|
+
}
|
package/src/monitor.ts
ADDED
|
@@ -0,0 +1,472 @@
|
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
2
|
+
import { promisify } from "node:util";
|
|
3
|
+
import fs from "node:fs/promises";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import os from "node:os";
|
|
6
|
+
import lockfile from "proper-lockfile";
|
|
7
|
+
import type { ChannelLogSink, InboundMessage } from "openclaw/plugin-sdk";
|
|
8
|
+
import type { ResolvedGmailAccount } from "./accounts.js";
|
|
9
|
+
import { loadHistoryId, saveHistoryId } from "./history-store.js";
|
|
10
|
+
import { parseInboundGmail, parseSearchGmail, type GogPayload, type GogSearchMessage } from "./inbound.js";
|
|
11
|
+
import { extractAttachments, type GmailAttachment } from "./attachments.js";
|
|
12
|
+
import { isAllowed } from "./normalize.js";
|
|
13
|
+
|
|
14
|
+
const execFileAsync = promisify(execFile);
|
|
15
|
+
|
|
16
|
+
// Polling interval: Default 60s, override via env for testing
|
|
17
|
+
const DEFAULT_POLL_INTERVAL = 60_000;
|
|
18
|
+
const POLL_INTERVAL_MS = process.env.GMAIL_POLL_INTERVAL_MS
|
|
19
|
+
? parseInt(process.env.GMAIL_POLL_INTERVAL_MS, 10)
|
|
20
|
+
: DEFAULT_POLL_INTERVAL;
|
|
21
|
+
const MAX_AUTO_DOWNLOAD_SIZE = 5 * 1024 * 1024; // 5MB
|
|
22
|
+
const QUARANTINE_LABEL = "not-allow-listed";
|
|
23
|
+
|
|
24
|
+
const sleep = (ms: number, signal?: AbortSignal) => new Promise<void>((resolve) => {
|
|
25
|
+
const timeout = setTimeout(resolve, ms);
|
|
26
|
+
signal?.addEventListener("abort", () => {
|
|
27
|
+
clearTimeout(timeout);
|
|
28
|
+
resolve();
|
|
29
|
+
}, { once: true });
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
const GOG_TIMEOUT_MS = 30_000;
|
|
33
|
+
const SYNC_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
|
|
34
|
+
|
|
35
|
+
// Local deduplication cache to prevent re-dispatching messages before Gmail updates labels
|
|
36
|
+
const dispatchedMessageIds = new Set<string>();
|
|
37
|
+
// Clear cache periodically to prevent memory growth (every hour)
|
|
38
|
+
setInterval(() => dispatchedMessageIds.clear(), 60 * 60 * 1000).unref();
|
|
39
|
+
|
|
40
|
+
interface CircuitState {
|
|
41
|
+
consecutiveFailures: number;
|
|
42
|
+
lastFailureAt: number;
|
|
43
|
+
backoffUntil: number;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const CIRCUIT_CONFIG = {
|
|
47
|
+
maxFailures: 3,
|
|
48
|
+
initialBackoffMs: 60_000, // 1 minute
|
|
49
|
+
maxBackoffMs: 15 * 60_000, // 15 minutes
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
const circuitStates = new Map<string, CircuitState>();
|
|
53
|
+
|
|
54
|
+
function getCircuit(email: string): CircuitState {
|
|
55
|
+
if (!circuitStates.has(email)) {
|
|
56
|
+
circuitStates.set(email, { consecutiveFailures: 0, lastFailureAt: 0, backoffUntil: 0 });
|
|
57
|
+
}
|
|
58
|
+
return circuitStates.get(email)!;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
async function checkGogExists(): Promise<boolean> {
|
|
62
|
+
try {
|
|
63
|
+
await execFileAsync("gog", ["--version"]);
|
|
64
|
+
return true;
|
|
65
|
+
} catch {
|
|
66
|
+
return false;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
async function runGog(args: string[], accountEmail: string, retries = 3): Promise<Record<string, unknown> | null> {
|
|
71
|
+
const circuit = getCircuit(accountEmail);
|
|
72
|
+
const now = Date.now();
|
|
73
|
+
|
|
74
|
+
if (now < circuit.backoffUntil) {
|
|
75
|
+
throw new Error(`Circuit breaker active for ${accountEmail}. Backing off until ${new Date(circuit.backoffUntil).toISOString()}`);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const allArgs = ["--json", "--account", accountEmail, ...args];
|
|
79
|
+
let lastErr: any;
|
|
80
|
+
|
|
81
|
+
for (let i = 0; i < retries; i++) {
|
|
82
|
+
try {
|
|
83
|
+
const { stdout } = await execFileAsync("gog", allArgs, {
|
|
84
|
+
maxBuffer: 10 * 1024 * 1024,
|
|
85
|
+
timeout: GOG_TIMEOUT_MS,
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
// Success - reset circuit
|
|
89
|
+
circuit.consecutiveFailures = 0;
|
|
90
|
+
circuit.backoffUntil = 0;
|
|
91
|
+
|
|
92
|
+
if (!stdout.trim()) return null;
|
|
93
|
+
return JSON.parse(stdout) as Record<string, unknown>;
|
|
94
|
+
} catch (err: unknown) {
|
|
95
|
+
lastErr = err;
|
|
96
|
+
const msg = String(err);
|
|
97
|
+
|
|
98
|
+
// Don't retry/backoff on certain errors (e.g. 404 means no messages, not a failure)
|
|
99
|
+
if (msg.includes("404")) {
|
|
100
|
+
return null;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
if (msg.includes("403") || msg.includes("invalid_grant") || msg.includes("ETIMEDOUT") || msg.includes("500")) {
|
|
104
|
+
circuit.consecutiveFailures++;
|
|
105
|
+
circuit.lastFailureAt = Date.now();
|
|
106
|
+
|
|
107
|
+
if (circuit.consecutiveFailures >= CIRCUIT_CONFIG.maxFailures) {
|
|
108
|
+
const backoff = Math.min(
|
|
109
|
+
CIRCUIT_CONFIG.initialBackoffMs * Math.pow(2, circuit.consecutiveFailures - CIRCUIT_CONFIG.maxFailures),
|
|
110
|
+
CIRCUIT_CONFIG.maxBackoffMs
|
|
111
|
+
);
|
|
112
|
+
circuit.backoffUntil = Date.now() + backoff;
|
|
113
|
+
}
|
|
114
|
+
break;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
if (i < retries - 1) {
|
|
118
|
+
await new Promise(resolve => setTimeout(resolve, 1000 * (i + 1)));
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const error = lastErr as { stderr?: string; message?: string };
|
|
124
|
+
throw new Error(`gog failed: ${error.stderr || error.message || String(lastErr)}`);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export async function quarantineMessage(id: string, accountEmail: string, log: ChannelLogSink) {
|
|
128
|
+
try {
|
|
129
|
+
// Add 'not-allow-listed', remove 'INBOX', leave UNREAD
|
|
130
|
+
await runGog(["gmail", "labels", "modify", id, "--add", "not-allow-listed", "--remove", "INBOX"], accountEmail);
|
|
131
|
+
log.info(`Quarantined message ${id} from disallowed sender (moved to not-allow-listed, removed from INBOX)`);
|
|
132
|
+
} catch (err) {
|
|
133
|
+
log.error(`Failed to quarantine message ${id}: ${String(err)}`);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
async function markAsRead(id: string, threadId: string | undefined, accountEmail: string, log: ChannelLogSink) {
|
|
138
|
+
try {
|
|
139
|
+
// Prefer thread-level modification as it's more robust in Gmail for label propagation
|
|
140
|
+
if (threadId) {
|
|
141
|
+
await runGog(["gmail", "thread", "modify", threadId, "--remove", "UNREAD"], accountEmail);
|
|
142
|
+
} else {
|
|
143
|
+
await runGog(["gmail", "labels", "modify", id, "--remove", "UNREAD"], accountEmail);
|
|
144
|
+
}
|
|
145
|
+
} catch (err) {
|
|
146
|
+
log.error(`Failed to mark ${id} as read: ${String(err)}`);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Prune old Gmail sessions and their associated attachments.
|
|
152
|
+
*/
|
|
153
|
+
async function pruneGmailSessions(account: ResolvedGmailAccount, log: ChannelLogSink) {
|
|
154
|
+
const ttlMs = account.sessionTtlDays * 24 * 60 * 60 * 1000;
|
|
155
|
+
const stateDir = path.join(os.homedir(), ".clawdbot", "agents", "main", "sessions");
|
|
156
|
+
const storePath = path.join(stateDir, "sessions.json");
|
|
157
|
+
|
|
158
|
+
// Base directory for agent workspace (where attachments are stored)
|
|
159
|
+
const agentDir = process.env.CLAWDBOT_AGENT_DIR || path.join(os.homedir(), "keith");
|
|
160
|
+
const attachmentsDir = path.join(agentDir, ".attachments");
|
|
161
|
+
|
|
162
|
+
try {
|
|
163
|
+
// Check if store exists
|
|
164
|
+
await fs.access(storePath);
|
|
165
|
+
|
|
166
|
+
const release = await lockfile.lock(storePath, {
|
|
167
|
+
stale: 10000,
|
|
168
|
+
retries: {
|
|
169
|
+
retries: 5,
|
|
170
|
+
factor: 3,
|
|
171
|
+
minTimeout: 1000,
|
|
172
|
+
maxTimeout: 5000,
|
|
173
|
+
randomize: true,
|
|
174
|
+
},
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
try {
|
|
178
|
+
const data = await fs.readFile(storePath, "utf-8");
|
|
179
|
+
const store = JSON.parse(data);
|
|
180
|
+
let changed = false;
|
|
181
|
+
const now = Date.now();
|
|
182
|
+
|
|
183
|
+
for (const key of Object.keys(store)) {
|
|
184
|
+
if (key.startsWith(`gmail:${account.email}:`)) {
|
|
185
|
+
const entry = store[key];
|
|
186
|
+
if (entry.updatedAt && now - entry.updatedAt > ttlMs) {
|
|
187
|
+
// Found an expired session
|
|
188
|
+
const threadId = key.split(":").pop();
|
|
189
|
+
|
|
190
|
+
// Delete associated attachments directory if it exists
|
|
191
|
+
if (threadId) {
|
|
192
|
+
const threadAttachmentsDir = path.join(attachmentsDir, threadId);
|
|
193
|
+
try {
|
|
194
|
+
await fs.rm(threadAttachmentsDir, { recursive: true, force: true });
|
|
195
|
+
log.info(`Pruned attachments for expired Gmail session: ${threadId}`);
|
|
196
|
+
} catch (err) {
|
|
197
|
+
log.error(`Failed to prune attachments for ${threadId}: ${String(err)}`);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
delete store[key];
|
|
202
|
+
changed = true;
|
|
203
|
+
log.info(`Pruned expired Gmail session: ${key}`);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
if (changed) {
|
|
209
|
+
await fs.writeFile(storePath, JSON.stringify(store, null, 2), "utf-8");
|
|
210
|
+
}
|
|
211
|
+
} finally {
|
|
212
|
+
await release();
|
|
213
|
+
}
|
|
214
|
+
} catch (err) {
|
|
215
|
+
// Ignore errors (e.g. file not found)
|
|
216
|
+
if ((err as any).code !== "ENOENT") {
|
|
217
|
+
log.error(`Failed to prune Gmail sessions: ${String(err)}`);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
async function fetchMessageDetails(
|
|
223
|
+
id: string,
|
|
224
|
+
account: ResolvedGmailAccount,
|
|
225
|
+
log: ChannelLogSink,
|
|
226
|
+
ignoreLabels = false
|
|
227
|
+
): Promise<InboundMessage | null> {
|
|
228
|
+
try {
|
|
229
|
+
const res = await runGog(["gmail", "get", id], account.email);
|
|
230
|
+
if (!res) return null;
|
|
231
|
+
|
|
232
|
+
const message = (res.message || res) as Record<string, unknown>;
|
|
233
|
+
const labelIds = (message.labelIds || []) as string[];
|
|
234
|
+
|
|
235
|
+
// Must be INBOX + UNREAD unless ignoring labels (e.g. from explicit search)
|
|
236
|
+
if (!ignoreLabels && (!labelIds.includes("INBOX") || !labelIds.includes("UNREAD"))) {
|
|
237
|
+
return null;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
const payload: GogPayload = {
|
|
241
|
+
...message,
|
|
242
|
+
account: account.email,
|
|
243
|
+
} as GogPayload;
|
|
244
|
+
|
|
245
|
+
return parseInboundGmail(payload, account.accountId);
|
|
246
|
+
} catch (err) {
|
|
247
|
+
log.error(`Failed to fetch message ${id}: ${String(err)}`);
|
|
248
|
+
return null;
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
async function downloadAttachmentsIfSmall(
|
|
253
|
+
msg: InboundMessage,
|
|
254
|
+
account: ResolvedGmailAccount,
|
|
255
|
+
log: ChannelLogSink
|
|
256
|
+
): Promise<string[]> {
|
|
257
|
+
if (!msg.raw || !msg.raw.payload) return [];
|
|
258
|
+
|
|
259
|
+
const attachments = extractAttachments(msg.raw.payload);
|
|
260
|
+
const downloaded: string[] = [];
|
|
261
|
+
|
|
262
|
+
const agentDir = process.env.CLAWDBOT_AGENT_DIR || path.join(os.homedir(), "keith");
|
|
263
|
+
const threadAttachmentsDir = path.join(agentDir, ".attachments", msg.threadId);
|
|
264
|
+
|
|
265
|
+
for (const att of attachments) {
|
|
266
|
+
if (att.size <= MAX_AUTO_DOWNLOAD_SIZE) {
|
|
267
|
+
try {
|
|
268
|
+
// Determine extension and safe filename
|
|
269
|
+
const ext = path.extname(att.filename) || "";
|
|
270
|
+
const safeName = path.basename(att.filename, ext).replace(/[^a-z0-9]/gi, '_') + ext;
|
|
271
|
+
const outPath = path.join(threadAttachmentsDir, safeName);
|
|
272
|
+
|
|
273
|
+
await fs.mkdir(threadAttachmentsDir, { recursive: true });
|
|
274
|
+
|
|
275
|
+
// Use gog to download
|
|
276
|
+
await execFileAsync("gog", [
|
|
277
|
+
"gmail", "attachment",
|
|
278
|
+
msg.channelMessageId,
|
|
279
|
+
att.attachmentId,
|
|
280
|
+
"--account", account.email,
|
|
281
|
+
"--out", outPath
|
|
282
|
+
]);
|
|
283
|
+
|
|
284
|
+
downloaded.push(outPath);
|
|
285
|
+
log.info(`Auto-downloaded attachment ${att.filename} to ${outPath}`);
|
|
286
|
+
} catch (err) {
|
|
287
|
+
log.error(`Failed to auto-download attachment ${att.filename}: ${err}`);
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
return downloaded;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
async function performFullSync(
|
|
295
|
+
account: ResolvedGmailAccount,
|
|
296
|
+
onMessage: (msg: InboundMessage) => Promise<void>,
|
|
297
|
+
signal: AbortSignal,
|
|
298
|
+
log: ChannelLogSink
|
|
299
|
+
): Promise<string | null> {
|
|
300
|
+
// Use label:INBOX label:UNREAD for the most reliable bot inbox pattern
|
|
301
|
+
// We explicitly ask for JSON output and include-body
|
|
302
|
+
const searchResult = await runGog([
|
|
303
|
+
"gmail", "messages", "search",
|
|
304
|
+
"label:INBOX label:UNREAD",
|
|
305
|
+
"--include-body",
|
|
306
|
+
"--max", "50"
|
|
307
|
+
], account.email);
|
|
308
|
+
|
|
309
|
+
if (!searchResult || !Array.isArray((searchResult as any).messages)) {
|
|
310
|
+
return null;
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
const rawMessages = (searchResult as any).messages as GogSearchMessage[];
|
|
314
|
+
const inboundMessages: InboundMessage[] = [];
|
|
315
|
+
|
|
316
|
+
for (const raw of rawMessages) {
|
|
317
|
+
if (signal.aborted) break;
|
|
318
|
+
|
|
319
|
+
// Parse the simplified search result
|
|
320
|
+
const msg = parseSearchGmail(raw, account.accountId, account.email);
|
|
321
|
+
|
|
322
|
+
if (msg) {
|
|
323
|
+
if (!isAllowed(msg.sender.id, account.allowFrom || [])) {
|
|
324
|
+
log.warn(`Quarantining email from non-whitelisted sender: ${msg.sender.id}`);
|
|
325
|
+
await quarantineMessage(msg.channelMessageId, account.email, log);
|
|
326
|
+
continue;
|
|
327
|
+
}
|
|
328
|
+
inboundMessages.push(msg);
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
const threads = new Map<string, InboundMessage[]>();
|
|
333
|
+
for (const msg of inboundMessages) {
|
|
334
|
+
const list = threads.get(msg.threadId) || [];
|
|
335
|
+
list.push(msg);
|
|
336
|
+
threads.set(msg.threadId, list);
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
for (const [threadId, messages] of threads) {
|
|
340
|
+
if (signal.aborted) break;
|
|
341
|
+
|
|
342
|
+
// Filter out messages we've already dispatched in this session
|
|
343
|
+
const newMessages = messages.filter(msg => !dispatchedMessageIds.has(msg.channelMessageId));
|
|
344
|
+
if (newMessages.length === 0) continue;
|
|
345
|
+
|
|
346
|
+
log.info(`[Sync] Processing thread ${threadId} with ${newMessages.length} new messages`);
|
|
347
|
+
newMessages.sort((a, b) => (a.timestamp || 0) - (b.timestamp || 0));
|
|
348
|
+
|
|
349
|
+
for (const msg of newMessages) {
|
|
350
|
+
if (signal.aborted) break;
|
|
351
|
+
|
|
352
|
+
// Add to local dedupe set to prevent race conditions during async processing
|
|
353
|
+
dispatchedMessageIds.add(msg.channelMessageId);
|
|
354
|
+
|
|
355
|
+
// To get attachments, we need the full message details (search --include-body only gives text)
|
|
356
|
+
const fullMsg = await fetchMessageDetails(msg.channelMessageId, account, log, true);
|
|
357
|
+
const msgToDispatch = fullMsg || msg;
|
|
358
|
+
|
|
359
|
+
try {
|
|
360
|
+
// Auto-download small attachments
|
|
361
|
+
if (fullMsg) {
|
|
362
|
+
const downloadedPaths = await downloadAttachmentsIfSmall(fullMsg, account, log);
|
|
363
|
+
if (downloadedPaths.length > 0) {
|
|
364
|
+
msgToDispatch.text += "\n\n### Auto-downloaded Files\n" +
|
|
365
|
+
downloadedPaths.map(p => `- \`${p}\``).join("\n");
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
await onMessage(msgToDispatch);
|
|
370
|
+
|
|
371
|
+
// CRITICAL: Only mark as read after successful dispatch
|
|
372
|
+
await markAsRead(msg.channelMessageId, msg.threadId, account.email, log);
|
|
373
|
+
} catch (err) {
|
|
374
|
+
log.error(`Failed to dispatch message ${msg.channelMessageId}, leaving as UNREAD: ${String(err)}`);
|
|
375
|
+
// Remove from dedupe so it's retried next tick
|
|
376
|
+
dispatchedMessageIds.delete(msg.channelMessageId);
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
// Get latest history ID to resume polling
|
|
382
|
+
const latest = await runGog(["gmail", "search", "label:INBOX", "--max", "1"], account.email);
|
|
383
|
+
if ((latest as any)?.threads?.[0]) {
|
|
384
|
+
// We still need a separate fetch for historyId as search result (thread list) might not have it
|
|
385
|
+
// But this is just 1 call.
|
|
386
|
+
const thread = await runGog(["gmail", "get", (latest as any).threads[0].id], account.email);
|
|
387
|
+
const nextId = (thread as any)?.message?.historyId || (thread as any)?.historyId || null;
|
|
388
|
+
return nextId;
|
|
389
|
+
}
|
|
390
|
+
return null;
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
async function ensureQuarantineLabel(accountEmail: string, log: ChannelLogSink) {
|
|
394
|
+
try {
|
|
395
|
+
const res = await runGog(["gmail", "labels", "list"], accountEmail);
|
|
396
|
+
// res.labels is likely the array from the JSON output
|
|
397
|
+
const labels = (res as any)?.labels || [];
|
|
398
|
+
const exists = labels.some((l: any) => l.name === QUARANTINE_LABEL);
|
|
399
|
+
|
|
400
|
+
if (!exists) {
|
|
401
|
+
log.info(`Creating quarantine label '${QUARANTINE_LABEL}'...`);
|
|
402
|
+
await runGog(["gmail", "labels", "create", QUARANTINE_LABEL], accountEmail);
|
|
403
|
+
}
|
|
404
|
+
} catch (err) {
|
|
405
|
+
// If this fails, quarantine attempts will also fail, but we don't block startup
|
|
406
|
+
log.error(`Failed to ensure quarantine label exists: ${String(err)}`);
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
export async function monitorGmail(params: {
|
|
411
|
+
account: ResolvedGmailAccount;
|
|
412
|
+
onMessage: (msg: InboundMessage) => Promise<void>;
|
|
413
|
+
signal: AbortSignal;
|
|
414
|
+
log: ChannelLogSink;
|
|
415
|
+
setStatus: (status: any) => void;
|
|
416
|
+
}) {
|
|
417
|
+
const { account, onMessage, signal, log, setStatus } = params;
|
|
418
|
+
|
|
419
|
+
// Doctor check
|
|
420
|
+
if (!(await checkGogExists())) {
|
|
421
|
+
log.error("gog CLI not found in PATH. Gmail channel disabled.");
|
|
422
|
+
setStatus({ accountId: account.accountId, running: false, connected: false, error: "gog CLI missing" });
|
|
423
|
+
return;
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
log.info(`Starting monitor for ${account.email}`);
|
|
427
|
+
|
|
428
|
+
// Ensure quarantine label exists
|
|
429
|
+
await ensureQuarantineLabel(account.email, log);
|
|
430
|
+
|
|
431
|
+
// Prune on start
|
|
432
|
+
await pruneGmailSessions(account, log);
|
|
433
|
+
let lastPruneAt = Date.now();
|
|
434
|
+
|
|
435
|
+
let isSyncing = false;
|
|
436
|
+
|
|
437
|
+
// Polling Loop
|
|
438
|
+
while (!signal.aborted) {
|
|
439
|
+
try {
|
|
440
|
+
const interval = account.pollIntervalMs || POLL_INTERVAL_MS;
|
|
441
|
+
await sleep(interval, signal);
|
|
442
|
+
if (signal.aborted) break;
|
|
443
|
+
|
|
444
|
+
if (isSyncing) {
|
|
445
|
+
log.warn(`Sync already in progress for ${account.email}, skipping this tick`);
|
|
446
|
+
continue;
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
// Periodically prune (once a day)
|
|
450
|
+
if (Date.now() - lastPruneAt > 24 * 60 * 60 * 1000) {
|
|
451
|
+
await pruneGmailSessions(account, log);
|
|
452
|
+
lastPruneAt = Date.now();
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
// Use Search-based polling (simpler and more robust than history API for this use case)
|
|
456
|
+
// We rely on the "UNREAD" label as our queue state.
|
|
457
|
+
isSyncing = true;
|
|
458
|
+
try {
|
|
459
|
+
log.debug("Performing full search sync...");
|
|
460
|
+
await performFullSync(account, onMessage, signal, log);
|
|
461
|
+
setStatus({ accountId: account.accountId, running: true, connected: true, lastError: undefined });
|
|
462
|
+
} finally {
|
|
463
|
+
isSyncing = false;
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
} catch (err: unknown) {
|
|
467
|
+
const msg = String(err);
|
|
468
|
+
log.error(`Monitor loop error: ${msg}`);
|
|
469
|
+
setStatus({ accountId: account.accountId, running: true, connected: false, lastError: msg });
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
}
|