@chrysb/alphaclaw 0.4.0 → 0.4.1-beta.0

package/lib/server/webhook-middleware.js

@@ -3,6 +3,8 @@ const https = require("https");
 const { URL } = require("url");
 
 const kRedactedHeaderKeys = new Set(["authorization", "cookie", "x-webhook-token"]);
+const kGmailDedupeTtlMs = 24 * 60 * 60 * 1000;
+const kGmailDedupeCleanupIntervalMs = 60 * 1000;
 
 const normalizeIp = (ip) => String(ip || "").replace(/^::ffff:/, "");
 
@@ -74,6 +76,51 @@ const resolveGatewayPath = ({ pathname, search }) => {
   return `${pathname}${search || ""}`;
 };
 
+const parseJsonSafe = (rawValue) => {
+  try {
+    return JSON.parse(String(rawValue || "").trim() || "{}");
+  } catch {
+    return null;
+  }
+};
+
+const getGmailPayloadData = (parsedBody) => {
+  if (!parsedBody || typeof parsedBody !== "object") return null;
+  if (parsedBody.payload && typeof parsedBody.payload === "object") {
+    return parsedBody.payload;
+  }
+  return parsedBody;
+};
+
+const getGmailMessageId = (message = {}) => {
+  const preferredId = String(message?.id || "").trim();
+  if (preferredId) return preferredId;
+  const fallbackId = String(message?.messageId || "").trim();
+  return fallbackId;
+};
+
+const buildGmailDedupedBodyBuffer = ({ parsedBody, filteredMessages }) => {
+  if (parsedBody?.payload && typeof parsedBody.payload === "object") {
+    return Buffer.from(
+      JSON.stringify({
+        ...parsedBody,
+        payload: {
+          ...parsedBody.payload,
+          messages: filteredMessages,
+        },
+      }),
+      "utf8",
+    );
+  }
+  return Buffer.from(
+    JSON.stringify({
+      ...(parsedBody || {}),
+      messages: filteredMessages,
+    }),
+    "utf8",
+  );
+};
+
 const createWebhookMiddleware = ({
   gatewayUrl,
   insertRequest,
@@ -81,6 +128,18 @@ const createWebhookMiddleware = ({
 }) => {
   const gateway = new URL(gatewayUrl);
   const protocolClient = gateway.protocol === "https:" ? https : http;
+  const gmailSeenMessageIds = new Map();
+  let lastGmailDedupeCleanupAt = 0;
+
+  const pruneGmailSeenMessageIds = (nowMs) => {
+    if (nowMs - lastGmailDedupeCleanupAt < kGmailDedupeCleanupIntervalMs) return;
+    for (const [messageKey, seenAt] of gmailSeenMessageIds.entries()) {
+      if (nowMs - seenAt > kGmailDedupeTtlMs) {
+        gmailSeenMessageIds.delete(messageKey);
+      }
+    }
+    lastGmailDedupeCleanupAt = nowMs;
+  };
 
   return (req, res) => {
     const inboundUrl = new URL(req.url, `http://${req.headers.host || "localhost"}`);
@@ -90,8 +149,47 @@ const createWebhookMiddleware = ({
       inboundUrl.searchParams.delete("token");
     }
 
-    const bodyBuffer = extractBodyBuffer(req);
+    let bodyBuffer = extractBodyBuffer(req);
     const hookName = resolveHookName(req);
+
+    if (hookName === "gmail" && bodyBuffer.length > 0) {
+      const parsedBody = parseJsonSafe(bodyBuffer.toString("utf8"));
+      const payloadData = getGmailPayloadData(parsedBody);
+      const accountKey = String(
+        payloadData?.account || payloadData?.email || payloadData?.inbox || "unknown",
+      )
+        .trim()
+        .toLowerCase();
+      const messages = Array.isArray(payloadData?.messages) ? payloadData.messages : [];
+      if (messages.length > 0) {
+        const nowMs = Date.now();
+        pruneGmailSeenMessageIds(nowMs);
+        const unseenMessages = [];
+        for (const message of messages) {
+          const messageId = getGmailMessageId(message);
+          if (!messageId) {
+            unseenMessages.push(message);
+            continue;
+          }
+          const dedupeKey = `${accountKey}:${messageId}`;
+          if (gmailSeenMessageIds.has(dedupeKey)) {
+            continue;
+          }
+          gmailSeenMessageIds.set(dedupeKey, nowMs);
+          unseenMessages.push(message);
+        }
+        if (unseenMessages.length === 0) {
+          return res.status(200).json({ ok: true, deduped: true });
+        }
+        if (unseenMessages.length < messages.length && parsedBody) {
+          bodyBuffer = buildGmailDedupedBodyBuffer({
+            parsedBody,
+            filteredMessages: unseenMessages,
+          });
+        }
+      }
+    }
+
     const sourceIp = normalizeIp(
       req.ip || req.headers["x-forwarded-for"] || req.socket?.remoteAddress || "",
     );

package/lib/server/webhooks.js

@@ -2,6 +2,14 @@ const path = require("path");
 
 const kNamePattern = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
 const kTransformsDir = "hooks/transforms";
+const kManagedWebhookConfigs = [
+  {
+    name: "gmail",
+    preset: "gmail",
+    description:
+      "Managed by AlphaClaw Gmail Watch setup. Required for internal Gmail watch delivery.",
+  },
+];
 
 const getConfigPath = ({ OPENCLAW_DIR }) =>
   path.join(OPENCLAW_DIR, "openclaw.json");
@@ -45,7 +53,25 @@ const ensureHooksRoot = (cfg) => {
   if (typeof cfg.hooks.path !== "string" || !cfg.hooks.path.trim())
     cfg.hooks.path = "/hooks";
   if (typeof cfg.hooks.token !== "string" || !cfg.hooks.token.trim()) {
-    cfg.hooks.token = "${WEBHOOK_TOKEN}";
+    cfg.hooks.token = "${OPENCLAW_HOOKS_TOKEN}";
+  }
+  if (
+    typeof cfg.hooks.defaultSessionKey !== "string" ||
+    !cfg.hooks.defaultSessionKey.trim()
+  ) {
+    cfg.hooks.defaultSessionKey = "hook:ingress";
+  }
+  if (typeof cfg.hooks.allowRequestSessionKey !== "boolean") {
+    cfg.hooks.allowRequestSessionKey = false;
+  }
+  if (!Array.isArray(cfg.hooks.allowedSessionKeyPrefixes)) {
+    cfg.hooks.allowedSessionKeyPrefixes = ["hook:"];
+  }
+  if (!cfg.hooks.allowedSessionKeyPrefixes.includes("hook:")) {
+    cfg.hooks.allowedSessionKeyPrefixes = [
+      ...cfg.hooks.allowedSessionKeyPrefixes,
+      "hook:",
+    ];
   }
   return cfg.hooks.mappings;
 };
@@ -100,36 +126,155 @@ const normalizeMappingTransformModules = (mappings) => {
   return changed;
 };
 
+const buildDefaultTransformSource = (name) =>
+  [
+    "export default async function transform(payload, context) {",
+    "  const data = payload.payload || payload;",
+    "  return {",
+    "    message: data.message,",
+    `    name: data.name || "${name}",`,
+    '    wakeMode: data.wakeMode || "now",',
+    "  };",
+    "}",
+    "",
+  ].join("\n");
+
+const ensureWebhookTransform = ({ fs, constants, name, source = "" }) => {
+  const webhookName = validateWebhookName(name);
+  const transformAbsolutePath = getTransformAbsolutePath(
+    constants,
+    webhookName,
+  );
+  fs.mkdirSync(path.dirname(transformAbsolutePath), { recursive: true });
+  if (fs.existsSync(transformAbsolutePath)) {
+    return { changed: false, path: transformAbsolutePath };
+  }
+  fs.writeFileSync(
+    transformAbsolutePath,
+    String(source || "").trim()
+      ? `${String(source).replace(/\s+$/, "")}\n`
+      : buildDefaultTransformSource(webhookName),
+  );
+  return { changed: true, path: transformAbsolutePath };
+};
+
+const ensureWebhookMapping = ({ cfg, name, mapping = {} }) => {
+  const webhookName = validateWebhookName(name);
+  const mappings = ensureHooksRoot(cfg);
+  const normalizedModulesChanged = normalizeMappingTransformModules(mappings);
+  const index = findMappingIndexByName(mappings, webhookName);
+  const defaults = {
+    match: { path: webhookName },
+    action: "agent",
+    name: webhookName,
+    wakeMode: "now",
+    transform: { module: getTransformModulePath(webhookName) },
+  };
+  if (index === -1) {
+    mappings.push({
+      ...defaults,
+      ...mapping,
+      match: { ...defaults.match, ...(mapping.match || {}) },
+      transform: { ...defaults.transform, ...(mapping.transform || {}) },
+    });
+    return { changed: true, created: true, normalizedModulesChanged };
+  }
+  const current = mappings[index] || {};
+  const next = {
+    ...current,
+    ...mapping,
+    match: {
+      ...(current.match || {}),
+      ...(mapping.match || {}),
+      path: webhookName,
+    },
+    action: mapping.action || current.action || defaults.action,
+    wakeMode: mapping.wakeMode || current.wakeMode || defaults.wakeMode,
+    transform: {
+      ...(current.transform || {}),
+      ...(mapping.transform || {}),
+      module:
+        String(mapping?.transform?.module || "").trim() ||
+        String(current?.transform?.module || "").trim() ||
+        defaults.transform.module,
+    },
+  };
+  if (JSON.stringify(current) !== JSON.stringify(next)) {
+    mappings[index] = next;
+    return { changed: true, created: false, normalizedModulesChanged };
+  }
+  return {
+    changed: normalizedModulesChanged,
+    created: false,
+    normalizedModulesChanged,
+  };
+};
+
+const listManagedWebhooksFromConfig = ({ cfg }) => {
+  const presets = Array.isArray(cfg?.hooks?.presets) ? cfg.hooks.presets : [];
+  return kManagedWebhookConfigs
+    .filter((managed) => presets.includes(managed.preset))
+    .map((managed) => ({
+      name: managed.name,
+      enabled: true,
+      createdAt: null,
+      path: `/hooks/${managed.name}`,
+      transformPath: null,
+      transformExists: true,
+      managed: true,
+      managedReason: managed.description,
+    }));
+};
+
+const isManagedWebhook = ({ cfg, name }) => {
+  const normalized = String(name || "")
+    .trim()
+    .toLowerCase();
+  if (!normalized) return false;
+  return listManagedWebhooksFromConfig({ cfg }).some(
+    (webhook) => webhook.name === normalized,
+  );
+};
+
 const listWebhooks = ({ fs, constants }) => {
   const { cfg } = readConfig({ fs, constants });
   const mappings = ensureHooksRoot(cfg);
-  return mappings
-    .filter(isWebhookMapping)
-    .map((mapping) => {
-      const name = getMappingHookName(mapping);
-      const transformPath = resolveTransformPathFromMapping(name, mapping);
-      const transformAbsolutePath = path.join(
-        constants.OPENCLAW_DIR,
-        transformPath,
-      );
-      let createdAt = null;
-      try {
-        const stat = fs.statSync(transformAbsolutePath);
-        createdAt =
-          stat.birthtime?.toISOString?.() ||
-          stat.ctime?.toISOString?.() ||
-          null;
-      } catch {}
-      return {
-        name,
-        enabled: true,
-        createdAt,
-        path: `/hooks/${name}`,
-        transformPath,
-        transformExists: fs.existsSync(transformAbsolutePath),
-      };
-    })
-    .sort((a, b) => a.name.localeCompare(b.name));
+  const managedWebhooks = listManagedWebhooksFromConfig({ cfg });
+  const managedByName = new Map(
+    managedWebhooks.map((item) => [item.name, item]),
+  );
+  const mappingWebhooks = mappings.filter(isWebhookMapping).map((mapping) => {
+    const name = getMappingHookName(mapping);
+    const managed = managedByName.get(name);
+    const transformPath = resolveTransformPathFromMapping(name, mapping);
+    const transformAbsolutePath = path.join(
+      constants.OPENCLAW_DIR,
+      transformPath,
+    );
+    let createdAt = null;
+    try {
+      const stat = fs.statSync(transformAbsolutePath);
+      createdAt =
+        stat.birthtime?.toISOString?.() || stat.ctime?.toISOString?.() || null;
+    } catch {}
+    return {
+      name,
+      enabled: true,
+      createdAt,
+      path: `/hooks/${name}`,
+      transformPath,
+      transformExists: fs.existsSync(transformAbsolutePath),
+      managed: Boolean(managed),
+      managedReason: managed?.managedReason || "",
+    };
+  });
+  const mappingNames = new Set(mappingWebhooks.map((item) => item.name));
+  const syntheticManagedWebhooks = managedWebhooks.filter(
+    (item) => !mappingNames.has(item.name),
+  );
+  return [...mappingWebhooks, ...syntheticManagedWebhooks].sort((a, b) =>
+    a.name.localeCompare(b.name),
+  );
 };
 
 const getWebhookDetail = ({ fs, constants, name }) => {
@@ -137,6 +282,12 @@ const getWebhookDetail = ({ fs, constants, name }) => {
   const hooks = listWebhooks({ fs, constants });
   const detail = hooks.find((item) => item.name === webhookName);
   if (!detail) return null;
+  if (detail.managed || !detail.transformPath) {
+    return {
+      ...detail,
+      transformExists: true,
+    };
+  }
   const transformAbsolutePath = path.join(
     constants.OPENCLAW_DIR,
     detail.transformPath,
@@ -147,56 +298,54 @@ const getWebhookDetail = ({ fs, constants, name }) => {
   };
 };
 
-const ensureStarterTransform = ({ fs, constants, name }) => {
-  const transformAbsolutePath = getTransformAbsolutePath(constants, name);
-  fs.mkdirSync(path.dirname(transformAbsolutePath), { recursive: true });
-  if (fs.existsSync(transformAbsolutePath)) return transformAbsolutePath;
-  fs.writeFileSync(
-    transformAbsolutePath,
-    [
-      "export default async function transform(payload, context) {",
-      "  const data = payload.payload || payload;",
-      "  return {",
-      "    message: data.message,",
-      `    name: data.name || "${name}",`,
-      '    wakeMode: data.wakeMode || "now",',
-      "  };",
-      "}",
-      "",
-    ].join("\n"),
-  );
-  return transformAbsolutePath;
-};
-
-const createWebhook = ({ fs, constants, name }) => {
+const createWebhook = ({
+  fs,
+  constants,
+  name,
+  upsert = false,
+  allowManagedName = false,
+  mapping = {},
+  transformSource = "",
+}) => {
   const webhookName = validateWebhookName(name);
   const { cfg, configPath } = readConfig({ fs, constants });
-  if (!cfg.hooks) cfg.hooks = {};
-  const mappings = ensureHooksRoot(cfg);
-  const normalizedModules = normalizeMappingTransformModules(mappings);
-  if (findMappingIndexByName(mappings, webhookName) !== -1) {
+  if (!allowManagedName && isManagedWebhook({ cfg, name: webhookName })) {
+    throw new Error(
+      `Webhook "${webhookName}" is managed and cannot be created manually`,
+    );
+  }
+  const existingMappings = ensureHooksRoot(cfg);
+  const exists = findMappingIndexByName(existingMappings, webhookName) !== -1;
+  if (exists && !upsert) {
     throw new Error(`Webhook "${webhookName}" already exists`);
   }
-  mappings.push({
-    match: { path: webhookName },
-    action: "agent",
+  const ensuredMapping = ensureWebhookMapping({
+    cfg,
     name: webhookName,
-    wakeMode: "now",
-    transform: { module: getTransformModulePath(webhookName) },
+    mapping,
   });
-
-  if (normalizedModules) {
-    // Keep all existing mappings consistent with transformsDir-relative module paths.
-    cfg.hooks.mappings = mappings;
+  const ensuredTransform = ensureWebhookTransform({
+    fs,
+    constants,
+    name: webhookName,
+    source: transformSource,
+  });
+  if (ensuredMapping.changed || ensuredTransform.changed || !exists) {
+    writeConfig({ fs, configPath, cfg });
   }
-  writeConfig({ fs, configPath, cfg });
-  ensureStarterTransform({ fs, constants, name: webhookName });
   return getWebhookDetail({ fs, constants, name: webhookName });
 };
 
 const deleteWebhook = ({ fs, constants, name, deleteTransformDir = false }) => {
   const webhookName = validateWebhookName(name);
   const { cfg, configPath } = readConfig({ fs, constants });
+  if (isManagedWebhook({ cfg, name: webhookName })) {
+    return {
+      removed: false,
+      managed: true,
+      deletedTransformDir: false,
+    };
+  }
   const mappings = ensureHooksRoot(cfg);
   const normalizedModules = normalizeMappingTransformModules(mappings);
   const index = findMappingIndexByName(mappings, webhookName);

package/lib/server.js

@@ -98,6 +98,7 @@ const { registerTelegramRoutes } = require("./server/routes/telegram");
 const { registerWebhookRoutes } = require("./server/routes/webhooks");
 const { registerWatchdogRoutes } = require("./server/routes/watchdog");
 const { registerUsageRoutes } = require("./server/routes/usage");
+const { registerGmailRoutes } = require("./server/routes/gmail");
 
 const { PORT, GATEWAY_URL, kTrustProxyHops, SETUP_API_PREFIXES } = constants;
 
@@ -111,6 +112,7 @@ migrateManagedInternalFiles({
 const app = express();
 app.set("trust proxy", kTrustProxyHops);
 app.use(["/webhook", "/hooks"], express.raw({ type: "*/*", limit: "5mb" }));
+app.use("/gmail-pubsub", express.raw({ type: "*/*", limit: "5mb" }));
 app.use(express.json());
 
 const proxy = httpProxy.createProxyServer({
@@ -241,6 +243,18 @@ registerGoogleRoutes({
   getApiEnableUrl,
   constants,
 });
+const gmailWatchService = registerGmailRoutes({
+  app,
+  fs,
+  constants,
+  gogCmd,
+  getBaseUrl,
+  readGoogleCredentials,
+  readEnvFile,
+  writeEnvFile,
+  reloadEnv,
+  restartRequiredState,
+});
 const telegramApi = createTelegramApi(() => process.env.TELEGRAM_BOT_TOKEN);
 const discordApi = createDiscordApi(() => process.env.DISCORD_BOT_TOKEN);
 const watchdogNotifier = createWatchdogNotifier({ telegramApi, discordApi });
@@ -345,7 +359,20 @@ server.listen(PORT, "0.0.0.0", () => {
     ensureGatewayProxyConfig(null);
     startGateway();
     watchdog.start();
+    gmailWatchService.start();
   } else {
     console.log("[alphaclaw] Awaiting onboarding via Setup UI");
   }
 });
+
+const shutdownGmailWatchService = async () => {
+  try {
+    await gmailWatchService.stop();
+  } catch {}
+};
+process.on("SIGTERM", () => {
+  shutdownGmailWatchService();
+});
+process.on("SIGINT", () => {
+  shutdownGmailWatchService();
+});

package/lib/setup/gitignore

@@ -13,6 +13,9 @@ db/
 db/**
 !skills/
 !skills/**
+!hooks/
+!hooks/transforms/
+!hooks/transforms/**
 !cron/
 !cron/jobs.json
 !openclaw.json

package/lib/setup/hourly-git-sync.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-REPO="$(cd "$(dirname "$0")" && pwd)"
+REPO="$(cd "$(dirname "$0")/.." && pwd)"
 cd "$REPO"
 
 # Load persisted env vars when running under cron's minimal environment.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chrysb/alphaclaw",
-  "version": "0.4.0",
+  "version": "0.4.1-beta.0",
   "publishConfig": {
     "access": "public"
   },