askshepherd 0.1.40 → 0.1.42

package/bin/shepherd-onboard.js

@@ -1,18 +1,21 @@
 #!/usr/bin/env node
 import { execFile, execFileSync, spawn } from "node:child_process";
 import { createHash } from "node:crypto";
-import { constants as fsConstants, existsSync, mkdirSync, readFileSync, unlinkSync, watch, writeFileSync } from "node:fs";
+import { constants as fsConstants, existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, renameSync, unlinkSync, watch, writeFileSync } from "node:fs";
 import { access, chmod, mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
 import { createServer } from "node:http";
 import { homedir, platform } from "node:os";
 import { basename, dirname, join } from "node:path";
 import readline from "node:readline";
 import { fileURLToPath } from "node:url";
+import sourceSelectionHelpers from "../source-selection.cjs";
+
+const { sourceSelectionFromSession } = sourceSelectionHelpers;
 
 const DEFAULT_API_URL = "https://brain-api-customer-facing.up.railway.app";
 const PACKAGE_NAME = "askshepherd";
 const PACKAGE_SPEC = `${PACKAGE_NAME}@latest`;
-const PACKAGE_VERSION = "0.1.40";
+const PACKAGE_VERSION = "0.1.42";
 const MCP_SERVER_NAME = "shepherd";
 const PACKAGE_DIR = dirname(dirname(fileURLToPath(import.meta.url)));
 const DEFAULT_AGENT_STATE_PATH = join(homedir(), ".shepherd", "raw-onboarding-agent.json");
@@ -24,7 +27,7 @@ const MAX_QUEUE_MESSAGES = 10_000;
 const DEFAULT_MESSAGE_CHAT_SEARCH_LIMIT = 200;
 const INITIAL_MESSAGE_CHAT_ROWS = 20;
 const ALL_MESSAGES_CHATS = "__shepherd_all_messages_chats__";
-const AGENT_MODALITY_ORDER = ["google", "slack", "granola", "messages", "codingSessions"];
+const AGENT_MODALITY_ORDER = ["google", "slack", "github", "granola", "messages", "codingSessions"];
 const SHEPHERD_LOGO_PATH = join(PACKAGE_DIR, "assets", "shepherd_G_vector_136033.png");
 const GRANOLA_API_KEYS_PATH = "/settings/integrations/api-keys";
 const GOOGLE_WORKSPACE_DELEGATION_ADMIN_URL = "https://admin.google.com/ac/owl/domainwidedelegation";
@@ -38,8 +41,22 @@ const CLAUDE_PROJECTS_DIR = join(homedir(), ".claude", "projects");
 const CONTACTS_WAL_PATH = join(homedir(), "Library", "Application Support", "AddressBook", "AddressBook-v22.abcddb-wal");
 const CONTACT_SYNC_DEBOUNCE_MS = 5_000;
 const CONTACT_SYNC_FALLBACK_MS = 30 * 60_000;
+const MESSAGES_EDIT_SCAN_INTERVAL_MS = positiveIntFromEnv("SHEPHERD_MESSAGES_EDIT_SCAN_INTERVAL_MS", 60_000);
+const MESSAGES_EDIT_SCAN_WINDOW_MS = positiveIntFromEnv("SHEPHERD_MESSAGES_EDIT_SCAN_WINDOW_MS", 30 * 60_000);
+const MESSAGES_EDIT_SCAN_LIMIT = positiveIntFromEnv("SHEPHERD_MESSAGES_EDIT_SCAN_LIMIT", 500);
+const MESSAGES_MUTATION_RECONCILE_INTERVAL_MS = positiveIntFromEnv("SHEPHERD_MESSAGES_MUTATION_RECONCILE_INTERVAL_MS", 15 * 60_000);
+const MESSAGES_STATE_CACHE_MAX = positiveIntFromEnv("SHEPHERD_MESSAGES_STATE_CACHE_MAX", 5_000);
+const LEGACY_SHEPHERD_OWNED_MESSAGE_HANDLES = [
+  "+13054098546",
+  "+12054012556",
+];
 const SHEPHERD_OWNED_MESSAGE_HANDLES = parseMessageHandleList(
-  process.env.SHEPHERD_OWNED_MESSAGE_HANDLES ?? process.env.SENDBLUE_NUMBER ?? "",
+  [
+    ...LEGACY_SHEPHERD_OWNED_MESSAGE_HANDLES,
+    process.env.SENDBLUE_NUMBER,
+    process.env.SHEPHERD_OWNED_MESSAGE_HANDLES,
+    process.env.SHEPHERD_OWNED_IMESSAGE_HANDLES,
+  ].filter(Boolean).join(","),
 );
 const GOOGLE_WORKSPACE_DELEGATION_APP_NAME = "Shepherd";
 const GOOGLE_WORKSPACE_DELEGATION_SERVICE_ACCOUNT_EMAIL =
@@ -101,18 +118,30 @@ const command = rawArgv[0] && !rawArgv[0].startsWith("--") ? rawArgv[0] : "onboa
 const args = parseArgs(command === "onboard" ? rawArgv : rawArgv.slice(1));
 let messagesPermissionNoticePrinted = false;
 
-if (command === "help" || args.help) {
-  printHelp(command === "help" ? "onboard" : command);
-  process.exit(0);
+if (isCliEntrypoint()) {
+  if (command === "help" || args.help) {
+    printHelp(command === "help" ? "onboard" : command);
+    process.exit(0);
+  }
+
+  void dispatch().catch((err) => {
+    console.error(`\nShepherd onboarding failed: ${safeError(err)}`);
+    if (args.debug === true) {
+      console.error(rawErrorDetails(err));
+    }
+    process.exit(1);
+  });
 }
 
-void dispatch().catch((err) => {
-  console.error(`\nShepherd onboarding failed: ${safeError(err)}`);
-  if (args.debug === true) {
-    console.error(rawErrorDetails(err));
+function isCliEntrypoint() {
+  if (!process.argv[1]) return false;
+  const modulePath = fileURLToPath(import.meta.url);
+  try {
+    return realpathSync(modulePath) === realpathSync(process.argv[1]);
+  } catch {
+    return modulePath === process.argv[1];
   }
-  process.exit(1);
-});
+}
 
 async function dispatch() {
   if (command === "onboard") {
@@ -139,6 +168,10 @@ async function dispatch() {
     await runWriteMessagesConfig();
   } else if (command === "install-messages-agent") {
     await runInstallMessagesAgent();
+  } else if (command === "write-coding-sessions-config") {
+    await runWriteCodingSessionsConfig();
+  } else if (command === "install-coding-sessions-agent") {
+    await runInstallCodingSessionsAgent();
   } else if (command === "coding-sessions-agent") {
     await runCodingSessionsAgent();
   } else if (command === "coding-sessions-status") {
@@ -354,15 +387,16 @@ async function runAgentOnboarding() {
     authSessionToken: workosAuth.authSessionToken,
     sources,
   });
+  const sessionSources = sourceSelectionFromSession(session, sources);
 
   const statePath = await writeAgentState({
     apiUrl,
     sessionId: session.sessionId,
     sessionToken: session.sessionToken,
     account: session.account,
-    sources,
+    sources: sessionSources,
     authUrls: session.authUrls ?? {},
-    googleWorkspaceDelegation: sources.google
+    googleWorkspaceDelegation: sessionSources.google && session.googleWorkspaceDelegation
       ? googleWorkspaceDelegationSetup(session.googleWorkspaceDelegation)
       : undefined,
     workosAuth,
@@ -380,7 +414,8 @@ async function runAgentOnboarding() {
       status: "auth_required",
       account: publicAgentAccount(session.account),
       opened: currentAction?.opened ? [currentAction.source] : [],
-      googleWorkspaceDelegation: sources.google ? googleWorkspaceDelegationSetup(session.googleWorkspaceDelegation) : undefined,
+      sources: sessionSources,
+      googleWorkspaceDelegation: sessionSources.google && session.googleWorkspaceDelegation ? googleWorkspaceDelegationSetup(session.googleWorkspaceDelegation) : undefined,
       currentAction,
       statePath,
       messagesChatsCommand: sources.messages ? `${agentCommand()} messages-chats` : undefined,
@@ -491,7 +526,7 @@ async function runMcpLogin() {
   const statePath = await writeMcpStateFromLogin(login);
   const installTargets = await selectMcpInstallTargets();
   const installResults = installTargets.length > 0
-    ? await installMcpClients({ statePath, targets: installTargets })
+    ? await installMcpClients({ statePath, targets: installTargets, proxyProgram: parseJsonArrayArg("mcp-program") })
     : [];
 
   if (args.json) {
@@ -680,7 +715,7 @@ async function runMcpInstall() {
   const ensured = await ensureMcpState({ allowBrowser: process.stdin.isTTY, quiet: args.json === true });
   const targets = await selectMcpInstallTargets({ defaultTargets: MCP_INSTALL_TARGETS });
   const installResults = targets.length > 0
-    ? await installMcpClients({ statePath: ensured.statePath, targets })
+    ? await installMcpClients({ statePath: ensured.statePath, targets, proxyProgram: parseJsonArrayArg("mcp-program") })
     : [];
 
   if (args.json) {
@@ -982,16 +1017,17 @@ function parseMcpInstallTargets(value) {
   return [...new Set(targets)];
 }
 
-async function installMcpClients({ statePath, targets }) {
+async function installMcpClients({ statePath, targets, proxyProgram }) {
   const results = [];
+  const proxy = mcpProxyCommand(statePath, proxyProgram);
   for (const target of targets) {
     try {
       if (target === "codex") {
-        await installCodexMcp(statePath);
+        await installCodexMcp(proxy);
       } else if (target === "claude") {
-        await installClaudeMcp(statePath);
+        await installClaudeMcp(proxy);
       } else if (target === "cursor") {
-        await installCursorMcp(statePath);
+        await installCursorMcp(proxy);
       }
       results.push({ target, status: "installed" });
     } catch (err) {
@@ -1001,25 +1037,25 @@ async function installMcpClients({ statePath, targets }) {
   return results;
 }
 
-async function installCodexMcp(statePath) {
+async function installCodexMcp(proxy) {
   await execFileQuiet("codex", ["mcp", "remove", MCP_SERVER_NAME], { ignoreError: true });
-  await execFileQuiet("codex", ["mcp", "add", MCP_SERVER_NAME, "--", "npx", ...mcpProxyArgs(statePath)]);
+  await execFileQuiet("codex", ["mcp", "add", MCP_SERVER_NAME, "--", proxy.command, ...proxy.args]);
 }
 
-async function installClaudeMcp(statePath) {
+async function installClaudeMcp(proxy) {
   await execFileQuiet("claude", ["mcp", "remove", MCP_SERVER_NAME], { ignoreError: true });
-  await execFileQuiet("claude", ["mcp", "add", "--scope", "user", MCP_SERVER_NAME, "--", "npx", ...mcpProxyArgs(statePath)]);
+  await execFileQuiet("claude", ["mcp", "add", "--scope", "user", MCP_SERVER_NAME, "--", proxy.command, ...proxy.args]);
 }
 
-async function installCursorMcp(statePath) {
+async function installCursorMcp(proxy) {
   const path = join(homedir(), ".cursor", "mcp.json");
   const config = await readJsonObject(path);
   const mcpServers = config.mcpServers && typeof config.mcpServers === "object" && !Array.isArray(config.mcpServers)
     ? config.mcpServers
     : {};
   mcpServers[MCP_SERVER_NAME] = {
-    command: "npx",
-    args: mcpProxyArgs(statePath),
+    command: proxy.command,
+    args: proxy.args,
   };
   config.mcpServers = mcpServers;
   await mkdir(dirname(path), { recursive: true });
@@ -1027,8 +1063,18 @@ async function installCursorMcp(statePath) {
   await execFileQuiet("cursor-agent", ["mcp", "enable", MCP_SERVER_NAME], { ignoreError: true });
 }
 
+function mcpProxyCommand(statePath, proxyProgram) {
+  const prefix = Array.isArray(proxyProgram) && proxyProgram.length > 0
+    ? proxyProgram
+    : ["npx", "-y", PACKAGE_SPEC];
+  return {
+    command: prefix[0],
+    args: [...prefix.slice(1), "mcp", "--state", statePath],
+  };
+}
+
 function mcpProxyArgs(statePath) {
-  return ["-y", PACKAGE_SPEC, "mcp", "--state", statePath];
+  return mcpProxyCommand(statePath).args;
 }
 
 async function readJsonObject(path) {
@@ -1192,7 +1238,7 @@ async function runStatusCommand() {
 
 async function collectShepherdStatus() {
   const statePath = agentStatePath();
-  const state = await readOptionalAgentState();
+  let state = await readOptionalAgentState();
   let production = null;
   let productionError = null;
 
@@ -1202,7 +1248,7 @@ async function collectShepherdStatus() {
         `${trimTrailingSlash(state.apiUrl)}/onboarding/raw/session/${encodeURIComponent(state.sessionId)}/status`,
         { token: state.sessionToken },
       );
-      await updateAgentStateFromOnboardingResponse(state, production);
+      state = await updateAgentStateFromOnboardingResponse(state, production);
     } catch (err) {
       productionError = safeError(err);
     }
@@ -1296,8 +1342,11 @@ function statusSourceRows(providers, savedSources = {}) {
   const definitions = [
     ["google", "Google Workspace", "google"],
     ["slack", "Slack", "slack"],
+    ["github", "GitHub", "github"],
     ["granola", "Granola", "granola"],
     ["messages", "Messages", "messages"],
+    ["discord", "Discord", "discord"],
+    ["instagram", "Instagram", "instagram"],
     ["codingSessions", "Coding Sessions", "codingSessions"],
   ];
   return definitions.map(([key, label, sourceKey]) => ({
@@ -1327,6 +1376,9 @@ function renderLocalMessagesStatus(status) {
     lines.push("  LaunchAgent: not installed or unavailable");
   }
   lines.push(`  Messages database: ${status.storage.readable ? "readable" : `not readable (${status.storage.reason})`}`);
+  if (status.contacts) {
+    lines.push(`  Contacts: ${status.contacts.ok ? "resolved from AddressBook DB" : `degraded (${status.contacts.error ?? "unsupported local Contacts database schema"}; contact founders@askshepherd.ai)`}`);
+  }
   lines.push(`  Queued unsent messages: ${status.queueDepth}`);
   return lines;
 }
@@ -1379,6 +1431,7 @@ async function runWriteMessagesConfig() {
     apiUrl: trimTrailingSlash(requiredConfigString(input.apiUrl, "apiUrl")),
     userId: requiredConfigString(input.userId, "userId"),
     agentToken: requiredConfigString(input.agentToken, "agentToken"),
+    handle: optionalString(input.handle),
     backfillDays: parseBackfillDays(input.backfillDays, null),
     allowedChatIds: input.allowedChatIds,
     selectedChats: Array.isArray(input.selectedChats) ? input.selectedChats : [],
@@ -1386,6 +1439,20 @@ async function runWriteMessagesConfig() {
   console.log(JSON.stringify({ configPath }, null, 2));
 }
 
+async function runWriteCodingSessionsConfig() {
+  const input = await readJsonInput();
+  if (!input || typeof input !== "object" || Array.isArray(input)) {
+    throw new Error("write-coding-sessions-config expects a JSON object on stdin.");
+  }
+  const configPath = await writeCodingSessionsConfig({
+    apiUrl: trimTrailingSlash(requiredConfigString(input.apiUrl, "apiUrl")),
+    userId: requiredConfigString(input.userId, "userId"),
+    agentToken: requiredConfigString(input.agentToken, "agentToken"),
+    intervalSeconds: Number(input.intervalSeconds ?? args["coding-sessions-interval-seconds"] ?? 60),
+  });
+  console.log(JSON.stringify({ configPath }, null, 2));
+}
+
 async function runInstallMessagesAgent() {
   const configPath = stringArg("config");
   if (!configPath) throw new Error("install-messages-agent requires --config <path>");
@@ -1411,6 +1478,31 @@ async function runInstallMessagesAgent() {
   console.log(JSON.stringify(install, null, 2));
 }
 
+async function runInstallCodingSessionsAgent() {
+  const configPath = stringArg("config");
+  if (!configPath) throw new Error("install-coding-sessions-agent requires --config <path>");
+  let config;
+  try {
+    config = JSON.parse(await readFile(configPath, "utf8"));
+  } catch (err) {
+    if (err && typeof err === "object" && "code" in err) throw err;
+    throw new Error(`install-coding-sessions-agent: config file at ${configPath} does not contain valid JSON.`);
+  }
+  const userId = stringArg("user-id") ?? requiredConfigString(config.userId, "userId");
+  const overrides = {
+    programArguments: parseJsonArrayArg("program"),
+    environment: parseJsonObjectArg("env"),
+  };
+
+  if (args["dry-run"]) {
+    console.log(JSON.stringify(buildCodingSessionsAgentInstall(configPath, userId, overrides), null, 2));
+    return;
+  }
+
+  const install = await installCodingSessionsAgent(configPath, userId, overrides);
+  console.log(JSON.stringify(install, null, 2));
+}
+
 async function readJsonInput() {
   const chunks = [];
   for await (const chunk of process.stdin) chunks.push(chunk);
@@ -1492,7 +1584,9 @@ async function runMessagesAgent() {
   const userId = requiredConfigString(config.userId, "userId");
   const agentToken = requiredConfigString(config.agentToken, "agentToken");
   mergeShepherdOwnedMessageHandles(config.excludedMessageHandles);
-  const backfillDays = parseBackfillDays(args["backfill-days"] ?? process.env.SHEPHERD_BACKFILL_DAYS ?? config.backfillDays, null);
+  const backfillOverride = args["backfill-days"] ?? process.env.SHEPHERD_BACKFILL_DAYS;
+  const backfillExplicit = backfillOverride !== undefined && backfillOverride !== null && backfillOverride !== "";
+  const backfillDays = parseBackfillDays(backfillExplicit ? backfillOverride : config.backfillDays, null);
   const allowedChatIds = parseAllowedChatIds(config.allowedChatIds);
   const allChats = config.allChats === true || selectedChatIdsIncludeAll(allowedChatIds);
   if (!allChats && allowedChatIds.length === 0) {
@@ -1502,12 +1596,15 @@ async function runMessagesAgent() {
   const kit = await import("@photon-ai/imessage-kit");
   const sdk = new kit.IMessageSDK({ debug: args.debug === true });
   const sender = new MessagesBatchSender(apiUrl, agentToken, userId);
-  const contactLookup = createMutableContactLookup(buildContactLookup());
+  const contactLookup = createMutableContactLookup(buildContactLookup({ userId }));
   const serializer = createMessageSerializer(kit, contactLookup);
+  const stateCache = createMessageStateCache(userId);
   const contactSync = startMessagesContactSync(sender, contactLookup, {
+    userId,
     syncAllContacts: allChats,
     seedHandles: allChats ? [] : selectedChatContactSeedHandles(config.selectedChats, allowedChatIds),
   });
+  let editDetector = null;
 
   console.log("Shepherd Messages raw sync starting");
   console.log(allChats
@@ -1515,25 +1612,52 @@ async function runMessagesAgent() {
     : `Messages chat filter: ${allowedChatIds.length} selected chat(s)`);
 
   try {
+    await validateMessagesDatabaseAccess(sdk);
+    console.log("Messages local database access validated");
     await contactSync.syncNow({ forceAll: true, reason: "startup" }).catch((err) => {
       console.error("Initial Messages contact sync failed:", safeError(err));
     });
     await loadGroupChatNames(sdk, serializer);
     loadSelectedChatNames(config.selectedChats, serializer);
 
-    if (backfillDays !== 0) {
-      await runMessagesBackfill(sdk, sender, serializer, backfillDays, allChats ? null : allowedChatIds, contactSync);
+    const initialWatermark = loadMessagesWatermark(userId);
+    const backfillScope = messagesBackfillScope({ backfillDays, allChats, allowedChatIds });
+    let backfillComplete = loadMessagesBackfillComplete(userId, backfillScope);
+    if (!backfillComplete && !hasAnyMessagesBackfillComplete(userId) && initialWatermark > 0 && !backfillExplicit && backfillDays !== 0) {
+      backfillComplete = saveMessagesBackfillComplete(userId, backfillScope, {
+        legacyAssumedComplete: true,
+        watermark: initialWatermark,
+      });
+    }
+    if (shouldRunMessagesBackfill({ backfillDays, backfillExplicit, backfillComplete })) {
+      const backfillResult = await runMessagesBackfill(sdk, sender, serializer, backfillDays, allChats ? null : allowedChatIds, contactSync, stateCache);
+      if (backfillResult.complete) {
+        saveMessagesBackfillComplete(userId, backfillScope, backfillResult);
+      } else {
+        console.warn("Messages backfill paused before completion because messages were queued for retry");
+      }
       await contactSync.syncNow({ forceAll: true, reason: "post-backfill" }).catch((err) => {
         console.error("Post-backfill Messages contact sync failed:", safeError(err));
       });
+    } else if (backfillDays !== 0) {
+      console.log(`Skipping configured Messages backfill because this chat scope is already complete; use --backfill-days to force a historical backfill`);
     }
 
-    await gapFillFromWatermark(sdk, sender, serializer, userId, allChats ? null : allowedChatIds, contactSync);
+    await gapFillFromWatermark(sdk, sender, serializer, userId, allChats ? null : allowedChatIds, contactSync, stateCache);
     await contactSync.syncNow({ forceAll: true, reason: "post-gap-fill" }).catch((err) => {
       console.error("Post-gap-fill Messages contact sync failed:", safeError(err));
     });
-    await watchMessages(sdk, sender, serializer, userId, allChats ? null : allowedChatIds, { contactSync });
+    editDetector = startMessagesEditDetector(sdk, sender, serializer, allChats ? null : allowedChatIds, {
+      contactSync,
+      stateCache,
+    });
+    await watchMessages(sdk, sender, serializer, userId, allChats ? null : allowedChatIds, {
+      contactSync,
+      stateCache,
+      onShutdown: () => editDetector?.stop(),
+    });
   } catch (err) {
+    editDetector?.stop();
     contactSync.stop();
     await sdk.close?.().catch(() => undefined);
     throw err;
@@ -1600,6 +1724,7 @@ async function collectMessagesLocalStatus(preferredUserId = null) {
     allChats: config?.allChats === true,
     selectedChatCount: Array.isArray(config?.allowedChatIds) ? config.allowedChatIds.length : 0,
     storage: await probePath("messages", MESSAGES_CHAT_DB_PATH),
+    contacts: userId ? readJsonOptional(messagesContactStatusFile(userId)) : null,
     launch: localLaunchStatus(label),
     queueDepth: Array.isArray(queue) ? queue.length : 0,
   };
@@ -1827,6 +1952,7 @@ Options:
   --api <url>                Advanced: Shepherd API URL.
   --state <path>             Token state file. Defaults to ~/.shepherd/mcp.json.
   --onboarding-state <path>  Local onboarding state file. Defaults to ~/.shepherd/raw-onboarding-agent.json.
+  --mcp-program <json_array> Advanced: MCP proxy command prefix. The app uses this to install app-binary-backed MCP instead of npm.
   --no-local                 Skip local onboarding auth and use WorkOS browser login.
   --install <targets>        Install MCP after login. Use all, none, codex, claude, cursor, or comma-separated targets.
   --no-install               Save the MCP token without installing client config.
@@ -1855,6 +1981,7 @@ Installs the saved Shepherd MCP login into:
 Options:
   --state <path>             Token state file. Defaults to ~/.shepherd/mcp.json.
   --onboarding-state <path>  Local onboarding state file. Defaults to ~/.shepherd/raw-onboarding-agent.json.
+  --mcp-program <json_array> Advanced: MCP proxy command prefix. The app uses this to install app-binary-backed MCP instead of npm.
   --no-local                 Skip local onboarding auth refresh.
   --install <targets>        Use all, none, codex, claude, cursor, or comma-separated targets.
   --no-install               Skip client config writes.
@@ -1864,7 +1991,7 @@ Options:
     return;
   }
 
-  if (which === "write-agent-state" || which === "write-messages-config" || which === "install-messages-agent") {
+  if (which === "write-agent-state" || which === "write-messages-config" || which === "install-messages-agent" || which === "write-coding-sessions-config" || which === "install-coding-sessions-agent") {
     console.log(`Shepherd onboarding engine commands
 
 These non-interactive commands are used by GUI onboarding apps (for example the
@@ -1876,9 +2003,12 @@ Usage:
   shepherd-onboard write-messages-config        JSON object on stdin ({apiUrl, userId, agentToken, backfillDays?, allowedChatIds, selectedChats?}) is written to ~/.shepherd/raw-messages/<userId>.json. Prints {configPath}.
   shepherd-onboard install-messages-agent --config <path>
                                                 Installs and verifies the Messages launchd agent for an existing config. Prints install metadata.
+  shepherd-onboard write-coding-sessions-config JSON object on stdin ({apiUrl, userId, agentToken, intervalSeconds?}) is written to ~/.shepherd/coding-sessions/<userId>.json. Prints {configPath}.
+  shepherd-onboard install-coding-sessions-agent --config <path>
+                                                Installs and verifies the Coding Sessions launchd agent for an existing config. Prints install metadata.
 
-install-messages-agent options:
-  --config <path>            Messages agent config created by onboarding. Required.
+install agent options:
+  --config <path>            Agent config created by onboarding. Required.
   --user-id <id>             Override the user ID. Defaults to the config's userId.
   --program <json_array>     Replace the default npx launcher with custom ProgramArguments (e.g. a signed app binary). --config <path> is appended.
   --env <json_object>        Extra EnvironmentVariables merged into the launchd plist (e.g. ELECTRON_RUN_AS_NODE).
@@ -2064,7 +2194,7 @@ function printAgentContract() {
     agentStatusCommand: `${command} agent --status`,
     messagesChatsCommand: `${command} messages-chats`,
     messagesPermissions: {
-      macOS: "Local Messages raw sync needs Full Disk Access for the app running onboarding and for Node.js used by the background LaunchAgent. The Messages selector command validates local chat.db access, opens Full Disk Access settings if needed, and keeps checking until access works in interactive onboarding. Background sync install also checks that launchd can start the Messages agent. Contacts permission may also appear when resolving local contact names. The background Messages agent reloads Contacts on startup, watches AddressBook changes when available, and runs fallback contact sync so renamed contacts can hydrate prior ingested Messages rows for the token-bound customer account.",
+      macOS: "Local Messages raw sync needs Full Disk Access for the app running onboarding and for Node.js used by the background LaunchAgent, so launchd can start the Messages agent after onboarding. The Messages selector command validates local chat.db access, opens Full Disk Access settings if needed, and keeps checking until access works in interactive onboarding. Contacts permission is also needed for contact-name hydration; Shepherd reads contact names from the local macOS AddressBook database, reloads Contacts on startup, watches AddressBook changes, and scopes synced messages to the token-bound customer account. If that schema cannot be read, raw message sync still works but contact names are marked degraded and the user should contact founders@askshepherd.ai.",
       nodeBinary: process.execPath,
     },
     codingSessions: {
@@ -2133,7 +2263,7 @@ If Messages is selected, run:
 
 Before or during this step, ask the user to grant or confirm macOS Full Disk Access for local Messages sync. The command validates access to the local Messages database, opens System Settings -> Privacy & Security -> Full Disk Access if access is missing, and keeps checking until access works in interactive onboarding. The user should enable the app running onboarding, such as Terminal, iTerm, Claude Code, or Codex, and Node.js for background sync:
   ${payload.messagesPermissions.nodeBinary}
-Contacts permission may also appear when Shepherd resolves local contact names. The background Messages agent reloads Contacts on startup, watches AddressBook changes when available, and runs fallback contact sync so renamed contacts can hydrate prior ingested Messages rows for the token-bound customer account.
+Shepherd reads contact names from the local macOS AddressBook database. If that database schema cannot be read, raw Messages sync still works, but contact names will be marked degraded and the user should contact founders@askshepherd.ai.
 
 This opens a minimal local webpage with recent local Messages chats and search. Have the user select which contacts/groups Shepherd should sync. Do not select all chats by default. If the user explicitly wants everything, use the "Sync all current and future chats" checkbox or pass --messages-chat-ids all. All-chats mode backfills current chats and keeps watching chats that appear later. When the command returns, keep the printed chat IDs or the literal value all.
 
@@ -2230,6 +2360,7 @@ function sourceSelectionFromList(value) {
   const selected = {
     google: false,
     slack: false,
+    github: false,
     granola: false,
     messages: false,
     codingSessions: false,
@@ -2242,6 +2373,7 @@ function sourceSelectionFromList(value) {
     ["gdocs", "google"],
     ["calendar", "google"],
     ["slack", "slack"],
+    ["github", "github"],
     ["granola", "granola"],
     ["messages", "messages"],
     ["imessage", "messages"],
@@ -2261,6 +2393,7 @@ function sourceSelectionFromList(value) {
     if (part === "all") {
       selected.google = true;
       selected.slack = true;
+      selected.github = true;
       selected.granola = true;
       selected.messages = true;
       selected.codingSessions = true;
@@ -2268,7 +2401,7 @@ function sourceSelectionFromList(value) {
     }
     const source = aliases.get(part);
     if (!source) {
-      throw new Error(`Unknown source "${part}". Use google, slack, granola, messages, coding-sessions, or all.`);
+      throw new Error(`Unknown source "${part}". Use google, slack, github, granola, messages, coding-sessions, or all.`);
     }
     selected[source] = true;
   }
@@ -2324,7 +2457,9 @@ async function updateAgentStateFromOnboardingResponse(state, response) {
   const hasStatus = typeof response?.status === "string";
   const hasProcessing = typeof response?.processingEnabled === "boolean" || response?.processing;
   const hasProviders = response?.providers && typeof response.providers === "object" && !Array.isArray(response.providers);
-  if (!hasAuthUrls && !hasGoogleWorkspaceDelegation && !hasStatus && !hasProcessing && !hasProviders) return state;
+  const responseSources = sourceSelectionFromSession(response, null);
+  const hasSources = responseSources !== null;
+  if (!hasAuthUrls && !hasGoogleWorkspaceDelegation && !hasStatus && !hasProcessing && !hasProviders && !hasSources) return state;
 
   const next = {
     ...state,
@@ -2336,6 +2471,7 @@ async function updateAgentStateFromOnboardingResponse(state, response) {
     ...(typeof response?.processingEnabled === "boolean" ? { processingEnabled: response.processingEnabled } : {}),
     ...(response?.processing ? { processing: response.processing } : {}),
     ...(hasProviders ? { providers: response.providers } : {}),
+    ...(hasSources ? { sources: responseSources } : {}),
   };
   await writeAgentState(next);
   return next;
@@ -2442,6 +2578,20 @@ async function openNextAgentModality({ sources, authUrls = {}, noOpen = false, p
       return { source, label: "Slack", opened: !noOpen, url };
     }
 
+    if (source === "github") {
+      const url = typeof authUrls.github === "string" ? authUrls.github : null;
+      if (!url) {
+        return {
+          source,
+          label: "GitHub",
+          opened: false,
+          message: "GitHub authorization URL was not returned by Shepherd.",
+        };
+      }
+      await openOrPrint(url, { noOpen });
+      return { source, label: "GitHub", opened: !noOpen, url };
+    }
+
     if (source === "granola") {
       const result = await openGranolaApiKeys({ noOpen: noOpen || Boolean(args["no-open-granola"]) });
       return { source, label: "Granola", ...result };
@@ -2503,6 +2653,18 @@ function printAgentCurrentAction(action, opts = {}) {
     return;
   }
 
+  if (action.source === "github") {
+    if (action.opened) {
+      console.log("Opened GitHub authorization in the browser.");
+    } else if (action.url) {
+      console.log(`GitHub authorization URL: ${action.url}`);
+    } else if (action.message) {
+      console.log(action.message);
+    }
+    console.log("Ask the user to complete GitHub authorization before opening another source.");
+    return;
+  }
+
   if (action.source === "granola") {
     if (action.target) console.log(`Granola target: ${action.target}`);
     console.log("Ask the user to create/copy the Granola API key before opening another source.");
@@ -2526,6 +2688,7 @@ function agentNeedsUserAction(sources, action) {
   if (!action) return [];
   if (action.source === "google") return ["Have the customer's Google Workspace super admin authorize Shepherd's domain-wide delegation Client ID and scopes in Google Admin Console."];
   if (action.source === "slack") return ["Complete Slack browser authorization."];
+  if (action.source === "github") return ["Complete GitHub browser authorization."];
   if (action.source === "granola") return ["Create/copy a Granola API key from the Granola Mac app."];
   if (action.source === "messages") return ["Grant or confirm macOS Full Disk Access for the onboarding app and Node.js, run messages-chats, have the user select local Messages contacts/groups in the browser, then pass the printed chat IDs with the Messages handle."];
   if (action.source === "codingSessions") return ["Run the continue command to install local Codex and Claude Code session summary sync."];
@@ -2683,7 +2846,7 @@ async function explainMessagesBackgroundPermissions(opts = {}) {
   console.log("\nMessages background sync permissions");
   console.log("Local Messages raw sync runs as a macOS LaunchAgent using npx/Node.js. For continuous sync, macOS Full Disk Access must include the background Node.js binary, not just the current terminal.");
   printMessagesPermissionTargets();
-  console.log("Contacts permission may also appear when Shepherd resolves local contact names. The background agent keeps contact names hydrated for observed Messages conversations.");
+  console.log("Shepherd reads contact names from the local macOS AddressBook database. If that schema cannot be read, raw Messages sync still works but contact names are marked degraded; contact founders@askshepherd.ai.");
   await openFullDiskAccessSettings(opts);
 
   if (opts.waitForUser && process.stdin.isTTY && !args["no-permission-prompt"]) {
@@ -2809,6 +2972,7 @@ async function writeMessagesConfig(input) {
       apiUrl: input.apiUrl,
       userId: input.userId,
       agentToken: input.agentToken,
+      ...(input.handle ? { handle: input.handle } : {}),
       backfillDays: input.backfillDays,
       allChats,
       allowedChatIds: allChats ? [] : allowedChatIds,
@@ -2939,22 +3103,29 @@ async function writeCodingSessionsConfig(input) {
   return path;
 }
 
-async function installCodingSessionsAgent(configPath, userId) {
-  if (platform() !== "darwin") {
-    throw new Error("automatic local coding-session sync is only supported on macOS");
-  }
-
+function buildCodingSessionsAgentInstall(configPath, userId, overrides = {}) {
   const safeId = safeFileId(userId);
   const label = `ai.shepherd.coding-sessions.${safeId}`;
   const rawDir = join(homedir(), ".shepherd", "coding-sessions");
   const agentsDir = join(homedir(), "Library", "LaunchAgents");
-  await mkdir(rawDir, { recursive: true });
-  await mkdir(agentsDir, { recursive: true });
-
   const plistPath = join(agentsDir, `${label}.plist`);
   const stdoutPath = join(rawDir, `${safeId}.out.log`);
   const stderrPath = join(rawDir, `${safeId}.err.log`);
-  const launchPath = launchAgentPath();
+  const programPrefix = Array.isArray(overrides.programArguments) && overrides.programArguments.length > 0
+    ? overrides.programArguments
+    : ["/usr/bin/env", "npx", "-y", PACKAGE_SPEC, "coding-sessions-agent"];
+  const programArguments = [...programPrefix, "--config", configPath];
+  const environment = {
+    PATH: launchAgentPath(),
+    ...stringRecord(overrides.environment),
+  };
+
+  const programArgumentsXml = programArguments
+    .map((value) => `    <string>${xmlEscape(value)}</string>`)
+    .join("\n");
+  const environmentXml = Object.entries(environment)
+    .map(([key, value]) => `    <key>${xmlEscape(key)}</key>\n    <string>${xmlEscape(value)}</string>`)
+    .join("\n");
 
   const plist = `<?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
@@ -2964,39 +3135,50 @@ async function installCodingSessionsAgent(configPath, userId) {
   <string>${xmlEscape(label)}</string>
   <key>ProgramArguments</key>
   <array>
-    <string>/usr/bin/env</string>
-    <string>npx</string>
-    <string>-y</string>
-    <string>${PACKAGE_SPEC}</string>
-    <string>coding-sessions-agent</string>
-    <string>--config</string>
-    <string>${xmlEscape(configPath)}</string>
+${programArgumentsXml}
   </array>
   <key>KeepAlive</key>
   <true/>
   <key>RunAtLoad</key>
   <true/>
+  <key>ThrottleInterval</key>
+  <integer>10</integer>
+  <key>WorkingDirectory</key>
+  <string>${xmlEscape(rawDir)}</string>
   <key>StandardOutPath</key>
   <string>${xmlEscape(stdoutPath)}</string>
   <key>StandardErrorPath</key>
   <string>${xmlEscape(stderrPath)}</string>
   <key>EnvironmentVariables</key>
   <dict>
-    <key>PATH</key>
-    <string>${xmlEscape(launchPath)}</string>
+${environmentXml}
   </dict>
 </dict>
 </plist>
 `;
 
+  return { label, rawDir, agentsDir, plistPath, stdoutPath, stderrPath, programArguments, environment, plist };
+}
+
+async function installCodingSessionsAgent(configPath, userId, overrides = {}) {
+  if (platform() !== "darwin") {
+    throw new Error("automatic local coding-session sync is only supported on macOS");
+  }
+
+  const install = buildCodingSessionsAgentInstall(configPath, userId, overrides);
+  const { label, rawDir, agentsDir, plistPath, stdoutPath, stderrPath, plist } = install;
+  await mkdir(rawDir, { recursive: true });
+  await mkdir(agentsDir, { recursive: true });
+
   await writeFile(plistPath, plist, { mode: 0o600 });
+  await chmod(plistPath, 0o600);
   const stdoutOffset = await fileLength(stdoutPath);
   const stderrOffset = await fileLength(stderrPath);
   await execFileQuiet("launchctl", ["unload", plistPath], { ignoreError: true });
   await execFileQuiet("launchctl", ["load", plistPath]);
   await execFileQuiet("launchctl", ["start", label], { ignoreError: true });
   await verifyCodingSessionsAgentLaunch({ label, stdoutPath, stderrPath, stdoutOffset, stderrOffset });
-  return { label, plistPath, stdoutPath, stderrPath };
+  return install;
 }
 
 async function verifyCodingSessionsAgentLaunch({ label, stdoutPath, stderrPath, stdoutOffset, stderrOffset }) {
@@ -3035,7 +3217,7 @@ async function verifyMessagesAgentLaunch({ label, stdoutPath, stderrPath, stdout
       throw new Error("Messages background sync could not open local Messages storage. Grant macOS Full Disk Access to the app running onboarding and Node.js, then rerun or continue the Messages step.");
     }
 
-    if (/Watching for new Messages|Shepherd Messages raw sync starting|Running .*Messages backfill/i.test(stdout)
+    if (/Messages local database access validated|Watching for new Messages/i.test(stdout)
       && /state = running|job state = running/i.test(launchState)) {
       return;
     }
@@ -3958,7 +4140,27 @@ function selectedChatContactSeedHandles(selectedChats, allowedChatIds) {
   return [...new Set(handles)];
 }
 
-async function runMessagesBackfill(sdk, sender, serializer, days, allowedChatIds, contactSync = null) {
+async function validateMessagesDatabaseAccess(sdk) {
+  await sdk.getMessages({ limit: 1 });
+}
+
+function messagesBackfillScope({ backfillDays, allChats, allowedChatIds }) {
+  return {
+    version: 1,
+    backfillDays: backfillDays == null ? "all" : backfillDays,
+    scope: allChats
+      ? { allChats: true }
+      : { chatIds: [...allowedChatIds].sort() },
+  };
+}
+
+function shouldRunMessagesBackfill({ backfillDays, backfillExplicit, backfillComplete }) {
+  if (backfillDays === 0) return false;
+  if (backfillExplicit) return true;
+  return !backfillComplete;
+}
+
+async function runMessagesBackfill(sdk, sender, serializer, days, allowedChatIds, contactSync = null, stateCache = null) {
   const allChats = allowedChatIds == null;
   console.log(allChats
     ? `Running ${days == null ? "all-history" : `${days}-day`} Messages backfill for all current chats`
@@ -3979,6 +4181,8 @@ async function runMessagesBackfill(sdk, sender, serializer, days, allowedChatIds
       totalMessages += filtered.length;
       const result = await sender.send(filtered.map((msg) => serializer.serialize(msg)));
       totalStored += result.stored;
+      if ((result.queued ?? 0) > 0) return { complete: false, totalMessages, totalStored, queued: result.queued };
+      stateCache?.observe(filtered);
       saveMessagesWatermark(sender.userId, maxRowId(messages));
 
       if (messages.length < pageSize) break;
@@ -3986,7 +4190,7 @@ async function runMessagesBackfill(sdk, sender, serializer, days, allowedChatIds
     }
 
     console.log(`Messages backfill complete: stored ${totalStored} of ${totalMessages}`);
-    return;
+    return { complete: true, totalMessages, totalStored, queued: 0, watermark: loadMessagesWatermark(sender.userId) };
   }
 
   for (const chatId of allowedChatIds) {
@@ -4000,6 +4204,8 @@ async function runMessagesBackfill(sdk, sender, serializer, days, allowedChatIds
       totalMessages += filtered.length;
       const result = await sender.send(filtered.map((msg) => serializer.serialize(msg)));
       totalStored += result.stored;
+      if ((result.queued ?? 0) > 0) return { complete: false, totalMessages, totalStored, queued: result.queued };
+      stateCache?.observe(filtered);
       saveMessagesWatermark(sender.userId, maxRowId(messages));
 
       if (messages.length < pageSize) break;
@@ -4008,19 +4214,20 @@ async function runMessagesBackfill(sdk, sender, serializer, days, allowedChatIds
   }
 
   console.log(`Messages backfill complete: stored ${totalStored} of ${totalMessages}`);
+  return { complete: true, totalMessages, totalStored, queued: 0, watermark: loadMessagesWatermark(sender.userId) };
 }
 
-async function gapFillFromWatermark(sdk, sender, serializer, userId, allowedChatIds, contactSync = null) {
+async function gapFillFromWatermark(sdk, sender, serializer, userId, allowedChatIds, contactSync = null, stateCache = null) {
   const allChats = allowedChatIds == null;
   const lastWatermark = loadMessagesWatermark(userId);
   if (lastWatermark <= 0) return;
 
-  const missed = [];
+  let missed = [];
   if (allChats) {
-    missed.push(...await sdk.getMessages({ limit: 5000 }));
+    missed = await getMessagesAfterWatermark(sdk, { lastWatermark, pageSize: 1000 });
   } else {
     for (const chatId of allowedChatIds) {
-      missed.push(...await sdk.getMessages({ chatId, limit: 1000 }));
+      missed.push(...await getMessagesAfterWatermark(sdk, { chatId, lastWatermark, pageSize: 1000 }));
     }
   }
   const newMessages = missed.filter((msg) =>
@@ -4029,16 +4236,405 @@ async function gapFillFromWatermark(sdk, sender, serializer, userId, allowedChat
     && !messageTouchesShepherdAgent(msg));
   if (newMessages.length === 0) return;
 
+  newMessages.sort((a, b) => Number(a.rowId ?? 0) - Number(b.rowId ?? 0));
   contactSync?.observeMessages(newMessages);
   const result = await sender.send(newMessages.map((msg) => serializer.serialize(msg)));
-  if (result.stored > 0) saveMessagesWatermark(userId, maxRowId(newMessages));
+  if ((result.queued ?? 0) === 0) {
+    stateCache?.observe(newMessages);
+    saveMessagesWatermark(userId, maxRowId(newMessages));
+  }
   console.log(`Messages gap-fill complete: stored ${result.stored} of ${newMessages.length}`);
 }
 
+async function getMessagesAfterWatermark(sdk, { chatId = null, lastWatermark, pageSize }) {
+  const messages = [];
+  let offset = 0;
+  while (true) {
+    const page = await sdk.getMessages({ ...(chatId ? { chatId } : {}), limit: pageSize, offset });
+    if (!page.length) break;
+    messages.push(...page.filter((msg) => Number(msg.rowId) > lastWatermark));
+    if (page.length < pageSize) break;
+    offset += pageSize;
+  }
+  return messages;
+}
+
+function createMessageStateCache(userId, maxSize = MESSAGES_STATE_CACHE_MAX) {
+  const stateFile = messagesStateFile(userId);
+  const initial = readJsonOptional(stateFile);
+  const cache = new Map(Object.entries(
+    initial && typeof initial === "object" && !Array.isArray(initial) ? initial : {},
+  ));
+
+  const remember = (msg) => {
+    const key = messageIdentity(msg);
+    if (!key) return;
+    cache.set(key, messageState(msg));
+    trimMessageStateCache(cache, maxSize);
+  };
+
+  const persist = () => saveMessagesState(userId, Object.fromEntries(cache));
+
+  return {
+    observe(messages) {
+      for (const msg of messages ?? []) remember(msg);
+      persist();
+    },
+    changed(messages) {
+      const changed = [];
+      for (const msg of messages ?? []) {
+        const key = messageIdentity(msg);
+        if (!key) continue;
+        const current = messageState(msg);
+        const previous = cache.get(key);
+        if ((previous && !sameMessageState(previous, current))
+          || (!previous && hasMessageEditOrRetraction(msg))) {
+          changed.push(msg);
+        }
+      }
+      return changed;
+    },
+  };
+}
+
+function trimMessageStateCache(cache, maxSize) {
+  if (cache.size <= maxSize) return;
+  const excess = cache.size - maxSize;
+  for (const key of [...cache.keys()].slice(0, excess)) cache.delete(key);
+}
+
+function hasMessageEditOrRetraction(msg) {
+  return Boolean(msg?.editedAt || msg?.retractedAt);
+}
+
+function messageIdentity(msg) {
+  const value = msg?.id ?? msg?.messageId ?? msg?.rowId;
+  return value == null ? null : String(value);
+}
+
+function messageState(msg) {
+  const participant = messageParticipant(msg);
+  return {
+    text: msg?.text ?? null,
+    chatId: msg?.chatId ?? null,
+    chatKind: messageChatKind(msg),
+    participant,
+    isFromMe: Boolean(msg?.isFromMe),
+    editedAt: isoDate(msg?.editedAt),
+    retractedAt: isoDate(msg?.retractedAt),
+    replyToMessageId: msg?.replyToMessageId ?? null,
+    threadRootMessageId: msg?.threadRootMessageId ?? null,
+    affectedParticipant: msg?.affectedParticipant ?? null,
+    kind: msg?.kind ?? null,
+    hasAttachments: Boolean(msg?.hasAttachments ?? (Array.isArray(msg?.attachments) && msg.attachments.length > 0)),
+    attachments: messageAttachmentState(msg),
+  };
+}
+
+function sameMessageState(a, b) {
+  return a.text === b.text
+    && a.chatId === b.chatId
+    && a.chatKind === b.chatKind
+    && a.participant === b.participant
+    && a.isFromMe === b.isFromMe
+    && a.editedAt === b.editedAt
+    && a.retractedAt === b.retractedAt
+    && a.replyToMessageId === b.replyToMessageId
+    && a.threadRootMessageId === b.threadRootMessageId
+    && a.affectedParticipant === b.affectedParticipant
+    && a.kind === b.kind
+    && a.hasAttachments === b.hasAttachments
+    && stableLocalJson(a.attachments) === stableLocalJson(b.attachments);
+}
+
+function messageAttachmentState(msg) {
+  return (Array.isArray(msg?.attachments) ? msg.attachments : [])
+    .map((att) => ({
+      id: String(att?.id ?? ""),
+      fileName: att?.fileName ?? null,
+      mimeType: att?.mimeType ?? null,
+      sizeBytes: Number(att?.sizeBytes ?? 0),
+      transferStatus: att?.transferStatus ?? null,
+      isSticker: Boolean(att?.isSticker),
+      isSensitiveContent: Boolean(att?.isSensitiveContent),
+    }))
+    .sort((a, b) => a.id.localeCompare(b.id) || String(a.fileName ?? "").localeCompare(String(b.fileName ?? "")));
+}
+
+function stableLocalJson(value) {
+  if (Array.isArray(value)) return `[${value.map(stableLocalJson).join(",")}]`;
+  if (value && typeof value === "object") {
+    return `{${Object.entries(value)
+      .filter(([, entryValue]) => entryValue !== undefined)
+      .sort(([a], [b]) => a.localeCompare(b))
+      .map(([key, entryValue]) => `${JSON.stringify(key)}:${stableLocalJson(entryValue)}`)
+      .join(",")}}`;
+  }
+  return JSON.stringify(value);
+}
+
+function startMessagesEditDetector(sdk, sender, serializer, allowedChatIds, opts = {}) {
+  const contactSync = opts.contactSync ?? null;
+  const stateCache = opts.stateCache ?? createMessageStateCache(sender.userId);
+  const watchers = [];
+  let stopped = false;
+  let timer = null;
+  let scanning = false;
+  let lastMutationReconcileAt = 0;
+
+  const scan = async (reason) => {
+    if (stopped || scanning) return;
+    scanning = true;
+    try {
+      const recentResult = await scanMessagesForEditsAndUnsends(sdk, sender, serializer, allowedChatIds, {
+        contactSync,
+        stateCache,
+      });
+      const shouldReconcileMutations = reason === "startup"
+        || Date.now() - lastMutationReconcileAt >= MESSAGES_MUTATION_RECONCILE_INTERVAL_MS;
+      const mutationResult = shouldReconcileMutations
+        ? await reconcileMessagesMutations(sdk, sender, serializer, allowedChatIds, {
+            contactSync,
+            stateCache,
+          })
+        : { changed: 0, stored: 0, updated: 0, queued: 0 };
+      if (shouldReconcileMutations) lastMutationReconcileAt = Date.now();
+
+      const result = {
+        changed: recentResult.changed + mutationResult.changed,
+        stored: recentResult.stored + mutationResult.stored,
+        updated: recentResult.updated + mutationResult.updated,
+        queued: recentResult.queued + mutationResult.queued,
+      };
+      if (result.changed > 0) {
+        console.log(`Messages edit detector: ${result.changed} edit/unsend change(s) sent (${reason}); stored ${result.stored}, updated ${result.updated}, queued ${result.queued}`);
+      }
+    } catch (err) {
+      console.error("Messages edit detector failed:", safeError(err));
+    } finally {
+      scanning = false;
+    }
+  };
+
+  const schedule = (reason) => {
+    if (stopped) return;
+    if (timer) clearTimeout(timer);
+    timer = setTimeout(() => {
+      timer = null;
+      scan(reason).catch((err) => console.error("Messages edit detector failed:", safeError(err)));
+    }, 500);
+  };
+
+  for (const walPath of messagesWatchPaths()) {
+    try {
+      watchers.push(watch(walPath, () => {
+        schedule("messages-db");
+      }));
+    } catch (err) {
+      console.warn(`Could not watch Messages database file ${walPath}: ${safeError(err)}`);
+    }
+  }
+
+  if (watchers.length > 0) {
+    console.log(`Watching ${watchers.length} Messages database file(s) for edits and unsends`);
+  } else if (platform() === "darwin") {
+    console.warn("Messages database watch files not found; edit/unsend detection will use fallback polling only");
+  }
+
+  const interval = setInterval(() => {
+    scan("fallback").catch((err) => console.error("Messages edit detector failed:", safeError(err)));
+  }, MESSAGES_EDIT_SCAN_INTERVAL_MS);
+  scan("startup").catch((err) => console.error("Messages edit detector failed:", safeError(err)));
+
+  return {
+    stop() {
+      stopped = true;
+      if (timer) clearTimeout(timer);
+      clearInterval(interval);
+      for (const watcher of watchers) watcher.close();
+    },
+  };
+}
+
+function messagesWatchPaths() {
+  if (platform() !== "darwin") return [];
+  return [
+    MESSAGES_CHAT_DB_PATH,
+    `${MESSAGES_CHAT_DB_PATH}-wal`,
+    `${MESSAGES_CHAT_DB_PATH}-shm`,
+  ].filter((path) => existsSync(path));
+}
+
+async function scanMessagesForEditsAndUnsends(sdk, sender, serializer, allowedChatIds, opts = {}) {
+  const allChats = allowedChatIds == null;
+  const since = new Date(Date.now() - MESSAGES_EDIT_SCAN_WINDOW_MS);
+  const messages = await getScopedMessages(sdk, allowedChatIds, { since });
+
+  const filtered = messages.filter((msg) =>
+    msg?.chatId
+    && (allChats || allowedChatIds.includes(msg.chatId))
+    && !messageTouchesShepherdAgent(msg));
+  opts.contactSync?.observeMessages(filtered);
+
+  const changed = opts.stateCache?.changed(filtered) ?? [];
+  if (changed.length === 0) {
+    opts.stateCache?.observe(filtered);
+    return { changed: 0, stored: 0, updated: 0, queued: 0 };
+  }
+
+  const result = await sender.send(changed.map((msg) => serializer.serialize(msg)));
+  if ((result.queued ?? 0) === 0) opts.stateCache?.observe(filtered);
+  return {
+    changed: changed.length,
+    stored: result.stored ?? 0,
+    updated: result.updated ?? 0,
+    queued: result.queued ?? 0,
+  };
+}
+
+async function getScopedMessages(sdk, allowedChatIds, query) {
+  const messages = [];
+  if (allowedChatIds == null) {
+    messages.push(...await getPagedMessages(sdk, query));
+    return messages;
+  }
+
+  for (const chatId of allowedChatIds) {
+    messages.push(...await getPagedMessages(sdk, { ...query, chatId }));
+  }
+  return messages;
+}
+
+async function getPagedMessages(sdk, query) {
+  const messages = [];
+  let offset = 0;
+  while (true) {
+    const page = await sdk.getMessages({ ...query, limit: MESSAGES_EDIT_SCAN_LIMIT, offset });
+    if (!page.length) break;
+    messages.push(...page);
+    if (page.length < MESSAGES_EDIT_SCAN_LIMIT) break;
+    offset += MESSAGES_EDIT_SCAN_LIMIT;
+  }
+  return messages;
+}
+
+async function reconcileMessagesMutations(sdk, sender, serializer, allowedChatIds, opts = {}) {
+  const allChats = allowedChatIds == null;
+  const rowIds = queryMessagesMutationRowIds(sdk, allowedChatIds);
+  if (rowIds.length === 0) return { changed: 0, stored: 0, updated: 0, queued: 0 };
+
+  const messages = await getMessagesByRowIds(sdk, rowIds);
+  const filtered = messages.filter((msg) =>
+    msg?.chatId
+    && (allChats || allowedChatIds.includes(msg.chatId))
+    && !messageTouchesShepherdAgent(msg));
+  opts.contactSync?.observeMessages(filtered);
+
+  const changed = opts.stateCache?.changed(filtered) ?? filtered.filter(hasMessageEditOrRetraction);
+  if (changed.length === 0) {
+    opts.stateCache?.observe(filtered);
+    return { changed: 0, stored: 0, updated: 0, queued: 0 };
+  }
+
+  const result = await sender.send(changed.map((msg) => serializer.serialize(msg)));
+  if ((result.queued ?? 0) === 0) opts.stateCache?.observe(filtered);
+  return {
+    changed: changed.length,
+    stored: result.stored ?? 0,
+    updated: result.updated ?? 0,
+    queued: result.queued ?? 0,
+  };
+}
+
+function queryMessagesMutationRowIds(sdk, allowedChatIds = null) {
+  const db = sdk?.database;
+  if (!db || typeof db.all !== "function") {
+    console.warn("Messages mutation reconciliation unavailable: SDK database reader is not exposed");
+    return [];
+  }
+
+  const columns = messageTableColumns(db);
+  const predicates = [];
+  if (columns.has("date_edited")) predicates.push("COALESCE(message.date_edited, 0) != 0");
+  if (columns.has("date_retracted")) predicates.push("COALESCE(message.date_retracted, 0) != 0");
+  if (!predicates.length) return [];
+
+  const params = [];
+  let chatJoin = "";
+  let chatPredicate = "";
+  if (allowedChatIds != null) {
+    const values = uniqueStrings(allowedChatIds.flatMap(chatIdMatchValues));
+    if (values.length === 0) return [];
+    const placeholders = values.map(() => "?").join(", ");
+    chatJoin = `
+      INNER JOIN chat_message_join ON chat_message_join.message_id = message.ROWID
+      INNER JOIN chat ON chat.ROWID = chat_message_join.chat_id
+    `;
+    chatPredicate = `AND (chat.chat_identifier IN (${placeholders}) OR chat.guid IN (${placeholders}))`;
+    params.push(...values, ...values);
+  }
+
+  try {
+    const rows = db.all(`
+      SELECT DISTINCT message.ROWID AS rowId
+      FROM message
+      ${chatJoin}
+      WHERE (${predicates.join(" OR ")})
+      ${chatPredicate}
+      ORDER BY message.ROWID ASC
+    `, params);
+    return rows
+      .map((row) => Number(row.rowId))
+      .filter((rowId) => Number.isFinite(rowId) && rowId > 0);
+  } catch (err) {
+    console.warn(`Messages mutation reconciliation query failed: ${safeError(err)}`);
+    return [];
+  }
+}
+
+function messageTableColumns(db) {
+  try {
+    return new Set(db.all("PRAGMA table_info(message)").map((row) => String(row.name)));
+  } catch (err) {
+    console.warn(`Messages mutation reconciliation unavailable: ${safeError(err)}`);
+    return new Set();
+  }
+}
+
+function chatIdMatchValues(chatId) {
+  const raw = String(chatId ?? "").trim();
+  if (!raw) return [];
+  const core = raw.includes(";+;")
+    ? raw.slice(raw.indexOf(";+;") + 3)
+    : raw.includes(";-;")
+      ? raw.slice(raw.indexOf(";-;") + 3)
+      : raw;
+  return raw === core ? [raw] : [raw, core];
+}
+
+function uniqueStrings(values) {
+  return [...new Set(values.map((value) => String(value ?? "").trim()).filter(Boolean))];
+}
+
+async function getMessagesByRowIds(sdk, rowIds) {
+  const messages = [];
+  for (const rowId of rowIds) {
+    const page = await sdk.getMessages({
+      sinceRowId: rowId - 1,
+      orderByRowIdAsc: true,
+      limit: 1,
+    });
+    const msg = page.find((candidate) => Number(candidate.rowId) === rowId);
+    if (msg) messages.push(msg);
+  }
+  return messages;
+}
+
 async function watchMessages(sdk, sender, serializer, userId, allowedChatIds, opts = {}) {
   const allChats = allowedChatIds == null;
   const allowed = new Set(allowedChatIds ?? []);
   const contactSync = opts.contactSync ?? null;
+  const stateCache = opts.stateCache ?? null;
   let buffer = [];
   let timer = null;
 
@@ -4046,7 +4642,10 @@ async function watchMessages(sdk, sender, serializer, userId, allowedChatIds, op
     if (!buffer.length) return;
     const batch = buffer.splice(0, MAX_BATCH_SIZE);
     const result = await sender.send(batch.map((msg) => serializer.serialize(msg)));
-    if (result.stored > 0) saveMessagesWatermark(userId, maxRowId(batch));
+    if ((result.queued ?? 0) === 0) {
+      stateCache?.observe(batch);
+      saveMessagesWatermark(userId, maxRowId(batch));
+    }
   };
 
   const scheduleFlush = () => {
@@ -4088,6 +4687,7 @@ async function watchMessages(sdk, sender, serializer, userId, allowedChatIds, op
       stopping = true;
       if (timer) clearTimeout(timer);
       contactSync?.stop();
+      opts.onShutdown?.();
       await flush().catch(() => undefined);
       await sdk.close?.().catch(() => undefined);
       resolve();
@@ -4144,7 +4744,7 @@ function startMessagesContactSync(sender, contactLookup, opts = {}) {
 
     syncing = true;
     try {
-      const nextLookup = buildContactLookup();
+      const nextLookup = buildContactLookup({ userId: opts.userId });
       contactLookup.replace(nextLookup);
       const mappings = visibleMappings();
       const toSync = forceAll
@@ -4180,7 +4780,7 @@ function startMessagesContactSync(sender, contactLookup, opts = {}) {
   const observeMessages = (messages) => {
     let changed = false;
     for (const msg of messages ?? []) {
-      if (rememberObservedHandle(observedHandles, contactLookup, msg?.participant)) changed = true;
+      if (rememberObservedHandle(observedHandles, contactLookup, messageParticipant(msg))) changed = true;
       if (msg?.affectedParticipant) {
         if (rememberObservedHandle(observedHandles, contactLookup, msg.affectedParticipant)) changed = true;
       }
@@ -4265,6 +4865,8 @@ function createMessageSerializer(kit, contactLookup = emptyContactLookup()) {
     },
     serialize(msg) {
       const chatId = msg.chatId ?? "unknown";
+      const participant = messageParticipant(msg);
+      const chatKind = messageChatKind(msg);
       const attachments = Array.isArray(msg.attachments) ? msg.attachments : [];
       return {
         messageId: String(msg.id ?? msg.messageId ?? msg.rowId),
@@ -4272,9 +4874,9 @@ function createMessageSerializer(kit, contactLookup = emptyContactLookup()) {
         text: msg.text ?? null,
         service: msg.service ?? "iMessage",
         chatId,
-        chatKind: msg.chatKind ?? "unknown",
+        chatKind,
         chatName: chatNames.get(chatId) ?? null,
-        participant: msg.participant ?? null,
+        participant,
         isFromMe: Boolean(msg.isFromMe),
         createdAt: isoDate(msg.createdAt) ?? new Date().toISOString(),
         deliveredAt: isoDate(msg.deliveredAt),
@@ -4287,14 +4889,22 @@ function createMessageSerializer(kit, contactLookup = emptyContactLookup()) {
               targetMessageId: msg.reaction.targetMessageId ?? null,
               emoji: msg.reaction.emoji ?? null,
               isRemoved: Boolean(msg.reaction.isRemoved),
+              textRange: msg.reaction.textRange ?? null,
+              appBundleId: msg.reaction.appBundleId ?? null,
             }
           : null,
+        hasAttachments: Boolean(msg.hasAttachments ?? attachments.length > 0),
         attachments: attachments.map((att) => ({
           id: String(att.id ?? ""),
           fileName: att.fileName ?? null,
           mimeType: att.mimeType ?? "application/octet-stream",
+          uti: att.uti ?? null,
           sizeBytes: Number(att.sizeBytes ?? 0),
           transferStatus: att.transferStatus ?? "unknown",
+          createdAt: isoDate(att.createdAt),
+          altText: att.altText ?? null,
+          isFromMe: typeof att.isFromMe === "boolean" ? att.isFromMe : null,
+          isSensitiveContent: Boolean(att.isSensitiveContent),
           isSticker: Boolean(att.isSticker),
           isImage: isImageAttachment(att),
           isVideo: isVideoAttachment(att),
@@ -4308,16 +4918,47 @@ function createMessageSerializer(kit, contactLookup = emptyContactLookup()) {
         isForwarded: Boolean(msg.isForwarded),
         affectedParticipant: msg.affectedParticipant ?? null,
         newGroupName: msg.newGroupName ?? null,
-        _resolved_name: msg.participant ? contactLookup.resolveName(msg.participant) : null,
-        _is_self_handle: msg.participant ? contactLookup.isSelfHandle(msg.participant) : false,
+        _resolved_name: participant ? contactLookup.resolveName(participant) : null,
+        _is_self_handle: participant ? contactLookup.isSelfHandle(participant) : false,
       };
     },
   };
 }
 
+function messageParticipant(msg) {
+  const explicit = typeof msg?.participant === "string" ? msg.participant.trim() : "";
+  if (explicit) return explicit;
+  if (!isDirectMessageKind(messageChatKind(msg))) return null;
+  return parseDmHandleFromChatId(msg?.chatId);
+}
+
+function messageChatKind(msg) {
+  const explicit = typeof msg?.chatKind === "string" ? msg.chatKind.trim() : "";
+  if (explicit === "direct") return "dm";
+  if (explicit) return explicit;
+  return parseDmHandleFromChatId(msg?.chatId) ? "dm" : "unknown";
+}
+
+function isDirectMessageKind(kind) {
+  return kind === "dm";
+}
+
 function buildContactLookup(opts = {}) {
-  const contacts = opts.loadAll === false ? [] : loadContacts();
-  const myCard = loadMyCard();
+  const addressBook = platform() === "darwin" ? loadContactsFromAddressBookDb() : { contacts: [], myCard: null };
+  if (opts.userId) {
+    saveMessagesContactStatus(opts.userId, {
+      ok: addressBook.ok === true,
+      updatedAt: new Date().toISOString(),
+      contacts: addressBook.contacts.length,
+      hasMyCard: Boolean(addressBook.myCard),
+      error: addressBook.ok === true ? null : addressBook.error ?? "unsupported local Contacts database schema",
+    });
+  }
+  if (addressBook.ok !== true && platform() === "darwin") {
+    console.warn(`Messages contact resolution degraded: ${addressBook.error ?? "unsupported local Contacts database schema"}. Raw Messages sync will continue; contact founders@askshepherd.ai.`);
+  }
+  const contacts = opts.loadAll === false ? [] : addressBook.contacts;
+  const myCard = addressBook.myCard;
   const handleToName = new Map();
   const selfHandles = new Set();
 
@@ -4367,84 +5008,96 @@ function emptyContactLookup() {
   };
 }
 
-function loadContacts() {
-  if (platform() !== "darwin") return [];
-  const sqliteContacts = loadContactsFromAddressBookDb();
-  if (sqliteContacts.length > 0) return sqliteContacts;
-
-  const script = `
-set output to ""
-tell application "Contacts"
-  repeat with p in every person
-    set pName to name of p
-    set phList to ""
-    repeat with ph in phones of p
-      if phList is not "" then set phList to phList & ","
-      set phList to phList & (value of ph)
-    end repeat
-    set eList to ""
-    repeat with e in emails of p
-      if eList is not "" then set eList to eList & ","
-      set eList to eList & (value of e)
-    end repeat
-    set output to output & pName & "\\t" & phList & "\\t" & eList & "\\n"
-  end repeat
-end tell
-return output`;
-
-  try {
-    const raw = execFileSync("osascript", ["-e", script], {
-      encoding: "utf8",
-      timeout: 120_000,
-    });
-    return parseContacts(raw);
-  } catch (err) {
-    if (args.debug === true) console.error("Could not load Contacts:", safeError(err));
-    return [];
-  }
-}
-
 function loadContactsFromAddressBookDb() {
-  const contacts = new Map();
+  const contacts = [];
+  let myCard = null;
+  let readSucceeded = false;
+  let firstError = null;
   for (const dbPath of addressBookDatabasePaths()) {
     const query = `
-select r.Z_PK,
-       coalesce(nullif(r.ZNAME, ''), nullif(trim(coalesce(r.ZFIRSTNAME, '') || ' ' || coalesce(r.ZLASTNAME, '')), ''), nullif(r.ZORGANIZATION, ''), '') as display_name,
-       coalesce(p.ZFULLNUMBER, '') as phone,
-       '' as email
-from ZABCDRECORD r
-join ZABCDPHONENUMBER p on p.ZOWNER = r.Z_PK
-where p.ZFULLNUMBER is not null and p.ZFULLNUMBER != ''
-union all
-select r.Z_PK,
-       coalesce(nullif(r.ZNAME, ''), nullif(trim(coalesce(r.ZFIRSTNAME, '') || ' ' || coalesce(r.ZLASTNAME, '')), ''), nullif(r.ZORGANIZATION, ''), '') as display_name,
-       '' as phone,
-       coalesce(e.ZADDRESS, '') as email
-from ZABCDRECORD r
-join ZABCDEMAILADDRESS e on e.ZOWNER = r.Z_PK
-where e.ZADDRESS is not null and e.ZADDRESS != '';`;
+with records as (
+  select Z_PK as id,
+         coalesce(ZME, 0) as me,
+         coalesce(Z22_ME, 0) as me22,
+         coalesce(
+           nullif(ZNAME, ''),
+           nullif(trim(coalesce(ZFIRSTNAME, '') || ' ' || coalesce(ZMIDDLENAME, '') || ' ' || coalesce(ZLASTNAME, '')), ''),
+           nullif(ZNICKNAME, ''),
+           nullif(ZORGANIZATION, ''),
+           ''
+         ) as display_name
+  from ZABCDRECORD
+),
+phones as (
+  select coalesce(ZOWNER, Z22_OWNER) as owner, ZFULLNUMBER as value
+  from ZABCDPHONENUMBER
+  where nullif(ZFULLNUMBER, '') is not null
+),
+emails as (
+  select coalesce(ZOWNER, Z22_OWNER) as owner, coalesce(ZADDRESSNORMALIZED, ZADDRESS) as value
+  from ZABCDEMAILADDRESS
+  where nullif(coalesce(ZADDRESSNORMALIZED, ZADDRESS), '') is not null
+)
+select r.id,
+       r.me,
+       r.me22,
+       r.display_name,
+       coalesce(p.value, '') as phone,
+       coalesce(e.value, '') as email
+from records r
+left join phones p on p.owner = r.id
+left join emails e on e.owner = r.id
+where r.display_name != ''
+  and (p.value is not null or e.value is not null);`;
 
     try {
       const raw = execFileSync("sqlite3", ["-separator", "\t", dbPath, query], {
         encoding: "utf8",
         timeout: 10_000,
       });
+      readSucceeded = true;
       for (const line of raw.split("\n").filter(Boolean)) {
-        const [id, rawName, phone, email] = line.split("\t");
+        const [id, me, me22, rawName, phone, email] = line.split("\t");
         const name = rawName?.trim();
         if (!id || !name) continue;
         const key = `${dbPath}:${id}`;
-        const current = contacts.get(key) ?? { name, phones: [], emails: [] };
-        if (phone) current.phones.push(phone.trim());
-        if (email) current.emails.push(email.trim());
-        contacts.set(key, current);
+        let current = contacts.find((contact) => contact._key === key);
+        if (!current) {
+          current = { _key: key, name, phones: [], emails: [] };
+          contacts.push(current);
+        }
+        if (phone?.trim()) current.phones.push(phone.trim());
+        if (email?.trim()) current.emails.push(email.trim());
+        if (Number(me) !== 0 || Number(me22) !== 0) myCard = current;
       }
     } catch (err) {
+      firstError ??= safeError(err);
       if (args.debug === true) console.error(`Could not read Contacts DB ${dbPath}:`, safeError(err));
     }
   }
 
-  return [...contacts.values()].filter((contact) => contact.name);
+  const cleanContacts = contacts
+    .map(({ _key, ...contact }) => ({
+      name: contact.name,
+      phones: [...new Set(contact.phones)],
+      emails: [...new Set(contact.emails)],
+    }))
+    .filter((contact) => contact.name && (contact.phones.length > 0 || contact.emails.length > 0));
+
+  const cleanMyCard = myCard
+    ? {
+        name: myCard.name,
+        phones: [...new Set(myCard.phones)],
+        emails: [...new Set(myCard.emails)],
+      }
+    : null;
+
+  return {
+    ok: readSucceeded,
+    contacts: cleanContacts,
+    myCard: cleanMyCard,
+    error: readSucceeded ? null : firstError,
+  };
 }
 
 function addressBookDatabasePaths() {
@@ -4469,51 +5122,6 @@ function addressBookWalPaths() {
   return [...paths].filter((path) => existsSync(path));
 }
 
-function loadMyCard() {
-  if (platform() !== "darwin") return null;
-  const script = `
-tell application "Contacts"
-  set mc to my card
-  set pName to name of mc
-  set phList to ""
-  repeat with ph in phones of mc
-    if phList is not "" then set phList to phList & ","
-    set phList to phList & (value of ph)
-  end repeat
-  set eList to ""
-  repeat with e in emails of mc
-    if eList is not "" then set eList to eList & ","
-    set eList to eList & (value of e)
-  end repeat
-  return pName & "\\t" & phList & "\\t" & eList
-end tell`;
-
-  try {
-    const raw = execFileSync("osascript", ["-e", script], {
-      encoding: "utf8",
-      timeout: 10_000,
-    });
-    return parseContacts(raw)[0] ?? null;
-  } catch {
-    return null;
-  }
-}
-
-function parseContacts(raw) {
-  return String(raw)
-    .split("\n")
-    .filter(Boolean)
-    .map((line) => {
-      const [name, phones, emails] = line.split("\t");
-      return {
-        name: name?.trim() ?? "",
-        phones: phones ? phones.split(",").map((phone) => phone.trim()).filter(Boolean) : [],
-        emails: emails ? emails.split(",").map((email) => email.trim()).filter(Boolean) : [],
-      };
-    })
-    .filter((contact) => contact.name);
-}
-
 function addHandleMapping(map, handle, name) {
   for (const candidate of handleCandidates(handle)) {
     map.set(candidate, name);
@@ -5219,14 +5827,23 @@ class MessagesBatchSender {
     this.agentToken = agentToken;
     this.userId = userId;
     this.queueFile = join(homedir(), ".shepherd", "raw-messages", `${safeFileId(userId)}-queue.json`);
+    this.sendChain = Promise.resolve();
   }
 
   async send(messages) {
+    const run = () => this.sendUnlocked(messages);
+    const next = this.sendChain.then(run, run);
+    this.sendChain = next.catch(() => undefined);
+    return next;
+  }
+
+  async sendUnlocked(messages) {
     const queued = this.loadQueue();
-    const all = [...queued, ...messages];
-    if (!all.length) return { stored: 0, skipped: 0 };
+    const all = dedupeMessagePayloads([...queued, ...messages]);
+    if (!all.length) return { stored: 0, updated: 0, skipped: 0, queued: 0 };
 
     let totalStored = 0;
+    let totalUpdated = 0;
     let totalSkipped = 0;
 
     for (let i = 0; i < all.length; i += MAX_BATCH_SIZE) {
@@ -5234,16 +5851,18 @@ class MessagesBatchSender {
       try {
         const result = await this.postBatch(batch);
         totalStored += result.stored ?? 0;
+        totalUpdated += result.updated ?? 0;
         totalSkipped += result.skipped ?? 0;
       } catch (err) {
-        this.saveQueue(all.slice(i));
+        const remaining = all.slice(i);
+        this.saveQueue(remaining);
         console.error("Messages batch send failed:", safeError(err));
-        return { stored: totalStored, skipped: totalSkipped };
+        return { stored: totalStored, updated: totalUpdated, skipped: totalSkipped, queued: remaining.length };
       }
     }
 
     this.clearQueue();
-    return { stored: totalStored, skipped: totalSkipped };
+    return { stored: totalStored, updated: totalUpdated, skipped: totalSkipped, queued: 0 };
   }
 
   async postBatch(messages) {
@@ -5286,7 +5905,11 @@ class MessagesBatchSender {
 
   saveQueue(messages) {
     const capped = messages.slice(-MAX_QUEUE_MESSAGES);
+    mkdirSync(dirname(this.queueFile), { recursive: true });
     writeFileSync(this.queueFile, JSON.stringify(capped), { mode: 0o600 });
+    if (capped.length < messages.length) {
+      console.warn(`Messages queue trimmed from ${messages.length} to ${MAX_QUEUE_MESSAGES} messages`);
+    }
   }
 
   clearQueue() {
@@ -5298,6 +5921,37 @@ class MessagesBatchSender {
   }
 }
 
+function dedupeMessagePayloads(messages) {
+  const entries = [];
+  const keyed = new Map();
+  for (const message of messages) {
+    const key = messagePayloadIdentity(message);
+    if (!key) {
+      entries.push({ message });
+      continue;
+    }
+
+    let entry = keyed.get(key);
+    if (!entry) {
+      entry = { key, message };
+      keyed.set(key, entry);
+      entries.push(entry);
+    } else {
+      entry.message = message;
+    }
+  }
+  return entries.map((entry) => entry.message);
+}
+
+function messagePayloadIdentity(message) {
+  const messageId = message?.messageId == null ? null : String(message.messageId);
+  const chatId = message?.chatId == null ? null : String(message.chatId);
+  if (messageId && chatId) return `${chatId}:${messageId}`;
+  if (messageId) return messageId;
+  const rowId = Number(message?.rowId);
+  return Number.isFinite(rowId) && rowId > 0 ? `row:${rowId}` : null;
+}
+
 function loadMessagesWatermark(userId) {
   try {
     const raw = readFileSync(messagesWatermarkFile(userId), "utf8").trim();
@@ -5308,21 +5962,99 @@ function loadMessagesWatermark(userId) {
   }
 }
 
+function loadMessagesBackfillComplete(userId, scope) {
+  const record = readJsonOptional(messagesBackfillCompleteFile(userId, scope));
+  if (!record || typeof record !== "object" || Array.isArray(record)) return null;
+  return record;
+}
+
+function hasAnyMessagesBackfillComplete(userId) {
+  const safeId = safeFileId(userId);
+  try {
+    return readdirSync(messagesRawMessagesDir()).some((entry) =>
+      entry.startsWith(`${safeId}-backfill-`) && entry.endsWith(".json"));
+  } catch {
+    return false;
+  }
+}
+
+function saveMessagesBackfillComplete(userId, scope, result = {}) {
+  const record = {
+    completedAt: new Date().toISOString(),
+    scope,
+    ...result,
+  };
+  writeJsonAtomic(messagesBackfillCompleteFile(userId, scope), record);
+  return record;
+}
+
 function saveMessagesWatermark(userId, rowId) {
+  const numericRowId = Number(rowId);
+  if (!Number.isFinite(numericRowId) || numericRowId <= 0) return;
+
   try {
     const path = messagesWatermarkFile(userId);
-    writeFileSync(path, String(rowId), { mode: 0o600 });
+    const next = Math.max(loadMessagesWatermark(userId), Math.floor(numericRowId));
+    const tmpPath = `${path}.tmp`;
+    writeFileSync(tmpPath, String(next), { mode: 0o600 });
+    renameSync(tmpPath, path);
   } catch (err) {
     console.error("Could not save Messages watermark:", safeError(err));
   }
 }
 
 function messagesWatermarkFile(userId) {
-  const path = join(homedir(), ".shepherd", "raw-messages", `${safeFileId(userId)}-watermark`);
+  const path = join(messagesRawMessagesDir(), `${safeFileId(userId)}-watermark`);
+  mkdirSync(dirname(path), { recursive: true });
+  return path;
+}
+
+function messagesBackfillCompleteFile(userId, scope) {
+  const path = join(messagesRawMessagesDir(), `${safeFileId(userId)}-backfill-${hashObject(scope).slice(0, 24)}.json`);
   mkdirSync(dirname(path), { recursive: true });
   return path;
 }
 
+function messagesStateFile(userId) {
+  const path = join(messagesRawMessagesDir(), `${safeFileId(userId)}-message-state.json`);
+  mkdirSync(dirname(path), { recursive: true });
+  return path;
+}
+
+function saveMessagesState(userId, state) {
+  try {
+    writeJsonAtomic(messagesStateFile(userId), state);
+  } catch (err) {
+    console.error("Could not save Messages state:", safeError(err));
+  }
+}
+
+function messagesContactStatusFile(userId) {
+  const path = join(messagesRawMessagesDir(), `${safeFileId(userId)}-contacts-status.json`);
+  mkdirSync(dirname(path), { recursive: true });
+  return path;
+}
+
+function saveMessagesContactStatus(userId, status) {
+  try {
+    writeJsonAtomic(messagesContactStatusFile(userId), status);
+  } catch (err) {
+    console.error("Could not save Messages contact status:", safeError(err));
+  }
+}
+
+function messagesRawMessagesDir() {
+  const path = join(homedir(), ".shepherd", "raw-messages");
+  mkdirSync(path, { recursive: true });
+  return path;
+}
+
+function writeJsonAtomic(path, value) {
+  const tmpPath = `${path}.tmp`;
+  writeFileSync(tmpPath, JSON.stringify(value), { mode: 0o600 });
+  renameSync(tmpPath, path);
+}
+
 function maxRowId(messages) {
   return Math.max(0, ...messages.map((msg) => Number(msg.rowId ?? 0)).filter(Number.isFinite));
 }
@@ -5344,6 +6076,10 @@ function requiredConfigString(value, label) {
   return value.trim();
 }
 
+function optionalString(value) {
+  return typeof value === "string" && value.trim() ? value.trim() : undefined;
+}
+
 function execFileQuiet(file, argv, opts = {}) {
   return new Promise((resolve, reject) => {
     execFile(file, argv, { windowsHide: true }, (error) => {
@@ -5385,6 +6121,12 @@ function clampInt(value, min, max) {
   return Math.min(Math.max(Math.floor(value), min), max);
 }
 
+function positiveIntFromEnv(name, defaultValue) {
+  const parsed = Number(process.env[name]);
+  if (!Number.isFinite(parsed) || parsed <= 0) return defaultValue;
+  return Math.floor(parsed);
+}
+
 function parseBackfillDays(value, defaultValue) {
   if (value === undefined || value === null || value === "") return defaultValue;
   if (typeof value === "string" && value.trim().toLowerCase() === "all") return null;
@@ -5403,3 +6145,10 @@ function rawErrorDetails(err) {
   if (err instanceof Error) return err.stack ?? err.message;
   return String(err);
 }
+
+export const __test = {
+  messageChatKind,
+  messageParticipant,
+  messageState,
+  sameMessageState,
+};