agentel 0.2.6 → 0.3.0

package/src/sources.js

@@ -15,7 +15,7 @@ const SOURCE_GROUPS = [
     sources: [
       { source: "claude", provider: "claude_code", label: "Claude Code CLI" },
       { source: "claude-code-desktop", provider: "claude_desktop", sourceType: "claude-code-desktop-metadata", label: "Claude Code Desktop" },
-      { source: "claude-workspace", provider: "claude_desktop", sourceType: "claude-workspace-desktop", label: "Claude Workspace" },
+      { source: "claude-cowork", provider: "claude_desktop", sourceType: "claude-workspace-desktop", label: "Claude Cowork" },
       { source: "claude-web", provider: "claude_web", label: "Claude.ai" },
       { source: "claude-sdk", provider: "claude_sdk", label: "Claude SDK jobs" }
     ]
@@ -24,19 +24,42 @@ const SOURCE_GROUPS = [
     group: "Google",
     sources: [
       { source: "gemini-cli", provider: "gemini_cli", label: "Gemini CLI" },
-      { source: "antigravity", provider: "antigravity", label: "Antigravity" }
+      { source: "antigravity-cli", provider: "antigravity_cli", sourceType: "antigravity-cli-transcript-log", label: "Antigravity CLI" },
+      { source: "antigravity", provider: "antigravity", label: "Antigravity 2.0" },
+      { source: "antigravity-ide", provider: "antigravity_ide", sourceType: "antigravity-ide-transcript-log", label: "Antigravity IDE" }
     ]
   },
   {
     group: "Cognition",
     sources: [
-      { source: "devin-cli", provider: "devin", sourceType: "devin-cli-history", label: "Devin CLI" }
+      { source: "devin-cli", provider: "devin", sourceType: "devin-cli-history", label: "Devin CLI" },
+      { source: "devin-desktop", provider: "devin", sourceType: "devin-desktop-acp-events", label: "Devin Desktop" },
+      { source: "windsurf", provider: "windsurf", sourceType: "windsurf-cascade-brain", label: "Windsurf" }
+    ]
+  },
+  {
+    group: "GitHub",
+    sources: [
+      { source: "copilot-cli", provider: "copilot", sourceType: "copilot-cli-history", label: "GitHub Copilot CLI" }
+    ]
+  },
+  {
+    group: "Factory",
+    sources: [
+      { source: "factory", provider: "factory", sourceType: "factory-droid-history", label: "Factory Droid" }
+    ]
+  },
+  {
+    group: "xAI",
+    sources: [
+      { source: "grok-build", provider: "grok", sourceType: "grok-build-history", label: "Grok Build" }
     ]
   },
   {
     group: "Other",
     sources: [
       { source: "cursor", provider: "cursor", label: "Cursor" },
+      { source: "pi", provider: "pi", sourceType: "pi-cli-history", label: "pi" },
       { source: "cline", provider: "cline", sourceType: "cline-task-history", label: "Cline" },
       { source: "opencode-cli", provider: "opencode", sourceType: "opencode-cli-history", label: "OpenCode CLI" },
       { source: "opencode-desktop", provider: "opencode", sourceType: "opencode-desktop-history", label: "OpenCode Desktop" },
@@ -46,23 +69,28 @@ const SOURCE_GROUPS = [
   }
 ];
 
-const DISABLED_IMPORT_SOURCES = new Set(["windsurf"]);
+const DISABLED_IMPORT_SOURCES = new Set();
 
-const DISABLED_IMPORT_SOURCE_MESSAGES = {
-  windsurf:
-    "Windsurf import is disabled because current Cascade transcripts are encrypted binary stores. agentlog can detect those sessions, but cannot archive readable conversation text yet."
-};
+const DISABLED_IMPORT_SOURCE_MESSAGES = {};
 
 const IMPORT_SOURCE_ORDER = [
   "codex-cli",
   "codex-desktop",
   "claude",
   "claude-code-desktop",
-  "claude-workspace",
+  "claude-cowork",
   "gemini-cli",
+  "antigravity-cli",
   "antigravity",
+  "antigravity-ide",
   "devin-cli",
+  "devin-desktop",
+  "windsurf",
+  "copilot-cli",
+  "factory",
+  "grok-build",
   "cursor",
+  "pi",
   "cline",
   "opencode-cli",
   "opencode-desktop",
@@ -70,17 +98,38 @@ const IMPORT_SOURCE_ORDER = [
   "aider"
 ];
 
+const IMPORT_SOURCE_ALIASES = {
+  "claude-workspace": "claude-cowork",
+  claude_workspace: "claude-cowork",
+  claude_cowork: "claude-cowork"
+};
+
 const HISTORY_PROVIDER_OPTIONS = SOURCE_GROUPS.flatMap((group) => group.sources);
 
+function canonicalImportSource(source) {
+  const value = String(source || "").trim();
+  return IMPORT_SOURCE_ALIASES[value] || value;
+}
+
 function enabledImportSources(sources) {
-  return (sources || []).filter((source) => !DISABLED_IMPORT_SOURCES.has(source));
+  const seen = new Set();
+  const result = [];
+  for (const rawSource of sources || []) {
+    const source = canonicalImportSource(rawSource);
+    if (!source || seen.has(source) || DISABLED_IMPORT_SOURCES.has(source)) continue;
+    seen.add(source);
+    result.push(source);
+  }
+  return result;
 }
 
 function sourceLabel(source) {
-  return HISTORY_PROVIDER_OPTIONS.find((item) => item.source === source)?.label || { opencode: "OpenCode" }[source] || source;
+  const canonical = canonicalImportSource(source);
+  return HISTORY_PROVIDER_OPTIONS.find((item) => item.source === canonical)?.label || { opencode: "OpenCode" }[canonical] || canonical;
 }
 
 module.exports = {
+  canonicalImportSource,
   DISABLED_IMPORT_SOURCE_MESSAGES,
   DISABLED_IMPORT_SOURCES,
   HISTORY_PROVIDER_OPTIONS,

package/src/supervisor.js

@@ -6,12 +6,101 @@ const { spawn } = require("child_process");
 const { ensureBaseDirs, paths, readJson, writeJson } = require("./paths");
 const { effectiveImportSources, loadConfig } = require("./config");
 const { startCollector } = require("./collector");
+const { startSourceWatchers } = require("./source-watch");
 const { hasRemoteTarget } = require("./sync");
 
 let tickRunning = false;
 const CHILD_MAX_OUTPUT_BYTES = 10 * 1024 * 1024;
 const CHILD_TIMEOUT_MS = 10 * 60 * 1000;
 const CHILD_KILL_GRACE_MS = 2000;
+// Backoff so an idle machine doesn't spawn import/index children every 30s
+// forever: after a few empty imports a source drops to a slower cadence and
+// snaps back to every tick as soon as an import finds activity. Failing
+// sources back off exponentially so a permanently broken source doesn't
+// burn a child process (and a log line) per tick.
+const SOURCE_IDLE_TICKS_BEFORE_BACKOFF = 4;
+const SOURCE_IDLE_INTERVAL_MS = 5 * 60 * 1000;
+const SOURCE_ERROR_BACKOFF_MIN_MS = 60 * 1000;
+const SOURCE_ERROR_BACKOFF_MAX_MS = 30 * 60 * 1000;
+// Sources covered by filesystem watchers don't need the fast poll at all:
+// events mark them pending (imported on the next tick, triggered
+// immediately), and this heartbeat poll only backstops dropped events —
+// fs.watch loses events under load and emits nothing on network volumes.
+const WATCHED_SOURCE_HEARTBEAT_MS = 15 * 60 * 1000;
+const INDEX_MIN_INTERVAL_MS = 10 * 60 * 1000;
+// Memory files change rarely and the backup is a stat+hash sweep, so a
+// moderate cadence keeps provider memories continuously archived without
+// meaningful cost.
+const MEMORY_BACKUP_INTERVAL_MS = 15 * 60 * 1000;
+
+// Per-run tick state. Owned by runSupervisorForeground so direct tick()
+// callers (tests, one-shot runs) always start fresh.
+function createTickState() {
+  return {
+    sources: new Map(),
+    lastIndexAt: 0,
+    indexDirty: true,
+    lastMemoryBackupAt: 0,
+    pendingSources: new Set(),
+    tickAgain: false,
+    watch: null
+  };
+}
+
+function sourceTickStateFor(state, source) {
+  let sourceState = state.sources.get(source);
+  if (!sourceState) {
+    sourceState = { nextEligibleAt: 0, idleTicks: 0, failures: 0, idleLogged: false };
+    state.sources.set(source, sourceState);
+  }
+  return sourceState;
+}
+
+// Returns true when the source just transitioned into idle backoff (so the
+// caller can log the transition once instead of every tick).
+function applySourceImportOutcome(sourceState, activity, now) {
+  sourceState.failures = 0;
+  if (activity > 0) {
+    sourceState.idleTicks = 0;
+    sourceState.nextEligibleAt = 0;
+    sourceState.idleLogged = false;
+    return false;
+  }
+  sourceState.idleTicks += 1;
+  if (sourceState.idleTicks < SOURCE_IDLE_TICKS_BEFORE_BACKOFF) return false;
+  sourceState.nextEligibleAt = now + SOURCE_IDLE_INTERVAL_MS;
+  const firstTime = !sourceState.idleLogged;
+  sourceState.idleLogged = true;
+  return firstTime;
+}
+
+// Watched sources skip idle backoff entirely: filesystem events snap them
+// back via pendingSources, so after any import they rest until the heartbeat.
+function applyWatchedSourceImportOutcome(sourceState, now) {
+  sourceState.failures = 0;
+  sourceState.idleTicks = 0;
+  sourceState.idleLogged = false;
+  sourceState.nextEligibleAt = now + WATCHED_SOURCE_HEARTBEAT_MS;
+}
+
+// Pending filesystem activity bypasses idle/heartbeat scheduling but not
+// error backoff, so a chatty source that fails to import doesn't retry on
+// every coalesce window.
+function sourceEligibleForImport(sourceState, pending, now) {
+  if (now >= sourceState.nextEligibleAt) return true;
+  return pending && sourceState.failures === 0;
+}
+
+// Returns the backoff delay in ms applied after a failed import.
+function applySourceImportFailure(sourceState, now) {
+  sourceState.failures += 1;
+  const backoffMs = Math.min(
+    SOURCE_ERROR_BACKOFF_MAX_MS,
+    SOURCE_ERROR_BACKOFF_MIN_MS * 2 ** (sourceState.failures - 1)
+  );
+  sourceState.nextEligibleAt = now + backoffMs;
+  return backoffMs;
+}
 
 function startSupervisorDetached(env = process.env) {
   const p = paths(env);
@@ -37,57 +126,134 @@ async function runSupervisorForeground(env = process.env) {
   });
   if (collector) log(`collector listening on ${loadConfig(env).collector.otlpEndpoint}`, env);
 
+  const tickState = createTickState();
+
   let stopping = false;
   const stop = () => {
     if (stopping) return;
     stopping = true;
     log("supervisor stopping", env);
+    if (tickState.watch) tickState.watch.close();
     removeSupervisorPidIfOwned(process.pid, env);
     process.exit(0);
   };
   process.on("SIGTERM", stop);
   process.on("SIGINT", stop);
 
-  await tick(env);
+  const runTick = () => {
+    if (stopping) return;
+    tick(env, tickState)
+      .catch((error) => log(`tick failed: ${error.message}`, env))
+      .finally(() => {
+        // Sources marked dirty while a tick ran get picked up right away
+        // instead of waiting for the interval.
+        if (tickState.tickAgain && !stopping) {
+          tickState.tickAgain = false;
+          runTick();
+        }
+      });
+  };
+
+  try {
+    const sources = effectiveImportSources(loadConfig(env));
+    tickState.watch = startSourceWatchers(
+      Array.isArray(sources) && sources.length ? sources : [],
+      env,
+      (source) => {
+        tickState.pendingSources.add(source);
+        if (tickRunning) {
+          tickState.tickAgain = true;
+          return;
+        }
+        runTick();
+      },
+      { onLog: (message) => log(message, env) }
+    );
+    const active = tickState.watch.activeWatchCount();
+    if (active) log(`watching ${active} source root(s) for changes`, env);
+  } catch (error) {
+    log(`source watch disabled: ${error.message}`, env);
+  }
+
+  await tick(env, tickState);
+  if (tickState.tickAgain) {
+    tickState.tickAgain = false;
+    runTick();
+  }
   const timer = setInterval(() => {
     if (tickRunning) {
       log("tick skipped: previous tick still running", env);
       return;
     }
-    tick(env).catch((error) => log(`tick failed: ${error.message}`, env));
+    runTick();
   }, 30 * 1000);
 
   await new Promise(() => {});
 }
 
-async function tick(env) {
+async function tick(env, state = createTickState()) {
   tickRunning = true;
   try {
+    const now = Date.now();
     try {
       const cfg = loadConfig(env);
       const sources = effectiveImportSources(cfg);
       const importSources = Array.isArray(sources) && sources.length ? sources : ["all"];
       for (const source of importSources) {
+        const sourceState = sourceTickStateFor(state, source);
+        const pending = state.pendingSources.has(source);
+        if (!sourceEligibleForImport(sourceState, pending, now)) continue;
+        state.pendingSources.delete(source);
         try {
           const imports = await importSourceInChild(source, cfg, env);
+          let activity = 0;
           for (const result of imports) {
+            activity += (Number(result.imported) || 0) + (Number(result.pruned) || 0);
             const summary = supervisorImportResultLog(result, source);
             if (summary) log(summary, env);
             if (result.errors?.length) log(`${result.provider} import errors from ${source}: ${result.errors.length}`, env);
           }
+          if (activity > 0) state.indexDirty = true;
+          if (state.watch && state.watch.isWatched(source)) {
+            applyWatchedSourceImportOutcome(sourceState, now);
+          } else if (applySourceImportOutcome(sourceState, activity, now)) {
+            log(`${source} import idle: slowing to ${formatDuration(SOURCE_IDLE_INTERVAL_MS)} cadence until new activity`, env);
+          }
         } catch (error) {
-          log(`${source} history import skipped: ${error.message}`, env);
+          const backoffMs = applySourceImportFailure(sourceState, now);
+          log(`${source} history import skipped: ${error.message} (retrying in ${formatDuration(backoffMs)})`, env);
         }
       }
     } catch (error) {
       log(`history import skipped: ${error.message}`, env);
     }
 
-    try {
-      const result = await reindexInChild(env);
-      if (result.index?.docCount != null) log(`index ready (${result.index.docCount} chunk(s))`, env);
-    } catch (error) {
-      log(`index skipped: ${error.message}`, env);
+    if (state.indexDirty || now - state.lastIndexAt >= INDEX_MIN_INTERVAL_MS) {
+      try {
+        const result = await reindexInChild(env);
+        if (result.index?.docCount != null) log(`index ready (${result.index.docCount} chunk(s))`, env);
+      } catch (error) {
+        log(`index skipped: ${error.message}`, env);
+      } finally {
+        // Advance the clock even on failure so a persistent indexing error
+        // backs off to INDEX_MIN_INTERVAL_MS instead of retrying every tick.
+        state.lastIndexAt = now;
+        state.indexDirty = false;
+      }
+    }
+
+    if (now - state.lastMemoryBackupAt >= MEMORY_BACKUP_INTERVAL_MS) {
+      try {
+        const summaries = await memoryBackupInChild(env);
+        const changed = summaries.reduce((sum, item) => sum + (Number(item.changed) || 0), 0);
+        if (changed > 0) log(`memory backup captured ${changed} change(s)`, env);
+      } catch (error) {
+        log(`memory backup skipped: ${error.message}`, env);
+      } finally {
+        // Advance even on failure so a persistent error backs off to the
+        // backup interval instead of retrying every tick.
+        state.lastMemoryBackupAt = now;
+      }
     }
 
     try {
@@ -124,6 +290,22 @@ try {
   return runJsonChild(script, env, `${source} import`, title);
 }
 
+function memoryBackupInChild(env = process.env) {
+  const title = childProcessTitle("memory-backup");
+  const script = `
+process.title = ${JSON.stringify(title)};
+(async () => {
+  const { collectMemoryBackup } = require(${JSON.stringify(path.join(__dirname, "memory-sources"))});
+  const summaries = await collectMemoryBackup(process.env);
+  process.stdout.write(JSON.stringify(summaries));
+})().catch((error) => {
+  console.error(error && error.message ? error.message : String(error));
+  process.exit(1);
+});
+`;
+  return runJsonChild(script, env, "memory backup", title);
+}
+
 function reindexInChild(env = process.env) {
   const title = childProcessTitle("index");
   const script = `
@@ -350,6 +532,11 @@ function log(message, env = process.env) {
 }
 
 module.exports = {
+  applySourceImportFailure,
+  applySourceImportOutcome,
+  applyWatchedSourceImportOutcome,
+  createTickState,
+  sourceEligibleForImport,
   runSupervisorForeground,
   startSupervisorDetached,
   stopSupervisor,
@@ -359,6 +546,7 @@ module.exports = {
   removeSupervisorPidIfOwned,
   shouldRunScheduledSync,
   importSourceInChild,
+  memoryBackupInChild,
   reindexInChild,
   runJsonChild,
   syncInChild,

package/src/sync.js

@@ -526,6 +526,7 @@ function listLocalArchiveObjects(env = process.env, prefix = "agentlog") {
     const stat = safeStat(file);
     if (!stat || !stat.isFile()) return;
     const relative = path.relative(root, file).split(path.sep).join("/");
+    if (!syncableArchiveObject(relative)) return;
     const key = [normalizePrefix(prefix), relative].filter(Boolean).join("/");
     objects.push({
       file,
@@ -538,6 +539,11 @@ function listLocalArchiveObjects(env = process.env, prefix = "agentlog") {
   return objects.sort((a, b) => a.key.localeCompare(b.key));
 }
 
+function syncableArchiveObject(relative) {
+  const key = String(relative || "").replace(/\\/g, "/");
+  return !key.startsWith("indexes/");
+}
+
 function listDirectoryObjects(root) {
   const objects = [];
   walk(root, (file) => {