agentel 0.2.0 → 0.2.2

package/src/mcp.js

@@ -60,7 +60,10 @@ async function handleRequest(message, env = process.env) {
       {
         repo: args.repo,
         limit: args.limit || 10,
-        includeWebChats: args.include_web_chats === true
+        includeWebChats: args.include_web_chats === true,
+        noRebuild: true,
+        skipJsonIndex: true,
+        allowStaleFts: true
       },
       env
     );

package/src/parser-versions.js

@@ -1,34 +1,47 @@
 "use strict";
 
+const { version: PACKAGE_VERSION } = require("../package.json");
+
 const SOURCE_TYPE_ALIASES = {
   "cursor-sqlite-history": "cursor-workspace-sqlite",
   "antigravity-brain": "antigravity-history"
 };
 
-const PARSER_VERSIONS = {
-  "codex-cli-history": "1.0.0",
-  "codex-desktop-history": "1.0.0",
-  "cli-history": "1.0.0",
-  "claude-sdk-history": "1.0.0",
-  "claude-code-desktop-metadata": "1.0.0",
-  "claude-workspace-desktop": "1.0.0",
-  "cursor-workspace-sqlite": "1.0.0",
-  "cursor-global-sqlite": "1.0.0",
-  "cursor-raw-sqlite-salvage": "1.0.0",
-  "cursor-agent-transcripts": "1.0.0",
-  "devin-cli-history": "1.0.0",
-  "gemini-cli-history": "1.0.0",
-  "cline-task-history": "1.0.0",
-  "opencode-history": "1.0.0",
-  "aider-chat-history": "1.0.0",
-  "antigravity-history": "1.0.0",
-  "web-chat-export": "1.0.0",
-  "chatgpt-export": "1.0.0",
-  "claude-web-export": "1.0.0",
-  "claude-web-memory": "1.0.0",
-  import: "1.0.0"
+const PARSER_VERSION_SUFFIXES = {
+  "codex-cli-history": 0,
+  "codex-desktop-history": 0,
+  "cli-history": 0,
+  "claude-sdk-history": 0,
+  "claude-code-desktop-metadata": 0,
+  "claude-workspace-desktop": 0,
+  "cursor-workspace-sqlite": 0,
+  "cursor-global-sqlite": 0,
+  "cursor-raw-sqlite-salvage": 0,
+  "cursor-agent-transcripts": 0,
+  "devin-cli-history": 0,
+  "gemini-cli-history": 0,
+  "cline-task-history": 0,
+  "opencode-history": 0,
+  "opencode-sqlite-history": 0,
+  "aider-chat-history": 0,
+  "antigravity-history": 0,
+  "antigravity-trajectory-summary": 0,
+  "windsurf-trajectory-export": 0,
+  "web-chat-export": 0,
+  "chatgpt-export": 0,
+  "claude-web-export": 0,
+  "claude-web-memory": 0,
+  import: 0
 };
 
+const PARSER_VERSIONS = Object.fromEntries(
+  Object.entries(PARSER_VERSION_SUFFIXES).map(([sourceType, suffix]) => [sourceType, parserVersion(suffix)])
+);
+
+function parserVersion(suffix = 0) {
+  return `${PACKAGE_VERSION}.${Number(suffix) || 0}`;
+}
+
 function canonicalSourceType(sourceType) {
   const key = String(sourceType || "import").trim() || "import";
   return SOURCE_TYPE_ALIASES[key] || key;
@@ -53,6 +66,8 @@ function assertKnownSourceType(sourceType) {
 }
 
 module.exports = {
+  PACKAGE_VERSION,
+  PARSER_VERSION_SUFFIXES,
   PARSER_VERSIONS,
   SOURCE_TYPE_ALIASES,
   assertKnownSourceType,

package/src/paths.js

@@ -23,6 +23,7 @@ function paths(env = process.env) {
     imports: path.join(home, "state", "imports.json"),
     webAccounts: path.join(home, "state", "web-accounts.json"),
     index: path.join(home, "data", "agentlog", "indexes", "bm25", "index.json"),
+    ftsIndex: path.join(home, "data", "agentlog", "indexes", "fts5", "index.sqlite"),
     revealCache: path.join(home, "cache", "unredacted")
   };
 }
@@ -46,9 +47,10 @@ function readJson(file, fallback) {
   }
 }
 
-function writeJson(file, value) {
+function writeJson(file, value, options = {}) {
   ensureDir(path.dirname(file));
-  fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 });
+  const json = options.pretty === false ? JSON.stringify(value) : JSON.stringify(value, null, 2);
+  fs.writeFileSync(file, `${json}\n`, { mode: 0o600 });
 }
 
 module.exports = {

package/src/redaction.js

@@ -16,23 +16,47 @@ const BUILT_INS = [
     regex: /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g
   }
 ];
+const HIGH_ENTROPY_ENV_BUILT_IN = "high_entropy_env";
+const BUILT_IN_NAMES = [...BUILT_INS.map((item) => item.name), HIGH_ENTROPY_ENV_BUILT_IN];
+const REDACTION_LABELS = {
+  aws_key: "AWS ACCESS KEY ID",
+  anthropic_key: "ANTHROPIC KEY",
+  openai_key: "OPENAI KEY",
+  github_token: "GITHUB TOKEN",
+  slack_token: "SLACK TOKEN",
+  jwt: "JWT",
+  private_key: "PRIVATE KEY",
+  high_entropy_env: "HIGH ENTROPY ENV VALUE"
+};
 
-const REDACTION_MARKER_RE = /\[REDACTED:([^\]\n]+)\]/g;
+const REDACTION_MARKER_RE = /\[REDACTED(?::|\s+)([^\]\n]+)\]/g;
 
 function loadRedactionConfig(env = process.env) {
   const file = paths(env).redaction;
   try {
     return parseRedactionYaml(fs.readFileSync(file, "utf8"));
   } catch (error) {
-    if (error.code === "ENOENT") return { patterns: [], envVars: [], allowlistRepos: [] };
+    if (error.code === "ENOENT") return defaultRedactionConfig({ enabled: false });
     throw error;
   }
 }
 
+function defaultRedactionConfig(options = {}) {
+  return {
+    enabled: Boolean(options.enabled),
+    builtIns: Object.fromEntries(BUILT_IN_NAMES.map((name) => [name, true])),
+    patterns: [],
+    envVars: [],
+    allowlistRepos: []
+  };
+}
+
 function parseRedactionYaml(text) {
+  const config = defaultRedactionConfig({ enabled: true });
   const patterns = [];
   const envVars = [];
   const allowlistRepos = [];
+  const builtIns = {};
   const lines = text.split(/\r?\n/);
   let section = "";
   let currentPattern = null;
@@ -46,10 +70,15 @@ function parseRedactionYaml(text) {
     if (top && !line.startsWith(" ")) {
       section = top[1];
       const inline = top[2].trim();
-      if (section === "env_vars" && inline.startsWith("[")) {
+      if (section === "enabled") {
+        config.enabled = parseYamlBoolean(inline, config.enabled);
+      } else if (section === "env_vars" && inline.startsWith("[")) {
         envVars.push(...parseInlineList(inline));
       } else if (section === "allowlist_repos" && inline.startsWith("[")) {
         allowlistRepos.push(...parseInlineList(inline));
+      } else if (section === "built_ins" && inline.startsWith("[")) {
+        const enabled = new Set(parseInlineList(inline));
+        for (const name of BUILT_IN_NAMES) builtIns[name] = enabled.has(name);
       }
       continue;
     }
@@ -71,16 +100,28 @@ function parseRedactionYaml(text) {
     } else if (section === "allowlist_repos") {
       const item = trimmed.match(/^-\s*(.+)$/);
       if (item) allowlistRepos.push(stripYamlString(item[1].trim()).toLowerCase());
+    } else if (section === "built_ins") {
+      const pair = trimmed.match(/^([A-Za-z0-9_.:-]+)\s*:\s*(.+)$/);
+      if (pair) builtIns[pair[1].trim()] = parseYamlBoolean(pair[2].trim(), true);
     }
   }
 
   return {
+    enabled: config.enabled,
+    builtIns: { ...config.builtIns, ...builtIns },
     patterns: patterns.filter((pattern) => pattern.name && pattern.regex),
     envVars: [...new Set(envVars.filter(Boolean))],
     allowlistRepos: [...new Set(allowlistRepos.filter(Boolean))]
   };
 }
 
+function parseYamlBoolean(value, fallback) {
+  const normalized = String(value || "").trim().toLowerCase();
+  if (["true", "yes", "on", "1"].includes(normalized)) return true;
+  if (["false", "no", "off", "0"].includes(normalized)) return false;
+  return fallback;
+}
+
 function parseInlineList(value) {
   return value
     .replace(/^\[/, "")
@@ -135,33 +176,38 @@ function redactText(input, options = {}) {
   const repo = options.repoCanonical || "";
   const cfg = options.config || loadRedactionConfig(options.env);
 
+  if (!cfg.enabled) return { text, summary };
+
   if (repo && cfg.allowlistRepos.includes(repo.toLowerCase())) {
     return { text, summary };
   }
 
   for (const pattern of BUILT_INS) {
-    text = replaceMatches(text, pattern.regex, `[REDACTED:${pattern.name}]`, summary, pattern.name);
+    if (!redactionBuiltInEnabled(cfg, pattern.name)) continue;
+    text = replaceMatches(text, pattern.regex, redactionMarker(pattern.name), summary, pattern.name);
   }
 
   const envValues = options.envValues || loadEnvValues(options.cwd, cfg.envVars, options.env);
   for (const item of envValues) {
     const escaped = escapeRegExp(item.value);
     if (!escaped) continue;
-    text = replaceMatches(text, new RegExp(escaped, "g"), `[REDACTED:env:${item.name}]`, summary, `env:${item.name}`);
+    text = replaceMatches(text, new RegExp(escaped, "g"), redactionMarker(`env:${item.name}`), summary, `env:${item.name}`);
   }
 
-  text = text.replace(
-    /(\b[A-Z0-9_]*(?:KEY|TOKEN|SECRET|PASSWORD|PWD)[A-Z0-9_]*\s*=\s*)(["']?)([A-Za-z0-9+/=_-]{32,})(\2)/gi,
-    (_, prefix, quote, value, suffix) => {
-      if (!looksHighEntropy(value)) return `${prefix}${quote}${value}${suffix}`;
-      increment(summary, "high_entropy_env");
-      return `${prefix}${quote}[REDACTED:high_entropy_env]${suffix}`;
-    }
-  );
+  if (redactionBuiltInEnabled(cfg, HIGH_ENTROPY_ENV_BUILT_IN)) {
+    text = text.replace(
+      /(\b[A-Z0-9_]*(?:KEY|TOKEN|SECRET|PASSWORD|PWD)[A-Z0-9_]*\s*=\s*)(["']?)([A-Za-z0-9+/=_-]{32,})(\2)/gi,
+      (_, prefix, quote, value, suffix) => {
+        if (!looksHighEntropy(value)) return `${prefix}${quote}${value}${suffix}`;
+        increment(summary, HIGH_ENTROPY_ENV_BUILT_IN);
+        return `${prefix}${quote}${redactionMarker(HIGH_ENTROPY_ENV_BUILT_IN)}${suffix}`;
+      }
+    );
+  }
 
   for (const pattern of cfg.patterns) {
     try {
-      text = replaceMatches(text, new RegExp(pattern.regex, "g"), `[REDACTED:${pattern.name}]`, summary, pattern.name);
+      text = replaceMatches(text, new RegExp(pattern.regex, "g"), redactionMarker(pattern.name), summary, pattern.name);
     } catch {
       increment(summary, "invalid_user_pattern");
     }
@@ -170,15 +216,85 @@ function redactText(input, options = {}) {
   return { text, summary };
 }
 
+function redactionBuiltInEnabled(config, name) {
+  if (!BUILT_IN_NAMES.includes(name)) return false;
+  if (!config.builtIns || !(name in config.builtIns)) return true;
+  return config.builtIns[name] !== false;
+}
+
+function redactionBuiltInNames() {
+  return [...BUILT_IN_NAMES];
+}
+
+function redactionConfigTemplate(config = defaultRedactionConfig({ enabled: true })) {
+  const cfg = {
+    ...defaultRedactionConfig({ enabled: true }),
+    ...(config || {}),
+    builtIns: { ...defaultRedactionConfig({ enabled: true }).builtIns, ...((config && config.builtIns) || {}) }
+  };
+  const lines = [
+    "# Agentlog redaction is opt-in. Set enabled: true to redact future imports.",
+    `enabled: ${cfg.enabled ? "true" : "false"}`,
+    "",
+    "built_ins:"
+  ];
+  for (const name of BUILT_IN_NAMES) lines.push(`  ${name}: ${redactionBuiltInEnabled(cfg, name) ? "true" : "false"}`);
+  lines.push("", "env_vars:");
+  if (cfg.envVars?.length) {
+    for (const name of cfg.envVars) lines.push(`  - ${formatYamlString(name)}`);
+  } else {
+    lines[lines.length - 1] = "env_vars: []";
+  }
+  lines.push("", "patterns:");
+  if (cfg.patterns?.length) {
+    for (const pattern of cfg.patterns) {
+      lines.push(`  - name: ${formatYamlString(pattern.name)}`);
+      lines.push(`    regex: ${formatYamlString(pattern.regex)}`);
+    }
+  } else {
+    lines[lines.length - 1] = "patterns: []";
+  }
+  lines.push("", "allowlist_repos:");
+  if (cfg.allowlistRepos?.length) {
+    for (const repo of cfg.allowlistRepos) lines.push(`  - ${formatYamlString(repo)}`);
+  } else {
+    lines[lines.length - 1] = "allowlist_repos: []";
+  }
+  lines.push("");
+  return lines.join("\n");
+}
+
+function formatYamlString(value) {
+  return JSON.stringify(String(value || ""));
+}
+
+function saveRedactionConfig(config, env = process.env) {
+  const file = paths(env).redaction;
+  fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
+  fs.writeFileSync(file, redactionConfigTemplate(config), { mode: 0o600 });
+  return file;
+}
+
 function styleRedactionMarkersForMarkdown(input) {
   return String(input ?? "").replace(REDACTION_MARKER_RE, (_, rawKind) => {
-    const kind = safeRedactionKind(rawKind);
-    return `<mark class="agentlog-redaction" title="agentlog redacted ${escapeHtml(kind)}">[REDACTED:${escapeHtml(kind)}]</mark>`;
+    const label = redactionLabel(rawKind);
+    return `<mark class="agentlog-redaction" title="agentlog redacted ${escapeHtml(label)}">[REDACTED ${escapeHtml(label)}]</mark>`;
   });
 }
 
+function redactionMarker(kind) {
+  return `[REDACTED ${redactionLabel(kind)}]`;
+}
+
+function redactionLabel(kind) {
+  const safeKind = safeRedactionKind(kind);
+  if (REDACTION_LABELS[safeKind]) return REDACTION_LABELS[safeKind];
+  if (safeKind.startsWith("env:")) return `ENV ${safeKind.slice(4).replace(/[^A-Za-z0-9_.-]+/g, "_").toUpperCase()}`;
+  return safeKind.replace(/[_:.-]+/g, " ").replace(/\s+/g, " ").trim().toUpperCase() || "UNKNOWN";
+}
+
 function safeRedactionKind(value) {
-  return String(value || "unknown").replace(/[^\w:.-]/g, "_").slice(0, 120) || "unknown";
+  return String(value || "unknown").trim().replace(/[^\w:.\s-]/g, "_").replace(/\s+/g, "_").toLowerCase().slice(0, 120) || "unknown";
 }
 
 function escapeHtml(value) {
@@ -219,10 +335,17 @@ function escapeRegExp(value) {
 }
 
 module.exports = {
+  defaultRedactionConfig,
   loadEnvValues,
   loadRedactionConfig,
   mergeSummaries,
   parseRedactionYaml,
+  redactionBuiltInEnabled,
+  redactionBuiltInNames,
+  redactionConfigTemplate,
+  redactionLabel,
+  redactionMarker,
   redactText,
+  saveRedactionConfig,
   styleRedactionMarkersForMarkdown
 };