@link-assistant/hive-mind 1.64.1 → 1.64.3

package/src/token-sanitization.lib.mjs

@@ -28,6 +28,61 @@ const getFsModule = async () => (await import('fs')).promises;
 let secretlintCore = null;
 let secretlintConfig = null;
 
+// Issue #1745: process-wide counters for how many tokens were masked. The
+// final-summary path (solve.mjs / hive.mjs) reads these to print a one-line
+// "we masked N secrets — pass --dangerously-skip-output-sanitization to skip"
+// note when N > 0. Counters are intentionally simple (numbers, not arrays of
+// values) so we never accidentally retain raw tokens for any longer than the
+// masking pass itself.
+const sanitizationStats = {
+  totalMasked: 0,
+  knownTokenMasks: 0,
+  patternMasks: 0,
+  hexMasks: 0,
+  excluded: 0,
+};
+
+/**
+ * Read process-wide sanitization counters. Pure read; never mutates.
+ * @returns {{totalMasked:number, knownTokenMasks:number, patternMasks:number, hexMasks:number, excluded:number}}
+ */
+export const getSanitizationStats = () => ({ ...sanitizationStats });
+
+/**
+ * Reset process-wide counters. Tests use this between cases. Production code
+ * has no reason to reset mid-run.
+ */
+export const resetSanitizationStats = () => {
+  sanitizationStats.totalMasked = 0;
+  sanitizationStats.knownTokenMasks = 0;
+  sanitizationStats.patternMasks = 0;
+  sanitizationStats.hexMasks = 0;
+  sanitizationStats.excluded = 0;
+};
+
+/**
+ * Format a one-line operator-facing summary describing how many tokens were
+ * masked during this run, plus the dangerously-skip note required by the
+ * issue when the count is > 0. Returns an empty string if nothing was masked,
+ * so the caller can simply check truthiness before logging.
+ *
+ * @param {Object} [stats] override stats (defaults to module counters)
+ * @returns {string}
+ */
+export const formatSanitizationSummary = (stats = sanitizationStats) => {
+  const { totalMasked = 0, knownTokenMasks = 0, patternMasks = 0, hexMasks = 0, excluded = 0 } = stats;
+  if (totalMasked <= 0 && excluded <= 0) return '';
+  const breakdown = [`known-local: ${knownTokenMasks}`, `pattern: ${patternMasks}`, `hex: ${hexMasks}`].join(', ');
+  const lines = [`🔒 Output sanitization: masked ${totalMasked} token(s) (${breakdown}) before publishing.`];
+  if (excluded > 0) {
+    lines.push(`   ↳ left ${excluded} pre-existing token(s) untouched (user-provided content carve-out).`);
+  }
+  if (totalMasked > 0) {
+    lines.push('   ↳ Pass --dangerously-skip-output-sanitization if this blocks your workflow (active local tokens stay masked unless --dangerously-skip-active-tokens-output-sanitization is also set).');
+  }
+  return lines.join('\n');
+};
+
 /**
  * Initialize secretlint modules lazily
  * @returns {Promise<boolean>} True if secretlint is available
@@ -399,20 +454,25 @@ const compareDetectionResults = async (secretlintSecrets, customSecrets) => {
 };
 
 /**
- * Sanitize log content by masking sensitive tokens while avoiding false positives
+ * Sanitize arbitrary outbound output by masking sensitive tokens while avoiding false positives
  * Uses DUAL APPROACH: Both secretlint AND custom patterns run independently
  *
  * If only secretlint detects a secret (but our custom patterns miss it),
  * a warning is logged so we can improve our patterns.
  *
- * @param {string} logContent - The log content to sanitize
+ * @param {string} output - The output to sanitize
  * @param {Object} options - Optional configuration
  * @param {boolean} options.warnOnMismatch - Log warnings when detection approaches differ (default: true in verbose mode)
- * @returns {Promise<string>} Sanitized log content with tokens masked
+ * @param {boolean} options.skipOutputSanitization - Skip pattern-based output sanitization. Does not skip known active-token masking.
+ * @param {boolean} options.skipActiveTokensOutputSanitization - Also skip known active-token masking. Dangerous; intended only for explicit debugging.
+ * @param {Array<string>} options.excludeTokens - Issue #1745 carve-out: token VALUES that were already in user-provided content (issue body, non-bot comments, pre-existing code). These will be left untouched and counted in `excluded` stats so we don't shock users by mangling tokens they typed themselves.
+ * @returns {Promise<string>} Sanitized output with tokens masked
  */
-export const sanitizeLogContent = async (logContent, options = {}) => {
-  let sanitized = logContent;
-  const { warnOnMismatch = global.verboseMode } = options;
+export const sanitizeOutput = async (output, options = {}) => {
+  let sanitized = output;
+  const { warnOnMismatch = global.verboseMode, skipOutputSanitization = false, skipActiveTokensOutputSanitization = false, excludeTokens = [] } = options;
+  const excludedSet = new Set((excludeTokens || []).filter(t => typeof t === 'string' && t.length > 0));
+  const isExcluded = token => excludedSet.has(token);
 
   // Statistics for dual approach
   const stats = {
@@ -424,20 +484,34 @@ export const sanitizeLogContent = async (logContent, options = {}) => {
   };
 
   try {
-    // Step 1: Get known tokens from files and commands
-    const fileTokens = await getGitHubTokensFromFiles();
-    const commandTokens = await getGitHubTokensFromCommand();
-    const allKnownTokens = [...new Set([...fileTokens, ...commandTokens])];
-
-    // Mask known tokens first
-    for (const token of allKnownTokens) {
-      if (token && token.length >= 12) {
-        const maskedToken = maskToken(token);
-        sanitized = sanitized.split(token).join(maskedToken);
-        stats.knownTokens++;
+    if (!skipActiveTokensOutputSanitization) {
+      // Step 1: Get known tokens from files and commands
+      const fileTokens = await getGitHubTokensFromFiles();
+      const commandTokens = await getGitHubTokensFromCommand();
+      const allKnownTokens = [...new Set([...fileTokens, ...commandTokens])];
+
+      // Mask known tokens first
+      for (const token of allKnownTokens) {
+        if (token && token.length >= 12) {
+          if (isExcluded(token)) {
+            sanitizationStats.excluded++;
+            continue;
+          }
+          if (sanitized.includes(token)) {
+            const maskedToken = maskToken(token);
+            sanitized = sanitized.split(token).join(maskedToken);
+            stats.knownTokens++;
+            sanitizationStats.knownTokenMasks++;
+            sanitizationStats.totalMasked++;
+          }
+        }
       }
     }
 
+    if (skipOutputSanitization) {
+      return sanitized;
+    }
+
     // Step 2: DUAL APPROACH - Run both detection methods independently
     const [secretlintSecrets, customSecrets] = await Promise.all([detectSecretsWithSecretlint(sanitized), Promise.resolve(detectSecretsWithCustomPatterns(sanitized))]);
 
@@ -491,8 +565,14 @@ export const sanitizeLogContent = async (logContent, options = {}) => {
       // Verify the token is still in the content at the expected position
       const currentToken = sanitized.substring(start, end);
       if (currentToken === token) {
+        if (isExcluded(token)) {
+          sanitizationStats.excluded++;
+          continue;
+        }
         const masked = maskToken(token);
         sanitized = sanitized.substring(0, start) + masked + sanitized.substring(end);
+        sanitizationStats.patternMasks++;
+        sanitizationStats.totalMasked++;
       }
     }
 
@@ -516,13 +596,21 @@ export const sanitizeLogContent = async (logContent, options = {}) => {
 
       // Only mask if NOT in a safe git/gist context
       if (!isHexInSafeContext(tempContent, token, position)) {
+        if (isExcluded(token)) {
+          sanitizationStats.excluded++;
+          continue;
+        }
         hexReplacements.push({ token, masked: maskToken(token) });
       }
     }
 
     // Second pass: apply replacements
     for (const { token, masked } of hexReplacements) {
-      sanitized = sanitized.split(token).join(masked);
+      if (sanitized.includes(token)) {
+        sanitized = sanitized.split(token).join(masked);
+        sanitizationStats.hexMasks++;
+        sanitizationStats.totalMasked++;
+      }
     }
 
     // Summary logging
@@ -558,14 +646,263 @@ export const sanitizeLogContent = async (logContent, options = {}) => {
 // Export detection functions for testing and visibility
 export { detectSecretsWithSecretlint, detectSecretsWithCustomPatterns, compareDetectionResults };
 
+/**
+ * Backward-compatible alias for older log-specific call sites.
+ * New output paths should call sanitizeOutput().
+ */
+export const sanitizeLogContent = sanitizeOutput;
+
+// ============================================================================
+// Issue #1745 — known-local-token registry
+// ============================================================================
+// We mask all known LOCAL tokens (env vars + tokens we discovered via gh/etc.)
+// even when our regex/secretlint patterns miss them. This is the
+// "defense-in-depth" layer for the leak documented in case-studies/issue-1745.
+// ============================================================================
+
+/**
+ * Names of environment variables that hold local tokens. Order is irrelevant
+ * but we list AI-CLI tools first since those are the most common leak vectors
+ * (claude, codex, opencode, gemini, qwen + telegram + gh).
+ *
+ * Adding a name here means: any process.env value at this key will be masked
+ * in every comment body / log line the bridge emits.
+ */
+export const KNOWN_LOCAL_TOKEN_ENV_VARS = Object.freeze([
+  // Telegram bridge
+  'TELEGRAM_BOT_TOKEN',
+  'TELEGRAM_OWNER_CHAT_ID',
+  // GitHub CLI / API
+  'GH_TOKEN',
+  'GITHUB_TOKEN',
+  'GITHUB_PAT',
+  // Claude / Anthropic
+  'ANTHROPIC_API_KEY',
+  'CLAUDE_API_KEY',
+  'CLAUDE_CODE_OAUTH_TOKEN',
+  // OpenAI / Codex
+  'OPENAI_API_KEY',
+  'CODEX_API_KEY',
+  // Open-source agent CLIs
+  'OPENCODE_API_KEY',
+  'AGENT_CLI_TOKEN',
+  // Google Gemini / Qwen
+  'GEMINI_API_KEY',
+  'GOOGLE_API_KEY',
+  'QWEN_API_KEY',
+  'DASHSCOPE_API_KEY',
+  // Misc
+  'HUGGINGFACE_TOKEN',
+  'HF_TOKEN',
+]);
+
+/**
+ * Read every known local-token env var that is currently set.
+ *
+ * @returns {Array<{name: string, value: string}>} entries with non-empty values
+ */
+export const getEnvironmentTokens = () => {
+  const out = [];
+  for (const name of KNOWN_LOCAL_TOKEN_ENV_VARS) {
+    const value = process.env[name];
+    if (typeof value === 'string' && value.length >= 12) {
+      out.push({ name, value });
+    }
+  }
+  return out;
+};
+
+/**
+ * Build the union of every known-local token: env vars + GitHub tokens we
+ * already discover via `gh auth status` / hosts.yml (existing helpers).
+ *
+ * Each entry is `{ source, name, value }` where `source` is 'env' | 'gh-files'
+ * | 'gh-command'. The `name` field is human-readable for debug logs but is
+ * NEVER printed alongside the token to avoid creating a secondary leak.
+ *
+ * @returns {Promise<Array<{source: string, name: string, value: string}>>}
+ */
+export const getAllKnownLocalTokens = async () => {
+  const tokens = [];
+
+  for (const { name, value } of getEnvironmentTokens()) {
+    tokens.push({ source: 'env', name, value });
+  }
+
+  try {
+    const fileTokens = await getGitHubTokensFromFiles();
+    for (const value of fileTokens) {
+      tokens.push({ source: 'gh-files', name: 'github', value });
+    }
+  } catch {
+    /* swallow — best-effort */
+  }
+
+  try {
+    const commandTokens = await getGitHubTokensFromCommand();
+    for (const value of commandTokens) {
+      tokens.push({ source: 'gh-command', name: 'github', value });
+    }
+  } catch {
+    /* swallow — best-effort */
+  }
+
+  // Deduplicate by exact value
+  const seen = new Set();
+  return tokens.filter(({ value }) => {
+    if (seen.has(value)) return false;
+    seen.add(value);
+    return true;
+  });
+};
+
+/**
+ * Test whether `text` contains any known-local token verbatim.
+ * Used to decide whether to fire the Telegram leak-warning DM.
+ *
+ * @param {string} text
+ * @param {Array<{value: string, name?: string, source?: string}>} [tokens]
+ *   Pre-fetched token list (if you already called getAllKnownLocalTokens).
+ *   Pass an explicit list to avoid re-running `gh auth status` per check.
+ * @returns {Promise<Array<{name: string, source: string}>>} list of token
+ *   identifiers that were found in the text (NOT the values themselves).
+ */
+export const containsKnownToken = async (text, tokens) => {
+  if (typeof text !== 'string' || text.length === 0) return [];
+  const list = tokens || (await getAllKnownLocalTokens());
+  const hits = [];
+  for (const t of list) {
+    if (t.value && text.includes(t.value)) {
+      hits.push({ name: t.name, source: t.source });
+    }
+  }
+  return hits;
+};
+
+/**
+ * Mask every known-local token inside `body` and then run `sanitizeOutput`
+ * for the regex/secretlint sweep. This is the wrapper that comment-posting
+ * paths must call before publishing anything to GitHub.
+ *
+ * Env-token masking runs FIRST so that even if our regex misses the shape
+ * (custom token formats from new AI tools, etc.) the local secret never
+ * leaves the process. The regex/secretlint pass then catches anything else.
+ *
+ * @param {string} body
+ * @param {Object} [options]
+ * @param {Array<{value: string}>} [options.knownTokens] pre-fetched token list
+ * @returns {Promise<string>} sanitized body
+ */
+export const sanitizeCommentBody = async (body, options = {}) => {
+  if (typeof body !== 'string' || body.length === 0) return body;
+
+  let sanitized = body;
+  const excludedSet = new Set((options.excludeTokens || []).filter(t => typeof t === 'string' && t.length > 0));
+
+  // Pass 1: mask known-local tokens verbatim. This is the defense-in-depth
+  // layer that closes the gap from issue #1745.
+  if (!options.skipActiveTokensOutputSanitization) {
+    const knownTokens = options.knownTokens || (await getAllKnownLocalTokens());
+    for (const { value } of knownTokens) {
+      if (value && value.length >= 12 && sanitized.includes(value)) {
+        if (excludedSet.has(value)) {
+          sanitizationStats.excluded++;
+          continue;
+        }
+        sanitized = sanitized.split(value).join(maskToken(value));
+        sanitizationStats.knownTokenMasks++;
+        sanitizationStats.totalMasked++;
+      }
+    }
+  }
+
+  // Pass 2: regex + secretlint sweep for anything else.
+  sanitized = await sanitizeOutput(sanitized, {
+    warnOnMismatch: false,
+    skipOutputSanitization: options.skipOutputSanitization,
+    skipActiveTokensOutputSanitization: true,
+    excludeTokens: options.excludeTokens || [],
+  });
+
+  return sanitized;
+};
+
+/**
+ * Issue #1745 user-content carve-out helper.
+ *
+ * Comment #4364642786: "if issue description/comment/pull request comment from
+ * other users than our bot, contained access token, meaning access token was
+ * explicitly given, or access token was existing in code before, we don't
+ * touch it. That is not our responsibility by default."
+ *
+ * Given concatenated user-provided text (issue body, non-bot issue/PR
+ * comments, original code), this helper returns the token-shaped strings
+ * already present in that text. Callers pass this list as `excludeTokens`
+ * to `sanitizeOutput` / `sanitizeCommentBody` so the sanitizer leaves those
+ * tokens untouched.
+ *
+ * Active local tokens (env vars, gh CLI tokens) are NEVER returned even if
+ * they appear in user-provided content — the user couldn't have intended for
+ * us to leak our own bot tokens, so the carve-out doesn't apply to them.
+ *
+ * @param {string} text concatenated user-provided text
+ * @param {Object} [options]
+ * @param {Array<{value: string}>} [options.knownTokens] active local tokens to
+ *   filter out of the carve-out (so the bot's own tokens still get masked
+ *   even if the user pasted one verbatim).
+ * @returns {Promise<Array<string>>} token VALUES to exclude from sanitization
+ */
+export const extractTokensFromUserContent = async (text, options = {}) => {
+  if (typeof text !== 'string' || text.length === 0) return [];
+
+  const customSecrets = detectSecretsWithCustomPatterns(text);
+  const secretlintSecrets = await detectSecretsWithSecretlint(text);
+
+  const tokens = new Set();
+  for (const s of [...customSecrets, ...secretlintSecrets]) {
+    if (s.token && s.token.length >= 12) {
+      tokens.add(s.token);
+    }
+  }
+
+  // 40-char hex in user-provided text — only exclude when not in a safe
+  // git/gist context. We're conservative here: the carve-out only applies
+  // to things our regex would otherwise mask.
+  const hexPattern = /(?:^|[\s:=])([a-f0-9]{40})(?=[\s\n]|$)/gm;
+  hexPattern.lastIndex = 0;
+  let m;
+  while ((m = hexPattern.exec(text)) !== null) {
+    const token = m[1];
+    if (!isHexInSafeContext(text, token, m.index)) {
+      tokens.add(token);
+    }
+  }
+
+  // Filter out our own active local tokens. The user pasting our token in
+  // their issue body doesn't mean we should leak it — that's still our bot's
+  // secret and it stays masked.
+  const knownActive = new Set((options.knownTokens || []).map(t => t.value).filter(Boolean));
+  return [...tokens].filter(value => !knownActive.has(value));
+};
+
 // Default export for convenience
 export default {
   isSafeToken,
   isHexInSafeContext,
   getGitHubTokensFromFiles,
   getGitHubTokensFromCommand,
+  sanitizeOutput,
   sanitizeLogContent,
   detectSecretsWithSecretlint,
   detectSecretsWithCustomPatterns,
   compareDetectionResults,
+  getEnvironmentTokens,
+  getAllKnownLocalTokens,
+  containsKnownToken,
+  sanitizeCommentBody,
+  getSanitizationStats,
+  resetSanitizationStats,
+  formatSanitizationSummary,
+  extractTokensFromUserContent,
+  KNOWN_LOCAL_TOKEN_ENV_VARS,
 };

package/src/tool-comments.lib.mjs

@@ -198,7 +198,7 @@ export const resetTrackedToolCommentIds = () => {
  * @param {string} options.body
  * @returns {Promise<{ok: boolean, commentId: string|null, stderr?: string}>}
  */
-export const postTrackedComment = async ({ $, owner, repo, targetNumber, body }) => {
+export const postTrackedComment = async ({ $, owner, repo, targetNumber, body, sanitizationOptions }) => {
   if (!$) {
     throw new Error('postTrackedComment requires a command-stream $ helper');
   }
@@ -208,7 +208,11 @@ export const postTrackedComment = async ({ $, owner, repo, targetNumber, body })
   // We use the /issues/<n>/comments endpoint because it works identically
   // for both PRs and issues (a PR is an issue at this endpoint).
   const apiPath = `repos/${owner}/${repo}/issues/${targetNumber}/comments`;
-  const payload = JSON.stringify({ body });
+  const { sanitizeOutput } = await import('./token-sanitization.lib.mjs');
+  // Issue #1745: caller may pass dangerous-skip flags + carve-out tokens.
+  // Defaults preserve fail-closed behavior: full sanitization.
+  const sanitizedBody = await sanitizeOutput(body, sanitizationOptions || {});
+  const payload = JSON.stringify({ body: sanitizedBody });
 
   // command-stream's options key is `stdin`, not `input` — unknown keys are
   // silently ignored, which previously left stdin inherited from the parent