@link-assistant/hive-mind 0.54.4 → 0.54.6

package/CHANGELOG.md

@@ -1,5 +1,38 @@
 # @link-assistant/hive-mind
 
+## 0.54.6
+
+### Patch Changes
+
+- f734d5d: feat: Add --base-branch to /help and implement option typo suggestions
+  - Added --base-branch option to Telegram bot /help command
+  - Implemented intelligent option name suggestions using Levenshtein distance
+  - Added --base-branch to README.md solve options section
+  - Enhanced error messages with helpful suggestions for typos (e.g., --branch → --base-branch)
+
+## 0.54.5
+
+### Patch Changes
+
+- Fix duplicate APT sources warning in installation script
+  - Add `cleanup_duplicate_apt_sources()` function to detect and remove duplicate APT source files
+  - Clean up duplicate Microsoft Edge sources (`microsoft-edge.list` vs `microsoft-edge-stable.list`)
+  - Clean up duplicate Google Chrome sources (`google-chrome.list` vs `google-chrome-stable.list`)
+  - Run cleanup before `apt update` to prevent "Target Packages configured multiple times" warnings
+  - Ensures script supports clean upgrade mode when run on previously installed systems
+
+  Improve Telegram bot error messages for better user experience (issue #1070)
+  - Enhanced URL validation to provide specific, actionable error messages based on URL type (issues list, pulls list, repository)
+  - Added step-by-step fix instructions with examples when users provide wrong URL formats
+  - Improved global error handler to properly escape Markdown special characters, preventing "400: Bad Request: can't parse entities" errors
+  - Added special handling for Telegram API parsing errors with clearer messaging
+  - Added `cleanNonPrintableChars()` to automatically remove invisible Unicode characters from user input
+  - Added `makeSpecialCharsVisible()` to show users exactly where problematic special characters are in their input
+  - Enhanced error messages to display user input with special characters made visible for easier debugging
+  - Refactored telegram-bot.mjs to meet 1500 line limit requirement
+  - Created comprehensive test suites to verify URL validation improvements and special character handling
+  - Documented case study analysis in docs/case-studies/issue-1070/ANALYSIS.md
+
 ## 0.54.4
 
 ### Patch Changes

package/README.md

@@ -332,10 +332,11 @@ solve <issue-url> [options]
 
 **Most frequently used options:**
 
-| Option    | Alias | Description                             | Default |
-| --------- | ----- | --------------------------------------- | ------- |
-| `--model` | `-m`  | AI model to use (sonnet, opus, haiku)   | sonnet  |
-| `--think` |       | Thinking level (low, medium, high, max) | -       |
+| Option          | Alias | Description                             | Default   |
+| --------------- | ----- | --------------------------------------- | --------- |
+| `--model`       | `-m`  | AI model to use (sonnet, opus, haiku)   | sonnet    |
+| `--think`       |       | Thinking level (low, medium, high, max) | -         |
+| `--base-branch` | `-b`  | Target branch for PR                    | (default) |
 
 **Other useful options:**
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/hive-mind",
-  "version": "0.54.4",
+  "version": "0.54.6",
   "description": "AI-powered issue solver and hive mind for collaborative problem solving",
   "main": "src/hive.mjs",
   "type": "module",

package/src/option-suggestions.lib.mjs

@@ -0,0 +1,180 @@
+// Option suggestion utility for providing helpful error messages
+// when users mistype command-line option names
+
+/**
+ * Calculate Levenshtein distance between two strings
+ * Measures the minimum number of single-character edits (insertions, deletions, or substitutions)
+ * required to change one string into the other.
+ *
+ * @param {string} a - First string
+ * @param {string} b - Second string
+ * @returns {number} - Levenshtein distance
+ */
+export function calculateLevenshteinDistance(a, b) {
+  if (a.length === 0) return b.length;
+  if (b.length === 0) return a.length;
+
+  const matrix = [];
+
+  // Initialize first column of matrix
+  for (let i = 0; i <= b.length; i++) {
+    matrix[i] = [i];
+  }
+
+  // Initialize first row of matrix
+  for (let j = 0; j <= a.length; j++) {
+    matrix[0][j] = j;
+  }
+
+  // Fill in the rest of the matrix
+  for (let i = 1; i <= b.length; i++) {
+    for (let j = 1; j <= a.length; j++) {
+      if (b.charAt(i - 1) === a.charAt(j - 1)) {
+        matrix[i][j] = matrix[i - 1][j - 1];
+      } else {
+        matrix[i][j] = Math.min(
+          matrix[i - 1][j - 1] + 1, // substitution
+          matrix[i][j - 1] + 1, // insertion
+          matrix[i - 1][j] + 1 // deletion
+        );
+      }
+    }
+  }
+
+  return matrix[b.length][a.length];
+}
+
+/**
+ * Find similar option names based on Levenshtein distance
+ *
+ * @param {string} unknownOption - The option name that was not recognized (e.g., "branch")
+ * @param {Object} yargsInstance - The yargs instance with defined options
+ * @param {number} maxSuggestions - Maximum number of suggestions to return (default: 3)
+ * @param {number} distanceThreshold - Maximum distance to consider for suggestions (default: 5)
+ * @returns {string[]} - Array of suggested option names, sorted by similarity
+ */
+export function findSimilarOptions(unknownOption, yargsInstance, maxSuggestions = 3, distanceThreshold = 5) {
+  // Remove leading dashes from the unknown option
+  const cleanUnknown = unknownOption.replace(/^-+/, '');
+
+  // Get all available options from yargs
+  const availableOptions = yargsInstance.getOptions();
+  const allOptions = new Set();
+
+  // Collect all option names (both long form and aliases)
+  if (availableOptions.key) {
+    // Ensure it's an array before iterating
+    const keys = Array.isArray(availableOptions.key) ? availableOptions.key : Object.keys(availableOptions.key || {});
+    keys.forEach(opt => {
+      allOptions.add(opt);
+    });
+  }
+
+  // Collect aliases
+  if (availableOptions.alias) {
+    Object.entries(availableOptions.alias).forEach(([key, aliases]) => {
+      allOptions.add(key);
+      if (Array.isArray(aliases)) {
+        aliases.forEach(alias => allOptions.add(alias));
+      } else if (aliases) {
+        // If it's not an array but exists, add it as a single alias
+        allOptions.add(String(aliases));
+      }
+    });
+  }
+
+  // Calculate distance for each option
+  const distances = [];
+  allOptions.forEach(option => {
+    const distance = calculateLevenshteinDistance(cleanUnknown, option);
+    if (distance <= distanceThreshold) {
+      // Calculate bonus score for substring matches
+      // If the unknown option is a substring of the valid option, it's likely what the user meant
+      // Check both directions: is unknown a substring of option, or is option a substring of unknown
+      const unknownInOption = option.includes(cleanUnknown);
+      const optionInUnknown = cleanUnknown.includes(option);
+
+      // Strong bonus for when user typed a word that appears in the option name
+      // e.g., "branch" appears in "base-branch"
+      const substringBonus = unknownInOption ? -10 : optionInUnknown ? -5 : 0;
+
+      // Also prioritize options with similar length (user likely tried to type the full name)
+      const lengthDiff = Math.abs(option.length - cleanUnknown.length);
+      const lengthBonus = lengthDiff < 3 ? -1 : 0;
+
+      distances.push({
+        option,
+        distance,
+        effectiveDistance: distance + substringBonus + lengthBonus,
+      });
+    }
+  });
+
+  // Sort by effective distance (closest first), then by actual distance
+  return distances
+    .sort((a, b) => {
+      if (a.effectiveDistance !== b.effectiveDistance) {
+        return a.effectiveDistance - b.effectiveDistance;
+      }
+      return a.distance - b.distance;
+    })
+    .slice(0, maxSuggestions)
+    .map(item => item.option);
+}
+
+/**
+ * Format suggestions into a user-friendly error message
+ *
+ * @param {string[]} suggestions - Array of suggested option names
+ * @returns {string} - Formatted suggestion message
+ */
+export function formatSuggestions(suggestions) {
+  if (suggestions.length === 0) {
+    return '';
+  }
+
+  if (suggestions.length === 1) {
+    return `\n\nDid you mean --${suggestions[0]}?`;
+  }
+
+  // For multiple suggestions, format them nicely
+  const formattedOptions = suggestions.map(opt => {
+    // If it's a single character, show as -x, otherwise --option-name
+    return opt.length === 1 ? `-${opt}` : `--${opt}`;
+  });
+
+  return `\n\nDid you mean one of these?\n${formattedOptions.map(opt => `  • ${opt}`).join('\n')}`;
+}
+
+/**
+ * Create an enhanced error message with suggestions for unknown arguments
+ *
+ * @param {string} originalError - The original error message from yargs
+ * @param {Object} yargsInstance - The yargs instance with defined options
+ * @returns {string} - Enhanced error message with suggestions
+ */
+export function enhanceErrorMessage(originalError, yargsInstance) {
+  // Extract the unknown option name from the error message
+  // Typical format: "Unknown argument: branch" or "Unknown arguments: branch, test"
+  const unknownMatch = originalError.match(/Unknown arguments?:\s*(.+?)(?:\s|$)/i);
+
+  if (!unknownMatch) {
+    return originalError;
+  }
+
+  // Get the first unknown argument (if multiple, focus on the first)
+  const unknownArgs = unknownMatch[1].split(',').map(arg => arg.trim());
+  const firstUnknown = unknownArgs[0];
+
+  // Find similar options
+  const suggestions = findSimilarOptions(firstUnknown, yargsInstance);
+
+  // Format the enhanced message
+  let enhancedMessage = originalError;
+
+  if (suggestions.length > 0) {
+    enhancedMessage += formatSuggestions(suggestions);
+  }
+
+  return enhancedMessage;
+}

package/src/solve.config.lib.mjs

@@ -7,6 +7,8 @@
 // Note: Strict options validation is now handled by yargs built-in .strict() mode (see below)
 // This approach was adopted per issue #482 feedback to minimize custom code maintenance
 
+import { enhanceErrorMessage } from './option-suggestions.lib.mjs';
+
 // Export an initialization function that accepts 'use'
 export const initializeConfig = async use => {
   // Import yargs with specific version for hideBin support
@@ -310,6 +312,7 @@ export const parseArguments = async (yargs, hideBin) => {
   // See: https://github.com/yargs/yargs/issues - .strict() only works with .parse()
 
   let argv;
+  let yargsInstance;
   try {
     // Suppress stderr output from yargs during parsing to prevent validation errors from appearing
     // This prevents "YError: Not enough arguments" from polluting stderr (issue #583)
@@ -330,7 +333,8 @@ export const parseArguments = async (yargs, hideBin) => {
     };
 
     try {
-      argv = await createYargsConfig(yargs()).parse(rawArgs);
+      yargsInstance = createYargsConfig(yargs());
+      argv = await yargsInstance.parse(rawArgs);
     } finally {
       // Always restore stderr.write
       process.stderr.write = originalStderrWrite;
@@ -345,9 +349,29 @@ export const parseArguments = async (yargs, hideBin) => {
     }
   } catch (error) {
     // Yargs throws errors for validation issues
-    // If the error is about unknown arguments (strict mode), re-throw it
-    if (error.message && error.message.includes('Unknown arguments')) {
-      throw error;
+    // If the error is about unknown arguments (strict mode), enhance it with suggestions
+    // Check if this error has already been enhanced to avoid re-processing
+    if (error.message && /Unknown argument/.test(error.message) && !error._enhanced) {
+      try {
+        // Enhance the error message with helpful suggestions
+        // Use the yargsInstance we already created, or create a new one if needed
+        const yargsWithConfig = yargsInstance || createYargsConfig(yargs());
+        const enhancedMessage = enhanceErrorMessage(error.message, yargsWithConfig);
+        const enhancedError = new Error(enhancedMessage);
+        enhancedError.name = error.name;
+        enhancedError._enhanced = true; // Mark as enhanced to prevent re-processing
+        throw enhancedError;
+      } catch (enhanceErr) {
+        // If enhancing fails, just throw the original error
+        if (global.verboseMode) {
+          console.error('[VERBOSE] Failed to enhance error message:', enhanceErr.message);
+        }
+        // If the enhance error itself is already enhanced, throw it
+        if (enhanceErr._enhanced) {
+          throw enhanceErr;
+        }
+        throw error;
+      }
     }
     // For other validation errors, show a warning in verbose mode
     if (error.message && global.verboseMode) {

package/src/solve.mjs

@@ -104,7 +104,16 @@ await log('🔧 Raw command executed:');
 await log(`   ${rawCommand}`);
 await log('');
 
-const argv = await parseArguments(yargs, hideBin);
+let argv;
+try {
+  argv = await parseArguments(yargs, hideBin);
+} catch (error) {
+  // Handle argument parsing errors with helpful messages
+  await log(`❌ ${error.message}`, { level: 'error' });
+  await log('', { level: 'error' });
+  await log('Use /help to see available options', { level: 'error' });
+  await safeExit(1, 'Invalid command-line arguments');
+}
 global.verboseMode = argv.verbose;
 
 // If user specified a custom log directory, we would need to move the log file

package/src/telegram-bot.mjs

@@ -47,7 +47,7 @@ const { validateModelName } = await import('./model-validation.lib.mjs');
 // Import libraries for /limits, /version, and markdown escaping
 const { formatUsageMessage, getAllCachedLimits } = await import('./limits.lib.mjs');
 const { getVersionInfo, formatVersionMessage } = await import('./version-info.lib.mjs');
-const { escapeMarkdown, escapeMarkdownV2 } = await import('./telegram-markdown.lib.mjs');
+const { escapeMarkdown, escapeMarkdownV2, cleanNonPrintableChars, makeSpecialCharsVisible } = await import('./telegram-markdown.lib.mjs');
 const { getSolveQueue, getRunningClaudeProcesses, createQueueExecuteCallback } = await import('./telegram-solve-queue.lib.mjs');
 
 const config = yargs(hideBin(process.argv))
@@ -643,10 +643,21 @@ function validateGitHubUrl(args, options = {}) {
   // Check if the URL type is allowed for this command
   if (!allowedTypes.includes(parsed.type)) {
     const allowedTypesStr = allowedTypes.map(t => (t === 'pull' ? 'pull request' : t)).join(', ');
-    return {
-      valid: false,
-      error: `URL must be a GitHub ${allowedTypesStr} (not ${parsed.type})`,
-    };
+    const baseUrl = `https://github.com/${parsed.owner}/${parsed.repo}`;
+
+    // Provide specific, helpful error messages based on the URL type
+    let error;
+    if (parsed.type === 'issues_list') {
+      error = `URL points to the issues list page, but you need a specific issue\n\n💡 How to fix:\n1. Open the repository: ${url}\n2. Click on a specific issue\n3. Copy the URL (it should end with /issues/NUMBER)\n\nExample: \`${baseUrl}/issues/1\``;
+    } else if (parsed.type === 'pulls_list') {
+      error = `URL points to the pull requests list page, but you need a specific pull request\n\n💡 How to fix:\n1. Open the repository: ${url}\n2. Click on a specific pull request\n3. Copy the URL (it should end with /pull/NUMBER)\n\nExample: \`${baseUrl}/pull/1\``;
+    } else if (parsed.type === 'repo') {
+      error = `URL points to a repository, but you need a specific ${allowedTypesStr}\n\n💡 How to fix:\n1. Go to: ${url}/issues\n2. Click on an issue to solve\n3. Use the full URL with the issue number\n\nExample: \`${baseUrl}/issues/1\``;
+    } else {
+      error = `URL must be a GitHub ${allowedTypesStr} (not ${parsed.type.replace('_', ' ')})`;
+    }
+
+    return { valid: false, error };
   }
 
   return { valid: true };
@@ -707,7 +718,7 @@ function extractGitHubUrl(text) {
     return { url: null, error: null, linkCount: 0 };
   }
 
-  // Split text into words and check each one
+  text = cleanNonPrintableChars(text); // Clean non-printable chars before processing
   const words = text.split(/\s+/);
   const foundUrls = [];
 
@@ -796,11 +807,12 @@ bot.command('help', async ctx => {
   message += '*/version* - Show bot and runtime versions\n';
   message += '*/help* - Show this help message\n\n';
   message += '⚠️ *Note:* /solve, /hive, /limits and /version commands only work in group chats.\n\n';
-  message += '🔧 *Available Options:*\n';
-  message += '• `--model <model>` - Specify AI model (sonnet, opus, haiku, haiku-3-5, haiku-3)\n';
+  message += '🔧 *Common Options:*\n';
+  message += '• `--model <model>` or `-m` - Specify AI model (sonnet, opus, haiku, haiku-3-5, haiku-3)\n';
+  message += '• `--base-branch <branch>` or `-b` - Target branch for PR (default: repo default branch)\n';
   message += '• `--think <level>` - Thinking level (low/medium/high/max)\n';
-  message += '• `--verbose` - Verbose output\n';
-  message += '• `--attach-logs` - Attach logs to PR\n';
+  message += '• `--verbose` or `-v` - Verbose output | `--attach-logs` - Attach logs to PR\n';
+  message += '\n💡 *Tip:* Many more options available. See full documentation for complete list.\n';
 
   if (allowedChats) {
     message += '\n🔒 *Restricted Mode:* This bot only accepts commands from authorized chats.\n';
@@ -1313,38 +1325,46 @@ bot.catch((error, ctx) => {
 
   // Try to notify the user about the error with more details
   if (ctx?.reply) {
-    // Build a more informative error message
-    let errorMessage = '❌ An error occurred while processing your request.\n\n';
-
-    // Add error type/name if available
-    if (error.name && error.name !== 'Error') {
-      errorMessage += `**Error type:** ${error.name}\n`;
-    }
-
-    // Add sanitized error message (avoid leaking sensitive info)
-    if (error.message) {
-      // Filter out potentially sensitive information
-      const sanitizedMessage = error.message
-        .replace(/token[s]?\s*[:=]\s*[\w-]+/gi, 'token: [REDACTED]')
-        .replace(/password[s]?\s*[:=]\s*[\w-]+/gi, 'password: [REDACTED]')
-        .replace(/api[_-]?key[s]?\s*[:=]\s*[\w-]+/gi, 'api_key: [REDACTED]');
-
-      errorMessage += `**Details:** ${sanitizedMessage}\n`;
-    }
-
-    errorMessage += '\n💡 **Troubleshooting:**\n';
-    errorMessage += '• Try running the command again\n';
-    errorMessage += '• Check if all required parameters are correct\n';
-    errorMessage += '• If the issue persists, contact support with the error details above\n';
-
-    if (VERBOSE) {
-      errorMessage += `\n🔍 **Debug info:** Update ID: ${ctx.update.update_id}`;
+    // Detect if this is a Telegram API parsing error
+    const isTelegramParsingError = error.message && (error.message.includes("can't parse entities") || error.message.includes("Can't parse entities") || error.message.includes("can't find end of") || (error.message.includes('Bad Request') && error.message.includes('400')));
+
+    let errorMessage;
+
+    if (isTelegramParsingError) {
+      // Special handling for Telegram API parsing errors caused by unescaped special characters
+      errorMessage = `❌ A message formatting error occurred.\n\n💡 This usually means there was a problem with special characters in the response.\nPlease try your command again with a different URL or contact support.`;
+      // Show the user's input with special characters visible (if available)
+      if (ctx.message?.text) {
+        const cleanedInput = cleanNonPrintableChars(ctx.message.text);
+        const visibleInput = makeSpecialCharsVisible(cleanedInput, { maxLength: 150 });
+        if (visibleInput !== cleanedInput) errorMessage += `\n\n📝 Your input (with special chars visible):\n\`${escapeMarkdown(visibleInput)}\``;
+      }
+      if (VERBOSE) {
+        const escapedError = escapeMarkdown(error.message || 'Unknown error');
+        errorMessage += `\n\n🔍 Debug info: ${escapedError}\nUpdate ID: ${ctx.update.update_id}`;
+      }
+    } else {
+      // Build informative error message for other errors
+      errorMessage = '❌ An error occurred while processing your request.\n\n';
+      if (error.message) {
+        // Filter out sensitive info and escape markdown
+        const sanitizedMessage = escapeMarkdown(
+          error.message
+            .replace(/token[s]?\s*[:=]\s*[\w-]+/gi, 'token: [REDACTED]')
+            .replace(/password[s]?\s*[:=]\s*[\w-]+/gi, 'password: [REDACTED]')
+            .replace(/api[_-]?key[s]?\s*[:=]\s*[\w-]+/gi, 'api_key: [REDACTED]')
+        );
+        errorMessage += `Details: ${sanitizedMessage}\n`;
+      }
+      errorMessage += '\n💡 Troubleshooting:\n• Try running the command again\n• Check if all required parameters are correct\n• Use /help to see command examples\n• If the issue persists, contact support with the error details above';
+      if (VERBOSE) errorMessage += `\n\n🔍 Debug info: Update ID: ${ctx.update.update_id}`;
     }
 
     ctx.reply(errorMessage, { parse_mode: 'Markdown' }).catch(replyError => {
       console.error('Failed to send error message to user:', replyError);
       // Try sending a simple text message without Markdown if Markdown parsing failed
-      ctx.reply('❌ An error occurred while processing your request. Please try again or contact support.').catch(fallbackError => {
+      const plainMessage = `An error occurred while processing your request. Please try again or contact support.\n\nError: ${error.message || 'Unknown error'}`;
+      ctx.reply(plainMessage).catch(fallbackError => {
         console.error('Failed to send fallback error message:', fallbackError);
       });
     });
@@ -1361,29 +1381,17 @@ if (allowedChats && allowedChats.length > 0) {
 } else {
   console.log('Allowed chats: All (no restrictions)');
 }
-console.log('Commands enabled:', {
-  solve: solveEnabled,
-  hive: hiveEnabled,
-});
-if (solveOverrides.length > 0) {
-  console.log('Solve overrides (lino):', lino.format(solveOverrides));
-}
-if (hiveOverrides.length > 0) {
-  console.log('Hive overrides (lino):', lino.format(hiveOverrides));
-}
+console.log('Commands enabled:', { solve: solveEnabled, hive: hiveEnabled });
+if (solveOverrides.length > 0) console.log('Solve overrides (lino):', lino.format(solveOverrides));
+if (hiveOverrides.length > 0) console.log('Hive overrides (lino):', lino.format(hiveOverrides));
 if (VERBOSE) {
   console.log('[VERBOSE] Verbose logging enabled');
   console.log('[VERBOSE] Bot start time (Unix):', BOT_START_TIME);
   console.log('[VERBOSE] Bot start time (ISO):', new Date(BOT_START_TIME * 1000).toISOString());
 }
 
-// Delete any existing webhook before starting polling
-// This is critical because a webhook prevents polling from working
-// If the bot was previously configured with a webhook (or if one exists),
-// we must delete it to allow polling mode to receive messages
-if (VERBOSE) {
-  console.log('[VERBOSE] Deleting webhook...');
-}
+// Delete existing webhook (critical: webhooks prevent polling from working)
+if (VERBOSE) console.log('[VERBOSE] Deleting webhook...');
 bot.telegram
   .deleteWebhook({ drop_pending_updates: true })
   .then(result => {
@@ -1398,19 +1406,12 @@ bot.telegram
       });
     }
     return bot.launch({
-      // Receive message updates (commands, text messages) and callback queries (button clicks)
-      // This ensures the bot receives all message types including commands and button interactions
-      allowedUpdates: ['message', 'callback_query'],
-      // Drop any pending updates that were sent before the bot started
-      // This ensures we only process new messages sent after this bot instance started
-      dropPendingUpdates: true,
+      allowedUpdates: ['message', 'callback_query'], // Receive messages and callback queries
+      dropPendingUpdates: true, // Drop pending updates sent before bot started
     });
   })
   .then(async () => {
-    // Check if shutdown was initiated before printing success messages
-    if (isShuttingDown) {
-      return; // Skip success messages if shutting down
-    }
+    if (isShuttingDown) return; // Skip success messages if shutting down
 
     console.log('✅ SwarmMindBot is now running!');
     console.log('Press Ctrl+C to stop');

package/src/telegram-markdown.lib.mjs

@@ -62,3 +62,79 @@ export function escapeMarkdownV2(text, options = {}) {
 
   return parts.join('');
 }
+
+/**
+ * Clean non-printable and problematic Unicode characters from text.
+ * Removes zero-width characters, control characters, and other invisible/problematic sequences.
+ * @param {string} text - Text to clean
+ * @returns {string} Cleaned text
+ */
+export function cleanNonPrintableChars(text) {
+  if (!text || typeof text !== 'string') return text;
+
+  return (
+    text
+      // Remove zero-width characters
+      .replace(/[\u200B-\u200D\uFEFF]/g, '')
+      // Remove other non-printable control characters (except newline, tab, carriage return)
+      // eslint-disable-next-line no-control-regex
+      .replace(/[\u0000-\u0008\u000B-\u000C\u000E-\u001F\u007F-\u009F]/g, '')
+      // Remove soft hyphens
+      .replace(/\u00AD/g, '')
+      // Normalize whitespace (replace multiple spaces with single space)
+      .replace(/[ \t]+/g, ' ')
+      // Trim leading/trailing whitespace from each line
+      .split('\n')
+      .map(line => line.trim())
+      .join('\n')
+      .trim()
+  );
+}
+
+/**
+ * Make special characters visible for debugging purposes.
+ * Replaces special characters with their Unicode escape sequences or names.
+ * Useful for showing users where problematic characters are in their input.
+ * @param {string} text - Text to make visible
+ * @param {Object} options - Configuration options
+ * @param {number} options.maxLength - Maximum length of output (default: 200)
+ * @returns {string} Text with special characters made visible
+ */
+export function makeSpecialCharsVisible(text, options = {}) {
+  if (!text || typeof text !== 'string') return text;
+
+  const { maxLength = 200 } = options;
+
+  // Map of special characters to their visible representations
+  const specialChars = {
+    '\u200B': '[ZWSP]', // Zero-width space
+    '\u200C': '[ZWNJ]', // Zero-width non-joiner
+    '\u200D': '[ZWJ]', // Zero-width joiner
+    '\uFEFF': '[BOM]', // Byte order mark / zero-width no-break space
+    '\u00AD': '[SHY]', // Soft hyphen
+    '\t': '[TAB]',
+    '\r': '[CR]',
+    '\n': '[LF]',
+  };
+
+  let result = '';
+  for (let i = 0; i < text.length && result.length < maxLength; i++) {
+    const char = text[i];
+    const code = char.charCodeAt(0);
+
+    if (specialChars[char]) {
+      result += specialChars[char];
+    } else if (code < 32 || (code >= 127 && code < 160)) {
+      // Control characters
+      result += `[U+${code.toString(16).toUpperCase().padStart(4, '0')}]`;
+    } else {
+      result += char;
+    }
+  }
+
+  if (text.length > maxLength) {
+    result += '... (truncated)';
+  }
+
+  return result;
+}