i18ntk 1.10.2 → 2.0.3

package/main/manage/commands/ScannerCommand.js

@@ -0,0 +1,654 @@
+#!/usr/bin/env node
+
+/**
+ * I18NTK SCANNER COMMAND
+ *
+ * Handles scanning functionality for translation keys.
+ * Contains embedded business logic from I18nTextScanner.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { getUnifiedConfig, displayHelp } = require('../../../utils/config-helper');
+const { loadTranslations } = require('../../../utils/i18n-helper');
+const SecurityUtils = require('../../../utils/security');
+const SetupEnforcer = require('../../../utils/setup-enforcer');
+
+class ScannerCommand {
+    constructor(config = {}, ui = null) {
+        this.config = config;
+        this.ui = ui;
+        this.prompt = null;
+        this.isNonInteractiveMode = false;
+        this.safeClose = null;
+
+        // Initialize scanner properties
+        this.sourceDir = null;
+        this.patterns = [];
+        this.exclusions = [];
+        this.locale = this.loadLocale();
+        this.results = [];
+        this.framework = null;
+    }
+
+    /**
+     * Set runtime dependencies for interactive operations
+     */
+    setRuntimeDependencies(prompt, isNonInteractiveMode, safeClose) {
+        this.prompt = prompt;
+        this.isNonInteractiveMode = isNonInteractiveMode;
+        this.safeClose = safeClose;
+    }
+
+    loadLocale() {
+        const uiLocalesDir = path.join(__dirname, '../../../resources', 'i18n', 'ui-locales');
+        const localeFile = path.join(uiLocalesDir, 'en.json');
+
+        try {
+            const localeContent = SecurityUtils.safeReadFileSync(localeFile, uiLocalesDir, 'utf8');
+            return SecurityUtils.safeParseJSON(localeContent);
+        } catch (error) {
+            return {
+                scanner: {
+                    help_options: {
+                        source_dir: "Source directory to scan (default: ./src)",
+                        framework: "Framework type: react, vue, angular, vanilla (auto-detected)",
+                        patterns: "Custom patterns to match (comma-separated)",
+                        exclude: "Exclude patterns (comma-separated)",
+                        output_report: "Generate detailed report",
+                        output_dir: "Report output directory (default: ./reports)",
+                        min_length: "Minimum text length to consider (default: 3)",
+                        max_length: "Maximum text length to consider (default: 100)",
+                        include_tests: "Include test files in scan"
+                    },
+                    starting: "🔍 Starting text analysis for {framework} project...",
+                    sourceDirectory: "📁 Source directory: {sourceDir}",
+                    framework: "🏗️ Framework: {framework}",
+                    scanningFiles: "📊 Scanning {count} files...",
+                    foundText: "📝 Found {count} potential hardcoded text instances",
+                    reportGenerated: "📊 Report generated: {path}",
+                    noTextFound: "✅ No hardcoded text found!",
+                    analysisTitle: "🔍 TEXT ANALYSIS RESULTS",
+                    summary: {
+                        totalFiles: "📄 Total files scanned: {count}",
+                        textInstances: "📝 Text instances found: {count}",
+                        filesWithText: "📂 Files with hardcoded text: {count}",
+                        framework: "🏗️ Framework detected: {framework}"
+                    }
+                }
+            };
+        }
+    }
+
+    t(key, params = {}) {
+        const keyStr = String(key || '');
+        const keys = keyStr.split('.');
+        let value = this.locale;
+
+        for (const k of keys) {
+            value = value?.[k];
+            if (value === undefined) break;
+        }
+
+        if (typeof value !== 'string') {
+            return key;
+        }
+
+        return value.replace(/\{([^}]+)\}/g, (match, param) => {
+            return params[param] !== undefined ? params[param] : match;
+        });
+    }
+
+    parseArgs() {
+        const args = process.argv.slice(2);
+        const parsed = {};
+
+        args.forEach(arg => {
+            if (arg.startsWith('--')) {
+                const [key, ...valueParts] = arg.substring(2).split('=');
+                const value = valueParts.join('=');
+
+                switch (key) {
+                    case 'source-dir':
+                        parsed.sourceDir = value || '';
+                        break;
+                    case 'framework':
+                        parsed.framework = value || '';
+                        break;
+                    case 'patterns':
+                        parsed.patterns = value ? value.split(',').map(p => p.trim()).filter(Boolean) : [];
+                        break;
+                    case 'exclude':
+                        parsed.exclude = value ? value.split(',').map(e => e.trim()).filter(Boolean) : [];
+                        break;
+                    case 'output-dir':
+                        parsed.outputDir = value || '';
+                        break;
+                    case 'min-length':
+                        parsed.minLength = parseInt(value) || 3;
+                        break;
+                    case 'max-length':
+                        parsed.maxLength = parseInt(value) || 100;
+                        break;
+                    case 'output-report':
+                        parsed.outputReport = true;
+                        break;
+                    case 'include-tests':
+                        parsed.includeTests = true;
+                        break;
+                    case 'help':
+                    case 'h':
+                        parsed.help = true;
+                        break;
+                }
+            }
+        });
+
+        return parsed;
+    }
+
+    detectFramework(projectRoot) {
+        const packagePath = path.join(projectRoot, 'package.json');
+
+        // Check for Python frameworks
+        const requirementsPath = path.join(projectRoot, 'requirements.txt');
+        const setupPath = path.join(projectRoot, 'setup.py');
+        const pyprojectPath = path.join(projectRoot, 'pyproject.toml');
+
+        try {
+            // Check Python frameworks first
+            if (SecurityUtils.safeExistsSync(requirementsPath, projectRoot)) {
+                const requirements = SecurityUtils.safeReadFileSync(requirementsPath, projectRoot, 'utf8');
+                if (requirements.includes('Django')) return 'django';
+                if (requirements.includes('Flask') || requirements.includes('flask-babel')) return 'flask';
+            }
+
+            if (SecurityUtils.safeExistsSync(setupPath, projectRoot)) {
+                const setup = SecurityUtils.safeReadFileSync(setupPath, projectRoot, 'utf8');
+                if (setup.includes('Django')) return 'django';
+                if (setup.includes('Flask')) return 'flask';
+            }
+
+            if (SecurityUtils.safeExistsSync(pyprojectPath, projectRoot)) {
+                const pyproject = SecurityUtils.safeReadFileSync(pyprojectPath, projectRoot, 'utf8');
+                if (pyproject.includes('Django')) return 'django';
+                if (pyproject.includes('Flask')) return 'flask';
+            }
+
+            // Check for Python files
+            const hasPythonFiles = fs.readdirSync(projectRoot, { recursive: true })
+                .some(file => file.endsWith && file.endsWith('.py'));
+            if (hasPythonFiles) return 'python';
+        } catch (error) {
+            // Continue to JS frameworks
+        }
+
+        try {
+            const packageJsonContent = SecurityUtils.safeReadFileSync(packagePath, projectRoot, 'utf8');
+            const packageJson = SecurityUtils.safeParseJSON(packageJsonContent);
+            const deps = { ...packageJson.dependencies, ...packageJson.devDependencies };
+
+            if (deps.react || deps['react-dom']) return 'react';
+            if (deps.vue || deps['vue-router']) return 'vue';
+            if (deps['@angular/core'] || deps.angular) return 'angular';
+            if (deps.next) return 'next';
+            if (deps.svelte) return 'svelte';
+
+            return 'vanilla';
+        } catch (error) {
+            return 'vanilla';
+        }
+    }
+
+    getFrameworkPatterns(framework) {
+        const basePatterns = [
+            // String literals in JSX/TSX - enhanced for Unicode
+            /(?<![\w])["'`]([^"'`]{2,99})["'`]/g,
+            // Template literals - enhanced for Unicode
+            /`([^`]{2,99})`/g,
+            // Text content in HTML - enhanced for Unicode
+            />([^<]{2,99})</g,
+            // Title attributes - enhanced for Unicode
+            /title=["']([^"']{2,99})["']/g,
+            // Alt attributes - enhanced for Unicode
+            /alt=["']([^"']{2,99})["']/g,
+            // Placeholder attributes - enhanced for Unicode
+            /placeholder=["']([^"']{2,99})["']/g
+        ];
+
+        const frameworkSpecific = {
+            react: [
+                // React specific patterns - enhanced for i18next detection
+                /children:\s*["']([^"']{2,99})["']/g,
+                /dangerouslySetInnerHTML={{\s*__html:\s*["']([^"']{2,99})["']/g,
+                // JSX text content without translation
+                />([^<{][^<>{]*[^}>])</g,
+                // Button text
+                /<button[^>]*>([^<]{2,99})<\/button>/g,
+                // Span text
+                /<span[^>]*>([^<]{2,99})<\/span>/g
+            ],
+            vue: [
+                // Vue specific patterns - enhanced for vue-i18n detection
+                /v-text=["']([^"']{2,99})["']/g,
+                /v-html=["']([^"']{2,99})["']/g,
+                // Vue template text
+                />([^<{][^<>{]*[^}>])</g,
+                // Button text
+                /<button[^>]*>([^<]{2,99})<\/button>/g,
+                // Span text
+                /<span[^>]*>([^<]{2,99})<\/span>/g
+            ],
+            angular: [
+                // Angular specific patterns - enhanced for ngx-translate detection
+                /\[innerHTML\]=["']([^"']{2,99})["']/g,
+                /\[textContent\]=["']([^"']{2,99})["']/g,
+                // Angular template text
+                />([^<{][^<>{]*[^}>])</g,
+                // Button text
+                /<button[^>]*>([^<]{2,99})<\/button>/g,
+                // Span text
+                /<span[^>]*>([^<]{2,99})<\/span>/g
+            ],
+            django: [
+                // Django template patterns
+                /\{\%\s*trans\s+["']([^"']{2,99})["']\s*%\}/g,
+                /\{\%\s*blocktrans\s*%\}([^%]{2,99})\{\%\s*endblocktrans\s*%\}/g,
+                /{{\s*_["']([^"']{2,99})["']\s*}}/g,
+                /{{\s*gettext\(["']([^"']{2,99})["']\)\s*}}/g
+            ],
+            flask: [
+                // Flask/Jinja2 template patterns
+                /\{\{\s*_["']([^"']{2,99})["']\s*}}/g,
+                /\{\{\s*gettext\(["']([^"']{2,99})["']\)\s*}}/g,
+                /\{\{\s*lazy_gettext\(["']([^"']{2,99})["']\)\s*}}/g
+            ],
+            python: [
+                // Python source patterns
+                /gettext\(["']([^"']{2,99})["']\)/g,
+                /_\(["']([^"']{2,99})["']\)/g,
+                /gettext_lazy\(["']([^"']{2,99})["']\)/g,
+                /lazy_gettext\(["']([^"']{2,99})["']\)/g
+            ]
+        };
+
+        return [...basePatterns, ...(frameworkSpecific[framework] || [])];
+    }
+
+    shouldExcludeFile(filePath, exclusions) {
+        const fileName = path.basename(filePath);
+        return exclusions.some(pattern => {
+            if (pattern.includes('*')) {
+                const regex = new RegExp(pattern.replace(/\*/g, '.*'));
+                return regex.test(fileName) || regex.test(filePath);
+            }
+            return fileName.includes(pattern) || filePath.includes(pattern);
+        });
+    }
+
+    isEnglishText(text) {
+        // Enhanced text detection for Unicode and multilingual support
+        const trimmed = text.trim();
+        if (trimmed.length < 3) return false;
+
+        // Skip if it's just numbers or special characters
+        if (/^\d+$/.test(trimmed)) return false;
+        if (/^[!@#$%^&*()_+\-=\[\]{};':"\\|,.<>?]+$/.test(trimmed)) return false;
+
+        // Allow Unicode characters including CJK, Cyrillic, etc.
+        const validChars = trimmed.match(/[\p{L}\p{N}\s\-,.!?':"()\[\]{}]/gu) || [];
+        const validRatio = validChars.length / trimmed.length;
+
+        // Must have at least 50% valid characters and some alphabetic characters
+        const hasAlpha = /[a-zA-Z\u00C0-\u024F\u1E00-\u1EFF\u0400-\u04FF\u4E00-\u9FFF\uAC00-\uD7AF]/u.test(trimmed);
+
+        return validRatio >= 0.5 && hasAlpha;
+    }
+
+    scanFile(filePath, patterns, minLength, maxLength) {
+        try {
+            const content = SecurityUtils.safeReadFileSync(filePath, path.dirname(filePath), 'utf8');
+            const lines = content.split('\n');
+            const results = [];
+
+            patterns.forEach(pattern => {
+                let match;
+                while ((match = pattern.exec(content)) !== null) {
+                    const text = match[1] || match[0];
+
+                    // Skip translation function calls
+                    const beforeMatch = content.substring(Math.max(0, match.index - 20), match.index);
+                    if (beforeMatch.includes('t(') || beforeMatch.includes('i18next.t(') ||
+                        beforeMatch.includes('$t(') || beforeMatch.includes('translate(')) {
+                        continue;
+                    }
+
+                    if (text && this.isEnglishText(text) &&
+                        text.length >= minLength && text.length <= maxLength) {
+
+                        const lineNumber = content.substring(0, match.index).split('\n').length;
+                        const lineContent = lines[lineNumber - 1] || '';
+
+                        results.push({
+                            text: text.trim(),
+                            line: lineNumber,
+                            column: match.index - content.lastIndexOf('\n', match.index),
+                            context: lineContent.trim(),
+                            pattern: pattern.toString(),
+                            suggestion: this.generateSuggestion(text)
+                        });
+                    }
+                }
+            });
+
+            return results;
+        } catch (error) {
+            console.warn(`Warning: Could not read file ${filePath}: ${error.message}`);
+            return [];
+        }
+    }
+
+    generateSuggestion(text) {
+        const key = text.toLowerCase()
+            .replace(/[^a-z0-9\s]/g, '')
+            .replace(/\s+/g, '_')
+            .substring(0, 50);
+
+        return {
+            key: `ui.${key}`,
+            original: text,
+            translationKey: `t('ui.${key}')`,
+            frameworkSpecific: this.getFrameworkSpecific(text)
+        };
+    }
+
+    getFrameworkSpecific(text) {
+        const frameworks = {
+            react: {
+                hook: `const { t } = useTranslation();`,
+                usage: `{t('ui.${text.toLowerCase().replace(/[^a-z0-9\s]/g, '').replace(/\s+/g, '_')}')}`,
+                component: `<Trans i18nKey="ui.${text.toLowerCase().replace(/[^a-z0-9\s]/g, '').replace(/\s+/g, '_')}">${text}</Trans>`
+            },
+            vue: {
+                directive: `{{ $t('ui.${text.toLowerCase().replace(/[^a-z0-9\s]/g, '').replace(/\s+/g, '_')}') }}`,
+                method: `this.$t('ui.${text.toLowerCase().replace(/[^a-z0-9\s]/g, '').replace(/\s+/g, '_')}')`
+            },
+            angular: {
+                pipe: `{{ '${text}' | translate }}`,
+                service: `this.translateService.instant('ui.${text.toLowerCase().replace(/[^a-z0-9\s]/g, '').replace(/\s+/g, '_')}')`
+            },
+            django: {
+                template: `{% trans '${text}' %}`,
+                python: `from django.utils.translation import gettext as _\n_('${text}')`,
+                model: `from django.utils.translation import gettext_lazy as _\n_('${text}')`
+            },
+            flask: {
+                template: `{{ _('${text}') }}`,
+                python: `from flask_babel import gettext as _\n_('${text}')`,
+                lazy: `from flask_babel import lazy_gettext as _\n_('${text}')`
+            },
+            python: {
+                gettext: `import gettext\ngettext.gettext('${text}')`,
+                underscore: `from gettext import gettext as _\n_('${text}')`,
+                lazy: `from gettext import gettext_lazy as _\n_('${text}')`
+            }
+        };
+
+        return frameworks[this.framework] || frameworks.vanilla;
+    }
+
+    async scanDirectory(dir, options = {}) {
+        const {
+            patterns = [],
+            exclusions = [],
+            minLength = 3,
+            maxLength = 100,
+            includeTests = false
+        } = options;
+
+        if (!SecurityUtils.safeExistsSync(dir, path.dirname(dir))) {
+            throw new Error(`Directory does not exist: ${dir}`);
+        }
+
+        const allResults = [];
+        const extensions = ['.js', '.jsx', '.ts', '.tsx', '.vue', '.html', '.svelte', '.py', '.pyx', '.pyi'];
+
+        const scanRecursive = (currentDir) => {
+            const items = fs.readdirSync(currentDir);
+
+            for (const item of items) {
+                const fullPath = path.join(currentDir, item);
+                const stat = fs.statSync(fullPath);
+
+                if (stat.isDirectory()) {
+                    if (!item.startsWith('.') && !this.shouldExcludeFile(fullPath, exclusions)) {
+                        scanRecursive(fullPath);
+                    }
+                } else if (stat.isFile()) {
+                    const ext = path.extname(item);
+                    if (extensions.includes(ext) && !this.shouldExcludeFile(fullPath, exclusions)) {
+                        if (!includeTests && (item.includes('.test.') || item.includes('.spec.'))) {
+                            continue;
+                        }
+
+                        const results = this.scanFile(fullPath, patterns, minLength, maxLength);
+                        if (results.length > 0) {
+                            allResults.push({
+                                file: fullPath,
+                                results
+                            });
+                        }
+                    }
+                }
+            }
+        };
+
+        scanRecursive(dir);
+        return allResults;
+    }
+
+    async generateReport(results, outputDir) {
+        if (!SecurityUtils.safeExistsSync(outputDir, path.dirname(outputDir))) {
+            fs.mkdirSync(outputDir, { recursive: true });
+        }
+
+        const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+        const reportFile = path.join(outputDir, `text-analysis-${timestamp}.json`);
+        const summaryFile = path.join(outputDir, `text-analysis-${timestamp}.md`);
+
+        const summary = {
+            totalFiles: results.length,
+            totalInstances: results.reduce((sum, file) => sum + file.results.length, 0),
+            filesWithText: results.length,
+            framework: this.framework,
+            timestamp: new Date().toISOString(),
+            results
+        };
+
+        // JSON report
+        SecurityUtils.safeWriteFileSync(reportFile, JSON.stringify(summary, null, 2), outputDir);
+
+        // Markdown summary
+        const mdContent = this.generateMarkdownReport(summary);
+        SecurityUtils.safeWriteFileSync(summaryFile, mdContent, outputDir);
+
+        return { reportFile, summaryFile, summary };
+    }
+
+    generateMarkdownReport(summary) {
+        let content = `# Text Analysis Report
+
+**Framework:** ${summary.framework}
+**Total Files Scanned:** ${summary.totalFiles}
+**Text Instances Found:** ${summary.totalInstances}
+**Files with Hardcoded Text:** ${summary.filesWithText}
+**Generated:** ${summary.timestamp}
+
+## Summary
+
+| Metric | Count |
+|--------|-------|
+| Total Files | ${summary.totalFiles} |
+| Text Instances | ${summary.totalInstances} |
+| Files with Text | ${summary.filesWithText} |
+
+## Files with Hardcoded Text
+
+`;
+
+        summary.results.forEach(file => {
+            content += `### ${file.file}
+
+| Text | Line | Suggestion |
+|------|------|------------|
+`;
+            file.results.forEach(result => {
+                const suggestion = result.suggestion;
+                content += `| "${result.text}" | ${result.line} | \`${suggestion.translationKey}\` |
+`;
+            });
+            content += '\n';
+        });
+
+        content += `
+## Recommendations
+
+1. **Create Translation Keys**: Add the suggested keys to your translation files
+2. **Replace Text**: Replace hardcoded text with the suggested translation patterns
+3. **Test Changes**: Verify translations work correctly in your application
+4. **Update Framework**: Ensure your i18n framework is properly configured
+
+## Next Steps
+
+- Run \`i18ntk init\` to set up translation infrastructure if needed
+- Use \`i18ntk fixer\` to fix any placeholder translations
+- Run \`i18ntk validate\` to ensure all translations are properly configured
+`;
+
+        return content;
+    }
+
+    async initialize() {
+        const args = this.parseArgs();
+        if (args.help) {
+            displayHelp('i18ntk-scanner', {
+                'source-dir': this.t('scanner.help_options.source_dir'),
+                'framework': this.t('scanner.help_options.framework'),
+                'patterns': this.t('scanner.help_options.patterns'),
+                'exclude': this.t('scanner.help_options.exclude'),
+                'output-report': this.t('scanner.help_options.output_report'),
+                'output-dir': this.t('scanner.help_options.output_dir'),
+                'min-length': this.t('scanner.help_options.min_length'),
+                'max-length': this.t('scanner.help_options.max_length'),
+                'include-tests': this.t('scanner.help_options.include_tests')
+            });
+            process.exit(0);
+        }
+
+        const baseConfig = await getUnifiedConfig('scanner', args);
+        this.config = { ...baseConfig, ...(this.config || {}) };
+
+        this.sourceDir = this.config.sourceDir || './src';
+
+        // Resolve framework with precedence: CLI arg > config.framework.preference|string > auto-detect > fallback
+        const cliFramework = args.framework;
+        const cfgFramework = this.config.framework;
+        const fwPref = typeof cfgFramework === 'string' ? cfgFramework : (cfgFramework?.preference || 'auto');
+        const fwDetectEnabled = typeof cfgFramework === 'object' ? (cfgFramework.detect !== false) : true;
+        const fwFallback = typeof cfgFramework === 'object' ? (cfgFramework.fallback || 'vanilla') : 'vanilla';
+
+        if (cliFramework && typeof cliFramework === 'string') {
+            this.framework = cliFramework;
+        } else if (fwPref && fwPref !== 'auto') {
+            this.framework = fwPref;
+        } else if (fwDetectEnabled) {
+            const detected = this.detectFramework(process.cwd());
+            this.framework = detected || fwFallback;
+        } else {
+            this.framework = fwFallback;
+        }
+
+        // Validate source directory
+        if (!SecurityUtils.safeExistsSync(this.sourceDir, path.dirname(this.sourceDir))) {
+            console.error(`❌ Source directory does not exist: ${this.sourceDir}`);
+            process.exit(1);
+        }
+
+        const validatedPath = SecurityUtils.validatePath(this.sourceDir);
+        if (!validatedPath) {
+            console.error(`❌ Security validation failed: Path validation returned null`);
+            process.exit(1);
+        }
+        this.sourceDir = validatedPath;
+
+        return this;
+    }
+
+    async run() {
+        console.log(this.t('scanner.starting', { framework: this.framework }));
+        console.log(this.t('scanner.sourceDirectory', { sourceDir: this.sourceDir }));
+
+        const patterns = this.getFrameworkPatterns(this.framework);
+        const exclusions = this.config.exclude || ['node_modules', '.git', 'dist', 'build'];
+        const minLength = this.config.minLength || 3;
+        const maxLength = this.config.maxLength || 100;
+        const includeTests = this.config.includeTests || false;
+
+        try {
+            const results = await this.scanDirectory(this.sourceDir, {
+                patterns,
+                exclusions,
+                minLength,
+                maxLength,
+                includeTests
+            });
+
+            console.log(this.t('scanner.foundText', { count: results.reduce((sum, file) => sum + file.results.length, 0) }));
+
+            if (results.length > 0 && this.config.outputReport) {
+                const outputDir = this.config.outputDir || './reports';
+                const { reportFile, summaryFile } = await this.generateReport(results, outputDir);
+                console.log(this.t('scanner.reportGenerated', { path: summaryFile }));
+            }
+
+            return results;
+        } catch (error) {
+            console.error(`❌ Error during scanning: ${error.message}`);
+            process.exit(1);
+        }
+    }
+
+    /**
+     * Execute the scanner command
+     */
+    async execute(options = {}) {
+        try {
+            await this.initialize();
+            await this.run();
+            return { success: true, command: 'scanner' };
+        } catch (error) {
+            console.error(`Scanner command failed: ${error.message}`);
+            throw error;
+        }
+    }
+
+    /**
+     * Get command metadata
+     */
+    getMetadata() {
+        return {
+            name: 'scanner',
+            description: 'Scan for translation keys in source code',
+            category: 'analysis',
+            aliases: [],
+            usage: 'scanner [options]',
+            examples: [
+                'scanner',
+                'scanner --source-dir=./src',
+                'scanner --output-dir=./reports'
+            ]
+        };
+    }
+}
+
+module.exports = ScannerCommand;