kotoshu 0.3.0

data/lib/kotoshu/commands/check_command.rb

@@ -0,0 +1,312 @@
+# frozen_string_literal: true
+
+require_relative '../documents/document'
+require_relative '../analyzers/semantic_analyzer'
+require_relative '../models/fasttext_model'
+require_relative '../cache/model_cache'
+require_relative '../cli/interactive_reviewer'
+require_relative '../cli/batch_reporter'
+require_relative '../language/identifier'
+
+module Kotoshu
+  class CheckCommand < Thor
+    namespace :check
+
+    class_option :language, aliases: '-l', type: :string, default: 'auto',
+             desc: 'Language code (auto, de, en, es, fr, pt, ru)'
+    class_option :interactive, aliases: '-i', type: :boolean, default: false,
+             desc: 'Interactive mode for error review'
+    class_option :output, aliases: '-o', type: :string,
+             desc: 'Output file path (for batch mode)'
+    class_option :format, type: :string, enum: %w[text json yaml csv sarif], default: 'text',
+             desc: 'Output format (text, json, yaml, csv, sarif)'
+    class_option :model, type: :string, enum: %w[fasttext hunspell], default: 'hunspell',
+             desc: 'Analysis model (fasttext, hunspell)'
+    class_option :download, type: :boolean, default: true,
+             desc: 'Automatically download models if missing'
+    class_option :verbose, aliases: '-v', type: :boolean, default: false,
+             desc: 'Verbose output'
+
+    desc 'check FILE', 'Check spelling/grammar in a file'
+    def check(file)
+      # Validate file exists
+      unless File.exist?(file)
+        puts "Error: File not found: #{file}"
+        exit 1
+      end
+
+      # Detect language if auto
+      language = detect_language(file, options[:language])
+
+      # Load document
+      document = load_document(file, language)
+
+      # Load analyzer based on model type
+      analyzer = load_analyzer(language, options[:model])
+
+      puts "Analyzing #{file} (language: #{language})..." if options[:verbose]
+
+      # Run interactive or batch mode
+      if options[:interactive]
+        run_interactive_mode(document, analyzer)
+      else
+        run_batch_mode(document, analyzer)
+      end
+    end
+
+    desc 'string TEXT', 'Check spelling/grammar in a text string'
+    option :format, type: :string, enum: %w[text markdown], default: 'text',
+             desc: 'Text format (text, markdown)'
+    def string(text)
+      language_code = options[:language]
+
+      # Create document from string
+      format_sym = options[:format].to_sym
+      document = Documents::Document.from_string(text, language_code: language_code)
+
+      # Load analyzer
+      analyzer = load_analyzer(language_code, options[:model])
+
+      puts "Analyzing..." if options[:verbose]
+
+      # Always use batch mode for string input
+      reporter = run_batch_mode(document, analyzer)
+
+      # Print report
+      reporter.print(format: options[:format].to_sym)
+
+      # Exit with appropriate code
+      exit reporter.exit_code
+    end
+
+    desc 'stdin', 'Check spelling/grammar from stdin'
+    option :format, type: :string, enum: %w[text markdown], default: 'text',
+             desc: 'Text format (text, markdown)'
+    def stdin
+      text = $stdin.read
+
+      if text.nil? || text.empty?
+        puts "Error: No input provided"
+        exit 1
+      end
+
+      # Delegate to string command
+      invoke :string, [text], options
+    end
+
+    private
+
+    # Detect language from file or use specified language.
+    #
+    # @param filepath [String] Path to file
+    # @param language_code [String] Specified language code or 'auto'
+    # @return [String] Detected or specified language code
+    def detect_language(filepath, language_code)
+      return language_code unless language_code == 'auto'
+
+      puts "Detecting language..." if options[:verbose]
+
+      begin
+        lid = Language::LanguageIdentifier.new
+        result = lid.detect_from_file(filepath, top_k: 1).first
+
+        if result && result.confidence > 0.8
+          detected = result.language
+          puts "  Detected: #{detected} (#{(result.confidence * 100).round(0)}% confidence)" if options[:verbose]
+          detected
+        else
+          puts "  Language detection uncertain, using 'en'" if options[:verbose]
+          'en'
+        end
+      rescue StandardError => e
+        puts "  Language detection failed: #{e.message}" if options[:verbose]
+        puts "  Using 'en' as default" if options[:verbose]
+        'en'
+      end
+    end
+
+    # Load document from file.
+    #
+    # @param filepath [String] Path to file
+    # @param language_code [String] Language code
+    # @return [Documents::Document] Loaded document
+    def load_document(filepath, language_code = 'en')
+      Documents::Document.from_file(filepath, language_code: language_code)
+    rescue StandardError => e
+      puts "Error loading document: #{e.message}"
+      exit 1
+    end
+
+    # Load analyzer based on model type.
+    #
+    # @param language_code [String] Language code
+    # @param model_type [String] Model type
+    # @return [Object] Analyzer instance
+    def load_analyzer(language_code, model_type)
+      case model_type
+      when 'fasttext'
+        load_fasttext_analyzer(language_code)
+      when 'hunspell'
+        load_hunspell_analyzer(language_code)
+      else
+        raise ArgumentError, "Unknown model type: #{model_type}"
+      end
+    end
+
+    # Load FastText analyzer using ONNX model.
+    #
+    # ONNX is the ONLY supported format. No fallbacks.
+    #
+    # @param language_code [String] Language code
+    # @return [Analyzers::SemanticAnalyzer] FastText analyzer with ONNX model
+    def load_fasttext_analyzer(language_code)
+      cache = Cache::ModelCache.new
+      onnx_file = cache.get_onnx_model(language_code, force_download: options[:download])
+
+      unless onnx_file && File.exist?(onnx_file)
+        puts "Error: ONNX model not found for #{language_code}"
+        puts ""
+        puts "Download the model first:"
+        puts "  kotoshu model download #{language_code} --type onnx"
+        puts ""
+        puts "Or convert from FastText .vec file:"
+        puts "  kotoshu model convert cc.#{language_code}.300.vec fasttext.#{language_code}.onnx -l #{language_code}"
+        exit 1
+      end
+
+      puts "Loading ONNX model for #{language_code}..." if options[:verbose]
+      model = Models::OnnxModel.from_file(onnx_file)
+      model.preload_embedding_matrix if options[:verbose]
+      Analyzers::SemanticAnalyzer.new(model)
+    rescue StandardError => e
+      puts "Error loading FastText analyzer: #{e.message}"
+      puts ""
+      puts "Ensure ONNX Runtime is installed:"
+      puts "  gem install onnxruntime"
+      exit 1
+    end
+
+    # Load Hunspell analyzer.
+    #
+    # @param language_code [String] Language code
+    # @return [Object] Hunspell analyzer
+    def load_hunspell_analyzer(language_code)
+      require_relative '../dictionary/hunspell'
+
+      # Load Hunspell dictionary
+      if options[:download]
+        puts "Loading Hunspell dictionary for #{language_code}..." if options[:verbose]
+        dict = Dictionary::Hunspell.from_github(language_code)
+      else
+        # Try local paths
+        dict = Dictionary::Hunspell.for_language(language_code)
+      end
+
+      # Create Hunspell-based analyzer
+      # Note: This would use HunspellDictionary for checking + EditDistanceStrategy for suggestions
+      # For now, we'll use a placeholder
+      require_relative '../spell_checker'
+      SpellChecker.new(dictionary: dict, language: language_code)
+    rescue StandardError => e
+      puts "Error loading Hunspell analyzer: #{e.message}"
+      exit 1
+    end
+
+    # Run interactive mode.
+    #
+    # @param document [Documents::Document] Document to check
+    # @param analyzer [Object] Analyzer instance
+    def run_interactive_mode(document, analyzer)
+      # Create interactive reviewer
+      reviewer = Cli::InteractiveReviewer.new(document, analyzer)
+
+      unless reviewer.has_errors?
+        puts "No errors found!"
+        return
+      end
+
+      # Run interactive loop
+      reviewer.run
+
+      # Apply corrections if user accepted any
+      if reviewer.navigation.modified.any?
+        apply_corrections(document, reviewer.navigation)
+      end
+    end
+
+    # Run batch mode.
+    #
+    # @param document [Documents::Document] Document to check
+    # @param analyzer [Object] Analyzer instance
+    # @return [Cli::BatchReporter] Batch reporter
+    def run_batch_mode(document, analyzer)
+      # For batch mode with Hunspell, we need different approach
+      if analyzer.is_a?(SpellChecker)
+        # Use traditional spell checker
+        result = analyzer.check_string(document.content)
+        # Convert result to navigation...
+        # This is a placeholder - full implementation would convert
+      end
+
+      # For SemanticAnalyzer, create reviewer and get batch reporter
+      if analyzer.is_a?(Analyzers::SemanticAnalyzer)
+        reviewer = Cli::InteractiveReviewer.new(document, analyzer)
+        reporter = reviewer.run_batch
+
+        # Write to file if specified
+        if options[:output]
+          case options[:format]
+          when 'json'
+            reporter.to_json(filepath: options[:output])
+          when 'yaml'
+            reporter.to_yaml(filepath: options[:output])
+          when 'csv'
+            reporter.to_csv(filepath: options[:output])
+          when 'sarif'
+            reporter.to_sarif(filepath: options[:output])
+          else
+            File.write(options[:output], reporter.to_text)
+          end
+
+          puts "Report written to: #{options[:output]}" if options[:verbose]
+        end
+
+        return reporter
+      end
+
+      # Fallback
+      nil
+    end
+
+    # Apply corrections to document.
+    #
+    # @param document [Documents::Document] Original document
+    # @param navigation [Cli::NavigationManager] Navigation state with corrections
+    def apply_corrections(document, navigation)
+      corrections = navigation.export_corrections
+
+      if corrections.empty?
+        return
+      end
+
+      # Apply corrections
+      corrected_doc = document.apply(corrections.map { |c|
+        # Convert correction hash to SemanticError
+        # This is a placeholder - full implementation would reconstruct errors
+      }.compact)
+
+      # Write corrected document
+      backup_path = document.name + ".bak"
+      output_path = document.name
+
+      # Create backup
+      File.write(backup_path, document.content)
+
+      # Write corrected version
+      File.write(output_path, corrected_doc.content)
+
+      puts "Created backup: #{backup_path}" if options[:verbose]
+      puts "Wrote corrections to: #{output_path}"
+    end
+  end
+end

data/lib/kotoshu/commands/model_command.rb

@@ -0,0 +1,295 @@
+# frozen_string_literal: true
+
+require 'thor'
+require_relative '../models/fasttext_model'
+require_relative '../models/onnx_model'
+require_relative '../cache/model_cache'
+
+module Kotoshu
+  class ModelCommand < Thor
+    namespace :model
+
+    desc 'convert INPUT OUTPUT', 'Convert FastText .vec file to ONNX format'
+    option :language, aliases: '-l', type: :string, required: true,
+             desc: 'Language code (de, en, es, fr, pt, ru)'
+    option :max_vectors, type: :numeric, default: 500_000,
+             desc: 'Maximum vectors to convert (default: 500k)'
+    option :validate, type: :boolean, default: true,
+             desc: 'Validate model after conversion'
+    def convert(input, output)
+      puts "Converting #{input} to #{output}..."
+
+      # Check if input file exists
+      unless File.exist?(input)
+        puts "Error: Input file not found: #{input}"
+        exit 1
+      end
+
+      # Build Python command
+      script_path = File.join(File.dirname(__FILE__), '../../scripts/convert_fasttext_to_onnx.py')
+
+      unless File.exist?(script_path)
+        puts "Error: Conversion script not found: #{script_path}"
+        exit 1
+      end
+
+      # Build command
+      cmd = [
+        'python3',
+        script_path,
+        '--input', input,
+        '--output', output,
+        '--language', options[:language],
+        '--max-vectors', options[:max_vectors].to_s
+      ]
+
+      cmd << '--validate' if options[:validate]
+
+      puts "Running: #{cmd.join(' ')}"
+
+      # Execute conversion
+      system(*cmd)
+
+      if $?.success?
+        puts "\n✓ Conversion successful!"
+        puts "  Model: #{output}"
+        puts "  Vocab: #{output.sub('.onnx', '.vocab.json')}"
+        puts "  Metadata: #{output.sub('.onnx', '.metadata.json')}"
+        puts "  Optimized: #{output.sub('.onnx', '.ort.onnx')}"
+      else
+        puts "\n✗ Conversion failed!"
+        exit 1
+      end
+    end
+
+    desc 'download LANGUAGE', 'Download FastText model for a language'
+    option :type, type: :string, enum: %w[fasttext onnx], default: 'fasttext',
+             desc: 'Model type to download'
+    option :output, type: :string,
+             desc: 'Output path (default: $XDG_CACHE_HOME/kotoshu/languages/{code}/models/)'
+    option :force, type: :boolean, default: false,
+             desc: 'Force re-download even if cached'
+    def download(language)
+      puts "Downloading #{options[:type]} model for #{language}..."
+
+      cache = Cache::ModelCache.new
+
+      case options[:type]
+      when 'fasttext'
+        vec_file = cache.get_fasttext_model(language, force_download: options[:force])
+        puts "✓ Downloaded to: #{vec_file}"
+      when 'onnx'
+        onnx_file = cache.get_onnx_model(language, force_download: options[:force])
+        puts "✓ Downloaded to: #{onnx_file}"
+      end
+
+      # Show file info
+      show_model_info(language)
+    end
+
+    desc 'info LANGUAGE', 'Show information about available models'
+    option :type, type: :string, enum: %w[fasttext onnx],
+             desc: 'Model type to show (default: all)'
+    def info(language)
+      cache = Cache::ModelCache.new
+
+      puts "Model information for #{language}:"
+      puts ""
+
+      if options[:type].nil? || options[:type] == 'fasttext'
+        model_info = cache.model_info(language, :fasttext)
+        if model_info
+          puts "FastText:"
+          puts "  File: #{model_info[:file]}"
+          puts "  Size: #{model_info[:size].to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse} vectors"
+          puts "  Source: #{model_info[:source]}"
+          puts ""
+        end
+      end
+
+      if options[:type].nil? || options[:type] == 'onnx'
+        model_info = cache.model_info(language, :onnx)
+        if model_info
+          puts "ONNX:"
+          puts "  File: #{model_info[:file]}"
+          puts "  Source: #{model_info[:source]}"
+          puts ""
+        end
+      end
+    end
+
+    desc 'list', 'List all available models'
+    def list
+      cache = Cache::ModelCache.new
+      all_models = cache.all_available_models
+
+      puts "Available models:"
+      puts ""
+
+      all_models.each do |model_type, languages|
+        puts "#{model_type.to_s.capitalize}:"
+
+        languages.each do |code, info|
+          puts "  #{code}:"
+          puts "    File: #{info[:file]}"
+          puts "    Source: #{info[:source]}"
+        end
+
+        puts ""
+      end
+    end
+
+    desc 'validate MODEL_PATH', 'Validate an ONNX model'
+    def validate(model_path)
+      puts "Validating #{model_path}..."
+
+      unless File.exist?(model_path)
+        puts "Error: Model file not found: #{model_path}"
+        exit 1
+      end
+
+      # Try to load the model
+      begin
+        model = Models::OnnxModel.from_file(model_path)
+
+        puts "✓ Model loaded successfully"
+        puts "  Language: #{model.language_code}"
+        puts "  Dimension: #{model.dimension}"
+        puts "  Vocabulary: #{model.vocabulary_size} words"
+
+        # Test lookup
+        test_word = model.vocabulary.first
+        if test_word
+          embedding = model.embedding_for(test_word)
+          puts "  Test lookup: '#{test_word}' -> vector of size #{embedding.vector.size}"
+        end
+
+        puts "\n✓ Model is valid!"
+
+      rescue StandardError => e
+        puts "✗ Validation failed: #{e.message}"
+        exit 1
+      end
+    end
+
+    desc 'upload LANGUAGE MODEL_FILE', 'Upload model to dictionaries repository'
+    option :repo, type: :string, default: 'kotoshu/dictionaries',
+             desc: 'GitHub repository'
+    option :branch, type: :string, default: 'main',
+             desc: 'Target branch'
+    option :create_pr, type: :boolean, default: false,
+             desc: 'Create pull request instead of direct push'
+    def upload(language, model_file)
+      puts "Uploading #{model_file} to #{options[:repo]}..."
+
+      # Check if file exists
+      unless File.exist?(model_file)
+        puts "Error: File not found: #{model_file}"
+        exit 1
+      end
+
+      # Determine model type and destination path
+      if model_file.end_with?('.vec')
+        model_type = 'fasttext'
+        filename = File.basename(model_file)
+        dest_path = "#{language}/models/fasttext/#{filename}"
+      elsif model_file.end_with?('.onnx')
+        model_type = 'onnx'
+        filename = File.basename(model_file)
+        dest_path = "#{language}/models/onnx/#{filename}"
+
+        # Also upload vocab and metadata files
+        vocab_file = model_file.sub('.onnx', '.vocab.json')
+        metadata_file = model_file.sub('.onnx', '.metadata.json')
+        ort_file = model_file.sub('.onnx', '.ort.onnx')
+      else
+        puts "Error: Unknown file type. Expected .vec or .onnx"
+        exit 1
+      end
+
+      # Build gh command
+      cmd = [
+        'gh', 'repo', 'clone', options[:repo], '/tmp/kotoshu-dictionaries'
+      ]
+
+      puts "Cloning repository..."
+      system(*cmd)
+
+      unless $?.success?
+        puts "Error: Failed to clone repository"
+        exit 1
+      end
+
+      # Copy files to destination
+      target_dir = File.join('/tmp/kotoshu-dictionaries', File.dirname(dest_path))
+      FileUtils.mkdir_p(target_dir)
+
+      FileUtils.cp(model_file, File.join('/tmp/kotoshu-dictionaries', dest_path))
+
+      if model_type == 'onnx'
+        if File.exist?(vocab_file)
+          FileUtils.cp(vocab_file, File.join('/tmp/kotoshu-dictionaries', dest_path.sub('.onnx', '.vocab.json')))
+        end
+        if File.exist?(metadata_file)
+          FileUtils.cp(metadata_file, File.join('/tmp/kotoshu-dictionaries', dest_path.sub('.onnx', '.metadata.json')))
+        end
+        if File.exist?(ort_file)
+          FileUtils.cp(ort_file, File.join('/tmp/kotoshu-dictionaries', dest_path.sub('.onnx', '.ort.onnx')))
+        end
+      end
+
+      # Commit and push
+      Dir.chdir('/tmp/kotoshu-dictionaries') do
+        system('git', 'add', '.')
+
+        message = "Add #{model_type} model for #{language}\n\n"
+        message += "Model: #{filename}\n"
+        message += "Language: #{language}\n"
+
+        system('git', 'commit', '-m', message)
+
+        if options[:create_pr]
+          # Create branch and PR
+          branch_name = "add-#{model_type}-#{language}"
+          system('git', 'checkout', '-b', branch_name)
+          system('git', 'push', 'origin', branch_name)
+          system('gh', 'pr', 'create', '--title', "Add #{model_type} model for #{language}", '--body', message)
+        else
+          # Direct push
+          system('git', 'push')
+        end
+      end
+
+      if $?.success?
+        puts "✓ Upload successful!"
+        puts "  Path: #{dest_path}"
+        puts "  Repository: #{options[:repo]}"
+      else
+        puts "✗ Upload failed!"
+        exit 1
+      end
+    end
+
+    private
+
+    # Show model file information.
+    #
+    # @param language [String] Language code
+    def show_model_info(language)
+      cache = Cache::ModelCache.new
+      model_path = File.join(cache.instance_variable_get(:@cache_path), language, 'models')
+
+      if Dir.exist?(model_path)
+        puts "\nModel files:"
+        Dir.glob(File.join(model_path, '**/*')).each do |file|
+          next if File.directory?(file)
+
+          size = File.size(file)
+          size_mb = (size.to_f / 1024 / 1024).round(2)
+
+          puts "  #{File.basename(file)}: #{size_mb} MB"
+        end
+      end
+    end
+  end
+end

data/lib/kotoshu/components/passthrough_spell_checker.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require_relative 'spell_checker'
+
+module Kotoshu
+  module Components
+    # Passthrough spell checker for languages that don't use spell checking.
+    #
+    # This checker always returns that words are "found" (correct). It's used
+    # for languages that don't have traditional spell checking, such as:
+    # - CJK languages (Japanese, Chinese) - use confusion rules instead
+    # - Languages with purely rule-based checking
+    #
+    # @example
+    #   checker = PassthroughSpellChecker.new
+    #   result = checker.check('任意のテキスト')
+    #   # => { found: true, stem: nil, flags: [] }
+    #
+    # @example Getting suggestions (always empty)
+    #   suggestions = checker.suggest('テキスト')
+    #   # => []
+    class PassthroughSpellChecker < SpellChecker
+      # Create a new passthrough spell checker.
+      #
+      # @param reason [String] Optional reason why spell checking is not used
+      def initialize(reason: nil)
+        @reason = reason || "Language does not use spell checking"
+      end
+
+      # Always returns that the word is "found" (correct).
+      #
+      # @param _word [String] The word to check (ignored)
+      # @return [Hash] Always returns { found: true, stem: nil, flags: [] }
+      def check(_word)
+        { found: true, stem: nil, flags: [] }
+      end
+
+      # Returns no suggestions.
+      #
+      # Passthrough spell checkers don't provide suggestions.
+      #
+      # @param _word [String] The word (ignored)
+      # @param _max_suggestions [Integer] Max suggestions (ignored)
+      # @return [Array<Hash>] Always returns empty array
+      def suggest(_word, _max_suggestions: 10)
+        []
+      end
+
+      # Always returns true (all words are "correct").
+      #
+      # @param _word [String] The word to check (ignored)
+      # @return [Boolean] Always true
+      def correct?(_word)
+        true
+      end
+
+      # Get the reason why spell checking is not used.
+      #
+      # @return [String] Reason text
+      def reason
+        @reason
+      end
+
+      # Check if this is a passthrough checker.
+      #
+      # @return [Boolean] Always true for this class
+      def passthrough?
+        true
+      end
+    end
+  end
+end