better_translate 0.3.0 → 0.4.0

data/lib/better_translate/similarity_analyzer.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+require "yaml"
+require "set"
+require "json"
+require "time"
+
+module BetterTranslate
+  class SimilarityAnalyzer
+    SIMILARITY_THRESHOLD = 0.75 # Abbassiamo la soglia per trovare più similarità
+    REPORT_FILE = "translation_similarities.json"
+
+    def initialize(yaml_files)
+      @yaml_files = yaml_files
+      @similarities = {}
+    end
+
+    def analyze
+      translations_by_language = load_translations
+      find_similarities(translations_by_language)
+      generate_report
+    end
+
+    private
+
+    def load_translations
+      translations = {}
+      puts "Loading YAML files..."
+      
+      @yaml_files.each do |file|
+        lang_code = File.basename(file, ".yml")
+        translations[lang_code] = YAML.load_file(file)
+        puts "  - Loaded #{lang_code}.yml"
+      end
+
+      translations
+    end
+
+    def find_similarities(translations_by_language)
+      translations_by_language.each do |lang, translations|
+        puts "\nAnalyzing #{lang} translations..."
+        flattened = flatten_translations(translations)
+        keys = flattened.keys
+        similar_found = 0
+        
+        keys.each_with_index do |key1, i|
+          value1 = flattened[key1]
+          
+          # Confronta solo con le chiavi successive per evitare duplicati
+          keys[(i + 1)..-1].each do |key2|
+            value2 = flattened[key2]
+            
+            similarity = calculate_similarity(value1.to_s, value2.to_s)
+            if similarity >= SIMILARITY_THRESHOLD
+              record_similarity(lang, key1, key2, value1, value2, similarity)
+              similar_found += 1
+            end
+          end
+        end
+        
+        puts "  Found #{similar_found} similar translations"
+      end
+    end
+
+    def flatten_translations(hash, prefix = "", result = {})
+      hash.each do |key, value|
+        current_key = prefix.empty? ? key.to_s : "#{prefix}.#{key}"
+        
+        if value.is_a?(Hash)
+          flatten_translations(value, current_key, result)
+        else
+          result[current_key] = value
+        end
+      end
+      
+      result
+    end
+
+    def calculate_similarity(str1, str2)
+      # Implementazione della distanza di Levenshtein normalizzata
+      matrix = Array.new(str1.length + 1) { Array.new(str2.length + 1) }
+      
+      (0..str1.length).each { |i| matrix[i][0] = i }
+      (0..str2.length).each { |j| matrix[0][j] = j }
+      
+      (1..str1.length).each do |i|
+        (1..str2.length).each do |j|
+          cost = str1[i-1] == str2[j-1] ? 0 : 1
+          matrix[i][j] = [
+            matrix[i-1][j] + 1,
+            matrix[i][j-1] + 1,
+            matrix[i-1][j-1] + cost
+          ].min
+        end
+      end
+      
+      max_length = [str1.length, str2.length].max
+      1 - (matrix[str1.length][str2.length].to_f / max_length)
+    end
+
+    def record_similarity(lang, key1, key2, value1, value2, similarity)
+      @similarities[lang] ||= []
+      @similarities[lang] << {
+        key1: key1,
+        key2: key2,
+        value1: value1,
+        value2: value2,
+        similarity: similarity.round(2)
+      }
+    end
+
+    def generate_report
+      puts "\nGenerating reports..."
+      report = {
+        generated_at: Time.now.iso8601,
+        similarity_threshold: SIMILARITY_THRESHOLD,
+        findings: @similarities
+      }
+
+      File.write(REPORT_FILE, JSON.pretty_generate(report))
+      puts "  - Generated #{REPORT_FILE}"
+      
+      summary = generate_summary(report)
+      File.write("translation_similarities_summary.txt", summary)
+      puts "  - Generated translation_similarities_summary.txt"
+    end
+
+    def generate_summary(report)
+      summary = ["Translation Similarities Report", "=" * 30, ""]
+      
+      report[:findings].each do |lang, similarities|
+        summary << "Language: #{lang}"
+        summary << "-" * 20
+        
+        similarities.each do |sim|
+          summary << "Similar translations found:"
+          summary << "  Key 1: #{sim[:key1]}"
+          summary << "  Value 1: #{sim[:value1]}"
+          summary << "  Key 2: #{sim[:key2]}"
+          summary << "  Value 2: #{sim[:value2]}"
+          summary << "  Similarity: #{(sim[:similarity] * 100).round(1)}%"
+          summary << ""
+        end
+        
+        summary << ""
+      end
+      
+      summary.join("\n")
+    end
+  end
+end

data/lib/better_translate/translator.rb

@@ -2,65 +2,75 @@ module BetterTranslate
   class Translator
     class << self
       def work
-        puts "Avvio della traduzione dei file..."
+        message = "Starting file translation..."
+        BetterTranslate::Utils.logger(message: message)
 
         translations = read_yml_source
 
-        # Rimuove le chiavi da escludere (global_exclusions) dalla struttura letta
+        # Removes the keys to exclude (global_exclusions) from the read structure
         global_filtered_translations = remove_exclusions(
           translations, BetterTranslate.configuration.global_exclusions
         )
 
-        BetterTranslate.configuration.target_languages.each do |target_lang|
+        start_time = Time.now
+        threads = BetterTranslate.configuration.target_languages.map do |target_lang|
+          Thread.new do
 
-          # Fase 2: Applica il filtro specifico per la lingua target
+          # Phase 2: Apply the target language specific filter
           lang_exclusions = BetterTranslate.configuration.exclusions_per_language[target_lang[:short_name]] || []
           filtered_translations = remove_exclusions(
             global_filtered_translations, lang_exclusions
           )
 
-          puts "Inizio traduzione da #{BetterTranslate.configuration.source_language} a #{target_lang[:short_name]}"
+          message = "Starting translation from #{BetterTranslate.configuration.source_language} to #{target_lang[:short_name]}"
+          BetterTranslate::Utils.logger(message: message)
           service = BetterTranslate::Service.new
           translated_data = translate_with_progress(filtered_translations, service, target_lang[:short_name], target_lang[:name])
           BetterTranslate::Writer.write_translations(translated_data, target_lang[:short_name])
-          puts "Traduzione completata da #{BetterTranslate.configuration.source_language} a #{target_lang[:short_name]}"
+          end_time = Time.now
+          duration = end_time - start_time
+          BetterTranslate::Utils.track_metric("translation_duration", duration)
+          
+          message = "Translation completed from #{BetterTranslate.configuration.source_language} to #{target_lang[:short_name]} in #{duration.round(2)} seconds"
+          BetterTranslate::Utils.logger(message: message)
+        end
         end
       end
 
       private
 
-      # Legge il file YAML in base al percorso fornito.
+      # Reads the YAML file based on the provided path.
       #
-      # @param file_path [String] percorso del file YAML da leggere
-      # @return [Hash] struttura dati contenente le traduzioni
-      # @raise [StandardError] se il file non esiste
+      # @param file_path [String] path of the YAML file to read
+      # @return [Hash] data structure containing the translations
+      # @raise [StandardError] if the file does not exist
       def read_yml_source
         file_path = BetterTranslate.configuration.input_file
         unless File.exist?(file_path)
-          raise "File non trovato: #{file_path}"
+          raise "File not found: #{file_path}"
         end
 
         YAML.load_file(file_path)
       end
 
-      # Rimuove le chiavi globali da escludere dalla struttura dati,
-      # calcolando i percorsi a partire dal contenuto della lingua di partenza.
+      # Removes the global keys to exclude from the data structure,
+      # calculating paths starting from the source language content.
       #
-      # Ad esempio, se il file YAML è:
+      # For example, if the YAML file is:
       #   { "en" => { "sample" => { "valid" => "valid", "excluded" => "Excluded" } } }
-      # e global_exclusions = ["sample.excluded"],
-      # il risultato sarà:
+      # and global_exclusions = ["sample.excluded"],
+      # the result will be:
       #   { "en" => { "sample" => { "valid" => "valid" } } }
       #
-      # @param data [Hash, Array, Object] La struttura dati da filtrare.
-      # @param global_exclusions [Array<String>] Lista dei percorsi (in dot notation) da escludere globalmente.
-      # @param current_path [Array] Il percorso corrente (usato in maniera ricorsiva, default: []).
-      # @return [Hash, Array, Object] La struttura dati filtrata.
+      # @param data [Hash, Array, Object] The data structure to filter.
+      # @param global_exclusions [Array<String>] List of paths (in dot notation) to exclude globally.
+      # @param current_path [Array] The current path (used recursively, default: []).
+      # @return [Hash, Array, Object] The filtered data structure.
       def remove_exclusions(data, exclusion_list, current_path = [])
         if data.is_a?(Hash)
           data.each_with_object({}) do |(key, value), result|
-            # Se siamo al livello top-level e la chiave corrisponde alla lingua di partenza,
-            # resettare il percorso (così da escludere "en" dal percorso finale)
+            # If we are at the top-level and the key matches the source language,
+            # reset the path (to exclude "en" from the final path)
             new_path = if current_path.empty? && key == BetterTranslate.configuration.source_language
                          []
                        else
@@ -81,14 +91,14 @@ module BetterTranslate
         end
       end
 
-      # Metodo ricorsivo che percorre la struttura, traducendo ogni stringa e aggiornando il progresso.
+      # Recursive method that traverses the structure, translating each string and updating progress.
       #
-      # @param data [Hash, Array, String] La struttura dati da tradurre.
-      # @param provider [Object] Il provider che risponde al metodo translate.
-      # @param target_lang_code [String] Codice della lingua target (es. "en").
-      # @param target_lang_name [String] Nome della lingua target (es. "English").
-      # @param progress [Hash] Un hash con le chiavi :count e :total per monitorare il progresso.
-      # @return [Hash, Array, String] La struttura tradotta.
+      # @param data [Hash, Array, String] The data structure to translate.
+      # @param provider [Object] The provider that responds to the translate method.
+      # @param target_lang_code [String] Target language code (e.g. "en").
+      # @param target_lang_name [String] Target language name (e.g. "English").
+      # @param progress [Hash] A hash with :count and :total keys to monitor progress.
+      # @return [Hash, Array, String] The translated structure.
       def deep_translate_with_progress(data, service, target_lang_code, target_lang_name, progress)
         if data.is_a?(Hash)
           data.each_with_object({}) do |(key, value), result|
@@ -106,20 +116,90 @@ module BetterTranslate
         end
       end
 
-      # Metodo principale per tradurre l'intera struttura dati, con monitoraggio del progresso.
+      # Main method to translate the entire data structure, with progress monitoring.
       #
-      # @param data [Hash, Array, String] la struttura dati da tradurre
-      # @param provider [Object] il provider da usare per tradurre (deve implementare translate)
-      # @param target_lang_code [String] codice della lingua target
-      # @param target_lang_name [String] nome della lingua target
-      # @return la struttura tradotta
+      # @param data [Hash, Array, String] the data structure to translate
+      # @param provider [Object] the provider to use for translation (must implement translate)
+      # @param target_lang_code [String] target language code
+      # @param target_lang_name [String] target language name
+      # @return the translated structure
       def translate_with_progress(data, service, target_lang_code, target_lang_name)
         total = count_strings(data)
         progress = ProgressBar.create(total: total, format: '%a %B %p%% %t')
-        deep_translate_with_progress(data, service, target_lang_code, target_lang_name, progress)
+
+        start_time = Time.now
+        result = if total > 50 # Usa il batch processing per dataset grandi
+          batch_translate_with_progress(data, service, target_lang_code, target_lang_name, progress)
+        else
+          deep_translate_with_progress(data, service, target_lang_code, target_lang_name, progress)
+        end
+
+        duration = Time.now - start_time
+        BetterTranslate::Utils.track_metric("translation_method_duration", {
+          method: total > 50 ? 'batch' : 'deep',
+          duration: duration,
+          total_strings: total
+        })
+
+        result
+      end
+
+      def batch_translate_with_progress(data, service, target_lang_code, target_lang_name, progress)
+        texts = extract_translatable_texts(data)
+        translations = {}
+
+        texts.each_slice(10).each_with_index do |batch, index|
+          batch_start = Time.now
+          
+          batch_translations = batch.map do |text|
+            translated = service.translate(text, target_lang_code, target_lang_name)
+            progress.increment
+            [text, translated]
+          end.to_h
+
+          translations.merge!(batch_translations)
+          
+          batch_duration = Time.now - batch_start
+          BetterTranslate::Utils.track_metric("batch_translation_duration", {
+            batch_number: index + 1,
+            size: batch.size,
+            duration: batch_duration
+          })
+        end
+
+        replace_translations(data, translations)
+      end
+
+      def extract_translatable_texts(data)
+        texts = Set.new
+        traverse_structure(data) do |value|
+          texts.add(value) if value.is_a?(String) && !value.strip.empty?
+        end
+        texts.to_a
+      end
+
+      def replace_translations(data, translations)
+        traverse_structure(data) do |value|
+          if value.is_a?(String) && !value.strip.empty? && translations.key?(value)
+            translations[value]
+          else
+            value
+          end
+        end
+      end
+
+      def traverse_structure(data, &block)
+        case data
+        when Hash
+          data.transform_values { |v| traverse_structure(v, &block) }
+        when Array
+          data.map { |v| traverse_structure(v, &block) }
+        else
+          yield data
+        end
       end
 
-      # Conta ricorsivamente il numero di stringhe traducibili nella struttura dati.
+      # Recursively counts the number of translatable strings in the data structure.
       def count_strings(data)
         if data.is_a?(Hash)
           data.values.sum { |v| count_strings(v) }

data/lib/better_translate/utils.rb

@@ -4,21 +4,21 @@
 #
 # = BetterSeeder::Utils
 #
-# Questo modulo fornisce metodi di utilità per la gestione dei seed. In particolare,
-# consente di trasformare i nomi delle classi in formato snake_case con il suffisso "_structure.rb",
-# gestire i messaggi di log e configurare il livello del logger per ActiveRecord.
+# This module provides utility methods for seed management. In particular,
+# allows transforming class names into snake_case format with the "_structure.rb" suffix,
+# manage log messages and configure the logger level for ActiveRecord.
 
 module BetterTranslate
   class Utils
     class << self
       ##
-      # Registra un messaggio usando il logger di Rails se disponibile, altrimenti lo stampa su standard output.
+      # Logs a message using the Rails logger if available, otherwise prints it to standard output.
       #
       # ==== Parametri
-      # * +message+ - Il messaggio da loggare (può essere una stringa o nil).
+      # * +message+ - The message to log (can be a string or nil).
       #
       # ==== Ritorno
-      # Non ritorna un valore significativo.
+      # Does not return a significant value.
       #
       def logger(message: nil)
         if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
@@ -27,6 +27,23 @@ module BetterTranslate
           puts message
         end
       end
+
+      def track_metric(name, value)
+        @metrics ||= {}
+        @metrics[name] ||= []
+        @metrics[name] << { value: value, timestamp: Time.now }
+
+        # Mantieni solo le ultime 1000 metriche per tipo
+        @metrics[name] = @metrics[name].last(1000) if @metrics[name].size > 1000
+      end
+
+      def get_metrics
+        @metrics || {}
+      end
+
+      def clear_metrics
+        @metrics = {}
+      end
     end
   end
 end

data/lib/better_translate/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module BetterTranslate
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/lib/better_translate/writer.rb

@@ -1,19 +1,19 @@
 module BetterTranslate
   class Writer
     class << self
-      # Scrive il file di traduzione per una lingua target.
-      # Se il metodo di traduzione è :override, il file viene riscritto da zero;
-      # se è :incremental, il file esistente viene aggiornato inserendo le nuove traduzioni nelle posizioni corrette.
+      # Writes the translation file for a target language.
+      # If the translation method is :override, the file is rewritten from scratch;
+      # if it's :incremental, the existing file is updated by inserting new translations in the correct positions.
       #
-      # @param translated_data [Hash] La struttura dati tradotta (ad es. { "sample" => { "valid" => "tradotto", "new_key" => "nuova traduzione" } }).
-      # @param target_lang_code [String] Codice della lingua target (es. "ru").
+      # @param translated_data [Hash] The translated data structure (e.g. { "sample" => { "valid" => "translated", "new_key" => "new translation" } }).
+      # @param target_lang_code [String] Target language code (e.g. "ru").
       def write_translations(translated_data, target_lang_code)
         output_folder = BetterTranslate.configuration.output_folder
         output_file = File.join(output_folder, "#{target_lang_code}.yml")
 
-        # Riformatta la struttura per utilizzare il codice target come chiave principale.
+        # Reformats the structure to use the target code as the main key.
         output_content = if translated_data.is_a?(Hash) && translated_data.key?(BetterTranslate.configuration.source_language)
-                           # Sostituisce la chiave della lingua di partenza con quella target.
+                           # Replaces the source language key with the target one.
                            { target_lang_code => translated_data[BetterTranslate.configuration.source_language] }
                          else
                            { target_lang_code => translated_data }
@@ -23,11 +23,13 @@ module BetterTranslate
           existing_data = YAML.load_file(output_file)
           merged = deep_merge(existing_data, output_content)
           File.write(output_file, merged.to_yaml(line_width: -1))
-          puts "File aggiornato in modalità incremental: #{output_file}"
+          message = "File updated in incremental mode: #{output_file}"
+          BetterTranslate::Utils.logger(message: message)
         else
           FileUtils.mkdir_p(output_folder) unless Dir.exist?(output_folder)
           File.write(output_file, output_content.to_yaml(line_width: -1))
-          puts "File riscritto in modalità override: #{output_file}"
+          message = "File rewritten in override mode: #{output_file}"
+          BetterTranslate::Utils.logger(message: message)
         end
       end
 
@@ -35,12 +37,12 @@ module BetterTranslate
 
       # Metodo di deep merge: unisce in modo ricorsivo i due hash.
       # Se una chiave esiste in entrambi gli hash e i valori sono hash, li unisce ricorsivamente.
-      # Altrimenti, se la chiave esiste già nell'hash esistente, la mantiene e non la sovrascrive.
+      # Otherwise, if the key already exists in the existing hash, it keeps it and doesn't overwrite it.
       # Se la chiave non esiste, la aggiunge.
       #
-      # @param existing [Hash] hash esistente (file corrente)
-      # @param new_data [Hash] nuovo hash con le traduzioni da unire
-      # @return [Hash] hash unito
+      # @param existing [Hash] existing hash (current file)
+      # @param new_data [Hash] new hash with translations to merge
+      # @return [Hash] merged hash
       def deep_merge(existing, new_data)
         merged = existing.dup
         new_data.each do |key, value|
@@ -48,8 +50,8 @@ module BetterTranslate
             if merged[key].is_a?(Hash) && value.is_a?(Hash)
               merged[key] = deep_merge(merged[key], value)
             else
-              # Se la chiave esiste già, non sovrascrivo il valore (modalità incrementale)
-              # oppure potresti decidere di aggiornare comunque, a seconda delle esigenze.
+              # If the key already exists, don't overwrite the value (incremental mode)
+              # or you might decide to update anyway, depending on the requirements.
               merged[key] = merged[key]
             end
           else

data/lib/better_translate.rb

@@ -7,50 +7,53 @@ require "better_translate/translator"
 require "better_translate/service"
 require "better_translate/writer"
 require "better_translate/helper"
+require "better_translate/similarity_analyzer"
 
 require 'better_translate/providers/base_provider'
 require 'better_translate/providers/chatgpt_provider'
 require 'better_translate/providers/gemini_provider'
 
+require 'ostruct'
+
 module BetterTranslate
   class << self
     attr_accessor :configuration
 
-    # Metodo per configurare la gemma
+    # Method to configure the gem
     def configure
       self.configuration ||= OpenStruct.new
       yield(configuration) if block_given?
     end
 
-    # Metodo install per generare il file di configurazione (initializer)
+    # Install method to generate the configuration file (initializer)
     def install
       unless defined?(Rails) && Rails.respond_to?(:root)
-        message = "Il metodo install è disponibile solo in un'applicazione Rails."
+        message = "The install method is only available in a Rails application."
         BetterTranslate::Utils.logger(message: message)
         return
       end
 
-      # Costruisce il percorso della cartella template all'interno della gemma
+      # Builds the path to the template folder inside the gem
       source = File.expand_path("../generators/better_translate/templates/better_translate.rb", __dir__)
       destination = File.join(Rails.root, "config", "initializers", "better_translate.rb")
 
       if File.exist?(destination)
-        message = "Il file initializer esiste già: #{destination}"
+        message = "The initializer file already exists: #{destination}"
         BetterTranslate::Utils.logger(message: message)
       else
         FileUtils.mkdir_p(File.dirname(destination))
         FileUtils.cp(source, destination)
-        message = "Il file initializer esiste già: #{destination}"
+        message = "The initializer file already exists: #{destination}"
         BetterTranslate::Utils.logger(message: message)
       end
     end
 
     def magic
-      message = "Metodo magic invocato: eseguirò la traduzione dei file..."
+      message = "Magic method invoked: Translation will begin..."
       BetterTranslate::Utils.logger(message: message)
 
       BetterTranslate::Translator.work
-      message = "Metodo magic invocato: Traduzione completata con successo!"
+      message = "Magic method invoked: Translation completed successfully!"
       BetterTranslate::Utils.logger(message: message)
     end
 

data/lib/generators/better_translate/analyze_generator.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module BetterTranslate
+  module Generators
+    class AnalyzeGenerator < Rails::Generators::Base
+      desc "Analyze translation files for similarities"
+
+      def analyze_translations
+        say "Starting translation similarity analysis...", :blue
+
+        # Find all YAML files in the locales directory
+        locale_dir = Rails.root.join("config", "locales")
+        yaml_files = Dir[locale_dir.join("*.yml")]
+
+        if yaml_files.empty?
+          say "No YAML files found in #{locale_dir}", :red
+          return
+        end
+
+        say "Found #{yaml_files.length} YAML files to analyze", :green
+
+        # Run analysis
+        analyzer = BetterTranslate::SimilarityAnalyzer.new(yaml_files)
+        analyzer.analyze
+
+        # Show results
+        say "\nAnalysis complete!", :green
+        say "Reports generated:", :green
+        say "  * #{BetterTranslate::SimilarityAnalyzer::REPORT_FILE} (detailed JSON report)"
+        say "  * translation_similarities_summary.txt (human-readable summary)"
+        
+        # Show quick summary from the text file
+        if File.exist?("translation_similarities_summary.txt")
+          summary = File.read("translation_similarities_summary.txt")
+          say "\nQuick Summary:", :blue
+          say summary
+        end
+      end
+    end
+  end
+end

data/lib/generators/better_translate/install_generator.rb

@@ -4,7 +4,7 @@ module BetterTranslate
   module Generators
     class InstallGenerator < Rails::Generators::Base
       source_root File.expand_path('templates', __dir__)
-      desc "Genera il file config/initializers/better_translate.rb con la configurazione di default"
+      desc "Generates the config/initializers/better_translate.rb file with default configuration"
 
       def copy_initializer
         template "better_translate.rb", "config/initializers/better_translate.rb"