better_translate 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a2ff034a6db3498226ddff2111cf66e94a371f65741f82986b6ec8324daaf2dc
-  data.tar.gz: 991a78781adfc647d789729f385bca6e7688a6269bf102e0bff8b40b7bf8aca7
+  metadata.gz: 6941ef1c334c2ddbbb7141c8257f8906957fd0b60eadeee867d9a11b83788ae6
+  data.tar.gz: 8678aa23790a06d905592efc1571a91408e58a9f1359e70ee90174d5d2ee8975
 SHA512:
-  metadata.gz: 68f06a1006c8fc3e43184485d8f71ec9b080649fa68e2f8c5563d55b20c356b501a20a308f0c00f967b5387ac8d3f62abf11d837ca25b325a646658bf8126146
-  data.tar.gz: 719cb1a9bb0217934ca0ecacd455bcc6c81afab081fbc43d24b8add5a50eb33b7947ab715a37938460d162fdcf88e9e629b8cc18d08e1f47ecec085101c35e8c
+  metadata.gz: 548433e5223f37320a42c7ba35644b46470dc89baf42581612753d0542d039e2729cad8dd8bf4fbde1cd962ac970f397fdd1a3d98374f83fbc0380805771daae
+  data.tar.gz: 704eaeb04b49d7f165631443567e89afdff27a30c42893d0d04e424fe340fb113a9a4553ded29c1b4f8e9af36d75eccea0e138d0056c72442f75570340b9fa97

data/CHANGELOG.md

@@ -5,6 +5,19 @@ All notable changes to BetterTranslate will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.4.0] - 2025-03-11
+
+### Added
+- New Translation Similarity Analyzer:
+  - Identifies similar translations across language files
+  - Generates detailed JSON reports and human-readable summaries
+  - Uses Levenshtein distance for similarity calculation
+  - Configurable similarity threshold
+- New Rails generator: `rails generate better_translate:analyze`
+  - Analyzes all YAML files in the locales directory
+  - Provides immediate feedback in the console
+  - Generates comprehensive similarity reports
+
 ## [0.3.1] - 2025-03-11
 
 ### Added

data/README.md

@@ -14,6 +14,7 @@ BetterTranslate simplifies the translation of YAML files in your Ruby/Rails appl
 - 📊 Real-time progress tracking
 - 🧪 Comprehensive test coverage
 - ⚡️ LRU caching for performance
+- 🔍 Translation similarity analysis
 
 ## Why BetterTranslate? 🤔
 
@@ -30,6 +31,10 @@ BetterTranslate simplifies the translation of YAML files in your Ruby/Rails appl
   - Comprehensive test suite
   - LRU caching for performance
   - Progress tracking
+- 🔍 **Translation Analysis**:
+  - Similarity detection
+  - Detailed reports
+  - Optimization suggestions
 - 🔧 **Flexible Helpers**:
   - Single text translation
   - Bulk text translation
@@ -146,6 +151,25 @@ The gem includes generators to simplify tasks:
 
   The `better_translate:translate` generator will trigger the translation process (via `BetterTranslate.magic`) and display progress in the terminal.
 
+- **Analyze Translations:**
+
+  ```bash
+  rails generate better_translate:analyze
+  ```
+
+  The `better_translate:analyze` generator will:
+  - Scan all YAML files in your locales directory
+  - Find similar translations using Levenshtein distance
+  - Generate two reports:
+    - `translation_similarities.json`: Detailed JSON report
+    - `translation_similarities_summary.txt`: Human-readable summary
+  
+  This helps you:
+  - Identify potentially redundant translations
+  - Maintain consistency across your translations
+  - Optimize your translation files
+  - Reduce translation costs
+
 ### Translation Helpers
 
 BetterTranslate provides helper methods to simplify translation tasks.

data/lib/better_translate/similarity_analyzer.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+require "yaml"
+require "set"
+require "json"
+require "time"
+
+module BetterTranslate
+  class SimilarityAnalyzer
+    SIMILARITY_THRESHOLD = 0.75 # Abbassiamo la soglia per trovare più similarità
+    REPORT_FILE = "translation_similarities.json"
+
+    def initialize(yaml_files)
+      @yaml_files = yaml_files
+      @similarities = {}
+    end
+
+    def analyze
+      translations_by_language = load_translations
+      find_similarities(translations_by_language)
+      generate_report
+    end
+
+    private
+
+    def load_translations
+      translations = {}
+      puts "Loading YAML files..."
+      
+      @yaml_files.each do |file|
+        lang_code = File.basename(file, ".yml")
+        translations[lang_code] = YAML.load_file(file)
+        puts "  - Loaded #{lang_code}.yml"
+      end
+
+      translations
+    end
+
+    def find_similarities(translations_by_language)
+      translations_by_language.each do |lang, translations|
+        puts "\nAnalyzing #{lang} translations..."
+        flattened = flatten_translations(translations)
+        keys = flattened.keys
+        similar_found = 0
+        
+        keys.each_with_index do |key1, i|
+          value1 = flattened[key1]
+          
+          # Confronta solo con le chiavi successive per evitare duplicati
+          keys[(i + 1)..-1].each do |key2|
+            value2 = flattened[key2]
+            
+            similarity = calculate_similarity(value1.to_s, value2.to_s)
+            if similarity >= SIMILARITY_THRESHOLD
+              record_similarity(lang, key1, key2, value1, value2, similarity)
+              similar_found += 1
+            end
+          end
+        end
+        
+        puts "  Found #{similar_found} similar translations"
+      end
+    end
+
+    def flatten_translations(hash, prefix = "", result = {})
+      hash.each do |key, value|
+        current_key = prefix.empty? ? key.to_s : "#{prefix}.#{key}"
+        
+        if value.is_a?(Hash)
+          flatten_translations(value, current_key, result)
+        else
+          result[current_key] = value
+        end
+      end
+      
+      result
+    end
+
+    def calculate_similarity(str1, str2)
+      # Implementazione della distanza di Levenshtein normalizzata
+      matrix = Array.new(str1.length + 1) { Array.new(str2.length + 1) }
+      
+      (0..str1.length).each { |i| matrix[i][0] = i }
+      (0..str2.length).each { |j| matrix[0][j] = j }
+      
+      (1..str1.length).each do |i|
+        (1..str2.length).each do |j|
+          cost = str1[i-1] == str2[j-1] ? 0 : 1
+          matrix[i][j] = [
+            matrix[i-1][j] + 1,
+            matrix[i][j-1] + 1,
+            matrix[i-1][j-1] + cost
+          ].min
+        end
+      end
+      
+      max_length = [str1.length, str2.length].max
+      1 - (matrix[str1.length][str2.length].to_f / max_length)
+    end
+
+    def record_similarity(lang, key1, key2, value1, value2, similarity)
+      @similarities[lang] ||= []
+      @similarities[lang] << {
+        key1: key1,
+        key2: key2,
+        value1: value1,
+        value2: value2,
+        similarity: similarity.round(2)
+      }
+    end
+
+    def generate_report
+      puts "\nGenerating reports..."
+      report = {
+        generated_at: Time.now.iso8601,
+        similarity_threshold: SIMILARITY_THRESHOLD,
+        findings: @similarities
+      }
+
+      File.write(REPORT_FILE, JSON.pretty_generate(report))
+      puts "  - Generated #{REPORT_FILE}"
+      
+      summary = generate_summary(report)
+      File.write("translation_similarities_summary.txt", summary)
+      puts "  - Generated translation_similarities_summary.txt"
+    end
+
+    def generate_summary(report)
+      summary = ["Translation Similarities Report", "=" * 30, ""]
+      
+      report[:findings].each do |lang, similarities|
+        summary << "Language: #{lang}"
+        summary << "-" * 20
+        
+        similarities.each do |sim|
+          summary << "Similar translations found:"
+          summary << "  Key 1: #{sim[:key1]}"
+          summary << "  Value 1: #{sim[:value1]}"
+          summary << "  Key 2: #{sim[:key2]}"
+          summary << "  Value 2: #{sim[:value2]}"
+          summary << "  Similarity: #{(sim[:similarity] * 100).round(1)}%"
+          summary << ""
+        end
+        
+        summary << ""
+      end
+      
+      summary.join("\n")
+    end
+  end
+end

data/lib/better_translate/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module BetterTranslate
-  VERSION = "0.3.1"
+  VERSION = "0.4.0"
 end

data/lib/better_translate.rb

@@ -7,6 +7,7 @@ require "better_translate/translator"
 require "better_translate/service"
 require "better_translate/writer"
 require "better_translate/helper"
+require "better_translate/similarity_analyzer"
 
 require 'better_translate/providers/base_provider'
 require 'better_translate/providers/chatgpt_provider'

data/lib/generators/better_translate/analyze_generator.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module BetterTranslate
+  module Generators
+    class AnalyzeGenerator < Rails::Generators::Base
+      desc "Analyze translation files for similarities"
+
+      def analyze_translations
+        say "Starting translation similarity analysis...", :blue
+
+        # Find all YAML files in the locales directory
+        locale_dir = Rails.root.join("config", "locales")
+        yaml_files = Dir[locale_dir.join("*.yml")]
+
+        if yaml_files.empty?
+          say "No YAML files found in #{locale_dir}", :red
+          return
+        end
+
+        say "Found #{yaml_files.length} YAML files to analyze", :green
+
+        # Run analysis
+        analyzer = BetterTranslate::SimilarityAnalyzer.new(yaml_files)
+        analyzer.analyze
+
+        # Show results
+        say "\nAnalysis complete!", :green
+        say "Reports generated:", :green
+        say "  * #{BetterTranslate::SimilarityAnalyzer::REPORT_FILE} (detailed JSON report)"
+        say "  * translation_similarities_summary.txt (human-readable summary)"
+        
+        # Show quick summary from the text file
+        if File.exist?("translation_similarities_summary.txt")
+          summary = File.read("translation_similarities_summary.txt")
+          say "\nQuick Summary:", :blue
+          say summary
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: better_translate
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Alessio Bussolari
@@ -222,10 +222,12 @@ files:
 - lib/better_translate/providers/chatgpt_provider.rb
 - lib/better_translate/providers/gemini_provider.rb
 - lib/better_translate/service.rb
+- lib/better_translate/similarity_analyzer.rb
 - lib/better_translate/translator.rb
 - lib/better_translate/utils.rb
 - lib/better_translate/version.rb
 - lib/better_translate/writer.rb
+- lib/generators/better_translate/analyze_generator.rb
 - lib/generators/better_translate/install_generator.rb
 - lib/generators/better_translate/templates/better_translate.rb
 - lib/generators/better_translate/translate_generator.rb