i18n-tasks 1.0.14 → 1.0.15

data/lib/i18n/tasks/translators/deepl_translator.rb

@@ -35,6 +35,8 @@ module I18n::Tasks::Translators
         else
           results += res.map(&:text)
         end
+
+        @progress_bar.progress += parts.size
       end
       results
     end
@@ -83,6 +85,9 @@ module I18n::Tasks::Translators
 
     # Convert 'es-ES' to 'ES' but warn about locales requiring a specific variant
     def to_deepl_target_locale(locale)
+      locale_aliases = @i18n_tasks.translation_config[:deepl_locale_aliases]
+      locale = locale_aliases[locale.to_s.downcase] || locale if locale_aliases.is_a?(Hash)
+
       loc, sub = locale.to_s.split('-')
       if SPECIFIC_TARGETS.include?(loc)
         # Must see how the deepl api evolves, so this could be an error in the future

data/lib/i18n/tasks/translators/google_translator.rb

@@ -17,14 +17,18 @@ module I18n::Tasks::Translators
     protected
 
     def translate_values(list, **options)
-      restore_newlines(
+      result = restore_newlines(
         EasyTranslate.translate(
           replace_newlines_with_placeholder(list, options[:html]),
           options,
-          format: :text
+          format: options[:html] ? :html : :text
         ),
         options[:html]
       )
+
+      @progress_bar.progress += result.size
+
+      result
     end
 
     def options_for_translate_values(from:, to:, **options)
@@ -69,7 +73,9 @@ module I18n::Tasks::Translators
       return list unless html
 
       list.map do |value|
-        value.gsub("\n", NEWLINE_PLACEHOLDER)
+        value.gsub(/\n(\s*)/) do
+          "<Z__#{::Regexp.last_match(1)&.length || 0}>"
+        end
       end
     end
 
@@ -77,7 +83,9 @@ module I18n::Tasks::Translators
       return translations unless html
 
       translations.map do |translation|
-        translation.gsub("#{NEWLINE_PLACEHOLDER} ", "\n")
+        translation.gsub(/<Z__(\d+)>/) do
+          "\n#{' ' * ::Regexp.last_match(1).to_i}"
+        end
       end
     end
   end

data/lib/i18n/tasks/translators/openai_translator.rb

@@ -18,6 +18,10 @@ module I18n::Tasks::Translators
       Variables (starting with %%{ and ending with }) must not be changed under any circumstance.
 
       Keep in mind the context of all the strings for a more accurate translation.
+      It is CRITICAL you output only the result, without any additional information, code block syntax or comments.
+    PROMPT
+    JSON_FORMAT_INSTRUCTIONS_SYSTEM_PROMPT = <<~PROMPT.squish
+      Return the translations as a JSON object with a 'translations' array containing the translated strings.
     PROMPT
 
     def initialize(*)
@@ -51,7 +55,7 @@ module I18n::Tasks::Translators
     private
 
     def translator
-      @translator ||= OpenAI::Client.new(access_token: api_key)
+      @translator ||= OpenAI::Client.new(access_token: api_key, log_errors: true)
     end
 
     def api_key
@@ -64,11 +68,14 @@ module I18n::Tasks::Translators
     end
 
     def model
-      @model ||= @i18n_tasks.translation_config[:openai_model].presence || 'gpt-3.5-turbo'
+      @model ||= @i18n_tasks.translation_config[:openai_model].presence || 'gpt-4o-mini'
     end
 
     def system_prompt
-      @system_prompt ||= @i18n_tasks.translation_config[:openai_system_prompt].presence || DEFAULT_SYSTEM_PROMPT
+      @system_prompt ||=
+        (@i18n_tasks.translation_config[:openai_system_prompt].presence || DEFAULT_SYSTEM_PROMPT)
+        .concat("\n#{JSON_FORMAT_INSTRUCTIONS_SYSTEM_PROMPT}")
+      @system_prompt
     end
 
     def translate_values(list, from:, to:)
@@ -76,15 +83,37 @@ module I18n::Tasks::Translators
 
       list.each_slice(BATCH_SIZE) do |batch|
         translations = translate(batch, from, to)
+        result = JSON.parse(translations)
+        results << result
 
-        results << JSON.parse(translations)
+        @progress_bar.progress += result.size
       end
 
       results.flatten
     end
 
     def translate(values, from, to)
-      messages = [
+      response = translator.chat(
+        parameters: {
+          model: model,
+          messages: build_messages(values, from, to),
+          temperature: 0.0,
+          response_format: { type: 'json_object' }
+        }
+      )
+
+      translations = response.dig('choices', 0, 'message', 'content')
+      error = response['error']
+
+      fail "AI error: #{error}" if error.present?
+
+      # Extract the array from the JSON object response
+      result = JSON.parse(translations)
+      result['translations'].to_json
+    end
+
+    def build_messages(values, from, to)
+      [
         {
           role: 'system',
           content: format(system_prompt, from: from, to: to)
@@ -98,21 +127,6 @@ module I18n::Tasks::Translators
           content: values.to_json
         }
       ]
-
-      response = translator.chat(
-        parameters: {
-          model: model,
-          messages: messages,
-          temperature: 0.0
-        }
-      )
-
-      translations = response.dig('choices', 0, 'message', 'content')
-      error = response['error']
-
-      fail "AI error: #{error}" if error.present?
-
-      translations
     end
   end
 end

data/lib/i18n/tasks/translators/watsonx_translator.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+require 'i18n/tasks/translators/base_translator'
+require 'active_support/core_ext/string/filters'
+
+module I18n::Tasks::Translators
+  class WatsonxTranslator < BaseTranslator
+    # max allowed texts per request
+    BATCH_SIZE = 50
+    DEFAULT_SYSTEM_PROMPT = <<~PROMPT.squish
+      You are a helpful assistant that translates content from the %{from} locale
+      to the %{to} locale in an i18n locale array.
+      You always preserve the structure and formatting exactly as it is.
+
+      The array has a structured format and contains multiple strings. Your task is to translate
+      each of these strings and create a new array with the translated strings.
+
+      Reminder:
+      - Translate only the text, preserving the structure and formatting.
+      - Do not translate any URLs.
+      - Do not translate HTML tags like `<details>` and `<summary>`.
+      - HTML markups (enclosed in < and > characters) must not be changed under any circumstance.
+      - Variables (starting with %%{ and ending with }) must not be changed under any circumstance.
+      - Output only the result, without any additional information or comments.
+    PROMPT
+
+    def options_for_translate_values(from:, to:, **options)
+      options.merge(
+        from: from,
+        to: to
+      )
+    end
+
+    def options_for_html
+      {}
+    end
+
+    def options_for_plain
+      {}
+    end
+
+    def no_results_error_message
+      I18n.t('i18n_tasks.watsonx_translate.errors.no_results')
+    end
+
+    private
+
+    def translator
+      @translator ||= WatsonxClient.new(key: api_key)
+    end
+
+    def api_key
+      @api_key ||= begin
+        key = @i18n_tasks.translation_config[:watsonx_api_key]
+        fail ::I18n::Tasks::CommandError, I18n.t('i18n_tasks.watsonx_translate.errors.no_api_key') if key.blank?
+
+        key
+      end
+    end
+
+    def project_id
+      @project_id ||= begin
+        project_id = @i18n_tasks.translation_config[:watsonx_project_id]
+        if project_id.blank?
+          fail ::I18n::Tasks::CommandError,
+               I18n.t('i18n_tasks.watsonx_translate.errors.no_project_id')
+        end
+
+        project_id
+      end
+    end
+
+    def model
+      @model ||= @i18n_tasks.translation_config[:watsonx_model].presence || 'meta-llama/llama-3-2-90b-vision-instruct'
+    end
+
+    def system_prompt
+      @system_prompt ||= @i18n_tasks.translation_config[:watsonx_system_prompt].presence || DEFAULT_SYSTEM_PROMPT
+    end
+
+    def translate_values(list, from:, to:)
+      results = []
+
+      list.each_slice(BATCH_SIZE) do |batch|
+        translations = translate(batch, from, to)
+        result = JSON.parse(translations)
+        results << result
+
+        @progress_bar.progress += results.size
+      end
+
+      results.flatten
+    end
+
+    def translate(values, from, to)
+      prompt = [
+        '<|eot_id|><|start_header_id|>system<|end_header_id|>',
+        format(system_prompt, from: from, to: to),
+        '<|eot_id|><|start_header_id|>user<|end_header_id|>Translate this array:',
+        "<|eot_id|><|start_header_id|>user<|end_header_id|>#{values.to_json}",
+        '<|eot_id|><|start_header_id|>assistant<|end_header_id|>'
+      ].join
+
+      response = translator.generate_text(
+        model_id: model,
+        project_id: project_id,
+        input: prompt,
+        parameters: {
+          decoding_method: :greedy,
+          max_new_tokens: 2048,
+          repetition_penalty: 1
+        }
+      )
+      response.dig('results', 0, 'generated_text')
+    end
+  end
+end
+
+class WatsonxClient
+  WATSONX_BASE_URL = 'https://us-south.ml.cloud.ibm.com/ml/'
+  IBM_CLOUD_IAM_URL = 'https://iam.cloud.ibm.com/identity/token'
+
+  def initialize(key:)
+    begin
+      require 'faraday'
+    rescue LoadError
+      raise ::I18n::Tasks::CommandError, "Add gem 'faraday' to your Gemfile to use this command"
+    end
+
+    @http = Faraday.new(url: WATSONX_BASE_URL) do |conn|
+      conn.use Faraday::Response::RaiseError
+      conn.request :json
+      conn.response :json
+      conn.options.timeout = 600
+      conn.request :authorization, :Bearer, token(key)
+    end
+  end
+
+  def generate_text(**opts)
+    @http.post('v1/text/generation?version=2024-05-20', **opts).body
+  end
+
+  private
+
+  def token(key)
+    Faraday.new(url: IBM_CLOUD_IAM_URL) do |conn|
+      conn.use Faraday::Response::RaiseError
+      conn.response :json
+      conn.params = {
+        grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
+        apikey: key
+      }
+    end.post.body['access_token']
+  end
+end

data/lib/i18n/tasks/translators/yandex_translator.rb

@@ -16,7 +16,11 @@ module I18n::Tasks::Translators
     protected
 
     def translate_values(list, **options)
-      list.map { |item| translator.translate(item, options) }
+      result = list.map { |item| translator.translate(item, options) }
+
+      @progress_bar.progress += result.size
+
+      result
     end
 
     def options_for_translate_values(from:, to:, **options)

data/lib/i18n/tasks/used_keys.rb

@@ -4,6 +4,7 @@ require 'find'
 require 'i18n/tasks/scanners/pattern_with_scope_scanner'
 require 'i18n/tasks/scanners/ruby_ast_scanner'
 require 'i18n/tasks/scanners/erb_ast_scanner'
+require 'i18n/tasks/scanners/prism_scanner'
 require 'i18n/tasks/scanners/scanner_multiplexer'
 require 'i18n/tasks/scanners/files/caching_file_finder_provider'
 require 'i18n/tasks/scanners/files/caching_file_reader'
@@ -21,8 +22,8 @@ module I18n::Tasks
       relative_roots: %w[app/controllers app/helpers app/mailers app/presenters app/views].freeze,
       scanners: [
         ['::I18n::Tasks::Scanners::RubyAstScanner', { only: %w[*.rb] }],
-        ['::I18n::Tasks::Scanners::ErbAstScanner', { only: %w[*.html.erb] }],
-        ['::I18n::Tasks::Scanners::PatternWithScopeScanner', { exclude: %w[*.html.erb *.rb] }]
+        ['::I18n::Tasks::Scanners::ErbAstScanner', { only: %w[*.erb] }],
+        ['::I18n::Tasks::Scanners::PatternWithScopeScanner', { exclude: %w[*.erb *.rb] }]
       ],
       ast_matchers: [],
       strict: true

data/lib/i18n/tasks/version.rb

@@ -2,6 +2,6 @@
 
 module I18n
   module Tasks
-    VERSION = '1.0.14'
+    VERSION = '1.0.15'
   end
 end

data/lib/i18n/tasks.rb

@@ -66,6 +66,7 @@ require 'active_support/core_ext/module/delegation'
 require 'active_support/core_ext/object/blank'
 require 'active_support/core_ext/object/try'
 
+require 'ruby-progressbar'
 require 'rainbow'
 require 'erubi'
 

data/templates/config/i18n-tasks.yml

@@ -34,7 +34,7 @@ data:
     ## Example (replace %#= with %=):
     # - "<%#= %x[bundle info vagrant --path].chomp %>/templates/locales/%{locale}.yml"
 
-  ## Specify the router (see Readme for details). Valid values: conservative_router, pattern_router, or a custom class.
+  ## Specify the router (see Readme for details). Valid values: conservative_router, pattern_router, isolating_router, or a custom class.
   # router: conservative_router
 
   yaml:
@@ -122,7 +122,7 @@ search:
 #     formality: prefer_less
 #   # OpenAI
 #   openai_api_key: "sk-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
-#   # openai_model: "gpt-3.5-turbo" # see https://platform.openai.com/docs/models
+#   # openai_model: "gpt-4o-mini" # see https://platform.openai.com/docs/models
 #   # may contain `%{from}` and `%{to}`, which will be replaced by source and target locale codes, respectively (using `Kernel.format`)
 #   # openai_system_prompt: >-
 #   #   You are a professional translator that translates content from the %{from} locale

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: i18n-tasks
 version: !ruby/object:Gem::Version
-  version: 1.0.14
+  version: 1.0.15
 platform: ruby
 authors:
 - glebm
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-05-10 00:00:00.000000000 Z
+date: 2025-03-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -128,6 +127,26 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.1
 - !ruby/object:Gem::Dependency
   name: terminal-table
   requirement: !ruby/object:Gem::Requirement
@@ -402,12 +421,17 @@ files:
 - lib/i18n/tasks/scanners/pattern_mapper.rb
 - lib/i18n/tasks/scanners/pattern_scanner.rb
 - lib/i18n/tasks/scanners/pattern_with_scope_scanner.rb
+- lib/i18n/tasks/scanners/prism_scanner.rb
+- lib/i18n/tasks/scanners/prism_scanners/arguments_visitor.rb
+- lib/i18n/tasks/scanners/prism_scanners/nodes.rb
+- lib/i18n/tasks/scanners/prism_scanners/visitor.rb
 - lib/i18n/tasks/scanners/relative_keys.rb
 - lib/i18n/tasks/scanners/results/key_occurrences.rb
 - lib/i18n/tasks/scanners/results/occurrence.rb
 - lib/i18n/tasks/scanners/ruby_ast_call_finder.rb
 - lib/i18n/tasks/scanners/ruby_ast_scanner.rb
 - lib/i18n/tasks/scanners/ruby_key_literals.rb
+- lib/i18n/tasks/scanners/ruby_parser_factory.rb
 - lib/i18n/tasks/scanners/scanner.rb
 - lib/i18n/tasks/scanners/scanner_multiplexer.rb
 - lib/i18n/tasks/split_key.rb
@@ -418,6 +442,7 @@ files:
 - lib/i18n/tasks/translators/deepl_translator.rb
 - lib/i18n/tasks/translators/google_translator.rb
 - lib/i18n/tasks/translators/openai_translator.rb
+- lib/i18n/tasks/translators/watsonx_translator.rb
 - lib/i18n/tasks/translators/yandex_translator.rb
 - lib/i18n/tasks/unused_keys.rb
 - lib/i18n/tasks/used_keys.rb
@@ -429,8 +454,10 @@ homepage: https://github.com/glebm/i18n-tasks
 licenses:
 - MIT
 metadata:
+  changelog_uri: https://github.com/glebm/i18n-tasks/blob/main/CHANGES.md
   issue_tracker: https://github.com/glebm/i18n-tasks
   rubygems_mfa_required: 'true'
+  source_code_uri: https://github.com/glebm/i18n-tasks
 post_install_message: |
   # Install default configuration:
   cp $(bundle exec i18n-tasks gem-path)/templates/config/i18n-tasks.yml config/
@@ -455,8 +482,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.3
-signing_key:
+rubygems_version: 3.6.3
 specification_version: 4
 summary: Manage localization and translation with the awesome power of static analysis
 test_files: []