i18n-tasks 1.0.14 → 1.1.0

data/lib/i18n/tasks/translators/deepl_translator.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'i18n/tasks/translators/base_translator'
+require "i18n/tasks/translators/base_translator"
 
 module I18n::Tasks::Translators
   class DeeplTranslator < BaseTranslator
@@ -11,7 +11,7 @@ module I18n::Tasks::Translators
 
     def initialize(*)
       begin
-        require 'deepl'
+        require "deepl"
       rescue LoadError
         raise ::I18n::Tasks::CommandError, "Add gem 'deepl-rb' to your Gemfile to use this command"
       end
@@ -35,6 +35,8 @@ module I18n::Tasks::Translators
         else
           results += res.map(&:text)
         end
+
+        @progress_bar.progress += parts.size
       end
       results
     end
@@ -42,15 +44,15 @@ module I18n::Tasks::Translators
     def options_for_translate_values(**options)
       extra_options = @i18n_tasks.translation_config[:deepl_options]&.symbolize_keys || {}
 
-      extra_options.merge({ ignore_tags: %w[i18n] }).merge(options)
+      extra_options.merge({ignore_tags: %w[i18n]}).merge(options)
     end
 
     def options_for_html
-      { tag_handling: 'xml' }
+      {tag_handling: "xml"}
     end
 
     def options_for_plain
-      { preserve_formatting: true, tag_handling: 'xml', html_escape: true }
+      {preserve_formatting: true, tag_handling: "xml", html_escape: true}
     end
 
     # @param [String] value
@@ -65,28 +67,31 @@ module I18n::Tasks::Translators
     def restore_interpolations(untranslated, translated)
       return translated if untranslated !~ INTERPOLATION_KEY_RE
 
-      translated.gsub(%r{</?i18n>}, '')
-    rescue StandardError => e
+      translated.gsub(%r{</?i18n>}, "")
+    rescue => e
       raise_interpolation_error(untranslated, translated, e)
     end
 
     def no_results_error_message
-      I18n.t('i18n_tasks.deepl_translate.errors.no_results')
+      I18n.t("i18n_tasks.deepl_translate.errors.no_results")
     end
 
     private
 
     # Convert 'es-ES' to 'ES', en-us to EN
     def to_deepl_source_locale(locale)
-      locale.to_s.split('-', 2).first.upcase
+      locale.to_s.split("-", 2).first.upcase
     end
 
     # Convert 'es-ES' to 'ES' but warn about locales requiring a specific variant
     def to_deepl_target_locale(locale)
-      loc, sub = locale.to_s.split('-')
+      locale_aliases = @i18n_tasks.translation_config[:deepl_locale_aliases]
+      locale = locale_aliases[locale.to_s.downcase] || locale if locale_aliases.is_a?(Hash)
+
+      loc, sub = locale.to_s.split("-")
       if SPECIFIC_TARGETS.include?(loc)
         # Must see how the deepl api evolves, so this could be an error in the future
-        warn_deprecated I18n.t('i18n_tasks.deepl_translate.errors.specific_target_missing') unless sub
+        warn_deprecated I18n.t("i18n_tasks.deepl_translate.errors.specific_target_missing") unless sub
         locale.to_s.upcase
       else
         loc.upcase
@@ -97,7 +102,7 @@ module I18n::Tasks::Translators
       api_key = @i18n_tasks.translation_config[:deepl_api_key]
       host = @i18n_tasks.translation_config[:deepl_host]
       version = @i18n_tasks.translation_config[:deepl_version]
-      fail ::I18n::Tasks::CommandError, I18n.t('i18n_tasks.deepl_translate.errors.no_api_key') if api_key.blank?
+      fail ::I18n::Tasks::CommandError, I18n.t("i18n_tasks.deepl_translate.errors.no_api_key") if api_key.blank?
 
       DeepL.configure do |config|
         config.auth_key = api_key
@@ -106,9 +111,12 @@ module I18n::Tasks::Translators
       end
     end
 
+    # The Free API endpoint doesn’t expose glossaries via DeepL.glossaries.list,
+    # so if no API-backed glossary is found, fall back to the first ID from i18n-config.yml.
     def options_with_glossary(options, from, to)
-      glossary = find_glossary(from, to)
-      glossary ? { glossary_id: glossary.id }.merge(options) : options
+      configured = @i18n_tasks.translation_config[:deepl_glossary_ids]
+      gid = find_glossary(from, to)&.id || configured&.first
+      gid ? {glossary_id: gid}.merge(options) : options
     end
 
     def all_ready_glossaries

data/lib/i18n/tasks/translators/google_translator.rb

@@ -1,30 +1,47 @@
 # frozen_string_literal: true
 
-require 'i18n/tasks/translators/base_translator'
+require "cgi"
+require "net/http"
+require "uri"
+require "json"
+require "i18n/tasks/translators/base_translator"
+
+# https://cloud.google.com/translate/docs/reference/rest/v2/translate
 
 module I18n::Tasks::Translators
   class GoogleTranslator < BaseTranslator
-    NEWLINE_PLACEHOLDER = '<br id=i18n />'
+    API_ENDPOINT = "https://translation.googleapis.com/language/translate/v2"
+    MAX_TEXTS_PER_REQUEST = 128
+    MAX_CHARS_PER_REQUEST = 30_000
+
+    class Error < StandardError; end
+
+    class RateLimitError < Error; end
+
+    class QuotaExceededError < Error; end
+
     def initialize(*)
-      begin
-        require 'easy_translate'
-      rescue LoadError
-        raise ::I18n::Tasks::CommandError, "Add gem 'easy_translate' to your Gemfile to use this command"
-      end
       super
     end
 
     protected
 
     def translate_values(list, **options)
-      restore_newlines(
-        EasyTranslate.translate(
-          replace_newlines_with_placeholder(list, options[:html]),
-          options,
-          format: :text
+      html = options[:html].present?
+      result = restore_newlines(
+        translate(
+          texts: replace_newlines_with_placeholder(list),
+          api_key: api_key,
+          to: options[:to],
+          from: options[:from],
+          html:
         ),
-        options[:html]
+        html:
       )
+
+      @progress_bar.progress += result.size
+
+      result
     end
 
     def options_for_translate_values(from:, to:, **options)
@@ -36,15 +53,15 @@ module I18n::Tasks::Translators
     end
 
     def options_for_html
-      { html: true }
+      {html: true}
     end
 
     def options_for_plain
-      { format: 'text' }
+      {format: "text"}
     end
 
     def no_results_error_message
-      I18n.t('i18n_tasks.google_translate.errors.no_results')
+      I18n.t("i18n_tasks.google_translate.errors.no_results")
     end
 
     private
@@ -55,29 +72,164 @@ module I18n::Tasks::Translators
         # fallback with deprecation warning
         if @i18n_tasks.translation_config[:api_key]
           warn_deprecated(
-            'Please rename Google Translate API Key from `api_key` to `google_translate_api_key`.'
+            "Please rename Google Translate API Key from `api_key` to `google_translate_api_key`."
           )
           key ||= translation_config[:api_key]
         end
-        fail ::I18n::Tasks::CommandError, I18n.t('i18n_tasks.google_translate.errors.no_api_key') if key.blank?
+        fail ::I18n::Tasks::CommandError, I18n.t("i18n_tasks.google_translate.errors.no_api_key") if key.blank?
 
         key
       end
     end
 
-    def replace_newlines_with_placeholder(list, html)
-      return list unless html
-
+    def replace_newlines_with_placeholder(list)
       list.map do |value|
-        value.gsub("\n", NEWLINE_PLACEHOLDER)
+        value.gsub(/\n(\s*)/) do
+          "<Z__#{::Regexp.last_match(1)&.length || 0}>"
+        end
+      end
+    end
+
+    def restore_newlines(translations, html:)
+      restored = translations.map do |translation|
+        translation.gsub(/<Z__(\d+)>/) do
+          "\n#{" " * ::Regexp.last_match(1).to_i}"
+        end
+      end
+
+      # Need to unescape if translating HTML content
+      html ? restored.map { |t| CGI.unescapeHTML(t) } : restored
+    end
+
+    # @param texts [Array<String>] texts to translate
+    # @param api_key [String] Google Translate API key (required)
+    # @param to [String] target language code (required)
+    # @param format [Symbol] :text or :html (required)
+    # @param from [String] source language code
+    # @return [Array<String>] translated texts
+    def translate(texts:, api_key:, to:, html:, from: nil)
+      texts = Array(texts)
+
+      return [] if texts.empty?
+
+      raise ArgumentError, "api_key is required" if api_key.nil? || api_key.empty?
+
+      raise ArgumentError, "target language (:to) is required" if to.nil? || to.empty?
+
+      # Split into batches if needed
+      batches = batch_texts(texts)
+
+      batches.flat_map do |batch|
+        translate_batch(texts: batch, api_key:, to:, from:, html:)
+      end
+    end
+
+    def batch_texts(texts)
+      batches = []
+      current_batch = []
+      current_chars = 0
+
+      texts.each do |text|
+        text_length = text.to_s.length
+
+        # If adding this text would exceed limits, start new batch
+        if current_batch.size >= MAX_TEXTS_PER_REQUEST ||
+            (current_chars + text_length > MAX_CHARS_PER_REQUEST && current_batch.any?)
+          batches << current_batch
+          current_batch = []
+          current_chars = 0
+        end
+
+        current_batch << text
+        current_chars += text_length
+      end
+
+      batches << current_batch if current_batch.any?
+      batches
+    end
+
+    def translate_batch(texts:, api_key:, to:, from:, html:)
+      uri = URI(API_ENDPOINT)
+      # Provide API key via query string as expected by v2 API
+      uri.query = URI.encode_www_form(key: api_key)
+
+      body = {
+        q: texts,
+        target: to,
+        format: html ? "html" : "text"
+      }
+
+      # Add source language if specified
+      if from
+        body[:source] = from
       end
+
+      response = make_request(uri, body)
+      parse_response(response, texts.size)
     end
 
-    def restore_newlines(translations, html)
-      return translations unless html
+    def make_request(uri, body)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = true
+      http.read_timeout = 60
+      http.open_timeout = 30
+
+      request = Net::HTTP::Post.new(uri.request_uri)
+      request["Content-Type"] = "application/json"
+      request["Accept"] = "application/json"
+      request["User-Agent"] = "i18n-tasks/GoogleTranslateApi"
+      request.body = JSON.generate(body)
+
+      response = http.request(request)
+
+      handle_error_response(response) unless response.is_a?(Net::HTTPSuccess)
+      response
+    end
+
+    def parse_response(response, expected_count)
+      data = JSON.parse(response.body)
+
+      unless data["data"] && data["data"]["translations"]
+        raise Error, "Unexpected API response format"
+      end
+
+      translations = data["data"]["translations"].map do |translation|
+        translation["translatedText"]
+      end
+
+      if translations.size != expected_count
+        raise Error, "Expected #{expected_count} translations, got #{translations.size}"
+      end
+
+      translations
+    rescue JSON::ParserError => e
+      raise Error, "Failed to parse API response: #{e.message}"
+    end
+
+    def handle_error_response(response)
+      begin
+        error_data = JSON.parse(response.body)
+        error_message = error_data.dig("error", "message") || "Unknown error"
+        error_code = error_data.dig("error", "code")
+      rescue JSON::ParserError
+        error_message = response.body
+        error_code = response.code
+      end
 
-      translations.map do |translation|
-        translation.gsub("#{NEWLINE_PLACEHOLDER} ", "\n")
+      status = response.code.to_i
+      diagnostic = "(status=#{status}, code=#{error_code})"
+      case status
+      when 400
+        raise Error, "Bad request #{diagnostic}: #{error_message}"
+      when 401, 403
+        raise Error, "Authentication/authorization failed #{diagnostic}: #{error_message}. " \
+          "Ensure the Cloud Translation API v2 is enabled, billing is active, and the key is unrestricted for this API."
+      when 429
+        raise RateLimitError, "Rate limit exceeded #{diagnostic}: #{error_message}"
+      when 503
+        raise QuotaExceededError, "Quota exceeded #{diagnostic}: #{error_message}"
+      else
+        raise Error, "API error #{diagnostic}: #{error_message}"
       end
     end
   end

data/lib/i18n/tasks/translators/openai_translator.rb

@@ -1,10 +1,12 @@
 # frozen_string_literal: true
 
-require 'i18n/tasks/translators/base_translator'
-require 'active_support/core_ext/string/filters'
+require "i18n/tasks/translators/base_translator"
+require "active_support/core_ext/string/filters"
 
 module I18n::Tasks::Translators
   class OpenAiTranslator < BaseTranslator
+    include ::I18n::Tasks::Data::LanguageNames
+
     # max allowed texts per request
     BATCH_SIZE = 50
     DEFAULT_SYSTEM_PROMPT = <<~PROMPT.squish
@@ -18,11 +20,15 @@ module I18n::Tasks::Translators
       Variables (starting with %%{ and ending with }) must not be changed under any circumstance.
 
       Keep in mind the context of all the strings for a more accurate translation.
+      It is CRITICAL you output only the result, without any additional information, code block syntax or comments.
+    PROMPT
+    JSON_FORMAT_INSTRUCTIONS_SYSTEM_PROMPT = <<~PROMPT.squish
+      Return the translations as a JSON object with a 'translations' array containing the translated strings.
     PROMPT
 
     def initialize(*)
       begin
-        require 'openai'
+        require "openai"
       rescue LoadError
         raise ::I18n::Tasks::CommandError, "Add gem 'ruby-openai' to your Gemfile to use this command"
       end
@@ -45,74 +51,93 @@ module I18n::Tasks::Translators
     end
 
     def no_results_error_message
-      I18n.t('i18n_tasks.openai_translate.errors.no_results')
+      I18n.t("i18n_tasks.openai_translate.errors.no_results")
     end
 
     private
 
     def translator
-      @translator ||= OpenAI::Client.new(access_token: api_key)
+      @translator ||= OpenAI::Client.new(access_token: api_key, log_errors: true)
     end
 
     def api_key
       @api_key ||= begin
         key = @i18n_tasks.translation_config[:openai_api_key]
-        fail ::I18n::Tasks::CommandError, I18n.t('i18n_tasks.openai_translate.errors.no_api_key') if key.blank?
+        fail ::I18n::Tasks::CommandError, I18n.t("i18n_tasks.openai_translate.errors.no_api_key") if key.blank?
 
         key
       end
     end
 
     def model
-      @model ||= @i18n_tasks.translation_config[:openai_model].presence || 'gpt-3.5-turbo'
+      @model ||= @i18n_tasks.translation_config[:openai_model].presence || "gpt-4o-mini"
+    end
+
+    def temperature
+      @temperature ||= @i18n_tasks.translation_config[:openai_temperature].presence || 0.0
+    end
+
+    def system_prompt(to_locale)
+      prompt = if locale_prompts[to_locale].present?
+        locale_prompts[to_locale]
+      else
+        @i18n_tasks.translation_config[:openai_system_prompt].presence || DEFAULT_SYSTEM_PROMPT
+      end
+
+      prompt.concat("\n#{JSON_FORMAT_INSTRUCTIONS_SYSTEM_PROMPT}")
     end
 
-    def system_prompt
-      @system_prompt ||= @i18n_tasks.translation_config[:openai_system_prompt].presence || DEFAULT_SYSTEM_PROMPT
+    def locale_prompts
+      @locale_prompts ||= @i18n_tasks.translation_config[:openai_locale_prompts] || {}
     end
 
     def translate_values(list, from:, to:)
       results = []
 
       list.each_slice(BATCH_SIZE) do |batch|
-        translations = translate(batch, from, to)
+        result = translate(batch, from, to)
+        results << result
 
-        results << JSON.parse(translations)
+        @progress_bar.progress += result.size
       end
 
       results.flatten
     end
 
     def translate(values, from, to)
-      messages = [
-        {
-          role: 'system',
-          content: format(system_prompt, from: from, to: to)
-        },
-        {
-          role: 'user',
-          content: "Translate this array: \n\n\n"
-        },
-        {
-          role: 'user',
-          content: values.to_json
-        }
-      ]
-
       response = translator.chat(
         parameters: {
           model: model,
-          messages: messages,
-          temperature: 0.0
+          messages: build_messages(values, from, to),
+          temperature: temperature,
+          response_format: {type: "json_object"}
         }
       )
 
-      translations = response.dig('choices', 0, 'message', 'content')
-      error = response['error']
+      translations = response.dig("choices", 0, "message", "content")
+      error = response["error"]
 
       fail "AI error: #{error}" if error.present?
 
-      translations
+      # Extract the array from the JSON object response
+      JSON.parse(translations)["translations"]
+    end
+
+    def build_messages(values, from, to)
+      [
+        {
+          role: "system",
+          content: format(system_prompt(to), from: language_name(from), to: language_name(to))
+        },
+        {
+          role: "user",
+          content: "Translate this array: \n\n\n"
+        },
+        {
+          role: "user",
+          content: values.to_json
+        }
+      ]
     end
   end
 end

data/lib/i18n/tasks/translators/watsonx_translator.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+require "i18n/tasks/translators/base_translator"
+require "active_support/core_ext/string/filters"
+
+module I18n::Tasks::Translators
+  class WatsonxTranslator < BaseTranslator
+    # max allowed texts per request
+    BATCH_SIZE = 50
+    DEFAULT_SYSTEM_PROMPT = <<~PROMPT.squish
+      You are a helpful assistant that translates content from the %{from} locale
+      to the %{to} locale in an i18n locale array.
+      You always preserve the structure and formatting exactly as it is.
+
+      The array has a structured format and contains multiple strings. Your task is to translate
+      each of these strings and create a new array with the translated strings.
+
+      Reminder:
+      - Translate only the text, preserving the structure and formatting.
+      - Do not translate any URLs.
+      - Do not translate HTML tags like `<details>` and `<summary>`.
+      - HTML markups (enclosed in < and > characters) must not be changed under any circumstance.
+      - Variables (starting with %%{ and ending with }) must not be changed under any circumstance.
+      - Output only the result, without any additional information or comments.
+    PROMPT
+
+    def options_for_translate_values(from:, to:, **options)
+      options.merge(
+        from: from,
+        to: to
+      )
+    end
+
+    def options_for_html
+      {}
+    end
+
+    def options_for_plain
+      {}
+    end
+
+    def no_results_error_message
+      I18n.t("i18n_tasks.watsonx_translate.errors.no_results")
+    end
+
+    private
+
+    def translator
+      @translator ||= WatsonxClient.new(key: api_key)
+    end
+
+    def api_key
+      @api_key ||= begin
+        key = @i18n_tasks.translation_config[:watsonx_api_key]
+        fail ::I18n::Tasks::CommandError, I18n.t("i18n_tasks.watsonx_translate.errors.no_api_key") if key.blank?
+
+        key
+      end
+    end
+
+    def project_id
+      @project_id ||= begin
+        project_id = @i18n_tasks.translation_config[:watsonx_project_id]
+        if project_id.blank?
+          fail ::I18n::Tasks::CommandError,
+            I18n.t("i18n_tasks.watsonx_translate.errors.no_project_id")
+        end
+
+        project_id
+      end
+    end
+
+    def model
+      @model ||= @i18n_tasks.translation_config[:watsonx_model].presence || "meta-llama/llama-3-2-90b-vision-instruct"
+    end
+
+    def system_prompt
+      @system_prompt ||= @i18n_tasks.translation_config[:watsonx_system_prompt].presence || DEFAULT_SYSTEM_PROMPT
+    end
+
+    def translate_values(list, from:, to:)
+      results = []
+
+      list.each_slice(BATCH_SIZE) do |batch|
+        translations = translate(batch, from, to)
+        result = JSON.parse(translations)
+        results << result
+
+        @progress_bar.progress += results.size
+      end
+
+      results.flatten
+    end
+
+    def translate(values, from, to)
+      prompt = [
+        "<|eot_id|><|start_header_id|>system<|end_header_id|>",
+        format(system_prompt, from: from, to: to),
+        "<|eot_id|><|start_header_id|>user<|end_header_id|>Translate this array:",
+        "<|eot_id|><|start_header_id|>user<|end_header_id|>#{values.to_json}",
+        "<|eot_id|><|start_header_id|>assistant<|end_header_id|>"
+      ].join
+
+      response = translator.generate_text(
+        model_id: model,
+        project_id: project_id,
+        input: prompt,
+        parameters: {
+          decoding_method: :greedy,
+          max_new_tokens: 2048,
+          repetition_penalty: 1
+        }
+      )
+      response.dig("results", 0, "generated_text")
+    end
+  end
+end
+
+class WatsonxClient
+  WATSONX_BASE_URL = "https://us-south.ml.cloud.ibm.com/ml/"
+  IBM_CLOUD_IAM_URL = "https://iam.cloud.ibm.com/identity/token"
+
+  def initialize(key:)
+    begin
+      require "faraday"
+    rescue LoadError
+      raise ::I18n::Tasks::CommandError, "Add gem 'faraday' to your Gemfile to use this command"
+    end
+
+    @http = Faraday.new(url: WATSONX_BASE_URL) do |conn|
+      conn.use Faraday::Response::RaiseError
+      conn.request :json
+      conn.response :json
+      conn.options.timeout = 600
+      conn.request :authorization, :Bearer, token(key)
+    end
+  end
+
+  def generate_text(**opts)
+    @http.post("v1/text/generation?version=2024-05-20", **opts).body
+  end
+
+  private
+
+  def token(key)
+    Faraday.new(url: IBM_CLOUD_IAM_URL) do |conn|
+      conn.use Faraday::Response::RaiseError
+      conn.response :json
+      conn.params = {
+        grant_type: "urn:ibm:params:oauth:grant-type:apikey",
+        apikey: key
+      }
+    end.post.body["access_token"]
+  end
+end