i18n-tasks 1.0.12 → 1.0.14

data/lib/i18n/tasks/translators/openai_translator.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require 'i18n/tasks/translators/base_translator'
+require 'active_support/core_ext/string/filters'
+
+module I18n::Tasks::Translators
+  class OpenAiTranslator < BaseTranslator
+    # max allowed texts per request
+    BATCH_SIZE = 50
+    DEFAULT_SYSTEM_PROMPT = <<~PROMPT.squish
+      You are a professional translator that translates content from the %{from} locale
+      to the %{to} locale in an i18n locale array.
+
+      The array has a structured format and contains multiple strings. Your task is to translate
+      each of these strings and create a new array with the translated strings.
+
+      HTML markups (enclosed in < and > characters) must not be changed under any circumstance.
+      Variables (starting with %%{ and ending with }) must not be changed under any circumstance.
+
+      Keep in mind the context of all the strings for a more accurate translation.
+    PROMPT
+
+    def initialize(*)
+      begin
+        require 'openai'
+      rescue LoadError
+        raise ::I18n::Tasks::CommandError, "Add gem 'ruby-openai' to your Gemfile to use this command"
+      end
+      super
+    end
+
+    def options_for_translate_values(from:, to:, **options)
+      options.merge(
+        from: from,
+        to: to
+      )
+    end
+
+    def options_for_html
+      {}
+    end
+
+    def options_for_plain
+      {}
+    end
+
+    def no_results_error_message
+      I18n.t('i18n_tasks.openai_translate.errors.no_results')
+    end
+
+    private
+
+    def translator
+      @translator ||= OpenAI::Client.new(access_token: api_key)
+    end
+
+    def api_key
+      @api_key ||= begin
+        key = @i18n_tasks.translation_config[:openai_api_key]
+        fail ::I18n::Tasks::CommandError, I18n.t('i18n_tasks.openai_translate.errors.no_api_key') if key.blank?
+
+        key
+      end
+    end
+
+    def model
+      @model ||= @i18n_tasks.translation_config[:openai_model].presence || 'gpt-3.5-turbo'
+    end
+
+    def system_prompt
+      @system_prompt ||= @i18n_tasks.translation_config[:openai_system_prompt].presence || DEFAULT_SYSTEM_PROMPT
+    end
+
+    def translate_values(list, from:, to:)
+      results = []
+
+      list.each_slice(BATCH_SIZE) do |batch|
+        translations = translate(batch, from, to)
+
+        results << JSON.parse(translations)
+      end
+
+      results.flatten
+    end
+
+    def translate(values, from, to)
+      messages = [
+        {
+          role: 'system',
+          content: format(system_prompt, from: from, to: to)
+        },
+        {
+          role: 'user',
+          content: "Translate this array: \n\n\n"
+        },
+        {
+          role: 'user',
+          content: values.to_json
+        }
+      ]
+
+      response = translator.chat(
+        parameters: {
+          model: model,
+          messages: messages,
+          temperature: 0.0
+        }
+      )
+
+      translations = response.dig('choices', 0, 'message', 'content')
+      error = response['error']
+
+      fail "AI error: #{error}" if error.present?
+
+      translations
+    end
+  end
+end

data/lib/i18n/tasks/used_keys.rb

@@ -21,15 +21,15 @@ module I18n::Tasks
       relative_roots: %w[app/controllers app/helpers app/mailers app/presenters app/views].freeze,
       scanners: [
         ['::I18n::Tasks::Scanners::RubyAstScanner', { only: %w[*.rb] }],
-        ['::I18n::Tasks::Scanners::ErbAstScanner', { only: %w[*.erb] }],
-        ['::I18n::Tasks::Scanners::PatternWithScopeScanner', { exclude: %w[*.erb *.rb] }]
+        ['::I18n::Tasks::Scanners::ErbAstScanner', { only: %w[*.html.erb] }],
+        ['::I18n::Tasks::Scanners::PatternWithScopeScanner', { exclude: %w[*.html.erb *.rb] }]
       ],
       ast_matchers: [],
       strict: true
     }.freeze
 
-    ALWAYS_EXCLUDE = %w[*.jpg *.jpeg *.png *.gif *.svg *.ico *.eot *.otf *.ttf *.woff *.woff2 *.pdf *.css *.sass *.scss *.less
-                        *.yml *.json *.zip *.tar.gz *.swf *.flv *.mp3 *.wav *.flac *.webm *.mp4 *.ogg *.opus
+    ALWAYS_EXCLUDE = %w[*.jpg *.jpeg *.png *.gif *.svg *.ico *.eot *.otf *.ttf *.woff *.woff2 *.pdf *.css *.sass *.scss
+                        *.less *.yml *.json *.zip *.tar.gz *.swf *.flv *.mp3 *.wav *.flac *.webm *.mp4 *.ogg *.opus
                         *.webp *.map *.xlsx].freeze
 
     # Find all keys in the source and return a forest with the keys in absolute form and their occurrences.
@@ -142,7 +142,7 @@ module I18n::Tasks
 
     # keys in the source that end with a ., e.g. t("category.#{ cat.i18n_key }") or t("category." + category.key)
     # @param [String] replacement for interpolated values.
-    def expr_key_re(replacement: ':')
+    def expr_key_re(replacement: '*:')
       @expr_key_re ||= begin
         # disallow patterns with no keys
         ignore_pattern_re = /\A[.#{replacement}]*\z/

data/lib/i18n/tasks/version.rb

@@ -2,6 +2,6 @@
 
 module I18n
   module Tasks
-    VERSION = '1.0.12'
+    VERSION = '1.0.14'
   end
 end

data/templates/config/i18n-tasks.yml

@@ -13,7 +13,7 @@ data:
   ## Provide a custom adapter:
   # adapter: I18n::Tasks::Data::FileSystem
 
-  # Locale files or `File.find` patterns where translations are read from:
+  # Locale files or `Dir.glob` patterns where translations are read from:
   read:
     ## Default:
     # - config/locales/%{locale}.yml
@@ -52,7 +52,7 @@ data:
 
 # Find translate calls
 search:
-  ## Paths or `File.find` patterns to search in:
+  ## Paths or `Find.find` patterns to search in:
   # paths:
   #  - app/
 
@@ -92,9 +92,13 @@ search:
   ## - RailsModelMatcher
   ##     Matches ActiveRecord translations like
   ##     User.human_attribute_name(:email) and User.model_name.human
+  ## - DefaultI18nSubjectMatcher
+  ##     Matches ActionMailer's default_i18n_subject method
   ##
   ## To implement your own, please see `I18n::Tasks::Scanners::AstMatchers::BaseMatcher`.
-  <%# I18n::Tasks.add_ast_matcher('I18n::Tasks::Scanners::AstMatchers::RailsModelMatcher') %>
+  # ast_matchers:
+  #   - 'I18n::Tasks::Scanners::AstMatchers::RailsModelMatcher'
+  #   - 'I18n::Tasks::Scanners::AstMatchers::DefaultI18nSubjectMatcher'
 
   ## Multiple scanners can be used. Their results are merged.
   ## The options specified above are passed down to each scanner. Per-scanner options can be specified as well.
@@ -110,6 +114,27 @@ search:
 #   deepl_api_key: "48E92789-57A3-466A-9959-1A1A1A1A1A1A"
 #   # deepl_host: "https://api.deepl.com"
 #   # deepl_version: "v2"
+#   # deepl_glossary_ids:
+#   #   - f28106eb-0e06-489e-82c6-8215d6f95089
+#   #   - 2c6415be-1852-4f54-9e1b-d800463496b4
+#   # add additional options to the DeepL.translate call: https://www.deepl.com/docs-api/translate-text/translate-text/
+#   deepl_options:
+#     formality: prefer_less
+#   # OpenAI
+#   openai_api_key: "sk-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+#   # openai_model: "gpt-3.5-turbo" # see https://platform.openai.com/docs/models
+#   # may contain `%{from}` and `%{to}`, which will be replaced by source and target locale codes, respectively (using `Kernel.format`)
+#   # openai_system_prompt: >-
+#   #   You are a professional translator that translates content from the %{from} locale
+#   #   to the %{to} locale in an i18n locale array.
+#   #
+#   #   The array has a structured format and contains multiple strings. Your task is to translate
+#   #   each of these strings and create a new array with the translated strings.
+#   #
+#   #   HTML markups (enclosed in < and > characters) must not be changed under any circumstance.
+#   #   Variables (starting with %%{ and ending with }) must not be changed under any circumstance.
+#   #
+#   #   Keep in mind the context of all the strings for a more accurate translation.
 
 ## Do not consider these keys missing:
 # ignore_missing:

data/templates/minitest/i18n_test.rb

@@ -5,18 +5,18 @@ require 'i18n/tasks'
 class I18nTest < ActiveSupport::TestCase
   def setup
     @i18n = I18n::Tasks::BaseTask.new
-    @missing_keys = @i18n.missing_keys
-    @unused_keys = @i18n.unused_keys
   end
 
   def test_no_missing_keys
-    assert_empty @missing_keys,
-                 "Missing #{@missing_keys.leaves.count} i18n keys, run `i18n-tasks missing' to show them"
+    missing_keys = @i18n.missing_keys
+    assert_empty missing_keys,
+                 "Missing #{missing_keys.leaves.count} i18n keys, run `i18n-tasks missing' to show them"
   end
 
   def test_no_unused_keys
-    assert_empty @unused_keys,
-                 "#{@unused_keys.leaves.count} unused i18n keys, run `i18n-tasks unused' to show them"
+    unused_keys = @i18n.unused_keys
+    assert_empty unused_keys,
+                 "#{unused_keys.leaves.count} unused i18n keys, run `i18n-tasks unused' to show them"
   end
 
   def test_files_are_normalized

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: i18n-tasks
 version: !ruby/object:Gem::Version
-  version: 1.0.12
+  version: 1.0.14
 platform: ruby
 authors:
 - glebm
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-09-18 00:00:00.000000000 Z
+date: 2024-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -38,26 +38,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.1.0
-- !ruby/object:Gem::Dependency
-  name: better_html
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.0'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.0'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: erubi
   requirement: !ruby/object:Gem::Requirement
@@ -106,14 +86,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.3.0
+        version: 3.2.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.3.0
+        version: 3.2.2.1
 - !ruby/object:Gem::Dependency
   name: rails-i18n
   requirement: !ruby/object:Gem::Requirement
@@ -230,14 +210,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.27.0
+        version: 1.50.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.50.1
+- !ruby/object:Gem::Dependency
+  name: rubocop-rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.19.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.27.0
+        version: 2.19.0
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -360,6 +368,7 @@ files:
 - lib/i18n/tasks/data/file_system.rb
 - lib/i18n/tasks/data/file_system_base.rb
 - lib/i18n/tasks/data/router/conservative_router.rb
+- lib/i18n/tasks/data/router/isolating_router.rb
 - lib/i18n/tasks/data/router/pattern_router.rb
 - lib/i18n/tasks/data/tree/node.rb
 - lib/i18n/tasks/data/tree/nodes.rb
@@ -378,9 +387,9 @@ files:
 - lib/i18n/tasks/reports/base.rb
 - lib/i18n/tasks/reports/terminal.rb
 - lib/i18n/tasks/scanners/ast_matchers/base_matcher.rb
+- lib/i18n/tasks/scanners/ast_matchers/default_i18n_subject_matcher.rb
 - lib/i18n/tasks/scanners/ast_matchers/message_receivers_matcher.rb
 - lib/i18n/tasks/scanners/ast_matchers/rails_model_matcher.rb
-- lib/i18n/tasks/scanners/erb_ast_processor.rb
 - lib/i18n/tasks/scanners/erb_ast_scanner.rb
 - lib/i18n/tasks/scanners/file_scanner.rb
 - lib/i18n/tasks/scanners/files/caching_file_finder.rb
@@ -408,6 +417,7 @@ files:
 - lib/i18n/tasks/translators/base_translator.rb
 - lib/i18n/tasks/translators/deepl_translator.rb
 - lib/i18n/tasks/translators/google_translator.rb
+- lib/i18n/tasks/translators/openai_translator.rb
 - lib/i18n/tasks/translators/yandex_translator.rb
 - lib/i18n/tasks/unused_keys.rb
 - lib/i18n/tasks/used_keys.rb
@@ -445,8 +455,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.5.3
+signing_key:
 specification_version: 4
 summary: Manage localization and translation with the awesome power of static analysis
 test_files: []

data/lib/i18n/tasks/scanners/erb_ast_processor.rb

@@ -1,74 +0,0 @@
-# frozen_string_literal: true
-
-require 'ast'
-require 'set'
-require 'i18n/tasks/scanners/local_ruby_parser'
-
-module I18n::Tasks::Scanners
-  class ErbAstProcessor
-    include AST::Processor::Mixin
-    def initialize
-      super()
-      @ruby_parser = LocalRubyParser.new(ignore_blocks: true)
-      @comments = []
-    end
-
-    def process_and_extract_comments(ast)
-      result = process(ast)
-      [result, @comments]
-    end
-
-    def on_code(node)
-      parsed, comments = @ruby_parser.parse(
-        node.children[0],
-        location: node.location
-      )
-      @comments.concat(comments)
-
-      unless parsed.nil?
-        parsed = parsed.updated(
-          nil,
-          parsed.children.map { |child| node?(child) ? process(child) : child }
-        )
-        node = node.updated(:send, parsed)
-      end
-      node
-    end
-
-    # @param node [::Parser::AST::Node]
-    # @return [::Parser::AST::Node]
-    def handler_missing(node)
-      node = handle_comment(node)
-      return if node.nil?
-
-      node.updated(
-        nil,
-        node.children.map { |child| node?(child) ? process(child) : child }
-      )
-    end
-
-    private
-
-    # Convert ERB-comments to ::Parser::Source::Comment and skip processing node
-    #
-    # @param node Parser::AST::Node Potential comment node
-    # @return Parser::AST::Node or nil
-    def handle_comment(node)
-      if node.type == :erb && node.children.size == 4 &&
-         node.children[0]&.type == :indicator && node.children[0].children[0] == '#' &&
-         node.children[2]&.type == :code
-
-        # Do not continue parsing this node
-        comment = node.children[2]
-        @comments << ::Parser::Source::Comment.new(comment.location.expression)
-        return
-      end
-
-      node
-    end
-
-    def node?(node)
-      node.is_a?(::Parser::AST::Node)
-    end
-  end
-end