i18n-tasks 0.9.37 → 1.0.13

data/lib/i18n/tasks/translators/openai_translator.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require 'i18n/tasks/translators/base_translator'
+
+module I18n::Tasks::Translators
+  class OpenAiTranslator < BaseTranslator
+    # max allowed texts per request
+    BATCH_SIZE = 50
+
+    def initialize(*)
+      begin
+        require 'openai'
+      rescue LoadError
+        raise ::I18n::Tasks::CommandError, "Add gem 'ruby-openai' to your Gemfile to use this command"
+      end
+      super
+    end
+
+    def options_for_translate_values(from:, to:, **options)
+      options.merge(
+        from: from,
+        to: to
+      )
+    end
+
+    def options_for_html
+      {}
+    end
+
+    def options_for_plain
+      {}
+    end
+
+    def no_results_error_message
+      I18n.t('i18n_tasks.openai_translate.errors.no_results')
+    end
+
+    private
+
+    def translator
+      @translator ||= OpenAI::Client.new(access_token: api_key)
+    end
+
+    def api_key
+      @api_key ||= begin
+        key = @i18n_tasks.translation_config[:openai_api_key]
+        fail ::I18n::Tasks::CommandError, I18n.t('i18n_tasks.openai_translate.errors.no_api_key') if key.blank?
+
+        key
+      end
+    end
+
+    def translate_values(list, from:, to:)
+      results = []
+
+      list.each_slice(BATCH_SIZE) do |batch|
+        translations = translate(batch, from, to)
+
+        results << JSON.parse(translations)
+      end
+
+      results.flatten
+    end
+
+    def translate(values, from, to) # rubocop:disable Metrics/MethodLength
+      messages = [
+        {
+          role: 'system',
+          content: "You are a helpful assistant that translates content from the #{from} to #{to} locale in an i18n
+          locale array. The array has a structured format and contains multiple strings. Your task is to translate
+          each of these strings and create a new array with the translated strings. Keep in mind the context of all
+          the strings for a more accurate translation.\n"
+        },
+        {
+          role: 'user',
+          content: "Translate this array: \n\n\n"
+        },
+        {
+          role: 'user',
+          content: values.to_json
+        }
+      ]
+
+      response = translator.chat(
+        parameters: {
+          model: 'gpt-3.5-turbo',
+          messages: messages,
+          temperature: 0.7
+        }
+      )
+
+      translations = response.dig('choices', 0, 'message', 'content')
+      error = response['error']
+
+      fail "AI error: #{error}" if error.present?
+
+      translations
+    end
+  end
+end

data/lib/i18n/tasks/used_keys.rb

@@ -3,6 +3,7 @@
 require 'find'
 require 'i18n/tasks/scanners/pattern_with_scope_scanner'
 require 'i18n/tasks/scanners/ruby_ast_scanner'
+require 'i18n/tasks/scanners/erb_ast_scanner'
 require 'i18n/tasks/scanners/scanner_multiplexer'
 require 'i18n/tasks/scanners/files/caching_file_finder_provider'
 require 'i18n/tasks/scanners/files/caching_file_reader'
@@ -20,14 +21,16 @@ module I18n::Tasks
       relative_roots: %w[app/controllers app/helpers app/mailers app/presenters app/views].freeze,
       scanners: [
         ['::I18n::Tasks::Scanners::RubyAstScanner', { only: %w[*.rb] }],
-        ['::I18n::Tasks::Scanners::PatternWithScopeScanner', { exclude: %w[*.rb] }]
+        ['::I18n::Tasks::Scanners::ErbAstScanner', { only: %w[*.erb] }],
+        ['::I18n::Tasks::Scanners::PatternWithScopeScanner', { exclude: %w[*.erb *.rb] }]
       ],
+      ast_matchers: [],
       strict: true
     }.freeze
 
-    ALWAYS_EXCLUDE = %w[*.jpg *.jpeg *.png *.gif *.svg *.ico *.eot *.otf *.ttf *.woff *.woff2 *.pdf *.css *.sass *.scss *.less
-                        *.yml *.json *.zip *.tar.gz *.swf *.flv *.mp3 *.wav *.flac *.webm *.mp4 *.ogg *.opus
-                        *.webp *.map].freeze
+    ALWAYS_EXCLUDE = %w[*.jpg *.jpeg *.png *.gif *.svg *.ico *.eot *.otf *.ttf *.woff *.woff2 *.pdf *.css *.sass *.scss
+                        *.less *.yml *.json *.zip *.tar.gz *.swf *.flv *.mp3 *.wav *.flac *.webm *.mp4 *.ogg *.opus
+                        *.webp *.map *.xlsx].freeze
 
     # Find all keys in the source and return a forest with the keys in absolute form and their occurrences.
     #
@@ -139,7 +142,7 @@ module I18n::Tasks
 
     # keys in the source that end with a ., e.g. t("category.#{ cat.i18n_key }") or t("category." + category.key)
     # @param [String] replacement for interpolated values.
-    def expr_key_re(replacement: ':')
+    def expr_key_re(replacement: '*:')
       @expr_key_re ||= begin
         # disallow patterns with no keys
         ignore_pattern_re = /\A[.#{replacement}]*\z/
@@ -175,7 +178,7 @@ module I18n::Tasks
           braces << '{'
         end
       end
-      result << key[scanner.pos..-1] unless scanner.eos?
+      result << key[scanner.pos..] unless scanner.eos?
       result.join
     end
   end

data/lib/i18n/tasks/version.rb

@@ -2,6 +2,6 @@
 
 module I18n
   module Tasks
-    VERSION = '0.9.37'
+    VERSION = '1.0.13'
   end
 end

data/lib/i18n/tasks.rb

@@ -34,6 +34,17 @@ module I18n
         ::I18n::Tasks::Commands.send :include, commands_module
         self
       end
+
+      # Add AST-matcher to i18n-tasks
+      #
+      # @param matcher_class_name
+      # @return self
+      def add_ast_matcher(matcher_class_name)
+        matchers = I18n::Tasks::Configuration::DEFAULTS[:search][:ast_matchers]
+        matchers << matcher_class_name
+        matchers.uniq!
+        self
+      end
     end
 
     @verbose = !ENV['VERBOSE'].nil?

data/templates/config/i18n-tasks.yml

@@ -13,7 +13,7 @@ data:
   ## Provide a custom adapter:
   # adapter: I18n::Tasks::Data::FileSystem
 
-  # Locale files or `File.find` patterns where translations are read from:
+  # Locale files or `Find.find` patterns where translations are read from:
   read:
     ## Default:
     # - config/locales/%{locale}.yml
@@ -52,7 +52,7 @@ data:
 
 # Find translate calls
 search:
-  ## Paths or `File.find` patterns to search in:
+  ## Paths or `Find.find` patterns to search in:
   # paths:
   #  - app/
 
@@ -72,11 +72,13 @@ search:
   #  -
 
   ## Files or `File.fnmatch` patterns to exclude from search. Some files are always excluded regardless of this setting:
-  ##   %w(*.jpg *.png *.gif *.svg *.ico *.eot *.otf *.ttf *.woff *.woff2 *.pdf *.css *.sass *.scss *.less *.yml *.json *.map)
+  ##   *.jpg *.jpeg *.png *.gif *.svg *.ico *.eot *.otf *.ttf *.woff *.woff2 *.pdf *.css *.sass *.scss *.less
+  ##   *.yml *.json *.zip *.tar.gz *.swf *.flv *.mp3 *.wav *.flac *.webm *.mp4 *.ogg *.opus *.webp *.map *.xlsx
   exclude:
     - app/assets/images
     - app/assets/fonts
     - app/assets/videos
+    - app/assets/builds
 
   ## Alternatively, the only files or `File.fnmatch patterns` to search in `paths`:
   ## If specified, this settings takes priority over `exclude`, but `exclude` still applies.
@@ -85,6 +87,15 @@ search:
   ## If `strict` is `false`, guess usages such as t("categories.#{category}.title"). The default is `true`.
   # strict: true
 
+  ## Allows adding ast_matchers for finding translations using the AST-scanners
+  ## The available matchers are:
+  ## - RailsModelMatcher
+  ##     Matches ActiveRecord translations like
+  ##     User.human_attribute_name(:email) and User.model_name.human
+  ##
+  ## To implement your own, please see `I18n::Tasks::Scanners::AstMatchers::BaseMatcher`.
+  # <%# I18n::Tasks.add_ast_matcher('I18n::Tasks::Scanners::AstMatchers::RailsModelMatcher') %>
+
   ## Multiple scanners can be used. Their results are merged.
   ## The options specified above are passed down to each scanner. Per-scanner options can be specified as well.
   ## See this example of a custom scanner: https://github.com/glebm/i18n-tasks/wiki/A-custom-scanner-example
@@ -99,7 +110,9 @@ search:
 #   deepl_api_key: "48E92789-57A3-466A-9959-1A1A1A1A1A1A"
 #   # deepl_host: "https://api.deepl.com"
 #   # deepl_version: "v2"
-
+#   # add additional options to the DeepL.translate call: https://www.deepl.com/docs-api/translate-text/translate-text/
+#   deepl_options:
+#     formality: prefer_less
 ## Do not consider these keys missing:
 # ignore_missing:
 # - 'errors.messages.{accepted,blank,invalid,too_short,too_long}'

data/templates/minitest/i18n_test.rb

@@ -5,18 +5,18 @@ require 'i18n/tasks'
 class I18nTest < ActiveSupport::TestCase
   def setup
     @i18n = I18n::Tasks::BaseTask.new
-    @missing_keys = @i18n.missing_keys
-    @unused_keys = @i18n.unused_keys
   end
 
   def test_no_missing_keys
-    assert_empty @missing_keys,
-                 "Missing #{@missing_keys.leaves.count} i18n keys, run `i18n-tasks missing' to show them"
+    missing_keys = @i18n.missing_keys
+    assert_empty missing_keys,
+                 "Missing #{missing_keys.leaves.count} i18n keys, run `i18n-tasks missing' to show them"
   end
 
   def test_no_unused_keys
-    assert_empty @unused_keys,
-                 "#{@unused_keys.leaves.count} unused i18n keys, run `i18n-tasks unused' to show them"
+    unused_keys = @i18n.unused_keys
+    assert_empty unused_keys,
+                 "#{unused_keys.leaves.count} unused i18n keys, run `i18n-tasks unused' to show them"
   end
 
   def test_files_are_normalized

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: i18n-tasks
 version: !ruby/object:Gem::Version
-  version: 0.9.37
+  version: 1.0.13
 platform: ruby
 authors:
 - glebm
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-26 00:00:00.000000000 Z
+date: 2023-10-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -38,6 +38,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: better_html
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: erubi
   requirement: !ruby/object:Gem::Requirement
@@ -86,14 +106,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.3.0
+        version: 3.2.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.3.0
+        version: 3.2.2.1
 - !ruby/object:Gem::Dependency
   name: rails-i18n
   requirement: !ruby/object:Gem::Requirement
@@ -143,12 +163,15 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.5.1
 - !ruby/object:Gem::Dependency
-  name: axlsx
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -156,26 +179,23 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.1
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: overcommit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.0.1
+        version: 0.58.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.0.1
+        version: 0.58.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -210,14 +230,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.1
+        version: 1.50.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.50.1
+- !ruby/object:Gem::Dependency
+  name: rubocop-rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.19.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.1
+        version: 2.19.0
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -357,12 +405,18 @@ files:
 - lib/i18n/tasks/references.rb
 - lib/i18n/tasks/reports/base.rb
 - lib/i18n/tasks/reports/terminal.rb
+- lib/i18n/tasks/scanners/ast_matchers/base_matcher.rb
+- lib/i18n/tasks/scanners/ast_matchers/message_receivers_matcher.rb
+- lib/i18n/tasks/scanners/ast_matchers/rails_model_matcher.rb
+- lib/i18n/tasks/scanners/erb_ast_processor.rb
+- lib/i18n/tasks/scanners/erb_ast_scanner.rb
 - lib/i18n/tasks/scanners/file_scanner.rb
 - lib/i18n/tasks/scanners/files/caching_file_finder.rb
 - lib/i18n/tasks/scanners/files/caching_file_finder_provider.rb
 - lib/i18n/tasks/scanners/files/caching_file_reader.rb
 - lib/i18n/tasks/scanners/files/file_finder.rb
 - lib/i18n/tasks/scanners/files/file_reader.rb
+- lib/i18n/tasks/scanners/local_ruby_parser.rb
 - lib/i18n/tasks/scanners/occurrence_from_position.rb
 - lib/i18n/tasks/scanners/pattern_mapper.rb
 - lib/i18n/tasks/scanners/pattern_scanner.rb
@@ -382,6 +436,7 @@ files:
 - lib/i18n/tasks/translators/base_translator.rb
 - lib/i18n/tasks/translators/deepl_translator.rb
 - lib/i18n/tasks/translators/google_translator.rb
+- lib/i18n/tasks/translators/openai_translator.rb
 - lib/i18n/tasks/translators/yandex_translator.rb
 - lib/i18n/tasks/unused_keys.rb
 - lib/i18n/tasks/used_keys.rb
@@ -394,11 +449,14 @@ licenses:
 - MIT
 metadata:
   issue_tracker: https://github.com/glebm/i18n-tasks
+  rubygems_mfa_required: 'true'
 post_install_message: |
   # Install default configuration:
   cp $(bundle exec i18n-tasks gem-path)/templates/config/i18n-tasks.yml config/
   # Add an RSpec for missing and unused keys:
   cp $(bundle exec i18n-tasks gem-path)/templates/rspec/i18n_spec.rb spec/
+  # Or for minitest:
+  cp $(bundle exec i18n-tasks gem-path)/templates/minitest/i18n_test.rb test/
 rdoc_options: []
 require_paths:
 - lib