extract_i18n 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e3434dc96f1e175bb499bfcdf6d4bf1b5dcc5a1a76190639c8689c0eee069820
-  data.tar.gz: a6bd0093f55639253d1f9dd5589f9ebdefda5197e4ccb04d9b01dd1ce2fade88
+  metadata.gz: 7e8c03edd94a9dd0fd99ecb17c2f2140b9ba96894a3e36b29ae623b5a79cfec8
+  data.tar.gz: 2c7bd0bae8c25e443d7f33c31dbdc5eb0832ad0d9bf2367a051555e16dcc8176
 SHA512:
-  metadata.gz: 82b5167ce206cc40b507d9e7eafae30f8b04384670483ed328c8a924498c252581a87bf02cf03fa103e7591c13090b6a8c97046028af974c18f15489e328b23a
-  data.tar.gz: 0e94e6d16ffe2e67d18be3d5a06d82051c3a713fdd03e0be8983b4f4a5f2097777f68935437605ef0cd788025c8dd5d4a0fbeb9c209c3d8074b07cf7915211c6
+  metadata.gz: a3b44e7bdc8c7b4db6a8391c6cdfc2b82ec9bde4c0fc4cf4feffc982cbeb016dec5ee979799a77b7ed96029693cc059082161882b8e7443aceee50be8651e9fe
+  data.tar.gz: 637756d6127ccb0a719e723a6d155206a5b87a485e3b91c2698bd8749ab24f73765df1fb66cf79ec3e1af253e3fe28c8333bf009ffce7db79ea58f535e1e13c1

data/.gitignore

@@ -7,3 +7,4 @@
 /spec/reports/
 /tmp/
 spec/.examples.txt
+Gemfile.lock

data/README.md

@@ -1,5 +1,7 @@
 # ExtractI18n
 
+[![Gem Version](https://badge.fury.io/rb/extract_i18n.svg)](https://badge.fury.io/rb/extract_i18n)
+
 CLI helper program to automatically extract bare text strings into Rails I18n interactively.
 
 Useful when adding i18n to a medium/large Rails app.
@@ -7,13 +9,14 @@ Useful when adding i18n to a medium/large Rails app.
 This Gem **supports** the following source files:
 
 - Ruby files (controllers, models etc.) via Ruby-Parser, e.g. walking all Ruby Strings
-- Slim Views (via Regexp parser by SlimKeyfy)
+- Slim Views (via Regexp parser by [SlimKeyfy](https://github.com/phrase/slimkeyfy) (MIT License))
 - Vue Pug views
   - Pug is very similar to slim and thus relatively good extractable via Regexp.
+- ERB views
+  - by vendoring/extending https://github.com/ProGM/i18n-html_extractor (MIT License)
 
 CURRENTLY THERE IS **NO SUPPORT** FOR:
 
-- erb ( integrating/forking https://github.com/zigzag/ready_for_i18n or https://github.com/ProGM/i18n-html_extractor)
 - haml ( integrating https://github.com/shaiguitar/haml-i18n-extractor)
 - vue html templates ([Check out my vue pug converting script](https://gist.github.com/zealot128/6c41df1d33a810856a557971a04989f6))
 

data/extract_i18n.gemspec

@@ -25,6 +25,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.add_runtime_dependency 'nokogiri'
   spec.add_runtime_dependency 'parser', '>= 2.6'
   spec.add_runtime_dependency 'slim'
   spec.add_runtime_dependency 'tty-prompt'

data/lib/extract_i18n.rb

@@ -4,11 +4,14 @@ require "extract_i18n/version"
 
 require "zeitwerk"
 loader = Zeitwerk::Loader.for_gem
+loader.inflector.inflect(
+  "html_extractor"   => "HTMLExtractor",
+)
 loader.setup # ready!
 
 module ExtractI18n
   class << self
-    attr_accessor :strip_path, :ignore_hash_keys, :ignore_functions, :ignorelist
+    attr_accessor :strip_path, :ignore_hash_keys, :ignore_functions, :ignorelist, :html_fields_with_plaintext
   end
 
   self.strip_path = %r{^app/(javascript|controllers|views)|^lib|^src|^app}
@@ -20,6 +23,7 @@ module ExtractI18n
     '_',
     '::'
   ]
+  self.html_fields_with_plaintext = %w[title placeholder alt label aria-label modal-title]
 
   def self.key(string, length: 25)
     string.strip.

data/lib/extract_i18n/adapters/erb_adapter.rb

@@ -0,0 +1,43 @@
+module ExtractI18n::Adapters
+  class ErbAdapter < Adapter
+    def run(original_content)
+      unless valid_erb?(original_content)
+        puts "ERB invalid!"
+        return original_content
+      end
+      document = ExtractI18n::HTMLExtractor::ErbDocument.parse_string(original_content)
+      nodes_to_translate = ExtractI18n::HTMLExtractor::Match::Finder.new(document).matches
+      nodes_to_translate.each { |node|
+        next if node.text == ""
+
+        process_change(node)
+      }
+      result = document.save
+
+      result
+    end
+
+    def valid_erb?(content)
+      Parser::CurrentRuby.parse(ERB.new(content).src)
+      true
+    rescue StandardError => e
+      warn e.inspect
+      false
+    end
+
+    def process_change(node)
+      change = ExtractI18n::SourceChange.new(
+        i18n_key: "#{@file_key}.#{ExtractI18n.key(node.text.strip)}",
+        i18n_string: node.text,
+        interpolate_arguments: {},
+        source_line: node.to_s,
+        remove: node.text,
+        t_template: %{ t('%s') },
+        interpolation_type: :ruby
+      )
+      if @on_ask.call(change)
+        node.replace_text!(change.key, change.i18n_t)
+      end
+    end
+  end
+end

data/lib/extract_i18n/html_extractor/erb_document.rb

@@ -0,0 +1,83 @@
+require 'nokogiri'
+
+module ExtractI18n
+  module HTMLExtractor
+    class ErbDocument
+      ERB_REGEXPS = [
+        TwoWayRegexp.new(/<%=(?<inner_text>.+?)%>/m, /@@=(?<inner_text>[a-z0-9\-\._]+)@@/m),
+        TwoWayRegexp.new(/<%#(?<inner_text>.+?)%>/m, /@@#(?<inner_text>[a-z0-9\-\._]+)@@/m),
+        TwoWayRegexp.new(/<%(?<inner_text>.+?)%>/m, /@@(?<inner_text>[a-z0-9\-\._]+)@@/m)
+      ].freeze
+
+      attr_reader :erb_directives
+
+      def initialize(document, erb_directives)
+        @document = document
+        @erb_directives = erb_directives
+      end
+
+      def save
+        result = @document.to_html(indent: 2, encoding: 'UTF-8')
+        ERB_REGEXPS.each do |regexp|
+          regexp.inverse_replace!(result) do |string_format, data|
+            string_format % { inner_text: erb_directives[data[:inner_text]] }
+          end
+        end
+        result
+      end
+
+      def method_missing(name, *args, &block)
+        @document.public_send(name, *args, &block) if @document.respond_to? name
+      end
+
+      class <<self
+        def parse(filename, verbose: false)
+          file_content = ''
+          File.open(filename) do |file|
+            file.read(nil, file_content)
+            return parse_string(file_content, verbose: verbose)
+          end
+        end
+
+        def parse_string(string, verbose: false)
+          erb_directives = extract_erb_directives! string
+          document = create_document(string)
+          log_errors(document.errors, string) if verbose
+          ErbDocument.new(document, erb_directives)
+        end
+
+        private
+
+        def create_document(file_content)
+          if file_content.start_with?('<!DOCTYPE')
+            Nokogiri::HTML(file_content)
+          else
+            Nokogiri::HTML.fragment(file_content)
+          end
+        end
+
+        def log_errors(errors, file_content)
+          return if errors.empty?
+          text = file_content.split("\n")
+          errors.each do |e|
+            puts "Error at line #{e.line}: #{e}".red
+            puts text[e.line - 1]
+          end
+        end
+
+        def extract_erb_directives!(text)
+          erb_directives = {}
+
+          ERB_REGEXPS.each do |regexp|
+            regexp.replace!(text) do |string_format, data|
+              key = SecureRandom.uuid
+              erb_directives[key] = data[:inner_text]
+              string_format % { inner_text: key }
+            end
+          end
+          erb_directives
+        end
+      end
+    end
+  end
+end

data/lib/extract_i18n/html_extractor/match.rb

@@ -0,0 +1,43 @@
+module ExtractI18n
+  module HTMLExtractor
+    module Match
+      class Finder
+        attr_reader :document
+
+        def initialize(document)
+          @document = document
+        end
+
+        def matches
+          erb_nodes(document) + plain_text_nodes(document) + form_fields(document)
+        end
+
+        private
+
+        def erb_nodes(document)
+          document.erb_directives.map do |fragment_id, _|
+            ErbDirectiveMatch.create(document, fragment_id)
+          end.flatten.compact
+        end
+
+        def plain_text_nodes(document)
+          leaf_nodes.map! { |node| PlainTextMatch.create(document, node) }.flatten.compact
+        end
+
+        def form_fields(document)
+          ExtractI18n.html_fields_with_plaintext.flat_map do |field|
+            document.
+              css("[#{field}]").
+              select { |input| input[field] && !input[field].empty? }.
+              reject { |n| n[field] =~ /\@\@(=?)[a-z0-9\-]+\@\@/ }.
+              flat_map { |node| AttributeMatch.create(document, node, field) }
+          end.compact
+        end
+
+        def leaf_nodes
+          @leaf_nodes ||= document.css('*:not(:has(*))').select { |n| n.text && !n.text.empty? }
+        end
+      end
+    end
+  end
+end

data/lib/extract_i18n/html_extractor/match/attribute_match.rb

@@ -0,0 +1,25 @@
+module ExtractI18n
+  module HTMLExtractor
+    module Match
+      class AttributeMatch < BaseMatch
+        def initialize(document, node, text, attribute)
+          super(document, node, text)
+          @attribute = attribute
+        end
+
+        def self.create(document, node, attribute)
+          if node[attribute] && !node[attribute].empty?
+            [new(document, node, node[attribute], attribute)]
+          else
+            []
+          end
+        end
+
+        def replace_text!(key, i18n_t)
+          document.erb_directives[key] = i18n_t
+          node[@attribute] = "@@=#{key}@@"
+        end
+      end
+    end
+  end
+end

data/lib/extract_i18n/html_extractor/match/base_match.rb

@@ -0,0 +1,18 @@
+module ExtractI18n
+  module HTMLExtractor
+    module Match
+      class BaseMatch < NodeMatch
+        attr_reader :node
+
+        def initialize(document, node, text)
+          super(document, text)
+          @node = node
+        end
+
+        def replace_text!
+          node.content = translation_key_object
+        end
+      end
+    end
+  end
+end

data/lib/extract_i18n/html_extractor/match/erb_directive_match.rb

@@ -0,0 +1,33 @@
+module ExtractI18n
+  module HTMLExtractor
+    module Match
+      class ErbDirectiveMatch < NodeMatch
+        REGEXPS = [
+          [/^([ \t]*link_to )(("[^"]+")|('[^']+'))/, '\1%s', 2],
+          [/^([ \t]*link_to (.*),[ ]?title:[ ]?)(("[^"]+")|('[^']+'))/, '\1%s', 3],
+          [/^([ \t]*[a-z_]+\.[a-z_]+_field (.*),[ ]?placeholder:[ ]?)(("[^"]+")|('[^']+'))/, '\1%s', 3],
+          [/^([ \t]*[a-z_]+\.text_area (.*),[ ]?placeholder:[ ]?)(("[^"]+")|('[^']+'))/, '\1%s', 3],
+          [/^([ \t]*[a-z_]+\.submit )(("[^"]+")|('[^']+'))/, '\1%s', 2],
+          [/^([ \t]*[a-z_]+\.label\s+\:[a-z_]+\,\s+)(("[^"]+")|('[^']+'))/, '\1%s', 2]
+        ].freeze
+
+        def initialize(document, fragment_id, text, regexp)
+          super(document, text)
+          @fragment_id = fragment_id
+          @regexp = regexp
+        end
+
+        def replace_text!(key, i18n_t)
+          document.erb_directives[@fragment_id].gsub!(@regexp[0], @regexp[1] % i18n_t.strip)
+        end
+
+        def self.create(document, fragment_id)
+          REGEXPS.map do |r|
+            match = document.erb_directives[fragment_id].match(r[0])
+            new(document, fragment_id, match[r[2]][1...-1], r) if match && match[r[2]]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/extract_i18n/html_extractor/match/node_match.rb

@@ -0,0 +1,28 @@
+module ExtractI18n
+  module HTMLExtractor
+    module Match
+      class NodeMatch
+        attr_reader :document, :text
+
+        def initialize(document, text)
+          @document = document
+          @text = text
+        end
+
+        def translation_key_object
+          "t('.#{key}')"
+        end
+
+        def replace_text!
+          raise NotImplementedError
+        end
+
+        attr_writer :key
+
+        def key
+          @key ||= text.parameterize.underscore
+        end
+      end
+    end
+  end
+end

data/lib/extract_i18n/html_extractor/match/plain_text_match.rb

@@ -0,0 +1,19 @@
+module ExtractI18n
+  module HTMLExtractor
+    module Match
+      class PlainTextMatch < BaseMatch
+        def self.create(document, node)
+          return nil if node.name.start_with?('script')
+          node.text.split(/\@\@(=?)[a-z0-9\-]+\@\@/).map! do |text|
+            new(document, node, text.strip) if !text.nil? && !text.empty?
+          end
+        end
+
+        def replace_text!(key, i18n_t)
+          document.erb_directives[key] = i18n_t
+          node.content = node.content.gsub(text, "@@=#{key}@@")
+        end
+      end
+    end
+  end
+end

data/lib/extract_i18n/html_extractor/runner.rb

@@ -0,0 +1,94 @@
+module ExtractI18
+  module HTMLExtractor
+    class Runner
+      include Cli
+
+      def initialize(args = {})
+        @files = file_list_from_pattern(args[:file_pattern])
+        @locale = args[:locale].presence
+        @verbose = args[:verbose]
+      end
+
+      def run_interactive
+        each_translation do |file, document, node|
+          puts "Found \"#{node.text}\" in #{file}:#{node.text}".green
+          next unless confirm 'Create a translation?', 'Yes', 'No', default: 'Yes'
+
+          node.key = prompt 'Choose i18n key', default: node.key
+          node.replace_text!
+
+          document.save!(file)
+
+          add_translations! node.key, node.text, default_locale: @locale
+          puts
+        end
+      end
+
+      def run
+        each_translation do |file, document, node|
+          puts "Found \"#{node.text}\" in #{file}:#{node.text}".green
+          node.replace_text!
+          document.save!(file)
+
+          add_translation! I18n.default_locale, node.key, node.text
+        end
+      end
+
+      def test_run
+        each_translation do |file, _, node|
+          puts "Found \"#{node.text}\" in #{file}:#{node.text}".green
+        end
+      end
+
+      private
+
+      def file_list_from_pattern(pattern)
+        if pattern.present?
+          Dir[Rails.root.join(pattern)]
+        else
+          Dir[Rails.root.join('app', 'views', '**', '*.erb')] -
+            Dir[Rails.root.join('app', 'views', '**', '*.*.*.erb')]
+        end
+      end
+
+      def add_translations!(key, text, default_locale: nil)
+        return prompt_and_add_translation!(default_locale, key, default_text: text) if default_locale
+        prompt_and_add_translation!(I18n.default_locale, key, default_text: text)
+
+        I18n.available_locales.each do |locale|
+          next if locale == I18n.default_locale
+
+          prompt_and_add_translation!(locale.to_s, key)
+        end
+      end
+
+      def prompt_and_add_translation!(locale, key, default_text: nil)
+        out_text = prompt "Choose #{locale} value", default: default_text
+        add_translation! locale, key, out_text
+      end
+
+      def add_translation!(locale, key, value)
+        new_keys = i18n.missing_keys(locales: [locale]).set_each_value!(value)
+        i18n.data.merge! new_keys
+        puts "Added t(.#{key}), translated in #{locale} as #{value}:".green
+        puts new_keys.inspect
+      end
+
+      def i18n
+        I18n::Tasks::BaseTask.new
+      end
+
+      def each_translation
+        @files.each do |file|
+          document = I18n::HTMLExtractor::ErbDocument.parse file
+          nodes_to_translate = extract_all_nodes_to_translate(document)
+          nodes_to_translate.each { |node| yield(file, document, node) }
+        end
+      end
+
+      def extract_all_nodes_to_translate(document)
+        Match::Finder.new(document).matches
+      end
+    end
+  end
+end

data/lib/extract_i18n/html_extractor/two_way_regexp.rb

@@ -0,0 +1,70 @@
+module ExtractI18n
+  module HTMLExtractor
+    class TwoWayRegexp
+      attr_reader :from, :to
+
+      def initialize(from, to)
+        @from = from
+        @to = to
+      end
+
+      def replace(text)
+        if block_given?
+          text.gsub(@from) do |matched_text|
+            yield(to_as_format, Regexp.last_match, matched_text)
+          end
+        else
+          text.gsub(@from, reverse_to)
+        end
+      end
+
+      def replace!(text)
+        if block_given?
+          text.gsub!(@from) do |matched_text|
+            yield(to_as_format, Regexp.last_match, matched_text)
+          end
+        else
+          text.gsub!(@from, reverse_to)
+        end
+      end
+
+      def inverse_replace(text)
+        if block_given?
+          text.gsub(@to) do |matched_text|
+            yield(from_as_format, Regexp.last_match, matched_text)
+          end
+        else
+          text.gsub(@to, reverse_from)
+        end
+      end
+
+      def inverse_replace!(text)
+        if block_given?
+          text.gsub!(@to) do |matched_text|
+            yield(from_as_format, Regexp.last_match, matched_text)
+          end
+        else
+          text.gsub!(@to, reverse_from)
+        end
+      end
+
+      private
+
+      def to_as_format
+        @to_as_format ||= @to.source.gsub('%', '%%').gsub!(/\(\?<([a-z_]+)>.*\)/, '%{\1}')
+      end
+
+      def from_as_format
+        @from_as_format ||= @from.source.gsub('%', '%%').gsub!(/\(\?<([a-z_]+)>.*\)/, '%{\1}')
+      end
+
+      def reverse_from
+        @reverse_from ||= @from.source.gsub(/\(\?<([a-z_]+)>.*\)/, '\k{\1}')
+      end
+
+      def reverse_to
+        @reverse_to ||= @to.source.gsub(/\(\?<([a-z_]+)>.*\)/, '\k{\1}')
+      end
+    end
+  end
+end

data/lib/extract_i18n/version.rb

@@ -1,3 +1,3 @@
 module ExtractI18n
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: extract_i18n
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Stefan Wienert
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-10-06 00:00:00.000000000 Z
+date: 2020-10-07 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: parser
   requirement: !ruby/object:Gem::Requirement
@@ -117,12 +131,22 @@ files:
 - extract_i18n.gemspec
 - lib/extract_i18n.rb
 - lib/extract_i18n/adapters/adapter.rb
+- lib/extract_i18n/adapters/erb_adapter.rb
 - lib/extract_i18n/adapters/ruby_adapter.rb
 - lib/extract_i18n/adapters/slim_adapter.rb
 - lib/extract_i18n/adapters/slim_adapter_wip.rb
 - lib/extract_i18n/adapters/vue_adapter.rb
 - lib/extract_i18n/cli.rb
 - lib/extract_i18n/file_processor.rb
+- lib/extract_i18n/html_extractor/erb_document.rb
+- lib/extract_i18n/html_extractor/match.rb
+- lib/extract_i18n/html_extractor/match/attribute_match.rb
+- lib/extract_i18n/html_extractor/match/base_match.rb
+- lib/extract_i18n/html_extractor/match/erb_directive_match.rb
+- lib/extract_i18n/html_extractor/match/node_match.rb
+- lib/extract_i18n/html_extractor/match/plain_text_match.rb
+- lib/extract_i18n/html_extractor/runner.rb
+- lib/extract_i18n/html_extractor/two_way_regexp.rb
 - lib/extract_i18n/slimkeyfy/slim_transformer.rb
 - lib/extract_i18n/slimkeyfy/vue_transformer.rb
 - lib/extract_i18n/slimkeyfy/whitespacer.rb