extract_i18n 0.1.0

data/lib/extract_i18n/adapters/slim_adapter.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module ExtractI18n::Adapters
+  class SlimAdapter < Adapter
+    def self.supports_relative_keys?
+      true
+    end
+
+    def run(original_content)
+      @content = self.class.join_multiline(original_content.split("\n"))
+      @content << "" # new line at the end
+      @transformer = ExtractI18n::Slimkeyfy::SlimTransformer
+
+      @new_content =
+        @content.map do |old_line|
+          process_line(old_line)
+        end
+      @new_content.join("\n")
+    end
+
+    def process_line(old_line)
+      word = ExtractI18n::Slimkeyfy::Word.for('.slim').new(old_line)
+      ExtractI18n::Slimkeyfy::SlimTransformer.new(word, @file_key).transform do |change|
+        if change.nil? # nothing to do
+          return old_line
+        end
+
+        if @on_ask.call(change)
+          change.i18n_t(relative: @options[:relative])
+        else
+          old_line
+        end
+      end
+    end
+
+    def self.join_multiline(strings_array)
+      result = []
+      joining_str = ''
+      indent_length = 0
+      long_str_start = /^[ ]+\| */
+      long_str_indent = /^[ ]+/
+      long_str_indent_with_vertical_bar = /^[ ]+\| */
+      strings_array.each do |str|
+        if joining_str.empty?
+          if str[long_str_start]
+            joining_str = str
+            indent_length = str[long_str_start].length
+          else
+            result << str
+          end
+          # multiline string continues with spaces
+        elsif str[long_str_indent] && str[long_str_indent].length.to_i >= indent_length
+          joining_str << str.gsub(long_str_indent, ' ')
+          # muliline string continues with spaces and vertical bar with same indentation
+        elsif str[long_str_indent_with_vertical_bar] && str[long_str_indent_with_vertical_bar].length.to_i == indent_length
+          joining_str << str.gsub(long_str_indent_with_vertical_bar, ' ')
+          # multiline string ends
+        else
+          result << joining_str
+          joining_str = ''
+          indent_length = 0
+          result << str
+        end
+      end
+      result << joining_str unless joining_str.empty?
+
+      result
+    end
+  end
+end

data/lib/extract_i18n/adapters/slim_adapter_wip.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require 'slim'
+
+module ExtractI18n::Adapters
+  class SlimAdapterWip < Adapter
+    def run(original_content)
+      parser = SlimRewriterParser.new(file_key: file_key, on_ask: on_ask)
+      parser.call(original_content)
+    end
+  end
+
+  class SlimRewriterParser < Slim::Engine
+    class CustomFilter < Slim::Filter
+      define_options file_key: :dynamic, on_ask: :dynamic
+      def call(exp)
+        @indent = 0
+        super.to_s
+      end
+
+      def on_multi(*exps)
+        super[1..-1].join
+      end
+
+      def on_newline
+        "\n"
+      end
+
+      def on_static(*args)
+        args[0].include?('"') ? "'#{args.join(' ')}'" : %["#{args.join(' ')}"]
+      end
+
+      def on_html_doctype(name)
+        "#{indent}doctype #{name}"
+      end
+
+      def on_html_tag(name, attrs, content = nil)
+        ret = "#{indent}#{name}#{compile attrs}"
+        if content
+          ret << ' ' << block(content)
+        end
+        ret
+      end
+
+      def block(content)
+        @indent += 1
+        result = compile(content).to_s # todo remove to_s
+        if result.is_a?(String)
+          extract_from_string(result)
+        else
+          result
+        end
+      ensure
+        @indent -= 1
+      end
+
+      def extract_from_string(string)
+        i18n_key = ExtractI18n.key(string)
+        change = ExtractI18n::SourceChange.new(
+          i18n_key: "#{@options[:file_key]}.#{i18n_key}",
+          i18n_string: string,
+          interpolate_arguments: {},
+          source_line: string,
+          remove: string
+        )
+        puts change.format
+        if @options[:on_ask].call(change)
+          change.i18n_t
+        else
+          string
+        end
+      end
+
+      def on_html_attrs(*attrs)
+        attrs.empty? ? '' : "(#{super[2..-1].join(' ')})"
+      end
+
+      def on_slim_interpolate(text)
+        compile(text)
+      end
+
+      def on_slim_text(type, text)
+        block(text)
+      end
+
+      def on_slim_embedded(type, text)
+        "#{indent}#{block text}"
+      end
+
+      def on_html_comment(exp)
+        "#{indent}/ #{block exp}"
+      end
+
+      def on_slim_control(line, block)
+        "#{indent}- #{line}#{block block}"
+      end
+
+      def on_html_attr(name, content)
+        "#{name}=#{compile content}"
+      end
+
+      def indent
+        '  ' * @indent
+      end
+
+      # def parse_text_block(first_line = nil, text_indent = nil)
+      #   source_line = @_lines[@lineno - 1]
+
+      #   binding.pry
+      #   result = super
+
+      #   change = ExtractI18n::SourceChange.new(
+      #     i18n_key: "#{@file_key}.#{i18n_key}",
+      #     i18n_string: i18n_string,
+      #     interpolate_arguments: {},
+      #     source_line: source_line,
+      #     remove: node.loc.expression.source
+      #   )
+      #   binding.pry
+
+      #   result
+      # end
+    end
+    remove //
+    filter :Encoding
+    filter :RemoveBOM
+    use Slim::Parser
+    use CustomFilter
+  end
+end

data/lib/extract_i18n/adapters/vue_adapter.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module ExtractI18n::Adapters
+  class VueAdapter < SlimAdapter
+    def process_line(old_line)
+      @mode ||= :template
+      if old_line[/^<template/]
+        @mode = :template
+      elsif old_line[/^<script/]
+        @mode = :script
+      elsif old_line[/^<style/]
+        @mode = :style
+      end
+      if @mode != :template
+        return old_line
+      end
+      word = ExtractI18n::Slimkeyfy::Word.for('.vue').new(old_line)
+      ExtractI18n::Slimkeyfy::VueTransformer.new(word, @file_key).transform do |change|
+        if change.nil? # nothing to do
+          return old_line
+        end
+
+        if @on_ask.call(change)
+          change.i18n_t
+        else
+          old_line
+        end
+      end
+    end
+  end
+end

data/lib/extract_i18n/cli.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+# rubocop:disable Layout/LineLength
+
+require 'optparse'
+require 'extract_i18n'
+require 'extract_i18n/file_processor'
+require 'extract_i18n/version'
+require 'open-uri'
+
+module ExtractI18n
+  # Cli Class
+  class CLI
+    def initialize
+      @options = {}
+      ARGV << '-h' if ARGV.empty?
+      OptionParser.new do |opts|
+        opts.banner = "Usage: extract-i18n -l <locale> -w <target-yml> [path*]"
+
+        opts.on('--version', 'Print version number') do
+          puts ExtractI18n::VERSION
+          exit 1
+        end
+
+        opts.on('-lLOCALE', '--locale=LOCALE', 'default locale for extraction (Default = en)') do |f|
+          @options[:locale] = f || 'en'
+        end
+
+        opts.on('-nNAMESPACE', '--namespace=NAMESPACE', 'Locale base key to wrap locations in') do |f|
+          @options[:base_key] = f
+        end
+
+        opts.on('-r', '--slim-relative', 'When activated, will use relative keys like t(".title")') do |f|
+          @options[:relative] = f
+        end
+
+        opts.on('-yYAML', '--yaml=YAML-FILE', 'Write extracted keys to YAML file (default = config/locales/unsorted.LOCALE.yml)') do |f|
+          @options[:write_to] = f || "config/locales/unsorted.#{@options[:locale]}"
+        end
+
+        opts.on('-h', '--help', 'Prints this help') do
+          puts opts
+          exit 1
+        end
+      end.parse!
+
+      @options[:write_to] ||= "config/locales/unsorted.#{@options[:locale]}.yml"
+      @options[:locale] ||= 'en'
+      @files = ARGV
+    end
+
+    def run
+      paths = @files.empty? ? [] : @files
+      paths.each do |path|
+        if File.directory?(path)
+          glob_path = File.join(path, '**', '*.rb')
+          Dir.glob(glob_path) do |file_path|
+            process_file file_path
+          end
+        else
+          process_file path
+        end
+      end
+    end
+
+    def process_file(file_path)
+      puts "Processing: #{file_path}"
+      ExtractI18n::FileProcessor.new(
+        file_path: file_path,
+        write_to: @options[:write_to],
+        locale: @options[:locale],
+        options: @options
+      ).run
+    end
+  end
+end

data/lib/extract_i18n/file_processor.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require 'parser/current'
+require 'tty/prompt'
+require 'diffy'
+require 'yaml'
+
+module ExtractI18n
+  class FileProcessor
+    PROMPT = TTY::Prompt.new
+    PASTEL = Pastel.new
+
+    def initialize(file_path:, write_to:, locale:, options: {})
+      @file_path = file_path
+      @file_key = ExtractI18n.file_key(@file_path)
+
+      @locale = locale
+      @write_to = write_to
+      @options = options
+      @i18n_changes = {}
+    end
+
+    def run
+      puts " reading #{@file_path}"
+      read_and_transform do |result|
+        puts Diffy::Diff.new(original_content, result, context: 1).to_s(:color)
+        if PROMPT.yes?("Save changes?")
+          File.write(@file_path, result)
+          update_i18n_yml_file
+          puts PASTEL.green("Saved #{@file_path}")
+        end
+      end
+    end
+
+    private
+
+    def read_and_transform(&_block)
+      key = if @options[:namespace]
+              "#{@options[:namespace]}.#{@file_key}"
+            else
+              @file_key
+            end
+      adapter_class = ExtractI18n::Adapters::Adapter.for(@file_path)
+      if adapter_class
+        adapter = adapter_class.new(
+          file_key: key,
+          on_ask: ->(change) { ask_one_change?(change) },
+          options: @options,
+        )
+        output = adapter.run(original_content)
+        if output != original_content
+          yield(output)
+        end
+      end
+    end
+
+    def ask_one_change?(change)
+      check_for_unique!(change)
+      puts change.format
+      if PROMPT.no?("replace line ?")
+        false
+      else
+        @i18n_changes[change.key] = change.i18n_string
+        true
+      end
+    end
+
+    def check_for_unique!(change)
+      if @i18n_changes[change.key] && @i18n_changes[change.key] != change.i18n_string
+        change.increment_key!
+        check_for_unique!(change)
+      end
+    end
+
+    def update_i18n_yml_file
+      base = if File.exist?(@write_to)
+               YAML.load_file(@write_to)
+             else
+               {}
+             end
+      @i18n_changes.each do |key, value|
+        tree = base
+        keys = key.split('.').unshift(@locale)
+        keys.each_with_index do |part, i|
+          if i == keys.length - 1
+            tree[part] = value
+          else
+            tree = tree[part] ||= {}
+          end
+        end
+      end
+      File.write(@write_to, base.to_yaml)
+    end
+
+    def original_content
+      @original_content ||= File.read(@file_path)
+    end
+  end
+end

data/lib/extract_i18n/slimkeyfy/slim_transformer.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+module ExtractI18n
+  class Slimkeyfy::SlimTransformer
+    TRANSLATED = /t\s*\(?\s*(".*?"|'.*?')\s*\)?/.freeze
+    STRING = /(\".*\"|\'.*\')/.freeze
+    STRING_WITHOUT_QUOTES = /("(?<double_quot>.*)"|'(?<single_quot>.*)')/.freeze
+
+    HTML_TAGS = /^(?<html_tag>'|\||([a-z\.]+[0-9\-]*)+)/.freeze
+    EQUALS = /#?([a-z0-9\.\-\s]+)?\=.*/.freeze
+
+    BEFORE =        /(?<before>.*)/.freeze
+    TRANSLATION =   /(?<translation>(".*?"|'.*?'))/.freeze
+    AFTER =         /(?<after>,?.*)?/.freeze
+
+    HTML_ARGUMENTS = {
+      hint: /(?<html_tag>hint:\s*)/,
+      link_to: /(?<html_tag>link_to\s*\(?)/,
+      inconified: /(?<html_tag>(iconified\s*\(?))/,
+      placeholder: /(?<html_tag>placeholder:\s*)/,
+      title: /(?<html_tag>title:\s*)/,
+      prepend: /(?<html_tag>prepend:\s*)/,
+      append: /(?<html_tag>append:\s*)/,
+      label: /(?<html_tag>[a-z]*_?label:\s*)/,
+      optionals: /(?<html_tag>(default|include_blank|alt):\s*)/,
+      input: /(?<html_tag>[a-z]*\.?input:?\s*)/,
+      button: /(?<html_tag>[a-z]*\.?button:?\s*(\:[a-z]+\s*,\s*)?)/,
+      tag: /(?<html_tag>(submit|content)_tag[\:\(]?\s*)/,
+      data_naive: /(?<html_tag>data:\s*\{\s*(confirm|content):\s*)/
+    }.freeze
+
+    LINK_TO = /#{HTML_ARGUMENTS[:link_to]}#{TRANSLATION}/.freeze
+
+    def transform(&block)
+      return yield(nil) if should_not_be_processed?(@word.as_list)
+      unindented_line = @word.unindented_line
+
+      if unindented_line.match(EQUALS)
+        parse_html_arguments(unindented_line, &block)
+      elsif @word.head.match(HTML_TAGS)
+        parse_html(&block)
+      else
+        yield(nil)
+      end
+    end
+
+    def initialize(word, file_key)
+      @word = word
+      @file_key = file_key
+    end
+
+    private
+
+    def parse_html(&_block)
+      return @word.line if @word.line.match(TRANSLATED)
+
+      tagged_with_equals = Slimkeyfy::Whitespacer.convert_slim(@word.head)
+      body = @word.tail.join(" ")
+      body, tagged_with_equals = Slimkeyfy::Whitespacer.convert_nbsp(body, tagged_with_equals)
+
+      if body.match(LINK_TO) != nil
+        body = link_tos(body)
+      end
+
+      interpolate_arguments, body = extract_arguments(body)
+      change = ExtractI18n::SourceChange.new(
+        source_line: @word.line,
+        remove: body,
+        interpolate_arguments: interpolate_arguments,
+        i18n_key: "#{@file_key}.#{ExtractI18n.key(body)}",
+        i18n_string: body,
+        t_template: "#{@word.indentation}#{tagged_with_equals} t('%s'%s)"
+      )
+      yield(change)
+    end
+
+    def parse_html_arguments(line, token_skipped_before = nil, &block)
+      final_line = line
+      regex_list.each do |regex|
+        line.scan(regex) do |m_data|
+          next if m_data == token_skipped_before
+          before = m_data[0]
+          html_tag = m_data[1]
+          translation = match_string(m_data[2])
+          after = m_data[3]
+          interpolate_arguments, translation = extract_arguments(translation)
+          change = ExtractI18n::SourceChange.new(
+            source_line: final_line,
+            i18n_string: translation,
+            i18n_key: "#{@file_key}.#{ExtractI18n.key(translation)}",
+            remove: m_data[2],
+            interpolate_arguments: interpolate_arguments,
+            t_template: "#{before}#{html_tag}t('%s'%s)#{after}"
+          )
+          final_line = yield(change)
+          return parse_html_arguments(final_line, &block)
+        end
+      end
+      final_line
+    end
+
+    def link_tos(line)
+      m = line.match(LINK_TO)
+      if m != nil
+        _ = m[:html_tag]
+        translation = match_string(m[:translation])
+        translation_key = update_hashes(translation)
+        line = line.gsub(m[:translation], translation_key)
+        link_tos(line)
+      else
+        line
+      end
+    end
+
+    def should_not_be_processed?(tokens)
+      (tokens.nil? or tokens.size < 2)
+    end
+
+    def matches_string?(translation)
+      m = translation.match(STRING_WITHOUT_QUOTES)
+      return false if m.nil?
+      (m[:double_quot] != nil or m[:single_quot] != nil)
+    end
+
+    def match_string(translation)
+      m = translation.match(STRING_WITHOUT_QUOTES)
+      return translation if m.nil?
+      if m[:double_quot] != nil
+        m[:double_quot]
+      else
+        (m[:single_quot] != nil ? m[:single_quot] : translation)
+      end
+    end
+
+    def regex_list
+      HTML_ARGUMENTS.map { |_, regex| /#{BEFORE}#{regex}#{TRANSLATION}#{AFTER}/ }
+    end
+
+    def extract_arguments(translation)
+      args = {}
+      translation.scan(/\#{[^}]*}/).each_with_index do |arg, index|
+        stripped_arg = arg[2..-2]
+        key = ExtractI18n.key(arg)
+        key += index.to_s if index > 0
+        translation = translation.gsub(arg, "%{#{key}}")
+        args[key] = stripped_arg
+      end
+      [args, translation]
+    end
+  end
+end