i18nliner 0.0.1 → 0.0.2

data/lib/i18nliner/pre_processors/erb_pre_processor.rb

@@ -0,0 +1,334 @@
+require 'i18nliner/errors'
+require 'i18nliner/call_helpers'
+require 'i18nliner/extractors/sexp_helper'
+require 'nokogiri'
+require 'ruby_parser'
+require 'ruby2ruby'
+
+module I18nliner
+  module PreProcessors
+    class ErbPreProcessor
+
+      class Context
+        attr_reader :buffer, :parent
+
+        def initialize(parent = nil)
+          @parent = parent
+          @buffer = ''
+        end
+
+        def <<(string)
+          if string =~ ERB_T_BLOCK_EXPRESSION
+            TBlock.new(self, $&)
+          else
+            @buffer << string
+            self
+          end
+        end
+
+        def result
+          @buffer
+        end
+      end
+
+      class Helper
+        include Extractors::SexpHelper
+
+        DEFINITIONS = [
+          {:method => :link_to, :pattern => /link_to/, :arg => 0}
+        ]
+        RUBY2RUBY = Ruby2Ruby.new
+        PARSER = RubyParser.new
+
+        def self.match_for(string)
+          DEFINITIONS.each do |info|
+            return Helper.new(info, string) if string =~ info[:pattern]
+          end
+          nil
+        end
+
+        attr_reader :placeholder, :wrapper
+        attr_accessor :content
+
+        def initialize(info, source)
+          @arg = info[:arg]
+          @method = info[:method]
+          @source = source
+        end
+
+        SEXP_ARG_OFFSET = 3
+        def wrappable?
+          return @wrappable if !@wrappable.nil?
+          begin
+            sexps = PARSER.parse(@source)
+            @wrappable = sexps.sexp_type == :call &&
+                         sexps[1].nil? &&
+                         sexps[2] == @method &&
+                         sexps[@arg + SEXP_ARG_OFFSET]
+            extract_content!(sexps) if @wrappable
+            @wrappable
+          end
+        end
+
+        def extract_content!(sexps)
+          sexp = sexps[@arg + SEXP_ARG_OFFSET]
+          if stringish?(sexp)
+            @content = string_from(sexp)
+          else
+            @placeholder = RUBY2RUBY.process(sexp)
+          end
+          sexps[@arg + SEXP_ARG_OFFSET] = Sexp.new(:str, "\\1")
+          @wrapper = RUBY2RUBY.process(sexps)
+        end
+      end
+
+      class TBlock < Context
+        include CallHelpers
+
+        def initialize(parent, content)
+          super(parent)
+          @lines = content.count("\n")
+        end
+
+        def <<(string)
+          case string
+          when ERB_BLOCK_EXPRESSION
+            if string =~ ERB_T_BLOCK_EXPRESSION
+              TBlock.new(self, $&)
+            else
+              raise TBlockNestingError.new("can't nest block expressions inside a t block")
+            end
+          when ERB_STATEMENT
+            if string =~ ERB_END_STATEMENT
+              @parent << result
+            else
+              raise TBlockNestingError.new("can't nest statements inside a t block")
+            end
+          else
+            # expressions and the like are handled a bit later
+            # TODO: perhaps a tad more efficient to capture/transform them
+            # here?
+            @buffer << string
+            self
+          end
+        end
+
+        def result
+          @lines += @buffer.count("\n")
+          key, default, options, wrappers = normalize_call
+          result = "<%= t :#{key}, #{default}"
+          result << ", " << options if options
+          result << ", " << wrappers if wrappers
+          result << (@lines > 0 ? "\n" * @lines : " ")
+          result << "%>"
+        end
+
+        # get a unique and reasonable looking key for a given erb
+        # expression
+        def infer_interpolation_key(string, others)
+          key = string.downcase
+          key.sub!(/\.html_safe\z/, '')
+          key.gsub!(/[^a-z0-9]/, ' ')
+          key.strip!
+          key.gsub!(/ +/, '_')
+          key.slice!(20)
+          i = 0
+          base_key = key
+          while others.key?(key) && others[key] != string
+            key = "#{base_key}_#{i}"
+            i += 1
+          end
+          key
+        end
+
+        def extract_wrappers!(source, wrappers, placeholder_map)
+          source = extract_html_wrappers!(source, wrappers, placeholder_map)
+          source = extract_helper_wrappers!(source, wrappers, placeholder_map)
+          source
+        end
+
+        def find_or_add_wrapper(wrapper, wrappers)
+          unless pos = wrappers.index(wrapper)
+            pos = wrappers.size
+            wrappers << wrapper
+          end
+          pos
+        end
+
+        # incidentally this converts entities to their corresponding values
+        def extract_html_wrappers!(source, wrappers, placeholder_map)
+          default = ''
+          nodes = Nokogiri::HTML.fragment(source).children
+          nodes.each do |node|
+            if node.is_a?(Nokogiri::XML::Text)
+              default << node.content
+            elsif text = extract_text(node)
+              wrapper = node.to_s.sub(text, "\\\\1")
+              wrapper = prepare_wrapper(wrapper, placeholder_map)
+              pos = find_or_add_wrapper(wrapper, wrappers)
+              default << wrap(text, pos + 1)
+            else # no wrapped text (e.g. <input>)
+              key = "__I18NLINER_#{placeholder_map.size}__"
+              placeholder_map[key] = node.to_s.inspect << ".html_safe"
+              default << key
+            end
+          end
+          default
+        end
+
+        def extract_helper_wrappers!(source, wrappers, placeholder_map)
+          source.gsub(TEMP_PLACEHOLDER) do |string|
+            if (helper = Helper.match_for(placeholder_map[string])) && helper.wrappable?
+              placeholder_map.delete(string)
+              if helper.placeholder # e.g. link_to(name) -> *%{name}*
+                helper.content = "__I18NLINER_#{placeholder_map.size}__"
+                placeholder_map[helper.content] = helper.placeholder
+              end
+              pos = find_or_add_wrapper(helper.wrapper, wrappers)
+              wrap(helper.content, pos + 1)
+            else
+              string
+            end
+          end
+        end
+
+        def prepare_wrapper(content, placeholder_map)
+          content = content.inspect
+          content.gsub!(TEMP_PLACEHOLDER) do |key|
+            "\#{#{placeholder_map[key]}}"
+          end
+          content
+        end
+
+        def extract_temp_placeholders!
+          extract_placeholders!(@buffer, ERB_EXPRESSION, false) do |str, map|
+            ["__I18NLINER_#{map.size}__", str]
+          end
+        end
+
+        def extract_placeholders!(buffer = @buffer, pattern = ERB_EXPRESSION, wrap_placeholder = true)
+          map = {}
+          buffer.gsub!(pattern) do |str|
+            key, str = yield($~[:content], map)
+            map[key] = str
+            wrap_placeholder ? "%{#{key}}" : key
+          end
+          map
+        end
+
+        TEMP_PLACEHOLDER = /(?<content>__I18NLINER_\d+__)/
+        def normalize_call
+          wrappers = []
+
+          temp_map = extract_temp_placeholders!
+          default = extract_wrappers!(@buffer, wrappers, temp_map)
+          options = extract_placeholders!(default, TEMP_PLACEHOLDER) do |str, map|
+            [infer_interpolation_key(temp_map[str], map), temp_map[str]]
+          end
+
+          default.strip!
+          default.gsub!(/\s+/, ' ')
+
+          key = infer_key(default)
+          default = default.inspect
+          options = options_to_ruby(options)
+          wrappers = wrappers_to_ruby(wrappers)
+          [key, default, options, wrappers]
+        end
+
+        def options_to_ruby(options)
+          return if options.size == 0
+          options.map do |key, value|
+            ":" << key << " => (" << value << ")"
+          end.join(", ")
+        end
+
+        def wrappers_to_ruby(wrappers)
+          return if wrappers.size == 0
+          ":wrappers => [" << wrappers.join(", ") << "]"
+        end
+
+        def extract_text(root_node)
+          text = nil
+          nodes = root_node.children.to_a
+          while node = nodes.shift
+            if node.is_a?(Nokogiri::XML::Text) && !node.content.strip.empty?
+              raise UnwrappableContentError.new "multiple text nodes in html markup" if text
+              text = node.content
+            else
+              nodes.concat node.children
+            end
+          end
+          text
+        end
+
+        def wrap(text, index)
+          delimiter = "*" * index
+          "" << delimiter << text << delimiter
+        end
+
+        def infer_wrappers(source)
+          wrappers = []
+          [source, wrappers]
+        end
+      end
+
+      # need to evaluate all expressions and statements, so we can
+      # correctly match the start/end of the `t` block expression
+      # (including nested ones)
+      ERB_EXPRESSION = /<%=\s*(?<content>.*?)\s*%>/
+      ERB_BLOCK_EXPRESSION = /
+        \A
+        <%=
+        .*?
+        (\sdo|\{)
+        (\s*\|[^\|]+\|)?
+        \s*
+        %>
+        \z
+      /xm
+      ERB_T_BLOCK_EXPRESSION = /
+        \A
+        <%=
+        \s*
+        t
+        \s*?
+        (\(\)\s*)?
+        (\sdo|\{)
+        \s*
+        %>
+        \z
+      /xm
+      ERB_STATEMENT = /\A<%[^=]/
+      ERB_END_STATEMENT = /
+        \A
+        <%
+        \s*
+        (end|\})
+        (\W|%>\z)
+      /xm
+      ERB_TOKENIZER = /(<%.*?%>)/m
+
+      def self.process(source)
+        new(source).result
+      end
+
+      def initialize(source)
+        @source = source
+      end
+
+      def result
+        # the basic idea:
+        # 1. whenever we find a t block expr, go till we find the end
+        # 2. if we find another t block expr before the end, goto step 1
+        # 3. capture any inline expressions along the way
+        # 4. if we find *any* other statement or block expr, abort,
+        #    since it's a no-go
+        # TODO get line numbers for errors
+        ctx = @source.split(ERB_TOKENIZER).inject(Context.new, :<<)
+        raise MalformedErbError.new('possibly unterminated block expression') if ctx.parent
+        ctx.result
+      end
+    end
+  end
+end

data/lib/i18nliner/processors/abstract_processor.rb

@@ -1,10 +1,19 @@
+require 'globby'
+require 'i18nliner/base'
+require 'i18nliner/processors'
+
 module I18nliner
   module Processors
     class AbstractProcessor
+      attr_reader :translation_count, :file_count
+
       def initialize(translations, options = {})
         @translations = translations
+        @translation_count = 0
+        @file_count = 0
         @only = options[:only]
-        @checker = options[:checker] || methods(:noop_checker)
+        @checker = options[:checker] || method(:noop_checker)
+        @pattern = options[:pattern] || self.class.default_pattern
       end
 
       def noop_checker(file)
@@ -13,21 +22,33 @@ module I18nliner
 
       def files
         @files ||= begin
-          files = Globby.select(@pattern)
-          files = files.select(@only) if @only
+          files = Globby.select(Array(@pattern))
+          files = files.select(Array(@only.dup)) if @only
           files.reject(I18nliner.ignore)
         end
       end
 
       def check_files
-        files.each do |file|
-          @checker.call file, &methods(:check_file)
+        Dir.chdir(I18nliner.base_path) do
+          files.each do |file|
+            @checker.call file, &method(:check_file)
+          end
         end
       end
 
+      def check_file(file)
+        @file_count += 1
+        check_contents(source_for(file), scope_for(file))
+      end
+
       def self.inherited(klass)
         Processors.register klass
       end
+
+      def self.default_pattern(*pattern)
+        @pattern ||= []
+        @pattern.concat(pattern)
+      end
     end
   end
 end

data/lib/i18nliner/processors/erb_processor.rb

@@ -1,9 +1,25 @@
+if defined?(::Rails)
+  require 'i18nliner/erubis'
+else
+  require 'erubis'
+end
+require 'i18nliner/processors/ruby_processor'
+require 'i18nliner/pre_processors/erb_pre_processor'
+
 module I18nliner
   module Processors
     class ErbProcessor < RubyProcessor
-      def source_for(file)
-        # TODO: pre-process for block fu
-        Erubis::Eruby.new(super).src
+      default_pattern '*.erb'
+
+      if defined?(::Rails) # block expressions and all that jazz
+        def pre_process(source)
+          I18nliner::Erubis.new(source).src
+        end
+      else
+        def pre_process(source)
+          source = PreProcessors::ErbPreProcessor.process(source)
+          Erubis::Eruby.new(source).src
+        end
       end
 
       def scope_for(path)

data/lib/i18nliner/processors/ruby_processor.rb

@@ -1,11 +1,18 @@
+require 'i18nliner/processors/abstract_processor'
+require 'i18nliner/extractors/ruby_extractor'
+require 'i18nliner/scope'
+
 module I18nliner
   module Processors
     class RubyProcessor < AbstractProcessor
-      def check_file(file)
-        sexps = RubyParser.new.parse(source_for(file))
-        extractor = Extractors::RubyExtractor.new(sexps, scope_for(file))
+      default_pattern '*.rb'
+
+      def check_contents(source, scope = Scope.new)
+        sexps = RubyParser.new.parse(pre_process(source))
+        extractor = Extractors::RubyExtractor.new(sexps, scope)
         extractor.each_translation do |key, value|
-          @translations.line = extractor.line
+          @translation_count += 1
+          @translations.line = extractor.current_line
           @translations[key] = value
         end
       end
@@ -15,7 +22,11 @@ module I18nliner
       end
 
       def scope_for(path)
-        Scope.new
+        Scope.root
+      end
+
+      def pre_process(source)
+        source
       end
     end
   end

data/lib/i18nliner/railtie.rb

@@ -0,0 +1,27 @@
+require 'i18nliner/extensions/controller'
+require 'i18nliner/extensions/view'
+require 'i18nliner/extensions/model'
+
+module I18nliner
+  class Railtie < Rails::Railtie
+    ActiveSupport.on_load :action_controller do
+      ActionController::Base.send :include, I18nliner::Extensions::Controller
+    end
+
+    ActiveSupport.on_load :action_view do
+      require 'i18nliner/erubis'
+      ActionView::Template::Handlers::ERB.erb_implementation = I18nliner::Erubis
+      ActionView::Base.send :include, I18nliner::Extensions::View
+    end
+
+    ActiveSupport.on_load :active_record do
+      ActiveRecord::Base.send :include, I18nliner::Extensions::Model
+    end
+
+    rake_tasks do
+      load "tasks/i18nliner.rake"
+    end
+  end
+end
+
+

data/lib/i18nliner/scope.rb

@@ -20,5 +20,9 @@ module I18nliner
         key
       end
     end
+
+    def self.root
+      @root ||= new
+    end
   end
 end

data/lib/i18nliner.rb

@@ -1,31 +1,7 @@
-require 'active_support/core_ext/string/inflections'
+require 'i18n'
+require 'i18nliner/base'
 
-module I18nliner
-  def self.translations
-  end
+require 'i18nliner/extensions/core'
+I18n.send :extend, I18nliner::Extensions::Core
 
-  def self.look_up(*args)
-  end
-
-  def self.setting(key, value)
-    instance_eval <<-CODE
-      def #{key}(value = nil)
-        if value && block_given?
-          begin
-            value_was = @#{key}
-            @#{key} = value
-            yield
-          ensure
-            @#{key} = value_was
-          end
-        else
-          @#{key} = #{value.inspect} if @#{key}.nil?
-          @#{key}
-        end
-      end
-    CODE
-  end
-
-  setting :inferred_key_format,        :underscored_crc32
-  setting :infer_interpolation_values, true
-end
+require 'i18nliner/railtie' if defined?(Rails)

data/lib/tasks/i18nliner.rake

@@ -1,14 +1,20 @@
 namespace :i18nliner do
   desc "Verifies all translation calls"
   task :check => :environment do
-    options = {:only => ENV['ONLY'])}
-    @command = I18nliner::Commands::Check.run(options) or exit 1
+    require 'i18nliner/commands/check'
+
+    options = {:only => ENV['ONLY']}
+    @command = I18nliner::Commands::Check.run(options)
+    @command.success? or exit 1
   end
 
   desc "Generates a new [default_locale].yml file for all translations"
   task :dump => :check do
-    options = {:translations => @command.translations}
-    @command = I18nliner::Commands::Dump.run(options) or exit 1
+    require 'i18nliner/commands/dump'
+
+    options = {:translations => @command.translations, :file => ENV['YML_FILE']}
+    @command = I18nliner::Commands::Dump.run(options)
+    @command.success? or exit 1
   end
 end
 

data/spec/commands/check_spec.rb

@@ -0,0 +1,22 @@
+require 'i18nliner/commands/check'
+
+describe I18nliner::Commands::Check do
+  describe ".run" do
+
+    around do |example|
+      I18nliner.base_path "spec/fixtures" do
+        example.run
+      end
+    end
+
+    it "should find errors" do
+      checker = I18nliner::Commands::Check.new({:silent => true})
+      checker.check_files
+      checker.translations.values.should == ["welcome, %{name}", "Hello World", "*This* is a test, %{user}"]
+      checker.errors.size.should == 2
+      checker.errors.each do |error|
+        error.should =~ /\Ainvalid signature/
+      end
+    end
+  end
+end

data/spec/commands/dump_spec.rb

@@ -0,0 +1,27 @@
+# encoding: UTF-8
+require 'i18nliner/commands/dump'
+require 'tmpdir'
+
+describe I18nliner::Commands::Dump do
+  describe ".run" do
+
+    around do |example|
+      Dir.mktmpdir do |dir|
+        I18nliner.base_path dir do
+          example.run
+        end
+      end
+    end
+
+    it "should dump translations in utf8" do
+      dumper = I18nliner::Commands::Dump.new({:silent => true, :translations => {'i18n' => "Iñtërnâtiônàlizætiøn"}})
+      dumper.run
+      File.read(dumper.yml_file).gsub(/\s+$/, '').should == <<-YML.strip_heredoc.strip
+        ---
+        #{I18n.default_locale}:
+          i18n: Iñtërnâtiônàlizætiøn
+      YML
+    end
+  end
+end
+

data/spec/extensions/core_spec.rb

@@ -0,0 +1,71 @@
+require 'i18n'
+require 'i18nliner/extensions/core'
+
+describe I18nliner::Extensions::Core do
+  let(:i18n) do
+    Module.new do
+      extend(Module.new do
+        def translate(*args)
+          simple_translate(args[0], args[1])
+        end
+
+        def simple_translate(key, options)
+          string = options.delete(:default)
+          interpolate_hash(string, options)
+        end
+
+        def interpolate_hash(string, values)
+          I18n.interpolate_hash(string, values)
+        end
+      end)
+      extend I18nliner::Extensions::Core
+    end
+  end
+
+  describe ".translate" do
+    it "should should normalize the arguments passed into the original translate" do
+      expect(i18n).to receive(:simple_translate).with("hello_name_84ff273f", :default => "Hello %{name}", :name => "bob")
+      i18n.translate("Hello %{name}", :name => "bob")
+    end
+
+    it "should apply wrappers" do
+      result = i18n.translate("Hello *bob*. Click **here**", :wrappers => ['<b>\1</b>', '<a href="/">\1</a>'])
+      result.should == "Hello <b>bob</b>. Click <a href=\"/\">here</a>"
+      result.should be_html_safe
+    end
+
+    it "should html-escape the default when applying wrappers" do
+      i18n.translate("*bacon* > narwhals", :wrappers => ['<b>\1</b>']).
+        should == "<b>bacon</b> &gt; narwhals"
+    end
+  end
+
+  describe ".translate!" do
+    it "should behave like translate" do
+      expect(i18n).to receive(:simple_translate).with("hello_name_84ff273f", :default => "Hello %{name}", :name => "bob")
+      i18n.translate!("Hello %{name}", :name => "bob")
+    end
+  end
+
+  describe ".interpolate_hash" do
+    it "should not mark the result as html-safe if none of the components are html-safe" do
+      result = i18n.interpolate_hash("hello %{name}", :name => "<script>")
+      result.should == "hello <script>"
+      result.should_not be_html_safe
+    end
+
+    it "should html-escape values if the string is html-safe" do
+      result = i18n.interpolate_hash("some markup: %{markup}".html_safe, :markup => "<html>")
+      result.should == "some markup: &lt;html&gt;"
+      result.should be_html_safe
+    end
+
+    it "should html-escape the string and other values if any value is html-safe" do
+      markup = "<input>"
+      result = i18n.interpolate_hash("type %{input} & you get this: %{output}", :input => markup, :output => markup.html_safe)
+      result.should == "type &lt;input&gt; &amp; you get this: <input>"
+      result.should be_html_safe
+    end
+  end
+end
+