haml-i18n-extractor 0.0.5

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.yml

data/.rvmrc

@@ -0,0 +1 @@
+rvm use 1.9.3

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in haml-i18n-extractor.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Shai Rosenfeld
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,110 @@
+# Haml::I18n::Extractor
+
+Extract strings likely to be translated from haml into locale file
+
+## Usage
+
+After installing this gem, you can run the binary in a rails app:
+
+`cd your-rails-app; haml-i18n-extractor .`
+
+## Notes
+
+Right now the design works on a per-file basis:
+
+<pre>
+begin
+  @ex1 = Haml::I18n::Extractor.new(haml_path)
+  @ex1.run
+rescue Haml::I18n::Extractor::InvalidSyntax
+  puts "There was an error with #{haml_path}"
+rescue Haml::I18n::Extractor::NothingToTranslate
+  puts "Nothing to translate for #{haml_path}"
+end  
+</pre>
+
+This is pretty much beta - but it does work! Since it is beta, it does not overwrite anything but lets you decide what you want, and don't want.
+
+The workflow at the moment is, to run the binary, then do a `git diff` and see all the changes. 
+
+What you should be seeing for a "foo.haml" file is a dumped version "foo.i18n-extractor.haml" file which has the temporary representation of the file it tried to translate. It also dumps the corresponding i18n locale .yml file for the haml just translated in the current working directory, suffixed with the path of the haml file it was working on.
+
+## Before
+
+<pre>
+  shai@comp ~/p/project ‹master*› » cat app/views/admin/notifications/index.html.haml
+  %h1 Consumer Notifications
+
+  .nav= will_paginate(@consumer_notifications)
+  %table.themed{cellspacing: 0}
+    %thead
+      %tr
+        %th.first Type
+        %th Identifier
+        %th Data
+        %th Success
+        %th Reported To
+        %th.last &nbsp;
+    - @consumer_notifications.each do |cn|
+      %tr
+        %td.type= cn.notification.type
+        %td.identifier= cn.notification.identifier
+        %td.data= cn.notification.data
+        %td.success= cn.success
+        %td.reported_to= cn.reported_to
+  .nav= will_paginate(@consumer_notifications)
+</pre>
+
+## After: new haml, new yaml.
+
+- Note how some of the strings are replaced, and the ones that shouldn't, aren't. Yup.
+
+Haml:
+
+<pre>
+  shai@comp ~/p/project ‹master*› » cat app/views/admin/notifications/index.html.i18n-extractor.haml 
+  %h1= t('.consumer_notifications')
+
+  .nav= will_paginate(@consumer_notifications)
+  %table.themed{cellspacing: 0}
+    %thead
+      %tr
+        %th.first= t('.type')
+        %th= t('.identifier')
+        %th= t('.data')
+        %th= t('.success')
+        %th= t('.reported_to')
+        %th.last= t('.nbsp;')
+    - @consumer_notifications.each do |cn|
+      %tr
+        %td.type= cn.notification.type
+        %td.identifier= cn.notification.identifier
+        %td.data= cn.notification.data
+        %td.success= cn.success
+        %td.reported_to= cn.reported_to
+  .nav= will_paginate(@consumer_notifications)
+</pre>
+
+Yaml: 
+
+<pre>
+  shai@comp ~/p/project ‹master*› » cat notifications.index.html.haml.yml  
+  --- !ruby/hash:ActiveSupport::HashWithIndifferentAccess
+  en: !ruby/hash:ActiveSupport::HashWithIndifferentAccess
+    notifications: !ruby/hash:ActiveSupport::HashWithIndifferentAccess
+      consumer_notifications: Consumer Notifications
+      type: Type
+      identifier: Identifier
+      data: Data
+      success: Success
+      reported_to: Reported To
+      nbsp;: ! '&nbsp;'
+</pre>
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,54 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+def with_each_gemfile
+  old_env = ENV['BUNDLE_GEMFILE']
+  gemfiles.each do |gemfile|
+    puts "Using gemfile: #{gemfile}"
+    ENV['BUNDLE_GEMFILE'] = gemfile
+    yield
+  end
+ensure
+  ENV['BUNDLE_GEMFILE'] = old_env
+end
+
+def silence_warnings
+  the_real_stderr, $stderr = $stderr, StringIO.new
+  yield
+ensure
+  $stderr = the_real_stderr
+end
+
+Rake::TestTask.new do |t|
+  t.libs << 'lib' << 'test'
+  t.test_files = Dir["test/**/*_test.rb"]
+  t.warning = true
+  t.verbose = true
+end
+
+task :default => :test
+namespace :test do
+  namespace :bundles do
+    desc "Install all dependencies necessary to test."
+    task :install do
+      with_each_gemfile {sh "bundle"}
+    end
+
+    desc "Update all dependencies for testing."
+    task :update do
+      with_each_gemfile {sh "bundle update"}
+    end
+  end
+
+  desc "Test all supported versions of rails. This takes a while."
+  task :rails_compatibility => 'test:bundles:install' do
+    with_each_gemfile {sh "bundle exec rake test"}
+  end
+  task :rc => :rails_compatibility
+end
+
+# FIXME, jsut run this
+task :force_build_and_install do
+  %{ gem uninstall -x haml-i18n-extractor; rm *gem; gem build *gemspec; gem install --local *gem }
+end

data/TODO

@@ -0,0 +1,5 @@
+- Have an interactive mode: define workflow(s) for using this lib/tool.
+- Remove the ex1.output ex1.i18n-extractor duplication. (or move to tmp dir)
+- Add {} to disallowed characters
+- Add yaml fixtures and compare those 
+- handled email suffixes (foo.en.html.haml etc)

data/bin/haml-i18n-extractor

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+
+require 'haml-i18n-extractor'
+
+if ARGV.empty?
+  puts "Usage: <haml-i18n-extractor> [path]"
+  abort
+end
+
+relative_path = ARGV[0]
+absolute_path = File.expand_path(relative_path)
+
+all_files = Dir.glob("**/*")
+haml_paths = all_files.select do |file|
+  file.match /.haml$/
+end
+
+# right now, this should create a copy of the haml file in that directory with a special suffix for review.
+# review it, see what fails, fix, etc.
+
+haml_paths.each do |haml_path|
+  begin
+    puts "extracting strings for #{haml_path}"
+    @ex1 = Haml::I18n::Extractor.new(haml_path)
+    @ex1.run
+  rescue Haml::I18n::Extractor::InvalidSyntax
+    puts "There was an error with #{haml_path}"
+  rescue Haml::I18n::Extractor::NothingToTranslate
+    puts "Nothing to translate for #{haml_path}"
+  end
+end

data/haml-i18n-extractor.gemspec

@@ -0,0 +1,29 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/haml-i18n-extractor/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Shai Rosenfeld"]
+  gem.email         = ["shaiguitar@gmail.com"]
+  gem.description   = %q{Parse the texts out of the haml files into localization files}
+  gem.summary       = %q{Parse the texts out of the haml files into localization files}
+  gem.homepage      = ""
+  gem.executables = ['haml-i18n-extractor']
+  gem.default_executable = 'haml-i18n-extractor'
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "haml-i18n-extractor"
+  gem.require_paths = ["lib"]
+  gem.version       = Haml::I18n::Extractor::VERSION
+
+  gem.add_dependency "tilt"
+  gem.add_dependency "haml"
+
+  gem.add_development_dependency 'rails', '>= 3.0.0'
+  gem.add_development_dependency 'rbench'
+  gem.add_development_dependency 'pry'
+  gem.add_development_dependency 'minitest'
+  gem.add_development_dependency 'nokogiri'
+
+end

data/lib/haml-i18n-extractor/core-ext/hash.rb

@@ -0,0 +1,8 @@
+# @date = Hash.recursive_init
+# @date[month][day][hours][min][sec] = 1
+# @date now equals {month=>{day=>{hours=>{min=>{sec=>1}}}}}
+class Hash
+  def self.recursive_init
+    new { |hash, key| hash[key] = recursive_init }
+  end
+end

data/lib/haml-i18n-extractor/extractor.rb

@@ -0,0 +1,84 @@
+module Haml
+  module I18n
+    class Extractor
+
+      def self.debug?
+        ENV['DEBUG']
+      end
+      
+      class InvalidSyntax < StandardError ; end
+      class NothingToTranslate < StandardError ; end
+      class NotDefinedLineType < StandardError ; end
+
+      LINE_TYPES_ALL = [:text, :not_text, :loud, :silent, :element]
+      LINE_TYPES_ADD_EVAL = [:text, :element]
+      
+      attr_reader :haml_reader, :haml_writer
+      attr_reader :locale_hash, :yaml_tool
+
+      def initialize(haml_path)
+        @haml_reader = Haml::I18n::Extractor::HamlReader.new(haml_path)
+        validate_haml(@haml_reader.body)
+        @haml_writer = Haml::I18n::Extractor::HamlWriter.new(haml_path)
+        @yaml_tool = Haml::I18n::Extractor::YamlTool.new
+        @locale_hash = {}
+      end
+
+      def run
+        assign_replacements
+        validate_haml(@haml_writer.body)
+        @haml_writer.write_file
+        @yaml_tool.write_file
+      end
+      
+      def assign_new_body
+        @haml_writer.body = new_body
+      end
+      
+      def assign_yaml
+        @yaml_tool.locale_hash = @locale_hash
+      end
+      
+      def assign_replacements
+        assign_new_body
+        assign_yaml
+      end
+
+      # processes the haml document, replaces the input body to an output body of replacements. also sets an in memory hash of replacement info
+      # to be used later by the yaml tool.
+      # keep indentation to use later when printing out.
+      def new_body
+        new_lines = []
+        file_has_been_parsed = false
+        @haml_reader.lines.each_with_index do |orig_line, line_no|
+          orig_line.chomp!
+          orig_line.rstrip.match(/([ \t]+)?(.*)/)
+          whitespace_indentation = $1
+          orig_line = $2
+          line_type, line_match = Haml::I18n::Extractor::TextFinder.new(orig_line).process_by_regex
+          if line_match && !line_match.empty?
+            file_has_been_parsed = true
+            replacer = Haml::I18n::Extractor::TextReplacer.new(orig_line, line_match, line_type)
+            @locale_hash[line_no] = replacer.replace_hash.dup.merge!({:path => @haml_reader.path })
+            new_lines << "#{whitespace_indentation}#{replacer.replace_hash[:modified_line]}"
+          else
+            @locale_hash[line_no] = { :modified_line => nil,:keyname => nil,:replaced_text => nil, :path => nil }
+            new_lines << "#{whitespace_indentation}#{orig_line}"
+          end
+        end
+        raise NothingToTranslate if !file_has_been_parsed
+        new_lines.join("\n")
+      end
+      
+      private
+      
+      def validate_haml(haml)
+        parser = Haml::Parser.new(haml, Haml::Options.new)
+        parser.parse
+        rescue Haml::SyntaxError
+          raise InvalidSyntax, "invalid syntax for haml #{@haml_reader.path}"
+      end
+      
+    end
+  end
+end

data/lib/haml-i18n-extractor/haml_reader.rb

@@ -0,0 +1,20 @@
+module Haml
+  module I18n
+    class Extractor
+      class HamlReader
+
+        attr_reader :body, :path, :lines
+
+        def initialize(path)
+          file = File.open(path, "r")
+          @path = path
+          @body = file.read
+          file.rewind
+          @lines = file.readlines
+          file.rewind
+        end
+        
+      end
+    end
+  end
+end

data/lib/haml-i18n-extractor/haml_writer.rb

@@ -0,0 +1,21 @@
+module Haml
+  module I18n
+    class Extractor
+      class HamlWriter
+
+        attr_accessor :path, :lines, :body
+
+        def initialize(orig_path)
+          @path = orig_path.gsub(/.haml$/, ".i18n-extractor.haml")
+        end
+
+        def write_file
+          f = File.open(@path, "w+")
+          f.puts(self.body)
+          f.close
+        end
+
+      end
+    end
+  end
+end

data/lib/haml-i18n-extractor/text_finder.rb

@@ -0,0 +1,94 @@
+require 'haml'
+require 'haml/parser'
+module Haml
+  module I18n
+    class Extractor
+      class TextFinder
+
+        THINGS_THAT_ARE_NOT_POTENTIAL_TEXT = [ Haml::Parser::DIV_CLASS, Haml::Parser::DIV_ID,
+            Haml::Parser::COMMENT, Haml::Parser::SANITIZE, Haml::Parser::FLAT_SCRIPT,
+            Haml::Parser::FILTER, Haml::Parser::DOCTYPE, Haml::Parser::ESCAPE ]
+
+        ## hint: Use http://rubular.com
+        SIMPLE_STRING_REGEX = /["'](.*)["']/
+        # = "yes"
+        # = 'yes'
+        LINK_TO_REGEX = /link_to\s*\(?\s*['"](.*)['"]\s*,\s*(.*)\)?/
+        # = link_to "yes", "http://bla.com"
+        # = link_to('yes'    , "http://bla.com")
+        # = link_to(  "yes's w space", "http://bla.com")
+        ELEMENT_REGEX = /%([\w.#-]+)({.+?})?(=)?(.*)/
+        # %foo.bar yes
+        # %foo.bar= no
+        # %foo{:a => 'b'} yes
+        # rubular.com against %foo#bar#cheez{:cheeze => 'hell'}= "what #{var}"
+
+        def initialize(haml)
+          @haml = haml
+          @parser = Haml::Parser.new(haml, Haml::Options.new)
+        end
+        
+        def line
+          if @haml == ""
+            return Haml::Parser::Line.new("", "", "", 0, @parser, false)
+          end
+          
+          match = @haml.rstrip.scan(/(([ \t]+)?(.*?))(?:\Z|\r\n|\r|\n)/m)
+          match.pop
+          haml_line ||= match.each_with_index.map do |(full, whitespace, text), index|
+            Haml::Parser::Line.new(whitespace, text.rstrip, full, index, @parser, false)
+          end.first
+          haml_line
+        end
+        
+        def find
+          text_match = process_by_regex.last
+        end
+
+        # returns [ line_type, text_found ]
+        def process_by_regex
+          case line.full[0]
+          when *THINGS_THAT_ARE_NOT_POTENTIAL_TEXT
+            [:not_text, ""]
+          when Haml::Parser::SILENT_SCRIPT
+            parse_silent_script
+          when Haml::Parser::ELEMENT
+            parse_element
+          when Haml::Parser::SCRIPT
+            parse_loud_script
+          else
+            [:text, line.full.strip]
+          end
+        end
+  
+        private
+
+        def parse_silent_script
+          line.full.match(/-[\s\t]*#{SIMPLE_STRING_REGEX}/)
+          [:silent, $1.to_s]
+        end
+  
+        def parse_loud_script
+          line.full.match(/=[\s\t]*#{LINK_TO_REGEX}/)
+          return [:loud, $1.to_s] if text = $1
+          line.full.match(/=[\s\t]*#{SIMPLE_STRING_REGEX}/)
+          return [:loud, $1.to_s]
+        end
+  
+        def parse_element
+          line.full.match(ELEMENT_REGEX)
+          elem_with_class_and_ids = $1
+          attributes_ruby_style = $2
+          is_loud_script = $3
+          text = $4.to_s
+          if is_loud_script
+            self.class.new("= #{text}").process_by_regex # treat like a loud script.
+          else
+            [:element, text.strip]
+          end
+        end
+  
+      end
+    end
+  end
+end

data/lib/haml-i18n-extractor/text_replacer.rb

@@ -0,0 +1,78 @@
+module Haml
+  module I18n
+    class Extractor
+      class TextReplacer
+
+        attr_reader :full_line, :text_to_replace, :line_type
+
+        T_REGEX = /t\('\..*'\)/
+        # limit the number of chars
+        LIMIT_KEY_NAME = 30
+        # do not pollute the key space it will make it invalid yaml
+        NOT_ALLOWED_IN_KEYNAME =  %w( ~ ` ! @ # $ % ^ & * - ( ) , )
+        
+        def initialize(full_line, text_to_replace,line_type)
+          @full_line = full_line
+          @text_to_replace = text_to_replace
+          if LINE_TYPES_ALL.include?(line_type)
+            @line_type = line_type
+          else
+            raise Extractor::NotDefinedLineType, "line type #{line_type} for #{full_line} does not make sense!"
+          end
+        end
+
+        def replace_hash
+          @replace_hash ||= { :modified_line => modified_line, :keyname => keyname, :replaced_text => @text_to_replace }
+        end
+        
+        # the new full line, including a `t()` replacement instead of the `text_to_replace` portion.
+        def modified_line
+          full_line = @full_line.dup
+          return @full_line if has_been_translated?(full_line)
+          remove_surrounding_quotes(full_line)
+          apply_ruby_evaling(full_line)
+          full_line
+        end
+        
+        private
+        
+        def keyname
+          text_to_replace = @text_to_replace.dup
+          has_been_translated?(text_to_replace) ? text_to_replace : "t('.#{to_keyname(text_to_replace)}')"
+        end
+
+        def apply_ruby_evaling(str)
+          if LINE_TYPES_ADD_EVAL.include?(@line_type)
+            if @line_type == :element
+              str.match /^([^\s\t]*)(.*)$/
+              elem, keyname = $1, $2
+              str.gsub!($2, "= #{$2.strip}")
+            elsif @line_type == :text
+              str.gsub!(str, "= "+str)
+            end
+          end
+        end
+
+        def has_been_translated?(str)
+          str.match T_REGEX
+        end
+
+        def remove_surrounding_quotes(str)
+          # if there are quotes surrounding the string, we want them removed as well...
+          unless str.gsub!('"' + @text_to_replace + '"', keyname)
+            unless str.gsub!("'" + @text_to_replace + "'", keyname)
+              str.gsub!(@text_to_replace, keyname)
+            end
+          end
+        end
+        
+        def to_keyname(str)
+          NOT_ALLOWED_IN_KEYNAME.each{ |rm_me| str.gsub!(rm_me, "") }
+          str = str.gsub(/\s+/, " ").strip
+          str.downcase.tr(' ', '_')[0..LIMIT_KEY_NAME-1]
+        end
+        
+      end
+    end
+  end
+end

data/lib/haml-i18n-extractor/version.rb

@@ -0,0 +1,7 @@
+module Haml
+  module I18n
+    class Extractor
+      VERSION = "0.0.5"
+    end
+  end
+end

data/lib/haml-i18n-extractor/yaml_tool.rb

@@ -0,0 +1,74 @@
+require 'yaml'
+require 'pathname'
+require 'haml-i18n-extractor/core-ext/hash'
+require 'active_support/hash_with_indifferent_access'
+
+module Haml
+  module I18n
+    class Extractor
+      class YamlTool
+        
+        attr_accessor :locales_dir, :locale_hash
+
+        def initialize(locales_dir = nil)
+          if locales_dir
+            @locales_dir = locales_dir
+          else
+            # locales_dir = rails_mode? ? (Rails.root.to_s + "/config/locales") : "."
+            @locales_dir = File.expand_path(".")
+          end
+        end
+
+        # {:en => {:view_name => {:key_name => :string_name } } }
+        def yaml_hash
+          h = HashWithIndifferentAccess.recursive_init
+          @locale_hash.map do |line_no, info|
+            unless info[:keyname].nil?
+              h[i18n_scope][standardized_viewname(info[:path])][standarized_keyname(info[:keyname])] = info[:replaced_text]
+            end
+          end
+          h
+        end
+
+        def locale_file
+          if @locale_hash
+            pth = @locale_hash.map{|_,h| h[:path] }.compact.first
+            if pth
+              full_path = Pathname.new(pth)
+              base_name = full_path.basename.to_s
+              File.expand_path(File.join( @locales_dir, standardized_viewname(full_path) + ".#{base_name}.yml"))
+            end
+          end.to_s
+        end
+        
+        def write_file
+          f = File.open(locale_file, "w+")
+          f.puts yaml_hash.to_yaml
+          f.flush
+        end
+        
+        private
+
+        def i18n_scope
+          :en
+        end
+
+        # comes in like "t('.some_place')", return .some_place
+        def standarized_keyname(name)
+          name.match(/t\('\.(.*)'\)/)
+          $1
+        end
+
+        # assuming rails format, app/views/users/index.html.haml return users
+        def standardized_viewname(pth)
+          Pathname.new(pth).dirname.to_s.split("/").last
+        end
+
+        # def rails_mode?
+        #   defined?(Rails)
+        # end
+
+      end
+    end
+  end
+end

data/lib/haml-i18n-extractor.rb

@@ -0,0 +1,7 @@
+require "haml-i18n-extractor/version"
+require "haml-i18n-extractor/text_finder"
+require "haml-i18n-extractor/text_replacer"
+require "haml-i18n-extractor/haml_reader"
+require "haml-i18n-extractor/haml_writer"
+require "haml-i18n-extractor/yaml_tool"
+require "haml-i18n-extractor/extractor"