herbgobbler 0.1.0

data/bin/gobble

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+require 'lib/herbgobbler'
+
+def print_usage
+  puts "ruby scripts/rails_gobble_all_views.rb <rails_root>"
+end
+
+# remove the file extension and the app/views/ so that
+# when the context is set that rails will be able to find
+# it through the default . key syntax
+def convert_path_to_key_path( path )
+  path.split('.').first.gsub( '/app/views/', '')
+end
+
+if( ARGV.length < 1 )
+  print_usage
+else
+  
+  rails_root = ARGV[0]
+  locale_file_name = "en.yml"
+  
+  rails_view_directory = "#{rails_root}/app/views"
+  full_yml_file_path = "#{rails_root}/config/locales/#{locale_file_name}"
+  
+  rails_translation_store = RailsTranslationStore.new
+  text_extractor = RailsTextExtractor.new( rails_translation_store )
+  
+  Dir["#{rails_view_directory}/**/*html.erb" ].each do |full_erb_file_path|
+    
+    erb_file = full_erb_file_path.gsub( rails_root, '' )
+    rails_translation_store.start_new_context( convert_path_to_key_path( erb_file.to_s ) )
+    erb_file = ErbFile.load( full_erb_file_path )
+    erb_file.extract_text( text_extractor )
+
+    File.open(full_erb_file_path, 'w') {|f| f.write(erb_file.to_s) }
+    puts "Wrote #{full_erb_file_path}"
+  end
+
+  File.open(full_yml_file_path, 'w') {|f| f.write(rails_translation_store.serialize) }
+  puts "Wrote #{full_yml_file_path}"
+  
+end
+

data/grammer/erb_grammer.treetop

@@ -0,0 +1,256 @@
+grammar ERBGrammer
+        rule erb
+             ( erb_string /
+               erb_block  / 
+               ignorable_script_tag_content /
+               html_end_tag / 
+               html_start_tag / 
+               html_self_contained / 
+               whitespace / 
+               text 
+             ) (erb)*
+         end
+
+        rule method_names             
+             "link_to" <NonTextNode>
+             {
+                def node_name  
+                    "method_names"
+                end
+                def can_be_combined?
+                    true
+                end
+             }
+        end
+
+        rule comma
+            "," <NonTextNode>
+            {
+                def node_name
+                    "method_names"
+                end
+
+                def can_be_combined?
+                    true
+                end
+
+            }
+        end 
+       
+        rule method_call_content
+             method_names whitespace double_quoted_ruby_string (comma erb_string_non_text_content)? <MethodCallNode>
+        end
+       
+        rule erb_string_non_text_content
+             (!erb_block_end .)* <ErbStringNonTextContent>
+             {
+                def can_be_combined?
+                    true
+                end
+                        
+                def node_name
+                   "erb_string_non_text_content"
+                end
+             }
+        end
+
+        rule erb_string
+             start_erb:erb_string_start start_space:(space) text:(method_call_content / double_quoted_ruby_string)? everything_else:erb_string_non_text_content end_erb:(erb_block_end) <NonTextNode>
+             {
+                def node_name
+                    "erb_string"
+                end
+             }
+
+        end
+        
+        rule string_variable 
+             '#{' variable:((!'}' .)*) '}' <HerbStringVariable>
+        end
+
+        rule double_quoted_ruby_string
+        start_quote:'"' unquoted_text:((!'"' (string_variable / .))*) end_quote:'"' <DoubleQuotedTextNode>
+             {
+                def can_be_combined?
+                    false
+                end
+                
+                def html?
+                    false
+                end 
+                
+                def text_value
+                    self.elements.inject("") { |text, node| text += node.text_value }
+                end
+
+                def node_name
+                    "double_quoted_ruby_string"
+                end
+             }
+
+        end
+
+        rule erb_block
+             start_block:'<%' block_contents:((!erb_block_end .)*) end_block:not_combindable_erb_block_end <NonTextNode>
+             {
+                def node_name
+                    "erb_block"
+                end
+             }
+
+        end
+        
+        rule erb_string_start
+           '<%=' <NonExtractingNonTextNode>
+             {
+                def node_name
+                    "erb_string_start"
+                end
+                
+                def can_be_combined?
+                    true
+                end
+             }
+
+        end
+
+        rule not_combindable_erb_block_end
+             ('%>' / '-%>') <NonTextNode>
+             {
+                def node_name
+                    "not_combindable_erb_block_end"
+                end
+
+                def can_be_combined?
+                    false
+                end
+
+             }
+
+        end
+        
+        rule erb_block_end
+             ('%>' / '-%>') <NonExtractingNonTextNode>
+             {
+                def node_name
+                    "erb_block_end"
+                end
+
+                def can_be_combined?
+                    true
+                end
+
+             }
+
+        end
+
+        rule ignorable_script_tag_content
+             '<script' (!'</script' .)+ <IgnorableTagNode>
+             {
+                def can_be_combined?
+                    false
+                end
+
+                def node_name
+                   "ignorable_tag_name"
+                end
+             }
+        end
+
+
+        rule html_self_contained
+             start_tag:'<' tag_name:([a-zA-Z])+ tag_contents:(!'/>' (erb_string / .))* end_tag:'/>' <NonTextNode> 
+             {
+                def can_be_combined?
+                   true
+                end
+
+                def node_name
+                    "html_self_contained"
+                end
+
+             }
+        end     
+
+        rule html_end_tag
+             '</' tag_name:([a-zA-Z])+ (!'>' .)* '>' <NonTextNode>
+             {
+                def can_be_combined?
+                    !(['a', 'b', 'span', 'strong'].index(tag_name.text_value.downcase).nil?)
+                end
+
+                def node_name
+                    "html_end_tag"
+                end
+             }
+
+        end
+        
+        rule tag_text
+            (!'>' !'/>' (erb_string / .) )* <NonTextNode>
+            {
+               def node_name
+                   "tag_text"
+               end
+            }
+        end
+
+        rule html_start_tag
+             start_bracket:'<'  tag_name:([a-zA-Z])+  extra_text:tag_text end_bracket:'>' <NonTextNode>
+             {
+                def can_be_combined?
+                    !(['a', 'b', 'span','strong'].index(tag_name.text_value.downcase).nil?)
+                end
+
+                def node_name
+                    "html_start_tag"
+                end
+             }
+
+        end     
+
+        rule text
+             ((!html_self_contained !html_start_tag !html_end_tag !erb_string !erb_block) . )+ <TextNode>
+             {
+                def can_be_combined?
+                    true
+                end
+
+                def node_name
+                    "text"
+                end
+             }
+
+        end
+
+        rule space
+           (' ')+ <NonTextNode>
+           {
+                      def node_name
+                         "space"
+                      end
+
+                      def can_be_combined?
+                          true
+                      end                      
+           }
+        end
+
+        rule whitespace
+            ((![a-zA-Z] !('<' [a-zA-Z]+) !('</' [a-zA-Z]+) !'<%' !'"') . )+ <NonTextNode>
+            {
+                def can_be_combined?
+                   true
+                end
+               
+               def node_name    
+                    "whitespace"
+               end
+               def white_space?
+                  true
+               end
+            }
+        end
+        
+end
+        

data/lib/core/base_node.rb

@@ -0,0 +1,29 @@
+module BaseNode
+  @@keys = []
+  @@hash = nil
+  
+  def shatter?
+    false
+  end
+  
+  def shattered_node_list( *nodes )
+      nodes.reject {|terminal| terminal.nil? || terminal.text_value.empty? }
+  end
+
+  protected
+
+  def generate_i18n_key( text_extractor, node_tree )
+    I18nKey.new( self.text_value, get_key_hash(text_extractor, node_tree) )    
+  end
+  
+  def get_key_hash( text_extractor, node_tree )
+    # here we just try to use different keysets based on where they are in the tree since
+    # there can only be key collisions when generating the same node
+    if( @@hash != text_extractor.hash.to_s + node_tree.last.hash.to_s )
+      @@keys = []
+      @@hash = text_extractor.hash.to_s + node_tree.last.hash.to_s
+    end
+    @@keys
+  end
+  
+end

data/lib/core/base_text_extractor.rb

@@ -0,0 +1,19 @@
+class BaseTextExtractor < TextExtractor
+
+  # This is called when text extraction has begun
+  def starting_text_extraction
+  end
+  
+  
+  # This takes in a text node and returns one or more nodes that will
+  # then be output.  The nodes that are output should implement
+  # node_name and text_value
+  def html_text( text_node )
+    self.remove_leading_and_trailing_whitespace( text_node )
+  end
+
+  # This is called when text extraction has finished
+  def completed_text_extraction
+  end
+  
+end

data/lib/core/base_translation_store.rb

@@ -0,0 +1,49 @@
+class BaseTranslationStore
+
+  attr_reader :context
+  
+  def initialize
+    @context_maps = {}
+    @context_array = []
+  end
+
+  def each( &block )
+    @context_array.each do |context|
+      @context_maps[context].each do |key_value_pair|
+        key_name, key_value = *key_value_pair
+        yield context, key_name, key_value
+      end
+    end
+  end
+  
+  def start_new_context( new_context )
+    @context = new_context
+  end
+  
+  def add_translation( key, value )
+    context_map() << [key, value]
+  end
+
+  def serialize
+    raise "Implement me"
+  end
+
+  private
+
+
+  def context_map
+    if( @context.nil? )
+      context_name = ''
+    else
+      context_name = @context
+    end
+    
+    local_context_map = []
+    @context_maps[context_name] =  [] if @context_maps[context_name].nil?
+    @context_array << context_name  if @context_array.index( context_name ).nil?
+    @context_maps[context_name]
+  end
+  
+
+  
+end

data/lib/core/core.rb

@@ -0,0 +1,17 @@
+require 'core/base_node'
+require 'core/node_processing'
+require 'core/erb_file'
+require 'core/text_node'
+require 'core/non_text_node'
+require 'core/text_extractor'
+require 'core/base_text_extractor'
+require 'core/rails_text_extractor'
+require 'core/i18n_key'
+require 'core/ignorable_tag_node'
+require 'core/base_translation_store'
+require 'core/rails_translation_store'
+require 'core/double_quoted_text_node'
+require 'core/erb_string_non_text_content'
+require 'core/non_extracting_non_text_node'
+require 'core/method_call_node'
+

data/lib/core/double_quoted_text_node.rb

@@ -0,0 +1,89 @@
+class DoubleQuotedTextNode < Treetop::Runtime::SyntaxNode  
+  include TextNode
+
+  
+  def has_variables?
+    leaves.any? { |node| node.is_a?( HerbStringVariable ) }
+  end
+  
+  def leaves
+    leaves_to_return = []
+    original_elements.each do |element|
+      leaves_to_return += flatten element
+    end unless original_elements.nil? || original_elements.empty?
+    leaves_to_return
+  end
+  
+  alias :original_elements :elements
+  alias :original_text_value :text_value
+  
+  def elements
+    if( has_variables? )
+      strip_leading_and_trailing_double_quotes( variable_elements )
+    else
+      strip_leading_and_trailing_double_quotes( original_elements )
+    end
+  end
+
+  def text_value
+    if( has_variables? )
+      elements.inject('') { |string, current_element| string += current_element.text_value }
+    else
+      original_text_value
+    end
+    
+  end
+  
+  def variable_elements
+    found_elements = []
+    current_text = ''
+    leaves.each do |leaf|
+      if( leaf.is_a?( HerbStringVariable ) )
+        found_elements = add_text_to_array( found_elements, current_text )
+        current_text = ''
+        found_elements << leaf
+      else
+        current_text += leaf.text_value unless( leaf.nil? || leaf.text_value.empty? )
+      end
+    end
+    found_elements = add_text_to_array( found_elements, current_text )
+    found_elements
+  end
+
+  private
+
+  def strip_leading_and_trailing_double_quotes( list_of_elements )
+    first_element = list_of_elements.first
+    last_element = list_of_elements.last
+
+    list_of_elements = list_of_elements[ 1..list_of_elements.length - 2 ]
+    first_element = remove_double_quote_at_position( first_element, 0 )
+    last_element = remove_double_quote_at_position( last_element, last_element.text_value.length - 1 )
+    ([first_element,list_of_elements,last_element]).flatten.compact
+  end
+
+  def remove_double_quote_at_position( element, position )
+    if( element.text_value[position..position] == '"' )
+      new_text = ""
+      new_text += element.text_value[0..position-1] if position > 0
+      new_text += element.text_value[position+1..element.text_value.length - 1] if (position + 1) < (element.text_value.length - 1)
+      if( new_text.length == 0 )
+        element = nil
+      else
+        element = HerbTextNode.new( new_text, false )
+      end
+    end
+
+    element
+  end
+  
+  
+  def add_text_to_array( array, text )
+    unless( text.empty? )
+      array << HerbTextNode.new( text )
+    end
+    array
+  end
+  
+
+end

data/lib/core/erb_file.rb

@@ -0,0 +1,159 @@
+class ErbFile
+
+  include NodeProcessing
+  attr_accessor :nodes
+  attr_accessor :node_set
+  
+  def initialize( node_set )
+    @node_set = node_set
+    @nodes = flatten_elements if compiled?
+    @debug = false
+  end
+
+  def flatten_elements
+    terminals = []
+    flatten( @node_set, terminals )
+    terminals
+  end
+
+  def compiled?
+    !@node_set.nil?
+  end
+
+  def ErbFile.debug( file_path )
+    ErbFile.parse( File.read( file_path ) )
+    @parser.failure_reason
+  end
+
+  def combine_nodes( leaves )
+    combined_nodes = []  
+    leaves.each do |node|
+      last_combined_node = combined_nodes.pop
+      if( last_combined_node.nil? )
+        combined_nodes << node
+      elsif( combindable_node?( node ) && combindable_node?( last_combined_node ) )
+        new_node = HerbNodeRetainingNonTextNode.new( last_combined_node )
+        new_node << node
+        combined_nodes << new_node
+      elsif( !node.is_a?(TextNode)  && !last_combined_node.is_a?(TextNode) && !(last_combined_node.is_a?(BaseNode) && last_combined_node.can_be_combined? ) && !(node.is_a?(NonTextNode) && node.can_be_combined? ) )
+        combined_nodes << combine_two_nodes( last_combined_node, node, HerbNodeRetainingNonTextNode )
+      elsif( last_combined_node.is_a?(TextNode) && node.is_a?(TextNode) )
+        combined_nodes << combine_two_nodes( last_combined_node, node, HerbNodeRetainingTextNode )
+      elsif( combindable_node?( last_combined_node ) && node.is_a?(TextNode) )
+        combined_nodes << combine_two_nodes( last_combined_node, node, HerbNodeRetainingTextNode )
+      elsif( combindable_node?( node ) && last_combined_node.is_a?(TextNode))
+        combined_nodes << combine_two_nodes( last_combined_node, node, HerbNodeRetainingTextNode )        
+      elsif( node.is_a?(NonTextNode) && node.can_be_combined? && last_combined_node.is_a?(TextNode ) )
+        combined_nodes << combine_two_nodes( last_combined_node, node, HerbNodeRetainingTextNode )
+      else
+        combined_nodes << last_combined_node
+        combined_nodes << node
+      end
+    end
+    remove_edge_tags_from_combined_text_nodes( combined_nodes )
+  end
+  
+
+  def extract_text( text_extractor )
+    # 1.  Going to have to loop over all of the elements and find all of
+    # the text.
+    # 2.  Then make the call back to the text_extractor
+    # 3.  Finally replace the nodes in the top levels (not sure how
+    # this is going to work with any of the nested stuff)
+    text_extractor.starting_text_extraction
+    new_node_set = []
+    @nodes = combine_nodes( @nodes )    
+    @nodes.each do |node|
+      if( node.text? )
+        text_extractor.start_html_text
+        node.extract_text( text_extractor, new_node_set )
+        new_node_set += text_extractor.end_html_text
+      else
+        new_node_set << node
+      end
+      @nodes = new_node_set
+      self
+    end
+    text_extractor.completed_text_extraction
+  end
+  
+  def ErbFile.from_string( string_to_parse )
+    ErbFile.parse( string_to_parse )
+  end
+  
+  def ErbFile.load( file_path )
+    ErbFile.parse( File.read( file_path ) )
+  end
+
+  def serialize
+    to_return = ""
+    nodes.each do |node|
+      to_return << node.text_value
+    end
+
+    to_return
+  end
+  
+  
+  def to_s
+    to_return = ""
+    nodes.each do |node|
+      to_return += node.text_value
+    end
+    to_return
+  end
+  
+  private
+
+  def remove_edge_tags_from_combined_text_nodes( nodes )
+    node_count = nodes.size
+    to_return = []
+
+    nodes.each do |node|
+      if( node.is_a?( HerbNodeRetainingTextNode ) )
+        to_return += remove_edge_tags_from_combined_text_node( node )
+      else
+        to_return << node
+      end
+    end
+
+    if( node_count != to_return.length )
+      to_return = remove_edge_tags_from_combined_text_nodes( to_return )
+    end
+
+    return to_return
+  end
+
+  def remove_edge_tags_from_combined_text_node( node )
+    if( node.can_remove_starting_or_ending_html_tags? )
+      node.break_out_start_and_end_tags
+    else
+      [node]
+    end
+  end
+  
+  
+  
+  def combine_two_nodes( node_a, node_b, resulting_node_type )
+    if node_a.class == resulting_node_type
+      node_a << node_b
+      node_a
+    else
+      new_node = resulting_node_type.new( node_a )
+      new_node << node_b
+      new_node
+    end
+  end
+  
+
+  def combindable_node?( node )
+    node.is_a?( NonTextNode) && node.can_be_combined?
+  end
+    
+  def ErbFile.parse( data_to_parse )
+    Treetop.load( $ERB_GRAMMER_FILE )
+    @parser = ERBGrammerParser.new
+    ErbFile.new( @parser.parse( data_to_parse ) )
+  end
+  
+end

data/lib/core/erb_string_non_text_content.rb

@@ -0,0 +1,8 @@
+module ErbStringNonTextContent
+  include NonTextNode
+
+  def extract_text( text_extractor, node_tree )
+    text_extractor.add_variable( generate_i18n_key( text_extractor, node_tree ).to_s, self.text_value.strip ) unless self.contains_only_whitespace?
+  end
+  
+end