scrubyt 0.2.8 → 0.3.0

data/lib/scrubyt/output/scrubyt_result.rb

@@ -0,0 +1,28 @@
+module Scrubyt
+  class ScrubytResult < ResultNode
+    attr_accessor :root_pattern
+
+    def export(arg1, output_file_name=nil, extractor_result_file_name=nil)
+      #      require 'scrubyt/output/export_old'; Scrubyt::ExportOld.export(arg1, self, output_file_name, extractor_result_file_name) ; return
+      if File.exists? arg1
+        old_export(arg1, output_file_name, extractor_result_file_name)
+      else
+        new_export(arg1, output_file_name, extractor_result_file_name)
+      end
+    end
+
+    def show_stats
+      #Implement me...
+    end
+
+    def old_export(input_file, output_file_name=nil, extractor_result_file_name=nil)
+      contents = open(input_file).read
+      wrapper_name = contents.scan(/\s+(.+)\s+=.*Extractor\.define.*/)[0][0]
+      Scrubyt::Export.export(@root_pattern, wrapper_name, output_file_name, extractor_result_file_name)
+    end
+
+    def new_export(wrapper_name, output_file_name=nil, extractor_result_file_name=nil)
+      Scrubyt::Export.export(@root_pattern, wrapper_name, output_file_name, extractor_result_file_name)
+    end
+  end
+end

data/lib/scrubyt/utils/ruby_extensions.rb

@@ -59,6 +59,14 @@ module Math
   end
 end
 
+#just some hack here to allow current examples' syntax:
+#table_data.to_xml.write(open('result.xml', 'w'), 1)
+class String
+  def write(stringio, add_indent=0)
+    stringio.write((self.split("\n").collect { |line| ('  ' * add_indent) + line }).join("\n"))
+  end
+end
+
 class Array
   def to_sexp
     [:array, *to_sexp_array]

data/lib/scrubyt/utils/simple_example_lookup.rb

@@ -16,8 +16,21 @@ module Scrubyt
     #In this case, <a>'s text is considered to be "Bon nuit, monsieur"
     def self.find_node_from_text(doc, text, next_link=false, index = 0)
       text.gsub!('»', '&#187;')
+      #Process immediate attribute extraction (like "go to google.com[@href]")
+      if text =~ /.+\/@.+$/
+        text = text.scan(/^(.+?)\/@.+$/)[0][0]
+      elsif text =~ /.+\[\d+\]$/
+        res = text.scan(/(.+)\[(\d+)\]$/)
+        text = res[0][0]
+        index = res[0][1].to_i
+      elsif text =~ /.+\[.+\]$/
+        final_element_name = text.scan(/^(.+?)\[/)[0][0]
+        text = text.scan(/\[(.+?)\]/)[0][0]
+      end
       text = Regexp.escape(text) if text.is_a? String
-      SharedUtils.traverse_for_match(doc,/#{text}/)[index]
+      result = SharedUtils.traverse_for_match(doc,/^#{text}$/)[index]
+      result = XPathUtils.traverse_up_until_name(result,final_element_name) if final_element_name
+      result
     end
   end #End of class SimpleExampleLookup
 end #End of module Scrubyt

data/lib/scrubyt/utils/xpathutils.rb

@@ -94,6 +94,11 @@ module Scrubyt
     #most typically the user will need the 0th - but if this is not the
     #case, there is the possibility to override this
     def self.find_image(doc, example, index=0)
+      if example =~ /\.(jpg|png|gif|jpeg)(\[\d+\])$/
+        res = example.scan(/(.+)\[(\d+)\]$/)
+        example = res[0][0]
+        index = res[0][1].to_i
+      end
       (doc/"//img[@src='#{example}']")[index]
     end
 
@@ -136,6 +141,12 @@ module Scrubyt
       "/" + original_child_xpath_parts[i..-1].join('/')
     end
 
+    def self.to_full_XPath(doc, xpath, generalize)
+      elem = doc/xpath
+      elem = elem.map[0] if elem.is_a? Hpricot::Elements
+      XPathUtils.generate_XPath(elem, nil, generalize)
+    end
+
 private
     #Find the index of the child inside the parent
     #For example:

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.0
 specification_version: 1
 name: scrubyt
 version: !ruby/object:Gem::Version 
-  version: 0.2.8
-date: 2007-04-12 00:00:00 +02:00
+  version: 0.3.0
+date: 2007-05-28 00:00:00 +02:00
 summary: A powerful Web-scraping framework
 require_paths: 
 - lib
@@ -34,10 +34,13 @@ files:
 - CHANGELOG
 - Rakefile
 - lib/scrubyt.rb
+- lib/scrubyt/logging.rb
 - lib/scrubyt/output/result_dumper.rb
 - lib/scrubyt/output/result.rb
 - lib/scrubyt/output/export.rb
 - lib/scrubyt/output/post_processor.rb
+- lib/scrubyt/output/result_node.rb
+- lib/scrubyt/output/scrubyt_result.rb
 - lib/scrubyt/utils/compound_example_lookup.rb
 - lib/scrubyt/utils/simple_example_lookup.rb
 - lib/scrubyt/utils/ruby_extensions.rb
@@ -61,16 +64,8 @@ files:
 - lib/scrubyt/core/shared/u_r_i_builder.rb
 - lib/scrubyt/core/shared/extractor.rb
 - lib/scrubyt/core/shared/evaluation_context.rb
-test_files: 
-- test/unittests/input
-- test/unittests/constraint_test.rb
-- test/unittests/filter_test.rb
-- test/unittests/xpathutils_test.rb
-- test/unittests/extractor_test.rb
-- test/unittests/pattern_test.rb
-- test/unittests/simple_example_lookup_test.rb
-- test/unittests/input/test.html
-- test/unittests/input/constraint_test.html
+test_files: []
+
 rdoc_options: []
 
 extra_rdoc_files: []

data/test/unittests/constraint_test.rb

@@ -1,107 +0,0 @@
-require 'scrubyt'
-require 'test/unit'
-
-class ConstraintTest < Test::Unit::TestCase
-
-  def test_presence_of_attribute_constraints
-    data = Scrubyt::Extractor.define do
-      fetch File.join(File.dirname(__FILE__), "input/constraint_test.html")
-      
-      (shape 'ruby_diamond').ensure_presence_of_attribute('color' => 'red').
-                          ensure_absence_of_attribute('fill' => 'small_circles')      
-    end
-    
-    assert_equal(data.children[0].constraints[0].type,
-                 Scrubyt::Constraint::CONSTRAINT_TYPE_ENSURE_PRESENCE_OF_ATTRIBUTE)
-    assert_equal(data.children[0].constraints[1].type,
-                 Scrubyt::Constraint::CONSTRAINT_TYPE_ENSURE_ABSENCE_OF_ATTRIBUTE)
-  end
-  
-  def test_presence_of_ancestor_node_constraints
-    data = Scrubyt::Extractor.define do
-      fetch File.join(File.dirname(__FILE__), "input/constraint_test.html")
-      
-      (shape 'funky_rectangle').ensure_presence_of_ancestor_node(:contains, 'name' => 'crispy_ham').
-                     ensure_absence_of_ancestor_node(:intersects_with, 'name' => 'spaghetti_ice')      
-    end
-  
-    assert_equal(data.children[0].constraints[0].type,
-                 Scrubyt::Constraint::CONSTRAINT_TYPE_ENSURE_PRESENCE_OF_ANCESTOR_NODE)
-    assert_equal(data.children[0].constraints[1].type,
-                 Scrubyt::Constraint::CONSTRAINT_TYPE_ENSURE_ABSENCE_OF_ANCESTOR_NODE)
-  end
-  
-  def test_ancestor_node_constraints
-    data0 = Scrubyt::Extractor.define do
-      fetch File.join(File.dirname(__FILE__), "input/constraint_test.html")
-      
-      (shape 'funky_rectangle').ensure_presence_of_ancestor_node(:contains, 'name' => 'crispy_ham')                     
-    end
-    
-    data1 = Scrubyt::Extractor.define do
-      fetch File.join(File.dirname(__FILE__), "input/constraint_test.html")
-      
-      (shape 'funky_rectangle').ensure_presence_of_ancestor_node(:intersects_with, 'name' => 'spaghetti_ice')      
-    end
-    
-    data2 = Scrubyt::Extractor.define do
-      fetch File.join(File.dirname(__FILE__), "input/constraint_test.html")
-      
-      (shape 'ruby_diamond').ensure_presence_of_ancestor_node(:contains, 'name' => 'crispy_ham').
-                     ensure_absence_of_ancestor_node(:intersects_with, 'name' => 'spaghetti_ice')      
-    end
-        
-    data3 = Scrubyt::Extractor.define do
-      fetch File.join(File.dirname(__FILE__), "input/constraint_test.html")
-       
-      shape 'line'#.ensure_presence_of_ancestor_node(:contains, 'name' => 'fungus_ooze').
-                    # ensure_presence_of_ancestor_node(:intersects_with, 'object' => 'funky_lemon')
-    end
-    
-    p data3.to_xml.to_s
-    exit
-    
-    
-    data4 = Scrubyt::Extractor.define do
-      fetch File.join(File.dirname(__FILE__), "input/constraint_test.html")
-      
-      (shape 'ruby_diamond').ensure_presence_of_ancestor_node(:contains, 'name' => 'chunky_bacon').
-                     ensure_absence_of_attribute 'thickness' => '2'
-    end    
-    
-    assert_equal(data0.to_xml.to_s, "<root><shape>blue_circle</shape><shape>splatted_ellipse</shape></root>")
-    assert_equal(data1.to_xml.to_s, "<root><shape>splatted_ellipse</shape></root>")
-    assert_equal(data2.to_xml.to_s, "<root><shape>blue_circle</shape></root>")
-    assert_equal(data3.to_xml.to_s, "<root><shape>big_rectangle</shape></root>")
-    assert_equal(data4.to_xml.to_s, "<root><shape>ruby_diamond</shape></root>")
-  end
-  
-  def test_attribute_constraints
-    data0 = Scrubyt::Extractor.define do
-      fetch File.join(File.dirname(__FILE__), "input/constraint_test.html")
-      
-      (shape 'ruby_diamond').ensure_presence_of_attribute 'color' => 'red'
-    end
-
-    data1 = Scrubyt::Extractor.define do
-      fetch File.join(File.dirname(__FILE__), "input/constraint_test.html")
-      
-      (shape 'ruby_diamond').ensure_presence_of_attribute 'color' => 'red', 'size' => '10x20'
-    end
-    
-    data2 = Scrubyt::Extractor.define do
-      fetch File.join(File.dirname(__FILE__), "input/constraint_test.html")
-      (shape 'ruby_diamond').ensure_presence_of_attribute 'color' => 'red', 'size' => nil
-    end    
-
-    data3 = Scrubyt::Extractor.define do
-      fetch File.join(File.dirname(__FILE__), "input/constraint_test.html")
-      (shape 'ruby_diamond').ensure_presence_of_attribute 'thickness' => nil
-    end    
-            
-    assert_equal(data0.to_xml.to_s, "<root><shape>funky_rectangle</shape><shape>blue_circle</shape><shape>shiny_diamond</shape><shape>clunky_ellipse</shape><shape>twinky_line</shape></root>")
-    assert_equal(data1.to_xml.to_s, "<root><shape>shiny_diamond</shape><shape>clunky_ellipse</shape></root>")
-    assert_equal(data2.to_xml.to_s, "<root><shape>funky_rectangle</shape><shape>blue_circle</shape><shape>shiny_diamond</shape><shape>clunky_ellipse</shape></root>")
-    assert_equal(data3.to_xml.to_s, "<root><shape>twinky_line</shape><shape>line</shape><shape>chunky_line</shape></root>")    
-  end
-end

data/test/unittests/extractor_test.rb

@@ -1,91 +0,0 @@
-require 'scrubyt'
-require 'test/unit'
-
-class ExtractorTest < Test::Unit::TestCase
-  def test_create_one_pattern
-    pattern = Scrubyt::Extractor.define do 
-      fetch File.join(File.dirname(__FILE__), "input/test.html")
-      pattern "1"
-    end
-    assert_instance_of(Scrubyt::Pattern, pattern)
-    
-    assert_equal(pattern.name, "root")
-    assert_equal(pattern.children[0].name, 'pattern')
-    assert_equal(pattern.type, :root)
-    assert_equal(pattern.output_type, :model)
-    
-    assert_equal(pattern.generalize, false)
-    assert_equal(pattern.children[0].generalize, true)
-  end
-
-  def test_create_child_pattern
-    pattern = Scrubyt::Extractor.define do 
-      fetch File.join(File.dirname(__FILE__), "input/test.html")
-      parent { child "2" }
-    end
-      
-    assert_equal(pattern.name, "root")
-    assert_equal(pattern.type, :root)
-    assert_equal(pattern.output_type, :model)
-    
-    assert_equal(pattern.children[0].name, "parent")        
-    assert_equal(pattern.children[0].type, :tree)
-    assert_equal(pattern.children[0].output_type, :model)
-  end
-
-  def test_create_more_children
-    pattern = Scrubyt::Extractor.define do 
-      fetch File.join(File.dirname(__FILE__), "input/test.html")
-      parent do
-        child1 '1'
-        child2 '2'
-        child3 '3'
-        child4 '4'
-      end
-    end
-    
-    assert_equal(pattern.children[0].children.size, 4)
-    
-    i = 0    
-    3.times do
-      assert_equal(pattern.children[0].children[i].parent, 
-                   pattern.children[0].children[i+=1].parent) 
-      assert_equal(pattern.children[0].children[i].children, [])
-    end
-    assert_equal(pattern.children[0].children[3].parent, pattern.children[0])
-    assert_equal(pattern.children[0].children[3].parent.parent, pattern)
-  end
-
-  def test_create_hierarchy
-    tree = Scrubyt::Extractor.define do 
-      fetch File.join(File.dirname(__FILE__), "input/test.html")
-      a { b { c { d { e "1" } } } }
-    end
-    
-    assert_equal(tree.name,"root")
-    assert_equal(tree.children[0].name,"a")
-    assert_equal(tree.children[0].children[0].name,"b")
-    assert_equal(tree.children[0].children[0].children[0].name,"c")
-    assert_equal(tree.children[0].children[0].children[0].children[0].name,"d")    
-  end
-  
-
-  def test_empty_filter
-    tree = Scrubyt::Extractor.define do 
-      fetch File.join(File.dirname(__FILE__), "input/test.html")      
-      a do
-        b '1'
-        c '2'
-      end
-    end
-    
-    assert_not_nil(tree.filters[0])
-    assert_nil(tree.example)
-    assert_not_nil(tree.children[0].filters[0])
-    assert_nil(tree.children[0].example)
-    assert_not_nil(tree.children[0].children[0].filters[0])
-    assert_equal(tree.children[0].children[0].filters[0].example,'1')
-    assert_not_nil(tree.children[0].children[1].filters[0])
-    assert_equal(tree.children[0].children[1].filters[0].example,'2')
-  end  
-end

data/test/unittests/filter_test.rb

@@ -1,79 +0,0 @@
-#require File.join(File.dirname(__FILE__), '../..', 'lib', 'filter')
-require 'scrubyt'
-require 'test/unit'
-
-class FilterTest < Test::Unit::TestCase
-  def test_determine_example_type
-    #Test children example
-    assert_equal(Scrubyt::BaseFilter.determine_example_type(nil), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_CHILDREN)                 
-    #Test image example
-    assert_equal(Scrubyt::BaseFilter.determine_example_type('scrubyt.png'), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_IMAGE)
-    assert_equal(Scrubyt::BaseFilter.determine_example_type('scrubyt.gif'), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_IMAGE)
-    assert_equal(Scrubyt::BaseFilter.determine_example_type('scrubyt.jpg'), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_IMAGE)
-    assert_equal(Scrubyt::BaseFilter.determine_example_type('scrubyt.jpeg'), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_IMAGE)                 
-    assert_not_equal(Scrubyt::BaseFilter.determine_example_type('scrubyt.zip'), 
-                     Scrubyt::BaseFilter::EXAMPLE_TYPE_IMAGE)
-    assert_not_equal(Scrubyt::BaseFilter.determine_example_type('scrubyt.pif'), 
-                     Scrubyt::BaseFilter::EXAMPLE_TYPE_IMAGE)
-    #Test XPaths
-    assert_equal(Scrubyt::BaseFilter.determine_example_type('/p/img'), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_XPATH)                 
-    assert_equal(Scrubyt::BaseFilter.determine_example_type('/p/h3'), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_XPATH)                 
-    assert_equal(Scrubyt::BaseFilter.determine_example_type('/p/h3/a/h2'), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_XPATH)
-    assert_equal(Scrubyt::BaseFilter.determine_example_type('/h2'), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_XPATH)
-    assert_equal(Scrubyt::BaseFilter.determine_example_type('/h1/h3'), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_XPATH)                                 
-    assert_equal(Scrubyt::BaseFilter.determine_example_type('/p'), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_XPATH)
-    assert_equal(Scrubyt::BaseFilter.determine_example_type('//p'), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_XPATH)
-    assert_equal(Scrubyt::BaseFilter.determine_example_type('/p//img'), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_XPATH)
-    assert_equal(Scrubyt::BaseFilter.determine_example_type('//p//img'), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_XPATH)
-    assert_equal(Scrubyt::BaseFilter.determine_example_type('/p[0]/img'), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_XPATH)
-    assert_equal(Scrubyt::BaseFilter.determine_example_type('/p[0]'), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_XPATH)
-    assert_equal(Scrubyt::BaseFilter.determine_example_type('//p[1]'), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_XPATH)
-    assert_equal(Scrubyt::BaseFilter.determine_example_type('/p[1]//img[2]'), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_XPATH)
-    assert_equal(Scrubyt::BaseFilter.determine_example_type('//p[1]//img'), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_XPATH)
-    assert_equal(Scrubyt::BaseFilter.determine_example_type('/table/tr/td//span/b'), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_XPATH)
-    assert_equal(Scrubyt::BaseFilter.determine_example_type('/table[0]//tr/td[1]/span[2]/b'), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_XPATH)
-    assert_not_equal(Scrubyt::BaseFilter.determine_example_type('table[0]//tr/td[1]/span[2]/b'), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_XPATH)                 
-    assert_not_equal(Scrubyt::BaseFilter.determine_example_type('/table[a]//tr/td[1]/span[2]/b'),
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_XPATH)                 
-    assert_not_equal(Scrubyt::BaseFilter.determine_example_type('/tab2le[a]//tr/td[1]/span[2]/b'),
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_XPATH)
-    assert_not_equal(Scrubyt::BaseFilter.determine_example_type('/table[a]///tr/td[1]/span[2]/b'),
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_XPATH)
-    #Test string example
-    assert_equal(Scrubyt::BaseFilter.determine_example_type('Hello, world!'), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_STRING)
-    assert_equal(Scrubyt::BaseFilter.determine_example_type('$1022'), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_STRING)
-    assert_equal(Scrubyt::BaseFilter.determine_example_type('CANON'), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_STRING)
-    assert_equal(Scrubyt::BaseFilter.determine_example_type('This is a string'), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_STRING)
-    assert_equal(Scrubyt::BaseFilter.determine_example_type('45'), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_STRING)
-    assert_equal(Scrubyt::BaseFilter.determine_example_type('td'), 
-                 Scrubyt::BaseFilter::EXAMPLE_TYPE_STRING)
-
-  end
-end

data/test/unittests/input/constraint_test.html

@@ -1,55 +0,0 @@
-<shapes>
-
-  <!-- red shapes -->
-  <shape color='red' size='10x10' fill='none'>funky_rectangle<contains name='stuff' color='blue'/></shape>
-  <shape color='red' size='20x10' fill='small_circles'>blue_circle
-    <contains name='crispy_ham' color='blue'/>
-    <intersects_with object='banana'/>
-  </shape>
-  <shape color='red' size='10x20' fill='big_boxes'>
-    shiny_diamond
-    <intersects_with object='chunky_bacon'/>
-  </shape>
-  <shape color='red' size='10x20'>clunky_ellipse
-    <contains name='sliced_orange' color='blue'/>
-    <intersects_with object='nintendo_wii'/>
-  </shape>
-  <shape color='red' thickness='2'>twinky_line</shape>
-   
-  <!-- green shapes -->    
-  <shape color='green' size='18x5' fill='big_boxes'>boxy_rectangle
-    <contains name='banana_fluff' color='blue'/>
-    <intersects_with object='bacon_fudge'/>
-  </shape>
-  <shape color='green' size='20x10' fill='chunky_bacon'>whammed_circle
-    <contains name='big_ham' color='blue'/>
-    <intersects_with object='crispy_ham'/>
-  </shape>
-  <shape color='green' size='30x15' fill='chunky_bacon'>
-    spherical_diamond
-    <intersects_with object='crispy_orange'/>
-  </shape>
-  <shape color='green' size='20x30' fill='none'>avocado_ellipse
-    <contains name='avocado_fudge' color='blue'/>   
-  </shape>
-  <shape color='green' thickness='2'>line</shape>
-  
-  <!-- green shapes -->      
-  <shape color='blue' size='120x10' fill='small_circles'>big_rectangle
-    <contains name='fungus_ooze' color='blue'/>   
-    <intersects_with object='funky_lemon'/>
-  </shape>
-  <shape color='blue' size='50x20' fill='small_circles'>crazy_circle
-    <contains name='nut_shake' color='blue'/>
-    <intersects_with object='crazy_nut'/>
-  </shape>
-  <shape color='blue' size='30x30' fill='big_boxes'>ruby_diamond<contains name='chunky_bacon' color='blue'/></shape>
-  <shape color='blue' size='5x12' fill='chunky_bacon'>splatted_ellipse
-    <contains name='crispy_ham' color='blue'/>
-    <intersects_with name='spaghetti_ice' color='blue'/>
-  </shape>
-  <shape color='blue' thickness='2'>chunky_line
-    <contains name='chunky_bacon' color='blue'/>   
-  </shape>
-  
-</shapes>