scrubyt 0.3.0 → 0.3.4

metadata

@@ -3,9 +3,9 @@ rubygems_version: 0.9.0
 specification_version: 1
 name: scrubyt
 version: !ruby/object:Gem::Version 
-  version: 0.3.0
-date: 2007-05-28 00:00:00 +02:00
-summary: A powerful Web-scraping framework
+  version: 0.3.4
+date: 2007-09-26 00:00:00 +02:00
+summary: A powerful Web-scraping framework built on Mechanize and Hpricot
 require_paths: 
 - lib
 email: peter@rubyrailways.com
@@ -15,7 +15,7 @@ description: scRUBYt! is an easy to learn and use, yet powerful and effective we
 autorequire: 
 default_executable: 
 bindir: bin
-has_rdoc: "true"
+has_rdoc: false
 required_ruby_version: !ruby/object:Gem::Version::Requirement 
   requirements: 
   - - ">"
@@ -61,9 +61,10 @@ files:
 - lib/scrubyt/core/scraping/filters/html_subtree_filter.rb
 - lib/scrubyt/core/scraping/filters/detail_page_filter.rb
 - lib/scrubyt/core/scraping/filters/download_filter.rb
-- lib/scrubyt/core/shared/u_r_i_builder.rb
+- lib/scrubyt/core/scraping/filters/text_filter.rb
+- lib/scrubyt/core/scraping/filters/constant_filter.rb
+- lib/scrubyt/core/scraping/filters/script_filter.rb
 - lib/scrubyt/core/shared/extractor.rb
-- lib/scrubyt/core/shared/evaluation_context.rb
 test_files: []
 
 rdoc_options: []
@@ -95,3 +96,48 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: 0.6.3
     version: 
+- !ruby/object:Gem::Dependency 
+  name: ParseTreeReloaded
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: RubyInlineAcceleration
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: RubyInline
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        version: 3.6.3
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: ParseTree
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        version: 1.7.1
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: ruby2ruby
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        version: 1.1.6
+    version: 

data/lib/scrubyt/core/shared/evaluation_context.rb

@@ -1,57 +0,0 @@
-module Scrubyt
-  ##
-  #=<tt>Holding the evaluation context of the extraction process</tt>
-  #
-  #Every kind of data that is shared among patterns during the extraction process
-  #is held in this class, so it can be looked up anytime.
-  #
-  #This class provides also some high-level basic functionality in navigation, like
-  #crawling to new pages, attaching doucment to the root pattern once arrived at the
-  #desired page etc.
-  #
-  #It can be viewed as a glue between Extractor and NavigationActions as well - these
-  #two classes need to communicate frequently as well as share different information
-  #and this is accomplished through EvaluationContext.
-  class EvaluationContext
-    attr_accessor :root_pattern, :document_index, :extractor, :uri_builder, :evaluating_extractor_definition
-
-    def initialize
-      @root_pattern = nil
-      @next_page = nil
-      @document_index = 0
-      @extractor = nil
-      @evaluating_extractor_definition = false
-    end
-
-    ##
-    #Crawl to a new page. This function should not be called from the outside - it is automatically called
-    #if the next_page pattern is defined
-    def crawl_to_new_page(uri_builder)
-      #puts "Crawling to new page!"
-      #puts "example #{uri_builder.next_page_example}"
-      temp_document = uri_builder.next_page_example ?
-                        generate_next_page_link(uri_builder) :
-                        uri_builder.generate_next_uri
-      return false if temp_document == nil
-      FetchAction.restore_host_name
-      @extractor.fetch(temp_document)
-      return true
-    end
-
-    def generate_next_page_link(uri_builder)
-      return nil unless uri_builder.next_page_pattern.filters[0].generate_XPath_for_example(true)
-      xpath = uri_builder.next_page_pattern.filters[0].xpath
-      node = (@extractor.get_hpricot_doc/xpath).map.last
-      node = XPathUtils.find_nearest_node_with_attribute(node, 'href')
-      return nil if node == nil || node.attributes['href'] == nil
-      node.attributes['href'].gsub('&amp;') {'&'}
-    end
-
-    def setup_uri_builder(pattern,args)
-      if args[0] =~ /^http.+/
-        args.insert(0, @extractor.get_current_doc_url) if args[1] !~ /^http.+/
-      end
-      @uri_builder = URIBuilder.new(pattern,args)
-    end
-  end #end of class EvaluationContext
-end #end of module Scrubyt

data/lib/scrubyt/core/shared/u_r_i_builder.rb

@@ -1,67 +0,0 @@
-module Scrubyt
-  ##
-  #=<tt>Build URIs from different parameters</tt>
-  #
-  #When crawling to further pages which are machine-generated
-  #(most typically "next" pages) we need to detect the pattern
-  #and generate the next URI based on the edetected rule. This
-  #class provides methods to build URIs based on different criteria.
-  #
-  #The other possibility is to use constant objects ('Next' links,
-  #or image links (like right arrow) pointing to the next page).
-  #URIBUilder supports both possibilities.
-  class URIBuilder
-    attr_reader :next_page_example, :next_page_pattern, :limit, :next_param, :next_increment, :increment, :current_uri
-    
-    def initialize(pattern,args)
-      if args[0] =~ /^http.+/
-        #Figure out how are the URLs generated based on the next URL
-        get_next_param(string_diff(args[0], args[1]))
-        @increment = 0
-        @current_uri = args[1]
-        @limit = args[2][:limit] if args.size > 2
-      else
-        #Otherwise, do this in the 'classic' way (by clicking on the "next" link)
-        @next_page_pattern = pattern
-        @next_page_example = args[0]
-        @limit = args[1][:limit] if args.size > 1
-      end
-    end
-    
-    #Used when generating the next URI (as opposed to 'clicking' the next link)
-    def generate_next_uri      
-      @increment += @next_increment
-      return @current_uri if @increment == @next_increment
-      @next_increment = 1 if @next_increment == 2
-      if @current_uri !~ /#{@next_param}/
-        @current_uri += (@next_param + '=' + @next_increment.to_s)
-      else
-        @current_uri = @current_uri.sub(/#{@next_param}=#{@increment-@next_increment}/) do
-          "#{@next_param}=#{@increment}"
-        end
-      end
-    end 
-    
-private
-    def get_next_param(pair)
-      param_and_value = pair.split('=')
-      @next_param = param_and_value[0]
-      @next_increment = param_and_value[1].to_i
-    end
-    
-    def find_difference_index(s1,s2)
-      cmp = s2.scan(/./).zip(s1.scan(/./))    
-      i = 0
-      loop do
-        return i if cmp[i][0] != cmp[i][1]
-        i+=1
-      end
-    end
-
-    def string_diff(s1,s2)
-      s2[find_difference_index(s1, s2)..s2.size-find_difference_index(s1.reverse, s2.reverse)-1]
-    end #end of method string_diff
-  end #end of class URIBuilder
-end #end of module Scrubyt 
-
-