web_crawler 0.5.0 → 0.5.2

data/Gemfile

@@ -4,11 +4,15 @@ gem 'thor', '>=0.14.6'
 gem 'mime-types', '>=1.16'
 gem 'parallel', '>=0.5.5'
 gem 'activesupport'
+gem "rake", '0.8.7'
+gem 'i18n'
+gem 'hpricot'
 
 # Specify your gem's dependencies in web_crawler.gemspec
 gemspec
 
 group :development, :test do
+  gem "rake", '0.8.7'
   gem "rspec", ">=2.6"
   gem "autotest"
   gem "autotest-growl"

data/README

@@ -1,22 +1,32 @@
 Web crawler help you with parse and collect data from the web
 
-#TODO
+==How it works.
 
-Base web crawler class for API present
-Its showld work like this:
+class StackoverflowCrawler < WebCrawler::Base
 
-class MyCrawler < WebCrawler::Base
+    target "http://stackoverflow.com/questions/tagged/:tag", :tag=> %w{ruby ruby-on-rails ruby-on-rails-3}
+    logger "path/to/log/file" # or Logger.new(...)
 
-    target "www.example.com"
-    target "www.example.com/page2"
-    target %[www.example.com/contacts www.example.com/about]
-    target "www.example.com/category_:category/page:page/", :categories => [1,2,3,4], :page => 1..100
+    cache_to '/tmp/cache/stackoverflow'
 
-    target { call_advanced_logic_for_url_generating }
+    context "#questions .question-summary", :jobs do
 
-    logger "path/to/log/file" # or Logger.new(...)
+      #TODO: defaults :format => lambda{ |v| v.to_i }
 
+      map '.vote-count-post strong', :to => :vote_count, :format => lambda{ |v| v.to_i }
+      map '.views', :to => :view_count, :format => lambda{ |v| v.match(/\d+/)[0].to_i }
+      map '.status strong', :to => :answer_count, :format => lambda{ |v| v.to_i }
+      map '.summary h3 a', :to => :title, :format => lambda{ |v| v.to_i }
+      map '.summary .excerpt', :to => :excerpt, :format => lambda{ |v| v.to_i }
+      map '.user-action-time .relativetime', :to => :posted_at, :on => [:attr, :title]
+      map '.tags .post-tag', :to => :tags, :format => lambda{ |v| v.to_i }
 
-    
+    end
+end
 
-end
+
+#TODO
+ 1. Add documentation
+ 2. ...
+ 3. PROFIT!!!1
+ (:

data/bin/wcrawler

@@ -1,5 +1,4 @@
 #!/usr/bin/env ruby
-
 $:.unshift File.expand_path("../../lib", __FILE__)
 
 # Check if an older version of bundler is installed

data/lib/web_crawler/application.rb

@@ -1,6 +1,9 @@
 module WebCrawler
   class Application < CLI
 
+    map '-V' => 'version'
+    map '-v' => 'version'
+    
     desc "test", "Test task"
 
     def test
@@ -72,6 +75,11 @@ module WebCrawler
       end
     end
 
+    desc '-v or -V or version', 'Show gem version'
+    def version
+      WebCrawler::VERSION::STRING
+    end
+
     protected
     def allow_format(*allow)
       allow.flatten.select { |f| f == @options[:format] }.first

data/lib/web_crawler/version.rb

@@ -2,7 +2,7 @@ module WebCrawler
   module VERSION
     MAJOR  = 0
     MINOR  = 5
-    TINY   = 0
+    TINY   = 2
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/lib/web_crawler/view.rb

@@ -17,6 +17,8 @@ module WebCrawler::View
   class Base
     attr_reader :input
 
+    delegate :logger, :to => WebCrawler.logger
+    
     class << self
       attr_accessor :default_options
 
@@ -52,11 +54,22 @@ module WebCrawler::View
       @present_output = if override && override.respond_to?(:puts)
                           override
                         elsif @options['output'].is_a?(String)
-                          File.open(@options['output'], 'w+')
+                          output_to_file(@options['output'])
                         elsif @options['output'].respond_to? :puts
                           @options['output']
                         end
     end
+
+    def output_to_file(filename)
+      path = Pathname.new(filename)
+
+      unless path.dirname.exist?
+        info("#{path.dirname} not exist, try to create...")
+        path.dirname.mkpath
+      end
+      
+      path.open('w+')
+    end
   end
 
 end

data/lib/web_crawler.rb

@@ -7,6 +7,7 @@ require "ext/hash"
 require "ext/array"
 require "ext/http_response"
 require "active_support/core_ext"
+require "web_crawler/version"
 
 module WebCrawler
   autoload :Request, 'web_crawler/request'

data/web_crawler.gemspec

@@ -8,7 +8,7 @@ Gem::Specification.new do |s|
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Anton Sozontov"]
   s.email       = ["a.sozontov@gmail.com"]
-  s.homepage    = ""
+  s.homepage    = "https://github.com/webgago/web_crawler"
   s.summary     = %q{Web crawler help you with parse and collect data from the web}
   s.description = %q{Web crawler help you with parse and collect data from the web}
 
@@ -26,7 +26,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'thor', '>=0.14.6'
   s.add_dependency 'mime-types', '>=1.16'
   s.add_dependency 'parallel', '>=0.5.5'
-  s.add_dependency 'activesupport'
+  s.add_dependency 'activesupport', '>=3.0'
 
   s.add_development_dependency(%q<rspec>, [">=2.6"])
   s.add_development_dependency(%q<fakeweb>)

metadata

@@ -1,12 +1,8 @@
 --- !ruby/object:Gem::Specification 
 name: web_crawler
 version: !ruby/object:Gem::Version 
-  prerelease: false
-  segments: 
-  - 0
-  - 5
-  - 0
-  version: 0.5.0
+  prerelease: 
+  version: 0.5.2
 platform: ruby
 authors: 
 - Anton Sozontov
@@ -14,8 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-06-14 00:00:00 +04:00
-default_executable: 
+date: 2011-06-24 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: thor
@@ -25,10 +20,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        - 14
-        - 6
         version: 0.14.6
   type: :runtime
   version_requirements: *id001
@@ -40,9 +31,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        segments: 
-        - 1
-        - 16
         version: "1.16"
   type: :runtime
   version_requirements: *id002
@@ -54,10 +42,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        - 5
-        - 5
         version: 0.5.5
   type: :runtime
   version_requirements: *id003
@@ -69,9 +53,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        version: "0"
+        version: "3.0"
   type: :runtime
   version_requirements: *id004
 - !ruby/object:Gem::Dependency 
@@ -82,9 +64,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        segments: 
-        - 2
-        - 6
         version: "2.6"
   type: :development
   version_requirements: *id005
@@ -96,8 +75,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        segments: 
-        - 0
         version: "0"
   type: :development
   version_requirements: *id006
@@ -117,6 +94,7 @@ files:
 - README
 - Rakefile
 - bin/wcrawler
+- gem_graph.png
 - lib/ext/array.rb
 - lib/ext/hash.rb
 - lib/ext/http_response.rb
@@ -167,8 +145,7 @@ files:
 - spec/web_crawler/view_spec.rb
 - spec/web_crawler/web_crawler_api_base_class_spec.rb
 - web_crawler.gemspec
-has_rdoc: false
-homepage: ""
+homepage: https://github.com/webgago/web_crawler
 licenses: []
 
 post_install_message: 
@@ -181,21 +158,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      segments: 
-      - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      segments: 
-      - 0
       version: "0"
 requirements: []
 
 rubyforge_project: web_crawler
-rubygems_version: 1.3.7
+rubygems_version: 1.8.5
 signing_key: 
 specification_version: 3
 summary: Web crawler help you with parse and collect data from the web