scrapey 0.0.2 → 0.0.3

data/README.md

@@ -16,14 +16,22 @@ Or install it yourself as:
 
     $ gem install scrapey
 
-## Usage
+## Examples
 
-TODO: Write usage instructions here
+### Concurrent downloads
 
-## Contributing
+```ruby
+require 'scrapey'
+require 'scrapey/multi'
 
-1. Fork it
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Added some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create new Pull Request
+fields 'url', 'title'
+
+def scrape url, response
+  doc = Nokogiri::HTML response
+  @items << {'url' => url, 'title' => doc.at('title').text}
+end
+
+@items = []
+multi_get ['http://www.yahoo.com/', 'http://www.google.com.', 'http://www.bing.com/'], 3, :scrape
+@items.each{|item| save item}
+```

data/examples/multi.rb

@@ -0,0 +1,13 @@
+require 'scrapey'
+require 'scrapey/multi'
+
+fields 'url', 'title'
+
+def scrape url, response
+  doc = Nokogiri::HTML response
+  @items << {'url' => url, 'title' => doc.at('title').text}
+end
+
+@items = []
+multi_get ['http://www.yahoo.com/', 'http://www.google.com.', 'http://www.bing.com/'], 3, :scrape
+@items.each{|item| save item}

data/lib/scrapey/constants.rb

@@ -0,0 +1,4 @@
+module Scrapey
+  BASEDIR = File.expand_path(File.dirname($0)).gsub(/\/src$/,'')
+  ENV['SSL_FILE'] = "#{Gem.dir}/gems/scrapey-#{Scrapey::VERSION}/ssl/cacert.pem"
+end

data/lib/scrapey/database.rb

@@ -1,7 +1,6 @@
 module Scrapey
   def tables *args
     args.each do |arg|
-      #eval("class #{arg} < ActiveRecord::Base; end")
       Object.const_set(arg, Class.new(ActiveRecord::Base) {})
     end
   end

data/lib/scrapey/multi.rb

@@ -0,0 +1,25 @@
+require 'em-http-request'
+
+module Scrapey
+  def multi_get all_urls, num_threads = 20, callback = :save_cache
+    all_urls.each_slice(num_threads) do |urls|
+      next unless urls.size > 0
+      EventMachine.run do
+        multi = EventMachine::MultiRequest.new
+        urls.each_with_index do |url, i|
+          multi.add i, EventMachine::HttpRequest.new(url).get(:redirects => 10)
+        end
+        multi.callback do
+          (0...multi.requests.length).each do |i|				
+            if multi.responses[:callback][i]
+              send callback, urls[i], multi.responses[:callback][i].response
+            else
+              puts "problem downloading #{urls[i]}!"
+            end
+          end
+          EventMachine.stop
+        end
+      end
+    end
+  end
+end

data/lib/scrapey/scrapey.rb

@@ -1,5 +1,4 @@
 module Scrapey
-  BASEDIR = File.expand_path(File.dirname($0)).gsub(/\/src$/,'')
 
   def get_or_post method, url, options={}, *args
     agent = method == 'goto' ? @browser : @agent
@@ -13,7 +12,7 @@ module Scrapey
       end
       
       doc = load_cache(url) if @use_cache
-  		return doc if doc
+      return doc if doc
 
       page = agent.send *new_args
       save_cache(url, page.body) if @use_cache
@@ -23,8 +22,7 @@ module Scrapey
     rescue Exception => e
       case
         when defined? on_error
-          on_error e
-          get_or_post method, url, options, *args
+          return on_error e, method, url, options, *args
         when _retries && _retries > 0
           puts "Error. Retries remaining: #{options[:retries]}"
           sleep _sleep if _sleep
@@ -59,12 +57,12 @@ module Scrapey
     end
   end
 
-	def visited? url
-		@visited ||= []
-		return true if @visited.include? url
-		@visited << url
-		false
-	end
+  def visited? url
+    @visited ||= []
+    return true if @visited.include? url
+    @visited << url
+    false
+  end
 
   def ts
     Time.now.to_i.to_s

data/lib/scrapey/version.rb

@@ -1,3 +1,3 @@
 module Scrapey
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/lib/scrapey.rb

@@ -3,8 +3,9 @@ require 'csv'
 require 'json'
 require 'yaml'
 
-require "scrapey/version"
 require "scrapey/scrapey"
+require "scrapey/version"
+require "scrapey/constants"
 require "scrapey/cache"
 require "scrapey/database"
 
@@ -15,7 +16,7 @@ include Scrapey
 @agent.user_agent = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.56 Safari/536.5'
 
 # default output file
-@output = Time.now.strftime("#{BASEDIR}/Output/output_%Y_%m_%d_%H_%M_%S.csv")
+@output = 'output.csv'
 
 # read config file
 config_file = "#{BASEDIR}/config/config.yml"

data/template/config/config.yml

@@ -3,21 +3,17 @@
 email: foo@gmail.com
 password: 12345
 
-=begin
 # example database section
-database:
-  adapter: mysql
-  database: template
-  username: root
-  password: 12345
-  host: localhost
-  encoding: 'utf8'
-=end
+#database:
+#  adapter: mysql
+#  database: template
+#  username: root
+#  password: 12345
+#  host: localhost
+#  encoding: 'utf8'
 
-=begin
 # example proxies section
-proxies:
-- www.host1.com:80
-- localhost:8080
-- 127.0.0.1:8888
-=end
+#proxies:
+#- www.host1.com:80
+#- localhost:8080
+#- 127.0.0.1:8888

data/template/src/template.rb

@@ -0,0 +1,4 @@
+require 'scrapey'
+
+# customizations...
+# @output = Time.now.strftime("#{BASEDIR}/Output/output_%Y_%m_%d_%H_%M_%S.csv")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scrapey
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-24 00:00:00.000000000 Z
+date: 2012-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
-  requirement: &16919208 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,15 @@ dependencies:
         version: 2.5.0
   type: :runtime
   prerelease: false
-  version_requirements: *16919208
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.5.0
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &16115544 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,7 +37,12 @@ dependencies:
         version: 1.7.0
   type: :runtime
   prerelease: false
-  version_requirements: *16115544
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.7.0
 description: A simple scraping framework
 email:
 - pguardiario@gmail.com
@@ -46,8 +56,11 @@ files:
 - README.md
 - Rakefile
 - bin/scrapey
+- examples/multi.rb
 - lib/scrapey/cache.rb
+- lib/scrapey/constants.rb
 - lib/scrapey/database.rb
+- lib/scrapey/multi.rb
 - lib/scrapey/scrapey.rb
 - lib/scrapey/template.rb
 - lib/scrapey/version.rb
@@ -75,7 +88,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.11
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: A simple scraping framework