scrapey 0.0.13 → 0.0.16

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    Njk0YTkyMDY0MGYxMzM1ZDMwOWM5Yzg4YmQ4YTA2NGM3N2Q1ZGUxMw==
+  data.tar.gz: !binary |-
+    YjE2NjNlYjQwMTIwMWE3YmRjNWQzNDVlZWI0MjhjNWI1NWMzNTg1OQ==
+SHA512:
+  metadata.gz: !binary |-
+    NTcyZWFmYjdhZmMyMjVjNzc1ZTk5ZDRkMTM2YjYxMTE5NjZmMThmNTc2MGVl
+    MTI4Yjc0ODkwZWQ3NGNkNWM4NmI5ZGFlMmJiYWNiMjFhMDEyMGIzMjhkZmYz
+    ZDNlN2Y3NGFjYzdjNjlhMWE4Y2FiMzcxZmQyNDBlZTM4MTA1Yzg=
+  data.tar.gz: !binary |-
+    NmIwOGRjZTczMTE1YjE2MDQzNzc5MmJmYTQ4MzQzMGNlNGQ1Y2YxMzA1MDZk
+    NzkzMTdhOTI3MzAxODNiYmZhNzlhNjkzNGUwODAwZTVmYmRjZjY1ZjdkOGJm
+    ZTFmOWQ5OTFiZmQwY2U3NTBjYzA2ZjBkNjcwZmMzNjcxY2E2ODY=

data/lib/scrapey.rb

@@ -11,6 +11,16 @@ require "scrapey/database"
 require "scrapey/multi"
 require "scrapey/tee"
 
+require 'addressable/uri'
+ 
+class URI::Parser
+  def split url
+    a = Addressable::URI::parse url
+    [a.scheme, a.userinfo, a.host, a.port, nil, a.path, nil, a.query, a.fragment]
+  end
+end
+
+
 # don't do this stuff in rails:
 unless defined? Rails
   Scrapey::init binding
@@ -24,5 +34,26 @@ unless defined? Rails
 
   init_db if @config['database']
 
-  $stderr = Scrapey::Tee.new(STDERR, File.open("#{BASEDIR}/errors.log", "w"))
-end
+  #$stderr = Scrapey::Tee.new(STDERR, File.open("#{BASEDIR}/errors.log", "w"))
+end
+
+if defined?(Ocra)
+  puts "doing ocra stuff..."
+  Mechanize.new.cookies
+  HTTP::Cookie::Scanner.new ''
+  if @config['database'] || @config['databases']
+    puts "doing ocra db stuff..."
+    ActiveRecord::Relation::PredicateBuilder.new rescue nil
+    [
+    'active_record',
+    'active_record/schema',
+    'active_record/connection_adapters/abstract/schema_definitions',
+    @config['database'] ? @config['database']['adapter'] : 'mysql',
+    'tzinfo',
+    'active_support/all',
+    'active_support/multibyte/chars'
+    ].each{|lib| require lib}
+  end
+end
+
+Dir.chdir BASEDIR

data/lib/scrapey/cache/disk.rb

@@ -15,7 +15,7 @@ module Scrapey
     return nil unless File::exists?(filename)
     debug "Loading #{filename} from cache"
     begin
-      Nokogiri::HTML Marshal.load(File.open(filename, "rb"){|f| f.read})
+      Mechanize::Page.new URI.parse(url), [], Marshal.load(File.open(filename, "rb"){|f| f.read}), nil, @agent
     rescue Exception => e
       puts e.message
     end

data/lib/scrapey/cache/redis.rb

@@ -9,7 +9,7 @@ module Scrapey
   def load_cache url
     debug "Loading #{url} from cache"
     return nil unless str = @redis.get(url)
-    Nokogiri::HTML Marshal.load(str) rescue nil
+    Mechanize::Page.new(URI.parse(url), [], Marshal.load(str), nil, @agent) rescue nil
   end
 
   def save_cache url, body, options = {}

data/lib/scrapey/constants.rb

@@ -1,5 +1,5 @@
 module Scrapey
-  VERSION = "0.0.13"
+  VERSION = "0.0.16"
   BASEDIR = File.expand_path(File.dirname($0)).gsub(/\/src$/,'')
   URL = "https://github.com/monkeysuffrage/scrapey"
 end

data/lib/scrapey/core.rb

@@ -0,0 +1,34 @@
+require 'addressable/uri'
+
+class URI::Parser
+  def split url
+    a = Addressable::URI::parse url
+    [a.scheme, a.userinfo, a.host, a.port, nil, a.path, nil, a.query, a.fragment]
+  end
+end
+
+class Hash
+  def shuffle
+    Hash[self.to_a.shuffle]
+  end
+end
+
+class Nokogiri::XML::NodeSet
+  def shuffle
+    self.to_a.shuffle
+  end
+end
+
+class Enumerator
+  def shuffle
+    self.to_a.shuffle
+  end
+end
+
+class CSV::Table
+  def shuffle
+    arr = self.to_a
+    k = arr.shift
+    arr.map{|v| Hash[k.zip v]}.shuffle
+  end
+end

data/lib/scrapey/scrapey.rb

@@ -85,4 +85,10 @@ module Scrapey
   def ts
     Time.now.to_i.to_s
   end
+
+  def enqueue url
+    @url_list ||= File.open("#{BASEDIR}/config/urls.txt", 'w')
+    @url_list << url
+    @url_list << "\n"
+  end
 end

data/template/.gitignore

@@ -0,0 +1 @@
+config

data/template/Rakefile

@@ -18,7 +18,7 @@ task 'dropbox' do
     folder = [ENV['DROPBOX'], name].join('/').squeeze('/')
     FileUtils.mkdir(folder) unless File.exists?(folder)
     FileUtils.cp "Output/#{file}", folder
-    url = [ENV['DROPBOX_public_url'], name, file].join('/').squeeze('/')
+    url = [ENV['DROPBOX_public_url'], name, file].join('/')
     puts "uploaded to #{url}"
   end
 end

data/template/config/config.yml

@@ -12,6 +12,9 @@ password: 12345
 #  host: localhost
 #  encoding: 'utf8'
 
+#category: xxx
+#dataset_name: yyy
+
 # example proxies section
 #proxies:
 #- www.host1.com:80

data/template/src/emails.rb

@@ -0,0 +1,48 @@
+require 'scrapey'
+require 'pry'
+
+=begin
+@config = {
+  'category' => 'businesses',
+  'dataset_name' => 'brazilian_companies',
+  'database' => {
+    'adapter' => 'mysql',
+    'database' => 'stefan',
+    'username' => 'root',
+    'password' => '12345',
+    'host' => 'localhost',
+    'encoding' => 'utf8'
+  }
+}
+=end
+
+def post url, body
+  page = @agent.post url, body
+  JSON.parse(page.body).each{|k, v|}
+  raise 'x' unless page.body
+  page
+rescue Exception => e
+  print '!'
+  sleep 10
+  return post url, body
+end
+
+@agent.open_timeout = @agent.read_timeout = 10000
+
+tables = ActiveRecord::Base.connection.tables
+
+tables.each do |table|
+  puts table
+  tables table.camelize
+  klass = table.camelize.constantize
+  return unless klass.column_names.include?('website')
+
+  klass.where("website is not null and email is null").find_in_batches(:batch_size => 10) do |group|
+    page = post('http://www.pay4data.com/lookup/email_for_url', {urls: group.map(&:website).compact}.to_json)
+    JSON.parse(page.body).each do |k, v|
+      group.find{|r| r['website'] == k}.update_attributes(:email => v)
+      puts k
+    end
+  end
+end
+

data/template/src/export.rb

@@ -0,0 +1,133 @@
+require 'aws-sdk'
+require 'scrapey'
+require 'pry'
+
+=begin
+@config = {
+  'category' => 'businesses',
+  'dataset_name' => 'brazilian_companies',
+  'database' => {
+    'adapter' => 'mysql',
+    'database' => 'stefan',
+    'username' => 'root',
+    'password' => '12345',
+    'host' => 'localhost',
+    'encoding' => 'utf8'
+  }
+}
+
+
+
+  CSV.open("#{BASEDIR}/#{table}.csv", 'w') do |csv|
+    csv << fields
+    klass.where(:found => true).find_each do |row|
+      csv << fields.map{|f| row[f]}
+    end
+  end
+
+=end
+
+
+
+def new_csv filename
+  File.open(filename, 'w') do |file|
+    file << 0xEF.chr + 0xBB.chr + 0xBF.chr
+  end
+  CSV.open(filename, 'a') do |csv|
+    yield csv
+  end
+end
+
+unless @config['dataset_name'] && @config['category']
+  puts 'Please fill out dataset_name and category in config.yml to continue'
+  exit
+end
+init_db
+@tables = ActiveRecord::Base.connection.tables
+
+all_fields = []
+
+@tables.each do |table|
+  puts table
+  tables table.camelize
+  klass = table.camelize.constantize
+
+  all_fields << klass.column_names
+  fields = klass.column_names - ['id', 'updated_at', 'created_at', 'updated_on', 'created_on']
+
+  new_csv("#{BASEDIR}/#{table}.csv") do |csv|
+    csv << fields
+    klass.all.find_each do |row|
+      csv << fields.map{|f| row[f]}
+    end
+  end
+
+  new_csv("#{BASEDIR}/#{table}_sample.csv") do |csv|
+    csv << fields
+    klass.order(:id).order('rand()').limit(50).each do |row|
+      csv << fields.map{|f| row[f]}
+    end
+  end
+
+end
+
+if @tables.length == 0
+  table = @tables.first
+  `gzip -f #{BASEDIR}/#{table}_sample.csv`
+  `gzip -f #{BASEDIR}/#{table}.csv`
+  csv_name = "#{table}.csv.gz"
+  sample_name = "#{table}_sample.csv.gz"
+
+  csv_name = "#{@config['dataset_name']}.csv.gz"
+  `mv #{BASEDIR}/#{table}.csv.gz #{csv_name}`
+  sample_name = "#{@config['dataset_name']}_sample.csv.gz"
+  `mv #{BASEDIR}/#{table}_sample.csv.gz #{sample_name}`
+
+else
+  csv_name = "#{@config['dataset_name']}.csv.tar.gz"
+  sample_name = "#{@config['dataset_name']}.sample.tar.gz"
+  sample_sql = "#{@config['dataset_name']}_sample.sql"
+
+  cmd = "tar -czf #{csv_name} " + @tables.map{|x| x + '.csv'}.join(' ')
+  `#{cmd}`
+  File.open(sample_sql, 'w') do |f|
+    f << `"C:\\Program Files\\MySQL\\MySQL Server 5.6\\bin\\mysqldump.exe" -uroot -p12345 --where="true limit 100" #{@config['database']['database']}`
+  end
+  cmd = "tar -czf #{sample_name} #{sample_sql} " + @tables.map{|x| x + '_sample.csv'}.join(' ')
+  `#{cmd}`
+end
+
+# --where="true limit 100"
+File.open("#{@config['dataset_name']}.sql", 'w') do |f|  
+  f << `"C:\\Program Files\\MySQL\\MySQL Server 5.6\\bin\\mysqldump.exe" -uroot -p12345 #{@config['database']['database']}`
+end
+`gzip -f #{@config['dataset_name']}.sql`
+sql_name = "#{@config['dataset_name']}.sql.gz"
+
+s3 = AWS::S3.new :access_key_id => ENV['AMAZON_ACCESS_KEY_ID'], :secret_access_key => ENV['AMAZON_SECRET_ACCESS_KEY']
+bucket = s3.buckets['pay4data']
+
+sample_object = bucket.objects["#{@config['category']}/#{sample_name}"].write :file => sample_name, :content_type => 'application/gzip', :acl => :public_read
+csv_object    = bucket.objects["#{@config['category']}/#{csv_name}"].write :file => csv_name, :content_type => 'application/gzip'
+sql_object    = bucket.objects["#{@config['category']}/#{sql_name}"].write :file => sql_name, :content_type => 'application/gzip'
+
+sql = <<EOF
+insert into datasets(sample_url, csv_url, sql_url, last_crawled, fields) values(
+'#{sample_object.public_url.to_s}',
+'#{csv_object.public_url.to_s}',
+'#{sql_object.public_url.to_s}',
+now(),
+'#{fields.map{|t| (t - ['id', 'updated_at', 'created_at', 'updated_on', 'created_on']).join ', '}.join ", "}'
+);
+
+update datasets set category_id=5, name='', description='', price='', button_html='' where id=
+
+
+mysqldump pay4data datasets categories | mysql2 pay4data
+
+
+EOF
+
+puts sql
+
+

data/template/src/get_proxies.rb

@@ -0,0 +1,14 @@
+require 'scrapey'
+require 'pry'
+require 'syck'
+require "#{BASEDIR}/src/proxy.rb"
+
+
+YAML::ENGINE.yamler='syck'
+
+#proxies = Proxy::get_proxies :proxy_list
+proxies = Proxy::get_proxies :all
+
+@config['proxies'] = proxies.uniq
+File.open("#{BASEDIR}/config/config.yml", 'w') { |f| YAML.dump(@config, f) }
+

data/template/src/proxy.rb

@@ -0,0 +1,278 @@
+require "base64"
+
+class Proxy
+  attr_reader :current
+  BOOM = 'boom'
+
+  def initialize agent = nil, options = {}
+    @user_agents = [
+      'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.76 Safari/537.36',
+      'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.66 Safari/537.36',
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/536.30.1 (KHTML, like Gecko) Version/6.0.5 Safari/536.30.1',
+      'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0',
+      'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.69 Safari/537.36',
+      'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0',
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.76 Safari/537.36',
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.69 Safari/537.36',
+      'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.76 Safari/537.36',
+      'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)'
+    ]
+    @agent = agent
+    @min = options[:min] || 5
+    @sleep = options[:sleep] || 60 * 60 # 1 hour
+    @verbose = options[:verbose] || false
+    @timeout = options[:timeout] || 30
+    @round_time = options[:round_time] || 5 * 60 # 5 minutes
+    @agent.open_timeout = @agent.read_timeout = @timeout
+    proxies = options[:proxies] || []
+    set_proxies proxies
+  end
+
+  def set_proxies proxies
+    @proxies = proxies.select{|x| x[/:/]}.uniq{|x| x[/.*:/]}
+    self.shuffle
+  end
+
+  def debug str
+    puts str if @verbose
+  end
+
+  def shuffle
+    @proxies = [BOOM] + (@proxies - [BOOM]).shuffle
+    start_round
+    self.rotate
+  end
+
+  def to_yaml
+    @proxies.to_yaml
+  end
+
+  def start_round
+    now = Time.now.to_i
+    if @round_start
+      sleep_time = @round_time - (now - @round_start)
+      if sleep_time > 0
+        puts "sleeping for #{sleep_time}"
+        sleep sleep_time
+      end
+    end
+    @round_start = Time.now.to_i
+  end
+
+  def get_more_proxies
+    puts 'getting more proxies'
+    proxies = Proxy::get_proxies
+    set_proxies proxies
+  end
+
+  def rotate
+    debug "rotating"
+    @proxies.rotate!
+    @user_agents.rotate!
+    if @proxies.length < @min
+      get_more_proxies
+    end
+    @current = @proxies.first
+    if @current == BOOM
+      start_round
+      rotate
+      return
+    end
+
+    host, port = @current.split ':'
+    debug "setting proxy to #{host}:#{port}"
+    @agent.set_proxy host, port.to_i
+    debug "setting user_agent to #{@user_agents.first}"
+    @agent.user_agent = @user_agents.first
+  end
+
+  def remove
+    debug "--- removing #{@current}"
+    @proxies.shift
+    rotate
+    debug @proxies.join(', ')
+    debug @current
+  end
+
+  def pause
+    time = @sleep / @proxies.length
+    debug "sleeping for #{time}"
+    sleep time
+  end
+
+  def length
+    @proxies.length
+  end
+
+
+
+  def self.get_idcloak
+    proxies = []
+    ['http://www.idcloak.com/proxylist/free-proxy-servers-list.html'].each do |url|
+      page = @agent.get url
+
+      page.search('#sort td[7]').each do |td|
+        port = td.text.strip
+        host = td.at('+ td').text.strip
+        proxies << "#{host}:#{port}"
+      end
+
+    end
+    proxies
+  end
+
+  def self.get_proxynova
+    proxies = []
+    ['http://www.proxynova.com/proxy-server-list/'].each do |url|
+      page = @agent.get url
+
+      page.search('.row_proxy_ip').each do |span|
+        str = span.text[/long2ip\((.*?)\)/, 1]
+        next if str[/a-z/i]
+        i = eval str
+        host = Proxy::long2ip(i)
+        port = span.parent.at('+ td').text.strip
+        proxies << "#{host}:#{port}"
+      end
+    end
+    proxies
+  end
+
+  def self.get_proxy_list
+    proxies = []
+    ['http://proxy-list.org/en/index.php',
+    'http://proxy-list.org/en/index.php?sp=20',
+    'http://proxy-list.org/en/index.php?sp=40',
+    'http://proxy-list.org/en/index.php?sp=60',
+    'http://proxy-list.org/en/index.php?sp=80',
+    'http://proxy-list.org/en/index.php?sp=100',
+    'http://proxy-list.org/en/index.php?sp=120'].each do |url|
+      page = @agent.get url
+      proxies += page.body.scan(/(?:\d+\.){3}\d+:\d+/)
+    end
+    proxies
+  end
+
+  def self.get_hidemyass
+    proxies = []
+    ['http://hidemyass.com/proxy-list/search-227752',
+    'http://hidemyass.com/proxy-list/search-227752/2',
+    'http://hidemyass.com/proxy-list/search-227752/3',
+    'http://hidemyass.com/proxy-list/search-227752/4',
+    'http://hidemyass.com/proxy-list/search-227752/5',
+    'http://hidemyass.com/proxy-list/search-227752/6'].each do |url|
+      page = @agent.get url
+      page.search('*[style*="display:none"]').remove
+      page.search(page.body.scan(/(\..*?)\{display:none\}/).flatten.join(', ')).remove
+      page.search('style').remove
+      proxies += page.search('td[2]').map{|x| x.text.strip}.zip(page.search('td[3]').map{|x| x.text.strip}).map{|h,p| "#{h}:#{p}"}[1..-1]
+    end
+    proxies
+  end
+
+  def self.get_cool_proxy
+    proxies = []
+    page = @agent.get 'http://www.cool-proxy.net/proxies/http_proxy_list/sort:score/direction:desc'
+    page.search('tr')[1..-2].each do |tr|
+      next unless tr.at('td[2]')
+      host = Base64.decode64 tr.at('td[1]').text[/"(.*?)"/, 1]
+      port = tr.at('td[2]').text
+      proxies << [host, port].join(':')
+    end
+
+    while a = page.at('a[rel=next]')
+      url = URI.join('http://www.freeproxylists.net/', a[:href]).to_s
+      begin
+        page = @agent.get url
+      rescue
+        return proxies
+      end
+      page.search('tr')[1..-2].each do |tr|
+        next unless tr.at('td[2]')
+        host = Base64.decode64 tr.at('td[1]').text[/"(.*?)"/, 1]
+        port = tr.at('td[2]').text
+        proxies << [host, port].join(':')
+      end
+    end
+
+    proxies
+  end
+
+
+  def self.get_freeproxylists
+    proxies = []
+
+    @agent.follow_meta_refresh = true
+    page = @agent.get 'http://www.freeproxylists.net/'
+
+    page.body.scan(/IPDecode\("([^"]+)"\)<\/script><\/td><td align="center">(\d+)/).each do |row|
+      proxies << [URI.decode(row[0]), row[1]].join(':')
+    end
+
+    while a = page.at('a[text()^=Next]')
+      url = URI.join('http://www.freeproxylists.net/', a[:href]).to_s
+      puts url
+      page = @agent.get url
+      page.body.scan(/IPDecode\("([^"]+)"\)<\/script><\/td><td align="center">(\d+)/).each do |row|
+        proxies << [URI.decode(row[0]), row[1]].join(':')
+      end
+    end
+
+    proxies
+  end
+
+def self.long2ip(long)
+  ip = []
+  4.times do |i|
+    ip.push(long.to_i & 255)
+    long = long.to_i >> 8
+  end
+  ip.join(".")
+end
+
+  def self.get_proxies provider = :all
+
+    @agent ||= Mechanize.new{|a| a.history.max_size = 10}
+    @agent.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    @agent.user_agent = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.56 Safari/536.5'
+
+    case provider
+      when :proxy_list then return get_proxy_list
+      when :hidemyass then return get_hidemyass
+      when :freeproxylists then return get_freeproxylists
+      when :cool_proxy then return get_cool_proxy
+      when :proxynova then return get_proxynova
+      when :idcloak then return get_idcloak
+      when :all
+        proxies = []
+        [:proxy_list, :hidemyass, :freeproxylists, :cool_proxy, :proxynova, :idcloak].each do |key|
+          puts key
+          begin
+            part = get_proxies(key)
+          rescue Exception => e
+            part = []
+            puts e.message
+          end
+          puts part.length
+          proxies += part
+        end
+        proxies
+    end
+  end
+end
+
+if ARGV.include?('-p')
+  puts "refreshing proxies, please wait..."
+  require "#{BASEDIR}/src/get_proxies.rb"
+  puts "#{@config['proxies'].length} proxies found."
+  puts "Hit [enter] to exit"
+  $stdin.gets
+  exit
+end
+
+# for testing
+if __FILE__ == $0
+  require 'mechanize'
+  @agent = Mechanize.new
+  proxy = Proxy.new @agent, :verbose => true, :min => 5
+end

data/template/src/schema.rb

@@ -1,7 +1,7 @@
 =begin
 # put table schemas here. this will be included if the table is not found.
 ActiveRecord::Schema.define do
-  create_table "items" do |t|
+  create_table "items", options: 'ENGINE=InnoDB DEFAULT CHARSET=utf8' do |t|
     t.string   "string_field"
     t.text     "text_field"
     t.integer  "number_field"

data/template/src/template.rb

@@ -5,6 +5,12 @@ require 'pry'
 # @agent.user_agent = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.56 Safari/536.5'
 # @output = Time.now.strftime("#{BASEDIR}/Output/output_%Y_%m_%d_%H_%M_%S.csv")
 
+EMAIL_REGEX = /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,4}\b/i
+
+def clean str
+  str.gsub(/[[:space:]]+/, ' ').strip
+end
+
 def scrape div
   a = div.at('a')
   url = URI.join(@url, a[:href]).to_s

metadata

@@ -1,20 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: scrapey
 version: !ruby/object:Gem::Version
-  version: 0.0.13
-  prerelease: 
+  version: 0.0.16
 platform: ruby
 authors:
 - P Guardiario
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-26 00:00:00.000000000 Z
+date: 2014-04-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -22,7 +20,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -30,7 +27,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: httpclient
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -38,7 +34,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -46,7 +41,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -54,7 +48,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -83,6 +76,7 @@ files:
 - lib/scrapey/cache/redis.rb
 - lib/scrapey/cache.rb
 - lib/scrapey/constants.rb
+- lib/scrapey/core.rb
 - lib/scrapey/database.rb
 - lib/scrapey/multi.rb
 - lib/scrapey/scrapey.rb
@@ -91,37 +85,42 @@ files:
 - lib/scrapey/tor.rb
 - lib/scrapey.rb
 - scrapey.gemspec
+- template/.gitignore
 - template/config/config.yml
 - template/Gemfile
 - template/icon.ico
 - template/output.csv
 - template/Rakefile
 - template/src/downloader.rb
+- template/src/emails.rb
+- template/src/export.rb
+- template/src/get_proxies.rb
+- template/src/proxy.rb
 - template/src/schema.rb
 - template/src/template.rb
 - template/template.iss
 homepage: ''
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 2.1.5
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: A simple scraping framework
 test_files: []
+has_rdoc: