scrapey 0.0.12 → 0.0.13

data/lib/scrapey.rb

@@ -3,7 +3,6 @@ require 'csv'
 require 'json'
 require 'yaml'
 require 'unf_ext'
-require 'coderay/tokens_proxy'
 
 require "scrapey/scrapey"
 require "scrapey/constants"
@@ -12,19 +11,18 @@ require "scrapey/database"
 require "scrapey/multi"
 require "scrapey/tee"
 
-include Scrapey
+# don't do this stuff in rails:
+unless defined? Rails
+  Scrapey::init binding
 
-# some defaults that I like
-@agent ||= Mechanize.new{|a| a.history.max_size = 10}
-@agent.user_agent = "Scrapey v#{Scrapey::VERSION} - #{Scrapey::URL}"
-@agent.verify_mode = OpenSSL::SSL::VERIFY_NONE
-# default output file
-@output = File.join BASEDIR, 'output.csv'
+  # default output file
+  @output = File.join BASEDIR, 'output.csv'
 
-# read config file
-config_file = "#{BASEDIR}/config/config.yml"
-@config = File.exists?(config_file) ? YAML::load(File.open(config_file)) : {}
+  # read config file
+  config_file = "#{BASEDIR}/config/config.yml"
+  @config = File.exists?(config_file) ? YAML::load(File.open(config_file)) : {}
 
-init_db if @config['database']
+  init_db if @config['database']
 
-$stderr = Scrapey::Tee.new(STDERR, File.open("#{BASEDIR}/errors.log", "w"))
+  $stderr = Scrapey::Tee.new(STDERR, File.open("#{BASEDIR}/errors.log", "w"))
+end

data/lib/scrapey/constants.rb

@@ -1,5 +1,5 @@
 module Scrapey
-  VERSION = "0.0.12"
+  VERSION = "0.0.13"
   BASEDIR = File.expand_path(File.dirname($0)).gsub(/\/src$/,'')
   URL = "https://github.com/monkeysuffrage/scrapey"
 end

data/lib/scrapey/scrapey.rb

@@ -1,5 +1,15 @@
 module Scrapey
 
+  def self.init b
+    eval "include Scrapey", b
+
+    # some defaults that I like
+    eval "@agent ||= Mechanize.new{|a| a.history.max_size = 10}", b
+    eval "@agent.user_agent = 'Scrapey v#{Scrapey::VERSION} - #{Scrapey::URL}'", b
+    eval "@agent.verify_mode = OpenSSL::SSL::VERIFY_NONE", b
+  end
+
+
   def get_or_post method, url, options={}, *args
     agent = ['goto', 'visit'].include?(method) ? @browser : @agent
     _retries = options.delete :retries

data/template/Rakefile

@@ -13,13 +13,14 @@ end
 
 desc "Copy installer to dropbox folder"
 task 'dropbox' do
-  file = 'setup.exe'
-  raise 'no dropbox folder!' unless ENV['DROPBOX']
-  folder = [ENV['DROPBOX'], name].join('/').squeeze('/')
-  FileUtils.mkdir(folder) unless File.exists?(folder)
-  FileUtils.cp "Output/#{file}", folder
-  url = [ENV['DROPBOX_public_url'], name, file].join('/').squeeze('/')
-  puts "uploaded to #{url}"
+  Dir.glob('Output/*').map{|x| x[/[^\/]*$/]}.each do |file|
+    raise 'no dropbox folder!' unless ENV['DROPBOX']
+    folder = [ENV['DROPBOX'], name].join('/').squeeze('/')
+    FileUtils.mkdir(folder) unless File.exists?(folder)
+    FileUtils.cp "Output/#{file}", folder
+    url = [ENV['DROPBOX_public_url'], name, file].join('/').squeeze('/')
+    puts "uploaded to #{url}"
+  end
 end
 
 task 'git' do

data/template/src/downloader.rb

@@ -0,0 +1,28 @@
+require 'scrapey'
+require 'pry'
+
+use_cache
+
+# File.open("#{BASEDIR}/config/urls.txt", 'w'){|f| f<< (0..100).map{|i| "http://www.example.com/id=#{i}"} * "\n"}
+@queue = File.read("#{BASEDIR}/config/urls.txt").split("\n").shuffle
+
+def download agent
+  while url = @queue.shift
+    if is_cached? url
+      puts 'skipping'
+      next
+    end
+    page = agent.get url
+    save_cache url, page.body
+    puts url
+  end
+end
+
+threads = []
+5.times do
+  threads << Thread.new { download Mechanize.new{|a| a.history.max_size, a.verify_mode = 10, OpenSSL::SSL::VERIFY_NONE}}
+end
+
+threads.each { |t| t.join }
+
+binding.pry

data/template/src/template.rb

@@ -14,17 +14,19 @@ def scrape div
   save item  
   exit if defined? Ocra
 rescue StandardError => e
+  puts e.message, e.backtrace
   binding.pry
 end
 
-#use_cache :redis
 
 fields 'name', 'address', 'zip'
 
 @url = "http://www.example.com/"
 
-page = get @url
-scrape page.at('div')
+with_cache do
+  page = get @url
+  binding.pry
+end
 
 #@csv.close
 #%x{call #{@output}}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scrapey
 version: !ruby/object:Gem::Version
-  version: 0.0.12
+  version: 0.0.13
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-04 00:00:00.000000000 Z
+date: 2013-03-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -96,6 +96,7 @@ files:
 - template/icon.ico
 - template/output.csv
 - template/Rakefile
+- template/src/downloader.rb
 - template/src/schema.rb
 - template/src/template.rb
 - template/template.iss