scrapey 0.0.1 → 0.0.2

data/bin/scrapey

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 'scrapey/template'
+
+Scrapey::Template::generate *ARGV

data/lib/scrapey.rb

@@ -1,79 +1,12 @@
-require "scrapey/version"
-require "scrapey/src/cache"
-require "scrapey/src/database"
-
-module Scrapey
-  BASEDIR = File.expand_path(File.dirname($0)).gsub(/\/src$/,'')
-
-  def get_or_post method, url, options={}, *args
-    agent = method == 'goto' ? @browser : @agent
-    _retries = options.delete :retries
-    _sleep = options.delete :sleep
-    begin
-      new_args = method, url
-      unless options.empty? && args.empty? 
-        new_args << options
-        args.each{|arg| new_args << arg}
-      end
-      
-      doc = load_cache(url) if @use_cache
-  		return doc if doc
-
-      page = agent.send *new_args
-      save_cache(url, page.body) if @use_cache
-
-      #exit if Object.const_defined? :Ocra
-      page
-    rescue Exception => e
-      case
-        when defined? on_error
-          on_error e
-          get_or_post method, url, options, *args
-        when _retries && _retries > 0
-          puts "Error. Retries remaining: #{options[:retries]}"
-          sleep _sleep if _sleep
-          get_or_post method, url, options.merge({:retries => _retries - 1, :sleep => _sleep}), *args
-        else raise e
-      end
-    end
-  end
-
-  def get *args; get_or_post 'get', *args; end
-  def post *args; get_or_post 'post', *args; end
-  def head *args; get_or_post 'head', *args; end
-  def goto *args; get_or_post 'goto', *args; end
+require 'mechanize'
+require 'csv'
+require 'json'
+require 'yaml'
 
-  def set_proxy *args
-    @agent.set_proxy *args
-  end
-
-  def fields *args
-    @fields = args
-  end
-
-  def save item
-    unless @csv && !@csv.closed?
-      @csv = CSV.open @output, 'w'
-      @csv << @fields if @fields
-    end
-    case
-      when item.is_a?(Array) then @csv << item
-      when item.is_a?(Hash) then @csv << @fields.map{|f| item[f]}
-      else raise "unsupported type: #{item.class}"
-    end
-  end
-
-	def visited? url
-		@visited ||= []
-		return true if @visited.include? url
-		@visited << url
-		false
-	end
-
-  def ts
-    Time.now.to_i.to_s
-  end
-end
+require "scrapey/version"
+require "scrapey/scrapey"
+require "scrapey/cache"
+require "scrapey/database"
 
 include Scrapey
 
@@ -81,7 +14,7 @@ include Scrapey
 @agent ||= Mechanize.new{|a| a.history.max_size = 10}
 @agent.user_agent = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.56 Safari/536.5'
 
-# defaulkt output file
+# default output file
 @output = Time.now.strftime("#{BASEDIR}/Output/output_%Y_%m_%d_%H_%M_%S.csv")
 
 # read config file
@@ -89,7 +22,7 @@ config_file = "#{BASEDIR}/config/config.yml"
 @config = File.exists?(config_file) ? YAML::load(File.open(config_file)) : {}
 
 if @config['database']
-  ['active_record', @config['database']['adapter'], 'tzinfo', 'active_support/multibyte'].each{|lib| require lib}
+  ['active_record', @config['database']['adapter'], 'tzinfo', 'active_support/all'].each{|lib| require lib}
 	ActiveRecord::Base.establish_connection(@config['database']) 
 end
 

data/lib/scrapey/cache.rb

@@ -0,0 +1,22 @@
+module Scrapey
+  def use_cache
+    @use_cache = true
+    @config['cache_dir'] ||= "#{BASEDIR}/cache"
+    FileUtils.mkdir_p @config['cache_dir']
+  end
+
+  def cache_filename url
+    @config['cache_dir'] + "/" + Digest::MD5.hexdigest(url) + ".cache"
+  end
+
+  def load_cache url
+    filename = cache_filename url
+    return nil unless File::exists?(filename)
+    puts "Loading #{filename} from cache"
+    Nokogiri::HTML File.read(filename)
+  end
+
+  def save_cache url,doc
+    File.open(cache_filename(url), 'wb') {|f| f.write(doc) }
+  end
+end

data/lib/scrapey/database.rb

@@ -0,0 +1,14 @@
+module Scrapey
+  def tables *args
+    args.each do |arg|
+      #eval("class #{arg} < ActiveRecord::Base; end")
+      Object.const_set(arg, Class.new(ActiveRecord::Base) {})
+    end
+  end
+
+  def truncate *args
+    args.each do |arg|
+      ActiveRecord::Base.connection.execute("TRUNCATE TABLE #{Object.const_get(arg).table_name}")
+    end
+  end
+end

data/lib/scrapey/scrapey.rb

@@ -0,0 +1,72 @@
+module Scrapey
+  BASEDIR = File.expand_path(File.dirname($0)).gsub(/\/src$/,'')
+
+  def get_or_post method, url, options={}, *args
+    agent = method == 'goto' ? @browser : @agent
+    _retries = options.delete :retries
+    _sleep = options.delete :sleep
+    begin
+      new_args = method, url
+      unless options.empty? && args.empty? 
+        new_args << options
+        args.each{|arg| new_args << arg}
+      end
+      
+      doc = load_cache(url) if @use_cache
+  		return doc if doc
+
+      page = agent.send *new_args
+      save_cache(url, page.body) if @use_cache
+
+      #exit if Object.const_defined? :Ocra
+      page
+    rescue Exception => e
+      case
+        when defined? on_error
+          on_error e
+          get_or_post method, url, options, *args
+        when _retries && _retries > 0
+          puts "Error. Retries remaining: #{options[:retries]}"
+          sleep _sleep if _sleep
+          get_or_post method, url, options.merge({:retries => _retries - 1, :sleep => _sleep}), *args
+        else raise e
+      end
+    end
+  end
+
+  def get *args; get_or_post 'get', *args; end
+  def post *args; get_or_post 'post', *args; end
+  def head *args; get_or_post 'head', *args; end
+  def goto *args; get_or_post 'goto', *args; end
+
+  def set_proxy *args
+    @agent.set_proxy *args
+  end
+
+  def fields *args
+    @fields = args
+  end
+
+  def save item
+    unless @csv && !@csv.closed?
+      @csv = CSV.open @output, 'w'
+      @csv << @fields if @fields
+    end
+    case
+      when item.is_a?(Array) then @csv << item
+      when item.is_a?(Hash) then @csv << @fields.map{|f| item[f]}
+      else raise "unsupported type: #{item.class}"
+    end
+  end
+
+	def visited? url
+		@visited ||= []
+		return true if @visited.include? url
+		@visited << url
+		false
+	end
+
+  def ts
+    Time.now.to_i.to_s
+  end
+end

data/lib/scrapey/template.rb

@@ -0,0 +1,16 @@
+require 'fileutils'
+
+module Scrapey
+  module Template
+    def self.generate name, *args
+      puts "creating new scrapey project: #{name}..."
+      template = File.expand_path('../../../template', __FILE__)
+      FileUtils.cp_r template, name
+      Dir.chdir name
+      Dir.glob(['*/*.*', '*.*']).grep(/template/).each do |fn|
+        FileUtils.mv fn, fn.gsub('template', name)
+      end
+
+    end
+  end
+end

data/lib/scrapey/version.rb

@@ -1,3 +1,3 @@
 module Scrapey
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/scrapey.gemspec

@@ -1,9 +1,6 @@
 # -*- encoding: utf-8 -*-
 require File.expand_path('../lib/scrapey/version', __FILE__)
 
-gem 'mechanize'
-gem 'json'
-
 Gem::Specification.new do |gem|
   gem.authors       = ["P Guardiario"]
   gem.email         = ["pguardiario@gmail.com"]
@@ -11,11 +8,14 @@ Gem::Specification.new do |gem|
   gem.summary       = %q{A simple scraping framework}
   gem.homepage      = ""
 
-  gem.files         = `git ls-files`.split($\)
+#  gem.files         = `git ls-files`.split($\)
+  gem.files         = `find * -type f | grep -v pkg`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "scrapey"
   gem.require_paths = ["lib"]
   gem.version       = Scrapey::VERSION
+  gem.add_dependency(%q<mechanize>, ["~> 2.5.0"])
+  gem.add_dependency(%q<json>, ["~> 1.7.0"])
 end
 

data/template/config/config.yml

@@ -0,0 +1,23 @@
+# you can put any configurable options in here
+# login information for example
+email: foo@gmail.com
+password: 12345
+
+=begin
+# example database section
+database:
+  adapter: mysql
+  database: template
+  username: root
+  password: 12345
+  host: localhost
+  encoding: 'utf8'
+=end
+
+=begin
+# example proxies section
+proxies:
+- www.host1.com:80
+- localhost:8080
+- 127.0.0.1:8888
+=end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scrapey
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,23 +9,52 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-22 00:00:00.000000000 Z
-dependencies: []
+date: 2012-07-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: &16919208 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *16919208
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: &16115544 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.7.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *16115544
 description: A simple scraping framework
 email:
 - pguardiario@gmail.com
-executables: []
+executables:
+- scrapey
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
 - Gemfile
 - LICENSE
 - README.md
 - Rakefile
-- lib/scrapey.rb
+- bin/scrapey
+- lib/scrapey/cache.rb
+- lib/scrapey/database.rb
+- lib/scrapey/scrapey.rb
+- lib/scrapey/template.rb
 - lib/scrapey/version.rb
+- lib/scrapey.rb
 - scrapey.gemspec
+- template/config/config.yml
+- template/src/template.rb
 homepage: ''
 licenses: []
 post_install_message: 

data/.gitignore

@@ -1,17 +0,0 @@
-*.gem
-*.rbc
-.bundle
-.config
-.yardoc
-Gemfile.lock
-InstalledFiles
-_yardoc
-coverage
-doc/
-lib/bundler/man
-pkg
-rdoc
-spec/reports
-test/tmp
-test/version_tmp
-tmp