pba_crawler 0.0.4

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in crawler.gemspec
+gemspec

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks

data/app/controllers/proxies_controller.rb

@@ -0,0 +1,39 @@
+class ProxiesController < ApplicationController
+
+  def index
+    @proxies = Proxy.all
+  end
+
+  def new
+    @proxy = Proxy.new
+  end
+
+  def edit
+    @proxy = Proxy.find(params[:id])
+  end
+
+  def create
+    @proxy = Proxy.new(params[:proxy])
+    if @proxy.save
+      redirect_to(proxies_url, :notice => 'Proxy was successfully created.')
+    else
+      render :action => "new"
+    end
+  end
+
+  def update
+    @proxy = Proxy.find(params[:id])
+    if @proxy.update_attributes(params[:proxy])
+      redirect_to(proxies_url, :notice => 'Proxy was successfully updated.')
+    else
+      render :action => "edit"
+    end
+  end
+
+  def destroy
+    @proxy = Proxy.find(params[:id])
+    @proxy.destroy
+
+    redirect_to(proxies_url)
+  end
+end

data/app/models/proxy.rb

@@ -0,0 +1,3 @@
+class Proxy < ActiveRecord::Base
+  
+end

data/app/views/proxies/_form.html.erb

@@ -0,0 +1,8 @@
+<% semantic_form_for @proxy do |form| %>
+  <%= form.input :address %>
+  <%= form.input :port %>
+  <%= form.input :position %>
+  <% form.buttons do %>
+    <%= form.commit_button %>
+  <% end %>
+<% end %>

data/app/views/proxies/edit.html.erb

@@ -0,0 +1,6 @@
+<h1>Editing Proxy</h1>
+
+<%= render 'form' %>
+
+<%= link_to 'Show', @proxy %> |
+<%= link_to 'Back', proxies_path %>

data/app/views/proxies/index.html.erb

@@ -0,0 +1,24 @@
+<h1>Listing proxies</h1>
+
+<table>
+  <tr>
+    <th>Address</th>
+    <th>IP</th>
+    <th></th>
+    <th></th>
+    <th></th>
+  </tr>
+
+<% @proxies.each do |proxy| %>
+  <tr>
+    <td><%= proxy.address %></td>
+    <td><%= proxy.port %></td>
+    <td><%= link_to 'Show', proxy %></td>
+    <td><%= link_to 'Edit', edit_proxy_path(proxy) %></td>
+    <td><%= link_to 'Destroy', proxy, :confirm => 'Are you sure?', :method => :delete %></td>
+  </tr>
+<% end %>
+</table>
+<br />
+
+<%= link_to 'New Proxy', new_proxy_path %>

data/app/views/proxies/new.html.erb

@@ -0,0 +1,5 @@
+<h1>New Proxy</h1>
+
+<%= render 'form' %>
+
+<%= link_to 'Back', proxies_path %>

data/config/routes.rb

@@ -0,0 +1,3 @@
+ActionController::Routing::Routes.draw do
+  resources :proxies
+end

data/db/migrate/20110321122537_create_proxy.rb

@@ -0,0 +1,14 @@
+class CreateProxy < ActiveRecord::Migration
+  def self.up
+    create_table :proxies do |t|
+      t.string :address
+      t.integer :port
+      t.datetime :last_request_at
+      t.integer :position
+      t.timestamps
+    end
+  end
+  def self.down
+    drop_table :proxies
+  end
+end

data/lib/generators/crawler/install/install_generator.rb

@@ -0,0 +1,20 @@
+# encoding: utf-8
+require "crawler/file_utilz.rb"
+
+module Crawler
+  # Copies database migration files
+  #
+  # @example
+  #   $ rails generate crawler:install
+  #
+  # @todo Revisit in Rails 3.1 where public assets are treated differently
+  class InstallGenerator < Rails::Generators::Base
+    desc "Copies database migration files"
+    
+    def copy_files
+      source = File.join(File.dirname(__FILE__),'..','..', '..', '..', 'db')
+      destination = File.join(Rails.root, 'db')
+      Crawler::FileUtilz.mirror_files(source, destination)
+    end
+  end
+end

data/lib/pba_crawler.rb

@@ -0,0 +1,10 @@
+require "pba_crawler_engine"
+
+
+require 'nokogiri'
+require 'mechanize'
+require 'net/http'
+require 'open-uri'
+
+
+require 'pba_crawler/generic'

data/lib/pba_crawler/file_utilz.rb

@@ -0,0 +1,73 @@
+require 'fileutils'
+
+module PbaCrawler
+  class FileUtilz
+    # A general purpose method to mirror a directory (+source+) into a destination
+    # directory, including all files and subdirectories. Files will not be mirrored
+    # if they are identical already (checked via FileUtils#identical?).
+    #
+    # Copyright (c) 2008 James Adam (The MIT License)
+
+    # added: do mirroring with backup saving turned on
+    def self.mirror_with_backup(source, destination)
+      self.mirror_files(source, destination, true)
+    end
+
+    def self.mirror_files(source, destination, create_backups = false)
+      # TODO: use Rake::FileList#pathmap?
+      if File.directory?(source)
+        source_files  = Dir[source + "/**/*"]
+        source_dirs   = source_files.select { |d| File.directory?(d) }
+        source_files -= source_dirs
+      elsif File.exist? source
+        source_dirs   = []
+        source_files  = [source]
+        source        = File.dirname source
+      else
+        puts "Could not mirror #{source} - entity does not exist"
+        return
+      end
+
+      unless source_files.empty?
+        base_target_dir = File.join(destination)
+        base_target_dir = File.dirname base_target_dir unless File.directory? base_target_dir
+        FileUtils.mkdir_p(base_target_dir)
+      end
+
+      base_target_dir ||= File.join(destination)
+
+      source_dirs.each do |dir|
+        # strip down these paths so we have simple, relative paths we can
+        # add to the destination
+        target_dir = File.join(base_target_dir, dir.gsub(source, ''))
+        begin
+          FileUtils.mkdir_p(target_dir)
+        rescue Exception => e
+          raise "Could not create directory #{target_dir}: \n" + e
+        end
+      end
+
+      source_files.each do |file|
+        begin
+          target = File.join(base_target_dir, file.gsub(source, ''))
+          unless File.exist?(target) && self.same_contents(file, target)
+            # WAS FileUtils.identical?(file, target), but we want to test contents too
+            if create_backups && File.exist?(target)
+              File.rename(target, target + '~')
+            end
+            FileUtils.cp(file, target)
+          end
+        rescue Exception => e
+          raise "Could not copy #{file} to #{target}: \n" + e
+        end
+      end
+    end
+
+    # for windows users...
+    def self.same_contents(source,destination)
+      File.read(source) == File.read(destination)
+    end
+
+  end
+end
+

data/lib/pba_crawler/generic.rb

@@ -0,0 +1,78 @@
+module PbaCrawler
+  class Generic
+    
+    #list of proxies address
+    @@proxies = ""
+    @@cursor = 0
+    @@proxy = nil
+
+    def initialize
+      #load proxy list
+      @@proxies = Proxy.order(:position)
+    end
+
+    def crawl_url(url)
+      agent = Mechanize.new { |a|
+        a.html_parser = Nokogiri::HTML
+        a.open_timeout = 20
+        a.read_timeout = 20
+      }
+      @@proxy = @@proxies.fetch(@@cursor)
+      agent.set_proxy(@@proxy.address,@@proxy.port)
+      agent.user_agent = 'Mozilla/5.0 (Windows; U; Windows NT 6.1; nl; rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13'
+
+      get_url(agent,url)
+    end
+
+    def get_url(agent,url)
+      begin
+        response = agent.get(url)
+        if response
+          @@proxy.update_attributes(:last_request_at => Time.now)
+        else
+          rescue_get_url(agent,url)
+        end
+      rescue
+        rescue_get_url(agent,url)
+      end
+      response
+    end
+
+    def rescue_get_url(agent,url)
+      if @@cursor + 1 < @@proxies.size
+        p "#{agent.proxy_addr} #{agent.proxy_port}"
+        @@cursor = @@cursor + 1
+        @@proxy = @@proxies.fetch(@@cursor)
+        agent.set_proxy(@@proxy.address,@@proxy.port)
+        response = get_url(agent,url)
+       end
+    end
+
+    def load_sample
+      list = [
+        ["101.0.6.87","31280"],
+        ["109.120.130.2","3128"],
+        ["62.215.5.69","8080"],
+        ["62.215.5.66","8080"],
+        ["212.33.199.19","80"],
+        ["196.1.70.201","80"],
+        ["194.87.32.233","80"],
+        ["94.125.27.20","8080"],
+        ["95.66.35.1","80"],
+        ["124.164.247.43","3128"],
+        ["186.225.98.17","3128"],
+        ["122.155.13.135","3128"],
+        ["148.81.130.19","80"],
+        ["202.129.58.68","80"],
+        ["97.100.143.20","27977"],
+        ["24.12.13.154","27977"],
+        ["200.129.180.61","80"]
+      ]
+      Proxy.delete_all
+      list.each do |add|
+        Proxy.create(:address => add[0], :port => add[1].to_i, :last_request_at => Time.now)
+      end
+    end
+
+  end
+end

data/lib/pba_crawler/version.rb

@@ -0,0 +1,3 @@
+module PbaCrawler
+  VERSION = "0.0.4"
+end

data/lib/pba_crawler_engine.rb

@@ -0,0 +1,5 @@
+module PbaCrawlerEngine
+ class Engine < Rails::Engine
+    #engine_name :crawler_engine
+  end
+end

data/pba_crawler.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "pba_crawler/version"
+
+Gem::Specification.new do |s|
+  s.name        = "pba_crawler"
+  s.version     = PbaCrawler::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Pierre BASILE"]
+  s.email       = ["pierre.basile@gmail.com"]
+  s.homepage    = ""
+  s.summary     = %q{Crawl websites}
+  s.description = %q{Crawl websites}
+
+  s.rubyforge_project = "pba_crawler"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency('formtastic',  '1.2.3')  
+  s.add_dependency('nokogiri',  '1.4.4')
+  s.add_dependency('mechanize',  '1.0.0')
+end

data/readme.textile

@@ -0,0 +1,16 @@
+h1. Crawler Gem
+
+h2. Installation
+
+gem 'crawler-pba'
+
+
+<pre>
+rails g crawler:install
+rake db:migrate
+</pre>
+
+h2. Use
+
+c = Crawler::Generic.new
+c.crawl_url("http://www.google.fr")

metadata

@@ -0,0 +1,107 @@
+--- !ruby/object:Gem::Specification 
+name: pba_crawler
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.0.4
+platform: ruby
+authors: 
+- Pierre BASILE
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-03-22 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: formtastic
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        version: 1.2.3
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: nokogiri
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        version: 1.4.4
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: mechanize
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        version: 1.0.0
+  type: :runtime
+  version_requirements: *id003
+description: Crawl websites
+email: 
+- pierre.basile@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- Rakefile
+- app/controllers/proxies_controller.rb
+- app/models/proxy.rb
+- app/views/proxies/_form.html.erb
+- app/views/proxies/edit.html.erb
+- app/views/proxies/index.html.erb
+- app/views/proxies/new.html.erb
+- config/routes.rb
+- db/migrate/20110321122537_create_proxy.rb
+- lib/generators/crawler/install/install_generator.rb
+- lib/pba_crawler.rb
+- lib/pba_crawler/file_utilz.rb
+- lib/pba_crawler/generic.rb
+- lib/pba_crawler/version.rb
+- lib/pba_crawler_engine.rb
+- pba_crawler.gemspec
+- readme.textile
+has_rdoc: true
+homepage: ""
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: pba_crawler
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: Crawl websites
+test_files: []
+