botch 0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8ef71ba47deb27d75990a88721b652a6974b0ac4
+  data.tar.gz: 6e2db20439b70a7f34285b44ee79246cf88ccf05
+SHA512:
+  metadata.gz: f7d6b4d3cd2f94ff018fb70be5d8e71237f52bc82404167aafd34f08b4f473a830b3cda4d52aefe792d65bef5659fa3df0054b2e9d9554d62da2c77b1d39ec9c
+  data.tar.gz: 09725780996179fdd914675a15f06bcf792819f02e0a20bcc5a9403bfcde08a859e8ff8536a5f1cba006c2d49dfce23630249a2e540da0c4237fe712a29bd2fa

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/README.md

@@ -0,0 +1,45 @@
+# Botch
+
+Botch is a simple DSL for quickly creating web crawlers.
+
+Inspired by Sinatra.
+
+## Usage
+
+```ruby
+require 'lib/botch'
+require 'kconv'
+
+class SampleBotch < Botch::Base
+  set :user_agent, "SampleBotch"
+
+  filter(:all) { status == 200 }
+  rule(:all) { |response| body.toutf8 }
+end
+
+if $0 == __FILE__
+  SampleBotch.run("http://namusyaka.info/") do |response|
+    puts response
+  end
+end
+```
+
+## TODO
+
+- RSpec
+- GET/POST method
+- Documentation
+- Classic style
+
+## Contributing to Botch
+
+1. fork the project.
+2. create your feature branch. (`git checkout -b my-feature`)
+3. commit your changes. (`git commit -am 'commit message.'`)
+4. push to the branch. (`git push origin my-feature`)
+5. send pull request.
+
+## License
+
+MIT
+

data/Rakefile

@@ -0,0 +1,7 @@
+require 'rspec/core/rake_task'
+
+desc "Run all specs."
+RSpec::Core::RakeTask.new(:rspec) do |spec|
+  spec.pattern = 'spec/*_spec.rb'
+  spec.rspec_opts = %w(--format p --color)
+end

data/botch.gemspec

@@ -0,0 +1,16 @@
+require File.expand_path("../lib/botch", __FILE__)
+
+Gem::Specification.new "botch", Botch::VERSION do |s|
+  s.description      = "Botch is a DSL for quickly creating web crawlers. Inspired by Sinatra."
+  s.summary          = "A DSL for web clawler."
+  s.authors          = ["namusyaka"]
+  s.email            = "namusyaka@gmail.com"
+  s.homepage         = "https://github.com/namusyaka/botch"
+  s.files            = `git ls-files`.split("\n") - %w(.gitignore)
+  s.test_files       = s.files.select { |path| path =~ /^spec\/.*_spec\.rb/ }
+
+  s.add_dependency "faraday"
+  s.add_dependency "mechanize"
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "fakeweb", ["~> 1.3"]
+end

data/lib/botch.rb

@@ -0,0 +1,5 @@
+require File.expand_path('../botch/base', __FILE__)
+
+module Botch
+  VERSION = "0.1"
+end

data/lib/botch/base.rb

@@ -0,0 +1,204 @@
+require 'rubygems' unless defined?(Gem)
+require 'faraday'
+require 'mechanize'
+
+%w(
+  clients/abstract_client
+  clients/faraday_client
+  clients/mechanize_client
+).each{ |path| require File.expand_path("../#{path}", __FILE__)  }
+
+module Botch
+  class Route
+    attr_accessor :routes
+
+    def initialize
+      @routes = []
+      self
+    end
+
+    def add(label, options = {}, &block)
+      raise ArgumentError unless block_given?
+      if position = index(label)
+        route = @routes[position]
+        route[:block] = block
+        route[:label] = label
+      else
+        options[:block] = block
+        options[:label] = label
+        @routes << options
+      end
+    end
+
+    def del(label)
+      @routes.delete_if{ |route| route[:label] == label }
+    end
+
+    def exist?(label)
+      !!index(label)
+    end
+
+    alias :exists? :exist?
+
+    def index(label)
+      @routes.index{ |route| route[:label] === label }
+    end
+
+    def inject(url)
+      @routes.inject([]) do |result, route|
+        result << route if map_validation(url, route[:map])
+        result
+      end
+    end
+
+    private
+
+    def map_validation(url, map)
+      case map.class.to_s
+      when "Regexp" then url =~ map
+      when "String" then url.include?(map)
+      else               true
+      end
+    end
+  end
+
+  %w( Filter Rule ).each { |klass| Object.const_set(klass, Class.new(Route)) }
+
+  class Base
+    DEFAULT_INSTANCE_VARIABLES = { :header => nil, :body => nil, :status => nil }
+    attr_reader(*DEFAULT_INSTANCE_VARIABLES.keys)
+
+    def initialize
+      @header, @body = nil, nil
+    end
+
+    def client
+      self.class.client
+    end
+
+    def options
+      self.class.options
+    end
+
+    def settings
+      self.class.settings
+    end
+
+    class << self
+      @@routes = { :filter => Filter.new, :rule => Rule.new }
+
+      attr_reader :client
+
+      def instance
+        @instance ||= self.new
+      end
+
+      def helpers(*extensions, &block)
+        class_eval(&block) if block_given?
+        include(*extensions) if extensions.any?
+      end
+
+      def set(key, value = nil)
+        return __send__("#{key}=", value) if respond_to?("#{key}=")
+
+        key_symbol = key.to_sym
+        return settings[key_symbol] = value if settings.has_key?(key_symbol)
+
+        options[key_symbol] = value
+      end
+
+      def route(type, label, options = {}, &block)
+        unbound_method = generate_method("#{type} #{label}", &block).bind(instance)
+        wrapper = generate_wrapper(&unbound_method)
+
+        @@routes[type.to_sym].add(label, options, &wrapper)
+      end
+
+      def filter(label, options = {}, &block)
+        route(:filter, label, options, &block)
+      end
+
+      def rule(label, options = {}, &block)
+        route(:rule, label, options, &block)
+      end
+
+      def generate_wrapper(&method)
+        method.arity != 0 ? proc {|args| method.call(*args) } :
+                            proc {|args| method.call }
+      end
+
+      def generate_method(method_name, &block)
+        define_method(method_name, &block)
+        unbound_method = instance_method(method_name)
+        remove_method(method_name)
+        unbound_method
+      end
+
+      def reset!
+        settings = {}
+      end
+
+      def run(*urls, &block)
+        if block_given?
+          unbound_method = generate_method(:main_unbound_method, &block).bind(instance)
+          block = case unbound_method.arity
+                  when 2 then proc{|r,v| unbound_method.call(r, v) }
+                  when 1 then proc{|r,v| unbound_method.call(r) }
+                  else        proc{|r,v| unbound_method.call }
+                  end
+        end
+        set_default_options! unless self.client
+
+        urls.map do |url|
+          filters, rules = @@routes.map{ |k, v| v.inject(url) }
+          response = self.client.get(url, options)
+
+          set_instance_variables(:header => response[:header],
+                                 :body   => response[:body],
+                                 :status => response[:status])
+
+          response = response[:response]
+
+          unless filters.empty?
+            valid = filters.map{ |_filter| _filter[:block].call(response) }.all?
+            next if settings[:disabled_invalid] && !valid
+          end
+
+          response = rules.inject(nil) { |result, _rule|
+            _rule[:block].call((result || response))
+          } unless rules.empty?
+
+          response = block.call(response, valid) if block_given?
+          set_instance_variables(DEFAULT_INSTANCE_VARIABLES)
+          response
+        end
+      end
+
+      def client=(name)
+        @client = Client.const_get("#{name.to_s.capitalize}Client").new(settings) if clients.include?(name)
+      end
+
+      def settings
+        @settings ||= { :disabled_invalid => false }
+      end
+
+      def options
+        @options ||= {}
+      end
+
+      private
+
+        def set_instance_variables(pairs = {})
+          pairs.each_pair { |name, value| instance.instance_variable_set("@#{name}".to_sym, value) }
+        end
+
+        def clients
+          @_clients ||= [:faraday, :mechanize]
+        end
+
+        def set_default_options!
+          self.client = :faraday
+        end
+    end
+  end
+end

data/lib/botch/clients/abstract_client.rb

@@ -0,0 +1,15 @@
+module Botch
+  module Client
+    class AbstractClient
+      attr_reader :client
+
+      def initialize
+        @client = nil
+      end
+
+      def get(url, options = {})
+        # return a response object
+      end
+    end
+  end
+end

data/lib/botch/clients/faraday_client.rb

@@ -0,0 +1,28 @@
+module Botch
+  module Client
+    class FaradayClient < AbstractClient
+      def initialize(settings = {})
+        @client     = :faraday
+        @handler    = Faraday.new(settings) do |builder|
+          builder.use Faraday::Adapter::NetHttp
+          builder.use Faraday::Request::UrlEncoded
+        end
+      end
+
+      def get(url, options)
+        options.each_pair{ |key, value| @handler.headers[key] = value }
+        response = @handler.get(url)
+        parse_response(response)
+      end
+
+      def parse_response(response)
+        result = {}
+        result[:status]   = response.status
+        result[:header]   = response.headers
+        result[:body]     = response.body
+        result[:response] = response
+        result
+      end
+    end
+  end
+end

data/lib/botch/clients/mechanize_client.rb

@@ -0,0 +1,36 @@
+module Botch
+  module Client
+    class MechanizeResponseError
+      attr_accessor :code, :header, :body
+
+      def initialize(response_error)
+        @code     = response_error.response_code
+        @body     = ""
+        @header   = Mechanize::Headers.new
+        @response = response_error
+      end
+    end
+
+    class MechanizeClient < AbstractClient
+      def initialize(options = {})
+        @client  = :mechanize
+        @handler = Mechanize.new
+      end
+
+      def get(url, options = {})
+        @handler.user_agent = options[:user_agent] if options[:user_agent]
+        mechanize_page = @handler.get(url) rescue MechanizeResponseError.new($!)
+        parse_response(mechanize_page)
+      end
+
+      def parse_response(response)
+        result = {}
+        result[:header]   = response.header
+        result[:status]   = response.code.to_i
+        result[:body]     = response.body
+        result[:response] = response
+        result
+      end
+    end
+  end
+end

data/spec/botch_spec.rb

@@ -0,0 +1,67 @@
+$:.unshift(File.dirname(__FILE__))
+require 'spec_helper'
+
+module Botch
+  describe do
+    it "Should have a version." do
+      expect(Botch.const_defined?("VERSION")).to be_true
+    end
+  end
+
+  describe Base do
+    before(:all) do
+      @fake = Fake.new
+      class SampleBotch < Botch::Base; end
+      SampleBotch.run(@fake.url) {}
+    end
+
+    it 'Default client should be faraday.' do
+      expect(SampleBotch.client).to be_an_instance_of(Botch::Client::FaradayClient)
+    end
+
+    describe "settings and options" do
+      before(:all) do
+        class SampleBotch < Botch::Base
+          set :user_agent, "SampleBotch User-Agent"
+          set :client, :mechanize
+          set :disabled_invalid, true
+          set :original_option, "foobar"
+        end
+        @options  = SampleBotch.options
+        @settings = SampleBotch.settings
+      end
+
+      it "Original options should be stored in options." do
+        expect(@options[:original_option]).to eq("foobar")
+      end
+
+      it ":user_agent should be stored in options." do
+        expect(@options[:user_agent]).to eq("SampleBotch User-Agent")
+      end
+
+      it ":disabled_invalid should be stored in settings." do
+        expect(@settings[:disabled_invalid]).to be_true
+      end
+
+      it "Client setter should be valid." do
+        expect(SampleBotch.client).to be_an_instance_of(Botch::Client::MechanizeClient)
+      end
+    end
+
+    describe "instance variable" do
+      before(:all) do
+        class SampleBotch < Botch::Base
+          set :user_agent, "SampleBotch User-Agent"
+          set :disabled_invalid, false
+
+          filter(:all) { @test = "test" }
+          rule(:all) { @test }
+        end
+      end
+
+      it "should be able to use instance variable." do
+        expect(SampleBotch.run(@fake.url)[0]).to eq("test")
+      end
+    end
+  end
+end

data/spec/faraday_spec.rb

@@ -0,0 +1,57 @@
+$:.unshift(File.dirname(__FILE__))
+require 'spec_helper'
+
+module Botch
+  describe Client::FaradayClient do
+    before(:all) do
+      FakeWeb.clean_registry
+      @fakes = []
+      @fakes << Fake.new("/", :status => [200, "OK"], :content_type => "text/html")
+      @fakes << Fake.new("/test1", :status => [404, "Not Found"], :content_type => "text/html")
+      @fakes << Fake.new("/test2", :status => [500, "Internal Server Error"], :content_type => "text/html")
+
+      class SampleBotch < Botch::Base
+        set :user_agent, "SampleBotch User-Agent"
+        set :client, :faraday
+        set :disabled_invalid, nil
+        filter(:all){ status == 200 }
+        rule(:all){ status }
+      end
+    end
+
+    it 'client should be faraday if set :faraday to :client.' do
+      expect(SampleBotch.run(@fakes[0].url) { client }[0]).to be_an_instance_of(Botch::Client::FaradayClient)
+    end
+
+    it 'helpers should return valid data.' do
+      result = SampleBotch.run(@fakes[0].url) do
+        { :status => status, :header => header, :body => body }
+      end
+      result = result[0]
+      expect(result[:status]).to eq(200)
+      expect(result[:header]).to be_an_instance_of(Faraday::Utils::Headers)
+      expect(result[:body]).to be_an_instance_of(String)
+    end
+
+    it 'block argument of #rule should replace last expression.' do
+      result = SampleBotch.run(*@fakes.map(&:url))
+      expect(result[0]).to eq(200)
+      expect(result[1]).to eq(404)
+      expect(result[2]).to eq(500)
+    end
+
+    it 'block argument of #run should replace last expression.' do
+      result = SampleBotch.run(*@fakes.map(&:url)){ "Foo" }
+      expect(result[0]).to eq("Foo")
+      expect(result[1]).to eq("Foo")
+      expect(result[2]).to eq("Foo")
+    end
+
+    it 'the second argument should be boolean.' do
+      result = SampleBotch.run(*@fakes.map(&:url)){ |response, valid| valid }
+      expect(result[0]).to be_true
+      expect(result[1]).to be_false
+      expect(result[2]).to be_false
+    end
+  end
+end

data/spec/mechanize_spec.rb

@@ -0,0 +1,57 @@
+$:.unshift(File.dirname(__FILE__))
+require 'spec_helper'
+
+module Botch
+  describe Client::MechanizeClient do
+    before(:all) do
+      FakeWeb.clean_registry
+      @fakes = []
+      @fakes << Fake.new("/", :status => [200, "OK"], :content_type => "text/html")
+      @fakes << Fake.new("/test1", :status => [404, "Not Found"], :content_type => "text/html")
+      @fakes << Fake.new("/test2", :status => [500, "Internal Server Error"], :content_type => "text/html")
+
+      class SampleBotch < Botch::Base
+        set :user_agent, "SampleBotch User-Agent"
+        set :client, :mechanize
+        set :disabled_invalid, nil
+        filter(:all){ status == 200 }
+        rule(:all){ status }
+      end
+    end
+
+    it 'client should be faraday if set :mechanize to :client.' do
+      expect(SampleBotch.run(@fakes[0].url) { client }[0]).to be_an_instance_of(Botch::Client::MechanizeClient)
+    end
+
+    it 'helpers should return valid data.' do
+      result = SampleBotch.run(@fakes[0].url) do
+        { :status => status, :header => header, :body => body }
+      end
+      result = result[0]
+      expect(result[:status]).to eq(200)
+      expect(result[:header]).to be_an_instance_of(Mechanize::Headers)
+      expect(result[:body]).to be_an_instance_of(String)
+    end
+
+    it 'block argument of #rule should replace last expression.' do
+      result = SampleBotch.run(*@fakes.map(&:url))
+      expect(result[0]).to eq(200)
+      expect(result[1]).to eq(404)
+      expect(result[2]).to eq(500)
+    end
+
+    it 'block argument of #run should replace last expression.' do
+      result = SampleBotch.run(*@fakes.map(&:url)){ "Foo" }
+      expect(result[0]).to eq("Foo")
+      expect(result[1]).to eq("Foo")
+      expect(result[2]).to eq("Foo")
+    end
+
+    it 'the second argument should be boolean.' do
+      result = SampleBotch.run(*@fakes.map(&:url)){ |response, valid| valid }
+      expect(result[0]).to be_true
+      expect(result[1]).to be_false
+      expect(result[2]).to be_false
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,39 @@
+require 'botch'
+require 'fakeweb'
+
+FakeWeb.allow_net_connect = false
+
+module Botch
+  SPEC_DOMAIN = 'example.com'
+
+  class Fake
+    def initialize(path = "/", options = {})
+      @path         = path
+      @scheme       = options[:scheme] || "http"
+      @content_type = options[:content_type] || "text/html"
+      @status       = options[:status] || [200, "OK"]
+      add_to_fakeweb
+    end
+
+    def url
+      @scheme + "://" + SPEC_DOMAIN + @path
+    end
+
+    def body
+      @body ||= <<-HTML
+<html>
+<head>
+<title>Fake page #{@path}</title>
+</head>
+<body>
+</body>
+</html>
+      HTML
+    end
+
+    def add_to_fakeweb
+      options = { :body => @body, :content_type => @content_type, :status => @status }
+      FakeWeb.register_uri(:get, url, options)
+    end
+  end
+end

metadata

@@ -0,0 +1,115 @@
+--- !ruby/object:Gem::Specification
+name: botch
+version: !ruby/object:Gem::Version
+  version: '0.1'
+platform: ruby
+authors:
+- namusyaka
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-07-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fakeweb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+description: Botch is a DSL for quickly creating web crawlers. Inspired by Sinatra.
+email: namusyaka@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README.md
+- Rakefile
+- botch.gemspec
+- lib/botch.rb
+- lib/botch/base.rb
+- lib/botch/clients/abstract_client.rb
+- lib/botch/clients/faraday_client.rb
+- lib/botch/clients/mechanize_client.rb
+- spec/botch_spec.rb
+- spec/faraday_spec.rb
+- spec/mechanize_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/namusyaka/botch
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.2
+signing_key: 
+specification_version: 4
+summary: A DSL for web clawler.
+test_files:
+- spec/botch_spec.rb
+- spec/faraday_spec.rb
+- spec/mechanize_spec.rb
+has_rdoc: