rumba-crawler 0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d37db998d18c3cfbe4e350026c474f37a05e2469
+  data.tar.gz: 1df7bf2ad9d67e46440bd5a0fa57e6b73893a52d
+SHA512:
+  metadata.gz: 92455f66a6efcfc923b4d0d9f760f405418f9673e65dad7908e00ea0b103c5b2f0f103eabc9c6bb1fbcbbb90dfddc0ecf7189e87b0b5df24b23658515f6aa097
+  data.tar.gz: a8dc4d46b1a2db70b222ad383cebcd756a5e2e2dcc866a35da78cab04dea11a113722f69b79022b69a70829f8271e80ea4f583a1ca5715f1c7c42d7cc857f97c

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+script: bundle exec rspec spec/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in rumba-crawler.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Volodymyr Ladnik
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,86 @@
+# Rumba::Crawler
+[![Dependency Status](https://gemnasium.com/vladnik/rumba-crawler.png)](https://gemnasium.com/vladnik/rumba-crawler)
+[![Build Status](https://travis-ci.org/vladnik/rumba-crawler.png?branch=master)](https://travis-ci.org/vladnik/rumba-crawler)
+[![Code Climate](https://codeclimate.com/github/vladnik/rumba-crawler.png)](https://codeclimate.com/github/vladnik/rumba-crawler)
+
+Web crawler with JSON-based DSL and EventMachine-powered page fetching
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'rumba-crawler'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install rumba-crawler
+
+## Usage
+Gem supports ```"css", "root" and "regexp"``` service keys:
+* css: CSS locator for node
+* root: custom root for current node (parent node is used by default )
+* regexp: regular expression to extract data
+
+If you have multiple objects of the same type, put object description into ```[ ]```.
+
+Besides it supports shortcut for ```"css"``` attribute, so if you don't have any nested nodes(leaf node) 
+and you don't need ```"root" and "regexp"``` keys, you can omit "css" key and provide locator as string value 
+(```"name": "span.name"``` in example).
+
+Map your models to page structure like this:
+```json
+[
+  {
+    "game": {
+      "css":".games",
+      "teams":[
+        { 
+          "css": ".teams",
+          "name": "span.name",
+          "scores": { 
+            "css":".score",
+            "regexp":"[0-9]+"
+          },
+          "sport": {
+            "css":"#sport",
+            "root":"body"
+          }
+        }
+	  ]	
+	}
+  }
+]
+```
+This will allow you to extract data from page for current model structure:
+```ruby
+class Game
+  # has_many :teams
+  def teams=(array_value)
+    array_value.each do |value|
+      Team.find_or_create(value)
+    end
+  end
+end
+class Team
+  # belongs_to :game
+  # belongs_to :sport
+  attr_writer :name, :scores
+  def sport=(value)
+    Sport.find_or_create(value)
+  end
+end
+class Sport
+  # has_many :teams
+end
+```
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,4 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+RSpec::Core::RakeTask.new('spec')
+task default: :spec

data/lib/rumba/crawler.rb

@@ -0,0 +1,28 @@
+require "date"
+require "json"
+require "em-http-request"
+require "nokogiri"
+require "rumba/crawler/version"
+require "rumba/crawler/exceptions"
+require "rumba/crawler/models"
+require "rumba/crawler/parser"
+
+module Rumba
+  module Crawler
+
+    def self.get_data(url, query)
+      EventMachine.run {
+        http = EventMachine::HttpRequest.new(url).get query: query
+        http.errback { raise Exceptions::BadResponse; EventMachine.stop }
+        http.callback {
+          if http.response_header.status >= 400
+            raise Exceptions::BadResponse
+          else
+            yield(http.response)
+          end
+          EventMachine.stop
+        }
+      }
+    end
+  end
+end

data/lib/rumba/crawler/exceptions.rb

@@ -0,0 +1,8 @@
+module Rumba
+  module Crawler
+    module Exceptions
+      class BadResponse < StandardError
+      end
+    end
+  end
+end

data/lib/rumba/crawler/models.rb

@@ -0,0 +1,6 @@
+module Rumba
+  module Crawler
+    module Models
+    end
+  end
+end

data/lib/rumba/crawler/parser.rb

@@ -0,0 +1,72 @@
+module Rumba
+  module Crawler
+    class Parser
+      include Rumba::Crawler::Models
+      # Service Keys
+      SK = ['css', 'root', 'regexp']
+
+      def process(response, template)
+        template = JSON.parse(template)
+        @doc = Nokogiri::HTML(response)
+        if template.is_a? Array
+          parse_multi(@doc, template.first)
+        else
+          parse_node(@doc, template, template.keys.first)
+        end
+      end
+
+      def parse_multi(doc, template)
+        result = []
+        template.each do |key, value|
+          get_node(doc, value).each do |node|
+            result << create_object(key, node, value)
+          end
+        end
+        return result
+      end
+
+      def parse_node(doc, template, name)
+        node = get_node(doc, template).first
+        if leaf_node?(template)
+          get_content(node, template)
+        else
+          create_object(name, node, template)
+        end
+      end
+
+      def create_object(name, node, template)
+        object = send(name)
+        template.reject{|key, _| SK.include?(key)}.each do |key, value|
+          if value.is_a? Array
+            object.send("#{key}=", parse_multi(node, value.first))
+          else
+            object.send("#{key}=", parse_node(node, value, key))
+          end
+        end
+        return object
+      end
+
+      def get_content(node, template)
+        if template['regexp']
+          /#{template['regexp']}/i.match(node.content).to_s
+        else
+          node.content
+        end
+      end
+
+      def get_node(doc, template)
+        if template.is_a?(String)
+          doc.css(template)
+        elsif template['root']
+          @doc.css(template['root']).css(template['css'])
+        else
+          doc.css(template['css'])
+        end
+      end
+
+      def leaf_node?(template)
+        template.is_a?(String) || template.reject{|key, _| SK.include?(key)}.empty?
+      end
+    end
+  end
+end

data/lib/rumba/crawler/version.rb

@@ -0,0 +1,5 @@
+module Rumba
+  module Crawler
+    VERSION = "0.1"
+  end
+end

data/rumba-crawler.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'rumba/crawler/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "rumba-crawler"
+  spec.version       = Rumba::Crawler::VERSION
+  spec.authors       = ["Volodymyr Ladnik"]
+  spec.email         = ["Volodymyr.UA@gmail.com"]
+  spec.summary       = %q{Web crawler with JSON-based DSL and EventMachine-powered page fetching}
+  spec.description   = spec.summary
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "json", "~> 1.8.0"
+  spec.add_dependency "em-http-request", "~> 1.0.3"
+  spec.add_dependency "nokogiri", "~> 1.5.9"
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "webmock"
+end

data/spec/crawler/parser_spec.rb

@@ -0,0 +1,101 @@
+require 'spec_helper'
+
+module Rumba::Crawler
+  describe Parser do
+    let(:parser) { Parser.new }
+
+    describe '#process' do
+      it "invokes parse_multi for Array template" do
+        Nokogiri::HTML::Document.stub(parse: :parsed_doc)
+        parser.should_receive(:parse_multi).with(:parsed_doc, 'games').and_return([:result])
+        parser.process('<html><html>','["games"]').should eq [:result]
+      end
+
+      it "invokes parse_node for Object template" do
+        Nokogiri::HTML::Document.stub(parse: :parsed_doc)
+        parser.should_receive(:parse_node).with(:parsed_doc, {"game" => 123}, 'game').and_return(:result)
+        parser.process('<html><html>','{"game": 123}').should eq :result
+      end
+    end
+
+    describe '#parse_multi' do
+      it "creates object for every matching HTML node" do
+        parser.stub(get_node: [:node, :node])
+        parser.should_receive(:create_object).with(:key, :node, :value).twice.and_return(:result)
+        parser.parse_multi(:doc, {key: :value}).should eq [:result, :result]
+      end
+    end
+
+    describe '#parse_node' do
+      it "gets content for template leaf node" do
+        parser.stub(get_node: [:node])
+        parser.stub(leaf_node?: true)
+        parser.should_receive(:get_content).with(:node, :template).and_return('some content')
+        parser.parse_node(:doc, :template, :name).should eq 'some content'
+      end
+      it "creates object for template parent node" do
+        parser.stub(get_node: [:node])
+        parser.stub(leaf_node?: false)
+        parser.should_receive(:create_object).with(:name, :node, :template).and_return(:object)
+        parser.parse_node(:doc, :template, :name).should eq :object
+      end
+    end
+
+    describe "#create_object" do
+      it "creates an object" do
+        parser.should_receive(:name).and_return(:object)
+        parser.create_object(:name, :node, {'css' => 'body'}).should eq :object
+      end
+      it "invokes a setter with parse_multi method for object attributes with Array value" do
+        object = double("object")
+        object.should_receive(:name=).with(:result)
+        parser.should_receive(:parse_multi).with(:node, 'attr').and_return(:result)
+        parser.stub(game: object)
+        parser.create_object(:game, :node, {'css' => 'body', 'name' => ['attr']})
+      end
+      it "invokes a setter with parse_node method for object other attributes" do
+        object = double("object")
+        object.should_receive(:name=).with(:result)
+        parser.should_receive(:parse_node).with(:node, {"attr"=>"value"}, "name").and_return(:result)
+        parser.stub(game: object)
+        parser.create_object(:game, :node, {'css' => 'body', 'name' => {'attr' => 'value'}})
+      end
+    end
+
+    describe "#get_content" do
+      it "get node content" do
+        node = double("node")
+        node.should_receive(:content).and_return('some content')
+        parser.get_content(node, {}).should eq 'some content'
+      end
+      it "uses provided regexp to extract content" do
+        node = double("node")
+        node.should_receive(:content).and_return('content123')
+        parser.get_content(node, {'regexp' => '[a-z]+'}).should eq 'content'
+      end
+    end
+
+    describe "#get_node" do
+      it "uses css locator provided" do
+        node = double("node")
+        node.should_receive(:css).with('locator').and_return(:node)
+        parser.get_node(node, {'css' => 'locator'}).should eq :node
+      end
+      it "uses shortcut css locator" do
+        node = double("node")
+        node.should_receive(:css).with('locator').and_return(:node)
+        parser.get_node(node, 'locator').should eq :node
+      end
+    end
+
+    describe "#leaf_node?" do
+      it "identifies shortcut locator" do
+        parser.leaf_node?('locator').should eq true
+      end
+      it "identifies non-service attributes" do
+        parser.leaf_node?({'css' => 'locator'}).should eq true
+        parser.leaf_node?({'name' => 'locator'}).should eq false
+      end
+    end
+  end
+end

data/spec/crawler_spec.rb

@@ -0,0 +1,23 @@
+require 'spec_helper'
+
+module Rumba::Crawler
+  describe '#get_data' do
+    it "makes GET request to specified url with specified parameters" do
+      stub = stub_request(:get, "http://example.com/?date=now").to_return(status: 200, body: '', headers: {})
+      Rumba::Crawler.get_data('http://example.com', {date: 'now'}) { |response| }
+      stub.should have_been_requested
+    end
+
+    it "invokes callback method with response data" do
+      callback = double("callback")
+      callback.should_receive(:process_response).with('response data')
+      stub_request(:get, "http://example.com/?date=now").to_return(status: 200, body: 'response data', headers: {})
+      Rumba::Crawler.get_data('http://example.com', {date: 'now'}) { |response| callback.process_response(response)}
+    end
+
+    it "should raise exception on bad response" do
+      stub_request(:get, "http://example.com/?date=now").to_return(status: 400)
+      expect {Rumba::Crawler.get_data('http://example.com', {date: 'now'}) { |response| }}.to raise_error(Exceptions::BadResponse)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+require 'rspec'
+require 'webmock/rspec'
+require 'rumba/crawler'
+
+
+RSpec.configure do |config|
+  config.color_enabled = true
+  config.formatter     = :documentation
+end

metadata

@@ -0,0 +1,160 @@
+--- !ruby/object:Gem::Specification
+name: rumba-crawler
+version: !ruby/object:Gem::Version
+  version: '0.1'
+platform: ruby
+authors:
+- Volodymyr Ladnik
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-06-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+- !ruby/object:Gem::Dependency
+  name: em-http-request
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.3
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.9
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.9
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Web crawler with JSON-based DSL and EventMachine-powered page fetching
+email:
+- Volodymyr.UA@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/rumba/crawler.rb
+- lib/rumba/crawler/exceptions.rb
+- lib/rumba/crawler/models.rb
+- lib/rumba/crawler/parser.rb
+- lib/rumba/crawler/version.rb
+- rumba-crawler.gemspec
+- spec/crawler/parser_spec.rb
+- spec/crawler_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Web crawler with JSON-based DSL and EventMachine-powered page fetching
+test_files:
+- spec/crawler/parser_spec.rb
+- spec/crawler_spec.rb
+- spec/spec_helper.rb