RubyGems - raspar - Versions diffs - 0.0.1 - Mend

raspar 0.0.1

Files changed (25) hide show

checksums.yaml +7 -0
data/.gitignore +8 -0
data/.travis.yml +10 -0
data/Gemfile +6 -0
data/LICENSE +21 -0
data/README.md +197 -0
data/Rakefile +1 -0
data/example/Gemfile +5 -0
data/example/ccode.rb +22 -0
data/example/debug.rb +4 -0
data/example/dynamic.rb +63 -0
data/example/leguide.rb +56 -0
data/lib/raspar.rb +94 -0
data/lib/raspar/dynamic_parser.rb +44 -0
data/lib/raspar/parser.rb +152 -0
data/lib/raspar/result.rb +20 -0
data/lib/raspar/version.rb +3 -0
data/raspar.gemspec +23 -0
data/spec/add_parser_spec.rb +58 -0
data/spec/dynamic_parser_spec.rb +74 -0
data/spec/parser_spec.rb +86 -0
data/spec/raspar_spec.rb +33 -0
data/spec/sample_parser.rb +94 -0
data/spec/spec_helper.rb +29 -0
metadata +100 -0

checksums.yaml ADDED

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: df5341823f5ad794338001c6a0bbf53de1ac4089
+  data.tar.gz: 540306d13e95a15312ba517fbc470fa136270bf0
+SHA512:
+  metadata.gz: d3966a090724b5435ba847c738b59f73d5fd23f59555db20b0317297759d0084032071bdf46cfba0171d84d7716080da09fbd58e8a56003bb4124ee3fa603cbc
+  data.tar.gz: 9e169aff0a96dc1f6b7ccdf8e9e92fd8c8d4ae4d2543fbae983b5fa148a91c448577b2117cde2164758432abc5ddabd4e987ffc974936024b411faf686fb8cb6

data/.gitignore ADDED

@@ -0,0 +1,8 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+*.swp
+1.html
+test.rn
+*.DS_Store

data/.travis.yml ADDED

@@ -0,0 +1,10 @@
+branches:
+  only:
+    - 'master'
+language: ruby
+rvm:
+  - 2.0.0
+script: bundle exec rspec spec

data/Gemfile ADDED

@@ -0,0 +1,6 @@
+source "http://rubygems.org"
+gem 'simplecov', :require => false
+gem 'coveralls', require: false
+# Specify your gem's dependencies in raspar.gemspec
+gemspec

data/LICENSE ADDED

@@ -0,0 +1,21 @@
+The MIT License
+Copyright (c) 2013 Jiren Patel[jirenpatel@gmail.com]
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md ADDED

@@ -0,0 +1,197 @@
+## Raspar - scraping library
+Raspar is a html scraping library which help to map html elements to ruby object using 'css' or 'xpath' selector.Using this library user can define multiple parser for different websites and it select parser according to input html page url.
+[![Build Status](https://travis-ci.org/jiren/raspar.png?branch=master)](https://travis-ci.org/jiren/raspar)
+ [![Coverage Status](https://coveralls.io/repos/jiren/raspar/badge.png?branch=master)](https://coveralls.io/r/jiren/raspar?branch=master)
+## Installation
+Add this line to your application's Gemfile:
+    gem 'raspar', :git => 'git://github.com/jiren/raspar.git'
+And then execute:
+    $ bundle
+## Usage
+```ruby
+  result = Rapsar.parse(url, html) #This will return parsed result object array.
+  #Result
+  [
+    #<Raspar::Result:0x007ffc91e4d640
+      @attrs={:name=>"Test1", :price=>"10", :image=>"1", :desc=>"Description"},
+      @domain="example.com",
+      @name=:product>,
+    #<Raspar::Result:0x007ffc91e57be0
+      @attrs={:name=>"Test2", :price=>"20", :image=>"2", :desc=>"Description"},
+      @domain="example.com",
+      @name=:product>
+   ]
+```
+## Example
+### Sample HTML
+```html
+<!DOCTYPE html>
+<html>
+    <body>
+    <span class="desc">Description</span>
+    <divi class="item">
+      <img src="1">
+      <span>Test1</span>
+      <span class="price">10</span>
+    </div>
+    <div class="item">
+      <img src="2">
+      <span>Test2</span>
+      <span class="price">20</span>
+    </div>
+    <span class="second">
+      <img src="2">
+      <span>Test2</span>
+      <span class="price">20</span>
+    </span>
+    <div class="offer">
+      <span class="name">First Offer</span>
+      <span class="percentage">10% off</span>
+    </div>
+  </body>
+</html>
+```
+#### Parser for above HTML
+```ruby
+class SampleParser
+  include Raspar
+  domain 'http://sample.com'
+  attr :desc, '.desc', :eval => :format_desc
+  collection :product, '.item,span.second' do
+    attr :image_url, 'img', :prop => 'src', :eval => :make_image_url
+    attr :name,  'span:first'
+    attr :price, 'span.price', :eval => Proc.new{|price, ele| price.to_i}
+    attr :price_map do |text, ele|
+      val = ele.search('span').collect{|s| s.content.strip}
+      {val[0] => val[1].to_f}
+    end
+  end
+  collection :offer, '.offer' do
+    attr :name, '.name'
+    attr :discount, '.discount' do |text, ele|
+      test.split('%').first.to_f
+    end
+  end
+  def name_price(val, ele)
+    val = ele.search('span').collect{|s| s.content.strip}
+    {val[0] => val[1].to_f}
+  end
+  def make_image_url(path, ele)
+    URI(@domain_url).merge(path).to_s
+  end
+  def format_desc(text, ele)
+    "Description: #{text}"
+  end
+end
+```
+- 'domain' method register parser for given domain value so raspar can differentiate parser at runtime.
+- Define 'attr' which is going to parse. First argument is 'css' or 'xpath' selector. Second argument contain options.
+  - Valid options are :field, :eval.
+  - :porp is selecting particular property/attribute for html element. In example for image, select image url using :prop => 'src'
+  - :eval is use to post process attr value. It can be proc, method or block. Each method, proc or block use for eval has two argument, first is html element text and second is html element as a Nokogiri doc.
+  - if :eval is not define then parser will return text of selected html element.
+- If your page has multiple type of objects or collections then define using 'collection' block. In above example '.item' and 'span.second' are product while '.offer' element contain offer detail.
+- In html page some of attributes are common which is not reside under particular collection and this attributes values are going to add for each parse object.
+### Add Parser in different way
+It takes only one argument domain url and block.
+```ruby
+Raspar.add('http://example.com') do
+  attr :desc, '.desc', :eval => :format_desc
+  collection :product, '.item,span.second' do
+    attr :image_url, 'img', :prop => 'src'
+    attr :name,  'span:first'
+    attr :price, 'span.price', :eval => Proc.new{|price, ele| price.to_i}
+    attr :price_map do |text, ele|
+      val = ele.search('span').collect{|s| s.content.strip}
+      {val[0] => val[1].to_f}
+    end
+  end
+  def format_desc(text, ele)
+    "Desc: #{text.downcase}"
+  end
+end
+```
+### Dynamically add Parser
+```ruby
+domain  = 'http://www.sample.com'
+selector_map = {
+  :common_attrs => {
+    :desc => {:select => '.desc'}
+  },
+  :collections =>{
+    :item => {
+      :select => 'div, span.second',
+      :attrs => {
+        :name =>  { :select => 'span:first'},
+        :price =>  { :select => 'span.price', :eval => :parse_price},
+        :image => { :select => 'img', :prop => 'src'}
+      }
+    }
+  }
+}
+module ParserHelper
+  def parse_price(val, ele)
+    val.gsub(/[ ,]/, ' ' => '', ',' => '.').to_f
+  end
+end
+Raspar.add(domain, selector_map, ParserHelper) //Add parser
+```
+For post processing user can add parser helper, but it is not mandatory.
+## Contributing
+Please send me a pull request so that this can be improved.
+## License
+This is released under the MIT license.

data/Rakefile ADDED

	@@ -0,0 +1 @@
1	+ require "bundler/gem_tasks"

data/example/Gemfile ADDED

@@ -0,0 +1,5 @@
+source :rubygems
+gem 'raspar', :path => "../"
+#gem 'raspar', :git => 'git@github.com:lemig/raspar.git'
+gem 'nokogiri'

data/example/ccode.rb ADDED

@@ -0,0 +1,22 @@
+require 'rubygems'
+require 'bundler/setup'
+require 'open-uri'
+require 'raspar'
+require 'pp'
+class CCode
+  include Raspar
+  domain 'http://www.exchange-rate.com'
+  collection :currency_code, 'table[cellpadding="2"] tr:gt(1)' do
+    attr :country,  'td:nth-child(1)'
+    attr :currency, 'td:nth-child(2)'
+    attr :code,     'td:nth-child(3)'
+  end
+end
+url = 'http://www.exchange-rate.com/currency-list.html'
+page = open(url).read
+Raspar.parse(url, page).each {|i| pp i }

data/example/debug.rb ADDED

@@ -0,0 +1,4 @@
+require 'rubygems'
+require 'bundler/setup'
+require 'open-uri'
+require 'raspar'

data/example/dynamic.rb ADDED

@@ -0,0 +1,63 @@
+require 'rubygems'
+require 'bundler/setup'
+require 'open-uri'
+require 'raspar'
+require 'pp'
+module ParserHelper
+  def build_specification(val, ele)
+    attrs = {}
+    ele.search('li').each do |li|
+      attrs[li.search('.title').first.content] =  li.search('.value').first.content
+    end
+    attrs
+  end
+  #For normal attr use instance method
+  def parse_price(val, ele)
+    val.gsub(/[ ,]/, ' ' => '', ',' => '.')
+  end
+  def parse_shipping_price(text, ele)
+    text.split(':').last.strip
+  end
+  def data_attr_parse(text, ele)
+    Nokogiri::HTML.parse(text).text.split(':').last.strip
+  end
+end
+domain  = 'http://www.leguide.com'
+selector_map = {
+  :common_attrs => {
+    :name => {:select => '.block_bpu_feature .p b'},
+    :specifications => {:select => '#page2', :eval => :build_specification}
+  },
+  :collections => {
+    :product =>{
+      :select => '.offers_list li',
+      :attrs => {
+        :image          => { :select => 'img', :prop => 'src'},
+        :price          => { :select => '.price .euro.gopt', :eval => :parse_price},
+        :orignal_price  => { :select => '.price .barre', :eval => :parse_price},
+        :desc           => { :select => '.gopt.description,.info .description'},
+        :vendor         => { :select => '.name a' },
+        :availability   => { :select => '.av', :prop => 'data-value', :eval => :data_attr_parse},
+        :delivery_time  => { :select => '.dv', :prop => 'data-value', :eval => :data_attr_parse},
+        :shipping_price => { :select => '.delivery.gopt'}
+      }
+    }
+  }
+}
+Raspar.add(domain, selector_map, ParserHelper)
+url = 'http://www.leguide.com/sb/bp/5010500/hotpoint_ariston/ECO9F_149_FRS/55743410.htm'
+page = open(url).read()
+Raspar.parse(url, page).each do |i|
+  pp i
+  p "*"*40
+end

data/example/leguide.rb ADDED

@@ -0,0 +1,56 @@
+require 'rubygems'
+require 'bundler/setup'
+require 'open-uri'
+require 'raspar'
+require 'pp'
+class Leguide
+  include Raspar
+  SHIPPING_PROC = Proc.new{|text, ele| text.split(':').last.strip}
+  DATA_PROC = Proc.new{|text, ele| Nokogiri::HTML.parse(text).text.split(':').last.strip}
+  domain 'http://www.leguide.com'
+  #External attrs
+  attr :name, '.block_bpu_feature .p b'
+  attr :specifications, '#page2', :eval => :build_specification
+  collection :product, '.offers_list li' do
+    attr :alt_name,       '.gopt.offer.t'
+    attr :image,          '.lg_photo img', :prop => 'src'
+    attr :price,          '.price .euro.gopt'
+    attr :orignal_price,  '.price .barre'
+    attr :desc,           '.gopt.description,.info .description'
+    attr :vendor,         '.name a'
+    attr :availability,   '.av', :prop => 'data-value', :eval => DATA_PROC
+    attr :delivery_time,  '.dv', :prop => 'data-value', :eval => DATA_PROC
+    attr :shipping_price, '.delivery.gopt', :eval => SHIPPING_PROC
+  end
+  #For External attr define class method because it evalute only once for all object in sigle html doc.
+  def build_specification(val, ele)
+    attrs = {}
+    ele.search('li').each do |li|
+      attrs[li.search('.title').first.content] =  li.search('.value').first.content
+    end
+    attrs
+  end
+  #For normal attr use instance method
+  def parse_price(val, ele)
+    val.gsub(/[ ,]/, ' ' => '', ',' => '.')
+  end
+end
+url = 'http://www.leguide.com/sb/bp/5010500/hotpoint_ariston/ECO9F_149_FRS/55743410.htm'
+url = 'http://www.leguide.com/electromenager.htm'
+p ARGV[0] || url
+#page = open(ARGV[0] || url).read().gsub(/[[:cntrl:]@]/, '')
+page = open(ARGV[0] || url).read()
+Raspar.parse(url, page).each do |o|
+  pp o
+  p "*"*40
+end

data/lib/raspar.rb ADDED

@@ -0,0 +1,94 @@
+require 'net/http'
+require 'net/https'
+require 'webrick/cookie'
+require 'delegate'
+require 'nokogiri'
+require 'raspar/version'
+require 'raspar/result'
+require 'raspar/parser'
+require 'raspar/dynamic_parser'
+module Raspar
+  def self.included(base)
+    base.extend Parser::ClassMethods
+    base.send :include, Parser::InstanceMethods
+    base._init_parser_
+  end
+  class RasparException < Exception; end
+  class << self
+    def _init
+      @parsers = {}
+    end
+    #Register parser class and domain
+    #
+    # === Example
+    # Raspar::Base.register('http://test.com', TestParser)
+    #
+    def register(domain, klass)
+      @parsers ||= {}
+      (URI(domain).host || domain).tap{ |host| @parsers[host] = klass }
+    end
+    # clear parser list
+    def clear_parser_list
+      @parsers = {}
+    end
+    def remove(domain)
+      @parsers.delete(URI(domain).host) if @parsers
+    end
+    def parsers
+      @parsers
+    end
+    def exist?(url)
+      @parsers.include?(URI(url).host)
+    end
+    def parse(url, html)
+      host = URI(url).host
+      if @parsers[host]
+        @parsers[host].parse(html)
+      else
+        puts "No parser define for #{host}"
+        nil
+      end
+    end
+    def add(url, selector_map = nil, helper_module = nil, &block)
+      if self.exist?(url)
+        raise RasparException.new("Parser already exist for '#{url}'")
+      end
+      if selector_map
+        return DynamicParser.register(url, selector_map, helper_module)
+      end
+      klass_name = URI(url).host
+                           .split('.')
+                           .reject{|w| w == 'www'}
+                           .collect{|w| w[0].upcase + w[1..-1] }
+                           .join
+                           .gsub(/\W/, '')
+      klass = Class.new
+      klass.send :include, Raspar
+      klass.domain(url)
+      klass.class_exec(&block) if block_given?
+      klass_name = 'R' + klass_name if Object.const_defined?(klass_name)
+      Object.const_set(klass_name, klass)
+    end
+  end
+  #Init Raspar parser list
+  self._init
+end

data/lib/raspar/dynamic_parser.rb ADDED

@@ -0,0 +1,44 @@
+module Raspar
+  class DynamicParser
+    include Parser::InstanceMethods
+    include Parser::ClassMethods
+    attr_accessor :domain, :domain_url,
+      :common_attrs, :collections, :_current_container_
+    def initialize
+      @common_attrs = {}
+      @collections = {}
+    end
+    def parse(html)
+      self.process(html, self)
+    end
+    def self.register(url, selector_map, helper_module = nil)
+      dp =  self.new
+      if selector_map[:common_attrs]
+        selector_map[:common_attrs].each { |attr, opts| dp.attr(attr, opts) }
+      end
+      if selector_map[:collections]
+        selector_map[:collections].each do |name, collection_opts|
+          dp.collections[name] = { :select => collection_opts[:select], :attrs => {} }
+          dp._current_container_ = name.to_sym
+          collection_opts[:attrs].each { |attr, opts| dp.attr(attr, opts) }
+          dp._current_container_ = nil
+        end
+      end
+      #TODO: Create constant from string and extend object.
+      dp.extend(helper_module) if helper_module
+      dp.domain_url = url
+      dp.domain = Raspar.register(url, dp)
+      dp
+    end
+  end
+end

data/lib/raspar/parser.rb ADDED

@@ -0,0 +1,152 @@
+module Raspar
+  module Parser
+    module ClassMethods
+      attr_reader :domain, :common_attrs, :collections
+      def _init_parser_
+        @common_attrs = {}
+        @collections = {}
+      end
+      #
+      # name  // opts = {:select => nil}
+      # name, '.name' // opts = {:select => ['.name']}
+      # name, '.name, .title' // opts = {:select => ['.name', '.title']}
+      # name, ['.name', .title] // opts = {:select => ['.name', '.title']}
+      # name, '.name', {:eval => :parse_name, :attr => 'name'}
+      #   opts = {:eval => :parse_name, :attr => 'name', :select => ['.name']}
+      # name, {:eval => :parse_name, :attr => 'name'}
+      #   opts = {:eval => :parse_name, :attr => 'name', :select => nil}
+      def attr(name, select = nil, opts = {}, &block)
+        if select.is_a?(Hash)
+          opts = select
+        else
+          opts[:select] = select
+        end
+        opts[:select] = case opts[:select]
+                        when Array
+                          opts[:select].flatten
+                        when String
+                          opts[:select].split(',').collect(&:strip)
+                        else
+                          opts[:select]
+                        end
+        opts[:select] = opts[:select].join(',') if opts[:as] == :array
+        opts[:eval] = opts[:eval].to_sym if opts[:eval].is_a?(String)
+        opts[:eval] = block if block_given?
+        if @_current_container_
+          @collections[@_current_container_][:attrs][name.to_sym] = opts
+        else
+          @common_attrs[name.to_sym] = opts
+        end
+      end
+      def collection(collection_name, select, &block)
+        collection_name = collection_name.to_sym
+        @collections[collection_name] = { :select => select, :attrs => {} }
+        @_current_container_ = collection_name
+        yield
+        @_current_container_ = nil
+      end
+      def domain(url = nil)
+        if url
+          @domain_url = url
+          @domain = Raspar.register(url, self)
+        end
+        @domain
+      end
+      def absolute_url(path)
+        URI(@domain_url).merge(path).to_s
+      end
+      def parse(html)
+        self.new.process(html)
+      end
+      def attrs
+        {:collections => @collections, :common_attrs => @common_attrs}
+      end
+      def info
+        {:domain => @domain, :collections => @collections.keys, :common_attrs => @common_attrs.keys}
+      end
+    end
+    module InstanceMethods
+      attr_reader :attributes
+      #Parse doc: html node accroding to attr selector
+      #If selector is :self then input doc is a selected doc
+      #Select first
+      def attr_parser(doc, attr_map)
+        attrs = {}
+        attr_map.each do |attr_name, opts|
+          ele = doc
+          if opts[:select]
+            if opts[:as] == :array
+              attrs[attr_name] = doc.search(opts[:select]).collect{|e| process_ele(e, opts)}
+            else
+              opts[:select].each do |s|
+                ele = doc.search(s).first
+                break if ele
+              end
+              attrs[attr_name] = process_ele(ele, opts) if ele
+            end
+          else
+            attrs[attr_name] = process_ele(ele, opts) if ele
+          end
+          #attrs[opts[:as]] ||= attrs[attr_name] if opts[:as]
+        end
+        attrs
+      end
+      def process(html, klass = nil)
+        @results = []
+        doc = Nokogiri::HTML(html)
+        klass = self.class unless klass
+        common_attrs = attr_parser(doc, klass.common_attrs)
+        klass.collections.each do |name, collection|
+          doc.search(collection[:select]).each do |ele|
+             attrs = attr_parser(ele, collection[:attrs]).merge!(common_attrs)
+             @results << Result.new(name, attrs, klass.domain)
+          end
+        end
+        @results << Result.new(:default, common_attrs, klass.domain) if @results.none?
+        @results
+      end
+      private
+      #Process selected html element
+      #
+      #- If process is false and check eval is present then pass
+      #  element to eval proc else html ele return and assign to
+      #  attribute.
+      #- If process is false then :value option not going to evaluate.
+      def process_ele(ele, opts)
+        val = opts[:prop] ? ele[opts[:prop]] : ele.content
+        val.strip! if val
+        if opts[:eval]
+          return opts[:eval].is_a?(Symbol) ? self.send(opts[:eval], val, ele) : opts[:eval].call(val, ele)
+        end
+        val
+      end
+    end
+  end
+end

data/lib/raspar/result.rb ADDED

@@ -0,0 +1,20 @@
+module Raspar
+  class Result
+    attr_reader :name, :attrs, :domain
+    def initialize(name, attrs, domain = nil)
+      @name = name
+      @attrs = attrs
+      @domain = domain if domain
+    end
+    def [](f)
+      @attrs[f]
+    end
+    def method_missing(name, *args, &block)
+      @attrs[name]
+    end
+  end
+end

data/lib/raspar/version.rb ADDED

@@ -0,0 +1,3 @@
+module Raspar
+  VERSION = "0.0.1"
+end

data/raspar.gemspec ADDED

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "raspar/version"
+Gem::Specification.new do |s|
+  s.name        = "raspar"
+  s.version     = Raspar::VERSION
+  s.authors     = ["Jiren Patel"]
+  s.email       = ["jiren@joshsoftware.com"]
+  s.homepage    = ""
+  s.summary     = %q{A generic html/xml parser}
+  s.description = %q{Raspar collects data from the html page and creates object from it.}
+  s.rubyforge_project = "raspar"
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  s.add_development_dependency "rspec"
+  s.add_dependency "nokogiri", "~> 1.5.5"
+end

data/spec/add_parser_spec.rb ADDED

@@ -0,0 +1,58 @@
+$:.unshift(File.dirname(__FILE__))
+require 'spec_helper'
+require 'sample_parser'
+describe 'Add Parser' do
+  def add_parser
+    Raspar.add(@site) do
+      attr :desc, '.desc', :common => true
+      collection :product, '.item,span.second' do
+        attr :name,  'span:first, .name', :eval => :full_name
+        attr :price, '.price', :eval => Proc.new{|i| i.to_i}
+      end
+      def full_name(val, ele)
+        "Full Name: #{val}"
+      end
+    end
+  end
+  before(:all) do
+    @site = 'http://addparser.com'
+    @domain = URI(@site).host
+    @parser_class = add_parser
+  end
+  it 'should register parser and parse data' do
+    Raspar.parsers.should include({@domain => @parser_class})
+  end
+  it "should have info" do
+    @parser_class.info.should == {
+      :domain => @domain,
+      :collections => [:product],
+      :common_attrs => [:desc]
+    }
+  end
+  it "should parse html and create object" do
+    parsed_objs = Raspar.parse(@site, FAKE_PAGE)
+    parsed_objs.length.should == 4
+    parsed_objs.count{|o| o.name == :product}.should == 4
+    count = 1
+    parsed_objs.select{|o| o.name == :product}.each do |o|
+      o[:name].should == "Full Name: Test#{count}"
+      o[:price].should == (count * 10)
+      o[:desc].should == "Description"
+      count = count + 1
+    end
+  end
+end

data/spec/dynamic_parser_spec.rb ADDED

@@ -0,0 +1,74 @@
+$:.unshift(File.dirname(__FILE__))
+require 'spec_helper'
+require 'sample_parser'
+module Raspar
+  describe DynamicParser do
+    before do
+      @site = 'http://dynmaicparser.com'
+      @domain = URI(@site).host
+      selector_map = {
+        :common_attrs => {
+          :desc => {:select => '.desc'}
+        },
+        :collections =>{
+          :product => {
+            :select => 'div.item, span.second',
+            :attrs => {
+              :name =>  { :select => 'span:first'},
+              :price =>  { :select => 'span.price'},
+              :image => { :select => 'img', :prop => 'src'}
+            }
+          }
+        }
+      }
+      Raspar.clear_parser_list
+      @dynmaic_parser = Raspar.add(@site, selector_map)
+    end
+    describe '#onload' do
+      it "should register DynamicParser to Raspar parser list" do
+        Raspar.parsers[@domain].class.should == Raspar::DynamicParser
+        @dynmaic_parser.domain.should == @domain
+      end
+    end
+    describe '#parse' do
+      it "should parse html and create object" do
+        parsed_objs = Raspar.parse(@site, FAKE_PAGE)
+        parsed_objs.length.should == 4
+        count = 1
+        parsed_objs.each do |o|
+          o.class.should == Raspar::Result
+          o[:name].should == "Test#{count}"
+          o[:image].should == count.to_s
+          #Price should eval using proc given in option which convert string value
+          #to integer
+          o[:price].should == (count * 10).to_s
+          #External Field check
+          o[:desc].should == "Description"
+          count = count + 1
+        end
+      end
+      it 'should return absoulte url' do
+        @dynmaic_parser.absolute_url('/test').should == @site + '/test'
+      end
+    end
+  end
+end

data/spec/parser_spec.rb ADDED

@@ -0,0 +1,86 @@
+$:.unshift(File.dirname(__FILE__))
+require 'spec_helper'
+require 'sample_parser'
+module Raspar
+  describe Parser do
+    before do
+      @site = 'http://sample.com'
+      @domain = URI(@site).host
+      Raspar.register(@site, SampleParser) unless Raspar.parsers[@domain]
+    end
+    #On load SampleParser class
+    describe 'onload' do
+      it "should register SampleParser to Raspar parser list" do
+        Raspar.parsers.should include({@domain => SampleParser})
+        SampleParser.domain.should == @domain
+      end
+      it 'should return absoulte url' do
+        SampleParser.absolute_url('/test').should == @site + '/test'
+      end
+      it "should have info" do
+        SampleParser.info.should == {
+                        :domain => @domain,
+                        :collections => [:product, :offer],
+                        :common_attrs => [:desc, :specs]
+                      }
+      end
+      it "should not define accessor if options not contail :selector" do
+        SampleParser.instance_methods.include?(:extra) == false
+      end
+    end
+    describe 'parse' do
+      it "should parse html and create object" do
+        parsed_objs = Raspar.parse(@site, FAKE_PAGE)
+        #Total parse objects
+        parsed_objs.length.should == 5
+        parsed_objs.count{|o| o.name == :product}.should == 4
+        parsed_objs.count{|o| o.name == :offer}.should == 1
+        count = 1
+        parsed_objs.select{|o| o.name == :product}.each do |o|
+          o[:name].should == "Full Name: Test#{count}"
+          o[:image].should == count.to_s
+          #Price should eval using proc given in option which convert string value
+          #to integer
+          o[:price].should == (count * 10)
+          #External Field check
+          o[:desc].should == "Description is full desc"
+          #self selector
+          o[:all_text].should == "Test#{count}\n    #{count*10}"
+          o[:price_map].should == {"Test#{count}" => (count*10).to_f}
+          o[:specs].should == ['spec 1', 'spec 2', 'spec 3']
+          count = count + 1
+        end
+        parsed_objs.select{|o| o.name == :offer}.each do |o|
+          o[:name].should == 'First Offer'
+          o[:percentage].should == '10% off'
+        end
+      end
+    end
+  end
+end

data/spec/raspar_spec.rb ADDED

@@ -0,0 +1,33 @@
+$:.unshift(File.dirname(__FILE__))
+require 'spec_helper'
+describe Raspar do
+  before(:each) do
+    @site = 'http://test.com'
+    @host = URI(@site).host
+  end
+  it "should add domain to register parser list" do
+    Raspar.register(@site, TestParser).should == @host
+    Raspar.parsers.should include({@host => TestParser})
+    Raspar.parsers.size.should > 0
+  end
+  it "should clear registered domains" do
+    Raspar.register(@site, TestParser)
+    Raspar.clear_parser_list
+    Raspar.parsers.size.should == 0
+  end
+  it "should able to remove parser from the registered list" do
+    Raspar.clear_parser_list
+    Raspar.register(@site, TestParser)
+    Raspar.remove(@site)
+    Raspar.parsers.should_not include(@host)
+  end
+end

data/spec/sample_parser.rb ADDED

@@ -0,0 +1,94 @@
+class SampleParser
+  include Raspar
+  domain 'http://sample.com'
+  attr :desc, '.desc', :common => true, :eval => :full_desc
+  attr :specs, '.specs li', :common => true, :as => :array, :eval => :format_specs
+  collection :product, '.item,span.second' do
+    attr :image, 'img', :prop => 'src'
+    attr :image_url, 'img', :prop => 'src', :eval => :make_image_url
+    attr :name,  'span:first, .name', :eval => :full_name
+    attr :price, '.price', :eval => Proc.new{|i| i.to_i}
+    attr :all_text
+    attr :price_map do |text, ele|
+      val = ele.search('span').collect{|s| s.content.strip}
+      {val[0] => val[1].to_f}
+    end
+  end
+  collection :offer, '.offer' do
+    attr :name, '.name'
+    attr :percentage, '.percentage'
+  end
+  def full_name(val, ele)
+    "Full Name: #{val}"
+  end
+  def name_price(val, ele)
+    val = ele.search('span').collect{|s| s.content.strip}
+    {val[0] => val[1].to_f}
+  end
+  def make_image_url(path, ele)
+    self.class.absolute_url(path)
+  end
+  def full_desc(text, ele)
+    "#{text} is full desc"
+  end
+  def format_specs(text, ele)
+    text.downcase
+  end
+end
+FAKE_PAGE = %q{
+  <!DOCTYPE html>
+  <html>
+  <body>
+  <span class="desc">Description</span>
+  <ul class="specs">
+    <li>Spec 1</li>
+    <li>Spec 2</li>
+    <li>Spec 3</li>
+  </ul>
+  <div class="item">
+    <img src="1">
+    <span>Test1</span>
+    <span class="price">10</span>
+  </div>
+  <div class="item">
+    <img src="2">
+    <span>Test2</span>
+    <span class="price">20</span>
+  </div>
+  <div class="item">
+    <img src="3">
+    <span>Test3</span>
+    <span class="price">30</span>
+  </div>
+  <span class="second">
+    <img src="4">
+    <span>Test4</span>
+    <span class="price">40</span>
+  </span>
+  <div class="offer">
+    <span class="name">First Offer</span>
+    <span class="percentage">10% off</span>
+  </div>
+  </body>
+  </html>
+}

data/spec/spec_helper.rb ADDED

@@ -0,0 +1,29 @@
+require 'rubygems'
+require 'bundler/setup'
+require 'open-uri'
+require 'simplecov'
+SimpleCov.start do
+  add_filter '/spec/'
+  add_filter '/example/'
+  add_group 'gem', 'lib'
+end
+require 'coveralls'
+Coveralls.wear!
+RSpec.configure do |config|
+  config.color_enabled = true
+  #config.tty = true
+  #config.formatter = :documentation
+end
+$:.unshift(File.dirname(__FILE__) + '/../lib/')
+require 'raspar'
+#REAL_PAGE = open('spec/html/test.htm')
+class TestParser
+end

metadata ADDED

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification
+name: raspar
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Jiren Patel
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2013-08-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.5
+description: Raspar collects data from the html page and creates object from it.
+email:
+- jiren@joshsoftware.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- example/Gemfile
+- example/ccode.rb
+- example/debug.rb
+- example/dynamic.rb
+- example/leguide.rb
+- lib/raspar.rb
+- lib/raspar/dynamic_parser.rb
+- lib/raspar/parser.rb
+- lib/raspar/result.rb
+- lib/raspar/version.rb
+- raspar.gemspec
+- spec/add_parser_spec.rb
+- spec/dynamic_parser_spec.rb
+- spec/parser_spec.rb
+- spec/raspar_spec.rb
+- spec/sample_parser.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses: []
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: raspar
+rubygems_version: 2.0.3
+signing_key:
+specification_version: 4
+summary: A generic html/xml parser
+test_files:
+- spec/add_parser_spec.rb
+- spec/dynamic_parser_spec.rb
+- spec/parser_spec.rb
+- spec/raspar_spec.rb
+- spec/sample_parser.rb
+- spec/spec_helper.rb