raspar 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: df5341823f5ad794338001c6a0bbf53de1ac4089
-  data.tar.gz: 540306d13e95a15312ba517fbc470fa136270bf0
+  metadata.gz: 144d5399e4105ea847e8d5441d80e15f022d2a8d
+  data.tar.gz: f73462c4d5be73a4f189cc430a539b2d7cd37577
 SHA512:
-  metadata.gz: d3966a090724b5435ba847c738b59f73d5fd23f59555db20b0317297759d0084032071bdf46cfba0171d84d7716080da09fbd58e8a56003bb4124ee3fa603cbc
-  data.tar.gz: 9e169aff0a96dc1f6b7ccdf8e9e92fd8c8d4ae4d2543fbae983b5fa148a91c448577b2117cde2164758432abc5ddabd4e987ffc974936024b411faf686fb8cb6
+  metadata.gz: a9b42d385112c7da047ba39a4bb2202ad4ded4767a6c510b526ab61bbf30420cf3e8ce9f9af8e00e5bbb81066605456ea805d4385863936cfae29508792cd499
+  data.tar.gz: 9588cf43acef8472bb0a9493c1dc364a99721401c0c5e55a13ee9fe2035a76d2eb01e536a29879a29a3f0e85c112ea50b8b409bcb34456fd69ef1bde4774b18f

data/.gitignore

@@ -6,3 +6,7 @@ pkg/*
 1.html
 test.rn
 *.DS_Store
+/coverage
+test.rb
+x.rb
+/tmp

data/README.md

@@ -23,16 +23,17 @@ And then execute:
   result = Rapsar.parse(url, html) #This will return parsed result object array.
 
   #Result
-  [
+  { :products => [
     #<Raspar::Result:0x007ffc91e4d640
       @attrs={:name=>"Test1", :price=>"10", :image=>"1", :desc=>"Description"},
       @domain="example.com",
-      @name=:product>,
+      @name=:products>,
     #<Raspar::Result:0x007ffc91e57be0
       @attrs={:name=>"Test2", :price=>"20", :image=>"2", :desc=>"Description"},
       @domain="example.com",
-      @name=:product>
-   ]
+      @name=:products>
+    ] 
+   }
 
 ```
 

data/example/Gemfile

@@ -1,5 +1,6 @@
-source :rubygems
+source 'https://rubygems.org'
 
-gem 'raspar', :path => "../"
+#gem 'raspar', :path => "../"
 #gem 'raspar', :git => 'git@github.com:lemig/raspar.git'
+gem 'raspar'
 gem 'nokogiri'

data/example/best_buy.rb

@@ -0,0 +1,30 @@
+require 'rubygems'
+require 'rest_client'
+require 'bundler/setup'
+require 'raspar'
+require 'pp'
+
+class BestBuy
+  include Raspar
+
+  domain 'www.bestbuy.com'
+
+  collection :products, '.hproduct' do
+    attr :image, '.image-col img', prop: 'src'
+    attr :name, '.info-main .name'
+    attr :price, 'span[itemprop="price"]'
+    attr :sku,  '.sku'
+    attr :description, '.product-short-description li', as: :array
+    attr :rating, 'span[itemprop="ratingValue"]', eval: ->(text, ele){ text.to_f }
+  end
+
+end
+
+url = ARGV[0] || "http://www.bestbuy.com/site/promo/htc-one-offer-118429"
+p url
+page = RestClient.get(url).to_str
+
+Raspar.parse(url, page).each do |product|
+  pp product
+  p "*"*40  
+end

data/example/{ccode.rb → country_code.rb}

@@ -1,10 +1,10 @@
 require 'rubygems'
+require 'rest_client'
 require 'bundler/setup'
-require 'open-uri'
 require 'raspar'
 require 'pp'
 
-class CCode
+class CountryCode
   include Raspar
 
   domain 'http://www.exchange-rate.com'
@@ -17,6 +17,6 @@ class CCode
 end
 
 url = 'http://www.exchange-rate.com/currency-list.html' 
-page = open(url).read
+page = RestClient.get(url).to_str
 
 Raspar.parse(url, page).each {|i| pp i }

data/example/leguide.rb

@@ -1,6 +1,6 @@
 require 'rubygems'
+require 'rest_client'
 require 'bundler/setup'
-require 'open-uri'
 require 'raspar'
 require 'pp'
 
@@ -16,13 +16,12 @@ class Leguide
   attr :name, '.block_bpu_feature .p b'
   attr :specifications, '#page2', :eval => :build_specification
 
-  collection :product, '.offers_list li' do
-    attr :alt_name,       '.gopt.offer.t'
-    attr :image,          '.lg_photo img', :prop => 'src'
-    attr :price,          '.price .euro.gopt'
-    attr :orignal_price,  '.price .barre'
+  collection :product, '.offers_item' do
+    attr :alt_name,       '.gopt.l'
+    attr :image,          '.lg_photo img, .img_container img', :prop => 'src'
+    attr :price,          '.gopt .prices'
+    attr :orignal_price,  '.gopt.old_price'
     attr :desc,           '.gopt.description,.info .description'
-    attr :vendor,         '.name a'
     attr :availability,   '.av', :prop => 'data-value', :eval => DATA_PROC
     attr :delivery_time,  '.dv', :prop => 'data-value', :eval => DATA_PROC
     attr :shipping_price, '.delivery.gopt', :eval => SHIPPING_PROC
@@ -44,11 +43,11 @@ class Leguide
 
 end
 
-url = 'http://www.leguide.com/sb/bp/5010500/hotpoint_ariston/ECO9F_149_FRS/55743410.htm'
 url = 'http://www.leguide.com/electromenager.htm'
+url = ARGV[0] || url
 p ARGV[0] || url
-#page = open(ARGV[0] || url).read().gsub(/[[:cntrl:]@]/, '')
-page = open(ARGV[0] || url).read()
+
+page = RestClient.get(url).to_str
 
 Raspar.parse(url, page).each do |o|
   pp o

data/example/{dynamic.rb → leguide_dynamic.rb}

@@ -1,6 +1,6 @@
 require 'rubygems'
+require 'rest_client'
 require 'bundler/setup'
-require 'open-uri'
 require 'raspar'
 require 'pp'
 
@@ -38,13 +38,9 @@ selector_map = {
       :select => '.offers_list li',
       :attrs => {
         :image          => { :select => 'img', :prop => 'src'},
-        :price          => { :select => '.price .euro.gopt', :eval => :parse_price},
-        :orignal_price  => { :select => '.price .barre', :eval => :parse_price},
+        :price          => { :select => '.gopt .prices', :eval => :parse_price},
         :desc           => { :select => '.gopt.description,.info .description'},
         :vendor         => { :select => '.name a' },
-        :availability   => { :select => '.av', :prop => 'data-value', :eval => :data_attr_parse},
-        :delivery_time  => { :select => '.dv', :prop => 'data-value', :eval => :data_attr_parse},
-        :shipping_price => { :select => '.delivery.gopt'}
       }
     }
   }
@@ -52,8 +48,9 @@ selector_map = {
 
 Raspar.add(domain, selector_map, ParserHelper)
 
-url = 'http://www.leguide.com/sb/bp/5010500/hotpoint_ariston/ECO9F_149_FRS/55743410.htm'
-page = open(url).read()
+url = 'http://www.leguide.com/electromenager.htm'
+p url
+page = RestClient.get(url).to_str
 
 Raspar.parse(url, page).each do |i|
   pp i

data/example/sample_parser.rb

@@ -0,0 +1,116 @@
+require 'rubygems'
+require 'bundler/setup'
+require 'raspar'
+require 'pp'
+
+FAKE_PAGE = %q{
+  <!DOCTYPE html>
+  <html>
+  <body>
+
+  <span class="desc">Description</span>
+  <ul class="specs">
+    <li>Spec 1</li>
+    <li>Spec 2</li>
+    <li>Spec 3</li>
+  </ul>
+
+  <div>
+    <img src="1">
+    <span>Test1</span>
+    <span class="price">10</span>
+  </div>
+
+  <div>
+    <img src="2">
+    <span>Test2</span>
+    <span class="price">20</span>
+  </div>
+
+  <span class="second">
+    <img src="3">
+    <span>Test3</span>
+    <span class="price">30</span>
+  </span>
+
+  <div class="offer">
+    <span class="name">First Offer</span>
+    <span class="percentage">10% off</span>
+  </div>
+
+  </body>
+  </html>
+}
+
+class SampleParser
+  include Raspar
+
+  domain 'http://sample.com'
+
+  attr :desc, '.desc', :eval => :full_desc
+  attr :specs, '.specs li', :as => :array, :eval => :format_specs
+
+  collection :product, 'div,span.second' do
+    attr :image, 'img', :attr => 'src'
+    attr :image_url, 'img', :attr => 'src', :eval => :make_image_url
+    attr :name,  'span:first, .name', :eval => :full_name
+    attr :price, '.price', :eval => Proc.new{|i| i.to_i}
+    attr :all_text 
+    attr :price_map do |text, ele|
+      val = ele.search('span').collect{|s| s.content.strip}
+      {val[0] => val[1].to_f}
+    end
+  end
+
+  collection :offer, '.offer' do
+    attr :name, '.name'
+    attr :percentage, '.percentage'
+  end
+
+  def full_name(val, ele)
+    "Full Name: #{val}"
+  end
+
+  def name_price(val, ele)
+    val = ele.search('span').collect{|s| s.content.strip}
+    {val[0] => val[1].to_f}
+  end
+
+  def make_image_url(path, ele)
+    self.class.absolute_url(path)
+  end
+
+  def full_desc(text, ele)
+    "#{text} full desc"
+  end
+
+  def format_specs(text, ele)
+    text.downcase
+  end
+
+end
+
+##pp SampleParser.attrs
+pp Raspar.parse('http://sample.com', FAKE_PAGE)
+
+
+selector_map = {
+  :common_attrs => {
+    :desc => {:select => '.desc'}
+  },
+  :collections =>{
+    :product => {
+      :select => 'div, span.second', 
+      :attrs => {
+        :name =>  { :select => 'span:first'},
+        :price =>  { :select => 'span.price'},
+        :image => { :select => 'img', :attr => 'src'}
+      }
+    }
+  }
+}
+
+@site = 'http://dynmaicparser.com'
+@dynmaic_parser = Raspar.add(@site, selector_map)
+pp @dynmaic_parser.attrs
+pp Raspar.parse(@site, FAKE_PAGE)

data/lib/raspar.rb

@@ -55,7 +55,7 @@ module Raspar
     def parse(url, html)
       host = URI(url).host
       if @parsers[host]
-        @parsers[host].parse(html)
+        @parsers[host].parse(html).group_by(&:name)
       else
         puts "No parser define for #{host}"
         nil
@@ -83,8 +83,7 @@ module Raspar
       klass.domain(url)
       klass.class_exec(&block) if block_given?
 
-      klass_name = 'R' + klass_name if Object.const_defined?(klass_name)
-      Object.const_set(klass_name, klass)
+      Raspar.const_set(klass_name, klass)
     end
 
   end

data/lib/raspar/parser.rb

@@ -50,7 +50,7 @@ module Raspar
         @collections[collection_name] = { :select => select, :attrs => {} } 
 
         @_current_container_ = collection_name
-        yield
+        yield if block_given?
         @_current_container_ = nil
       end
 

data/lib/raspar/version.rb

@@ -1,3 +1,3 @@
 module Raspar
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/raspar.gemspec

@@ -19,5 +19,5 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   s.add_development_dependency "rspec"
-  s.add_dependency "nokogiri", "~> 1.5.5"
+  s.add_dependency "nokogiri", ">= 1.5.5"
 end

data/spec/add_parser_spec.rb

@@ -8,7 +8,7 @@ describe 'Add Parser' do
     Raspar.add(@site) do
       attr :desc, '.desc', :common => true
 
-      collection :product, '.item,span.second' do
+      collection :products, '.item,span.second' do
         attr :name,  'span:first, .name', :eval => :full_name
         attr :price, '.price', :eval => Proc.new{|i| i.to_i}
       end
@@ -27,29 +27,26 @@ describe 'Add Parser' do
   end
 
   it 'should register parser and parse data' do
-    Raspar.parsers.should include({@domain => @parser_class})
+    expect(Raspar.parsers).to include({@domain => @parser_class})
   end
 
   it "should have info" do
-    @parser_class.info.should == {
+    expect(@parser_class.info).to eq({
       :domain => @domain, 
-      :collections => [:product], 
+      :collections => [:products], 
       :common_attrs => [:desc] 
-    }
+    })
   end
 
   it "should parse html and create object" do
     parsed_objs = Raspar.parse(@site, FAKE_PAGE)
 
-    parsed_objs.length.should == 4
-
-    parsed_objs.count{|o| o.name == :product}.should == 4
-
+    expect(parsed_objs[:products].length).to eq(4)
     count = 1
-    parsed_objs.select{|o| o.name == :product}.each do |o|
-      o[:name].should == "Full Name: Test#{count}"
-      o[:price].should == (count * 10)
-      o[:desc].should == "Description"
+    parsed_objs[:products].each do |o|
+      expect(o[:name]).to eq("Full Name: Test#{count}")
+      expect(o[:price]).to eq(count * 10)
+      expect(o[:desc]).to eq("Description")
 
       count = count + 1
     end

data/spec/dynamic_parser_spec.rb

@@ -15,7 +15,7 @@ module Raspar
           :desc => {:select => '.desc'}
         },
         :collections =>{
-          :product => {
+          :products => {
             :select => 'div.item, span.second', 
             :attrs => {
               :name =>  { :select => 'span:first'},
@@ -33,9 +33,9 @@ module Raspar
     describe '#onload' do
 
       it "should register DynamicParser to Raspar parser list" do
-        Raspar.parsers[@domain].class.should == Raspar::DynamicParser
+        expect(Raspar.parsers[@domain].class).to eq(Raspar::DynamicParser)
 
-        @dynmaic_parser.domain.should == @domain
+        expect(@dynmaic_parser.domain).to eq(@domain)
       end
 
     end
@@ -44,28 +44,28 @@ module Raspar
 
       it "should parse html and create object" do
         parsed_objs = Raspar.parse(@site, FAKE_PAGE)
-        parsed_objs.length.should == 4
+        parsed_objs[:products] == 4
 
         count = 1
-        parsed_objs.each do |o|
-          o.class.should == Raspar::Result
+        parsed_objs[:products].each do |o|
+          expect(o.class).to eq(Raspar::Result)
 
-          o[:name].should == "Test#{count}"
-          o[:image].should == count.to_s
+          expect(o[:name]).to eq("Test#{count}")
+          expect(o[:image]).to eq(count.to_s)
 
           #Price should eval using proc given in option which convert string value
           #to integer
-          o[:price].should == (count * 10).to_s
+          expect(o[:price]).to eq((count * 10).to_s)
 
           #External Field check
-          o[:desc].should == "Description"
+          expect(o[:desc]).to eq("Description")
           count = count + 1
         end
 
       end
 
       it 'should return absoulte url' do
-        @dynmaic_parser.absolute_url('/test').should == @site + '/test'
+        expect(@dynmaic_parser.absolute_url('/test')).to eq(@site + '/test')
       end
 
     end

data/spec/parser_spec.rb

@@ -17,21 +17,21 @@ module Raspar
     describe 'onload' do
 
       it "should register SampleParser to Raspar parser list" do
-        Raspar.parsers.should include({@domain => SampleParser})
+        expect(Raspar.parsers).to include({@domain => SampleParser})
 
-        SampleParser.domain.should == @domain
+        expect(SampleParser.domain).to eq(@domain)
       end
 
       it 'should return absoulte url' do
-        SampleParser.absolute_url('/test').should == @site + '/test'
+        expect(SampleParser.absolute_url('/test')).to eq(@site + '/test')
       end
 
       it "should have info" do
-        SampleParser.info.should == {
+        expect(SampleParser.info).to eq({
                         :domain => @domain, 
-                        :collections => [:product, :offer], 
+                        :collections => [:products, :offers, :related_products], 
                         :common_attrs => [:desc, :specs] 
-                      }
+                      })
       end
 
       it "should not define accessor if options not contail :selector" do
@@ -46,36 +46,37 @@ module Raspar
         parsed_objs = Raspar.parse(@site, FAKE_PAGE)
 
         #Total parse objects
-        parsed_objs.length.should == 5
+        expect(parsed_objs.keys.length).to eq(3)
 
-        parsed_objs.count{|o| o.name == :product}.should == 4
-        parsed_objs.count{|o| o.name == :offer}.should == 1
+        expect(parsed_objs[:products].length).to eq(4)
+        expect(parsed_objs[:offers].length).to eq(1)
+        expect(parsed_objs[:related_products].length).to eq(1)
 
         count = 1
-        parsed_objs.select{|o| o.name == :product}.each do |o|
-          o[:name].should == "Full Name: Test#{count}"
-          o[:image].should == count.to_s
+        parsed_objs[:products].each do |o|
+          expect(o[:name]).to eq("Full Name: Test#{count}")
+          expect(o[:image]).to eq(count.to_s)
 
           #Price should eval using proc given in option which convert string value
           #to integer
-          o[:price].should == (count * 10)
+          expect(o[:price]).to eq(count * 10)
 
           #External Field check
-          o[:desc].should == "Description is full desc"
+          expect(o[:desc]).to eq("Description is full desc")
 
           #self selector 
-          o[:all_text].should == "Test#{count}\n    #{count*10}"
+          expect(o[:all_text]).to eq("Test#{count}\n    #{count*10}")
 
-          o[:price_map].should == {"Test#{count}" => (count*10).to_f}
+          expect(o[:price_map]).to eq({"Test#{count}" => (count*10).to_f})
 
-          o[:specs].should == ['spec 1', 'spec 2', 'spec 3']
+          expect(o[:specs]).to eq(['spec 1', 'spec 2', 'spec 3'])
 
           count = count + 1
         end
 
-        parsed_objs.select{|o| o.name == :offer}.each do |o|
-          o[:name].should == 'First Offer'
-          o[:percentage].should == '10% off'
+        parsed_objs[:offers].each do |o|
+          expect(o[:name]).to eq('First Offer')
+          expect(o[:percentage]).to eq('10% off')
         end
 
       end

data/spec/raspar_spec.rb

@@ -9,16 +9,16 @@ describe Raspar do
   end
 
   it "should add domain to register parser list" do
-    Raspar.register(@site, TestParser).should == @host
-    Raspar.parsers.should include({@host => TestParser})
-    Raspar.parsers.size.should > 0
+    expect(Raspar.register(@site, TestParser)).to eq(@host)
+    expect(Raspar.parsers).to include({@host => TestParser})
+    expect(Raspar.parsers.size).to be > 0
   end
 
   it "should clear registered domains" do
     Raspar.register(@site, TestParser)
     Raspar.clear_parser_list
 
-    Raspar.parsers.size.should == 0
+    expect(Raspar.parsers.size).to eq(0)
   end
 
   it "should able to remove parser from the registered list" do
@@ -27,7 +27,7 @@ describe Raspar do
 
     Raspar.remove(@site)
 
-    Raspar.parsers.should_not include(@host) 
+    expect(Raspar.parsers).not_to include(@host) 
   end
 end
 

data/spec/sample_parser.rb

@@ -6,7 +6,7 @@ class SampleParser
   attr :desc, '.desc', :common => true, :eval => :full_desc
   attr :specs, '.specs li', :common => true, :as => :array, :eval => :format_specs
 
-  collection :product, '.item,span.second' do
+  collection :products, '.item,span.second' do
     attr :image, 'img', :prop => 'src'
     attr :image_url, 'img', :prop => 'src', :eval => :make_image_url
     attr :name,  'span:first, .name', :eval => :full_name
@@ -18,11 +18,15 @@ class SampleParser
     end
   end
 
-  collection :offer, '.offer' do
+  collection :offers, '.offer' do
     attr :name, '.name'
     attr :percentage, '.percentage'
   end
 
+  collection :related_products, 'ol.related_products' do
+    attr :name, 'li', as: :array
+  end
+
   def full_name(val, ele)
     "Full Name: #{val}"
   end
@@ -87,6 +91,12 @@ FAKE_PAGE = %q{
     <span class="percentage">10% off</span>
   </div>
 
+  <ol class="related_products">
+    <li> Product 1 </li>
+    <li> Product 2 </li>
+    <li> Product 3 </li>
+  </ol>
+
   </body>
   </html>
 }

data/spec/spec_helper.rb

@@ -14,7 +14,7 @@ require 'coveralls'
 Coveralls.wear!
 
 RSpec.configure do |config|
-  config.color_enabled = true
+  #config.color_enabled = true
   #config.tty = true
   #config.formatter = :documentation
 end

metadata

@@ -1,41 +1,41 @@
 --- !ruby/object:Gem::Specification
 name: raspar
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Jiren Patel
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-01 00:00:00.000000000 Z
+date: 2014-06-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.5.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.5.5
 description: Raspar collects data from the html page and creates object from it.
@@ -45,17 +45,19 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".gitignore"
+- ".travis.yml"
 - Gemfile
 - LICENSE
 - README.md
 - Rakefile
 - example/Gemfile
-- example/ccode.rb
+- example/best_buy.rb
+- example/country_code.rb
 - example/debug.rb
-- example/dynamic.rb
 - example/leguide.rb
+- example/leguide_dynamic.rb
+- example/sample_parser.rb
 - lib/raspar.rb
 - lib/raspar/dynamic_parser.rb
 - lib/raspar/parser.rb
@@ -77,17 +79,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: raspar
-rubygems_version: 2.0.3
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: A generic html/xml parser