scrapify 0.0.6 → 0.0.7

data/examples/pizza.rb

@@ -0,0 +1,10 @@
+class Pizza
+  include Scrapify::Base
+  html "http://www.dominos.co.in/menuDetails_ajx.php?catgId=1"
+
+  attribute :name, css: ".menu_lft li a"
+  attribute :image_url, xpath: "//li//input//@value"
+  attribute :price, css: ".price", regex: /([\d\.]+)/
+
+  key :name
+end

data/lib/scrapify.rb

@@ -1,9 +1,10 @@
 require 'scrapify/version'
 require 'active_support/core_ext/class/attribute_accessors'
+require 'active_support/core_ext/hash/keys'
 require 'nokogiri'
 require 'uri'
 require 'net/http'
-require 'meta_define'
 require 'scrapify/base'
+require 'scrapify/scraper'
 require 'json'
 require 'jsonify'

data/lib/scrapify/base.rb

@@ -27,11 +27,12 @@ module Scrapify
 
       def attribute(name, options={}, &block)
         add_attribute(name)
+        options = options.symbolize_keys
         parser = options[:xpath] ? :xpath : :css
         selector = options[parser]
         matcher = /#{options[:regex]}/ if options[:regex]
         to_array = options[:array]
-        meta_define "#{name}_values" do
+        define_singleton_method "#{name}_values" do
           self.doc ||= parse_html
           self.doc.send(parser, selector).map do |element|
             if block
@@ -86,21 +87,21 @@ module Scrapify
       end
 
       def define_finders
-        meta_define :all do
+        define_singleton_method :all do
           count.times.map do |index|
             find_by_index index
           end
         end
 
-        meta_define :first do
+        define_singleton_method :first do
           find_by_index 0
         end
 
-        meta_define :last do
+        define_singleton_method :last do
           find_by_index count - 1
         end
 
-        meta_define :find_by_index do |index|
+        define_singleton_method :find_by_index do |index|
           return if index.nil? or index < 0
           attributes = Hash[attribute_names.map {|attribute| [attribute, send("#{attribute}_values")[index]]}]
           self.new(attributes)
@@ -108,13 +109,13 @@ module Scrapify
       end
 
       def define_count(key_attribute)
-        meta_define :count do
+        define_singleton_method :count do
           send("#{key_attribute}_values").size
         end
       end
 
       def define_find_by_id(key_attribute)
-        meta_define :find do |key_value|
+        define_singleton_method :find do |key_value|
           index = send("#{key_attribute}_values").index(key_value)
           find_by_index index
         end

data/lib/scrapify/scraper.rb

@@ -0,0 +1,24 @@
+module Scrapify
+  class Scraper
+    def initialize(url, key_attr, attributes)
+      @scraper = Class.new
+      @scraper.send(:include, Scrapify::Base)
+      @scraper.html url
+      @scraper.key key_attr.to_sym
+      attributes.symbolize_keys.each do |attr, options|
+        options = options.symbolize_keys
+        custom_block = options.delete(:block)
+        @scraper.attribute attr, options, &custom_block
+      end
+    end
+
+    def method_missing(method, *args, &block)
+      return @scraper.send(method, *args, &block) if @scraper.respond_to?(method)
+      super
+    end
+
+    def respond_to?(method, include_private=false)
+      @scraper.respond_to?(method, include_private) || super
+    end
+  end
+end

data/lib/scrapify/version.rb

@@ -1,3 +1,3 @@
 module Scrapify
-  VERSION = "0.0.6"
+  VERSION = "0.0.7"
 end

data/spec/pizza.rb

@@ -8,7 +8,7 @@ class Pizza
   attribute :ingredients, css: ".ingredients", regex: /contains (\w+)/, array: true
   attribute :ingredient_urls, css: '.references ol li' do |element|
     element.children.map do |child|
-        child.attributes['href'].value if child.attributes['href']
+      child.attributes['href'].value if child.attributes['href']
     end.compact
   end
 

data/spec/scraper_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+require 'test_models'
+require 'active_support/core_ext/hash/keys'
+
+describe Scrapify::Scraper do
+  def self.attributes
+    {:name => {:css => ".menu_lft li a"},
+                :image_url => {:xpath => "//li//input//@value"},
+                :price => {:css => ".price", :regex => /([\d\.]+)/},
+                :ingredients => {:css => ".ingredients", :regex => /contains (\w+)/, :array => true},
+                :ingredient_urls => {:css => ".references ol li",
+                                     :block => lambda{|element| element.children.map{|child| child.attributes['href'].value if child.attributes['href']}.compact}
+                                    }}
+  end
+
+  def self.attributes_with_string_keys
+    attributes.each_with_object({}){|(k,v),h| h[k.to_s] = v.stringify_keys}
+  end
+
+  it_should_behave_like "Scrapify", Scrapify::Scraper.new(::Pizza.url, :name, attributes)
+  it_should_behave_like "Scrapify", Scrapify::Scraper.new(::Pizza.url, 'name', attributes_with_string_keys)
+end

data/spec/scrapify_spec.rb

@@ -2,181 +2,5 @@ require 'spec_helper'
 require 'test_models'
 
 describe Scrapify do
-
-  before do
-    @pizza_url = "http://www.dominos.co.in/menuDetails_ajx.php?catgId=1"
-    FakeWeb.register_uri :get, @pizza_url,
-                         :cache_control => "private, s-maxage=0, max-age=0, must-revalidate",
-                         :age           => 51592,
-                         :length        => 12312,
-                         :body          => <<-HTML
-      <ul class="menu_lft">
-        <li>
-          <a>  chicken supreme  </a><input value="chicken.jpg">
-          <span class='price'>(1.23)</span>
-          <span class='ingredients'>
-            <ol>
-              <li>contains corn</li>
-              <li>contains tomato</li>
-            <ol>
-          </span>
-          <span class='references'><ol><li></li></ol></span
-        </li>
-        <li>
-          <a>veg supreme</a><input value="veg.jpg">
-          <span class='price'>(2.34)</span>
-          <span class='ingredients'>
-            <ol>
-              <li>contains mushroom</li>
-              <li>contains jalapeno</li>
-            <ol>
-          </span>
-          <span class='references'><ol><li></li></ol></span
-        </li>
-        <li>
-          <a>pepperoni</a><input value="pepperoni.jpg">
-          <span class='price'>(3.45)</span>
-          <span class='ingredients'></span>
-          <span class='references'><ol><li></li></ol></span
-        </li>
-        <li>
-          <a>chicken golden delight</a><input value="golden.jpg">
-          <span class='price'>(4.56)</span>
-          <span class='ingredients'/>
-          <span class='references'>
-            <ol>
-              <li>
-                <div href='chicken.html'>chicken</div>
-                <div href='delight.html'>delight</div>
-              </li>
-            </ol>
-          </span>
-        </li>
-      </ul>
-    HTML
-  end
-
-  it "should return attribute names" do
-    ::Pizza.attribute_names.should == [:name, :image_url, :price, :ingredients, :ingredient_urls]
-  end
-
-  describe "html" do
-    it "should store url" do
-      ::Pizza.url.should == @pizza_url
-    end
-
-    it "should parse html and fetch attributes using css" do
-      ::Pizza.name_values.should == ['chicken supreme', 'veg supreme', 'pepperoni', 'chicken golden delight']
-    end
-
-    it "should parse html and fetch attributes using xpath" do
-      ::Pizza.image_url_values.should == ['chicken.jpg', 'veg.jpg', 'pepperoni.jpg', 'golden.jpg']
-    end
-
-    it "should parse html and extract attributes using regex" do
-      ::Pizza.price_values.should == ['1.23', '2.34', '3.45', '4.56']
-    end
-
-    it "should parse html and extract multiple attributes using regex" do
-      ::Pizza.ingredients_values.should == [['corn','tomato'], ['mushroom','jalapeno'], [], []]
-    end
-
-    it 'should accept block to yield attribute values' do
-      ::Pizza.ingredient_urls_values.should == [[], [], [], ['chicken.html', 'delight.html']]
-    end
-
-    it "should strip content" do
-      ::Pizza.first.name.should == 'chicken supreme'
-    end
-
-    describe "cache headers" do
-      it "should return the http headers" do
-        ::Pizza.http_cache_header.should == {
-           "cache-control" => "private, s-maxage=0, max-age=0, must-revalidate",
-           "age"           => 51592,
-        }
-      end
-    end
-  end
-
-  describe "find" do
-    it "should find element by key" do
-      pizza = ::Pizza.find('pepperoni')
-      pizza.should_not be_nil
-      pizza.name.should == 'pepperoni'
-      pizza.image_url.should == 'pepperoni.jpg'
-    end
-
-    it "should be nil if element does not exist" do
-      pizza = ::Pizza.find('mushroom')
-      pizza.should be_nil
-    end
-  end
-
-  describe "first" do
-    it "should fetch first matching element" do
-      first_pizza = ::Pizza.first
-      first_pizza.name.should == 'chicken supreme'
-      first_pizza.image_url.should == 'chicken.jpg'
-    end
-  end
-
-  describe "last" do
-    it "should fetch last matching element" do
-      last_pizza = ::Pizza.last
-      last_pizza.name.should == 'chicken golden delight'
-      last_pizza.image_url.should == 'golden.jpg'
-    end
-  end
-
-  describe "all" do
-    it "should fetch all objects" do
-      pizzas = ::Pizza.all
-      pizzas.size.should == 4
-      pizzas.map(&:name).should == ['chicken supreme', 'veg supreme', 'pepperoni', 'chicken golden delight']
-      pizzas.map(&:image_url).should == ['chicken.jpg', 'veg.jpg', 'pepperoni.jpg', 'golden.jpg']
-    end
-  end
-
-  describe "count" do
-    it "should return number of matching elements" do
-      ::Pizza.count.should == 4
-    end
-  end
-
-  describe "attributes" do
-    it "should return attributes hash" do
-      first_pizza = ::Pizza.first
-      first_pizza.attributes.should == {
-        name: "chicken supreme",
-        image_url: "chicken.jpg",
-        price: '1.23',
-        ingredients: ['corn', 'tomato'],
-        ingredient_urls: []
-      }
-    end
-  end
-
-  describe "to_json" do
-    it "should convert attributes to json" do
-      first_pizza = ::Pizza.first
-      first_pizza.to_json.should == {
-        name: "chicken supreme",
-        image_url: "chicken.jpg",
-        price: '1.23',
-        ingredients: ['corn', 'tomato'],
-        ingredient_urls: []
-      }.to_json
-    end
-
-    it "should convert array to json" do
-      pizzas = ::Pizza.all
-      pizzas.to_json.should == [
-        {name: "chicken supreme", image_url: "chicken.jpg", price: '1.23', ingredients: ['corn', 'tomato'], :ingredient_urls => []},
-        {name: "veg supreme", image_url: "veg.jpg", price: '2.34', ingredients: ['mushroom', 'jalapeno'], :ingredient_urls => []},
-        {name: "pepperoni", image_url: "pepperoni.jpg", price: '3.45', ingredients: [], :ingredient_urls => []},
-        {name: "chicken golden delight", image_url: "golden.jpg", price: '4.56', ingredients: [], :ingredient_urls => ['chicken.html', 'delight.html']},
-      ].to_json
-    end
-  end
+  it_should_behave_like "Scrapify", ::Pizza
 end

data/spec/shared/scrapify.rb

@@ -0,0 +1,178 @@
+shared_examples_for "Scrapify" do |klass_or_object|
+  before do
+    @pizza_url = "http://www.dominos.co.in/menuDetails_ajx.php?catgId=1"
+    FakeWeb.register_uri :get, @pizza_url,
+                         :cache_control => "private, s-maxage=0, max-age=0, must-revalidate",
+                         :age           => 51592,
+                         :length        => 12312,
+                         :body          => <<-HTML
+      <ul class="menu_lft">
+        <li>
+          <a>  chicken supreme  </a><input value="chicken.jpg">
+          <span class='price'>(1.23)</span>
+          <span class='ingredients'>
+            <ol>
+              <li>contains corn</li>
+              <li>contains tomato</li>
+            <ol>
+          </span>
+          <span class='references'><ol><li></li></ol></span
+        </li>
+        <li>
+          <a>veg supreme</a><input value="veg.jpg">
+          <span class='price'>(2.34)</span>
+          <span class='ingredients'>
+            <ol>
+              <li>contains mushroom</li>
+              <li>contains jalapeno</li>
+            <ol>
+          </span>
+          <span class='references'><ol><li></li></ol></span
+        </li>
+        <li>
+          <a>pepperoni</a><input value="pepperoni.jpg">
+          <span class='price'>(3.45)</span>
+          <span class='ingredients'></span>
+          <span class='references'><ol><li></li></ol></span
+        </li>
+        <li>
+          <a>chicken golden delight</a><input value="golden.jpg">
+          <span class='price'>(4.56)</span>
+          <span class='ingredients'/>
+          <span class='references'>
+            <ol>
+              <li>
+                <div href='chicken.html'>chicken</div>
+                <div href='delight.html'>delight</div>
+              </li>
+            </ol>
+          </span>
+        </li>
+      </ul>
+    HTML
+  end
+
+  it "should return attribute names" do
+    klass_or_object.attribute_names.should == [:name, :image_url, :price, :ingredients, :ingredient_urls]
+  end
+
+  describe "html" do
+    it "should store url" do
+      klass_or_object.url.should == @pizza_url
+    end
+
+    it "should parse html and fetch attributes using css" do
+      klass_or_object.name_values.should == ['chicken supreme', 'veg supreme', 'pepperoni', 'chicken golden delight']
+    end
+
+    it "should parse html and fetch attributes using xpath" do
+      klass_or_object.image_url_values.should == ['chicken.jpg', 'veg.jpg', 'pepperoni.jpg', 'golden.jpg']
+    end
+
+    it "should parse html and extract attributes using regex" do
+      klass_or_object.price_values.should == ['1.23', '2.34', '3.45', '4.56']
+    end
+
+    it "should parse html and extract multiple attributes using regex" do
+      klass_or_object.ingredients_values.should == [['corn','tomato'], ['mushroom','jalapeno'], [], []]
+    end
+
+    it 'should accept block to yield attribute values' do
+      klass_or_object.ingredient_urls_values.should == [[], [], [], ['chicken.html', 'delight.html']]
+    end
+
+    it "should strip content" do
+      klass_or_object.first.name.should == 'chicken supreme'
+    end
+
+    describe "cache headers" do
+      it "should return the http headers" do
+        klass_or_object.http_cache_header.should == {
+           "cache-control" => "private, s-maxage=0, max-age=0, must-revalidate",
+           "age"           => 51592,
+        }
+      end
+    end
+  end
+
+  describe "find" do
+    it "should find element by key" do
+      pizza = klass_or_object.find('pepperoni')
+      pizza.should_not be_nil
+      pizza.name.should == 'pepperoni'
+      pizza.image_url.should == 'pepperoni.jpg'
+    end
+
+    it "should be nil if element does not exist" do
+      pizza = klass_or_object.find('mushroom')
+      pizza.should be_nil
+    end
+  end
+
+  describe "first" do
+    it "should fetch first matching element" do
+      first_pizza = klass_or_object.first
+      first_pizza.name.should == 'chicken supreme'
+      first_pizza.image_url.should == 'chicken.jpg'
+    end
+  end
+
+  describe "last" do
+    it "should fetch last matching element" do
+      last_pizza = klass_or_object.last
+      last_pizza.name.should == 'chicken golden delight'
+      last_pizza.image_url.should == 'golden.jpg'
+    end
+  end
+
+  describe "all" do
+    it "should fetch all objects" do
+      pizzas = klass_or_object.all
+      pizzas.size.should == 4
+      pizzas.map(&:name).should == ['chicken supreme', 'veg supreme', 'pepperoni', 'chicken golden delight']
+      pizzas.map(&:image_url).should == ['chicken.jpg', 'veg.jpg', 'pepperoni.jpg', 'golden.jpg']
+    end
+  end
+
+  describe "count" do
+    it "should return number of matching elements" do
+      klass_or_object.count.should == 4
+    end
+  end
+
+  describe "attributes" do
+    it "should return attributes hash" do
+      first_pizza = klass_or_object.first
+      first_pizza.attributes.should == {
+        name: "chicken supreme",
+        image_url: "chicken.jpg",
+        price: '1.23',
+        ingredients: ['corn', 'tomato'],
+        ingredient_urls: []
+      }
+    end
+  end
+
+  describe "to_json" do
+    it "should convert attributes to json" do
+      first_pizza = klass_or_object.first
+      first_pizza.to_json.should == {
+        name: "chicken supreme",
+        image_url: "chicken.jpg",
+        price: '1.23',
+        ingredients: ['corn', 'tomato'],
+        ingredient_urls: []
+      }.to_json
+    end
+
+    it "should convert array to json" do
+      pizzas = klass_or_object.all
+      pizzas.to_json.should == [
+        {name: "chicken supreme", image_url: "chicken.jpg", price: '1.23', ingredients: ['corn', 'tomato'], :ingredient_urls => []},
+        {name: "veg supreme", image_url: "veg.jpg", price: '2.34', ingredients: ['mushroom', 'jalapeno'], :ingredient_urls => []},
+        {name: "pepperoni", image_url: "pepperoni.jpg", price: '3.45', ingredients: [], :ingredient_urls => []},
+        {name: "chicken golden delight", image_url: "golden.jpg", price: '4.56', ingredients: [], :ingredient_urls => ['chicken.html', 'delight.html']},
+      ].to_json
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -4,7 +4,8 @@ require 'rspec/mocks'
 require 'fakeweb'
 
 require 'scrapify'
+Dir["./spec/shared/*.rb"].each {|f| require File.expand_path(f)}
 
 RSpec.configure do |config|
   config.mock_with :mocha
-end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scrapify
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-19 00:00:00.000000000Z
+date: 2012-06-28 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70142157555980 !ruby/object:Gem::Requirement
+  requirement: &70233941360460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70142157555980
+  version_requirements: *70233941360460
 - !ruby/object:Gem::Dependency
   name: mocha
-  requirement: &70142157554660 !ruby/object:Gem::Requirement
+  requirement: &70233941359980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70142157554660
+  version_requirements: *70233941359980
 - !ruby/object:Gem::Dependency
   name: fakeweb
-  requirement: &70142157538940 !ruby/object:Gem::Requirement
+  requirement: &70233941359460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70142157538940
+  version_requirements: *70233941359460
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70142157538080 !ruby/object:Gem::Requirement
+  requirement: &70233941358800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70142157538080
+  version_requirements: *70233941358800
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &70142157537240 !ruby/object:Gem::Requirement
+  requirement: &70233941358220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70142157537240
+  version_requirements: *70233941358220
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70142157536340 !ruby/object:Gem::Requirement
+  requirement: &70233941351380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,7 +76,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70142157536340
+  version_requirements: *70233941351380
 description: ScrApify scraps static html sites to RESTlike APIs
 email:
 - sathish316@gmail.com
@@ -88,15 +88,18 @@ files:
 - Gemfile
 - README.md
 - Rakefile
+- examples/pizza.rb
 - lib/jsonify.rb
-- lib/meta_define.rb
 - lib/scrapify.rb
 - lib/scrapify/base.rb
+- lib/scrapify/scraper.rb
 - lib/scrapify/version.rb
 - scrapify.gemspec
 - spec/jsonify_spec.rb
 - spec/pizza.rb
+- spec/scraper_spec.rb
 - spec/scrapify_spec.rb
+- spec/shared/scrapify.rb
 - spec/spec_helper.rb
 - spec/test_models.rb
 homepage: http://www.github.com/sathish316/scrapify
@@ -126,6 +129,8 @@ summary: ScrApify scraps static html sites to scraESTlike APIs
 test_files:
 - spec/jsonify_spec.rb
 - spec/pizza.rb
+- spec/scraper_spec.rb
 - spec/scrapify_spec.rb
+- spec/shared/scrapify.rb
 - spec/spec_helper.rb
 - spec/test_models.rb

data/lib/meta_define.rb

@@ -1,5 +0,0 @@
-class Object # http://whytheluckystiff.net/articles/seeingMetaclassesClearly.html
-  def meta_define name, &blk
-    (class << self; self; end).instance_eval { define_method name, &blk }
-  end
-end