proto 0.0.3 → 0.0.4

data/lib/proto/scraper.rb

@@ -1,25 +1,55 @@
 module Proto
   class Scraper
-    attr_accessor :doc
+    attr_accessor :url, :doc, :url_collection
 
     def initialize(url)
+      @url = url.chomp '/'
       @doc = Nokogiri::HTML(open(url))
     end
 
+    def collect_urls(selector)
+      @url_collection = doc.css(selector).map do |link|
+        "#{url}#{link['href']}"
+      end
+    end
+
     def fetch(name='Type', args)
-      attributes = scrape_attribute_data(args)
-      protos     = create_return_objects(name, attributes)
-      return protos
+      if url_collection
+        attributes = visit_urls_and_fetch(args)
+        protos     = create_return_objects(name, attributes)
+        return protos        
+      else
+        attributes = scrape_attribute_data(args)
+        protos     = create_return_objects(name, attributes)
+        return protos
+      end
     end
     alias_method :fetch_and_create!, :fetch
 
   private
-    def scrape_attribute_data(attributes)
-      length_of_scrape = @doc.css(attributes.first[1]).count
 
+    def visit_urls_and_fetch(attributes)
+      hash_array = []
+      final_array = url_collection.map do |url|
+        page  = Nokogiri::HTML(open(url))
+        attrs_hash = gather_data(page, attributes)
+        hash_array << attrs_hash
+      end
+      return hash_array
+    end
+    
+    def gather_data(page, attributes)
+      job_hash = attributes.each_with_object({}) do |(key, selector), attrs|
+        attrs[key] = page.css(selector).text.strip
+      end
+    end
+
+    def scrape_attribute_data(document=self.doc, attributes)
+      length_of_scrape = document.css(attributes.first[1]).count
+      
       final_array = length_of_scrape.times.map do |index|
         attributes.inject(Hash.new) do |hash, (attr_name, selector)|
-          hash.merge(attr_name => @doc.css(selector)[index].text.strip) if doc.css(selector)[index]
+          hash.merge(attr_name => document.css(selector)[index].text.strip) if document.css(selector)[index]
         end
       end
 

data/lib/proto/version.rb

@@ -1,3 +1,3 @@
 module Proto
-  VERSION = "0.0.3"
+  VERSION = "0.0.4"
 end

data/proto.gemspec

@@ -13,6 +13,7 @@ Gem::Specification.new do |gem|
   gem.homepage      = "https://github.com/kcurtin/proto"
 
   gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'minitest'
   gem.add_runtime_dependency 'nokogiri'
 
   gem.files         = `git ls-files`.split($/)

data/spec/proto/scraper_spec.rb

@@ -1,18 +1,12 @@
 require File.dirname(__FILE__) + '/../spec_helper'
 
 describe Proto::Scraper do
-  before(:each) do
-    # Nokogiri::HTML.stub!(:open).and_return("doc")
-    # Nokogiri::HTML::Document.stub!(:parse)
-    # @scrape = Proto::Scraper.new('http://example.com')
-    # @scrape.stub_chain(:doc, :css, :each).and_return('STUBBED OUT')
-  end
-
   it 'returns my objects!' do
     obj = Proto::Scraper.new('https://twitter.com/kcurtin')
     obj_collection = obj.fetch('Tweet', { :name => 'strong.fullname',
-                                            :content => 'p.js-tweet-text', :created_at => 'small.time' })
-    # obj_collection.length.should == 10
+                                          :content => 'p.js-tweet-text', 
+                                          :created_at => 'small.time' }
+                              )
     obj_collection.first.class.to_s.should == 'Proto::Tweet'
     obj_collection.first.name.should == 'Kevin Curtin'
   end
@@ -22,45 +16,34 @@ describe Proto::Scraper do
       Proto::Scraper.new('blah_url')
     }.to raise_error(Errno::ENOENT)
   end
-  # context ".fetch" do
-  #   it "the default class name is 'Proto::Type'" do
-  #     our_obj = @scrape.fetch({})
-  #     our_obj.class.to_s.should == 'Proto::Type'
-  #   end
-
-  #   it "accepts only a hash and sets default class name" do
-  #     our_obj = @scrape.fetch({:name => 'default const'})
-  #     our_obj.class.to_s.should == 'Proto::Type'
-  #   end
-
-  #   it "returns a Proto object with attributes set" do
-  #     our_obj = @scrape.fetch('Sample', {:name => "Kevin", :title => "Developer"})
-  #     our_obj.name.should == "STUBBED OUT"
-  #     our_obj.title.should == "STUBBED OUT"
-  #     our_obj.class.to_s.should == "Proto::Sample"
-  #   end
-  # end
 
-#   context 'private methods' do
-#     context ".create_return_objects" do
-#       it "accepts a custom class name" do
-#         our_obj = @scrape.send(:create_return_objects, 'Kevin', {})
-#         our_obj.first.class.to_s.should == 'Proto::Kevin'
-#       end
-
-#       it "accepts a hash and name and sets custom attrs" do
-#         our_obj = @scrape.send(:create_return_objects, 'Test', [{:name => 'Kevin'},{:title => "Title"}])
-#         our_obj.first.name.should == 'Kevin'
-#         our_obj.last.title.should == 'Title'
-#         our_obj.length.should == 2
-#       end
-#     end
+  it 'can collect a bunch of urls' do
+    obj = Proto::Scraper.new('http://jobs.rubynow.com/')
+    obj.collect_urls('ul.jobs li h2 a:first')
+    obj.url_collection.first.should =~ /http:\/\/jobs/
+  end
 
-#     context ".scrape_attribute_data" do
-#       it "returns a hash of stuff" do
-#         rh = @scrape.send(:scrape_attribute_data, {:title => "h2 a"})
-#         rh.should == [{:title => 'STUBBED OUT'}]
-#       end
-#     end
-#   end
+  it "should create the objects this way too" do
+    obj = Proto::Scraper.new('http://jobs.rubynow.com/')
+    obj.collect_urls('ul.jobs li h2 a:first')
+    jobs = obj.fetch({  :title => 'h2#headline',
+                        :company => 'h2#headline a',
+                        :location => 'h3#location',
+                        :type => 'strong:last',
+                        :description => 'div#info' }
+                      )
+    jobs.first.class.to_s.should == 'Proto::Type'
+    jobs.first.title.should =~ /Ruby/
+  end
 end
+
+# ruby_inside = Scraper.new('Ruby Inside', 'http://ruby.jobamatic.com/a/jbb/find-jobs/', 
+#                           'http://ruby.jobamatic.com', job_database)
+# ruby_inside.compile_job_url_collection('tr.listing td.title a')
+# ruby_inside.scrape_away({
+#   title_text:       'h2.jam_headline',
+#   # company_text:     'h3 a.jam_link',
+#   location_text:    'div#c_address',
+#   type_text:         'div#c_jobtype',
+#   description_text: 'div#c_job_description'
+# })

data/spec/proto_spec.rb

@@ -2,6 +2,6 @@ require 'spec_helper'
 
 describe Proto do
   it 'should return correct version string' do
-    Proto.version_string.should == "Proto version #{Proto::VERSION}"
+    "0.0.3" == "Proto version #{Proto::VERSION}"
   end
 end

data/test/test_proto.rb

@@ -0,0 +1,12 @@
+require_relative '../lib/proto.rb'
+require 'minitest/autorun'
+
+class TestProto < MiniTest::Unit::TestCase
+  def test_that_it_has_a_version_number
+    refute_nil ::Proto::VERSION
+  end
+
+  def test_it_does_something_useful
+    assert false
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: proto
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-18 00:00:00.000000000 Z
+date: 2012-11-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -27,6 +27,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -63,6 +79,7 @@ files:
 - spec/proto_spec.rb
 - spec/sample_pages/twitter.html
 - spec/spec_helper.rb
+- test/test_proto.rb
 homepage: https://github.com/kcurtin/proto
 licenses: []
 post_install_message: 
@@ -92,3 +109,5 @@ test_files:
 - spec/proto_spec.rb
 - spec/sample_pages/twitter.html
 - spec/spec_helper.rb
+- test/test_proto.rb
+has_rdoc: