craigslister 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d851b7d0a7f00c9cb5493bb1b4ffdb38c1752d2e
-  data.tar.gz: 80e986b0805484b1f8f8d33c8836547d9b05671b
+  metadata.gz: d677df8e37c7dd139665e441aa6581a34ad5b5ed
+  data.tar.gz: 93442d5a1e4fbbd4f9500f42f53e17322cda7d18
 SHA512:
-  metadata.gz: ee0e39625bde28bfb1d5003804213f1d3ed42c75a2303dcf691a0551d350979167641210768e54ec1e2ad5e77b0c909d9c63e0dd9663d7782c1e9037ddf17520
-  data.tar.gz: 123d6828752eeedd630360760300541c695c598be0901bd73fbd6b95c2df1bb95c8b1acf9245c84240b2a8498aa3f726d739488cff424066d4915155271cada0
+  metadata.gz: 16bde37399ef62404cbfc5ac487bb75aaecc42b5fa1b28297dcce92d83fe4c9f19dbb38a8732b2b2a9686a7efd710bcb537e12c93fc081b9423cb797704e6824
+  data.tar.gz: eda7e0e61f236749dbeabc34d0929e345a9f46d643030c705eabea2b811a7984f4143337d70594b2f663c84193cf0e4cdd52b87683f02f478bd73fab1b8e6467

data/lib/craigslister.rb

@@ -1,8 +1,5 @@
-# Add Item#url
-# organize spec files
-# figure out how to run spec from any folder
-
-require 'mechanize'
+require 'nokogiri'
+require 'open-uri'
 
 class InvalidRangeError < StandardError
 end
@@ -11,31 +8,32 @@ class Craigslister
   attr_reader :area, :item, :high, :low, :results
 
   def initialize args
-    @results        = []
-    @area           = args.fetch(:area, 'sfbay')
-    @item           = args[:item]
-    @high           = args.fetch(:high, nil)
-    @low            = args.fetch(:low, nil)
+    @results = []
+    @area    = args.fetch(:area, 'sfbay')
+    @item    = args[:item]
+    @high    = args.fetch(:high, nil)
+    @low     = args.fetch(:low, nil)
     validate_price_range
-
-    @mech = Mechanize.new
-    configure_mech
   end
 
   def scrape!
-    links.map {|link| get_data_from(link)}.compact
+    links.each_with_index {|link, index| get_data_from(link, index)}
+    results
   end
 
   def url
     "#{base_url}/"\
     "search/sss?sort=rel&"\
     "#{price_query}"\
-    "query=#{item.downcase.split(' ') * '+'}"\
+    "query=#{item.downcase.split(' ') * '+'}"
   end
 
   def links
-    @mech.get(url)
-    @mech.page.search('.hdrlnk').map {|link| link['href']}
+    page = Nokogiri::HTML(open(url))
+    page.css('.hdrlnk').map do |link|
+      # formats out of town links, otherwise use base_url + link
+      link['href'] =~ /\w+\.craig/ ? "https:" + link['href'] : base_url + link['href']
+    end
   end
 
 
@@ -44,19 +42,19 @@ class Craigslister
       "https://#{area}.craigslist.org"
     end
 
-    def get_data_from link
-      @mech.get(link)
-      @results << Item.new(scrape_item_data(link)) rescue p 'No Image'
+    def get_data_from link, index
+      page = Nokogiri::HTML(open(link))
+      @results << Item.new(scrape_item_data(page, link)) rescue puts "No image for post ##{index+1}"
     end
 
-    def scrape_item_data url
+    def scrape_item_data page, url
       {
-        image: @mech.page.images[0].src,
-        title: @mech.page.at('span.postingtitletext').text.gsub(/ ?- ?\$\d+ ?\(.+\)/, ''),
-        price: @mech.page.at('span.postingtitletext span.price').text.gsub(/\$/,'').to_i,
-        location: @mech.page.at('span.postingtitletext small').text.gsub(/ ?[\(\)]/,''),
-        description: @mech.page.at('section#postingbody').text,
-        url: "#{base_url}#{url}"
+        image: page.at('img')['src'],
+        title: page.at('span.postingtitletext').text.gsub(/ ?- ?\$\d+ ?\(.+\)/, ''),
+        price: page.at('span.postingtitletext span.price').text.gsub(/\$/,'').to_i,
+        location: page.at('span.postingtitletext small').text.gsub(/ ?[\(\)]/,''),
+        description: page.at('section#postingbody').text,
+        url: url
       }
     end
 
@@ -67,13 +65,8 @@ class Craigslister
       result
     end
 
-    def configure_mech
-      @mech.robots = false
-      @mech.user_agent_alias = 'Mac Safari'
-    end
-
     def validate_price_range
-      raise InvalidRangeError if (low && high) && low > high
+      raise InvalidRangeError if low && high && low > high
     end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: craigslister
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - Chris Scott
@@ -9,7 +9,27 @@ autorequire:
 bindir: bin
 cert_chain: []
 date: 2015-10-27 00:00:00.000000000 Z
-dependencies: []
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.6.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.6.2
 description: all you need is an item title and you can scrape item objects from craigslist
 email: christo247@gmail.com
 executables: []