craigslister 2.0.0 → 2.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6e1d5d4462a643f2ffcc73f70e2fe18c26f2cd05
-  data.tar.gz: 784647815b81ed5fcb46cf717c57b3fe85a29058
+  metadata.gz: 83f273dd97db5b9e9322851b3b69a5818194c39e
+  data.tar.gz: a53505d6d77de5be42ff950c88d9379289fb3e8a
 SHA512:
-  metadata.gz: 07101df2df31e3acf6b4d5b14d44c4d9ad00cf13b72d0d5ee6e3816a1f57cc2236ef8d501f6b380956b7d39c8c0c1eb345caf65735bf08e621a00bd7069798c0
-  data.tar.gz: 2db66d6ac5ccf8de5f25d6393b0d721f36307e2d008717de5dba6d2849799e75322089dda412347ae41ed2bea9f80c18a352d5b00358b8937be00a2c480c03dc
+  metadata.gz: 813c4dc83d6f84642167220e7f8f25a5bff5aef84bd491b12a8065de2ff20a49f4645743169853645ced314de9a73595d8788857ea43aa1e5977acd81582521e
+  data.tar.gz: 7cac96de0d2b7852b90d9d57ba4a045ab5641e3f36b2c4a264c8a9f26c7722a5af1c34e151f7ca1f82be2eee85aec91b9c1db1b8230ccaac7b12f827eb8b5b4e

data/lib/craigslister/craigslister.rb

@@ -0,0 +1,52 @@
+# Thrown when low price is higher than high price
+class InvalidRangeError < StandardError
+end
+
+# Creates url from arguments and scrapes
+class Craigslister
+  attr_reader :area, :item, :high, :low
+
+  def initialize(args)
+    @area    = args.fetch(:area, 'sfbay')
+    @item    = args[:item]
+    @high    = args.fetch(:high, nil)
+    @low     = args.fetch(:low, nil)
+    validate_price_range
+  end
+
+  def scrape
+    scraper.scrape
+  end
+
+  def links
+    scraper.links
+  end
+
+  def url
+    "#{base_url}/search/sss?sort=rel&"\
+    "#{price_query}query="\
+    "#{item.downcase.split(' ') * '+'}"
+  end
+
+  private
+
+  def scraper
+    Scraper.new(url, base_url)
+  end
+
+  def base_url
+    "https://#{area}.craigslist.org"
+  end
+
+  def price_query
+    result = ''
+    result += "min_price=#{low}&" if low
+    result += "max_price=#{high}&" if high
+    result
+  end
+
+  def validate_price_range
+    return unless low && high && low > high
+    fail(InvalidRangeError, 'Price range is invalid.')
+  end
+end

data/lib/craigslister/post.rb

@@ -0,0 +1,13 @@
+# Used for packaging Craigslist post data
+class Post
+  attr_reader :title, :image, :price, :location, :url, :description
+
+  def initialize(args)
+    @title       = args[:title]
+    @image       = args[:image]
+    @price       = args[:price]
+    @location    = args[:location]
+    @description = args[:description]
+    @url         = args[:url]
+  end
+end

data/lib/craigslister/post_scraper.rb

@@ -0,0 +1,49 @@
+# Creates Post objects out of an HTML page
+class PostScraper
+  def initialize(page, link)
+    @page = page
+    @link = link
+  end
+
+  def new_post
+    Post.new(
+      image: image,
+      title: title,
+      price: price,
+      location: location,
+      description: description,
+      url: link
+    )
+  end
+
+  private
+
+  attr_reader :page, :link
+
+  def posting_title
+    page.at('span.postingtitletext')
+  end
+
+  def image
+    image = page.at('img')
+    image ? image['src'] : ''
+  end
+
+  def title
+    posting_title.text.gsub(/ ?- ?\$\d+ ?\(.+\)/, '')
+  end
+
+  def price
+    price = posting_title.at('span.price')
+    price ? price.text.gsub(/\$/, '').to_i : 0
+  end
+
+  def location
+    location = posting_title.at('small')
+    location ? location.text.gsub(/ ?[\(\)]/, '') : ''
+  end
+
+  def description
+    page.at('section#postingbody').text
+  end
+end

data/lib/craigslister/scraper.rb

@@ -0,0 +1,39 @@
+# Houses all higher level scraping logic
+class Scraper
+  def initialize(url, base_url)
+    @url = url
+    @base_url = base_url
+  end
+
+  def links
+    header_link.map { |link| format_link(link['href']) }
+  end
+
+  def scrape
+    links.flat_map { |link| post_from(link) }
+  end
+
+  private
+
+  attr_reader :url, :base_url
+
+  def page_from(url)
+    Nokogiri::HTML(open(url))
+  end
+
+  def post_from(link)
+    PostScraper.new(page_from(link), link).new_post
+  end
+
+  def header_link
+    page_from(url).css('.hdrlnk')
+  end
+
+  def format_link(link)
+    if link =~ /\w+\.craig/
+      'https:' + link
+    else
+      base_url + link
+    end
+  end
+end

data/lib/craigslister.rb

@@ -1,107 +1,6 @@
 require 'nokogiri'
 require 'open-uri'
-
-
-class InvalidRangeError < StandardError
-end
-
-
-class Craigslister
-  attr_reader :area, :item, :high, :low
-
-  def initialize args
-    @area    = args.fetch(:area, 'sfbay')
-    @item    = args[:item]
-    @high    = args.fetch(:high, nil)
-    @low     = args.fetch(:low, nil)
-    validate_price_range
-  end
-
-  def scrape!
-    links.map {|link| item_from(link)}.compact
-  end
-
-  def links
-    page_from(url).css('.hdrlnk').map {|link| format_link(link)}
-  end
-
-  def url
-    "#{base_url}/search/sss?sort=rel&"\
-    "#{price_query}query="\
-    "#{item.downcase.split(' ') * '+'}"
-  end
-
-
-  private
-    def base_url
-      "https://#{area}.craigslist.org"
-    end
-
-    def page_from url
-      Nokogiri::HTML(open(url))
-    end
-
-    def format_link link
-      link['href'] =~ /\w+\.craig/ ? "https:" + link['href'] : base_url + link['href']
-    end
-
-    def price_query
-      result = ''
-      result += "min_price=#{low}&" if low
-      result += "max_price=#{high}&" if high
-      result
-    end
-
-    def validate_price_range
-      raise InvalidRangeError if low && high && low > high
-    end
-
-    def item_from link
-      Item.new(get_item_data(page_from(link), link))
-    end
-
-    def get_item_data page, link
-      {
-        image: scrape_image(page),
-        title: page.at('span.postingtitletext').text.gsub(/ ?- ?\$\d+ ?\(.+\)/, ''),
-        price: scrape_price(page),
-        location: scrape_location(page),
-        description: page.at('section#postingbody').text,
-        url: link
-      }
-    end
-
-    def scrape_image page
-      page.at('img') ? page.at('img')['src'] : ""
-    end
-
-    def scrape_price page
-      if price = page.at('span.postingtitletext span.price')
-        price.text.gsub(/\$/,'').to_i
-      else
-        0
-      end
-    end
-
-    def scrape_location page
-      if location = page.at('span.postingtitletext small')
-        location.text.gsub(/ ?[\(\)]/,'')
-      else
-        ""
-      end
-    end
-end
-
-
-
-class Item
-  attr_reader :title, :image, :price, :location, :url
-
-  def initialize args
-    @title    = args[:title]
-    @image    = args[:image]
-    @price    = args[:price]
-    @location = args[:location]
-    @url      = args[:url]
-  end
-end
+require 'craigslister/scraper'
+require 'craigslister/post_scraper'
+require 'craigslister/post'
+require 'craigslister/craigslister'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: craigslister
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.1.1
 platform: ruby
 authors:
 - Chris Scott
@@ -30,13 +30,17 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.6.6.2
-description: all you need is an item title and you can scrape item objects from craigslist
+description: all you need is an item title and you can scrape posts from craigslist
 email: christo247@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - lib/craigslister.rb
+- lib/craigslister/craigslister.rb
+- lib/craigslister/post.rb
+- lib/craigslister/post_scraper.rb
+- lib/craigslister/scraper.rb
 homepage: https://github.com/Yago580/craigslister
 licenses:
 - MIT
@@ -57,8 +61,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
-summary: Scrape Craigslist for item objects
+summary: Scrape Craigslist for Posts
 test_files: []