classifieds_api 0.1.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 486b9412804f50e3cc7e010055abf47b683a7cb6
-  data.tar.gz: 4edab757b113ff0470117d37585b83d710671abc
+  metadata.gz: 714c4645bfb468ff19cb1512775b8933d91c3fb6
+  data.tar.gz: b3d558c70659613ce0cfad26b8dcb63296c16bd6
 SHA512:
-  metadata.gz: 3a08b4db4f33829aae03460c6cc5476e6491e3b0dfe59082ca4a0c1bc3d1adede678239dbcd0e1bbbc058820b36d8ff7b53e31b8d4fe0fa85d7a10cae747d52d
-  data.tar.gz: 34fec05b950afeefec26f909e5e168f124cf7fc6858bf95a394b25d331f54295e5de8c12c6858b16d8062355723fb9a6bb35f47f74345bada2cbf9aca39dde3d
+  metadata.gz: 0b4c19a643adf817a6dd7fb9a8333772cb466d7f88c8cbe31ee7e2e2ed39411565c6c4a38ee8d80b5078976377ef691a9879e1d25ff41430e84f550e5bb069bf
+  data.tar.gz: 8e5623695ce780cf04cd8f47ffb34645a3d0d130784f7fc778a96a177e50066d6ec1563ec82a1e84b4f0cce5eec2f6473d408ce7baf758605881331249aff0ab

data/.gitignore

@@ -34,5 +34,10 @@ bower.json
 # Ignore pow environment settings
 .powenv
 
-# Ignore the gem files
+# Ignore gem files
 *.gem
+
+# Ignore vim temporary files and swap files
+*~
+*.swp
+*.swo

data/README.rdoc

@@ -0,0 +1,9 @@
+= Classifieds API
+
+== Description
+
+Simple Ruby API for popular classifieds website.
+
+== Installation
+
+  gem install classifieds_api

data/Rakefile

@@ -1,2 +1,10 @@
 require "bundler/gem_tasks"
 
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+      t.libs << 'test'
+end
+
+desc "Run tests"
+task :default => :test

data/classifieds_api.gemspec

@@ -9,8 +9,8 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Jean-Francois Bisson"]
   spec.email         = ["jbisson@uwaterloo.ca"]
 
-  spec.summary       = %q{Ruby API for popular classifieds website}
-  spec.description   = %q{Simple Ruby API for filtered ad searchs on popular classifieds website}
+  spec.summary       = %q{Simple API for popular classifieds website}
+  spec.description   = %q{Simple API for popular classifieds website. Currently, the gem allows you to do search items on Kijiji in Ontario.}
   spec.homepage      = "https://github.com/jfbisson4/classifieds_api"
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
@@ -24,6 +24,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.8"
   spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "test-unit", "~> 3.0"
 
   spec.add_runtime_dependency "nokogiri", "~> 1.6"
 end

data/lib/classifieds_api.rb

@@ -1,5 +1,7 @@
-require "classifieds_api/version"
+require 'classifieds_api/version'
+require 'classifieds_api/search_filters'
+require 'classifieds_api/ad'
+require 'classifieds_api/kijiji'
 
 module ClassifiedsApi
-  # Your code goes here...
 end

data/lib/classifieds_api/ad.rb

@@ -0,0 +1,115 @@
+module ClassifiedsApi
+  class Ad
+    attr_reader :title, :date_listed, :price, :address, :description,
+                :image_urls, :offering, :featured, :url
+
+    def initialize(title, date_listed, price, address, description, image_urls,
+                  offering, featured, url)
+      self.title        = title
+      self.date_listed  = date_listed
+      self.price        = price
+      self.address      = address
+      self.description  = description
+      self.image_urls   = image_urls
+      self.offering     = offering
+      self.featured     = featured
+      self.url          = url
+    end
+
+    def title=(title)
+      unless title.is_a? String
+        raise TypeError, "Argument #{title} is not a string."
+      end
+      @title = title
+    end
+
+    def date_listed=(date_listed)
+      if date_listed.nil? == false
+        unless date_listed.is_a? DateTime
+          raise TypeError, "Argument #{date_listed} is not a DateTime."
+        end
+      end
+      @date_listed = date_listed
+    end
+
+    def price=(price)
+      if price.nil? == false
+        unless price.is_a? Numeric
+          raise TypeError, "Argument #{price} is not a numeric."
+        end
+      end
+      @price = price
+    end
+
+    def address=(address)
+      if address.nil? == false
+        unless address.is_a? String
+          raise TypeError, "Argument #{address} is not a string."
+        end
+      end
+      @address = address
+    end
+
+    def description=(description)
+      unless description.is_a? String
+        raise TypeError, "Argument #{description} is not a string."
+      end
+      @description = description
+    end
+
+    def image_urls=(image_urls)
+      if image_urls.nil? == false
+        unless image_urls.is_a? Array
+          raise TypeError, "Argument #{image_urls} is not a string."
+        end
+      end
+      @image_urls = image_urls
+    end
+
+    def offering=(offering)
+      unless !!offering == offering
+        raise TypeError "Argument #{offering} is not a string."
+      end
+      @offering = offering
+    end
+
+    def featured=(featured)
+      unless !!featured == featured
+        raise TypeError "Argument #{featured} is not a boolean."
+      end
+      @featured = featured
+    end
+
+    def url=(url)
+      unless url.is_a? String
+        raise TypeError, "Argument #{address} is no a string."
+      end
+      @url = url
+    end
+
+    def print_ad
+      puts '-------------------------------------------------------------------'
+      if @offering == true
+        print "Offering - "
+      else
+        print "Wanted - "
+      end
+      if @featured_ad == true
+        puts "Featured Ad"
+      else
+        puts "Regular Ad"
+      end
+      if (@date_listed.nil? == false)
+        puts "Date Listed:  #{@date_listed.strftime("%d/%m/%Y %H:%M")}"
+      else
+        puts "Date Listed:"
+      end
+      puts "Title:        #{@title}"
+      puts "Price:        #{@price}"
+      puts "Description:  #{@description}"
+      puts "Address:      #{@address}"
+      puts "URL:          #{@url}"
+      puts '-------------------------------------------------------------------'
+    end
+  end
+end

data/lib/classifieds_api/kijiji.rb

@@ -0,0 +1,322 @@
+module ClassifiedsApi
+  class Kijiji
+
+    def search(keyword, search_limit, search_filters)
+
+      unless keyword.is_a? String
+        raise TypeError, "Argument #{keyword} is not a string."
+      end
+      unless search_limit.is_a? Numeric
+        raise TypeError, "Argument #{search_limit} is not a numeric."
+      end
+      if search_limit <= 0
+        raise RangeError, "Argument #{search_limit} should be bigger than 0."
+      end
+      if search_limit > SEARCH_LIMIT
+        raise RangeError,
+          "Argument #{search_limit} should be smaller than the search limit " +
+          "of #{SEARCH_LIMIT}."
+      end
+      unless search_filters.instance_of? SearchFilters
+        raise TypeError,
+          "The search filters argument is not of the SearchFilters class."
+      end
+
+      ads = []
+      page_number = 1
+      until ads.count >= search_limit
+
+        search_page_url     = get_search_page_url(keyword,
+                                                  page_number,
+                                                  search_filters)
+        search_page_html    = get_search_page_html(search_page_url)
+        noko_search_html    = get_nokogiri_html(search_page_html)
+        noko_search_results = get_noko_results_on_search_page(
+                                noko_search_html,
+                                search_filters.include_sponsored_ads)
+
+        if noko_search_results.count == 0
+          return ads
+        end
+
+        noko_search_results.each do |noko_result|
+          if ads.count >= search_limit
+            return ads
+          end
+
+          ad_title_wanted       = get_title             (noko_result)
+          ad_title              = remove_wanted         (ad_title_wanted)
+          ad_price              = get_price             (noko_result)
+          ad_short_description  = get_short_description (noko_result)
+          ad_offering           = get_offering          (ad_title_wanted)
+          ad_featured           = get_ad_featured       (noko_result)
+          ad_url                = get_ad_url            (noko_result)
+
+
+          new_ad = Ad.new(ad_title,
+                          nil,
+                          ad_price,
+                          nil,
+                          ad_short_description,
+                          nil,
+                          ad_offering,
+                          ad_featured,
+                          ad_url)
+
+          ads.push(new_ad)
+        end
+        page_number += 1
+      end
+      return ads
+    end
+
+    private
+
+    def get_search_page_url(keyword, page_number, search_filters)
+      category_key        = get_category_key        (search_filters.category)
+      category_id         = get_category_id         (search_filters.category)
+      page_key            = get_page_key            (page_number)
+      location_string_key = get_location_string_key (search_filters.location)
+      location_id         = get_location_id         (search_filters.location)
+      distance_key        = get_distance_key        (search_filters.distance)
+      offer_type_key      = get_offer_type_key      (search_filters.offer_type)
+      price_key           = get_price_key(search_filters.price_minimum,
+                                          search_filters.price_maximum,
+                                          offer_type_key)
+
+
+      url  = ""
+      url += "http://www.kijiji.ca/b-"
+      url += "#{category_key}"
+      url += "#{location_string_key}/"
+      url += "#{keyword}/"
+      url += "#{page_key}"
+      url += "#{category_id}"
+      url += "#{location_id}"
+      url += "#{distance_key}"
+      url += "#{offer_type_key}"
+      url += "#{price_key}"
+      return url
+    end
+
+    def get_search_page_html(page_url)
+      html = ""
+      open(page_url) {|io| html = io.read}
+    end
+
+    def get_nokogiri_html(html)
+      noko_html = Nokogiri::HTML(html)
+    end
+
+    def get_noko_results_on_search_page(noko_html, include_sponsored_ads)
+      noko_regular_ads  = noko_html.css('table.regular-ad')
+      if include_sponsored_ads == true
+        noko_featured_ads = noko_html.css('table.top-feature')
+        noko_ads          = noko_featured_ads + noko_regular_ads
+      else
+        noko_ads = noko_regular_ads
+      end
+      return noko_ads
+    end
+
+    def get_title(noko_result)
+      title = noko_result.css('td.description').css('a.title').text.strip
+    end
+
+    def remove_wanted(title_wanted)
+      title = title_wanted.gsub('Wanted:', '').strip
+    end
+
+    def get_price(noko_result)
+      string_price =
+        noko_result.css('td.price').text.strip.gsub('$', '').gsub(',', '')
+      price = string_price.to_f
+      if string_price == "Free"
+        return 0
+      elsif string_price == "Please Contact" || string_price == ""
+        return nil
+      else
+        return price
+      end
+    end
+
+    def get_short_description(noko_result)
+      description = noko_result.css('td.description').css('p').text.strip
+    end
+
+    def get_offering(title_wanted)
+      if title_wanted.include? 'Wanted:'
+        return false
+      else
+        return true
+      end
+    end
+
+    def get_ad_featured(noko_result)
+      css_class = get_node_class(noko_result)
+      if (css_class.include? 'regular-ad')
+        return false
+      else
+        return true
+      end
+    end
+
+    def get_node_class(noko_result)
+      css_class = noko_result['class'].strip
+    end
+
+    def get_ad_url(noko_result)
+      extension = noko_result.css('td.description').css('a.title')[0]["href"]
+      url = "http://www.kijiji.ca" + extension
+    end
+
+    def get_category_key(category)
+      if category == 'all'
+        return ""
+      else
+        return "#{CATEGORY_KEYS[category]}/"
+      end
+    end
+
+    def get_category_id(category)
+      return CATEGORY_IDS[category]
+    end
+
+    def get_page_key(page_number)
+      if (page_number > 1)
+        return "page-#{page_number}/"
+      else
+        return ""
+      end
+    end
+
+    def get_location_string_key(location)
+      if location == 'Toronto (GTA)'
+        return 'gta-greater-toronto-area'
+      end
+      location_string = location.dup
+      location_string.gsub! '/', ''
+      location_string.gsub! '.', ''
+      location_string.downcase!
+      location_string.split(' ').join('-')
+    end
+
+    def get_location_id(location)
+      location_id = LOCATION_IDS[location]
+    end
+
+    def get_distance_key(distance)
+      if distance != 50
+        return "r#{distance}"
+      else
+        return ""
+      end
+    end
+
+    def get_offer_type_key(offer_type)
+      offer_key = ""
+      if offer_type == 'Offering'
+        offer_key = "?ad=offering"
+      elsif offer_type == 'Wanted'
+        offer_key = '?ad=wanted'
+      end
+      return offer_key
+    end
+
+    def get_price_key(min_price, max_price, previous_param)
+      price_key = ""
+      if min_price != 0 || max_price != 10000000
+        if previous_param != ""
+          price_key += "&price="
+        else
+          price_key += "?price="
+        end
+        if min_price != 0
+          price_key += "#{min_price}"
+        end
+        price_key += "__"
+        if max_price != 10000000
+          price_key += "#{max_price}"
+        end
+      end
+      return price_key
+    end
+
+    SEARCH_LIMIT = 10
+
+    CATEGORY_KEYS = Hash[ 'buy and sell',       'buy-sell',
+                          'services',           'services',
+                          'cars & vehicles',    'cars-vehicles',
+                          'pets',               'pets',
+                          'vacation rentals',   'vacation-rentals',
+                          'community',          'community',
+                          'real estate',        'real-estate',
+                          'jobs',               'jobs',
+                          'resumes',            'resumes']
+
+
+    CATEGORY_IDS = Hash['all',                'k0l',
+                        'buy and sell',       'k0c10l',
+                        'services',           'k0c72l',
+                        'cars & vehicles',    'k0c27l',
+                        'pets',               'k0c112l',
+                        'vacation rentals',   'k0c800l',
+                        'community',          'k0c1l',
+                        'real estate',        'k0c34l',
+                        'jobs',               'k0c45l',
+                        'resumes',            'k0c218l']
+
+    LOCATION_IDS = Hash['Berrie',                         '1700006',
+                        'Belleville Area',                '1700129',
+                        'Belleville',                     '1700130',
+                        'Trenton',                        '1700132',
+                        'Brantford',                      '1700206',
+                        'Brockville',                     '1700247',
+                        'Chatham-Kent',                   '1700239',
+                        'Cornwall',                       '1700133',
+                        'Guelph',                         '1700242',
+                        'Hamilton',                       '80014',
+                        'Kapuskasing',                    '1700237',
+                        'Kenora',                         '1700249',
+                        'Kingston Area',                  '1700181',
+                        'Kingston',                       '1700183',
+                        'Napanee',                        '1700182',
+                        'Kitchener Area',                 '1700209',
+                        'Cambridge',                      '1700210',
+                        'Kitchener / Waterloo',           '1700212',
+                        'Stratford',                      '1700213',
+                        'Leamington',                     '1700240',
+                        'London',                         '1700214',
+                        'Muskoka',                        '1700078',
+                        'Norfolk County',                 '1700248',
+                        'North Bay',                      '1700243',
+                        'Ottawa / Gatineau Area',         '1700184',
+                        'Gatineau',                       '1700186',
+                        'Ottawa',                         '1700185',
+                        'Owen Sound',                     '1700187',
+                        'Peterborough Area',              '1700217',
+                        'Kawartha Lakes',                 '1700219',
+                        'Peterborough',                   '1700218',
+                        'Renfrew County Area',            '1700074',
+                        'Pembroke',                       '1700075',
+                        'Petawawa',                       '1700076',
+                        'Renfrew',                        '1700077',
+                        'Sarnia Area',                    '1700189',
+                        'Grand Bend',                     '1700190',
+                        'Sarnia',                         '1700191',
+                        'Sault Ste. Marie',               '1700244',
+                        'St. Catharines',                 '80016',
+                        'Sudbury',                        '1700245',
+                        'Thunder Bay',                    '1700126',
+                        'Timmins',                        '1700238',
+                        'Toronto (GTA)',                  '1700272',
+                        'City of Toronto',                '1700273',
+                        'Markham / York Region',          '1700274',
+                        'Mississauga / Peel Region',      '1700276',
+                        'Oakville / Halton Region',       '1700277',
+                        'Oshawa / Durham Region',         '1700275',
+                        'Windsor Region',                 '1700220',
+                        'Woodstock',                      '1700241']
+
+  end
+end

data/lib/classifieds_api/search_filters.rb

@@ -0,0 +1,134 @@
+module ClassifiedsApi
+  class SearchFilters
+
+    attr_reader :category, :location, :distance, :offer_type, :price_minimum,
+                :price_maximum, :additional_info, :include_sponsored_ads
+
+    def initialize(params={})
+      self.category               = params[:category]               || 'all'
+      self.location               = params[:location]
+      self.distance               = params[:distance]               ||  50
+      self.offer_type             = params[:offer_type]             || 'all'
+      self.price_minimum          = params[:price_minimum]          ||  0
+      self.price_maximum          = params[:price_maximum]          ||  10000000
+      self.include_sponsored_ads  = params[:include_sponsored_ads]  || false
+    end
+
+    def category=(category)
+      unless CATEGORIES.include? category
+        #print_list('CATEGORIES', CATEGORIES)
+        raise ArgumentError,
+          "Argument #{category} is not a valid category."
+      end
+      @category = category
+    end
+
+    def location=(location)
+      unless LOCATIONS.include? location
+        #print_list('LOCATIONS', LOCATIONS)
+        raise ArgumentError,
+          "Argument #{location} is not a valid location."
+      end
+      @location = location
+    end
+
+    def distance=(distance)
+      unless distance.is_a? Numeric
+        raise TypeError, "Argument #{distance} is not numeric."
+      end
+      if 0 > distance || distance > 500
+        raise RangeError, "Argument #{distance} should be between 0 and 500."
+      end
+      @distance = distance
+    end
+
+    def offer_type=(offer_type)
+      unless OFFER_TYPES.include? offer_type
+        #print_list('OFFER TYPES', OFFER_TYPES)
+        raise ArgumentError,
+          "Argument #{offer_type} is not a valid offer type."
+      end
+      @offer_type = offer_type
+    end
+
+    def price_minimum=(price_minimum)
+      unless price_minimum.is_a? Numeric
+        raise TypeError, "Argument #{price_minimum} is not numeric."
+      end
+      if 0 > price_minimum
+        raise RangeError,
+          "Argument #{price_minimum} should be between bigger than 0."
+      end
+      if price_minimum > (@price_maximum || 10000000)
+        raise RangeError,
+          "Argument #{price_minimum} should be smaller than maximum price " +
+          "#{@price_maximum}."
+      end
+      @price_minimum = price_minimum
+    end
+
+    def price_maximum=(price_maximum)
+      unless price_maximum.is_a? Numeric
+        raise TypeError, "Argument #{price_maximum} is not numeric."
+      end
+      if price_maximum < (@price_minimum || 0)
+        raise RangeError,
+          "Argument #{price_maximum} should be bigger than minimum price " +
+          "#{@price_minimum}."
+      end
+      @price_maximum = price_maximum
+    end
+
+    def include_sponsored_ads=(include_sponsored_ads)
+      unless !!include_sponsored_ads == include_sponsored_ads
+        raise TypeError, "Argument #{include_sponsored_ads} is not a boolean."
+      end
+      @include_sponsored_ads = include_sponsored_ads
+    end
+
+    def print_filters
+      puts '-------------------------------------------------------------------'
+      puts 'Search Filters:'
+      puts "Category:               #{@category}"
+      puts "Location:               #{@location}"
+      puts "Distance:               #{@distance}"
+      puts "Offer Type:             #{@offer_type}"
+      puts "Minimum Price:          #{@price_minimum}"
+      puts "Maximum Price:          #{@price_maximum}"
+      puts "Include Sponsored Ads:  #{@include_sponsored_ads}"
+      puts '-------------------------------------------------------------------'
+    end
+
+    private
+
+    def print_list(list_name, list)
+      puts '-------------------------------------------------------------------'
+      puts "#{list_name}:"
+      list.each do |element|
+        puts element
+      end
+      puts '-------------------------------------------------------------------'
+    end
+
+    CATEGORIES = ['all', 'buy and sell', 'services', 'cars & vehicles', 'pets',
+                  'vacation rentals', 'community', 'real estate', 'jobs',
+                  'resumes']
+
+    LOCATIONS = ['Berrie', 'Belleville Area', 'Belleville', 'Trenton',
+                 'Brantford', 'Brockville', 'Chatham-Kent', 'Cornwall',
+                 'Guelph', 'Hamilton', 'Kapuskasing', 'Kenora', 'Kingston Area',
+                 'Kingston', 'Napanee', 'Kitchener Area', 'Cambridge',
+                 'Kitchener / Waterloo', 'Stratford', 'Leamington', 'London',
+                 'Muskoka', 'Norfolk County', 'North Bay',
+                 'Ottawa / Gatineau Area', 'Gatineau', 'Ottawa', 'Owen Sound',
+                 'Peterborough Area', 'Kawartha Lakes', 'Peterborough',
+                 'Renfrew County Area', 'Pembroke', 'Petawawa', 'Renfrew',
+                 'Sarnia Area', 'Grand Bend', 'Sarnia', 'Sault Ste. Marie',
+                 'St. Catharines', 'Sudbury', 'Thunder Bay', 'Timmins',
+                 'Toronto (GTA)', 'City of Toronto', 'Markham / York Region',
+                 'Mississauga / Peel Region', 'Oakville / Halton Region',
+                 'Oshawa / Durham Region', 'Windsor Region', 'Woodstock']
+
+    OFFER_TYPES = ['all', 'Offering', 'Wanted']
+  end
+end

data/lib/classifieds_api/version.rb

@@ -1,3 +1,3 @@
 module ClassifiedsApi
-  VERSION = "0.1.0"
+  VERSION = "1.0.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: classifieds_api
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Jean-Francois Bisson
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-04-17 00:00:00.000000000 Z
+date: 2015-04-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -52,7 +66,8 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
-description: Simple Ruby API for filtered ad searchs on popular classifieds website
+description: Simple API for popular classifieds website. Currently, the gem allows
+  you to do search items on Kijiji in Ontario.
 email:
 - jbisson@uwaterloo.ca
 executables: []
@@ -63,12 +78,15 @@ files:
 - ".travis.yml"
 - Gemfile
 - LICENSE
-- README.md
+- README.rdoc
 - Rakefile
 - bin/console
 - bin/setup
 - classifieds_api.gemspec
 - lib/classifieds_api.rb
+- lib/classifieds_api/ad.rb
+- lib/classifieds_api/kijiji.rb
+- lib/classifieds_api/search_filters.rb
 - lib/classifieds_api/version.rb
 homepage: https://github.com/jfbisson4/classifieds_api
 licenses: []
@@ -93,5 +111,5 @@ rubyforge_project:
 rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
-summary: Ruby API for popular classifieds website
+summary: Simple API for popular classifieds website
 test_files: []

data/README.md

@@ -1,2 +0,0 @@
-# classifieds_api
-Ruby API for popular classifieds website