gumtree_scraper 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 42e2924d09e2b7f023235c3a1c0a824bb4af9f4b
+  data.tar.gz: aa76be8fe4d7974a0149ae08eceb92ec52ecebef
+SHA512:
+  metadata.gz: 4f07747c6cb026f0028f802198cd2916a9c5ab1c1c6ff4364bc064a198aa6df7b71e8d3a21afa42f356e0714d672098428615f469207735900a9db02e3cf2fc0
+  data.tar.gz: 0fba809d91d3afbdfe2decc0dce1efa450ad9f019c5f7a3f4e618c588e9dc85ff0d722d84e343b4a67d6a5afb2cb6df076c1561c919fc25549c66d9b67092a59

data/.gitignore

@@ -0,0 +1,34 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,34 @@
+PATH
+  remote: .
+  specs:
+    gumtree_scraper (0.0.1)
+      bundler (~> 1.5)
+      docparser (~> 0.2)
+      thor (~> 0.19)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    axlsx (2.0.1)
+      htmlentities (~> 4.3.1)
+      nokogiri (>= 1.4.1)
+      rubyzip (~> 1.0.0)
+    docparser (0.2.3)
+      axlsx (~> 2.0.1)
+      log4r (~> 1.1.10)
+      nokogiri (~> 1.6.1)
+      parallel (~> 1.3.2)
+    htmlentities (4.3.2)
+    log4r (1.1.10)
+    mini_portile (0.6.1)
+    nokogiri (1.6.4)
+      mini_portile (~> 0.6.0)
+    parallel (1.3.3)
+    rubyzip (1.0.0)
+    thor (0.19.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  gumtree_scraper!

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 charles marshall
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

data/README.md

@@ -0,0 +1,20 @@
+gumtree_scrape
+==============
+
+Install with:
+
+`gem install gumtree_scrape`
+
+Usage:
+
+On terminal run:
+
+`gumtree_scrape go`
+
+If failing and IP has been blocked, there is a proxy built in. To use that, run:
+
+`gumtree_scrape go --proxy`
+
+At the moment only a single proxy is setup, http://anonymouse.org
+
+When running it will ask a series of questions, just enter one the values offered.

data/bin/gumtree_scrape

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+lib = File.expand_path(File.dirname(__FILE__) + '/../lib')
+$LOAD_PATH.unshift(lib) if File.directory?(lib) && !$LOAD_PATH.include?(lib)
+
+require "gum"
+GUM::Gum.start(ARGV)

data/gumtree_scrape.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "gum/version"
+
+Gem::Specification.new do |spec|
+    spec.name = "gumtree_scraper"
+    spec.version = GUM::VERSION
+    spec.authors = ["Charles Marshall"]
+    spec.email = ["cm56marshall@gmail.com"]
+    spec.summary = %q{ }
+    spec.description = %q{ }
+    spec.homepage = "https://github.com/charlesmarshall/gumtree_scraper/tree/#{GUM::VERSION}"
+    spec.license = "MIT"
+    spec.files = `git ls-files -z`.split("\x0")
+    spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+    spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
+    spec.require_paths = ["lib"]
+    spec.required_ruby_version = '>= 2.0'
+
+    spec.add_dependency "docparser", "~> 0.2"
+    spec.add_dependency "bundler", "~> 1.5"
+    spec.add_dependency "thor", "~> 0.19"
+
+    spec.post_install_message = ""
+
+end

data/lib/gum/base.rb

@@ -0,0 +1,33 @@
+module GUM
+	class Base < Thor
+		include Thor::Shell
+
+		DEFAULT_PROXIES = [
+			"http://anonymouse.org/cgi-bin/anon-www.cgi/%URL",
+		]
+
+		no_commands{
+			def log(out)
+				say("[#{Time.now.to_s}] #{out}", :black)
+			end
+
+			# fetch url and save to a local file for parsing
+			def url_to_file(url, proxi=nil)
+				url = URI.escape(url)
+				file = "/tmp/gum_#{Time.now.to_i.to_s}"
+				if proxi
+					proxy = GUM::Base::DEFAULT_PROXIES.sample
+					log("Using proxy: #{proxy}")
+					url = proxy.gsub("%URL", url)
+					File.open(file, "wb") { |f| f.write open(url).read.gsub(proxy.gsub("%URL", ""), "") }
+				else
+					File.open(file, "wb") { |f| f.write open(url).read }
+				end
+				log("'#{url}' saved to '#{file}'")
+				return file
+			end
+
+		}
+
+	end
+end

data/lib/gum/gum.rb

@@ -0,0 +1,50 @@
+module GUM
+	class Gum < Base
+		class_option :verbose, :type => :boolean, :default => true
+		class_option :proxy, :type => :boolean, :default => true
+
+
+		desc "go", "Run everything"
+		def go			
+
+			# first off, ask what we want to download
+			page = GUM::Pages.new(options)
+			# base url without the page number subbed out
+			url = page.url(1)
+			# now check if the page works
+			file = self.url_to_file(url, options["proxy"])
+			if File.exists?(file)
+				max_page = page.max_page(file)
+				start_page = ask("Enter page to start at: ", :yellow)
+				start_page = 1 if start_page.nil?
+				asked_pages = ask("Enter page to end at (found #{max_page}) : ", :yellow)
+				max_page = asked_pages.to_i if ! asked_pages.nil? && asked_pages != "y"
+				log("Using #{start_page}-#{max_page} as max page")
+				
+				adverts = page.advert_links(start_page.to_i ,max_page)
+				say("Found #{adverts.length} adverts", :green)
+				# now fetch data for each link
+				processed = page.advert_data(adverts, :telephone)
+				say("Found #{processed.length} with phone numbers", :green)
+				# output to a csv
+				path = Dir.pwd+"/gumtree_p#{start_page}_p#{max_page}_#{Time.now.to_i.to_s}.csv"
+				
+				CSV.open(path, "wb") do |csvfile|
+					processed.each do|v|
+						log("Saving #{v[:url]}")
+						csvfile << v.values if ! v.nil?
+					end
+				end
+				# clean up
+				say("Saved to : #{path}", :green)
+				`rm /tmp/gum_* 2>&1 /dev/null`
+				say("Complete", :blue)
+			else
+				say("Failed to download page", :red)
+				exit 1
+			end
+		end
+
+
+	end
+end

data/lib/gum/pages.rb

@@ -0,0 +1,156 @@
+module GUM
+	class Pages < Base
+		include DocParser
+		attr_accessor :top_level, :sub_level_values, :base_uri, :uri, :proxy
+		def initialize(options)
+			@proxy = nil
+			options.each do |k,v|
+				self.class.send(:attr_accessor, k)
+        		self.instance_variable_set("@#{k}",v)
+			end
+		end
+		
+		# gum tree page info
+		DOMAIN = "http://www.gumtree.com"
+		BASE_SEARCH = "#{GUM::Pages::DOMAIN}/search?q=&search_category=%CATEGORY&search_location=%LOCATION&distance=%DISTANCE&seller_type=%SELLER_TYPE&page=%PAGE"
+		LISTING_PAGES ={
+			"property" => {
+				"category" => "flatshare",
+				"location" => "birmingham",
+				"distance" => ["10", "50", "100", "500", "1000"],
+				"seller_type" => ["private", "trade"],
+			}
+		}
+
+		# page data
+		SELECTORS = {
+			"property" => {
+				"list" => {
+					:body => "#fullListings",
+					:max_page => "li.page-last a",
+					# anything with a link in the key name is presumed that you want the href value, not the content
+					:item_links => "a.listing-link"
+				},
+				"advert" => {
+					:body => ".main main",
+					:name => "h1.space-mbs",
+					:price => ".ad-price",
+					:telephone => ".truncate-number",
+					:description => "p.ad-description"
+				}
+			}
+		}
+
+		no_commands {
+			# select the top level versions
+			def top_level_selection
+				ask("Please select what type of search page you want to scrape:", :yellow, :limited_to =>GUM::Pages::LISTING_PAGES.keys)
+			end
+
+			# find the real values for each item in side the sub list
+			def get_values(section)
+				data = GUM::Pages::LISTING_PAGES[section]
+				values = {}
+				data.each do |k, options|
+					if options.class == Array
+						values[k] = ask("Enter value for #{k}: ", :yellow, :limited_to => options)
+					else
+						values[k] = options
+					end
+					log("selected '#{values[k]}'")
+				end
+				values
+			end
+
+			# use the base url and swap out values
+			def parse(values)
+				base = GUM::Pages::BASE_SEARCH				
+				values.each{|k,v| base = base.gsub("%#{k.upcase}", v) if ! v.nil?}
+				base
+			end
+			# grab the base url with the %page param still set
+			def base_url()
+				@top_level = self.top_level_selection
+				log("selected '#{@top_level}'")
+				# request all the data underneath top level
+				@sub_level_values = self.get_values(@top_level).merge({"page"=>nil})
+				# now sub out values and get the url
+				@base_uri = self.parse(@sub_level_values)
+				@base_uri
+			end
+
+			# get the url of a page
+			def url(page)
+				@base_uri = self.base_url if @base_uri.nil?
+				@uri = @base_uri.gsub("%PAGE", page.to_s)
+				log("url '#{@uri}'")
+				@uri
+			end
+
+			# load the page up
+			def load_without_output(file)
+				@parser = Parser.new(files: [file], parallel: false, quiet: true)
+				self
+			end
+			# find values for attributes
+			def values(type)
+				body = GUM::Pages::SELECTORS[@top_level][type][:body]
+				selectors = GUM::Pages::SELECTORS[@top_level][type].select{ |k,v| k.to_s != "body" }
+				data = {}
+				@parser.parse! do
+					css(body) do |post|
+						selectors.each do |n,selector|
+							data[n] = []
+							post.search(selector).each{ |i| data[n].push(if n.to_s.index("link") then i.attributes["href"].value.gsub(GUM::Pages::DOMAIN, "") else i.content.strip end) }
+						end
+					end
+				end
+				# remove dups
+				data.each{|k,v| data[k] = v.first if k.to_s.index("link").nil? && v.class == Array }
+				data
+			end
+
+			# find max value
+			def max_page(file)
+				# find the max page				
+				list_values = self.load_without_output(file).values("list")
+				max_page = list_values[:max_page].to_i
+				log("max page: '#{max_page}'")
+				max_page
+			end
+
+			# find all advert links
+			def advert_links(start, max_page)
+				adverts = []
+				range = (start..max_page).to_a
+				range.each do |i|
+					say("Page #{i}", :green)
+					url = self.url(i)
+					file = self.url_to_file(url, @proxy) if ! url.nil?
+					list_values = self.load_without_output(file).values("list") if ! file.nil?
+					log("found #{list_values[:item_links].length} adverts")
+					# merge together
+					adverts = adverts.concat(list_values[:item_links])
+					log("total length so far: #{adverts.length}")					
+				end
+				adverts
+			end
+
+			# find the data for each advert
+			def advert_data(adverts, with)
+				data = []
+				adverts.each_with_index do |post, i|
+					url = GUM::Pages::DOMAIN + post
+					say("Ad #{i}: #{url}", :green)
+					file = self.url_to_file(url, @proxy)
+					values = self.load_without_output(file).values("advert").merge({url:url}) if ! file.nil?
+					data.push(values) if ! values[with].nil? && values[with].length > 0
+					sleep 0.2
+				end
+				data
+			end
+		}
+
+	end
+
+end

data/lib/gum/version.rb

@@ -0,0 +1,3 @@
+module GUM
+	VERSION = "0.0.1"
+end

data/lib/gum.rb

@@ -0,0 +1,13 @@
+require "gum/version"
+require "thor"
+require "docparser"
+require "open-uri"
+require "uri"
+require "csv"
+
+module GUM
+	require "gum/base"
+	require "gum/pages"
+
+	require "gum/gum"
+end

metadata

@@ -0,0 +1,99 @@
+--- !ruby/object:Gem::Specification
+name: gumtree_scraper
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Charles Marshall
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-11-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: docparser
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+description: " "
+email:
+- cm56marshall@gmail.com
+executables:
+- gumtree_scrape
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- bin/gumtree_scrape
+- gumtree_scrape.gemspec
+- lib/gum.rb
+- lib/gum/base.rb
+- lib/gum/gum.rb
+- lib/gum/pages.rb
+- lib/gum/version.rb
+homepage: https://github.com/charlesmarshall/gumtree_scraper/tree/0.0.1
+licenses:
+- MIT
+metadata: {}
+post_install_message: ''
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: ''
+test_files: []