gumtree_scraper 0.0.1 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 42e2924d09e2b7f023235c3a1c0a824bb4af9f4b
-  data.tar.gz: aa76be8fe4d7974a0149ae08eceb92ec52ecebef
+  metadata.gz: 2657037c9ffa4200cc4fba6693fc1a6ccdf4ade4
+  data.tar.gz: 393b0f81f385de12272a496f8b5383d5e48e35d7
 SHA512:
-  metadata.gz: 4f07747c6cb026f0028f802198cd2916a9c5ab1c1c6ff4364bc064a198aa6df7b71e8d3a21afa42f356e0714d672098428615f469207735900a9db02e3cf2fc0
-  data.tar.gz: 0fba809d91d3afbdfe2decc0dce1efa450ad9f019c5f7a3f4e618c588e9dc85ff0d722d84e343b4a67d6a5afb2cb6df076c1561c919fc25549c66d9b67092a59
+  metadata.gz: ed75e5e9e9bd4b6eb31dc02724d701ef308d3846c0db9eed81a680d51a2ec81299abca64b57a5148a6254461dbe36e8ccb09f5c9e36d925bc175663ed122195e
+  data.tar.gz: 3a02bf158a85dcb797cb25c2a80d16f1afe21eaf64cd9c1fd9b86b8cba7352c87a79f36326da439289e50b4927a31b32df0f20552b820cb5848e8c0a17ceeb73

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gumtree_scraper (0.0.1)
+    gumtree_scraper (0.0.4)
       bundler (~> 1.5)
       docparser (~> 0.2)
       thor (~> 0.19)

data/README.md

@@ -3,17 +3,17 @@ gumtree_scrape
 
 Install with:
 
-`gem install gumtree_scrape`
+`gem install gumtree_scraper`
 
 Usage:
 
 On terminal run:
 
-`gumtree_scrape go`
+`gumtree_scraper go`
 
 If failing and IP has been blocked, there is a proxy built in. To use that, run:
 
-`gumtree_scrape go --proxy`
+`gumtree_scraper go --proxy`
 
 At the moment only a single proxy is setup, http://anonymouse.org
 

data/lib/gum/gum.rb

@@ -2,7 +2,7 @@ module GUM
 	class Gum < Base
 		class_option :verbose, :type => :boolean, :default => true
 		class_option :proxy, :type => :boolean, :default => true
-
+		class_option :delay, :type => :string, :default => "2"
 
 		desc "go", "Run everything"
 		def go			

data/lib/gum/pages.rb

@@ -47,6 +47,7 @@ module GUM
 				ask("Please select what type of search page you want to scrape:", :yellow, :limited_to =>GUM::Pages::LISTING_PAGES.keys)
 			end
 
+
 			# find the real values for each item in side the sub list
 			def get_values(section)
 				data = GUM::Pages::LISTING_PAGES[section]
@@ -59,6 +60,7 @@ module GUM
 					end
 					log("selected '#{values[k]}'")
 				end
+				@delay = ask("Enter delay time between calls: ", :yellow)
 				values
 			end
 
@@ -131,7 +133,8 @@ module GUM
 					log("found #{list_values[:item_links].length} adverts")
 					# merge together
 					adverts = adverts.concat(list_values[:item_links])
-					log("total length so far: #{adverts.length}")					
+					log("total length so far: #{adverts.length}")	
+					sleep @delay.to_f;				
 				end
 				adverts
 			end
@@ -145,7 +148,7 @@ module GUM
 					file = self.url_to_file(url, @proxy)
 					values = self.load_without_output(file).values("advert").merge({url:url}) if ! file.nil?
 					data.push(values) if ! values[with].nil? && values[with].length > 0
-					sleep 0.2
+					sleep @delay.to_f;
 				end
 				data
 			end

data/lib/gum/version.rb

@@ -1,3 +1,3 @@
 module GUM
-	VERSION = "0.0.1"
+	VERSION = "0.0.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gumtree_scraper
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.5
 platform: ruby
 authors:
 - Charles Marshall
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-05 00:00:00.000000000 Z
+date: 2014-11-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: docparser
@@ -72,7 +72,7 @@ files:
 - lib/gum/gum.rb
 - lib/gum/pages.rb
 - lib/gum/version.rb
-homepage: https://github.com/charlesmarshall/gumtree_scraper/tree/0.0.1
+homepage: https://github.com/charlesmarshall/gumtree_scraper/tree/0.0.5
 licenses:
 - MIT
 metadata: {}