craigslist-data 0.0.1.deprecated → 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +29 -1
- data/craigslist-data.gemspec +1 -5
- data/lib/craigslist/craigslist.rb +14 -31
- data/lib/craigslist/data/version.rb +1 -1
- data/spec/craigslist_spec.rb +0 -74
- metadata +5 -9
- data/lib/craigslist/cities.rb +0 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 537e6a35285b7c4a33340bcbefeeb410240a62ab
|
|
4
|
+
data.tar.gz: 92e078a3b187dc4ac91e221feaaa33c7e57c4cf7
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: ea80727f292c96956dfa12b77c8574b182d4cf50a3fee8ac3d3d1783a336bed9b0c05f65136746dc0504e46908776218efdb5cf521bff2054a49e88538b07c98
|
|
7
|
+
data.tar.gz: 86183fcfe7b6ef29dd759478c8255ccf20600bf70cb33bcb72439c508657d65db2b03c82f3ada78873dc542a73bc7933a8fc5b6dc8f462243855f7dc53dbf428
|
data/README.md
CHANGED
|
@@ -1 +1,29 @@
|
|
|
1
|
-
|
|
1
|
+
# Craigslist::Data
|
|
2
|
+
|
|
3
|
+
TODO: Write a gem description
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
Add this line to your application's Gemfile:
|
|
8
|
+
|
|
9
|
+
gem 'craigslist-data'
|
|
10
|
+
|
|
11
|
+
And then execute:
|
|
12
|
+
|
|
13
|
+
$ bundle
|
|
14
|
+
|
|
15
|
+
Or install it yourself as:
|
|
16
|
+
|
|
17
|
+
$ gem install craigslist-data
|
|
18
|
+
|
|
19
|
+
## Usage
|
|
20
|
+
|
|
21
|
+
TODO: Write usage instructions here
|
|
22
|
+
|
|
23
|
+
## Contributing
|
|
24
|
+
|
|
25
|
+
1. Fork it
|
|
26
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
|
27
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
|
28
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
|
29
|
+
5. Create new Pull Request
|
data/craigslist-data.gemspec
CHANGED
|
@@ -16,12 +16,8 @@ Gem::Specification.new do |spec|
|
|
|
16
16
|
spec.files = `git ls-files`.split($/)
|
|
17
17
|
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
|
18
18
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
|
19
|
-
spec.post_install_message = <<-MESSAGE
|
|
20
|
-
! The 'craigslist-data' gem has been deprecated and has been replaced by 'craigslist_scraper'.
|
|
21
|
-
! See: http://rubygems.org/gems/craigslist_scraper
|
|
22
|
-
! And: https://github.com/mark-nery/craigslist_scraper
|
|
23
|
-
MESSAGE
|
|
24
19
|
spec.require_paths = ["lib"]
|
|
20
|
+
|
|
25
21
|
spec.add_development_dependency "nokogiri"
|
|
26
22
|
spec.add_development_dependency "rspec", "~> 2.6"
|
|
27
23
|
spec.add_development_dependency "bundler", "~> 1.3"
|
|
@@ -1,11 +1,9 @@
|
|
|
1
1
|
require 'nokogiri'
|
|
2
2
|
require 'open-uri'
|
|
3
3
|
require 'cgi'
|
|
4
|
-
require_relative 'cities'
|
|
5
4
|
|
|
6
5
|
class CraigsList
|
|
7
|
-
|
|
8
|
-
|
|
6
|
+
|
|
9
7
|
VALID_FIELDS = [:query, :srchType]
|
|
10
8
|
|
|
11
9
|
def search(options ={})
|
|
@@ -16,47 +14,32 @@ class CraigsList
|
|
|
16
14
|
uri = "http://#{options[:city]}.craigslist.org/search/sss?#{to_query(options)}"
|
|
17
15
|
|
|
18
16
|
doc = Nokogiri::HTML(open(uri))
|
|
19
|
-
|
|
20
|
-
doc.css('p.row').
|
|
21
|
-
|
|
17
|
+
items = []
|
|
18
|
+
doc.css('p.row').each do |link|
|
|
19
|
+
items << {
|
|
22
20
|
data_id: link["data-pid"] ,
|
|
23
21
|
description: link.css("a").text,
|
|
24
22
|
url: "http://#{options[:city]}.craigslist.org#{link.css("a")[0]["href"]}",
|
|
25
23
|
price: extract_price(link.css("span.price").text)
|
|
26
|
-
|
|
24
|
+
}
|
|
27
25
|
end
|
|
26
|
+
items
|
|
28
27
|
end
|
|
29
28
|
|
|
30
|
-
def
|
|
31
|
-
|
|
29
|
+
def us_cities
|
|
30
|
+
doc = Nokogiri::HTML(open("http://www.craigslist.org/about/sites"))
|
|
32
31
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
32
|
+
cities = []
|
|
33
|
+
doc.css('a').each do |link|
|
|
34
|
+
if /http:\/\/([a-z]*).craigslist.org/ =~ link["href"]
|
|
35
|
+
puts link["href"]
|
|
36
|
+
end
|
|
37
|
+
end
|
|
37
38
|
end
|
|
38
39
|
|
|
39
|
-
Array.class_eval do
|
|
40
|
-
def average_price
|
|
41
|
-
return 0 if empty?
|
|
42
|
-
|
|
43
|
-
self.reject! { |item| item[:price] == nil }
|
|
44
40
|
|
|
45
|
-
flat_map { |item| [item[:price]] }.map { |price| price.to_i }.reduce(:+) / self.size
|
|
46
|
-
end
|
|
47
|
-
end
|
|
48
|
-
|
|
49
41
|
private
|
|
50
42
|
|
|
51
|
-
def extract_city(method_name)
|
|
52
|
-
|
|
53
|
-
if /titles/ =~ method_name
|
|
54
|
-
method_name.to_s.gsub("search_titles_in_","").gsub("_for","")
|
|
55
|
-
else
|
|
56
|
-
method_name.to_s.gsub("search_","").gsub("_for","")
|
|
57
|
-
end
|
|
58
|
-
end
|
|
59
|
-
|
|
60
43
|
def extract_price(dollar_string)
|
|
61
44
|
dollar_string[1..-1]
|
|
62
45
|
end
|
data/spec/craigslist_spec.rb
CHANGED
|
@@ -60,79 +60,5 @@ describe CraigsList do
|
|
|
60
60
|
cl.search(city: city)[0][:url].should == "http://#{city}.craigslist.org/mob/3849318365.html"
|
|
61
61
|
end
|
|
62
62
|
|
|
63
|
-
describe "dynamic method search_{cityname}_for" do
|
|
64
|
-
|
|
65
|
-
let(:craigslist) { CraigsList.new }
|
|
66
|
-
|
|
67
|
-
it "calls search for a valid city" do
|
|
68
|
-
CraigsList::CITIES.each do |city|
|
|
69
|
-
craigslist.should_receive(:search).with(city: city , query: nil)
|
|
70
|
-
|
|
71
|
-
craigslist.send("search_#{city}_for")
|
|
72
|
-
end
|
|
73
|
-
end
|
|
74
|
-
|
|
75
|
-
it "doesn't call search for an invalid city" do
|
|
76
|
-
expect { craigslist.search_yourmamaville_for }.to raise_error(NoMethodError)
|
|
77
|
-
end
|
|
78
|
-
|
|
79
|
-
it "passes a query" do
|
|
80
|
-
craigslist.should_receive(:search).with(city: "dallas", query: "cowboy hats")
|
|
81
|
-
|
|
82
|
-
craigslist.search_dallas_for("cowboy hats")
|
|
83
|
-
end
|
|
84
|
-
end
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
describe "dynamic method search_titles_in_{cityname}_for" do
|
|
88
|
-
let(:craigslist) { CraigsList.new }
|
|
89
|
-
|
|
90
|
-
it "calls search for a valid city" do
|
|
91
|
-
CraigsList::CITIES.each do |city|
|
|
92
|
-
craigslist.should_receive(:search).with(city: city , query: nil , title_only: true)
|
|
93
|
-
|
|
94
|
-
craigslist.send("search_titles_in_#{city}_for")
|
|
95
|
-
end
|
|
96
|
-
end
|
|
97
|
-
|
|
98
|
-
it "doesn't call search for an invalid city" do
|
|
99
|
-
expect { craigslist.search_titles_in_yourmamaville_for }.to raise_error(NoMethodError)
|
|
100
|
-
end
|
|
101
|
-
end
|
|
102
|
-
|
|
103
|
-
describe "Array#average_price" do
|
|
104
|
-
let(:craigslist) { CraigsList.new }
|
|
105
|
-
|
|
106
|
-
it "returns the average price for a search with multiple items" do
|
|
107
|
-
craigslist.stub(:search_denver_for).and_return([{price: "3"} , {price: "5"} , {price: "7"}])
|
|
108
|
-
|
|
109
|
-
craigslist.search_denver_for("uranium").average_price.should == 5
|
|
110
|
-
end
|
|
111
|
-
|
|
112
|
-
it "returns 0 for search with no results" do
|
|
113
|
-
craigslist.stub(:search_denver_for).and_return([])
|
|
114
|
-
|
|
115
|
-
craigslist.search_denver_for("uranium").average_price.should == 0
|
|
116
|
-
end
|
|
117
|
-
|
|
118
|
-
it "returns average for a search with two items" do
|
|
119
|
-
craigslist.stub(:search_denver_for).and_return([{price: "8"} , {price: "12"} ])
|
|
120
|
-
|
|
121
|
-
craigslist.search_denver_for("uranium").average_price.should == 10
|
|
122
|
-
end
|
|
123
|
-
|
|
124
|
-
it "returns the price for a search with one item" do
|
|
125
|
-
craigslist.stub(:search_denver_for).and_return([{price: 1}])
|
|
126
|
-
|
|
127
|
-
craigslist.search_denver_for("uranium").average_price.should == 1
|
|
128
|
-
end
|
|
129
|
-
|
|
130
|
-
it "discards nil prices" do
|
|
131
|
-
craigslist.stub(:search_denver_for).and_return([{price: 1} , {price: nil}])
|
|
132
|
-
|
|
133
|
-
craigslist.search_denver_for("uranium").average_price.should == 1
|
|
134
|
-
end
|
|
135
|
-
|
|
136
|
-
end
|
|
137
63
|
end
|
|
138
64
|
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: craigslist-data
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.1
|
|
4
|
+
version: 0.0.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- mark nery
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2013-06-
|
|
11
|
+
date: 2013-06-21 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: nokogiri
|
|
@@ -79,7 +79,6 @@ files:
|
|
|
79
79
|
- README.md
|
|
80
80
|
- Rakefile
|
|
81
81
|
- craigslist-data.gemspec
|
|
82
|
-
- lib/craigslist/cities.rb
|
|
83
82
|
- lib/craigslist/craigslist.rb
|
|
84
83
|
- lib/craigslist/data.rb
|
|
85
84
|
- lib/craigslist/data/version.rb
|
|
@@ -89,10 +88,7 @@ homepage: https://github.com/mark-nery/craigslist-data
|
|
|
89
88
|
licenses:
|
|
90
89
|
- MIT
|
|
91
90
|
metadata: {}
|
|
92
|
-
post_install_message:
|
|
93
|
-
! The 'craigslist-data' gem has been deprecated and has been replaced by 'craigslist_scraper'.
|
|
94
|
-
! See: http://rubygems.org/gems/craigslist_scraper
|
|
95
|
-
! And: https://github.com/mark-nery/craigslist_scraper
|
|
91
|
+
post_install_message:
|
|
96
92
|
rdoc_options: []
|
|
97
93
|
require_paths:
|
|
98
94
|
- lib
|
|
@@ -103,9 +99,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
103
99
|
version: '0'
|
|
104
100
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
105
101
|
requirements:
|
|
106
|
-
- - '
|
|
102
|
+
- - '>='
|
|
107
103
|
- !ruby/object:Gem::Version
|
|
108
|
-
version:
|
|
104
|
+
version: '0'
|
|
109
105
|
requirements: []
|
|
110
106
|
rubyforge_project:
|
|
111
107
|
rubygems_version: 2.0.3
|
data/lib/craigslist/cities.rb
DELETED
|
@@ -1,3 +0,0 @@
|
|
|
1
|
-
module Cities
|
|
2
|
-
CITIES = ["auburn", "bham", "dothan", "shoals", "gadsden", "huntsville", "mobile", "montgomery", "tuscaloosa", "anchorage", "fairbanks", "kenai", "juneau", "flagstaff", "mohave", "phoenix", "prescott", "showlow", "sierravista", "tucson", "yuma", "fayar", "fortsmith", "jonesboro", "littlerock", "texarkana", "bakersfield", "chico", "fresno", "goldcountry", "hanford", "humboldt", "imperial", "inlandempire", "losangeles", "mendocino", "merced", "modesto", "monterey", "orangecounty", "palmsprings", "redding", "sacramento", "sandiego", "sfbay", "slo", "santabarbara", "santamaria", "siskiyou", "stockton", "susanville", "ventura", "visalia", "yubasutter", "boulder", "cosprings", "denver", "eastco", "fortcollins", "rockies", "pueblo", "westslope", "newlondon", "hartford", "newhaven", "nwct", "delaware", "washingtondc", "daytona", "keys", "fortlauderdale", "fortmyers", "gainesville", "cfl", "jacksonville", "lakeland", "lakecity", "ocala", "okaloosa", "orlando", "panamacity", "pensacola", "sarasota", "miami", "spacecoast", "staugustine", "tallahassee", "tampa", "treasure", "westpalmbeach", "albanyga", "athensga", "atlanta", "augusta", "brunswick", "columbusga", "macon", "nwga", "savannah", "statesboro", "valdosta", "honolulu", "boise", "eastidaho", "lewiston", "twinfalls", "bn", "chambana", "chicago", "decatur", "lasalle", "mattoon", "peoria", "rockford", "carbondale", "springfieldil", "quincy", "bloomington", "evansville", "fortwayne", "indianapolis", "kokomo", "tippecanoe", "muncie", "richmondin", "southbend", "terrehaute", "ames", "cedarrapids", "desmoines", "dubuque", "fortdodge", "iowacity", "masoncity", "quadcities", "siouxcity", "ottumwa", "waterloo", "lawrence", "ksu", "nwks", "salina", "seks", "swks", "topeka", "wichita", "bgky", "eastky", "lexington", "louisville", "owensboro", "westky", "batonrouge", "cenla", "houma", "lafayette", "lakecharles", "monroe", "neworleans", "shreveport", "maine", "annapolis", "baltimore", "easternshore", "frederick", "smd", "westmd", "boston", "capecod", "southcoast", "westernmass", "worcester", "annarbor", "battlecreek", "centralmich", "detroit", "flint", "grandrapids", "holland", "jxn", "kalamazoo", "lansing", "monroemi", "muskegon", "nmi", "porthuron", "saginaw", "swmi", "thumb", "up", "bemidji", "brainerd", "duluth", "mankato", "minneapolis", "rmn", "marshall", "stcloud", "gulfport", "hattiesburg", "jackson", "meridian", "northmiss", "natchez", "columbiamo", "joplin", "kansascity", "kirksville", "loz", "semo", "springfield", "stjoseph", "stlouis", "billings", "bozeman", "butte", "greatfalls", "helena", "kalispell", "missoula", "montana", "grandisland", "lincoln", "northplatte", "omaha", "scottsbluff", "elko", "lasvegas", "reno", "nh", "cnj", "jerseyshore", "newjersey", "southjersey", "albuquerque", "clovis", "farmington", "lascruces", "roswell", "santafe", "albany", "binghamton", "buffalo", "catskills", "chautauqua", "elmira", "fingerlakes", "glensfalls", "hudsonvalley", "ithaca", "longisland", "newyork", "oneonta", "plattsburgh", "potsdam", "rochester", "syracuse", "twintiers", "utica", "watertown", "asheville", "boone", "charlotte", "eastnc", "fayetteville", "greensboro", "hickory", "onslow", "outerbanks", "raleigh", "wilmington", "winstonsalem", "bismarck", "fargo", "grandforks", "nd", "akroncanton", "ashtabula", "athensohio", "chillicothe", "cincinnati", "cleveland", "columbus", "dayton", "limaohio", "mansfield", "sandusky", "toledo", "tuscarawas", "youngstown", "zanesville", "lawton", "enid", "oklahomacity", "stillwater", "tulsa", "bend", "corvallis", "eastoregon", "eugene", "klamath", "medford", "oregoncoast", "portland", "roseburg", "salem", "altoona", "chambersburg", "erie", "harrisburg", "lancaster", "allentown", "meadville", "philadelphia", "pittsburgh", "poconos", "reading", "scranton", "pennstate", "williamsport", "york", "providence", "charleston", "columbia", "florencesc", "greenville", "hiltonhead", "myrtlebeach", "nesd", "csd", "rapidcity", "siouxfalls", "sd", "chattanooga", "clarksville", "cookeville", "jacksontn", "knoxville", "memphis", "nashville", "tricities", "abilene", "amarillo", "austin", "beaumont", "brownsville", "collegestation", "corpuschristi", "dallas", "nacogdoches", "delrio", "elpaso", "galveston", "houston", "killeen", "laredo", "lubbock", "mcallen", "odessa", "sanangelo", "sanantonio", "sanmarcos", "bigbend", "texoma", "easttexas", "victoriatx", "waco", "wichitafalls", "logan", "ogden", "provo", "saltlakecity", "stgeorge", "burlington", "charlottesville", "danville", "fredericksburg", "norfolk", "harrisonburg", "lynchburg", "blacksburg", "richmond", "roanoke", "swva", "winchester", "bellingham", "kpr", "moseslake", "olympic", "pullman", "seattle", "skagit", "spokane", "wenatchee", "yakima", "charlestonwv", "martinsburg", "huntington", "morgantown", "wheeling", "parkersburg", "swv", "wv", "appleton", "eauclaire", "greenbay", "janesville", "racine", "lacrosse", "madison", "milwaukee", "northernwi", "sheboygan", "wausau", "wyoming", "micronesia", "puertorico", "virgin", "brussels", "bulgaria", "zagreb", "copenhagen", "bordeaux", "rennes", "grenoble", "lille", "loire", "lyon", "marseilles", "montpellier", "cotedazur", "rouen", "paris", "strasbourg", "toulouse", "budapest", "reykjavik", "dublin", "luxembourg", "amsterdam", "oslo", "bucharest", "moscow", "stpetersburg", "ukraine", "bangladesh", "micronesia", "jakarta", "tehran", "baghdad", "haifa", "jerusalem", "telaviv", "ramallah", "kuwait", "beirut", "malaysia", "pakistan", "dubai", "vietnam", "auckland", "christchurch", "wellington", "buenosaires", "lapaz", "belohorizonte", "brasilia", "curitiba", "fortaleza", "portoalegre", "recife", "rio", "salvador", "saopaulo", "caribbean", "santiago", "colombia", "costarica", "santodomingo", "quito", "elsalvador", "guatemala", "managua", "panama", "lima", "puertorico", "montevideo", "caracas", "virgin", "cairo", "addisababa", "accra", "kenya", "casablanca", "tunis"]
|
|
3
|
-
end
|