popularity 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.document +5 -0
- data/.rspec +3 -0
- data/Gemfile +21 -0
- data/Gemfile.lock +126 -0
- data/LICENSE.txt +20 -0
- data/README.rdoc +92 -0
- data/Rakefile +54 -0
- data/VERSION +1 -0
- data/lib/.DS_Store +0 -0
- data/lib/popularity.rb +92 -0
- data/lib/popularity/crawler.rb +76 -0
- data/lib/popularity/facebook.rb +26 -0
- data/lib/popularity/github.rb +28 -0
- data/lib/popularity/google_plus.rb +22 -0
- data/lib/popularity/medium.rb +33 -0
- data/lib/popularity/pinterest.rb +21 -0
- data/lib/popularity/reddit_comment.rb +32 -0
- data/lib/popularity/reddit_post.rb +47 -0
- data/lib/popularity/reddit_share.rb +72 -0
- data/lib/popularity/results_container.rb +70 -0
- data/lib/popularity/search.rb +79 -0
- data/lib/popularity/soundcloud.rb +44 -0
- data/lib/popularity/twitter.rb +21 -0
- data/popularity.gemspec +124 -0
- data/spec/cassettes/facebook.yml +50 -0
- data/spec/cassettes/github-valid.yml +105 -0
- data/spec/cassettes/google_plus.yml +89 -0
- data/spec/cassettes/medium-valid.yml +110 -0
- data/spec/cassettes/pinterest.yml +44 -0
- data/spec/cassettes/reddit-comment.yml +207 -0
- data/spec/cassettes/reddit-post.yml +174 -0
- data/spec/cassettes/reddit.yml +183 -0
- data/spec/cassettes/result-container-test.yml +97 -0
- data/spec/cassettes/search-multi.yml +1211 -0
- data/spec/cassettes/search.yml +598 -0
- data/spec/cassettes/soundcloud-valid.yml +682 -0
- data/spec/cassettes/twitter.yml +64 -0
- data/spec/cassettes/unknown-reddit-post.yml +68 -0
- data/spec/facebook_spec.rb +23 -0
- data/spec/github_spec.rb +39 -0
- data/spec/google_plus_spec.rb +27 -0
- data/spec/medium_spec.rb +47 -0
- data/spec/pinterest_spec.rb +27 -0
- data/spec/reddit_comment_spec.rb +60 -0
- data/spec/reddit_post_spec.rb +64 -0
- data/spec/reddit_spec.rb +89 -0
- data/spec/results_container_spec.rb +61 -0
- data/spec/search_spec.rb +79 -0
- data/spec/soundcloud_spec.rb +66 -0
- data/spec/spec_helper.rb +95 -0
- data/spec/twitter_spec.rb +19 -0
- data/test/facebook_test.rb +28 -0
- data/test/fixtures/vcr_cassettes/facebook.yml +50 -0
- data/test/fixtures/vcr_cassettes/google.yml +425 -0
- data/test/helper.rb +34 -0
- data/test/test_non_specific.rb +27 -0
- data/test/test_twitter.rb +28 -0
- metadata +242 -0
@@ -0,0 +1,26 @@
|
|
1
|
+
module Popularity
|
2
|
+
class Facebook < Crawler
|
3
|
+
def shares
|
4
|
+
response_json['shares'].to_f.to_i
|
5
|
+
end
|
6
|
+
|
7
|
+
def comments
|
8
|
+
response_json['comments'].to_f.to_i
|
9
|
+
end
|
10
|
+
|
11
|
+
def as_json(options = {})
|
12
|
+
{ "shares" => shares,
|
13
|
+
"comments" => comments }
|
14
|
+
end
|
15
|
+
|
16
|
+
def total
|
17
|
+
shares + comments
|
18
|
+
end
|
19
|
+
|
20
|
+
protected
|
21
|
+
|
22
|
+
def request_url
|
23
|
+
"http://graph.facebook.com/?id=#{@url}"
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Popularity
|
2
|
+
class Github < Crawler
|
3
|
+
def stars
|
4
|
+
response_json.size
|
5
|
+
end
|
6
|
+
|
7
|
+
def as_json(options = {})
|
8
|
+
{ "stars" => stars }
|
9
|
+
end
|
10
|
+
|
11
|
+
def total
|
12
|
+
response_json.size
|
13
|
+
end
|
14
|
+
|
15
|
+
def valid?
|
16
|
+
host == 'github.com'
|
17
|
+
end
|
18
|
+
|
19
|
+
protected
|
20
|
+
|
21
|
+
def request_url
|
22
|
+
parts = @url.split("/").last(2)
|
23
|
+
repo = parts.last
|
24
|
+
owner = parts.first
|
25
|
+
"https://api.github.com/repos/#{owner}/#{repo}/stargazers"
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module Popularity
|
2
|
+
class GooglePlus < Crawler
|
3
|
+
def plus_ones
|
4
|
+
matches = response.scan(/window.__SSR = {c\: (\d+.\d+E?\d+)/)
|
5
|
+
matches.flatten.first.to_f.to_i
|
6
|
+
end
|
7
|
+
|
8
|
+
def as_json(options = {})
|
9
|
+
{"plus_ones" => plus_ones}
|
10
|
+
end
|
11
|
+
|
12
|
+
def total
|
13
|
+
plus_ones
|
14
|
+
end
|
15
|
+
|
16
|
+
protected
|
17
|
+
|
18
|
+
def request_url
|
19
|
+
"https://plusone.google.com/_/+1/fastbutton?url=#{URI::encode(@url)}"
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module Popularity
|
2
|
+
class Medium < Crawler
|
3
|
+
def recommends
|
4
|
+
response_json["payload"]["value"]["count"]
|
5
|
+
end
|
6
|
+
|
7
|
+
def as_json(options = {})
|
8
|
+
{"recommends" => recommends}
|
9
|
+
end
|
10
|
+
|
11
|
+
def total
|
12
|
+
recommends
|
13
|
+
end
|
14
|
+
|
15
|
+
def valid?
|
16
|
+
host == 'medium.com'
|
17
|
+
end
|
18
|
+
|
19
|
+
protected
|
20
|
+
|
21
|
+
def medium_id
|
22
|
+
@url.split("/").last.split("-").last
|
23
|
+
end
|
24
|
+
|
25
|
+
def request_url
|
26
|
+
"https://medium.com/p/#{medium_id}/upvotes"
|
27
|
+
end
|
28
|
+
|
29
|
+
def response_json
|
30
|
+
JSON.parse(response.sub("])}while(1);</x>", ""))
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Popularity
|
2
|
+
class Pinterest < Crawler
|
3
|
+
def pins
|
4
|
+
JSON.parse(response.gsub('receiveCount(','').gsub(')',''))['count'].to_f.to_i
|
5
|
+
end
|
6
|
+
|
7
|
+
def as_json(options = {})
|
8
|
+
{"pins" => pins}
|
9
|
+
end
|
10
|
+
|
11
|
+
def total
|
12
|
+
pins
|
13
|
+
end
|
14
|
+
|
15
|
+
protected
|
16
|
+
|
17
|
+
def request_url
|
18
|
+
"http://api.pinterest.com/v1/urls/count.json?url=#{@url}"
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
module Popularity
|
2
|
+
class RedditComment < Crawler
|
3
|
+
def score
|
4
|
+
response_json[1]["data"]["children"][0]["data"]["score"]
|
5
|
+
end
|
6
|
+
|
7
|
+
def as_json(options = {})
|
8
|
+
{"score" => score}
|
9
|
+
end
|
10
|
+
|
11
|
+
def total
|
12
|
+
score
|
13
|
+
end
|
14
|
+
|
15
|
+
def valid?
|
16
|
+
return false unless host == 'reddit.com'
|
17
|
+
|
18
|
+
path = URI.parse(@url).path
|
19
|
+
path.split('/').delete_if { |a| a.empty? }.size == 6
|
20
|
+
end
|
21
|
+
|
22
|
+
def name
|
23
|
+
"reddit"
|
24
|
+
end
|
25
|
+
|
26
|
+
protected
|
27
|
+
|
28
|
+
def request_url
|
29
|
+
"#{@url}.json"
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
module Popularity
|
2
|
+
class RedditPost < Crawler
|
3
|
+
def score
|
4
|
+
begin
|
5
|
+
response_json[0]["data"]["children"][0]["data"]["score"]
|
6
|
+
rescue
|
7
|
+
0
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
def comments
|
12
|
+
begin
|
13
|
+
response_json[0]["data"]["children"][0]["data"]["num_comments"]
|
14
|
+
rescue
|
15
|
+
0
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def as_json(options = {})
|
20
|
+
{
|
21
|
+
"comments" => comments,
|
22
|
+
"score" => score
|
23
|
+
}
|
24
|
+
end
|
25
|
+
|
26
|
+
def total
|
27
|
+
comments + score
|
28
|
+
end
|
29
|
+
|
30
|
+
def valid?
|
31
|
+
return false unless host == 'reddit.com'
|
32
|
+
|
33
|
+
path = URI.parse(@url).path
|
34
|
+
path.split('/').delete_if { |a| a.empty? }.size < 6
|
35
|
+
end
|
36
|
+
|
37
|
+
def name
|
38
|
+
"reddit"
|
39
|
+
end
|
40
|
+
|
41
|
+
protected
|
42
|
+
|
43
|
+
def request_url
|
44
|
+
"#{@url}.json"
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
module Popularity
|
2
|
+
class RedditShare < Crawler
|
3
|
+
include Popularity::ContainerMethods
|
4
|
+
|
5
|
+
class RedditResult < Popularity::RedditPost
|
6
|
+
def initialize(url, r)
|
7
|
+
super(url)
|
8
|
+
@response = r
|
9
|
+
|
10
|
+
self
|
11
|
+
end
|
12
|
+
|
13
|
+
def has_response?
|
14
|
+
true
|
15
|
+
end
|
16
|
+
|
17
|
+
def valid?
|
18
|
+
URI.parse(@url).host
|
19
|
+
end
|
20
|
+
|
21
|
+
def fetch
|
22
|
+
false
|
23
|
+
end
|
24
|
+
|
25
|
+
def fetch_async
|
26
|
+
false
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def initialize(*args)
|
31
|
+
super(*args)
|
32
|
+
posts_json = response_json["data"]["children"]
|
33
|
+
posts_json.each do |child|
|
34
|
+
new_json = response_json.clone
|
35
|
+
|
36
|
+
new_json["data"]["children"] = [child]
|
37
|
+
url = "http://reddit.com#{child["data"]["permalink"]}"
|
38
|
+
post = RedditResult.new(url, JSON.dump([new_json]))
|
39
|
+
|
40
|
+
self.add_result(post)
|
41
|
+
end
|
42
|
+
|
43
|
+
self
|
44
|
+
end
|
45
|
+
|
46
|
+
def to_json(options ={})
|
47
|
+
total = {"comments" => 0, "posts" => 0, "score" => 0}
|
48
|
+
return total unless @results
|
49
|
+
|
50
|
+
@results.collect(&:to_json).each do |json|
|
51
|
+
json.each do |key, value|
|
52
|
+
total[key] ||= 0
|
53
|
+
total[key] += value
|
54
|
+
end
|
55
|
+
end
|
56
|
+
total["posts"] = posts
|
57
|
+
total
|
58
|
+
end
|
59
|
+
|
60
|
+
def posts
|
61
|
+
@results.size rescue 0
|
62
|
+
end
|
63
|
+
|
64
|
+
def name
|
65
|
+
"reddit"
|
66
|
+
end
|
67
|
+
|
68
|
+
def request_url
|
69
|
+
"http://www.reddit.com/r/search/search.json?q=url:#{@url}"
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
|
2
|
+
module Popularity
|
3
|
+
module ContainerMethods
|
4
|
+
def self.included(base)
|
5
|
+
base.class_eval do
|
6
|
+
def results
|
7
|
+
@results
|
8
|
+
end
|
9
|
+
|
10
|
+
def add_result(result)
|
11
|
+
@results ||= []
|
12
|
+
|
13
|
+
if @results.size > 0
|
14
|
+
verify_type = @results.first.name
|
15
|
+
if verify_type == result.name
|
16
|
+
@results << result
|
17
|
+
else
|
18
|
+
raise "ResultTypeError", "types must be the same within a results container"
|
19
|
+
end
|
20
|
+
else
|
21
|
+
@results << result
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def to_json(options ={})
|
26
|
+
individual = {}
|
27
|
+
total = {}
|
28
|
+
@results.collect do |result|
|
29
|
+
json = result.to_json
|
30
|
+
individual[result.url] = json
|
31
|
+
|
32
|
+
json.each do |key, value|
|
33
|
+
next if key == "total"
|
34
|
+
|
35
|
+
if value.is_a?(Hash)
|
36
|
+
# RedditShare breaks out into separate results for each reddit link
|
37
|
+
# I'm not a big fan of this hacky stuff here
|
38
|
+
value.each do |k,v|
|
39
|
+
total[k] ||= 0
|
40
|
+
total[k] += v
|
41
|
+
end
|
42
|
+
else
|
43
|
+
total[key] ||= 0
|
44
|
+
total[key] += value
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
individual["total"] = total
|
50
|
+
individual
|
51
|
+
end
|
52
|
+
|
53
|
+
def method_missing(method_sym, *arguments, &block)
|
54
|
+
return 0 unless @results
|
55
|
+
collection = @results.collect do |result|
|
56
|
+
result.send(method_sym, *arguments)
|
57
|
+
end
|
58
|
+
|
59
|
+
if collection.all? { |t| t.is_a?(Fixnum) }
|
60
|
+
collection.reduce(:+)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
class ResultsContainer
|
68
|
+
include Popularity::ContainerMethods
|
69
|
+
end
|
70
|
+
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
module Popularity
|
2
|
+
class Search
|
3
|
+
attr_accessor :info
|
4
|
+
attr_accessor :results
|
5
|
+
attr_accessor :sources
|
6
|
+
attr_reader :total
|
7
|
+
attr_reader :url
|
8
|
+
|
9
|
+
def initialize(url)
|
10
|
+
@url = url
|
11
|
+
@info = {}
|
12
|
+
total_score = []
|
13
|
+
|
14
|
+
selected_types.each do |network|
|
15
|
+
network.fetch_async do |code, body|
|
16
|
+
add_result(network)
|
17
|
+
begin
|
18
|
+
if network.has_response?
|
19
|
+
total_score << network.total
|
20
|
+
end
|
21
|
+
rescue Exception => e
|
22
|
+
puts "#{network.name} had an accident"
|
23
|
+
puts e.message
|
24
|
+
puts e.backtrace.join("\n")
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
loop do
|
30
|
+
# we want the requests to be asyncronous, but we don't
|
31
|
+
# want gem users to have to deal with async code
|
32
|
+
#
|
33
|
+
break if selected_types.all? { |network| network.async_done? }
|
34
|
+
end
|
35
|
+
|
36
|
+
@total = total_score.reduce(:+)
|
37
|
+
end
|
38
|
+
|
39
|
+
def to_json(options ={})
|
40
|
+
json = {}
|
41
|
+
self.results.collect do |result|
|
42
|
+
json[result.name.to_s] = result.to_json
|
43
|
+
end
|
44
|
+
|
45
|
+
self.sources.collect do |source|
|
46
|
+
json[source.to_s] = self.send(source.to_sym).to_json
|
47
|
+
end
|
48
|
+
|
49
|
+
json["total"] = total
|
50
|
+
|
51
|
+
json
|
52
|
+
end
|
53
|
+
|
54
|
+
protected
|
55
|
+
|
56
|
+
def selected_types
|
57
|
+
# github.com stats only valid for github urls, etc
|
58
|
+
@types ||= Popularity::TYPES.collect { |n|
|
59
|
+
network = n.new(@url)
|
60
|
+
network if network.valid?
|
61
|
+
}.compact
|
62
|
+
end
|
63
|
+
|
64
|
+
def add_result(result)
|
65
|
+
self.sources ||= []
|
66
|
+
self.results ||= []
|
67
|
+
self.results << result
|
68
|
+
self.sources << result.name.to_sym
|
69
|
+
|
70
|
+
self.instance_variable_set "@#{result.name}", result
|
71
|
+
|
72
|
+
self.define_singleton_method(result.name.to_sym) { result }
|
73
|
+
end
|
74
|
+
|
75
|
+
def to_s
|
76
|
+
self.class.name
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
module Popularity
|
2
|
+
class Soundcloud < Crawler
|
3
|
+
def plays
|
4
|
+
response.scan(/\"soundcloud:play_count\" content=\"([0-9]*)\"/).flatten.first.to_f.to_i
|
5
|
+
end
|
6
|
+
|
7
|
+
def likes
|
8
|
+
response.scan(/\"soundcloud:like_count\" content=\"([0-9]*)\"/).flatten.first.to_f.to_i
|
9
|
+
end
|
10
|
+
|
11
|
+
def comments
|
12
|
+
response.scan(/\"soundcloud:comments_count\" content=\"([0-9]*)\"/).flatten.first.to_f.to_i
|
13
|
+
end
|
14
|
+
|
15
|
+
def downloads
|
16
|
+
response.scan(/\"soundcloud:download_count\" content=\"([0-9]*)\"/).flatten.first.to_f.to_i
|
17
|
+
end
|
18
|
+
|
19
|
+
def as_json(options = {})
|
20
|
+
{"plays" => plays,
|
21
|
+
"likes" => likes,
|
22
|
+
"comments" => comments,
|
23
|
+
"downloads" => downloads }
|
24
|
+
end
|
25
|
+
|
26
|
+
def total
|
27
|
+
plays + likes + downloads + comments
|
28
|
+
end
|
29
|
+
|
30
|
+
def valid?
|
31
|
+
host == 'soundcloud.com'
|
32
|
+
end
|
33
|
+
|
34
|
+
protected
|
35
|
+
|
36
|
+
def response_json
|
37
|
+
#not json!
|
38
|
+
end
|
39
|
+
|
40
|
+
def request_url
|
41
|
+
@url
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|