repocrawler 0.1.16 → 0.1.17
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/repocrawler/crawler.rb +49 -26
- data/lib/repocrawler/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c26e53deca5a1fdc7695619ad8c2b0e7e12c9b53
|
4
|
+
data.tar.gz: 99baff26279fcf60051973161f8f0bdda2bd32cf
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 557ddd7bc81605a3ad8bfd32de1b83cc275cd0f98f1809fcc2066cd41ca38cf94d51d5bb5181aaad6e439cc372acbd6f8f402a736934174948b097f3ebae4b30
|
7
|
+
data.tar.gz: 9d9f93e0e171091f0af2a26ffdd24650e646a64626de2d4d42cbaf2547aff77d3ca766c84b59b0ef989240d89a36eb75b2cc05652a0e5c8ac39c9a1e59323d1a
|
data/lib/repocrawler/crawler.rb
CHANGED
@@ -37,11 +37,16 @@ module Repos
|
|
37
37
|
def get_contributors
|
38
38
|
contributors = HTTParty.get(@GITHUB_API_BASE_URL + "/contributors?access_token=#{@access_token}", headers: {
|
39
39
|
"User-Agent" => @user_agent
|
40
|
-
})
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
40
|
+
})
|
41
|
+
if contributors['message'] === 'Not Found'
|
42
|
+
contributor = nil
|
43
|
+
else
|
44
|
+
contributors.map do |contributor|
|
45
|
+
{
|
46
|
+
'name' => contributor['login'],
|
47
|
+
'contributions' => contributor['contributions']
|
48
|
+
}
|
49
|
+
end
|
45
50
|
end
|
46
51
|
|
47
52
|
contributors
|
@@ -50,8 +55,13 @@ module Repos
|
|
50
55
|
# get the total commits
|
51
56
|
def get_total_commits
|
52
57
|
contributors = get_contributors
|
53
|
-
|
54
|
-
|
58
|
+
|
59
|
+
if contributors.nil?
|
60
|
+
commits = nil
|
61
|
+
else
|
62
|
+
commits = contributors.reduce(0) do |sum, num|
|
63
|
+
sum + num['contributions']
|
64
|
+
end
|
55
65
|
end
|
56
66
|
|
57
67
|
commits
|
@@ -110,6 +120,11 @@ module Repos
|
|
110
120
|
commits_fetch = HTTParty.get(@GITHUB_API_BASE_URL + "/commits?page=#{page}&access_token=#{@access_token}", headers: {
|
111
121
|
"User-Agent" => @user_agent
|
112
122
|
})
|
123
|
+
|
124
|
+
if commits_fetch['message'] === 'Not Found'
|
125
|
+
break
|
126
|
+
end
|
127
|
+
|
113
128
|
if commits_fetch.count === 0
|
114
129
|
stop = true
|
115
130
|
end
|
@@ -137,6 +152,11 @@ module Repos
|
|
137
152
|
issue_fetch = HTTParty.get(@GITHUB_API_BASE_URL + "/issues?state=closed&page=#{page}&access_token=#{@access_token}", headers: {
|
138
153
|
"User-Agent" => @user_agent
|
139
154
|
})
|
155
|
+
|
156
|
+
if issue_fetch['message'] === 'Not Found'
|
157
|
+
break
|
158
|
+
end
|
159
|
+
|
140
160
|
if issue_fetch.count === 0
|
141
161
|
stop = true
|
142
162
|
end
|
@@ -179,30 +199,33 @@ module Repos
|
|
179
199
|
"User-Agent" => @user_agent
|
180
200
|
})
|
181
201
|
|
202
|
+
if github_contents['message'] === 'Not Found'
|
203
|
+
return nil
|
204
|
+
else
|
205
|
+
readme_file = ''
|
206
|
+
github_contents.each do |content|
|
207
|
+
readme_file = content['name'] if content['name'] =~ /^README/
|
208
|
+
end
|
182
209
|
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
stop_words = []
|
189
|
-
File.open(File.expand_path("../../public/stop_words.txt", File.dirname(__FILE__)), "r") do |f|
|
190
|
-
f.each_line do |line|
|
191
|
-
stop_words << line.gsub(/\n/,"")
|
210
|
+
stop_words = []
|
211
|
+
File.open(File.expand_path("../../public/stop_words.txt", File.dirname(__FILE__)), "r") do |f|
|
212
|
+
f.each_line do |line|
|
213
|
+
stop_words << line.gsub(/\n/,"")
|
214
|
+
end
|
192
215
|
end
|
193
|
-
end
|
194
216
|
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
217
|
+
readme = HTTParty.get(@GITHUB_README_URL + "/#{readme_file}")
|
218
|
+
words = readme.split(' ')
|
219
|
+
freqs = Hash.new(0)
|
220
|
+
words.each do |word|
|
221
|
+
if word =~ /^\w+$/ && !stop_words.include?(word.downcase)
|
222
|
+
freqs[word] += 1
|
223
|
+
end
|
201
224
|
end
|
202
|
-
|
203
|
-
freqs = freqs.sort_by { |word, freq| freq }.reverse!
|
225
|
+
freqs = freqs.sort_by { |word, freq| freq }.reverse!
|
204
226
|
|
205
|
-
|
227
|
+
freqs
|
228
|
+
end
|
206
229
|
end
|
207
230
|
end
|
208
231
|
|
data/lib/repocrawler/version.rb
CHANGED