gh-trending 1.0.0 → 1.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/gh_trending/developer.rb +5 -1
- data/lib/gh_trending/developers.rb +2 -0
- data/lib/gh_trending/parsers/developers_parser.rb +22 -15
- data/lib/gh_trending/parsers/html_changed_error.rb +7 -4
- data/lib/gh_trending/parsers/parser.rb +1 -0
- data/lib/gh_trending/parsers/parser_arguments.rb +7 -2
- data/lib/gh_trending/parsers/repos_parser.rb +20 -9
- data/lib/gh_trending/repositories.rb +2 -0
- data/lib/gh_trending/repository.rb +9 -3
- data/lib/gh_trending/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 461ee5d745b9d3b316bd3cb709efbcdeeb7507ea
|
4
|
+
data.tar.gz: 2d2a4630245ab5e3bae4f35f64091fa8896fe942
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3b09330b0ae4a00ab8941a18e21e330a719ac0477dc7741e8c7e2b0b8ed78b867c6b331fd73ab92048b9aaef9843b2fa96efb995de6d43cba9ed95ac539b5aef
|
7
|
+
data.tar.gz: 66225f25a11535bca3c1004927cf28f87733fd5d51336a10841e830dd76ddca1349adbc8b1e4b8647b846b041468a7c3c68dcf18c25d4d23c7d5c7bce3e0aa36
|
@@ -5,7 +5,11 @@ module Gh
|
|
5
5
|
class Developer
|
6
6
|
attr_reader :full_name, :user_profile_url, :hot_repository_url
|
7
7
|
|
8
|
-
def initialize(
|
8
|
+
def initialize(
|
9
|
+
full_name: nil,
|
10
|
+
user_profile_url: nil,
|
11
|
+
hot_repository_url: nil
|
12
|
+
)
|
9
13
|
@full_name = full_name || ''
|
10
14
|
@user_profile_url = user_profile_url || ''
|
11
15
|
@hot_repository_url = hot_repository_url || ''
|
@@ -4,17 +4,21 @@ module Gh
|
|
4
4
|
# Parses the trending developers page
|
5
5
|
class DevelopersParser < Parser
|
6
6
|
LIST_WITH_DEVS_NODES_HTML_CLASS = 'ol.leaderboard-list'.freeze
|
7
|
-
H2_WITH_DEVELOPER_NAME
|
8
|
-
ANCHOR_WITH_HOT_REPOSITORY
|
7
|
+
H2_WITH_DEVELOPER_NAME = 'h2.user-leaderboard-list-name'.freeze
|
8
|
+
ANCHOR_WITH_HOT_REPOSITORY = 'a.repo-snipit'.freeze
|
9
9
|
SPAN_WITH_FULL_NAME = 'span.full-name'.freeze
|
10
|
-
REGEXP_FOR_FULLNAME = /\((.+)\)
|
10
|
+
REGEXP_FOR_FULLNAME = /\((.+)\)/
|
11
11
|
|
12
12
|
def parse(parser_args)
|
13
13
|
url = build_request_url_with(parser_args)
|
14
14
|
page = agent.get(url)
|
15
15
|
parsed_page = page.at(LIST_WITH_DEVS_NODES_HTML_CLASS)
|
16
16
|
|
17
|
-
|
17
|
+
if parsed_page.nil?
|
18
|
+
changed_page = :front_page_with_devs
|
19
|
+
raise Gh::Trending::Parsers::HtmlChangedError.new(changed_page)
|
20
|
+
end
|
21
|
+
|
18
22
|
devs = parsed_page.search('li')
|
19
23
|
build_developers_from_node devs
|
20
24
|
end
|
@@ -25,11 +29,13 @@ module Gh
|
|
25
29
|
|
26
30
|
private
|
27
31
|
|
28
|
-
def build_developers_from_node
|
29
|
-
developers_node.map
|
32
|
+
def build_developers_from_node(developers_node)
|
33
|
+
developers_node.map do |dev_node|
|
34
|
+
Developer.new extract_developer_info dev_node
|
35
|
+
end
|
30
36
|
end
|
31
37
|
|
32
|
-
def extract_developer_info
|
38
|
+
def extract_developer_info(developer_node)
|
33
39
|
{
|
34
40
|
full_name: get_full_name(developer_node),
|
35
41
|
hot_repository_url: get_hot_repository_url(developer_node),
|
@@ -37,21 +43,22 @@ module Gh
|
|
37
43
|
}
|
38
44
|
end
|
39
45
|
|
40
|
-
def get_profile_url
|
46
|
+
def get_profile_url(dev_node)
|
41
47
|
parsed_url = dev_node.search(H2_WITH_DEVELOPER_NAME)
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
48
|
+
.children
|
49
|
+
.search('a')[0]
|
50
|
+
.attributes['href']
|
51
|
+
.value
|
46
52
|
build_github_url(parsed_url)
|
47
53
|
end
|
48
54
|
|
49
|
-
def get_hot_repository_url
|
50
|
-
parsed_url = dev_node.search(ANCHOR_WITH_HOT_REPOSITORY)[0]
|
55
|
+
def get_hot_repository_url(dev_node)
|
56
|
+
parsed_url = dev_node.search(ANCHOR_WITH_HOT_REPOSITORY)[0]
|
57
|
+
.attributes['href'].value
|
51
58
|
build_github_url(parsed_url)
|
52
59
|
end
|
53
60
|
|
54
|
-
def get_full_name
|
61
|
+
def get_full_name(dev_node)
|
55
62
|
span_with_full_name = dev_node.search(SPAN_WITH_FULL_NAME)
|
56
63
|
return nil if span_with_full_name.empty?
|
57
64
|
|
@@ -12,7 +12,9 @@ module Gh
|
|
12
12
|
@page_that_changed = page_that_changed
|
13
13
|
page_changed_human_format = build_human_format_page_changed
|
14
14
|
|
15
|
-
msg ||=
|
15
|
+
msg ||= 'The page can\'t be crawled anymore because' \
|
16
|
+
"#{page_changed_human_format} changed its HTML format." \
|
17
|
+
'Submit a PR to fix it!'
|
16
18
|
super(msg)
|
17
19
|
end
|
18
20
|
|
@@ -21,11 +23,12 @@ module Gh
|
|
21
23
|
def build_human_format_page_changed
|
22
24
|
case page_that_changed
|
23
25
|
when :front_page_with_repos
|
24
|
-
|
26
|
+
'the front page that lists all trending repositories'
|
25
27
|
when :front_page_with_devs
|
26
|
-
|
28
|
+
'the front page that lists all trending developers'
|
27
29
|
else
|
28
|
-
|
30
|
+
error = "Unknown page_that_changed symbol: #{page_that_changed}"
|
31
|
+
raise ArgumentError, error
|
29
32
|
end
|
30
33
|
end
|
31
34
|
end
|
@@ -18,7 +18,7 @@ module Gh
|
|
18
18
|
TIME_PERIOD_ARG = :time_period
|
19
19
|
LANGUAGE_ARG = :language
|
20
20
|
|
21
|
-
KNOWN_TIME_PERIODS = %i
|
21
|
+
KNOWN_TIME_PERIODS = %i[daily weekly monthly].freeze
|
22
22
|
DEFAULT_TIME_PERIOD = KNOWN_TIME_PERIODS.first
|
23
23
|
|
24
24
|
def initialize(**args)
|
@@ -42,7 +42,12 @@ module Gh
|
|
42
42
|
|
43
43
|
def sanitize_time_period(time_period)
|
44
44
|
return nil if time_period.nil?
|
45
|
-
|
45
|
+
|
46
|
+
unless KNOWN_TIME_PERIODS.include? time_period
|
47
|
+
error_msg = "The time period '#{time_period}' is invalid. " \
|
48
|
+
"Supported: #{KNOWN_TIME_PERIODS}"
|
49
|
+
raise UnknownTimePeriod.new(error_msg)
|
50
|
+
end
|
46
51
|
|
47
52
|
time_period
|
48
53
|
end
|
@@ -13,7 +13,12 @@ module Gh
|
|
13
13
|
page = agent.get(url)
|
14
14
|
parsed_page = page.at(LIST_WITH_REPO_NODES_HTML_CLASS)
|
15
15
|
|
16
|
-
|
16
|
+
if parsed_page.nil?
|
17
|
+
changed_page = :front_page_with_repos
|
18
|
+
error = Gh::Trending::Parsers::HtmlChangedError.new(changed_page)
|
19
|
+
raise error
|
20
|
+
end
|
21
|
+
|
17
22
|
repositories = parsed_page.search('li')
|
18
23
|
build_repositories_from_node(repositories)
|
19
24
|
end
|
@@ -24,11 +29,13 @@ module Gh
|
|
24
29
|
|
25
30
|
private
|
26
31
|
|
27
|
-
def build_repositories_from_node
|
28
|
-
repositories_node.map
|
32
|
+
def build_repositories_from_node(repositories_node)
|
33
|
+
repositories_node.map do |repo_node|
|
34
|
+
Repository.new extract_repo_data repo_node
|
35
|
+
end
|
29
36
|
end
|
30
37
|
|
31
|
-
def extract_repo_data
|
38
|
+
def extract_repo_data(repo_node)
|
32
39
|
{
|
33
40
|
name: get_name(repo_node),
|
34
41
|
description: get_description(repo_node),
|
@@ -38,19 +45,19 @@ module Gh
|
|
38
45
|
}
|
39
46
|
end
|
40
47
|
|
41
|
-
def get_name
|
48
|
+
def get_name(repo_node)
|
42
49
|
repo_node.search('h3')[0].children[1].attributes['href'].value
|
43
50
|
end
|
44
51
|
|
45
|
-
def get_description
|
52
|
+
def get_description(repo_node)
|
46
53
|
repo_node.search(DIV_WITH_DESCRIPTION).text.strip
|
47
54
|
end
|
48
55
|
|
49
|
-
def get_language
|
56
|
+
def get_language(repo_node)
|
50
57
|
repo_node.search(SPAN_WITH_LANGUAGE).text.strip
|
51
58
|
end
|
52
59
|
|
53
|
-
def get_stars
|
60
|
+
def get_stars(repo_node)
|
54
61
|
string_with_stars = repo_node.search(SPAN_WITH_STARS).text.strip
|
55
62
|
# Github shows the string "1,263 stars this week"
|
56
63
|
# Lets strip only the number.
|
@@ -65,7 +72,11 @@ module Gh
|
|
65
72
|
end
|
66
73
|
|
67
74
|
def get_repo_url(repo_node)
|
68
|
-
parsed_url = repo_node.search('h3')
|
75
|
+
parsed_url = repo_node.search('h3')
|
76
|
+
.children
|
77
|
+
.search('a')[0]
|
78
|
+
.attributes['href']
|
79
|
+
.value
|
69
80
|
build_github_url(parsed_url)
|
70
81
|
end
|
71
82
|
end
|
@@ -6,6 +6,8 @@ require 'gh_trending/parsers/html_changed_error'
|
|
6
6
|
|
7
7
|
module Gh
|
8
8
|
module Trending
|
9
|
+
# Returns an array of representation of the trending repositories in github.
|
10
|
+
# Usage example: Gh::Trending::Repositories.new.get(language: 'ruby')
|
9
11
|
class Repositories
|
10
12
|
attr_reader :repos_list
|
11
13
|
|
@@ -4,9 +4,15 @@ module Gh
|
|
4
4
|
# repositories page.
|
5
5
|
class Repository
|
6
6
|
attr_reader :name, :description, :language, :period_stars, :url
|
7
|
-
# Period stars correspond to the stars that the repository has in the
|
8
|
-
# period being considered, for example: 283 stars THIS WEEK.
|
9
|
-
def initialize(
|
7
|
+
# Period stars correspond to the stars that the repository has in the
|
8
|
+
# current period being considered, for example: 283 stars THIS WEEK.
|
9
|
+
def initialize(
|
10
|
+
name: nil,
|
11
|
+
description: nil,
|
12
|
+
language: nil,
|
13
|
+
period_stars: nil,
|
14
|
+
url: nil
|
15
|
+
)
|
10
16
|
@name = name || ''
|
11
17
|
@description = description || ''
|
12
18
|
@language = language || ''
|
data/lib/gh_trending/version.rb
CHANGED