gems-status 0.38.0 → 0.39.0
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/gems-status +1 -1
- data/lib/gems-status.rb +26 -23
- data/lib/gems-status/checkers/exists_in_upstream.rb +17 -14
- data/lib/gems-status/checkers/gem_checker.rb +8 -6
- data/lib/gems-status/checkers/git_check_messages.rb +25 -22
- data/lib/gems-status/checkers/hg_check_messages.rb +25 -22
- data/lib/gems-status/checkers/not_a_security_alert_checker.rb +161 -158
- data/lib/gems-status/checkers/not_native_gem_checker.rb +33 -30
- data/lib/gems-status/checkers/not_rails_checker.rb +16 -13
- data/lib/gems-status/checkers/print_gem_versions.rb +27 -23
- data/lib/gems-status/checkers/scm_check_messages.rb +44 -41
- data/lib/gems-status/checkers/scm_security_messages.rb +5 -3
- data/lib/gems-status/checkers/security_alert.rb +7 -5
- data/lib/gems-status/checkers/svn_check_messages.rb +29 -26
- data/lib/gems-status/gem_simple.rb +42 -40
- data/lib/gems-status/gems_command.rb +30 -28
- data/lib/gems-status/gems_composite_command.rb +85 -82
- data/lib/gems-status/gems_status_metadata.rb +2 -2
- data/lib/gems-status/html_view.rb +240 -237
- data/lib/gems-status/sources/lockfile_gems.rb +64 -61
- data/lib/gems-status/sources/obs_gems.rb +86 -83
- data/lib/gems-status/sources/ruby_gems_gems.rb +32 -30
- data/lib/gems-status/sources/ruby_gems_gems_gem_simple.rb +29 -26
- data/lib/gems-status/utils.rb +77 -74
- data/test/test-gems_command.rb +52 -49
- data/test/test-gems_composite_command.rb +43 -40
- data/test/test-helper.rb +2 -0
- data/test/test-lockfile_gems.rb +64 -61
- data/test/test-not_rails_checker.rb +45 -42
- data/test/test-obs_gems.rb +31 -29
- data/test/test-ruby_gems_gems.rb +22 -20
- data/test/test-utils.rb +42 -39
- metadata +181 -176
data/bin/gems-status
CHANGED
data/lib/gems-status.rb
CHANGED
@@ -10,33 +10,36 @@ require "gems-status/sources.rb"
|
|
10
10
|
require "gems-status/gems_composite_command"
|
11
11
|
require "gems-status/checkers"
|
12
12
|
|
13
|
-
|
14
|
-
def initialize(conf_file)
|
15
|
-
@conf_file = conf_file
|
16
|
-
begin
|
17
|
-
@conf = YAML::load(File::open(conf_file))
|
18
|
-
rescue
|
19
|
-
Utils::log_error("?", "There was a problem opening #{conf_file}")
|
20
|
-
end
|
21
|
-
end
|
13
|
+
module GemsStatus
|
22
14
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
@conf["checkers"].each do |c|
|
31
|
-
checker = eval(c["classname"]).new(c)
|
32
|
-
gems_composite_command.add_checker(checker)
|
15
|
+
class GemStatus
|
16
|
+
def initialize(conf_file)
|
17
|
+
@conf_file = conf_file
|
18
|
+
begin
|
19
|
+
@conf = YAML::load(File::open(conf_file))
|
20
|
+
rescue
|
21
|
+
Utils::log_error("?", "There was a problem opening #{conf_file}")
|
33
22
|
end
|
34
23
|
end
|
35
|
-
|
36
|
-
|
24
|
+
|
25
|
+
def execute
|
26
|
+
gems_composite_command = GemsCompositeCommand.new(@conf["target"])
|
27
|
+
@conf["sources"].each do |c|
|
28
|
+
gems = eval(c["classname"]).new(c)
|
29
|
+
gems_composite_command.add_command(gems)
|
30
|
+
end
|
31
|
+
if @conf["checkers"]
|
32
|
+
@conf["checkers"].each do |c|
|
33
|
+
checker = eval(c["classname"]).new(c)
|
34
|
+
gems_composite_command.add_checker(checker)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
if @conf["comments"]
|
38
|
+
gems_composite_command.add_comments(@conf["comments"])
|
39
|
+
end
|
40
|
+
gems_composite_command.execute
|
41
|
+
gems_composite_command.print
|
37
42
|
end
|
38
|
-
gems_composite_command.execute
|
39
|
-
gems_composite_command.print
|
40
43
|
end
|
41
44
|
end
|
42
45
|
|
@@ -3,20 +3,23 @@ require 'open-uri'
|
|
3
3
|
require 'gems-status/checkers/gem_checker'
|
4
4
|
require 'gems-status/utils'
|
5
5
|
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
6
|
+
module GemsStatus
|
7
|
+
|
8
|
+
class ExistsInUpstream < GemChecker
|
9
|
+
def check?(gem)
|
10
|
+
Utils::log_debug("Looking for #{gem.name}")
|
11
|
+
result = nil
|
12
|
+
gem_uri = "#{gem.gems_url}/#{gem.name}-#{gem.version}.gem"
|
13
|
+
begin
|
14
|
+
source = open(gem_uri)
|
15
|
+
return true
|
16
|
+
rescue
|
17
|
+
return false
|
18
|
+
end
|
19
|
+
end
|
20
|
+
def description
|
21
|
+
"This gem does not exist in upstream: "
|
16
22
|
end
|
17
23
|
end
|
18
|
-
def description
|
19
|
-
"This gem does not exist in upstream: "
|
20
|
-
end
|
21
|
-
end
|
22
24
|
|
25
|
+
end
|
@@ -2,33 +2,36 @@ require "git"
|
|
2
2
|
|
3
3
|
require "gems-status/checkers/scm_check_messages"
|
4
4
|
|
5
|
-
|
6
|
-
# This value is the maximum log entries that git.log accepts.
|
7
|
-
# It is a magic number found through testing.
|
8
|
-
MAX_NUM_MESSAGES = 8688
|
5
|
+
module GemsStatus
|
9
6
|
|
10
|
-
|
7
|
+
class GitCheckMessages < ScmCheckMessages
|
8
|
+
# This value is the maximum log entries that git.log accepts.
|
9
|
+
# It is a magic number found through testing.
|
10
|
+
MAX_NUM_MESSAGES = 8688
|
11
11
|
|
12
|
-
|
13
|
-
return commit.message
|
14
|
-
end
|
12
|
+
private
|
15
13
|
|
16
|
-
|
17
|
-
|
18
|
-
g = Git.open(name)
|
19
|
-
rescue
|
20
|
-
g = Git.clone(source_repo, name)
|
14
|
+
def message(commit)
|
15
|
+
return commit.message
|
21
16
|
end
|
22
|
-
g.lib.send(:command, 'pull')
|
23
|
-
return g.log MAX_NUM_MESSAGES
|
24
|
-
end
|
25
17
|
|
26
|
-
|
27
|
-
|
28
|
-
|
18
|
+
def messages(name, source_repo)
|
19
|
+
begin
|
20
|
+
g = Git.open(name)
|
21
|
+
rescue
|
22
|
+
g = Git.clone(source_repo, name)
|
23
|
+
end
|
24
|
+
g.lib.send(:command, 'pull')
|
25
|
+
return g.log MAX_NUM_MESSAGES
|
26
|
+
end
|
29
27
|
|
30
|
-
|
31
|
-
|
32
|
-
|
28
|
+
def commit_key(commit)
|
29
|
+
return commit.sha
|
30
|
+
end
|
31
|
+
|
32
|
+
def date(commit)
|
33
|
+
commit.date
|
34
|
+
end
|
33
35
|
|
36
|
+
end
|
34
37
|
end
|
@@ -2,35 +2,38 @@ require "mercurial-ruby"
|
|
2
2
|
|
3
3
|
require "gems-status/checkers/scm_check_messages"
|
4
4
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
5
|
+
module GemsStatus
|
6
|
+
|
7
|
+
class HgCheckMessages < ScmCheckMessages
|
8
|
+
def initialize
|
9
|
+
Mercurial.configure do |conf|
|
10
|
+
conf.hg_binary_path = "/usr/bin/hg"
|
11
|
+
end
|
9
12
|
end
|
10
|
-
end
|
11
13
|
|
12
|
-
private
|
14
|
+
private
|
13
15
|
|
14
|
-
|
15
|
-
|
16
|
-
|
16
|
+
def message(commit)
|
17
|
+
return commit.message
|
18
|
+
end
|
17
19
|
|
18
|
-
|
19
|
-
|
20
|
-
|
20
|
+
def messages(name, source_repo)
|
21
|
+
if ! File.exists?(name)
|
22
|
+
Mercurial::Repository.clone(source_repo, name, {})
|
23
|
+
end
|
24
|
+
repo = Mercurial::Repository.open(name)
|
25
|
+
repo.pull
|
26
|
+
return repo.commits
|
21
27
|
end
|
22
|
-
repo = Mercurial::Repository.open(name)
|
23
|
-
repo.pull
|
24
|
-
return repo.commits
|
25
|
-
end
|
26
28
|
|
27
|
-
|
28
|
-
|
29
|
-
|
29
|
+
def commit_key(commit)
|
30
|
+
return commit.hash_id
|
31
|
+
end
|
32
|
+
|
33
|
+
def date(commit)
|
34
|
+
commit.date
|
35
|
+
end
|
30
36
|
|
31
|
-
def date(commit)
|
32
|
-
commit.date
|
33
37
|
end
|
34
38
|
|
35
39
|
end
|
36
|
-
|
@@ -9,192 +9,195 @@ require "gems-status/checkers/hg_check_messages"
|
|
9
9
|
require "gems-status/checkers/svn_check_messages"
|
10
10
|
require "gems-status/checkers/scm_security_messages"
|
11
11
|
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
12
|
+
module GemsStatus
|
13
|
+
|
14
|
+
class NotASecurityAlertChecker < GemChecker
|
15
|
+
def initialize(conf)
|
16
|
+
Utils::check_parameters('NotASecurityAlertChecker', conf, ["fixed", "source_repos", "email_username", "email_password", "mailing_lists", "email_to"])
|
17
|
+
begin
|
18
|
+
@fixed = YAML::load(File::open(conf["fixed"]))
|
19
|
+
rescue
|
20
|
+
Utils::log_error("?", "There was a problem opening #{conf["fixed"]}")
|
21
|
+
@fixed = []
|
22
|
+
end
|
23
|
+
@source_repos = conf["source_repos"]
|
24
|
+
@security_messages = {}
|
25
|
+
@email_username = conf["email_username"]
|
26
|
+
@email_password = conf["email_password"]
|
27
|
+
@mailing_lists = conf["mailing_lists"]
|
28
|
+
@email_to = conf["email_to"]
|
29
|
+
@emails = {}
|
30
|
+
download_emails
|
20
31
|
end
|
21
|
-
@source_repos = conf["source_repos"]
|
22
|
-
@security_messages = {}
|
23
|
-
@email_username = conf["email_username"]
|
24
|
-
@email_password = conf["email_password"]
|
25
|
-
@mailing_lists = conf["mailing_lists"]
|
26
|
-
@email_to = conf["email_to"]
|
27
|
-
@emails = {}
|
28
|
-
download_emails
|
29
|
-
end
|
30
32
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
33
|
+
def download_emails
|
34
|
+
#TODO: only download new emails and keep the old ones in a database
|
35
|
+
#puts "Security email alerts from #{mailing_list} #{gmail.inbox.count(:unread, :to => mailing_list}"
|
36
|
+
Gmail.new(@email_username, @email_password) do |gmail|
|
37
|
+
@mailing_lists.each do |mailing_list|
|
38
|
+
@emails[mailing_list] = []
|
39
|
+
Utils::log_debug "Security email alerts from #{mailing_list} #{gmail.inbox.count( :to => mailing_list)}"
|
40
|
+
#TODO: only read new emails
|
41
|
+
#gmail.inbox.emails(:unread, :to => "rubyonrails-security@googlegroups.com").each do |email|
|
42
|
+
gmail.inbox.emails(:to => mailing_list).each do |email|
|
43
|
+
Utils::log_debug "Read #{email.subject}"
|
44
|
+
@emails[mailing_list] << email
|
45
|
+
end
|
46
|
+
end
|
44
47
|
end
|
45
48
|
end
|
46
|
-
end
|
47
49
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
50
|
+
def send_emails(gem)
|
51
|
+
return if @security_messages.length == 0
|
52
|
+
#gems.origin == gems.gems_url if we are looking to an upstream gem,
|
53
|
+
#for example in rubygems.org. We only care about our application gems.
|
54
|
+
return if gem.origin == gem.gems_url
|
55
|
+
mssg = ""
|
56
|
+
mssg = "#{gem.name} #{gem.version} : #{gem.origin} \n"
|
57
|
+
@security_messages.each do |k,v|
|
58
|
+
mssg = mssg + "\n #{v.desc}"
|
59
|
+
mssg = mssg + "\nFixed in #{@fixed[k]}\n" if @fixed[k]
|
60
|
+
end
|
61
|
+
@email_to.each do |email_receiver|
|
62
|
+
Gmail.new(@email_username, @email_password) do |gmail|
|
63
|
+
gmail.deliver do
|
64
|
+
to email_receiver
|
65
|
+
subject "[gems-status] security alerts for #{gem.name}"
|
66
|
+
text_part do
|
67
|
+
body mssg
|
68
|
+
end
|
69
|
+
end
|
67
70
|
end
|
68
71
|
end
|
72
|
+
Utils::log_debug "Email sent to #{@email_to} "
|
73
|
+
Utils::log_debug "with body #{mssg} "
|
69
74
|
end
|
70
|
-
Utils::log_debug "Email sent to #{@email_to} "
|
71
|
-
Utils::log_debug "with body #{mssg} "
|
72
|
-
end
|
73
75
|
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
76
|
+
def look_in_scm(gem)
|
77
|
+
version = gem.version
|
78
|
+
source_repo = source_repo(gem)
|
79
|
+
if ! source_repo
|
80
|
+
Utils::log_error gem.name, "Not source URL for #{gem.name}"
|
81
|
+
return
|
82
|
+
end
|
83
|
+
Utils::log_debug "Source URL for #{gem.name} #{source_repo}"
|
84
|
+
look_for_security_messages(gem.name, source_repo, gem.origin)
|
80
85
|
end
|
81
|
-
Utils::log_debug "Source URL for #{gem.name} #{source_repo}"
|
82
|
-
look_for_security_messages(gem.name, source_repo, gem.origin)
|
83
|
-
end
|
84
86
|
|
85
|
-
|
86
|
-
|
87
|
-
|
87
|
+
def key_for_emails(listname, gem, email)
|
88
|
+
"email_#{listname}_#{gem.name}_#{gem.origin}_#{email.uid}"
|
89
|
+
end
|
88
90
|
|
89
|
-
|
90
|
-
|
91
|
-
|
91
|
+
def match_name(str, name)
|
92
|
+
str =~ /[gem|ruby].*\b#{name}\b/
|
93
|
+
end
|
92
94
|
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
95
|
+
def look_in_emails(gem)
|
96
|
+
@emails.each do |listname, emails|
|
97
|
+
emails.each do |email|
|
98
|
+
if match_name(listname, gem.name)
|
99
|
+
@security_messages[key_for_emails(listname, gem, email)] = SecurityAlert.new(email.subject)
|
100
|
+
Utils::log_debug "looking for security emails: listname matches gem #{gem.name}: #{listname}"
|
101
|
+
next
|
102
|
+
end
|
103
|
+
if match_name(email.subject, gem.name)
|
104
|
+
@security_messages[key_for_emails(listname, gem, email)] = SecurityAlert.new(email.subject)
|
105
|
+
Utils::log_debug "looking for security emails: subject matches gem #{gem.name}: #{email.subject}"
|
106
|
+
next
|
107
|
+
end
|
105
108
|
end
|
106
109
|
end
|
107
110
|
end
|
108
|
-
end
|
109
111
|
|
110
|
-
|
111
|
-
|
112
|
-
|
112
|
+
def check?(gem)
|
113
|
+
#ignore upstream checks
|
114
|
+
return true if gem.origin == gem.gems_url
|
113
115
|
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
116
|
+
@security_messages = {}
|
117
|
+
look_in_scm(gem)
|
118
|
+
look_in_emails(gem)
|
119
|
+
filter_security_messages_already_fixed(gem.version, gem.date)
|
120
|
+
send_emails(gem)
|
121
|
+
return @security_messages.length == 0
|
122
|
+
end
|
121
123
|
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
124
|
+
def description
|
125
|
+
result = ""
|
126
|
+
@security_messages.keys.sort.each do |k|
|
127
|
+
result = result + "[#{k}] - #{@security_messages[k].desc}"
|
128
|
+
result = result + "Fixed in #{@fixed[k]}" if @fixed[k]
|
129
|
+
result = result + "<br/>"
|
130
|
+
end
|
131
|
+
result = "Security alerts: #{result}" if result!=""
|
132
|
+
return result
|
128
133
|
end
|
129
|
-
result = "Security alerts: #{result}" if result!=""
|
130
|
-
return result
|
131
|
-
end
|
132
134
|
|
133
|
-
|
135
|
+
private
|
134
136
|
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
137
|
+
def filter_security_messages_already_fixed(version, date)
|
138
|
+
#TODO: let's use a database instead of having the info in yaml file
|
139
|
+
@security_messages.delete_if do |k,v|
|
140
|
+
@fixed[k] && Gem::Version.new(@fixed[k]) <= version
|
141
|
+
end
|
142
|
+
@security_messages.delete_if do |k,v|
|
143
|
+
v.date && date && v.date < date
|
144
|
+
end
|
139
145
|
end
|
140
|
-
@security_messages.delete_if do |k,v|
|
141
|
-
v.date && date && v.date < date
|
142
|
-
end
|
143
|
-
end
|
144
146
|
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
147
|
+
def source_repo(gem)
|
148
|
+
if @source_repos[gem.name]
|
149
|
+
return @source_repos[gem.name]
|
150
|
+
end
|
151
|
+
begin
|
152
|
+
gem_version_information = JSON.parse(open("http://rubygems.org/api/v1/gems/#{gem.name}.json").read)
|
153
|
+
rescue => e
|
154
|
+
Utils::log_error gem.name, "There was a problem downloading info for #{gem.name} #{e.to_s}"
|
155
|
+
return nil
|
156
|
+
end
|
157
|
+
uri = nil
|
158
|
+
if gem_version_information["project_uri"] &&
|
159
|
+
gem_version_information["project_uri"].include?("github")
|
160
|
+
uri = gem_version_information["project_uri"]
|
161
|
+
end
|
162
|
+
if gem_version_information["homepage_uri"] &&
|
163
|
+
gem_version_information["homepage_uri"].include?("github")
|
164
|
+
uri = gem_version_information["homepage_uri"]
|
165
|
+
end
|
166
|
+
if gem_version_information["source_code_uri"] &&
|
167
|
+
gem_version_information["source_code_uri"].include?("github")
|
168
|
+
uri = gem_version_information["source_code_uri"]
|
169
|
+
end
|
170
|
+
return uri
|
159
171
|
end
|
160
|
-
if gem_version_information["homepage_uri"] &&
|
161
|
-
gem_version_information["homepage_uri"].include?("github")
|
162
|
-
uri = gem_version_information["homepage_uri"]
|
163
|
-
end
|
164
|
-
if gem_version_information["source_code_uri"] &&
|
165
|
-
gem_version_information["source_code_uri"].include?("github")
|
166
|
-
uri = gem_version_information["source_code_uri"]
|
167
|
-
end
|
168
|
-
return uri
|
169
|
-
end
|
170
172
|
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
end
|
176
|
-
Dir.chdir("build_security_messages_check") do
|
177
|
-
if ! File.exists?(name)
|
178
|
-
Dir.mkdir(name)
|
173
|
+
def look_for_security_messages(name, source_repo, origin, counter = 0)
|
174
|
+
Utils::log_debug "looking for security messages on #{source_repo}"
|
175
|
+
if ! File.exists?("build_security_messages_check")
|
176
|
+
Dir.mkdir("build_security_messages_check")
|
179
177
|
end
|
180
|
-
Dir.chdir(
|
181
|
-
if
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
178
|
+
Dir.chdir("build_security_messages_check") do
|
179
|
+
if ! File.exists?(name)
|
180
|
+
Dir.mkdir(name)
|
181
|
+
end
|
182
|
+
Dir.chdir(name) do
|
183
|
+
if source_repo.include?("git")
|
184
|
+
scmCheckMessages = GitCheckMessages.new
|
185
|
+
Utils::log_debug "git repo"
|
186
|
+
elsif source_repo.include?("svn")
|
187
|
+
scmCheckMessages = SvnCheckMessages.new
|
188
|
+
Utils::log_debug "svn repo"
|
189
|
+
elsif source_repo.include?("bitbucket")
|
190
|
+
scmCheckMessages = HgCheckMessages.new
|
191
|
+
Utils::log_debug "mercurial repo"
|
192
|
+
else
|
193
|
+
Utils::log_error name, "Not a valid source repo #{source_repo}"
|
194
|
+
return {}
|
195
|
+
end
|
196
|
+
@security_messages = scmCheckMessages.check_messages(name, source_repo,
|
197
|
+
ScmSecurityMessages.new, origin)
|
193
198
|
end
|
194
|
-
@security_messages = scmCheckMessages.check_messages(name, source_repo,
|
195
|
-
ScmSecurityMessages.new, origin)
|
196
199
|
end
|
197
|
-
|
198
|
-
end
|
200
|
+
end
|
199
201
|
|
202
|
+
end
|
200
203
|
end
|