yavdb 0.5.1 → 0.5.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.circleci/config.yml +2 -1
- data/.github/ISSUE_TEMPLATE/bug_report.md +32 -0
- data/.github/ISSUE_TEMPLATE/feature_request.md +20 -0
- data/.rubocop.yml +24 -10
- data/.rubocop_todo.yml +13 -0
- data/.ruby-version +1 -1
- data/CONTRIBUTING.md +60 -0
- data/Gemfile.lock +80 -40
- data/lib/yavdb/constants.rb +1 -1
- data/lib/yavdb/crawler.rb +1 -1
- data/lib/yavdb/database.rb +0 -1
- data/lib/yavdb/sources/npmjs.rb +9 -9
- data/lib/yavdb/sources/ruby_advisory.rb +2 -2
- data/lib/yavdb/sources/rustsec.rb +7 -5
- data/lib/yavdb/sources/snyk_io.rb +21 -28
- data/lib/yavdb/sources/victims.rb +2 -2
- data/lib/yavdb/utils/http.rb +4 -4
- data/lib/yavdb/version.rb +1 -1
- data/yavdb.gemspec +10 -8
- metadata +56 -18
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: b419ba080c5a258be853b91c5e5c63bff95dc18ee4f5920432929aa3bb3f2e67
|
4
|
+
data.tar.gz: c195785e1b26437d64273659b48bf4266f92f6960886927f47e77228a1165b99
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 55288f2a25f94f9e8a514f95c5bb66b69b3d24968378a82894bab62525b16c9978efda200c84d1c3b36281b9872a7fa7d878eec2b185734497dc4a73936721d8
|
7
|
+
data.tar.gz: 9efbf7161bc446f64ed3334a51a56125e78fb8b112b477d6860bfbe151d93055a7cb6c02bee479057d10b2862b4398cf04755fe23dc83a87ef4b8c684c22ac47
|
data/.circleci/config.yml
CHANGED
@@ -4,7 +4,7 @@ jobs:
|
|
4
4
|
build-lint-test:
|
5
5
|
working_directory: ~/yavdb
|
6
6
|
docker:
|
7
|
-
- image: circleci/ruby:2.
|
7
|
+
- image: circleci/ruby:2.5.5
|
8
8
|
steps:
|
9
9
|
- checkout
|
10
10
|
|
@@ -27,6 +27,7 @@ jobs:
|
|
27
27
|
type: shell
|
28
28
|
command: |
|
29
29
|
sudo gem update --system
|
30
|
+
gem install bundler
|
30
31
|
bundle install --path /tmp/vendor/bundle
|
31
32
|
|
32
33
|
- name: Save bundler cache
|
@@ -0,0 +1,32 @@
|
|
1
|
+
---
|
2
|
+
name: Bug report
|
3
|
+
about: Create a report to help us improve
|
4
|
+
title: ''
|
5
|
+
labels: ''
|
6
|
+
assignees: ''
|
7
|
+
|
8
|
+
---
|
9
|
+
|
10
|
+
**Describe the bug**
|
11
|
+
A clear and concise description of what the bug is.
|
12
|
+
|
13
|
+
**To Reproduce**
|
14
|
+
Steps to reproduce the behavior:
|
15
|
+
1. Go to '...'
|
16
|
+
2. Click on '....'
|
17
|
+
3. Scroll down to '....'
|
18
|
+
4. See error
|
19
|
+
|
20
|
+
**Expected behavior**
|
21
|
+
A clear and concise description of what you expected to happen.
|
22
|
+
|
23
|
+
**Screenshots**
|
24
|
+
If applicable, add screenshots to help explain your problem.
|
25
|
+
|
26
|
+
**Desktop (please complete the following information):**
|
27
|
+
- OS: [e.g. Windows, Linux, Mac]
|
28
|
+
- Ruby Version [e.g. 2.5.5]
|
29
|
+
- Version [e.g. 22]
|
30
|
+
|
31
|
+
**Additional context**
|
32
|
+
Add any other context about the problem here.
|
@@ -0,0 +1,20 @@
|
|
1
|
+
---
|
2
|
+
name: Feature request
|
3
|
+
about: Suggest an idea for this project
|
4
|
+
title: ''
|
5
|
+
labels: ''
|
6
|
+
assignees: ''
|
7
|
+
|
8
|
+
---
|
9
|
+
|
10
|
+
**Is your feature request related to a problem? Please describe.**
|
11
|
+
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
|
12
|
+
|
13
|
+
**Describe the solution you'd like**
|
14
|
+
A clear and concise description of what you want to happen.
|
15
|
+
|
16
|
+
**Describe alternatives you've considered**
|
17
|
+
A clear and concise description of any alternative solutions or features you've considered.
|
18
|
+
|
19
|
+
**Additional context**
|
20
|
+
Add any other context or screenshots about the feature request here.
|
data/.rubocop.yml
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
inherit_from: .rubocop_todo.yml
|
2
|
+
|
3
|
+
require:
|
4
|
+
- rubocop-performance
|
5
|
+
- rubocop-rspec
|
6
|
+
|
1
7
|
AllCops:
|
2
8
|
# Include common Ruby source files.
|
3
9
|
Include:
|
@@ -57,7 +63,7 @@ AllCops:
|
|
57
63
|
DefaultFormatter: progress
|
58
64
|
UseCache: false
|
59
65
|
DisplayCopNames: false
|
60
|
-
TargetRubyVersion: 2.
|
66
|
+
TargetRubyVersion: 2.5.5
|
61
67
|
|
62
68
|
Gemspec/OrderedDependencies:
|
63
69
|
Enabled: true
|
@@ -94,18 +100,18 @@ Layout/EmptyLinesAroundModuleBody:
|
|
94
100
|
Layout/ExtraSpacing:
|
95
101
|
Enabled: true
|
96
102
|
|
97
|
-
Layout/
|
103
|
+
Layout/FirstArgumentIndentation:
|
98
104
|
Enabled: true
|
99
105
|
EnforcedStyle: consistent
|
100
106
|
IndentationWidth: 2
|
101
107
|
|
102
|
-
Layout/
|
108
|
+
Layout/FirstArrayElementIndentation:
|
103
109
|
Enabled: true
|
104
110
|
|
105
|
-
Layout/
|
111
|
+
Layout/AssignmentIndentation:
|
106
112
|
Enabled: true
|
107
113
|
|
108
|
-
Layout/
|
114
|
+
Layout/FirstHashElementIndentation:
|
109
115
|
Enabled: true
|
110
116
|
|
111
117
|
Layout/MultilineHashBraceLayout:
|
@@ -122,7 +128,7 @@ Layout/MultilineOperationIndentation:
|
|
122
128
|
Layout/SpaceAfterComma:
|
123
129
|
Enabled: true
|
124
130
|
|
125
|
-
Layout/
|
131
|
+
Layout/ParameterAlignment:
|
126
132
|
Enabled: true
|
127
133
|
EnforcedStyle: with_fixed_indentation
|
128
134
|
|
@@ -155,7 +161,7 @@ Lint/UselessAccessModifier:
|
|
155
161
|
Lint/UselessAssignment:
|
156
162
|
Enabled: true
|
157
163
|
|
158
|
-
Lint/
|
164
|
+
Lint/SuppressedException:
|
159
165
|
Enabled: true
|
160
166
|
Exclude:
|
161
167
|
- "lib/yavdb/sources/snyk_io.rb"
|
@@ -168,7 +174,7 @@ Metrics/BlockLength:
|
|
168
174
|
Enabled: true
|
169
175
|
Max: 51
|
170
176
|
Exclude:
|
171
|
-
- "spec
|
177
|
+
- "spec/**/*"
|
172
178
|
|
173
179
|
Metrics/ClassLength:
|
174
180
|
Enabled: false
|
@@ -202,6 +208,14 @@ Metrics/ParameterLists:
|
|
202
208
|
Performance/RedundantBlockCall:
|
203
209
|
Enabled: true
|
204
210
|
|
211
|
+
RSpec/ExampleLength:
|
212
|
+
Enabled: true
|
213
|
+
Max: 15
|
214
|
+
|
215
|
+
RSpec/MultipleExpectations:
|
216
|
+
Enabled: true
|
217
|
+
Max: 14
|
218
|
+
|
205
219
|
Security/MarshalLoad:
|
206
220
|
Enabled: true
|
207
221
|
Exclude:
|
@@ -324,10 +338,10 @@ Style/TrailingCommaInHashLiteral:
|
|
324
338
|
Enabled: true
|
325
339
|
EnforcedStyleForMultiline: no_comma
|
326
340
|
|
327
|
-
Style/
|
341
|
+
Style/RedundantInterpolation:
|
328
342
|
Enabled: true
|
329
343
|
|
330
|
-
Style/
|
344
|
+
Style/RedundantPercentQ:
|
331
345
|
Enabled: true
|
332
346
|
|
333
347
|
Style/WhileUntilDo:
|
data/.rubocop_todo.yml
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
# This configuration was generated by
|
2
|
+
# `rubocop --auto-gen-config`
|
3
|
+
# on 2019-10-09 19:40:20 +0300 using RuboCop version 0.75.0.
|
4
|
+
# The point is for the user to remove these configuration records
|
5
|
+
# one by one as the offenses are removed from the code base.
|
6
|
+
# Note that changes in the inspected code, or installation of new
|
7
|
+
# versions of RuboCop, may require this file to be generated again.
|
8
|
+
|
9
|
+
# Offense count: 1
|
10
|
+
# Configuration parameters: Max.
|
11
|
+
RSpec/ExampleLength:
|
12
|
+
Exclude:
|
13
|
+
- 'spec/crawler_spec.rb'
|
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
2.
|
1
|
+
2.5.5
|
data/CONTRIBUTING.md
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
## How to contribute to yavdb
|
2
|
+
|
3
|
+
### Main rules
|
4
|
+
|
5
|
+
* Before you open a ticket or send a pull request, [search](https://github.com/rtfpessoa/yavdb/issues) for previous discussions about the same feature or issue. Add to the earlier ticket if you find one.
|
6
|
+
|
7
|
+
* If you're proposing a new feature, make sure you create an issue to let other contributors know what you are working on.
|
8
|
+
|
9
|
+
* Before sending a pull request make sure your code is tested.
|
10
|
+
|
11
|
+
* Before sending a pull request for a feature, be sure to run tests.
|
12
|
+
|
13
|
+
* Use the same coding style as the rest of the codebase.
|
14
|
+
|
15
|
+
* Use `git rebase` (not `git merge`) to sync your work from time to time with the master branch.
|
16
|
+
|
17
|
+
* After creating your pull request make sure the build is passing on [CircleCI](https://circleci.com/gh/rtfpessoa/yavdb)
|
18
|
+
and that [Codacy](https://www.codacy.com/app/rtfpessoa/yavdb) is also confident in the code quality.
|
19
|
+
|
20
|
+
### Commit Style
|
21
|
+
|
22
|
+
Writing good commit logs is important. A commit log should describe what changed and why.
|
23
|
+
Follow these guidelines when writing one:
|
24
|
+
|
25
|
+
1. The first line should be 50 characters or less and contain a short
|
26
|
+
description of the change prefixed with the name of the changed
|
27
|
+
subsystem (e.g. "net: add localAddress and localPort to Socket").
|
28
|
+
2. Keep the second line blank.
|
29
|
+
3. Wrap all other lines at 72 columns.
|
30
|
+
|
31
|
+
A good commit log can look something like this:
|
32
|
+
|
33
|
+
```
|
34
|
+
subsystem: explaining the commit in one line
|
35
|
+
|
36
|
+
Body of commit message is a few lines of text, explaining things
|
37
|
+
in more detail, possibly giving some background about the issue
|
38
|
+
being fixed, etc. etc.
|
39
|
+
|
40
|
+
The body of the commit message can be several paragraphs, and
|
41
|
+
please do proper word-wrap and keep columns shorter than about
|
42
|
+
72 characters or so. That way `git log` will show things
|
43
|
+
nicely even when it is indented.
|
44
|
+
```
|
45
|
+
|
46
|
+
### Developer's Certificate of Origin 1.0
|
47
|
+
|
48
|
+
By making a contribution to this project, I certify that:
|
49
|
+
|
50
|
+
* (a) The contribution was created in whole or in part by me and I
|
51
|
+
have the right to submit it under the open source license indicated
|
52
|
+
in the file; or
|
53
|
+
* (b) The contribution is based upon previous work that, to the best
|
54
|
+
of my knowledge, is covered under an appropriate open source license
|
55
|
+
and I have the right under that license to submit that work with
|
56
|
+
modifications, whether created in whole or in part by me, under the
|
57
|
+
same open source license (unless I am permitted to submit under a
|
58
|
+
different license), as indicated in the file; or
|
59
|
+
* (c) The contribution was provided directly to me by some other
|
60
|
+
person who certified (a), (b) or (c) and I have not modified it.
|
data/Gemfile.lock
CHANGED
@@ -1,11 +1,11 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
yavdb (0.5.
|
5
|
-
execjs (~> 2.7
|
6
|
-
json (~> 2.
|
7
|
-
kramdown (~> 1
|
8
|
-
oga (
|
4
|
+
yavdb (0.5.6)
|
5
|
+
execjs (~> 2.7)
|
6
|
+
json (~> 2.2)
|
7
|
+
kramdown (~> 2.1)
|
8
|
+
oga (>= 2.15, < 4.0)
|
9
9
|
semantic_interval (~> 0.1)
|
10
10
|
therubyracer (~> 0.12)
|
11
11
|
thor (~> 0.20)
|
@@ -16,81 +16,121 @@ GEM
|
|
16
16
|
specs:
|
17
17
|
ansi (1.5.0)
|
18
18
|
ast (2.4.0)
|
19
|
+
bibliothecary (6.8.5)
|
20
|
+
commander
|
21
|
+
deb_control
|
22
|
+
librariesio-gem-parser
|
23
|
+
ox (>= 2.8.1)
|
24
|
+
sdl4r
|
25
|
+
strings
|
26
|
+
strings-ansi
|
27
|
+
toml-rb (~> 1.0)
|
28
|
+
typhoeus
|
19
29
|
citrus (3.0.2)
|
20
|
-
codacy-coverage (2.
|
30
|
+
codacy-coverage (2.2.0)
|
21
31
|
simplecov
|
32
|
+
colorize (0.8.1)
|
33
|
+
commander (4.4.7)
|
34
|
+
highline (~> 2.0.0)
|
35
|
+
deb_control (0.0.1)
|
36
|
+
dependency_spy (0.6.0)
|
37
|
+
bibliothecary (~> 6.6)
|
38
|
+
colorize (= 0.8.1)
|
39
|
+
semantic_range (~> 2.2)
|
40
|
+
thor (~> 0.20)
|
41
|
+
yavdb (~> 0.5)
|
22
42
|
diff-lcs (1.3)
|
23
|
-
docile (1.3.
|
43
|
+
docile (1.3.2)
|
44
|
+
ethon (0.12.0)
|
45
|
+
ffi (>= 1.3.0)
|
24
46
|
execjs (2.7.0)
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
47
|
+
ffi (1.11.3)
|
48
|
+
highline (2.0.3)
|
49
|
+
jaro_winkler (1.5.4)
|
50
|
+
json (2.3.0)
|
51
|
+
kramdown (2.2.1)
|
52
|
+
rexml
|
53
|
+
librariesio-gem-parser (1.0.0)
|
54
|
+
libv8 (3.16.14.19)
|
55
|
+
oga (3.2)
|
30
56
|
ast
|
31
57
|
ruby-ll (~> 2.1)
|
32
|
-
|
33
|
-
|
58
|
+
ox (2.12.0)
|
59
|
+
parallel (1.19.1)
|
60
|
+
parser (2.7.0.2)
|
34
61
|
ast (~> 2.4.0)
|
35
|
-
powerpack (0.1.2)
|
36
62
|
rainbow (3.0.0)
|
37
|
-
rake (
|
63
|
+
rake (13.0.1)
|
38
64
|
ref (2.0.0)
|
39
|
-
|
40
|
-
|
41
|
-
rspec-
|
42
|
-
rspec-
|
43
|
-
|
44
|
-
|
45
|
-
|
65
|
+
rexml (3.2.4)
|
66
|
+
rspec (3.9.0)
|
67
|
+
rspec-core (~> 3.9.0)
|
68
|
+
rspec-expectations (~> 3.9.0)
|
69
|
+
rspec-mocks (~> 3.9.0)
|
70
|
+
rspec-core (3.9.1)
|
71
|
+
rspec-support (~> 3.9.1)
|
72
|
+
rspec-expectations (3.9.0)
|
46
73
|
diff-lcs (>= 1.2.0, < 2.0)
|
47
|
-
rspec-support (~> 3.
|
48
|
-
rspec-mocks (3.
|
74
|
+
rspec-support (~> 3.9.0)
|
75
|
+
rspec-mocks (3.9.1)
|
49
76
|
diff-lcs (>= 1.2.0, < 2.0)
|
50
|
-
rspec-support (~> 3.
|
51
|
-
rspec-support (3.
|
77
|
+
rspec-support (~> 3.9.0)
|
78
|
+
rspec-support (3.9.2)
|
52
79
|
rspec_junit_formatter (0.4.1)
|
53
80
|
rspec-core (>= 2, < 4, != 2.12.0)
|
54
|
-
rubocop (0.
|
81
|
+
rubocop (0.79.0)
|
55
82
|
jaro_winkler (~> 1.5.1)
|
56
83
|
parallel (~> 1.10)
|
57
|
-
parser (>= 2.
|
58
|
-
powerpack (~> 0.1)
|
84
|
+
parser (>= 2.7.0.1)
|
59
85
|
rainbow (>= 2.2.2, < 4.0)
|
60
86
|
ruby-progressbar (~> 1.7)
|
61
|
-
unicode-display_width (
|
62
|
-
rubocop-
|
63
|
-
rubocop (>= 0.
|
87
|
+
unicode-display_width (>= 1.4.0, < 1.7)
|
88
|
+
rubocop-performance (1.5.2)
|
89
|
+
rubocop (>= 0.71.0)
|
90
|
+
rubocop-rspec (1.37.1)
|
91
|
+
rubocop (>= 0.68.1)
|
64
92
|
ruby-ll (2.1.2)
|
65
93
|
ansi
|
66
94
|
ast
|
67
|
-
ruby-progressbar (1.10.
|
95
|
+
ruby-progressbar (1.10.1)
|
96
|
+
sdl4r (0.9.11)
|
68
97
|
semantic_interval (0.1.0)
|
69
|
-
|
98
|
+
semantic_range (2.2.1)
|
99
|
+
simplecov (0.17.1)
|
70
100
|
docile (~> 1.1)
|
71
101
|
json (>= 1.8, < 3)
|
72
102
|
simplecov-html (~> 0.10.0)
|
73
103
|
simplecov-html (0.10.2)
|
104
|
+
strings (0.1.8)
|
105
|
+
strings-ansi (~> 0.1)
|
106
|
+
unicode-display_width (~> 1.5)
|
107
|
+
unicode_utils (~> 1.4)
|
108
|
+
strings-ansi (0.2.0)
|
74
109
|
therubyracer (0.12.3)
|
75
110
|
libv8 (~> 3.16.14.15)
|
76
111
|
ref
|
77
112
|
thor (0.20.3)
|
78
113
|
toml-rb (1.1.2)
|
79
114
|
citrus (~> 3.0, > 3.0)
|
80
|
-
|
115
|
+
typhoeus (1.3.1)
|
116
|
+
ethon (>= 0.9.0)
|
117
|
+
unicode-display_width (1.6.0)
|
118
|
+
unicode_utils (1.4.0)
|
81
119
|
|
82
120
|
PLATFORMS
|
83
121
|
ruby
|
84
122
|
|
85
123
|
DEPENDENCIES
|
86
124
|
codacy-coverage
|
87
|
-
|
125
|
+
dependency_spy
|
126
|
+
rake (~> 13.0)
|
88
127
|
rspec (~> 3.8)
|
89
128
|
rspec_junit_formatter (~> 0.4)
|
90
|
-
rubocop (~> 0.
|
91
|
-
rubocop-
|
129
|
+
rubocop (~> 0.75)
|
130
|
+
rubocop-performance (~> 1.5.0)
|
131
|
+
rubocop-rspec (~> 1.36)
|
92
132
|
simplecov
|
93
133
|
yavdb!
|
94
134
|
|
95
135
|
BUNDLED WITH
|
96
|
-
1.
|
136
|
+
2.1.2
|
data/lib/yavdb/constants.rb
CHANGED
data/lib/yavdb/crawler.rb
CHANGED
@@ -14,7 +14,7 @@
|
|
14
14
|
# You should have received a copy of the GNU Affero General Public License
|
15
15
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
16
16
|
|
17
|
-
Dir[File.expand_path('sources/*.rb', __dir__)].each do |file|
|
17
|
+
Dir[File.expand_path('sources/*.rb', __dir__)].sort.each do |file|
|
18
18
|
require file
|
19
19
|
end
|
20
20
|
|
data/lib/yavdb/database.rb
CHANGED
data/lib/yavdb/sources/npmjs.rb
CHANGED
@@ -40,13 +40,13 @@ module YAVDB
|
|
40
40
|
def fetch_packages_recursive(page_number)
|
41
41
|
page = get_page_html(get_page_url(page_number), false, 'npmjs/feed')
|
42
42
|
|
43
|
-
script_tag
|
44
|
-
context
|
43
|
+
script_tag = page.css('script').find { |script| script.text.include?('window.__context__') }.text
|
44
|
+
context = ExecJS.compile("var window = {};\n#{script_tag.force_encoding('utf-8')};")
|
45
45
|
advisory_data = context.exec('return window.__context__.context.advisoriesData')
|
46
46
|
|
47
47
|
packages = advisory_data['objects']
|
48
48
|
|
49
|
-
next_url
|
49
|
+
next_url = advisory_data['urls']['next']
|
50
50
|
next_packages = if next_url && !next_url&.include?("page=#{page_number}")
|
51
51
|
fetch_packages_recursive(page_number + 1)
|
52
52
|
else
|
@@ -62,7 +62,7 @@ module YAVDB
|
|
62
62
|
|
63
63
|
def create(package)
|
64
64
|
published_date = Date.strptime(package['created'], '%s')
|
65
|
-
updated_date
|
65
|
+
updated_date = Date.strptime(package['updated'], '%s')
|
66
66
|
|
67
67
|
cves = package['cves'] || []
|
68
68
|
|
@@ -103,18 +103,18 @@ module YAVDB
|
|
103
103
|
end
|
104
104
|
|
105
105
|
def get_page_url(page)
|
106
|
-
"#{API_URL}/advisories?page=#{page}&perPage=
|
106
|
+
"#{API_URL}/advisories?page=#{page}&perPage=100&order=-id"
|
107
107
|
end
|
108
108
|
|
109
109
|
def parse_severity(severity)
|
110
110
|
case severity
|
111
|
-
when 'low'
|
111
|
+
when 'low'
|
112
112
|
'low'
|
113
|
-
when 'moderate'
|
113
|
+
when 'moderate'
|
114
114
|
'medium'
|
115
|
-
when 'high'
|
115
|
+
when 'high'
|
116
116
|
'high'
|
117
|
-
when 'critical'
|
117
|
+
when 'critical'
|
118
118
|
'high'
|
119
119
|
else
|
120
120
|
'high'
|
@@ -25,7 +25,7 @@ module YAVDB
|
|
25
25
|
module RustSec
|
26
26
|
class Client
|
27
27
|
|
28
|
-
REPOSITORY_URL
|
28
|
+
REPOSITORY_URL = 'https://github.com/RustSec/advisory-db'.freeze
|
29
29
|
PACKAGE_MANAGER = 'cargo'.freeze
|
30
30
|
|
31
31
|
def self.advisories
|
@@ -44,19 +44,21 @@ module YAVDB
|
|
44
44
|
private
|
45
45
|
|
46
46
|
def create(advisory_hash)
|
47
|
-
date
|
48
|
-
severity
|
49
|
-
cve
|
47
|
+
date = Date.strptime(advisory_hash['date'].to_s, '%Y-%m-%d')
|
48
|
+
severity = 'high' # since no value is provided will use highest
|
49
|
+
cve = advisory_hash['aliases']&.select { |a| a.start_with?('CVE') }
|
50
50
|
references = advisory_hash['url'] && [advisory_hash['url']]
|
51
51
|
|
52
52
|
vuln_id = "rustsec:cargo:#{advisory_hash['package']}:#{advisory_hash['id']}"
|
53
53
|
|
54
|
+
vulnerable_versions = (['*'] if (advisory_hash['unaffected_versions'].nil? || advisory_hash['unaffected_versions'].empty?) && (advisory_hash['patched_versions'].nil? || advisory_hash['patched_versions'].empty?))
|
55
|
+
|
54
56
|
YAVDB::Advisory.new(
|
55
57
|
vuln_id,
|
56
58
|
advisory_hash['title'],
|
57
59
|
advisory_hash['description'],
|
58
60
|
advisory_hash['package'],
|
59
|
-
|
61
|
+
vulnerable_versions,
|
60
62
|
advisory_hash['unaffected_versions'],
|
61
63
|
advisory_hash['patched_versions'],
|
62
64
|
severity,
|
@@ -47,7 +47,7 @@ module YAVDB
|
|
47
47
|
urls.map do |advisory_url|
|
48
48
|
advisory_page = get_page_html(advisory_url, true, 'snyk.io/advisories')
|
49
49
|
create(advisory_url, advisory_page)
|
50
|
-
end
|
50
|
+
end.reject(&:nil?)
|
51
51
|
end
|
52
52
|
|
53
53
|
class << self
|
@@ -66,7 +66,7 @@ module YAVDB
|
|
66
66
|
page_vuln_urls = snykio
|
67
67
|
.css('table tbody tr td span a')
|
68
68
|
.map { |anchor| anchor.get('href') }
|
69
|
-
.map { |link| link if
|
69
|
+
.map { |link| link if %r{\/vuln\/.+}.match?(link) }.compact
|
70
70
|
|
71
71
|
next_urls = if page_vuln_urls.any?
|
72
72
|
next_url = snykio.css('a.pagination__next')
|
@@ -92,15 +92,16 @@ module YAVDB
|
|
92
92
|
severity = advisory_page.css('span.label__text').text.gsub(%r{(.*?) severity}, '\1')
|
93
93
|
|
94
94
|
package_manager = advisory_page.css('.breadcrumbs__list-item')[1].text.gsub(%r{\s+}, '').downcase
|
95
|
-
package_manager = PACKAGE_MANAGER_ALIAS[package_manager] ||
|
95
|
+
package_manager = PACKAGE_MANAGER_ALIAS[package_manager] || return
|
96
96
|
|
97
97
|
title = utf8(advisory_page.css('h1.header__title span.header__title__text').text)
|
98
98
|
|
99
99
|
affected_package = advisory_page.css('.custom-package-name').text
|
100
100
|
affected_package = advisory_page.css('.header__lede .breadcrumbs__list-item__link').text if affected_package.empty?
|
101
101
|
|
102
|
-
vulnerable_versions = advisory_page.css('.custom-affected-versions')
|
103
|
-
|
102
|
+
vulnerable_versions = (advisory_page.css('.custom-affected-versions') ||
|
103
|
+
advisory_page.css('.header__lede strong').drop(1).first).text.strip
|
104
|
+
vulnerable_versions = if vulnerable_versions.empty? || vulnerable_versions == 'ALL' || vulnerable_versions == '(,)'
|
104
105
|
['*']
|
105
106
|
elsif ['maven', 'nuget', 'pypi'].include?(package_manager)
|
106
107
|
[vulnerable_versions]
|
@@ -168,30 +169,22 @@ module YAVDB
|
|
168
169
|
body = section[:body]
|
169
170
|
|
170
171
|
case header.text
|
171
|
-
when
|
172
|
+
when %r{^(Overview|Details)$} then
|
172
173
|
overview_str = body
|
173
174
|
.map(&:to_xml)
|
175
|
+
.map { |e| e.force_encoding('UTF-8') }
|
174
176
|
.join("\n")
|
175
|
-
.force_encoding('UTF-8')
|
176
177
|
begin
|
177
|
-
|
178
|
-
|
178
|
+
if data[:description]
|
179
|
+
data[:description] += '\n'
|
180
|
+
else
|
181
|
+
data[:description] = ''
|
182
|
+
end
|
183
|
+
|
179
184
|
data[:description] += utf8(Kramdown::Document.new(overview_str, :html_to_native => true).to_kramdown)
|
180
185
|
rescue StandardError
|
181
186
|
# ignore
|
182
187
|
end
|
183
|
-
when 'Details' then
|
184
|
-
details_str = body
|
185
|
-
.map(&:to_xml)
|
186
|
-
.join("\n")
|
187
|
-
.force_encoding('UTF-8')
|
188
|
-
begin
|
189
|
-
data[:description] += '\n' if data[:description]
|
190
|
-
data[:description] = '' unless data[:description]
|
191
|
-
data[:description] += utf8(Kramdown::Document.new(details_str, :html_to_native => true).to_kramdown)
|
192
|
-
rescue StandardError
|
193
|
-
# ignore
|
194
|
-
end
|
195
188
|
when 'References' then
|
196
189
|
references = []
|
197
190
|
if body.any?
|
@@ -211,19 +204,19 @@ module YAVDB
|
|
211
204
|
|
212
205
|
advisory_page.css('.l-col .card .card__content dl > *').each_slice(2).to_a.map do |key, value|
|
213
206
|
case key.text
|
214
|
-
when 'Credit'
|
207
|
+
when 'Credit'
|
215
208
|
data[:credit] = utf8(value.text.split(',').map { |str| str.strip.sub(%r{-\s*}, '') }.reject(&:empty?))
|
216
|
-
when 'CVE'
|
209
|
+
when 'CVE'
|
217
210
|
data[:cve] = value.css('a').map { |a| a.text.strip.split(',') }.flatten.map(&:strip).reject(&:empty?)
|
218
|
-
when 'CWE'
|
211
|
+
when 'CWE'
|
219
212
|
data[:cwe] = value.css('a').map { |a| a.text.strip.split(',') }.flatten.map(&:strip).reject(&:empty?)
|
220
|
-
when 'Snyk ID'
|
213
|
+
when 'Snyk ID'
|
221
214
|
data[:id] = value.text.strip
|
222
|
-
when 'Disclosed'
|
215
|
+
when 'Disclosed'
|
223
216
|
data[:disclosed_date] = value.text.strip
|
224
|
-
when 'Published'
|
217
|
+
when 'Published'
|
225
218
|
data[:published_date] = value.text.strip
|
226
|
-
when 'Last modified'
|
219
|
+
when 'Last modified'
|
227
220
|
data[:last_modified_date] = value.text.strip
|
228
221
|
end
|
229
222
|
end
|
data/lib/yavdb/utils/http.rb
CHANGED
@@ -45,15 +45,15 @@ module YAVDB
|
|
45
45
|
begin
|
46
46
|
response = Net::HTTP.get_response(url)
|
47
47
|
case response
|
48
|
-
when Net::HTTPNotFound
|
48
|
+
when Net::HTTPNotFound
|
49
49
|
raise ArgumentError, 'page not found'
|
50
|
-
when Net::HTTPTooManyRequests
|
50
|
+
when Net::HTTPTooManyRequests
|
51
51
|
raise ArgumentError, 'too many requests'
|
52
52
|
else
|
53
53
|
response.body.lines
|
54
54
|
end
|
55
|
-
rescue StandardError =>
|
56
|
-
raise
|
55
|
+
rescue StandardError => e
|
56
|
+
raise e if retries.zero?
|
57
57
|
|
58
58
|
puts "Going to retry #{url}"
|
59
59
|
retries -= 1
|
data/lib/yavdb/version.rb
CHANGED
data/yavdb.gemspec
CHANGED
@@ -21,24 +21,26 @@ Gem::Specification.new do |spec|
|
|
21
21
|
spec.executables = ['yavdb', 'vulndb', 'vulnerabilitydb']
|
22
22
|
spec.require_paths = ['lib']
|
23
23
|
|
24
|
-
spec.required_ruby_version = '>= 2.
|
24
|
+
spec.required_ruby_version = '>= 2.5.5'
|
25
25
|
|
26
26
|
# Development
|
27
27
|
spec.add_development_dependency 'codacy-coverage'
|
28
|
-
spec.add_development_dependency 'rake',
|
28
|
+
spec.add_development_dependency 'rake', '~> 13.0'
|
29
29
|
spec.add_development_dependency 'rspec', ['~> 3.8']
|
30
30
|
spec.add_development_dependency 'rspec_junit_formatter', ['~> 0.4']
|
31
31
|
spec.add_development_dependency 'simplecov'
|
32
32
|
|
33
33
|
# Linters
|
34
|
-
spec.add_development_dependency '
|
35
|
-
spec.add_development_dependency 'rubocop
|
34
|
+
spec.add_development_dependency 'dependency_spy'
|
35
|
+
spec.add_development_dependency 'rubocop', ['~> 0.75']
|
36
|
+
spec.add_development_dependency 'rubocop-performance', ['~> 1.5.0']
|
37
|
+
spec.add_development_dependency 'rubocop-rspec', ['~> 1.36']
|
36
38
|
|
37
39
|
# Runtime
|
38
|
-
spec.add_runtime_dependency 'execjs', ['~> 2.7
|
39
|
-
spec.add_runtime_dependency 'json', ['~> 2.
|
40
|
-
spec.add_runtime_dependency 'kramdown', ['~> 1
|
41
|
-
spec.add_runtime_dependency 'oga',
|
40
|
+
spec.add_runtime_dependency 'execjs', ['~> 2.7']
|
41
|
+
spec.add_runtime_dependency 'json', ['~> 2.2']
|
42
|
+
spec.add_runtime_dependency 'kramdown', ['~> 2.1']
|
43
|
+
spec.add_runtime_dependency 'oga', '>= 2.15', '< 4.0'
|
42
44
|
spec.add_runtime_dependency 'semantic_interval', ['~> 0.1']
|
43
45
|
spec.add_runtime_dependency 'therubyracer', ['~> 0.12']
|
44
46
|
spec.add_runtime_dependency 'thor', ['~> 0.20']
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: yavdb
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Rodrigo Fernandes
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-05-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: codacy-coverage
|
@@ -30,14 +30,14 @@ dependencies:
|
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
33
|
+
version: '13.0'
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
40
|
+
version: '13.0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: rspec
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -80,90 +80,124 @@ dependencies:
|
|
80
80
|
- - ">="
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: dependency_spy
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
83
97
|
- !ruby/object:Gem::Dependency
|
84
98
|
name: rubocop
|
85
99
|
requirement: !ruby/object:Gem::Requirement
|
86
100
|
requirements:
|
87
101
|
- - "~>"
|
88
102
|
- !ruby/object:Gem::Version
|
89
|
-
version: '0.
|
103
|
+
version: '0.75'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0.75'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: rubocop-performance
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - "~>"
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: 1.5.0
|
90
118
|
type: :development
|
91
119
|
prerelease: false
|
92
120
|
version_requirements: !ruby/object:Gem::Requirement
|
93
121
|
requirements:
|
94
122
|
- - "~>"
|
95
123
|
- !ruby/object:Gem::Version
|
96
|
-
version:
|
124
|
+
version: 1.5.0
|
97
125
|
- !ruby/object:Gem::Dependency
|
98
126
|
name: rubocop-rspec
|
99
127
|
requirement: !ruby/object:Gem::Requirement
|
100
128
|
requirements:
|
101
129
|
- - "~>"
|
102
130
|
- !ruby/object:Gem::Version
|
103
|
-
version: '1.
|
131
|
+
version: '1.36'
|
104
132
|
type: :development
|
105
133
|
prerelease: false
|
106
134
|
version_requirements: !ruby/object:Gem::Requirement
|
107
135
|
requirements:
|
108
136
|
- - "~>"
|
109
137
|
- !ruby/object:Gem::Version
|
110
|
-
version: '1.
|
138
|
+
version: '1.36'
|
111
139
|
- !ruby/object:Gem::Dependency
|
112
140
|
name: execjs
|
113
141
|
requirement: !ruby/object:Gem::Requirement
|
114
142
|
requirements:
|
115
143
|
- - "~>"
|
116
144
|
- !ruby/object:Gem::Version
|
117
|
-
version: 2.7
|
145
|
+
version: '2.7'
|
118
146
|
type: :runtime
|
119
147
|
prerelease: false
|
120
148
|
version_requirements: !ruby/object:Gem::Requirement
|
121
149
|
requirements:
|
122
150
|
- - "~>"
|
123
151
|
- !ruby/object:Gem::Version
|
124
|
-
version: 2.7
|
152
|
+
version: '2.7'
|
125
153
|
- !ruby/object:Gem::Dependency
|
126
154
|
name: json
|
127
155
|
requirement: !ruby/object:Gem::Requirement
|
128
156
|
requirements:
|
129
157
|
- - "~>"
|
130
158
|
- !ruby/object:Gem::Version
|
131
|
-
version: '2.
|
159
|
+
version: '2.2'
|
132
160
|
type: :runtime
|
133
161
|
prerelease: false
|
134
162
|
version_requirements: !ruby/object:Gem::Requirement
|
135
163
|
requirements:
|
136
164
|
- - "~>"
|
137
165
|
- !ruby/object:Gem::Version
|
138
|
-
version: '2.
|
166
|
+
version: '2.2'
|
139
167
|
- !ruby/object:Gem::Dependency
|
140
168
|
name: kramdown
|
141
169
|
requirement: !ruby/object:Gem::Requirement
|
142
170
|
requirements:
|
143
171
|
- - "~>"
|
144
172
|
- !ruby/object:Gem::Version
|
145
|
-
version: '1
|
173
|
+
version: '2.1'
|
146
174
|
type: :runtime
|
147
175
|
prerelease: false
|
148
176
|
version_requirements: !ruby/object:Gem::Requirement
|
149
177
|
requirements:
|
150
178
|
- - "~>"
|
151
179
|
- !ruby/object:Gem::Version
|
152
|
-
version: '1
|
180
|
+
version: '2.1'
|
153
181
|
- !ruby/object:Gem::Dependency
|
154
182
|
name: oga
|
155
183
|
requirement: !ruby/object:Gem::Requirement
|
156
184
|
requirements:
|
157
|
-
- - "
|
185
|
+
- - ">="
|
158
186
|
- !ruby/object:Gem::Version
|
159
187
|
version: '2.15'
|
188
|
+
- - "<"
|
189
|
+
- !ruby/object:Gem::Version
|
190
|
+
version: '4.0'
|
160
191
|
type: :runtime
|
161
192
|
prerelease: false
|
162
193
|
version_requirements: !ruby/object:Gem::Requirement
|
163
194
|
requirements:
|
164
|
-
- - "
|
195
|
+
- - ">="
|
165
196
|
- !ruby/object:Gem::Version
|
166
197
|
version: '2.15'
|
198
|
+
- - "<"
|
199
|
+
- !ruby/object:Gem::Version
|
200
|
+
version: '4.0'
|
167
201
|
- !ruby/object:Gem::Dependency
|
168
202
|
name: semantic_interval
|
169
203
|
requirement: !ruby/object:Gem::Requirement
|
@@ -232,11 +266,15 @@ extensions: []
|
|
232
266
|
extra_rdoc_files: []
|
233
267
|
files:
|
234
268
|
- ".circleci/config.yml"
|
269
|
+
- ".github/ISSUE_TEMPLATE/bug_report.md"
|
270
|
+
- ".github/ISSUE_TEMPLATE/feature_request.md"
|
235
271
|
- ".gitignore"
|
236
272
|
- ".rspec"
|
237
273
|
- ".rubocop.yml"
|
274
|
+
- ".rubocop_todo.yml"
|
238
275
|
- ".ruby-version"
|
239
276
|
- CODE_OF_CONDUCT.md
|
277
|
+
- CONTRIBUTING.md
|
240
278
|
- Gemfile
|
241
279
|
- Gemfile.lock
|
242
280
|
- LICENSE
|
@@ -280,7 +318,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
280
318
|
requirements:
|
281
319
|
- - ">="
|
282
320
|
- !ruby/object:Gem::Version
|
283
|
-
version: 2.
|
321
|
+
version: 2.5.5
|
284
322
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
285
323
|
requirements:
|
286
324
|
- - ">="
|
@@ -288,7 +326,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
288
326
|
version: '0'
|
289
327
|
requirements: []
|
290
328
|
rubyforge_project:
|
291
|
-
rubygems_version: 2.
|
329
|
+
rubygems_version: 2.7.6.2
|
292
330
|
signing_key:
|
293
331
|
specification_version: 4
|
294
332
|
summary: The Free and Open Source vulnerability database.
|