nvd_feed_api 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitlab-ci.yml +5 -8
- data/.rubocop.yml +8 -4
- data/.tool-versions +1 -1
- data/Gemfile +12 -6
- data/Gemfile.lock +93 -39
- data/Rakefile +2 -0
- data/bin/nvd_feed_api +1 -0
- data/bin/nvd_feed_api_console +1 -0
- data/lib/nvd_feed_api/feed.rb +28 -27
- data/lib/nvd_feed_api/meta.rb +2 -0
- data/lib/nvd_feed_api/version.rb +3 -1
- data/lib/nvd_feed_api.rb +8 -11
- data/nvd_feed_api.gemspec +4 -2
- data/pages/CHANGELOG.md +18 -2
- data/pages/INSTALL.md +23 -23
- data/test/test_nvd_feed_api.rb +27 -14
- metadata +7 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 77fb8e2cb880ccd6376de4a63dc9ca016a3553cb8b52ad28463a6db7c8f28203
|
4
|
+
data.tar.gz: a6efdc897b58302b4fc8bf3f49a1cb31f3e00dcaa7deafcac9c1dc5e0642dfd9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8d2795f2b8b16563df4b8af0980fecdf167de7c875b4bc1c944931a65cebf02ee0ff689b299307bb3ec51c985c29cc2e830e5e01286095ccbb11167662f978ce
|
7
|
+
data.tar.gz: 3c9d6acb0c8a2736fd416dc4b6b6431f90a7f7ce30d1c748e85892f2c27e6c88e85082cbf2e0f6be39c90bab81d7179bd3010ff9930dbff8341edac3952b07e3
|
data/.gitlab-ci.yml
CHANGED
@@ -15,7 +15,8 @@ before_script:
|
|
15
15
|
# install make, gcc for building gem native extension (commonmarker)
|
16
16
|
# libc-dev for musl-dev dependency (stdlib.h) needed by gcc
|
17
17
|
- apk --no-cache add coreutils git make gcc libc-dev
|
18
|
-
- bundle
|
18
|
+
- bundle config set path 'vendor' # Set dependencies install dir to ./vendor/ruby
|
19
|
+
- bundle install -j $(nproc) # Install dependencies into ./vendor/ruby
|
19
20
|
- bundle exec rake install # install the gem
|
20
21
|
|
21
22
|
# Anchors: https://docs.gitlab.com/ee/ci/yaml/README.html#anchors
|
@@ -25,17 +26,13 @@ before_script:
|
|
25
26
|
- bundle exec rubocop
|
26
27
|
- bundle exec rake test
|
27
28
|
|
28
|
-
|
29
|
-
# <<: *job_definition
|
30
|
-
# image: ruby:2.4-alpine
|
31
|
-
|
32
|
-
test:3.1:
|
29
|
+
test:3.4:
|
33
30
|
<<: *job_definition
|
34
|
-
image: ruby:3.
|
31
|
+
image: ruby:3.4-alpine
|
35
32
|
|
36
33
|
pages:
|
37
34
|
stage: deploy
|
38
|
-
image: ruby:3.
|
35
|
+
image: ruby:3.4-alpine
|
39
36
|
script:
|
40
37
|
- bundle exec yard doc
|
41
38
|
- mkdir public
|
data/.rubocop.yml
CHANGED
@@ -1,6 +1,10 @@
|
|
1
1
|
AllCops:
|
2
|
-
TargetRubyVersion:
|
2
|
+
TargetRubyVersion: 3.1
|
3
3
|
NewCops: enable
|
4
|
+
SuggestExtensions: false
|
5
|
+
|
6
|
+
plugins:
|
7
|
+
- rubocop-minitest
|
4
8
|
|
5
9
|
Layout/HashAlignment:
|
6
10
|
EnforcedHashRocketStyle: table
|
@@ -33,6 +37,9 @@ Metrics/MethodLength:
|
|
33
37
|
Metrics/PerceivedComplexity:
|
34
38
|
Enabled: false
|
35
39
|
|
40
|
+
Minitest/MultipleAssertions:
|
41
|
+
Enabled: false
|
42
|
+
|
36
43
|
Naming/VariableName:
|
37
44
|
EnforcedStyle: snake_case
|
38
45
|
|
@@ -42,9 +49,6 @@ Security/JSONLoad:
|
|
42
49
|
Style/CaseLikeIf:
|
43
50
|
Enabled: true
|
44
51
|
|
45
|
-
Style/FrozenStringLiteralComment:
|
46
|
-
EnforcedStyle: never
|
47
|
-
|
48
52
|
Style/PerlBackrefs:
|
49
53
|
AutoCorrect: false
|
50
54
|
|
data/.tool-versions
CHANGED
@@ -1 +1 @@
|
|
1
|
-
ruby 3.1
|
1
|
+
ruby 3.4.1
|
data/Gemfile
CHANGED
@@ -1,9 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
source 'https://rubygems.org'
|
2
4
|
|
3
5
|
gemspec
|
4
6
|
|
5
7
|
group :runtime, :cli do
|
6
|
-
gem 'archive-zip', '
|
8
|
+
gem 'archive-zip', '0.13.0.pre1'
|
7
9
|
gem 'nokogiri', '~> 1.11'
|
8
10
|
gem 'oj', '>= 3.7.8', '<4'
|
9
11
|
end
|
@@ -13,15 +15,19 @@ group :development, :install do
|
|
13
15
|
end
|
14
16
|
|
15
17
|
group :development, :test do
|
16
|
-
gem 'minitest', '~> 5.
|
17
|
-
gem 'rake', '~> 13.
|
18
|
+
gem 'minitest', '~> 5.25'
|
19
|
+
gem 'rake', '~> 13.2'
|
18
20
|
end
|
19
21
|
|
20
22
|
group :development, :lint do
|
21
|
-
gem 'rubocop', '~> 1.
|
23
|
+
gem 'rubocop', '~> 1.71'
|
24
|
+
gem 'rubocop-minitest', '~> 0.36'
|
22
25
|
end
|
23
26
|
|
24
27
|
group :development, :docs do
|
25
|
-
gem 'commonmarker', '~> 0
|
26
|
-
gem '
|
28
|
+
gem 'commonmarker', '~> 2.0' # for markdown support in YARD
|
29
|
+
gem 'logger', '< 2.0'
|
30
|
+
# gem 'yard', ['>= 0.9.27', '< 0.10']
|
31
|
+
# https://github.com/lsegal/yard/issues/1528
|
32
|
+
gem 'yard', github: 'ParadoxV5/yard', ref: '9e869c940859570b07b81c5eadd6070e76f6291e', branch: 'commonmarker-1.0'
|
27
33
|
end
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,15 @@
|
|
1
|
+
GIT
|
2
|
+
remote: https://github.com/ParadoxV5/yard.git
|
3
|
+
revision: 9e869c940859570b07b81c5eadd6070e76f6291e
|
4
|
+
ref: 9e869c940859570b07b81c5eadd6070e76f6291e
|
5
|
+
branch: commonmarker-1.0
|
6
|
+
specs:
|
7
|
+
yard (0.9.36)
|
8
|
+
|
1
9
|
PATH
|
2
10
|
remote: .
|
3
11
|
specs:
|
4
|
-
nvd_feed_api (0.
|
12
|
+
nvd_feed_api (0.6.0)
|
5
13
|
archive-zip (~> 0.11)
|
6
14
|
nokogiri (~> 1.11)
|
7
15
|
oj (>= 3.7.8, < 4)
|
@@ -9,56 +17,102 @@ PATH
|
|
9
17
|
GEM
|
10
18
|
remote: https://rubygems.org/
|
11
19
|
specs:
|
12
|
-
archive-zip (0.
|
13
|
-
io-like (~> 0.
|
14
|
-
ast (2.4.
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
20
|
+
archive-zip (0.13.0.pre1)
|
21
|
+
io-like (~> 0.4.0.pre1)
|
22
|
+
ast (2.4.3)
|
23
|
+
bigdecimal (3.1.9)
|
24
|
+
commonmarker (2.3.0)
|
25
|
+
rb_sys (~> 0.9)
|
26
|
+
commonmarker (2.3.0-aarch64-linux)
|
27
|
+
commonmarker (2.3.0-aarch64-linux-musl)
|
28
|
+
commonmarker (2.3.0-arm64-darwin)
|
29
|
+
commonmarker (2.3.0-x86_64-darwin)
|
30
|
+
commonmarker (2.3.0-x86_64-linux)
|
31
|
+
commonmarker (2.3.0-x86_64-linux-musl)
|
32
|
+
io-like (0.4.0.pre1)
|
33
|
+
json (2.12.0)
|
34
|
+
language_server-protocol (3.17.0.5)
|
35
|
+
lint_roller (1.1.0)
|
36
|
+
logger (1.7.0)
|
37
|
+
minitest (5.25.5)
|
38
|
+
nokogiri (1.18.8-aarch64-linux-gnu)
|
39
|
+
racc (~> 1.4)
|
40
|
+
nokogiri (1.18.8-aarch64-linux-musl)
|
41
|
+
racc (~> 1.4)
|
42
|
+
nokogiri (1.18.8-arm-linux-gnu)
|
43
|
+
racc (~> 1.4)
|
44
|
+
nokogiri (1.18.8-arm-linux-musl)
|
45
|
+
racc (~> 1.4)
|
46
|
+
nokogiri (1.18.8-arm64-darwin)
|
47
|
+
racc (~> 1.4)
|
48
|
+
nokogiri (1.18.8-x86_64-darwin)
|
49
|
+
racc (~> 1.4)
|
50
|
+
nokogiri (1.18.8-x86_64-linux-gnu)
|
51
|
+
racc (~> 1.4)
|
52
|
+
nokogiri (1.18.8-x86_64-linux-musl)
|
21
53
|
racc (~> 1.4)
|
22
|
-
oj (3.
|
23
|
-
|
24
|
-
|
54
|
+
oj (3.16.10)
|
55
|
+
bigdecimal (>= 3.0)
|
56
|
+
ostruct (>= 0.2)
|
57
|
+
ostruct (0.6.1)
|
58
|
+
parallel (1.27.0)
|
59
|
+
parser (3.3.8.0)
|
25
60
|
ast (~> 2.4.1)
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
61
|
+
racc
|
62
|
+
prism (1.4.0)
|
63
|
+
racc (1.8.1)
|
64
|
+
rainbow (3.1.1)
|
65
|
+
rake (13.2.1)
|
66
|
+
rake-compiler-dock (1.9.1)
|
67
|
+
rb_sys (0.9.111)
|
68
|
+
rake-compiler-dock (= 1.9.1)
|
69
|
+
regexp_parser (2.10.0)
|
70
|
+
rubocop (1.75.7)
|
71
|
+
json (~> 2.3)
|
72
|
+
language_server-protocol (~> 3.17.0.2)
|
73
|
+
lint_roller (~> 1.1.0)
|
32
74
|
parallel (~> 1.10)
|
33
|
-
parser (>= 3.
|
75
|
+
parser (>= 3.3.0.2)
|
34
76
|
rainbow (>= 2.2.2, < 4.0)
|
35
|
-
regexp_parser (>=
|
36
|
-
|
37
|
-
rubocop-ast (>= 1.15.1, < 2.0)
|
77
|
+
regexp_parser (>= 2.9.3, < 3.0)
|
78
|
+
rubocop-ast (>= 1.44.0, < 2.0)
|
38
79
|
ruby-progressbar (~> 1.7)
|
39
|
-
unicode-display_width (>=
|
40
|
-
rubocop-ast (1.
|
41
|
-
parser (>= 3.
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
80
|
+
unicode-display_width (>= 2.4.0, < 4.0)
|
81
|
+
rubocop-ast (1.44.1)
|
82
|
+
parser (>= 3.3.7.2)
|
83
|
+
prism (~> 1.4)
|
84
|
+
rubocop-minitest (0.38.0)
|
85
|
+
lint_roller (~> 1.1)
|
86
|
+
rubocop (>= 1.75.0, < 2.0)
|
87
|
+
rubocop-ast (>= 1.38.0, < 2.0)
|
88
|
+
ruby-progressbar (1.13.0)
|
89
|
+
unicode-display_width (3.1.4)
|
90
|
+
unicode-emoji (~> 4.0, >= 4.0.4)
|
91
|
+
unicode-emoji (4.0.4)
|
47
92
|
|
48
93
|
PLATFORMS
|
49
|
-
|
94
|
+
aarch64-linux-gnu
|
95
|
+
aarch64-linux-musl
|
96
|
+
arm-linux-gnu
|
97
|
+
arm-linux-musl
|
98
|
+
arm64-darwin
|
99
|
+
x86_64-darwin
|
100
|
+
x86_64-linux-gnu
|
101
|
+
x86_64-linux-musl
|
50
102
|
|
51
103
|
DEPENDENCIES
|
52
|
-
archive-zip (
|
104
|
+
archive-zip (= 0.13.0.pre1)
|
53
105
|
bundler (~> 2.1)
|
54
|
-
commonmarker (~> 0
|
55
|
-
|
106
|
+
commonmarker (~> 2.0)
|
107
|
+
logger (< 2.0)
|
108
|
+
minitest (~> 5.25)
|
56
109
|
nokogiri (~> 1.11)
|
57
110
|
nvd_feed_api!
|
58
111
|
oj (>= 3.7.8, < 4)
|
59
|
-
rake (~> 13.
|
60
|
-
rubocop (~> 1.
|
61
|
-
|
112
|
+
rake (~> 13.2)
|
113
|
+
rubocop (~> 1.71)
|
114
|
+
rubocop-minitest (~> 0.36)
|
115
|
+
yard!
|
62
116
|
|
63
117
|
BUNDLED WITH
|
64
|
-
2.
|
118
|
+
2.6.2
|
data/Rakefile
CHANGED
data/bin/nvd_feed_api
CHANGED
data/bin/nvd_feed_api_console
CHANGED
data/lib/nvd_feed_api/feed.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# Ruby internal
|
2
4
|
require 'digest'
|
3
5
|
require 'net/https'
|
@@ -32,17 +34,17 @@ class NVDFeedScraper
|
|
32
34
|
|
33
35
|
# @return [String] the URL of the metadata file of the feed.
|
34
36
|
# @example
|
35
|
-
# 'https://
|
37
|
+
# 'https://nvd.nist.gov/feeds/json/cve/2.0/nvdcve-2.0-2007.meta'
|
36
38
|
attr_reader :meta_url
|
37
39
|
|
38
40
|
# @return [String] the URL of the gz archive of the feed.
|
39
41
|
# @example
|
40
|
-
# 'https://
|
42
|
+
# 'https://nvd.nist.gov/feeds/json/cve/2.0/nvdcve-2.0-2007.json.gz'
|
41
43
|
attr_reader :gz_url
|
42
44
|
|
43
45
|
# @return [String] the URL of the zip archive of the feed.
|
44
46
|
# @example
|
45
|
-
# 'https://
|
47
|
+
# 'https://nvd.nist.gov/feeds/json/cve/2.0/nvdcve-2.0-2007.json.zip'
|
46
48
|
attr_reader :zip_url
|
47
49
|
|
48
50
|
# @return [Meta] the {Meta} object of the feed.
|
@@ -168,11 +170,11 @@ class NVDFeedScraper
|
|
168
170
|
# Set data
|
169
171
|
if @data_type.nil?
|
170
172
|
doc = Oj::Doc.open(File.read(@json_file))
|
171
|
-
@data_type = doc.fetch('/CVE_data_type')
|
172
|
-
@data_format = doc.fetch('/
|
173
|
-
@data_version = doc.fetch('/
|
174
|
-
@data_number_of_cves = doc.fetch('/
|
175
|
-
@data_timestamp = Date.strptime(doc.fetch('/
|
173
|
+
@data_type = 'CVE' # @data_type was doc.fetch('/CVE_data_type') in 1.1 schema but was removed in 2.0 so keep this string for retrocompatibility
|
174
|
+
@data_format = doc.fetch('/format')
|
175
|
+
@data_version = doc.fetch('/version').to_f
|
176
|
+
@data_number_of_cves = doc.fetch('/totalResults').to_i
|
177
|
+
@data_timestamp = Date.strptime(doc.fetch('/timestamp'), '%FT%T.%N')
|
176
178
|
doc.close
|
177
179
|
end
|
178
180
|
else
|
@@ -187,11 +189,11 @@ class NVDFeedScraper
|
|
187
189
|
|
188
190
|
# update data
|
189
191
|
doc = Oj::Doc.open(File.read(@json_file))
|
190
|
-
@data_type = doc.fetch('/CVE_data_type')
|
191
|
-
@data_format = doc.fetch('/
|
192
|
-
@data_version = doc.fetch('/
|
193
|
-
@data_number_of_cves = doc.fetch('/
|
194
|
-
@data_timestamp = Date.strptime(doc.fetch('/
|
192
|
+
@data_type = 'CVE' # @data_type was doc.fetch('/CVE_data_type') in 1.1 schema but was removed in 2.0 so keep this string for retrocompatibility
|
193
|
+
@data_format = doc.fetch('/format')
|
194
|
+
@data_version = doc.fetch('/version').to_f
|
195
|
+
@data_number_of_cves = doc.fetch('/totalResults').to_i
|
196
|
+
@data_timestamp = Date.strptime(doc.fetch('/timestamp'), '%FT%T.%N')
|
195
197
|
doc.close
|
196
198
|
end
|
197
199
|
return @json_file
|
@@ -213,8 +215,7 @@ class NVDFeedScraper
|
|
213
215
|
# @return [Array] an Array of CVE, each CVE is a Ruby Hash. May not be in the same order as provided.
|
214
216
|
# @note {#json_pull} is needed before using this method. Remember you're searching only in the current feed.
|
215
217
|
# @todo implement a CVE Class instead of returning a Hash.
|
216
|
-
# @see https://
|
217
|
-
# @see https://scap.nist.gov/schema/nvd/feed/0.1/CVE_JSON_4.0_min.schema
|
218
|
+
# @see https://csrc.nist.gov/schema/nvd/api/2.0/cve_api_json_2.0.schema
|
218
219
|
# @example
|
219
220
|
# s = NVDFeedScraper.new
|
220
221
|
# s.scrap
|
@@ -234,10 +235,10 @@ class NVDFeedScraper
|
|
234
235
|
raise "bad CVE name (#{arg_cve[0]})" unless /^CVE-[0-9]{4}-[0-9]{4,}$/i.match?(arg_cve[0])
|
235
236
|
|
236
237
|
doc = Oj::Doc.open(File.read(@json_file))
|
237
|
-
# Quicker than doc.fetch('/
|
238
|
+
# Quicker than doc.fetch('/vulnerabilities').size
|
238
239
|
(1..@data_number_of_cves).each do |i|
|
239
|
-
if arg_cve[0].upcase == doc.fetch("/
|
240
|
-
return_value = doc.fetch("/
|
240
|
+
if arg_cve[0].upcase == doc.fetch("/vulnerabilities/#{i}/cve/id")
|
241
|
+
return_value = doc.fetch("/vulnerabilities/#{i}")
|
241
242
|
break
|
242
243
|
end
|
243
244
|
end
|
@@ -251,10 +252,10 @@ class NVDFeedScraper
|
|
251
252
|
raise 'bad CVE name' unless cves_to_find.all? { |x| /^CVE-[0-9]{4}-[0-9]{4,}$/i.match?(x) }
|
252
253
|
|
253
254
|
doc = Oj::Doc.open(File.read(@json_file))
|
254
|
-
# Quicker than doc.fetch('/
|
255
|
+
# Quicker than doc.fetch('/vulnerabilities').size
|
255
256
|
(1..@data_number_of_cves).each do |i|
|
256
|
-
doc.move("/
|
257
|
-
cve_id = doc.fetch('cve/
|
257
|
+
doc.move("/vulnerabilities/#{i}")
|
258
|
+
cve_id = doc.fetch('cve/id')
|
258
259
|
if cves_to_find.include?(cve_id)
|
259
260
|
return_value.push(doc.fetch)
|
260
261
|
cves_to_find.delete(cve_id)
|
@@ -281,11 +282,11 @@ class NVDFeedScraper
|
|
281
282
|
raise "json_file (#{@json_file}) doesn't exist" unless File.file?(@json_file)
|
282
283
|
|
283
284
|
doc = Oj::Doc.open(File.read(@json_file))
|
284
|
-
# Quicker than doc.fetch('/
|
285
|
+
# Quicker than doc.fetch('/vulnerabilities').size
|
285
286
|
cve_names = []
|
286
287
|
(1..@data_number_of_cves).each do |i|
|
287
|
-
doc.move("/
|
288
|
-
cve_names.push(doc.fetch('cve/
|
288
|
+
doc.move("/vulnerabilities/#{i}")
|
289
|
+
cve_names.push(doc.fetch('cve/id'))
|
289
290
|
end
|
290
291
|
doc.close
|
291
292
|
return cve_names
|
@@ -314,7 +315,7 @@ class NVDFeedScraper
|
|
314
315
|
# @param arg_meta_url [String] the new URL of the metadata file of the feed.
|
315
316
|
# @return [String] the new URL of the metadata file of the feed.
|
316
317
|
# @example
|
317
|
-
# 'https://
|
318
|
+
# 'https://nvd.nist.gov/feeds/json/cve/2.0/nvdcve-2.0-2007.meta'
|
318
319
|
def meta_url=(arg_meta_url)
|
319
320
|
raise "meta_url (#{arg_meta_url}) is not a string" unless arg_meta_url.is_a?(String)
|
320
321
|
|
@@ -324,7 +325,7 @@ class NVDFeedScraper
|
|
324
325
|
# @param arg_gz_url [String] the new URL of the gz archive of the feed.
|
325
326
|
# @return [String] the new URL of the gz archive of the feed.
|
326
327
|
# @example
|
327
|
-
# 'https://
|
328
|
+
# 'https://nvd.nist.gov/feeds/json/cve/2.0/nvdcve-2.0-2007.json.gz'
|
328
329
|
def gz_url=(arg_gz_url)
|
329
330
|
raise "gz_url (#{arg_gz_url}) is not a string" unless arg_gz_url.is_a?(String)
|
330
331
|
|
@@ -334,7 +335,7 @@ class NVDFeedScraper
|
|
334
335
|
# @param arg_zip_url [String] the new URL of the zip archive of the feed.
|
335
336
|
# @return [String] the new URL of the zip archive of the feed.
|
336
337
|
# @example
|
337
|
-
# 'https://
|
338
|
+
# 'https://nvd.nist.gov/feeds/json/cve/2.0/nvdcve-2.0-2007.json.zip'
|
338
339
|
def zip_url=(arg_zip_url)
|
339
340
|
raise "zip_url (#{arg_zip_url}) is not a string" unless arg_zip_url.is_a?(String)
|
340
341
|
|
data/lib/nvd_feed_api/meta.rb
CHANGED
data/lib/nvd_feed_api/version.rb
CHANGED
data/lib/nvd_feed_api.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# @author Alexandre ZANNI <alexandre.zanni@engineer.com>
|
2
4
|
|
3
5
|
# Ruby internal
|
@@ -18,7 +20,7 @@ require 'nvd_feed_api/feed'
|
|
18
20
|
# scraper.feeds("CVE-2007")
|
19
21
|
# cve2007, cve2015 = scraper.feeds("CVE-2007", "CVE-2015")
|
20
22
|
class NVDFeedScraper
|
21
|
-
BASE = 'https://nvd.nist.gov'
|
23
|
+
BASE = 'https://nvd.nist.gov'
|
22
24
|
# The NVD url where is located the data feeds.
|
23
25
|
URL = "#{BASE}/vuln/data-feeds".freeze
|
24
26
|
# Load constants
|
@@ -40,7 +42,7 @@ class NVDFeedScraper
|
|
40
42
|
doc = Nokogiri::HTML(html)
|
41
43
|
@feeds = []
|
42
44
|
tmp_feeds = {}
|
43
|
-
doc.css('
|
45
|
+
doc.css('div[data-testid=nvd-json-2-feed-table] table.xml-feed-table tr[data-testid]').each do |tr|
|
44
46
|
num, type = tr.attr('data-testid')[13..].split('-')
|
45
47
|
case type
|
46
48
|
when 'meta'
|
@@ -128,15 +130,11 @@ class NVDFeedScraper
|
|
128
130
|
# Return a list with the name of all available feeds. Returned feed names can be use as argument for {#feeds} method. Can only be called after {#scrap}.
|
129
131
|
# @return [Array<String>] List with the name of all available feeds.
|
130
132
|
# @example
|
131
|
-
# scraper.available_feeds => ["CVE-Modified", "CVE-Recent", "CVE-
|
133
|
+
# scraper.available_feeds => ["CVE-Modified", "CVE-Recent", "CVE-2025", "[…]", "CVE-2002"]
|
132
134
|
def available_feeds
|
133
135
|
raise 'call scrap method before using available_feeds method' if @feeds.nil?
|
134
136
|
|
135
|
-
|
136
|
-
@feeds.each do |feed| # feed is an objet
|
137
|
-
feed_names.push(feed.name)
|
138
|
-
end
|
139
|
-
feed_names
|
137
|
+
@feeds.map(&:name)
|
140
138
|
end
|
141
139
|
|
142
140
|
# Search for CVE in all year feeds.
|
@@ -155,8 +153,7 @@ class NVDFeedScraper
|
|
155
153
|
# @return [Array] an Array of CVE, each CVE is a Ruby Hash.
|
156
154
|
# @todo implement a CVE Class instead of returning a Hash. May not be in the same order as provided.
|
157
155
|
# @note {#scrap} is needed before using this method.
|
158
|
-
# @see https://
|
159
|
-
# @see https://scap.nist.gov/schema/nvd/feed/0.1/CVE_JSON_4.0_min.schema
|
156
|
+
# @see https://csrc.nist.gov/schema/nvd/api/2.0/cve_api_json_2.0.schema
|
160
157
|
# @example
|
161
158
|
# s = NVDFeedScraper.new
|
162
159
|
# s.scrap
|
@@ -211,7 +208,7 @@ class NVDFeedScraper
|
|
211
208
|
|
212
209
|
matched_feeds = feeds_to_match.intersection(feed_names)
|
213
210
|
# and now that the intersection is done remove those virtual feeds and add CVE-2002 instead if needed
|
214
|
-
|
211
|
+
if matched_feeds.intersect?(virtual_feeds.to_set)
|
215
212
|
matched_feeds.subtract(virtual_feeds)
|
216
213
|
matched_feeds.add('CVE-2002')
|
217
214
|
end
|
data/nvd_feed_api.gemspec
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative 'lib/nvd_feed_api/version'
|
2
4
|
|
3
5
|
Gem::Specification.new do |s|
|
@@ -13,7 +15,6 @@ Gem::Specification.new do |s|
|
|
13
15
|
|
14
16
|
s.files = `git ls-files`.split("\n")
|
15
17
|
s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
|
16
|
-
s.test_files = s.files.grep(%r{^(test)/})
|
17
18
|
s.require_paths = ['lib']
|
18
19
|
|
19
20
|
s.metadata = {
|
@@ -24,10 +25,11 @@ Gem::Specification.new do |s|
|
|
24
25
|
'homepage_uri' => 'https://noraj.gitlab.io/nvd_api/',
|
25
26
|
'source_code_uri' => 'https://gitlab.com/noraj/nvd_api/tree/master',
|
26
27
|
'wiki_uri' => 'https://gitlab.com/noraj/nvd_api/wikis/home',
|
28
|
+
'funding_uri' => 'https://github.com/sponsors/noraj',
|
27
29
|
'rubygems_mfa_required' => 'true'
|
28
30
|
}
|
29
31
|
|
30
|
-
s.required_ruby_version = ['>=
|
32
|
+
s.required_ruby_version = ['>= 3.1.0', '< 4.0']
|
31
33
|
|
32
34
|
s.add_dependency('archive-zip', '~> 0.11')
|
33
35
|
s.add_dependency('nokogiri', '~> 1.11')
|
data/pages/CHANGELOG.md
CHANGED
@@ -1,4 +1,20 @@
|
|
1
|
-
# [
|
1
|
+
# [0.6.0] - 29 May 2025
|
2
|
+
|
3
|
+
- Enhancements and fixes:
|
4
|
+
- migrating from CVE JSON 1.1 to 2.0 spec
|
5
|
+
- 1.1 feeds will be removed in the future
|
6
|
+
- 1.1 feeds do not contain Deferred status CVEs
|
7
|
+
- Chore:
|
8
|
+
- Update dependencies
|
9
|
+
|
10
|
+
# [0.5.0] - 30 March 2025
|
11
|
+
|
12
|
+
- Breaking changes:
|
13
|
+
- Drop support for Ruby 2.7 and 3.0
|
14
|
+
- Chore:
|
15
|
+
- Add support for Ruby 3.2 & 3.3 & 3.4
|
16
|
+
- Update dependencies
|
17
|
+
- Add freeze literal string comment
|
2
18
|
|
3
19
|
# [0.4.0] - 31 January 2021
|
4
20
|
|
@@ -78,7 +94,7 @@
|
|
78
94
|
|
79
95
|
[0.0.1.rc2]: https://gitlab.com/noraj/nvd_api/tags/v0.0.1.rc2
|
80
96
|
|
81
|
-
- Add some contribution guidelines,
|
97
|
+
- Add some contribution guidelines, issues and MR templates.
|
82
98
|
- Improve the README to be a good entrypoint.
|
83
99
|
- Improve the FEATURES.
|
84
100
|
|
data/pages/INSTALL.md
CHANGED
@@ -4,42 +4,42 @@
|
|
4
4
|
|
5
5
|
### Install from rubygems.org
|
6
6
|
|
7
|
-
```
|
8
|
-
|
7
|
+
```bash
|
8
|
+
gem install nvd_feed_api
|
9
9
|
```
|
10
10
|
|
11
11
|
## Development
|
12
12
|
|
13
|
-
It's better to use [rbenv](https://github.com/rbenv/rbenv) to have latests version of ruby and to avoid trashing your system ruby.
|
13
|
+
It's better to use [rbenv](https://github.com/rbenv/rbenv) or [asdf-vm](https://asdf-vm.com/) to have latests version of ruby and to avoid trashing your system ruby.
|
14
14
|
|
15
15
|
### Install from rubygems.org
|
16
16
|
|
17
|
-
```
|
18
|
-
|
17
|
+
```bash
|
18
|
+
gem install --development nvd_feed_api
|
19
19
|
```
|
20
20
|
|
21
21
|
### Build from git
|
22
22
|
|
23
23
|
Just replace `x.x.x` with the gem version you see after `gem build`.
|
24
24
|
|
25
|
-
```
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
25
|
+
```bash
|
26
|
+
git clone https://gitlab.com/noraj/nvd_api.git nvd_feed_api
|
27
|
+
cd nvd_feed_api
|
28
|
+
gem install bundle
|
29
|
+
./bin/nvd_feed_api_setup
|
30
|
+
rake install
|
31
31
|
```
|
32
32
|
|
33
33
|
You can use `nvd_feed_api_console` to launch `irb` with the API required.
|
34
34
|
|
35
35
|
Alternatively to build you can use:
|
36
36
|
|
37
|
-
```
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
37
|
+
```bash
|
38
|
+
git clone https://gitlab.com/noraj/nvd_api.git nvd_feed_api
|
39
|
+
cd nvd_feed_api
|
40
|
+
gem install bundle
|
41
|
+
gem build nvd_feed_api.gemspec
|
42
|
+
gem install --development nvd_feed_api-x.x.x.gem
|
43
43
|
```
|
44
44
|
|
45
45
|
Note: if an automatic install is needed you can get the version with `$ gem build nvd_feed_api.gemspec | grep Version | cut -d' ' -f4`.
|
@@ -48,10 +48,10 @@ Note: if an automatic install is needed you can get the version with `$ gem buil
|
|
48
48
|
|
49
49
|
Useful when you want to try your changes without building the gem and re-installing it each time.
|
50
50
|
|
51
|
-
```
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
51
|
+
```bash
|
52
|
+
git clone https://gitlab.com/noraj/nvd_api.git nvd_feed_api
|
53
|
+
cd nvd_feed_api
|
54
|
+
gem install bundle
|
55
|
+
./bin/nvd_feed_api_setup
|
56
|
+
irb -Ilib -rnvd_feed_api
|
57
57
|
```
|
data/test/test_nvd_feed_api.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'minitest/autorun'
|
2
4
|
require 'nvd_feed_api'
|
3
5
|
require 'date'
|
@@ -78,7 +80,7 @@ class NVDAPITest < Minitest::Test
|
|
78
80
|
f2017, f2016, f_modified = @s.feeds('CVE-2017', 'CVE-2016', 'CVE-Modified')
|
79
81
|
# one arg
|
80
82
|
# can't use assert_instance_of because there is no boolean class
|
81
|
-
|
83
|
+
assert_includes(['TrueClass', 'FalseClass'], @s.update_feeds(f2017).class.to_s, "update_feeds doesn't return a boolean")
|
82
84
|
# two args
|
83
85
|
assert_instance_of(Array, @s.update_feeds(f2017, f2016), "update_feeds doesn't return an array")
|
84
86
|
refute_empty(@s.update_feeds(f2017, f2016), 'update_feeds returns an empty array')
|
@@ -90,7 +92,7 @@ class NVDAPITest < Minitest::Test
|
|
90
92
|
err = assert_raises(RuntimeError) do
|
91
93
|
@s.update_feeds(1)
|
92
94
|
end
|
93
|
-
assert_equal(
|
95
|
+
assert_equal('the provided argument 1 is not a Feed or an Array', err.message)
|
94
96
|
## empty array
|
95
97
|
assert_empty(@s.update_feeds([]))
|
96
98
|
end
|
@@ -102,6 +104,7 @@ class NVDAPITest < Minitest::Test
|
|
102
104
|
assert_instance_of(String, default_val, "default_storage_location doesn't return a string")
|
103
105
|
# check new value
|
104
106
|
new_val = '/srv/downloads/'
|
107
|
+
|
105
108
|
assert_equal(new_val, NVDFeedScraper::Feed.default_storage_location = new_val, 'the new value was not set properly')
|
106
109
|
# put the default value back / restore context
|
107
110
|
NVDFeedScraper::Feed.default_storage_location = default_val
|
@@ -109,9 +112,9 @@ class NVDAPITest < Minitest::Test
|
|
109
112
|
|
110
113
|
def test_feed_attributes
|
111
114
|
name = 'CVE-2010'
|
112
|
-
meta_url = 'https://nvd.nist.gov/feeds/json/cve/
|
113
|
-
gz_url = 'https://nvd.nist.gov/feeds/json/cve/
|
114
|
-
zip_url = 'https://nvd.nist.gov/feeds/json/cve/
|
115
|
+
meta_url = 'https://nvd.nist.gov/feeds/json/cve/2.0/nvdcve-2.0-2010.meta'
|
116
|
+
gz_url = 'https://nvd.nist.gov/feeds/json/cve/2.0/nvdcve-2.0-2010.json.gz'
|
117
|
+
zip_url = 'https://nvd.nist.gov/feeds/json/cve/2.0/nvdcve-2.0-2010.json.zip'
|
115
118
|
f = @s.feeds('CVE-2010')
|
116
119
|
# Test name
|
117
120
|
assert_instance_of(String, f.name, "name doesn't return a string")
|
@@ -137,10 +140,12 @@ class NVDAPITest < Minitest::Test
|
|
137
140
|
# Test json_file
|
138
141
|
assert_nil(f.json_file)
|
139
142
|
f.json_pull
|
143
|
+
|
140
144
|
assert_instance_of(String, f.json_file, "json_file doesn't return a string")
|
141
145
|
refute_empty(f.json_file, 'json_file is empty')
|
142
146
|
# Test meta (after json_pull)
|
143
147
|
f.meta_pull
|
148
|
+
|
144
149
|
assert_instance_of(NVDFeedScraper::Meta, f.meta, "meta doesn't return a Meta object")
|
145
150
|
|
146
151
|
# Test data (require json_pull)
|
@@ -161,6 +166,7 @@ class NVDAPITest < Minitest::Test
|
|
161
166
|
def test_feed_available_cves
|
162
167
|
f = @s.feeds('CVE-2011')
|
163
168
|
f.json_pull
|
169
|
+
|
164
170
|
assert_instance_of(Array, f.available_cves, "available_cves doesn't return an array")
|
165
171
|
refute_empty(f.available_cves, 'available_cves returns an empty array')
|
166
172
|
end
|
@@ -199,6 +205,7 @@ class NVDAPITest < Minitest::Test
|
|
199
205
|
def test_feed_download_gz
|
200
206
|
f = @s.feeds('CVE-2013')
|
201
207
|
return_value = f.download_gz
|
208
|
+
|
202
209
|
assert_instance_of(String, return_value, "download_gz doesn't return a string")
|
203
210
|
refute_empty(return_value, 'download_gz returns an empty string')
|
204
211
|
assert(File.file?(return_value), 'download_gz returns an unexisting file')
|
@@ -207,6 +214,7 @@ class NVDAPITest < Minitest::Test
|
|
207
214
|
def test_feed_download_zip
|
208
215
|
f = @s.feeds('CVE-2003')
|
209
216
|
return_value = f.download_zip
|
217
|
+
|
210
218
|
assert_instance_of(String, return_value, "download_zip doesn't return a string")
|
211
219
|
refute_empty(return_value, 'download_zip returns an empty string')
|
212
220
|
assert(File.file?(return_value), 'download_zip returns an unexisting file')
|
@@ -215,6 +223,7 @@ class NVDAPITest < Minitest::Test
|
|
215
223
|
def test_feed_json_pull
|
216
224
|
f = @s.feeds('CVE-2004')
|
217
225
|
return_value = f.json_pull
|
226
|
+
|
218
227
|
assert_instance_of(String, return_value, "json_pull doesn't return a string")
|
219
228
|
refute_empty(return_value, 'json_pull returns an empty string')
|
220
229
|
assert(File.file?(return_value), 'json_pull returns an unexisting file')
|
@@ -222,6 +231,7 @@ class NVDAPITest < Minitest::Test
|
|
222
231
|
|
223
232
|
def test_feed_meta_pull
|
224
233
|
f = @s.feeds('CVE-2005')
|
234
|
+
|
225
235
|
assert_instance_of(NVDFeedScraper::Meta, f.meta_pull, "meta_pull doesn't return a Meta object")
|
226
236
|
end
|
227
237
|
|
@@ -231,7 +241,7 @@ class NVDAPITest < Minitest::Test
|
|
231
241
|
f_new = @s.feeds('CVE-2006')
|
232
242
|
# Right arg
|
233
243
|
# can't use assert_instance_of because there is no boolean class
|
234
|
-
|
244
|
+
assert_includes(['TrueClass', 'FalseClass'], f.update!(f_new).class.to_s, "update! doesn't return a boolean")
|
235
245
|
# Bad arg
|
236
246
|
err = assert_raises(RuntimeError) do
|
237
247
|
f.update!('bad_arg')
|
@@ -240,45 +250,48 @@ class NVDAPITest < Minitest::Test
|
|
240
250
|
end
|
241
251
|
|
242
252
|
def test_meta_parse_noarg
|
243
|
-
m = NVDFeedScraper::Meta.new('https://nvd.nist.gov/feeds/json/cve/
|
253
|
+
m = NVDFeedScraper::Meta.new('https://nvd.nist.gov/feeds/json/cve/2.0/nvdcve-2.0-2015.meta')
|
254
|
+
|
244
255
|
assert_equal(0, m.parse, 'parse method return nothing')
|
245
256
|
end
|
246
257
|
|
247
258
|
def test_meta_parse_witharg
|
248
259
|
m = NVDFeedScraper::Meta.new
|
249
|
-
meta_url = 'https://nvd.nist.gov/feeds/json/cve/
|
260
|
+
meta_url = 'https://nvd.nist.gov/feeds/json/cve/2.0/nvdcve-2.0-2015.meta'
|
261
|
+
|
250
262
|
assert_equal(0, m.parse(meta_url), 'parse method return nothing')
|
251
263
|
end
|
252
264
|
|
253
265
|
def test_meta_url_setter
|
254
266
|
m = NVDFeedScraper::Meta.new
|
255
|
-
meta_url = 'https://nvd.nist.gov/feeds/json/cve/
|
267
|
+
meta_url = 'https://nvd.nist.gov/feeds/json/cve/2.0/nvdcve-2.0-2015.meta'
|
268
|
+
|
256
269
|
assert_equal(meta_url, m.url = meta_url, 'the meta URL is not set correctly')
|
257
270
|
end
|
258
271
|
|
259
272
|
def test_meta_attributes
|
260
273
|
m = NVDFeedScraper::Meta.new
|
261
|
-
meta_url = 'https://nvd.nist.gov/feeds/json/cve/
|
274
|
+
meta_url = 'https://nvd.nist.gov/feeds/json/cve/2.0/nvdcve-2.0-2015.meta'
|
262
275
|
m.url = meta_url
|
263
276
|
m.parse
|
264
277
|
# Test gz_size
|
265
278
|
assert_instance_of(String, m.gz_size, "Meta gz_size method doesn't return a string")
|
266
|
-
|
279
|
+
assert_match(/[0-9]+/, m.gz_size, 'Meta gz_size is not an integer')
|
267
280
|
# Test last_modified_date
|
268
281
|
assert_instance_of(String, m.last_modified_date, "Meta last_modified_date method doesn't return a string")
|
269
282
|
## Date and time of day for calendar date (extended) '%FT%T%:z'
|
270
283
|
assert(Date.rfc3339(m.last_modified_date), 'Meta last_modified_date is not a rfc3339 date')
|
271
284
|
# Test sha256
|
272
285
|
assert_instance_of(String, m.sha256, "Meta sha256 method doesn't return a string")
|
273
|
-
|
286
|
+
assert_match(/[0-9A-F]{64}/, m.sha256, 'Meta sha256 is not a sha256 string matching /[0-9A-F]{64}/')
|
274
287
|
# Test size
|
275
288
|
assert_instance_of(String, m.size, "Meta size method doesn't return a string")
|
276
|
-
|
289
|
+
assert_match(/[0-9]+/, m.size, 'Meta size is not an integer')
|
277
290
|
# Test url
|
278
291
|
assert_instance_of(String, m.url, "Meta url method doesn't return a string")
|
279
292
|
assert_equal(meta_url, m.url, 'The Meta url was modified')
|
280
293
|
# Test zip_size
|
281
294
|
assert_instance_of(String, m.zip_size, "Meta zip_size method doesn't return a string")
|
282
|
-
|
295
|
+
assert_match(/[0-9]+/, m.zip_size, 'Meta zip_size is not an integer')
|
283
296
|
end
|
284
297
|
end
|
metadata
CHANGED
@@ -1,14 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: nvd_feed_api
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Alexandre ZANNI
|
8
|
-
autorequire:
|
9
8
|
bindir: bin
|
10
9
|
cert_chain: []
|
11
|
-
date:
|
10
|
+
date: 2025-05-29 00:00:00.000000000 Z
|
12
11
|
dependencies:
|
13
12
|
- !ruby/object:Gem::Dependency
|
14
13
|
name: archive-zip
|
@@ -107,8 +106,8 @@ metadata:
|
|
107
106
|
homepage_uri: https://noraj.gitlab.io/nvd_api/
|
108
107
|
source_code_uri: https://gitlab.com/noraj/nvd_api/tree/master
|
109
108
|
wiki_uri: https://gitlab.com/noraj/nvd_api/wikis/home
|
109
|
+
funding_uri: https://github.com/sponsors/noraj
|
110
110
|
rubygems_mfa_required: 'true'
|
111
|
-
post_install_message:
|
112
111
|
rdoc_options: []
|
113
112
|
require_paths:
|
114
113
|
- lib
|
@@ -116,19 +115,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
116
115
|
requirements:
|
117
116
|
- - ">="
|
118
117
|
- !ruby/object:Gem::Version
|
119
|
-
version:
|
118
|
+
version: 3.1.0
|
120
119
|
- - "<"
|
121
120
|
- !ruby/object:Gem::Version
|
122
|
-
version: '
|
121
|
+
version: '4.0'
|
123
122
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
124
123
|
requirements:
|
125
124
|
- - ">="
|
126
125
|
- !ruby/object:Gem::Version
|
127
126
|
version: '0'
|
128
127
|
requirements: []
|
129
|
-
rubygems_version: 3.
|
130
|
-
signing_key:
|
128
|
+
rubygems_version: 3.6.2
|
131
129
|
specification_version: 4
|
132
130
|
summary: API for NVD CVE feeds
|
133
|
-
test_files:
|
134
|
-
- test/test_nvd_feed_api.rb
|
131
|
+
test_files: []
|