nvd_feed_api 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitlab-ci.yml +5 -3
- data/.rubocop.yml +1 -1
- data/.tool-versions +1 -1
- data/Gemfile +5 -10
- data/Gemfile.lock +70 -50
- data/lib/nvd_feed_api/feed.rb +5 -4
- data/lib/nvd_feed_api/meta.rb +1 -1
- data/lib/nvd_feed_api/version.rb +1 -1
- data/lib/nvd_feed_api.rb +47 -17
- data/nvd_feed_api.gemspec +1 -1
- data/pages/CHANGELOG.md +14 -0
- data/pages/EXAMPLES.md +0 -2
- data/test/test_nvd_feed_api.rb +5 -6
- metadata +3 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 0b2135d36e474c39e03f1124211c464e57d7251aea33c3de29610ab2ceb781e1
|
|
4
|
+
data.tar.gz: d5a520e831cdb7104ac57f0fe676a55bcac2c3eaee3d9f072070fe0405dae992
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: cc5673883c2ffcd84e21e88a6ca84d2f518c390bb8622da6408e91051bee01822a37d2e60a9060e6f8289750cef4d165ea9d85d24be238f83ea2ebf87209bace
|
|
7
|
+
data.tar.gz: b22d6e16a3ec4d6e98b64e91b1eca26ad7f167475edc9a5f9619b3d1782b6a0cfebc0bc9295d1d389f9e7875af68290c71c5f75b58bc0414946099ac4ae659c1
|
data/.gitlab-ci.yml
CHANGED
|
@@ -12,9 +12,11 @@ before_script:
|
|
|
12
12
|
- gem install bundler --no-document # Bundler is not installed with the image
|
|
13
13
|
# install nproc (coreutils) for bundle -j
|
|
14
14
|
# install git for building the gemspec
|
|
15
|
-
# install make, gcc for building gem native extension (commonmarker)
|
|
16
|
-
# libc-dev for musl-dev dependency (stdlib.h) needed by gcc
|
|
17
|
-
|
|
15
|
+
# install make, gcc (build-base) for building gem native extension (commonmarker)
|
|
16
|
+
# libc-dev for musl-dev dependency (stdlib.h) needed by gcc (build-base)
|
|
17
|
+
# yaml-dev to build psych ruby native extension
|
|
18
|
+
# add gcompat is GNU compatibility for MUSL libc
|
|
19
|
+
- apk --no-cache add coreutils git build-base yaml-dev
|
|
18
20
|
- bundle config set path 'vendor' # Set dependencies install dir to ./vendor/ruby
|
|
19
21
|
- bundle install -j $(nproc) # Install dependencies into ./vendor/ruby
|
|
20
22
|
- bundle exec rake install # install the gem
|
data/.rubocop.yml
CHANGED
data/.tool-versions
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
ruby 4.0.
|
|
1
|
+
ruby 4.0.9
|
data/Gemfile
CHANGED
|
@@ -4,8 +4,8 @@ source 'https://rubygems.org'
|
|
|
4
4
|
|
|
5
5
|
gemspec
|
|
6
6
|
|
|
7
|
-
group :runtime
|
|
8
|
-
gem 'archive-zip', '0.13.
|
|
7
|
+
group :runtime do
|
|
8
|
+
gem 'archive-zip', '0.13.1'
|
|
9
9
|
gem 'nokogiri', '~> 1.11'
|
|
10
10
|
gem 'oj', '>= 3.7.8', '<4'
|
|
11
11
|
end
|
|
@@ -25,12 +25,7 @@ group :development, :lint do
|
|
|
25
25
|
end
|
|
26
26
|
|
|
27
27
|
group :development, :docs do
|
|
28
|
-
|
|
29
|
-
# https://github.com/
|
|
30
|
-
gem '
|
|
31
|
-
gem 'logger', '< 2.0'
|
|
32
|
-
# gem 'yard', ['>= 0.9.27', '< 0.10']
|
|
33
|
-
# yard not supporting recent commonmarker version yet https://github.com/lsegal/yard/issues/1528
|
|
34
|
-
# yard 0.9.38 not supporting ruby 4.0 yet https://github.com/lsegal/yard/issues/1636
|
|
35
|
-
gem 'yard', github: 'ParadoxV5/yard', ref: '9e869c940859570b07b81c5eadd6070e76f6291e', branch: 'commonmarker-1.0'
|
|
28
|
+
gem 'commonmarker', '~> 2.8'
|
|
29
|
+
gem 'irb' # to supress warning because using @overload from yard https://github.com/lsegal/yard/pull/1643#issuecomment-4322721189
|
|
30
|
+
gem 'yard', ['>= 0.9.43', '< 0.10']
|
|
36
31
|
end
|
data/Gemfile.lock
CHANGED
|
@@ -1,15 +1,7 @@
|
|
|
1
|
-
GIT
|
|
2
|
-
remote: https://github.com/ParadoxV5/yard.git
|
|
3
|
-
revision: 9e869c940859570b07b81c5eadd6070e76f6291e
|
|
4
|
-
ref: 9e869c940859570b07b81c5eadd6070e76f6291e
|
|
5
|
-
branch: commonmarker-1.0
|
|
6
|
-
specs:
|
|
7
|
-
yard (0.9.36)
|
|
8
|
-
|
|
9
1
|
PATH
|
|
10
2
|
remote: .
|
|
11
3
|
specs:
|
|
12
|
-
nvd_feed_api (0.
|
|
4
|
+
nvd_feed_api (0.8.0)
|
|
13
5
|
archive-zip (~> 0.11)
|
|
14
6
|
nokogiri (~> 1.11)
|
|
15
7
|
oj (>= 3.7.8, < 4)
|
|
@@ -17,73 +9,101 @@ PATH
|
|
|
17
9
|
GEM
|
|
18
10
|
remote: https://rubygems.org/
|
|
19
11
|
specs:
|
|
20
|
-
archive-zip (0.13.
|
|
21
|
-
io-like (~> 0.4.0
|
|
12
|
+
archive-zip (0.13.1)
|
|
13
|
+
io-like (~> 0.4.0)
|
|
22
14
|
ast (2.4.3)
|
|
23
|
-
bigdecimal (4.
|
|
24
|
-
commonmarker (2.
|
|
25
|
-
|
|
15
|
+
bigdecimal (4.1.3)
|
|
16
|
+
commonmarker (2.10.1-aarch64-linux)
|
|
17
|
+
commonmarker (2.10.1-aarch64-linux-musl)
|
|
18
|
+
commonmarker (2.10.1-arm-linux-gnu)
|
|
19
|
+
commonmarker (2.10.1-arm-linux-musl)
|
|
20
|
+
commonmarker (2.10.1-arm64-darwin)
|
|
21
|
+
commonmarker (2.10.1-x86_64-darwin)
|
|
22
|
+
commonmarker (2.10.1-x86_64-linux)
|
|
23
|
+
commonmarker (2.10.1-x86_64-linux-musl)
|
|
24
|
+
drb (2.2.3)
|
|
25
|
+
erb (6.0.7)
|
|
26
|
+
io-console (0.9.5)
|
|
26
27
|
io-like (0.4.0)
|
|
27
|
-
|
|
28
|
-
|
|
28
|
+
irb (1.18.0)
|
|
29
|
+
pp (>= 0.6.0)
|
|
30
|
+
prism (>= 1.3.0)
|
|
31
|
+
rdoc (>= 4.0.0)
|
|
32
|
+
reline (>= 0.4.2)
|
|
33
|
+
json (3.1.0)
|
|
34
|
+
language_server-protocol (3.17.0.6)
|
|
29
35
|
lint_roller (1.1.0)
|
|
30
36
|
logger (1.7.0)
|
|
31
|
-
minitest (6.0.
|
|
37
|
+
minitest (6.0.6)
|
|
38
|
+
drb (~> 2.0)
|
|
32
39
|
prism (~> 1.5)
|
|
33
|
-
nokogiri (1.19.
|
|
40
|
+
nokogiri (1.19.4-aarch64-linux-gnu)
|
|
34
41
|
racc (~> 1.4)
|
|
35
|
-
nokogiri (1.19.
|
|
42
|
+
nokogiri (1.19.4-aarch64-linux-musl)
|
|
36
43
|
racc (~> 1.4)
|
|
37
|
-
nokogiri (1.19.
|
|
44
|
+
nokogiri (1.19.4-arm-linux-gnu)
|
|
38
45
|
racc (~> 1.4)
|
|
39
|
-
nokogiri (1.19.
|
|
46
|
+
nokogiri (1.19.4-arm-linux-musl)
|
|
40
47
|
racc (~> 1.4)
|
|
41
|
-
nokogiri (1.19.
|
|
48
|
+
nokogiri (1.19.4-arm64-darwin)
|
|
42
49
|
racc (~> 1.4)
|
|
43
|
-
nokogiri (1.19.
|
|
50
|
+
nokogiri (1.19.4-x86_64-darwin)
|
|
44
51
|
racc (~> 1.4)
|
|
45
|
-
nokogiri (1.19.
|
|
52
|
+
nokogiri (1.19.4-x86_64-linux-gnu)
|
|
46
53
|
racc (~> 1.4)
|
|
47
|
-
nokogiri (1.19.
|
|
54
|
+
nokogiri (1.19.4-x86_64-linux-musl)
|
|
48
55
|
racc (~> 1.4)
|
|
49
|
-
oj (3.
|
|
56
|
+
oj (3.17.7)
|
|
50
57
|
bigdecimal (>= 3.0)
|
|
51
58
|
ostruct (>= 0.2)
|
|
52
59
|
ostruct (0.6.3)
|
|
53
|
-
parallel (
|
|
54
|
-
parser (3.3.
|
|
60
|
+
parallel (2.3.0)
|
|
61
|
+
parser (3.3.13.0)
|
|
55
62
|
ast (~> 2.4.1)
|
|
56
63
|
racc
|
|
57
|
-
|
|
64
|
+
pp (0.6.4)
|
|
65
|
+
prettyprint
|
|
66
|
+
prettyprint (0.2.1)
|
|
67
|
+
prism (1.9.0)
|
|
58
68
|
racc (1.8.1)
|
|
59
69
|
rainbow (3.1.1)
|
|
60
|
-
rake (13.
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
70
|
+
rake (13.4.2)
|
|
71
|
+
rbs (4.2.0)
|
|
72
|
+
logger
|
|
73
|
+
prism (>= 1.6.0)
|
|
74
|
+
tsort
|
|
75
|
+
rdoc (8.1.0)
|
|
76
|
+
erb
|
|
77
|
+
prism (>= 1.6.0)
|
|
78
|
+
rbs (>= 4.0.0)
|
|
79
|
+
tsort
|
|
80
|
+
regexp_parser (2.13.1)
|
|
81
|
+
reline (0.7.0)
|
|
82
|
+
io-console (~> 0.5)
|
|
83
|
+
rubocop (1.92.2)
|
|
84
|
+
json (>= 2.3)
|
|
67
85
|
language_server-protocol (~> 3.17.0.2)
|
|
68
86
|
lint_roller (~> 1.1.0)
|
|
69
|
-
parallel (
|
|
87
|
+
parallel (>= 1.10)
|
|
70
88
|
parser (>= 3.3.0.2)
|
|
71
89
|
rainbow (>= 2.2.2, < 4.0)
|
|
72
|
-
regexp_parser (>= 2.
|
|
73
|
-
rubocop-ast (>= 1.
|
|
90
|
+
regexp_parser (>= 2.11.3, < 3.0)
|
|
91
|
+
rubocop-ast (>= 1.49.0, < 2.0)
|
|
74
92
|
ruby-progressbar (~> 1.7)
|
|
75
93
|
unicode-display_width (>= 2.4.0, < 4.0)
|
|
76
|
-
rubocop-ast (1.
|
|
94
|
+
rubocop-ast (1.50.0)
|
|
77
95
|
parser (>= 3.3.7.2)
|
|
78
96
|
prism (~> 1.7)
|
|
79
|
-
rubocop-minitest (0.
|
|
97
|
+
rubocop-minitest (0.41.0)
|
|
80
98
|
lint_roller (~> 1.1)
|
|
81
|
-
rubocop (>= 1.
|
|
99
|
+
rubocop (>= 1.89.0, < 2.0)
|
|
82
100
|
rubocop-ast (>= 1.38.0, < 2.0)
|
|
83
101
|
ruby-progressbar (1.13.0)
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
102
|
+
tsort (0.2.0)
|
|
103
|
+
unicode-display_width (3.3.0)
|
|
104
|
+
unicode-emoji (~> 4.3)
|
|
105
|
+
unicode-emoji (4.3.0)
|
|
106
|
+
yard (0.9.45)
|
|
87
107
|
|
|
88
108
|
PLATFORMS
|
|
89
109
|
aarch64-linux-gnu
|
|
@@ -96,10 +116,10 @@ PLATFORMS
|
|
|
96
116
|
x86_64-linux-musl
|
|
97
117
|
|
|
98
118
|
DEPENDENCIES
|
|
99
|
-
archive-zip (= 0.13.
|
|
119
|
+
archive-zip (= 0.13.1)
|
|
100
120
|
bundler (~> 4.0)
|
|
101
|
-
commonmarker (~> 2.
|
|
102
|
-
|
|
121
|
+
commonmarker (~> 2.8)
|
|
122
|
+
irb
|
|
103
123
|
minitest (~> 6.0)
|
|
104
124
|
nokogiri (~> 1.11)
|
|
105
125
|
nvd_feed_api!
|
|
@@ -107,7 +127,7 @@ DEPENDENCIES
|
|
|
107
127
|
rake (~> 13.2)
|
|
108
128
|
rubocop (~> 1.71)
|
|
109
129
|
rubocop-minitest (~> 0.36)
|
|
110
|
-
yard
|
|
130
|
+
yard (>= 0.9.43, < 0.10)
|
|
111
131
|
|
|
112
132
|
BUNDLED WITH
|
|
113
|
-
4.0.
|
|
133
|
+
4.0.23
|
data/lib/nvd_feed_api/feed.rb
CHANGED
|
@@ -28,6 +28,7 @@ class NVDFeedScraper
|
|
|
28
28
|
attr_reader :name
|
|
29
29
|
|
|
30
30
|
# @return [String] the last update date of the feed information on the NVD website.
|
|
31
|
+
# @deprecated Is broken since {NVDFeedScraper#scrap} was replaced with {NVDFeedScraper#populate}.
|
|
31
32
|
# @example
|
|
32
33
|
# '10/19/2017 3:27:02 AM -04:00'
|
|
33
34
|
attr_reader :updated
|
|
@@ -53,7 +54,7 @@ class NVDFeedScraper
|
|
|
53
54
|
# Note that {#json_pull} also calls {#meta_pull}.
|
|
54
55
|
# @example
|
|
55
56
|
# s = NVDFeedScraper.new
|
|
56
|
-
# s.
|
|
57
|
+
# s.populate
|
|
57
58
|
# f = s.feeds("CVE-2014")
|
|
58
59
|
# f.meta # => nil
|
|
59
60
|
# f.meta_pull
|
|
@@ -64,7 +65,7 @@ class NVDFeedScraper
|
|
|
64
65
|
# @note Return nil if not previously loaded by {#json_pull}.
|
|
65
66
|
# @example
|
|
66
67
|
# s = NVDFeedScraper.new
|
|
67
|
-
# s.
|
|
68
|
+
# s.populate
|
|
68
69
|
# f = s.feeds("CVE-2014")
|
|
69
70
|
# f.json_file # => nil
|
|
70
71
|
# f.json_pull
|
|
@@ -218,7 +219,7 @@ class NVDFeedScraper
|
|
|
218
219
|
# @see https://csrc.nist.gov/schema/nvd/api/2.0/cve_api_json_2.0.schema
|
|
219
220
|
# @example
|
|
220
221
|
# s = NVDFeedScraper.new
|
|
221
|
-
# s.
|
|
222
|
+
# s.populate
|
|
222
223
|
# f = s.feeds("CVE-2014")
|
|
223
224
|
# f.json_pull
|
|
224
225
|
# f.cve("CVE-2014-0002", "cve-2014-0001")
|
|
@@ -248,7 +249,7 @@ class NVDFeedScraper
|
|
|
248
249
|
# Sorting CVE can allow us to parse quicker
|
|
249
250
|
# Upcase to be sure include? works
|
|
250
251
|
cves_to_find = arg_cve[0].map(&:upcase).sort
|
|
251
|
-
raise 'one of the provided arguments is not a String' unless cves_to_find.all?
|
|
252
|
+
raise 'one of the provided arguments is not a String' unless cves_to_find.all?(String)
|
|
252
253
|
raise 'bad CVE name' unless cves_to_find.all? { |x| /^CVE-[0-9]{4}-[0-9]{4,}$/i.match?(x) }
|
|
253
254
|
|
|
254
255
|
doc = Oj::Doc.open(File.read(@json_file))
|
data/lib/nvd_feed_api/meta.rb
CHANGED
data/lib/nvd_feed_api/version.rb
CHANGED
data/lib/nvd_feed_api.rb
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
# @author Alexandre ZANNI <alexandre.zanni@engineer.com>
|
|
4
4
|
|
|
5
5
|
# Ruby internal
|
|
6
|
-
require 'net/https'
|
|
6
|
+
# require 'net/https'
|
|
7
7
|
# External
|
|
8
8
|
require 'nokogiri'
|
|
9
9
|
# Project internal
|
|
@@ -13,7 +13,7 @@ require 'nvd_feed_api/feed'
|
|
|
13
13
|
# The class that parse NVD website to get information.
|
|
14
14
|
# @example Initialize a NVDFeedScraper object, get the feeds and see them:
|
|
15
15
|
# scraper = NVDFeedScraper.new
|
|
16
|
-
# scraper.
|
|
16
|
+
# # scraper.populate
|
|
17
17
|
# scraper.available_feeds
|
|
18
18
|
# scraper.feeds
|
|
19
19
|
# scraper.feeds("CVE-2007")
|
|
@@ -28,26 +28,56 @@ class NVDFeedScraper
|
|
|
28
28
|
# Initialize the scraper
|
|
29
29
|
def initialize
|
|
30
30
|
@url = URL
|
|
31
|
-
@feeds =
|
|
31
|
+
@feeds = populate
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Populate feeds deterministically without scraping the NVD website.
|
|
35
|
+
# Replaces {scrap}.
|
|
36
|
+
# @return [Integer] Number of populated feeds.
|
|
37
|
+
def populate
|
|
38
|
+
@feeds = []
|
|
39
|
+
current_year = Time.now.year
|
|
40
|
+
names = ['CVE-Modified', 'CVE-Recent'] + (2002..current_year).to_a.reverse.map { |year| "CVE-#{year}" }
|
|
41
|
+
names.each do |name|
|
|
42
|
+
file_part = case name
|
|
43
|
+
when 'CVE-Modified' then 'modified'
|
|
44
|
+
when 'CVE-Recent' then 'recent'
|
|
45
|
+
else name.split('-').last
|
|
46
|
+
end
|
|
47
|
+
base_path = "#{BASE}/feeds/json/cve/2.0/nvdcve-2.0-#{file_part}"
|
|
48
|
+
@feeds.push(Feed.new(name,
|
|
49
|
+
nil,
|
|
50
|
+
"#{base_path}.meta",
|
|
51
|
+
"#{base_path}.json.gz",
|
|
52
|
+
"#{base_path}.json.zip"))
|
|
53
|
+
end
|
|
54
|
+
return @feeds.size
|
|
32
55
|
end
|
|
33
56
|
|
|
34
57
|
# Scrap / parse the website to get the feeds and fill the {#feeds} attribute.
|
|
35
58
|
# @note {#scrap} need to be called only once but can be called again to update if the NVD feed page changed.
|
|
59
|
+
# @deprecated Since upstream use a JS SPA, an headless browser is required which is slow and unreliable. Replaced by {populate}.
|
|
36
60
|
# @return [Integer] Number of scrapped feeds.
|
|
37
61
|
def scrap
|
|
38
|
-
|
|
39
|
-
|
|
62
|
+
require 'lightpanda'
|
|
63
|
+
begin
|
|
64
|
+
browser = Lightpanda::Browser.new(timeout: 60)
|
|
65
|
+
browser.go_to(@url, wait: true)
|
|
66
|
+
html = browser.body
|
|
67
|
+
ensure
|
|
68
|
+
browser.quit
|
|
69
|
+
end
|
|
40
70
|
|
|
41
71
|
doc = Nokogiri::HTML(html)
|
|
42
72
|
@feeds = []
|
|
43
73
|
tmp_feeds = {}
|
|
44
74
|
doc.css('div[data-testid=nvd-json-2-feed-table] table.xml-feed-table tr[data-testid]').each do |tr|
|
|
45
|
-
num, type = tr.attr('data-testid')
|
|
75
|
+
num, type = tr.attr('data-testid').split('-')[1..2]
|
|
46
76
|
case type
|
|
47
77
|
when 'meta'
|
|
48
78
|
tmp_feeds[num] = {}
|
|
49
|
-
tmp_feeds[num][:name] = tr.css('td')[0].text
|
|
50
|
-
tmp_feeds[num][:updated] = tr.css('td')[1].text
|
|
79
|
+
tmp_feeds[num][:name] = tr.css('td')[0].text.strip
|
|
80
|
+
tmp_feeds[num][:updated] = tr.css('td')[1].text.strip
|
|
51
81
|
tmp_feeds[num][:meta] = BASE + tr.css('td')[2].css('> a').attr('href').value
|
|
52
82
|
when 'gz'
|
|
53
83
|
tmp_feeds[num][:gz] = BASE + tr.css('td > a').attr('href').value
|
|
@@ -63,7 +93,7 @@ class NVDFeedScraper
|
|
|
63
93
|
return @feeds.size
|
|
64
94
|
end
|
|
65
95
|
|
|
66
|
-
# Return feeds. Can only be called after {#
|
|
96
|
+
# Return feeds. Can only be called after {#populate}.
|
|
67
97
|
# @overload feeds
|
|
68
98
|
# All the feeds.
|
|
69
99
|
# @return [Array<Feed>] Attributes of all feeds. It's an array of {Feed} object.
|
|
@@ -86,7 +116,7 @@ class NVDFeedScraper
|
|
|
86
116
|
# scraper.feeds("CVE-2005", "CVE-2002") # => return CVE-2005 and CVE-2002 feeds
|
|
87
117
|
# @see https://nvd.nist.gov/vuln/data-feeds
|
|
88
118
|
def feeds(*arg_feeds)
|
|
89
|
-
raise 'call
|
|
119
|
+
raise 'call populate method before using feeds method' if @feeds.nil?
|
|
90
120
|
|
|
91
121
|
return_value = nil
|
|
92
122
|
if arg_feeds.empty?
|
|
@@ -99,7 +129,7 @@ class NVDFeedScraper
|
|
|
99
129
|
end
|
|
100
130
|
# if nothing found return nil
|
|
101
131
|
when Array
|
|
102
|
-
raise 'one of the provided arguments is not a String' unless arg_feeds[0].all?
|
|
132
|
+
raise 'one of the provided arguments is not a String' unless arg_feeds[0].all?(String)
|
|
103
133
|
|
|
104
134
|
# Sorting CVE can allow us to parse quicker
|
|
105
135
|
# Upcase to be sure include? works
|
|
@@ -126,12 +156,12 @@ class NVDFeedScraper
|
|
|
126
156
|
return return_value
|
|
127
157
|
end
|
|
128
158
|
|
|
129
|
-
# Return a list with the name of all available feeds. Returned feed names can be use as argument for {#feeds} method. Can only be called after {#
|
|
159
|
+
# Return a list with the name of all available feeds. Returned feed names can be use as argument for {#feeds} method. Can only be called after {#populate}.
|
|
130
160
|
# @return [Array<String>] List with the name of all available feeds.
|
|
131
161
|
# @example
|
|
132
162
|
# scraper.available_feeds => ["CVE-Modified", "CVE-Recent", "CVE-2025", "[…]", "CVE-2002"]
|
|
133
163
|
def available_feeds
|
|
134
|
-
raise 'call
|
|
164
|
+
raise 'call populate method before using available_feeds method' if @feeds.nil?
|
|
135
165
|
|
|
136
166
|
@feeds.map(&:name)
|
|
137
167
|
end
|
|
@@ -155,7 +185,7 @@ class NVDFeedScraper
|
|
|
155
185
|
# @see https://csrc.nist.gov/schema/nvd/api/2.0/cve_api_json_2.0.schema
|
|
156
186
|
# @example
|
|
157
187
|
# s = NVDFeedScraper.new
|
|
158
|
-
# s.
|
|
188
|
+
# s.populate
|
|
159
189
|
# s.cve("CVE-2014-0002", "cve-2014-0001")
|
|
160
190
|
def cve(*arg_cve)
|
|
161
191
|
return_value = nil
|
|
@@ -185,7 +215,7 @@ class NVDFeedScraper
|
|
|
185
215
|
f.json_pull
|
|
186
216
|
return_value = f.cve(arg_cve[0])
|
|
187
217
|
when Array
|
|
188
|
-
raise 'one of the provided arguments is not a String' unless arg_cve[0].all?
|
|
218
|
+
raise 'one of the provided arguments is not a String' unless arg_cve[0].all?(String)
|
|
189
219
|
raise 'bad CVE name' unless arg_cve[0].all? { |x| /^CVE-[0-9]{4}-[0-9]{4,}$/i.match?(x) }
|
|
190
220
|
|
|
191
221
|
return_value = []
|
|
@@ -250,14 +280,14 @@ class NVDFeedScraper
|
|
|
250
280
|
# @return [Array<Boolean>] `true` if the feed was updated, `false` if it wasn't.
|
|
251
281
|
# @example
|
|
252
282
|
# s = NVDFeedScraper.new
|
|
253
|
-
# s.
|
|
283
|
+
# s.populate
|
|
254
284
|
# f2015, f2017 = s.feeds("CVE-2015", "CVE-2017")
|
|
255
285
|
# s.update_feeds(f2015, f2017) # => [false, false]
|
|
256
286
|
def update_feeds(*arg_feed)
|
|
257
287
|
return_value = false
|
|
258
288
|
raise 'no argument provided, 1 or more expected' if arg_feed.empty?
|
|
259
289
|
|
|
260
|
-
|
|
290
|
+
populate
|
|
261
291
|
if arg_feed.length == 1
|
|
262
292
|
case arg_feed[0]
|
|
263
293
|
when Feed
|
data/nvd_feed_api.gemspec
CHANGED
|
@@ -29,7 +29,7 @@ Gem::Specification.new do |s|
|
|
|
29
29
|
'rubygems_mfa_required' => 'true'
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
-
s.required_ruby_version = ['>= 3.
|
|
32
|
+
s.required_ruby_version = ['>= 3.3.0', '< 5.0']
|
|
33
33
|
|
|
34
34
|
s.add_dependency('archive-zip', '~> 0.11')
|
|
35
35
|
s.add_dependency('nokogiri', '~> 1.11')
|
data/pages/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,17 @@
|
|
|
1
|
+
# Change log / Release notes
|
|
2
|
+
|
|
3
|
+
## unreleased
|
|
4
|
+
|
|
5
|
+
# [0.8.0] - 01 November 2026
|
|
6
|
+
|
|
7
|
+
- **Breaking change**:
|
|
8
|
+
- Drop support for Ruby 3.2
|
|
9
|
+
- Ruby 3.2 is ([EOL](https://www.ruby-lang.org/en/downloads/branches/))
|
|
10
|
+
- parallel-2.1.0 requires ruby version >= 3.3
|
|
11
|
+
- Deprecate `NVDFeedScraper.scrap` and replacted it with `NVDFeedScraper.populate` that populate feeds deterministically without scraping the NVD website. It's faster, more reliabled, but we can't have the last update date as a trade off.
|
|
12
|
+
- `NVDFeedScraper.scrap` because NVD changed from plain HTML page to a JavaScript SPA that need JavaScript evaluation for the HTML DOM to be generated. So now a headless broswer is required, which is slow and very unreliable.
|
|
13
|
+
- `updated` attribute of `Feed` doesn't work anymore as a result as it was a data scraped from HTML table
|
|
14
|
+
|
|
1
15
|
# [0.7.0] - 01 January 2026
|
|
2
16
|
|
|
3
17
|
- Breaking changes:
|
data/pages/EXAMPLES.md
CHANGED
|
@@ -7,8 +7,6 @@ For all examples don't forget to add `require 'nvd_feed_api'`.
|
|
|
7
7
|
```ruby
|
|
8
8
|
# Initialize the scraper.
|
|
9
9
|
s = NVDFeedScraper.new
|
|
10
|
-
# Scrap the NVD website to get the feeds attributes.
|
|
11
|
-
s.scrap
|
|
12
10
|
# Change the default feed storage location beacause default value is '/tmp/'.
|
|
13
11
|
# '/tmp/' is mounted as tmpFS and is cleaned at every start.
|
|
14
12
|
# This will considerably speed up your performance is you have to reboot.
|
data/test/test_nvd_feed_api.rb
CHANGED
|
@@ -7,11 +7,11 @@ require 'date'
|
|
|
7
7
|
class NVDAPITest < Minitest::Test
|
|
8
8
|
def setup
|
|
9
9
|
@s = NVDFeedScraper.new
|
|
10
|
-
@s.
|
|
10
|
+
@s.populate # needed for feeds method
|
|
11
11
|
end
|
|
12
12
|
|
|
13
13
|
def test_scraper_scrap
|
|
14
|
-
assert_operator(0, :<, @s.
|
|
14
|
+
assert_operator(0, :<, @s.populate, 'populate method returns nothing')
|
|
15
15
|
end
|
|
16
16
|
|
|
17
17
|
def test_scraper_feeds_noarg
|
|
@@ -120,9 +120,8 @@ class NVDAPITest < Minitest::Test
|
|
|
120
120
|
assert_instance_of(String, f.name, "name doesn't return a string")
|
|
121
121
|
refute_empty(f.name, 'name is empty')
|
|
122
122
|
assert_equal(name, f.name, 'The name of the feed was modified')
|
|
123
|
-
# Test updated
|
|
124
|
-
|
|
125
|
-
refute_empty(f.updated, 'updated is empty')
|
|
123
|
+
# Test updated (before json_pull)
|
|
124
|
+
assert_nil(f.updated)
|
|
126
125
|
# Test gz_url
|
|
127
126
|
assert_instance_of(String, f.gz_url, "gz_url doesn't return a string")
|
|
128
127
|
refute_empty(f.gz_url, 'gz_url is empty')
|
|
@@ -237,7 +236,7 @@ class NVDAPITest < Minitest::Test
|
|
|
237
236
|
|
|
238
237
|
def test_feed_update!
|
|
239
238
|
f = @s.feeds('CVE-2006')
|
|
240
|
-
@s.
|
|
239
|
+
@s.populate
|
|
241
240
|
f_new = @s.feeds('CVE-2006')
|
|
242
241
|
# Right arg
|
|
243
242
|
# can't use assert_instance_of because there is no boolean class
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: nvd_feed_api
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.8.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Alexandre ZANNI
|
|
@@ -115,7 +115,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
115
115
|
requirements:
|
|
116
116
|
- - ">="
|
|
117
117
|
- !ruby/object:Gem::Version
|
|
118
|
-
version: 3.
|
|
118
|
+
version: 3.3.0
|
|
119
119
|
- - "<"
|
|
120
120
|
- !ruby/object:Gem::Version
|
|
121
121
|
version: '5.0'
|
|
@@ -125,7 +125,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
125
125
|
- !ruby/object:Gem::Version
|
|
126
126
|
version: '0'
|
|
127
127
|
requirements: []
|
|
128
|
-
rubygems_version: 4.0.
|
|
128
|
+
rubygems_version: 4.0.20
|
|
129
129
|
specification_version: 4
|
|
130
130
|
summary: API for NVD CVE feeds
|
|
131
131
|
test_files: []
|