nvd_feed_api 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitlab-ci.yml +8 -6
- data/.rubocop.yml +1 -1
- data/.tool-versions +1 -1
- data/Gemfile +7 -9
- data/Gemfile.lock +76 -61
- data/README.md +2 -3
- data/lib/nvd_feed_api/feed.rb +5 -4
- data/lib/nvd_feed_api/meta.rb +1 -1
- data/lib/nvd_feed_api/version.rb +1 -1
- data/lib/nvd_feed_api.rb +47 -18
- data/nvd_feed_api.gemspec +1 -1
- data/pages/CHANGELOG.md +22 -0
- data/pages/EXAMPLES.md +0 -2
- data/test/test_nvd_feed_api.rb +5 -6
- metadata +5 -5
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 0b2135d36e474c39e03f1124211c464e57d7251aea33c3de29610ab2ceb781e1
|
|
4
|
+
data.tar.gz: d5a520e831cdb7104ac57f0fe676a55bcac2c3eaee3d9f072070fe0405dae992
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: cc5673883c2ffcd84e21e88a6ca84d2f518c390bb8622da6408e91051bee01822a37d2e60a9060e6f8289750cef4d165ea9d85d24be238f83ea2ebf87209bace
|
|
7
|
+
data.tar.gz: b22d6e16a3ec4d6e98b64e91b1eca26ad7f167475edc9a5f9619b3d1782b6a0cfebc0bc9295d1d389f9e7875af68290c71c5f75b58bc0414946099ac4ae659c1
|
data/.gitlab-ci.yml
CHANGED
|
@@ -12,9 +12,11 @@ before_script:
|
|
|
12
12
|
- gem install bundler --no-document # Bundler is not installed with the image
|
|
13
13
|
# install nproc (coreutils) for bundle -j
|
|
14
14
|
# install git for building the gemspec
|
|
15
|
-
# install make, gcc for building gem native extension (commonmarker)
|
|
16
|
-
# libc-dev for musl-dev dependency (stdlib.h) needed by gcc
|
|
17
|
-
|
|
15
|
+
# install make, gcc (build-base) for building gem native extension (commonmarker)
|
|
16
|
+
# libc-dev for musl-dev dependency (stdlib.h) needed by gcc (build-base)
|
|
17
|
+
# yaml-dev to build psych ruby native extension
|
|
18
|
+
# add gcompat is GNU compatibility for MUSL libc
|
|
19
|
+
- apk --no-cache add coreutils git build-base yaml-dev
|
|
18
20
|
- bundle config set path 'vendor' # Set dependencies install dir to ./vendor/ruby
|
|
19
21
|
- bundle install -j $(nproc) # Install dependencies into ./vendor/ruby
|
|
20
22
|
- bundle exec rake install # install the gem
|
|
@@ -26,13 +28,13 @@ before_script:
|
|
|
26
28
|
- bundle exec rubocop
|
|
27
29
|
- bundle exec rake test
|
|
28
30
|
|
|
29
|
-
test:
|
|
31
|
+
test:4.0:
|
|
30
32
|
<<: *job_definition
|
|
31
|
-
image: ruby:
|
|
33
|
+
image: ruby:4.0-alpine
|
|
32
34
|
|
|
33
35
|
pages:
|
|
34
36
|
stage: deploy
|
|
35
|
-
image: ruby:
|
|
37
|
+
image: ruby:4.0-alpine
|
|
36
38
|
script:
|
|
37
39
|
- bundle exec yard doc
|
|
38
40
|
- mkdir public
|
data/.rubocop.yml
CHANGED
data/.tool-versions
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
ruby
|
|
1
|
+
ruby 4.0.9
|
data/Gemfile
CHANGED
|
@@ -4,18 +4,18 @@ source 'https://rubygems.org'
|
|
|
4
4
|
|
|
5
5
|
gemspec
|
|
6
6
|
|
|
7
|
-
group :runtime
|
|
8
|
-
gem 'archive-zip', '0.13.
|
|
7
|
+
group :runtime do
|
|
8
|
+
gem 'archive-zip', '0.13.1'
|
|
9
9
|
gem 'nokogiri', '~> 1.11'
|
|
10
10
|
gem 'oj', '>= 3.7.8', '<4'
|
|
11
11
|
end
|
|
12
12
|
|
|
13
13
|
group :development, :install do
|
|
14
|
-
gem 'bundler', '~>
|
|
14
|
+
gem 'bundler', '~> 4.0'
|
|
15
15
|
end
|
|
16
16
|
|
|
17
17
|
group :development, :test do
|
|
18
|
-
gem 'minitest', '~>
|
|
18
|
+
gem 'minitest', '~> 6.0'
|
|
19
19
|
gem 'rake', '~> 13.2'
|
|
20
20
|
end
|
|
21
21
|
|
|
@@ -25,9 +25,7 @@ group :development, :lint do
|
|
|
25
25
|
end
|
|
26
26
|
|
|
27
27
|
group :development, :docs do
|
|
28
|
-
gem 'commonmarker', '~> 2.
|
|
29
|
-
gem '
|
|
30
|
-
|
|
31
|
-
# https://github.com/lsegal/yard/issues/1528
|
|
32
|
-
gem 'yard', github: 'ParadoxV5/yard', ref: '9e869c940859570b07b81c5eadd6070e76f6291e', branch: 'commonmarker-1.0'
|
|
28
|
+
gem 'commonmarker', '~> 2.8'
|
|
29
|
+
gem 'irb' # to supress warning because using @overload from yard https://github.com/lsegal/yard/pull/1643#issuecomment-4322721189
|
|
30
|
+
gem 'yard', ['>= 0.9.43', '< 0.10']
|
|
33
31
|
end
|
data/Gemfile.lock
CHANGED
|
@@ -1,15 +1,7 @@
|
|
|
1
|
-
GIT
|
|
2
|
-
remote: https://github.com/ParadoxV5/yard.git
|
|
3
|
-
revision: 9e869c940859570b07b81c5eadd6070e76f6291e
|
|
4
|
-
ref: 9e869c940859570b07b81c5eadd6070e76f6291e
|
|
5
|
-
branch: commonmarker-1.0
|
|
6
|
-
specs:
|
|
7
|
-
yard (0.9.36)
|
|
8
|
-
|
|
9
1
|
PATH
|
|
10
2
|
remote: .
|
|
11
3
|
specs:
|
|
12
|
-
nvd_feed_api (0.
|
|
4
|
+
nvd_feed_api (0.8.0)
|
|
13
5
|
archive-zip (~> 0.11)
|
|
14
6
|
nokogiri (~> 1.11)
|
|
15
7
|
oj (>= 3.7.8, < 4)
|
|
@@ -17,78 +9,101 @@ PATH
|
|
|
17
9
|
GEM
|
|
18
10
|
remote: https://rubygems.org/
|
|
19
11
|
specs:
|
|
20
|
-
archive-zip (0.13.
|
|
21
|
-
io-like (~> 0.4.0
|
|
12
|
+
archive-zip (0.13.1)
|
|
13
|
+
io-like (~> 0.4.0)
|
|
22
14
|
ast (2.4.3)
|
|
23
|
-
bigdecimal (
|
|
24
|
-
commonmarker (2.
|
|
25
|
-
|
|
26
|
-
commonmarker (2.
|
|
27
|
-
commonmarker (2.
|
|
28
|
-
commonmarker (2.
|
|
29
|
-
commonmarker (2.
|
|
30
|
-
commonmarker (2.
|
|
31
|
-
commonmarker (2.
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
15
|
+
bigdecimal (4.1.3)
|
|
16
|
+
commonmarker (2.10.1-aarch64-linux)
|
|
17
|
+
commonmarker (2.10.1-aarch64-linux-musl)
|
|
18
|
+
commonmarker (2.10.1-arm-linux-gnu)
|
|
19
|
+
commonmarker (2.10.1-arm-linux-musl)
|
|
20
|
+
commonmarker (2.10.1-arm64-darwin)
|
|
21
|
+
commonmarker (2.10.1-x86_64-darwin)
|
|
22
|
+
commonmarker (2.10.1-x86_64-linux)
|
|
23
|
+
commonmarker (2.10.1-x86_64-linux-musl)
|
|
24
|
+
drb (2.2.3)
|
|
25
|
+
erb (6.0.7)
|
|
26
|
+
io-console (0.9.5)
|
|
27
|
+
io-like (0.4.0)
|
|
28
|
+
irb (1.18.0)
|
|
29
|
+
pp (>= 0.6.0)
|
|
30
|
+
prism (>= 1.3.0)
|
|
31
|
+
rdoc (>= 4.0.0)
|
|
32
|
+
reline (>= 0.4.2)
|
|
33
|
+
json (3.1.0)
|
|
34
|
+
language_server-protocol (3.17.0.6)
|
|
35
35
|
lint_roller (1.1.0)
|
|
36
36
|
logger (1.7.0)
|
|
37
|
-
minitest (
|
|
38
|
-
|
|
37
|
+
minitest (6.0.6)
|
|
38
|
+
drb (~> 2.0)
|
|
39
|
+
prism (~> 1.5)
|
|
40
|
+
nokogiri (1.19.4-aarch64-linux-gnu)
|
|
39
41
|
racc (~> 1.4)
|
|
40
|
-
nokogiri (1.
|
|
42
|
+
nokogiri (1.19.4-aarch64-linux-musl)
|
|
41
43
|
racc (~> 1.4)
|
|
42
|
-
nokogiri (1.
|
|
44
|
+
nokogiri (1.19.4-arm-linux-gnu)
|
|
43
45
|
racc (~> 1.4)
|
|
44
|
-
nokogiri (1.
|
|
46
|
+
nokogiri (1.19.4-arm-linux-musl)
|
|
45
47
|
racc (~> 1.4)
|
|
46
|
-
nokogiri (1.
|
|
48
|
+
nokogiri (1.19.4-arm64-darwin)
|
|
47
49
|
racc (~> 1.4)
|
|
48
|
-
nokogiri (1.
|
|
50
|
+
nokogiri (1.19.4-x86_64-darwin)
|
|
49
51
|
racc (~> 1.4)
|
|
50
|
-
nokogiri (1.
|
|
52
|
+
nokogiri (1.19.4-x86_64-linux-gnu)
|
|
51
53
|
racc (~> 1.4)
|
|
52
|
-
nokogiri (1.
|
|
54
|
+
nokogiri (1.19.4-x86_64-linux-musl)
|
|
53
55
|
racc (~> 1.4)
|
|
54
|
-
oj (3.
|
|
56
|
+
oj (3.17.7)
|
|
55
57
|
bigdecimal (>= 3.0)
|
|
56
58
|
ostruct (>= 0.2)
|
|
57
|
-
ostruct (0.6.
|
|
58
|
-
parallel (
|
|
59
|
-
parser (3.3.
|
|
59
|
+
ostruct (0.6.3)
|
|
60
|
+
parallel (2.3.0)
|
|
61
|
+
parser (3.3.13.0)
|
|
60
62
|
ast (~> 2.4.1)
|
|
61
63
|
racc
|
|
62
|
-
|
|
64
|
+
pp (0.6.4)
|
|
65
|
+
prettyprint
|
|
66
|
+
prettyprint (0.2.1)
|
|
67
|
+
prism (1.9.0)
|
|
63
68
|
racc (1.8.1)
|
|
64
69
|
rainbow (3.1.1)
|
|
65
|
-
rake (13.2
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
70
|
+
rake (13.4.2)
|
|
71
|
+
rbs (4.2.0)
|
|
72
|
+
logger
|
|
73
|
+
prism (>= 1.6.0)
|
|
74
|
+
tsort
|
|
75
|
+
rdoc (8.1.0)
|
|
76
|
+
erb
|
|
77
|
+
prism (>= 1.6.0)
|
|
78
|
+
rbs (>= 4.0.0)
|
|
79
|
+
tsort
|
|
80
|
+
regexp_parser (2.13.1)
|
|
81
|
+
reline (0.7.0)
|
|
82
|
+
io-console (~> 0.5)
|
|
83
|
+
rubocop (1.92.2)
|
|
84
|
+
json (>= 2.3)
|
|
72
85
|
language_server-protocol (~> 3.17.0.2)
|
|
73
86
|
lint_roller (~> 1.1.0)
|
|
74
|
-
parallel (
|
|
87
|
+
parallel (>= 1.10)
|
|
75
88
|
parser (>= 3.3.0.2)
|
|
76
89
|
rainbow (>= 2.2.2, < 4.0)
|
|
77
|
-
regexp_parser (>= 2.
|
|
78
|
-
rubocop-ast (>= 1.
|
|
90
|
+
regexp_parser (>= 2.11.3, < 3.0)
|
|
91
|
+
rubocop-ast (>= 1.49.0, < 2.0)
|
|
79
92
|
ruby-progressbar (~> 1.7)
|
|
80
93
|
unicode-display_width (>= 2.4.0, < 4.0)
|
|
81
|
-
rubocop-ast (1.
|
|
94
|
+
rubocop-ast (1.50.0)
|
|
82
95
|
parser (>= 3.3.7.2)
|
|
83
|
-
prism (~> 1.
|
|
84
|
-
rubocop-minitest (0.
|
|
96
|
+
prism (~> 1.7)
|
|
97
|
+
rubocop-minitest (0.41.0)
|
|
85
98
|
lint_roller (~> 1.1)
|
|
86
|
-
rubocop (>= 1.
|
|
99
|
+
rubocop (>= 1.89.0, < 2.0)
|
|
87
100
|
rubocop-ast (>= 1.38.0, < 2.0)
|
|
88
101
|
ruby-progressbar (1.13.0)
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
102
|
+
tsort (0.2.0)
|
|
103
|
+
unicode-display_width (3.3.0)
|
|
104
|
+
unicode-emoji (~> 4.3)
|
|
105
|
+
unicode-emoji (4.3.0)
|
|
106
|
+
yard (0.9.45)
|
|
92
107
|
|
|
93
108
|
PLATFORMS
|
|
94
109
|
aarch64-linux-gnu
|
|
@@ -101,18 +116,18 @@ PLATFORMS
|
|
|
101
116
|
x86_64-linux-musl
|
|
102
117
|
|
|
103
118
|
DEPENDENCIES
|
|
104
|
-
archive-zip (= 0.13.
|
|
105
|
-
bundler (~>
|
|
106
|
-
commonmarker (~> 2.
|
|
107
|
-
|
|
108
|
-
minitest (~>
|
|
119
|
+
archive-zip (= 0.13.1)
|
|
120
|
+
bundler (~> 4.0)
|
|
121
|
+
commonmarker (~> 2.8)
|
|
122
|
+
irb
|
|
123
|
+
minitest (~> 6.0)
|
|
109
124
|
nokogiri (~> 1.11)
|
|
110
125
|
nvd_feed_api!
|
|
111
126
|
oj (>= 3.7.8, < 4)
|
|
112
127
|
rake (~> 13.2)
|
|
113
128
|
rubocop (~> 1.71)
|
|
114
129
|
rubocop-minitest (~> 0.36)
|
|
115
|
-
yard
|
|
130
|
+
yard (>= 0.9.43, < 0.10)
|
|
116
131
|
|
|
117
132
|
BUNDLED WITH
|
|
118
|
-
|
|
133
|
+
4.0.23
|
data/README.md
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
|
-
[](https://gitlab.com/noraj/nvd_api/commits/master)
|
|
1
|
+
[](https://gitlab.com/noraj/nvd_api/-/commits/master)
|
|
3
2
|
|
|
4
3
|
[][rubygems]
|
|
5
4
|
[][rubygems]
|
|
6
5
|
[][rubygems]
|
|
7
6
|
[][rubygems]
|
|
8
|
-
[](https://inventory.raw.pm/tools.html#nvd_feed_api)
|
|
9
8
|
|
|
10
9
|
[rubygems]:https://rubygems.org/gems/nvd_feed_api/
|
|
11
10
|
|
data/lib/nvd_feed_api/feed.rb
CHANGED
|
@@ -28,6 +28,7 @@ class NVDFeedScraper
|
|
|
28
28
|
attr_reader :name
|
|
29
29
|
|
|
30
30
|
# @return [String] the last update date of the feed information on the NVD website.
|
|
31
|
+
# @deprecated Is broken since {NVDFeedScraper#scrap} was replaced with {NVDFeedScraper#populate}.
|
|
31
32
|
# @example
|
|
32
33
|
# '10/19/2017 3:27:02 AM -04:00'
|
|
33
34
|
attr_reader :updated
|
|
@@ -53,7 +54,7 @@ class NVDFeedScraper
|
|
|
53
54
|
# Note that {#json_pull} also calls {#meta_pull}.
|
|
54
55
|
# @example
|
|
55
56
|
# s = NVDFeedScraper.new
|
|
56
|
-
# s.
|
|
57
|
+
# s.populate
|
|
57
58
|
# f = s.feeds("CVE-2014")
|
|
58
59
|
# f.meta # => nil
|
|
59
60
|
# f.meta_pull
|
|
@@ -64,7 +65,7 @@ class NVDFeedScraper
|
|
|
64
65
|
# @note Return nil if not previously loaded by {#json_pull}.
|
|
65
66
|
# @example
|
|
66
67
|
# s = NVDFeedScraper.new
|
|
67
|
-
# s.
|
|
68
|
+
# s.populate
|
|
68
69
|
# f = s.feeds("CVE-2014")
|
|
69
70
|
# f.json_file # => nil
|
|
70
71
|
# f.json_pull
|
|
@@ -218,7 +219,7 @@ class NVDFeedScraper
|
|
|
218
219
|
# @see https://csrc.nist.gov/schema/nvd/api/2.0/cve_api_json_2.0.schema
|
|
219
220
|
# @example
|
|
220
221
|
# s = NVDFeedScraper.new
|
|
221
|
-
# s.
|
|
222
|
+
# s.populate
|
|
222
223
|
# f = s.feeds("CVE-2014")
|
|
223
224
|
# f.json_pull
|
|
224
225
|
# f.cve("CVE-2014-0002", "cve-2014-0001")
|
|
@@ -248,7 +249,7 @@ class NVDFeedScraper
|
|
|
248
249
|
# Sorting CVE can allow us to parse quicker
|
|
249
250
|
# Upcase to be sure include? works
|
|
250
251
|
cves_to_find = arg_cve[0].map(&:upcase).sort
|
|
251
|
-
raise 'one of the provided arguments is not a String' unless cves_to_find.all?
|
|
252
|
+
raise 'one of the provided arguments is not a String' unless cves_to_find.all?(String)
|
|
252
253
|
raise 'bad CVE name' unless cves_to_find.all? { |x| /^CVE-[0-9]{4}-[0-9]{4,}$/i.match?(x) }
|
|
253
254
|
|
|
254
255
|
doc = Oj::Doc.open(File.read(@json_file))
|
data/lib/nvd_feed_api/meta.rb
CHANGED
data/lib/nvd_feed_api/version.rb
CHANGED
data/lib/nvd_feed_api.rb
CHANGED
|
@@ -3,8 +3,7 @@
|
|
|
3
3
|
# @author Alexandre ZANNI <alexandre.zanni@engineer.com>
|
|
4
4
|
|
|
5
5
|
# Ruby internal
|
|
6
|
-
require 'net/https'
|
|
7
|
-
require 'set'
|
|
6
|
+
# require 'net/https'
|
|
8
7
|
# External
|
|
9
8
|
require 'nokogiri'
|
|
10
9
|
# Project internal
|
|
@@ -14,7 +13,7 @@ require 'nvd_feed_api/feed'
|
|
|
14
13
|
# The class that parse NVD website to get information.
|
|
15
14
|
# @example Initialize a NVDFeedScraper object, get the feeds and see them:
|
|
16
15
|
# scraper = NVDFeedScraper.new
|
|
17
|
-
# scraper.
|
|
16
|
+
# # scraper.populate
|
|
18
17
|
# scraper.available_feeds
|
|
19
18
|
# scraper.feeds
|
|
20
19
|
# scraper.feeds("CVE-2007")
|
|
@@ -29,26 +28,56 @@ class NVDFeedScraper
|
|
|
29
28
|
# Initialize the scraper
|
|
30
29
|
def initialize
|
|
31
30
|
@url = URL
|
|
32
|
-
@feeds =
|
|
31
|
+
@feeds = populate
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Populate feeds deterministically without scraping the NVD website.
|
|
35
|
+
# Replaces {scrap}.
|
|
36
|
+
# @return [Integer] Number of populated feeds.
|
|
37
|
+
def populate
|
|
38
|
+
@feeds = []
|
|
39
|
+
current_year = Time.now.year
|
|
40
|
+
names = ['CVE-Modified', 'CVE-Recent'] + (2002..current_year).to_a.reverse.map { |year| "CVE-#{year}" }
|
|
41
|
+
names.each do |name|
|
|
42
|
+
file_part = case name
|
|
43
|
+
when 'CVE-Modified' then 'modified'
|
|
44
|
+
when 'CVE-Recent' then 'recent'
|
|
45
|
+
else name.split('-').last
|
|
46
|
+
end
|
|
47
|
+
base_path = "#{BASE}/feeds/json/cve/2.0/nvdcve-2.0-#{file_part}"
|
|
48
|
+
@feeds.push(Feed.new(name,
|
|
49
|
+
nil,
|
|
50
|
+
"#{base_path}.meta",
|
|
51
|
+
"#{base_path}.json.gz",
|
|
52
|
+
"#{base_path}.json.zip"))
|
|
53
|
+
end
|
|
54
|
+
return @feeds.size
|
|
33
55
|
end
|
|
34
56
|
|
|
35
57
|
# Scrap / parse the website to get the feeds and fill the {#feeds} attribute.
|
|
36
58
|
# @note {#scrap} need to be called only once but can be called again to update if the NVD feed page changed.
|
|
59
|
+
# @deprecated Since upstream use a JS SPA, an headless browser is required which is slow and unreliable. Replaced by {populate}.
|
|
37
60
|
# @return [Integer] Number of scrapped feeds.
|
|
38
61
|
def scrap
|
|
39
|
-
|
|
40
|
-
|
|
62
|
+
require 'lightpanda'
|
|
63
|
+
begin
|
|
64
|
+
browser = Lightpanda::Browser.new(timeout: 60)
|
|
65
|
+
browser.go_to(@url, wait: true)
|
|
66
|
+
html = browser.body
|
|
67
|
+
ensure
|
|
68
|
+
browser.quit
|
|
69
|
+
end
|
|
41
70
|
|
|
42
71
|
doc = Nokogiri::HTML(html)
|
|
43
72
|
@feeds = []
|
|
44
73
|
tmp_feeds = {}
|
|
45
74
|
doc.css('div[data-testid=nvd-json-2-feed-table] table.xml-feed-table tr[data-testid]').each do |tr|
|
|
46
|
-
num, type = tr.attr('data-testid')
|
|
75
|
+
num, type = tr.attr('data-testid').split('-')[1..2]
|
|
47
76
|
case type
|
|
48
77
|
when 'meta'
|
|
49
78
|
tmp_feeds[num] = {}
|
|
50
|
-
tmp_feeds[num][:name] = tr.css('td')[0].text
|
|
51
|
-
tmp_feeds[num][:updated] = tr.css('td')[1].text
|
|
79
|
+
tmp_feeds[num][:name] = tr.css('td')[0].text.strip
|
|
80
|
+
tmp_feeds[num][:updated] = tr.css('td')[1].text.strip
|
|
52
81
|
tmp_feeds[num][:meta] = BASE + tr.css('td')[2].css('> a').attr('href').value
|
|
53
82
|
when 'gz'
|
|
54
83
|
tmp_feeds[num][:gz] = BASE + tr.css('td > a').attr('href').value
|
|
@@ -64,7 +93,7 @@ class NVDFeedScraper
|
|
|
64
93
|
return @feeds.size
|
|
65
94
|
end
|
|
66
95
|
|
|
67
|
-
# Return feeds. Can only be called after {#
|
|
96
|
+
# Return feeds. Can only be called after {#populate}.
|
|
68
97
|
# @overload feeds
|
|
69
98
|
# All the feeds.
|
|
70
99
|
# @return [Array<Feed>] Attributes of all feeds. It's an array of {Feed} object.
|
|
@@ -87,7 +116,7 @@ class NVDFeedScraper
|
|
|
87
116
|
# scraper.feeds("CVE-2005", "CVE-2002") # => return CVE-2005 and CVE-2002 feeds
|
|
88
117
|
# @see https://nvd.nist.gov/vuln/data-feeds
|
|
89
118
|
def feeds(*arg_feeds)
|
|
90
|
-
raise 'call
|
|
119
|
+
raise 'call populate method before using feeds method' if @feeds.nil?
|
|
91
120
|
|
|
92
121
|
return_value = nil
|
|
93
122
|
if arg_feeds.empty?
|
|
@@ -100,7 +129,7 @@ class NVDFeedScraper
|
|
|
100
129
|
end
|
|
101
130
|
# if nothing found return nil
|
|
102
131
|
when Array
|
|
103
|
-
raise 'one of the provided arguments is not a String' unless arg_feeds[0].all?
|
|
132
|
+
raise 'one of the provided arguments is not a String' unless arg_feeds[0].all?(String)
|
|
104
133
|
|
|
105
134
|
# Sorting CVE can allow us to parse quicker
|
|
106
135
|
# Upcase to be sure include? works
|
|
@@ -127,12 +156,12 @@ class NVDFeedScraper
|
|
|
127
156
|
return return_value
|
|
128
157
|
end
|
|
129
158
|
|
|
130
|
-
# Return a list with the name of all available feeds. Returned feed names can be use as argument for {#feeds} method. Can only be called after {#
|
|
159
|
+
# Return a list with the name of all available feeds. Returned feed names can be use as argument for {#feeds} method. Can only be called after {#populate}.
|
|
131
160
|
# @return [Array<String>] List with the name of all available feeds.
|
|
132
161
|
# @example
|
|
133
162
|
# scraper.available_feeds => ["CVE-Modified", "CVE-Recent", "CVE-2025", "[…]", "CVE-2002"]
|
|
134
163
|
def available_feeds
|
|
135
|
-
raise 'call
|
|
164
|
+
raise 'call populate method before using available_feeds method' if @feeds.nil?
|
|
136
165
|
|
|
137
166
|
@feeds.map(&:name)
|
|
138
167
|
end
|
|
@@ -156,7 +185,7 @@ class NVDFeedScraper
|
|
|
156
185
|
# @see https://csrc.nist.gov/schema/nvd/api/2.0/cve_api_json_2.0.schema
|
|
157
186
|
# @example
|
|
158
187
|
# s = NVDFeedScraper.new
|
|
159
|
-
# s.
|
|
188
|
+
# s.populate
|
|
160
189
|
# s.cve("CVE-2014-0002", "cve-2014-0001")
|
|
161
190
|
def cve(*arg_cve)
|
|
162
191
|
return_value = nil
|
|
@@ -186,7 +215,7 @@ class NVDFeedScraper
|
|
|
186
215
|
f.json_pull
|
|
187
216
|
return_value = f.cve(arg_cve[0])
|
|
188
217
|
when Array
|
|
189
|
-
raise 'one of the provided arguments is not a String' unless arg_cve[0].all?
|
|
218
|
+
raise 'one of the provided arguments is not a String' unless arg_cve[0].all?(String)
|
|
190
219
|
raise 'bad CVE name' unless arg_cve[0].all? { |x| /^CVE-[0-9]{4}-[0-9]{4,}$/i.match?(x) }
|
|
191
220
|
|
|
192
221
|
return_value = []
|
|
@@ -251,14 +280,14 @@ class NVDFeedScraper
|
|
|
251
280
|
# @return [Array<Boolean>] `true` if the feed was updated, `false` if it wasn't.
|
|
252
281
|
# @example
|
|
253
282
|
# s = NVDFeedScraper.new
|
|
254
|
-
# s.
|
|
283
|
+
# s.populate
|
|
255
284
|
# f2015, f2017 = s.feeds("CVE-2015", "CVE-2017")
|
|
256
285
|
# s.update_feeds(f2015, f2017) # => [false, false]
|
|
257
286
|
def update_feeds(*arg_feed)
|
|
258
287
|
return_value = false
|
|
259
288
|
raise 'no argument provided, 1 or more expected' if arg_feed.empty?
|
|
260
289
|
|
|
261
|
-
|
|
290
|
+
populate
|
|
262
291
|
if arg_feed.length == 1
|
|
263
292
|
case arg_feed[0]
|
|
264
293
|
when Feed
|
data/nvd_feed_api.gemspec
CHANGED
|
@@ -29,7 +29,7 @@ Gem::Specification.new do |s|
|
|
|
29
29
|
'rubygems_mfa_required' => 'true'
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
-
s.required_ruby_version = ['>= 3.
|
|
32
|
+
s.required_ruby_version = ['>= 3.3.0', '< 5.0']
|
|
33
33
|
|
|
34
34
|
s.add_dependency('archive-zip', '~> 0.11')
|
|
35
35
|
s.add_dependency('nokogiri', '~> 1.11')
|
data/pages/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,25 @@
|
|
|
1
|
+
# Change log / Release notes
|
|
2
|
+
|
|
3
|
+
## unreleased
|
|
4
|
+
|
|
5
|
+
# [0.8.0] - 01 November 2026
|
|
6
|
+
|
|
7
|
+
- **Breaking change**:
|
|
8
|
+
- Drop support for Ruby 3.2
|
|
9
|
+
- Ruby 3.2 is ([EOL](https://www.ruby-lang.org/en/downloads/branches/))
|
|
10
|
+
- parallel-2.1.0 requires ruby version >= 3.3
|
|
11
|
+
- Deprecate `NVDFeedScraper.scrap` and replacted it with `NVDFeedScraper.populate` that populate feeds deterministically without scraping the NVD website. It's faster, more reliabled, but we can't have the last update date as a trade off.
|
|
12
|
+
- `NVDFeedScraper.scrap` because NVD changed from plain HTML page to a JavaScript SPA that need JavaScript evaluation for the HTML DOM to be generated. So now a headless broswer is required, which is slow and very unreliable.
|
|
13
|
+
- `updated` attribute of `Feed` doesn't work anymore as a result as it was a data scraped from HTML table
|
|
14
|
+
|
|
15
|
+
# [0.7.0] - 01 January 2026
|
|
16
|
+
|
|
17
|
+
- Breaking changes:
|
|
18
|
+
- Drop support for Ruby 3.1
|
|
19
|
+
- Chore:
|
|
20
|
+
- Add support for Ruby 4.0
|
|
21
|
+
- Update dependencies
|
|
22
|
+
|
|
1
23
|
# [0.6.0] - 29 May 2025
|
|
2
24
|
|
|
3
25
|
- Enhancements and fixes:
|
data/pages/EXAMPLES.md
CHANGED
|
@@ -7,8 +7,6 @@ For all examples don't forget to add `require 'nvd_feed_api'`.
|
|
|
7
7
|
```ruby
|
|
8
8
|
# Initialize the scraper.
|
|
9
9
|
s = NVDFeedScraper.new
|
|
10
|
-
# Scrap the NVD website to get the feeds attributes.
|
|
11
|
-
s.scrap
|
|
12
10
|
# Change the default feed storage location beacause default value is '/tmp/'.
|
|
13
11
|
# '/tmp/' is mounted as tmpFS and is cleaned at every start.
|
|
14
12
|
# This will considerably speed up your performance is you have to reboot.
|
data/test/test_nvd_feed_api.rb
CHANGED
|
@@ -7,11 +7,11 @@ require 'date'
|
|
|
7
7
|
class NVDAPITest < Minitest::Test
|
|
8
8
|
def setup
|
|
9
9
|
@s = NVDFeedScraper.new
|
|
10
|
-
@s.
|
|
10
|
+
@s.populate # needed for feeds method
|
|
11
11
|
end
|
|
12
12
|
|
|
13
13
|
def test_scraper_scrap
|
|
14
|
-
assert_operator(0, :<, @s.
|
|
14
|
+
assert_operator(0, :<, @s.populate, 'populate method returns nothing')
|
|
15
15
|
end
|
|
16
16
|
|
|
17
17
|
def test_scraper_feeds_noarg
|
|
@@ -120,9 +120,8 @@ class NVDAPITest < Minitest::Test
|
|
|
120
120
|
assert_instance_of(String, f.name, "name doesn't return a string")
|
|
121
121
|
refute_empty(f.name, 'name is empty')
|
|
122
122
|
assert_equal(name, f.name, 'The name of the feed was modified')
|
|
123
|
-
# Test updated
|
|
124
|
-
|
|
125
|
-
refute_empty(f.updated, 'updated is empty')
|
|
123
|
+
# Test updated (before json_pull)
|
|
124
|
+
assert_nil(f.updated)
|
|
126
125
|
# Test gz_url
|
|
127
126
|
assert_instance_of(String, f.gz_url, "gz_url doesn't return a string")
|
|
128
127
|
refute_empty(f.gz_url, 'gz_url is empty')
|
|
@@ -237,7 +236,7 @@ class NVDAPITest < Minitest::Test
|
|
|
237
236
|
|
|
238
237
|
def test_feed_update!
|
|
239
238
|
f = @s.feeds('CVE-2006')
|
|
240
|
-
@s.
|
|
239
|
+
@s.populate
|
|
241
240
|
f_new = @s.feeds('CVE-2006')
|
|
242
241
|
# Right arg
|
|
243
242
|
# can't use assert_instance_of because there is no boolean class
|
metadata
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: nvd_feed_api
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.8.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Alexandre ZANNI
|
|
8
8
|
bindir: bin
|
|
9
9
|
cert_chain: []
|
|
10
|
-
date:
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
11
|
dependencies:
|
|
12
12
|
- !ruby/object:Gem::Dependency
|
|
13
13
|
name: archive-zip
|
|
@@ -115,17 +115,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
115
115
|
requirements:
|
|
116
116
|
- - ">="
|
|
117
117
|
- !ruby/object:Gem::Version
|
|
118
|
-
version: 3.
|
|
118
|
+
version: 3.3.0
|
|
119
119
|
- - "<"
|
|
120
120
|
- !ruby/object:Gem::Version
|
|
121
|
-
version: '
|
|
121
|
+
version: '5.0'
|
|
122
122
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
123
123
|
requirements:
|
|
124
124
|
- - ">="
|
|
125
125
|
- !ruby/object:Gem::Version
|
|
126
126
|
version: '0'
|
|
127
127
|
requirements: []
|
|
128
|
-
rubygems_version:
|
|
128
|
+
rubygems_version: 4.0.20
|
|
129
129
|
specification_version: 4
|
|
130
130
|
summary: API for NVD CVE feeds
|
|
131
131
|
test_files: []
|