nvd_feed_api 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 421862fc8896856c94ac947b15d78b2538a9c13c
-  data.tar.gz: 23daab198d07a5da168a8884d2476a73feb21ee7
+  metadata.gz: 38ab69d805d125729995fc9ea26d79e0b324414f
+  data.tar.gz: 7c9838e134a5f503d21979a5915db3cd9d9e823b
 SHA512:
-  metadata.gz: ddfcce9f3483fee3d46e70a7a9fcc839f22c15a1bbc9c8090df47bfda8de118d0f71ea21d639df8b54ea6efb0c19dfdae383e7b7c71bea026a009a1122fff888
-  data.tar.gz: b8f1755ce9de8c03da15699f4c17f57433d60a02b9b421a38ce420ba4188d59f79ec3cc4069600ee7ed99e63f2981994f511a74ffeebd13fcebc3b2118ba42ce
+  metadata.gz: eda8d75faf07c0189cf6fdf9bbea8296fce70f0aaf380e8af2a3af83d1216f6ae360319cd01e727659f4cac527cd284ac9e9555dfb1dc79f1982f93309940129
+  data.tar.gz: eb9eb999eee1ef44b3b64de314f3fb37b0cc857df67a7ae580ed6f71ef51e827f3ec93de5b5fdfdd0205a0f280f6825e673a858e67248c9a684dc77a1342adea

data/lib/nvd_feed_api.rb

@@ -1,15 +1,13 @@
 # @author Alexandre ZANNI <alexandre.zanni@engineer.com>
 
 # Ruby internal
-require 'digest'
 require 'net/https'
 require 'set'
 # External
-require 'archive/zip'
 require 'nokogiri'
-require 'oj'
 # Project internal
 require 'nvd_feed_api/version'
+require 'nvd_feed_api/feed'
 
 # The class that parse NVD website to get information.
 # @example Initialize a NVDFeedScraper object, get the feeds and see them:
@@ -25,327 +23,6 @@ class NVDFeedScraper
   # Load constants
   include NvdFeedApi
 
-  # Feed object.
-  class Feed
-    class << self
-      # Get / set default feed storage location, where will be stored JSON feeds and archives by default.
-      # @return [String] default feed storage location. Default to +/tmp/+.
-      # @example
-      #   NVDFeedScraper::Feed.default_storage_location = '/srv/downloads/'
-      attr_accessor :default_storage_location
-    end
-    @default_storage_location = '/tmp/'
-
-    # @return [String] the name of the feed.
-    # @example
-    #   'CVE-2007'
-    attr_reader :name
-
-    # @return [String] the last update date of the feed information on the NVD website.
-    # @example
-    #   '10/19/2017 3:27:02 AM -04:00'
-    attr_reader :updated
-
-    # @return [String] the URL of the metadata file of the feed.
-    # @example
-    #   'https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-2007.meta'
-    attr_reader :meta_url
-
-    # @return [String] the URL of the gz archive of the feed.
-    # @example
-    #   'https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-2007.json.gz'
-    attr_reader :gz_url
-
-    # @return [String] the URL of the zip archive of the feed.
-    # @example
-    #   'https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-2007.json.zip'
-    attr_reader :zip_url
-
-    # @return [Meta] the {Meta} object of the feed.
-    # @note
-    #   Return nil if not previously loaded by {#meta_pull}.
-    #   Note that {#json_pull} also calls {#meta_pull}.
-    # @example
-    #   s = NVDFeedScraper.new
-    #   s.scrap
-    #   f = s.feeds("CVE-2014")
-    #   f.meta # => nil
-    #   f.meta_pull
-    #   f.meta # => #<NVDFeedScraper::Meta:0x00555b53027570 ... >
-    attr_reader :meta
-
-    # @return [String] the path of the saved JSON file.
-    # @note Return nil if not previously loaded by {#json_pull}.
-    # @example
-    #   s = NVDFeedScraper.new
-    #   s.scrap
-    #   f = s.feeds("CVE-2014")
-    #   f.json_file # => nil
-    #   f.json_pull
-    #   f.json_file # => "/tmp/nvdcve-1.0-2014.json"
-    attr_reader :json_file
-
-    # A new instance of Feed.
-    # @param name [String] see {#name}.
-    # @param updated [String] see {#updated}.
-    # @param meta_url [String] see {#meta_url}.
-    # @param gz_url [String] see {#gz_url}.
-    # @param zip_url [String] see {#zip_url}.
-    def initialize(name, updated, meta_url, gz_url, zip_url)
-      @name = name
-      @updated = updated
-      @meta_url = meta_url
-      @gz_url = gz_url
-      @zip_url = zip_url
-      # do not pull meta and json automatically for speed and memory footprint
-      @meta = nil
-      @json_file = nil
-    end
-
-    # Create or update the {Meta} object (fill the attribute).
-    # @return [Meta] the updated {Meta} object of the feed.
-    # @see #meta
-    def meta_pull
-      meta_content = NVDFeedScraper::Meta.new(@meta_url)
-      meta_content.parse
-      # update @meta
-      @meta = meta_content
-    end
-
-    # Download the gz archive of the feed.
-    # @param opts [Hash] see {#download_file}.
-    # @return [String] the saved gz file path.
-    # @example
-    #   afeed.download_gz
-    #   afeed.download_gz(destination_path: '/srv/save/')
-    def download_gz(opts = {})
-      download_file(@gz_url, opts)
-    end
-
-    # Download the zip archive of the feed.
-    # @param opts [Hash] see {#download_file}.
-    # @return [String] the saved zip file path.
-    # @example
-    #   afeed.download_zip
-    #   afeed.download_zip(destination_path: '/srv/save/')
-    def download_zip(opts = {})
-      download_file(@zip_url, opts)
-    end
-
-    # Download the JSON feed and fill the attribute.
-    # @param opts [Hash] see {#download_file}.
-    # @return [String] the path of the saved JSON file. Default use {Feed#default_storage_location}.
-    # @note Will downlaod and save the zip of the JSON file, unzip and save it. This massively consume time.
-    # @see #json_file
-    def json_pull(opts = {})
-      opts[:destination_path] ||= Feed.default_storage_location
-
-      skip_download = false
-      destination_path = opts[:destination_path]
-      destination_path += '/' unless destination_path[-1] == '/'
-      filename = URI(@zip_url).path.split('/').last.chomp('.zip')
-      # do not use @json_file for destination_file because of offline loading
-      destination_file = destination_path + filename
-      meta_pull
-      if File.file?(destination_file)
-        # Verify hash to see if it is the latest
-        computed_h = Digest::SHA256.file(destination_file)
-        skip_download = true if meta.sha256.casecmp(computed_h.hexdigest).zero?
-      end
-      if skip_download
-        @json_file = destination_file
-      else
-        zip_path = download_zip(opts)
-        Archive::Zip.open(zip_path) do |z|
-          z.extract(destination_path, flatten: true)
-        end
-        @json_file = zip_path.chomp('.zip')
-        # Verify hash integrity
-        computed_h = Digest::SHA256.file(@json_file)
-        raise "File corruption: #{@json_file}" unless meta.sha256.casecmp(computed_h.hexdigest).zero?
-      end
-      return @json_file
-    end
-
-    # Search for CVE in the feed.
-    # @overload cve(cve)
-    #   One CVE.
-    #   @param cve [String] CVE ID, case insensitive.
-    #   @return [Hash] a Ruby Hash corresponding to the CVE.
-    # @overload cve(cve_arr)
-    #   An array of CVEs.
-    #   @param cve_arr [Array<String>] Array of CVE ID, case insensitive.
-    #   @return [Array] an Array of CVE, each CVE is a Ruby Hash. May not be in the same order as provided.
-    # @overload cve(cve, *)
-    #   Multiple CVEs.
-    #   @param cve [String] CVE ID, case insensitive.
-    #   @param * [String] As many CVE ID as you want.
-    #   @return [Array] an Array of CVE, each CVE is a Ruby Hash. May not be in the same order as provided.
-    # @note {#json_pull} is needed before using this method. Remember you're searching only in the current feed.
-    # @todo implement a CVE Class instead of returning a Hash.
-    # @see https://scap.nist.gov/schema/nvd/feed/0.1/nvd_cve_feed_json_0.1_beta.schema
-    # @see https://scap.nist.gov/schema/nvd/feed/0.1/CVE_JSON_4.0_min.schema
-    # @example
-    #   s = NVDFeedScraper.new
-    #   s.scrap
-    #   f = s.feeds("CVE-2014")
-    #   f.json_pull
-    #   f.cve("CVE-2014-0002", "cve-2014-0001")
-    def cve(*arg_cve)
-      raise 'json_file is nil, it needs to be populated with json_pull' if @json_file.nil?
-      raise "json_file (#{@json_file}) doesn't exist" unless File.file?(@json_file)
-      return_value = nil
-      raise 'no argument provided, 1 or more expected' if arg_cve.empty?
-      if arg_cve.length == 1
-        if arg_cve[0].is_a?(String)
-          raise "bad CVE name (#{arg_cve[0]})" unless /^CVE-[0-9]{4}-[0-9]{4,}$/i.match?(arg_cve[0])
-          doc = Oj::Doc.open(File.read(@json_file))
-          # Quicker than doc.fetch('/CVE_Items').size
-          doc_size = doc.fetch('/CVE_data_numberOfCVEs').to_i
-          (1..doc_size).each do |i|
-            if arg_cve[0].upcase == doc.fetch("/CVE_Items/#{i}/cve/CVE_data_meta/ID")
-              return_value = doc.fetch("/CVE_Items/#{i}")
-              break
-            end
-          end
-          doc.close
-        elsif arg_cve[0].is_a?(Array)
-          return_value = []
-          # Sorting CVE can allow us to parse quicker
-          # Upcase to be sure include? works
-          cves_to_find = arg_cve[0].map(&:upcase).sort
-          raise 'one of the provided arguments is not a String' unless cves_to_find.all? { |x| x.is_a?(String) }
-          raise 'bad CVE name' unless cves_to_find.all? { |x| /^CVE-[0-9]{4}-[0-9]{4,}$/i.match?(x) }
-          doc = Oj::Doc.open(File.read(@json_file))
-          # Quicker than doc.fetch('/CVE_Items').size
-          doc_size = doc.fetch('/CVE_data_numberOfCVEs').to_i
-          (1..doc_size).each do |i|
-            doc.move("/CVE_Items/#{i}")
-            cve_id = doc.fetch('cve/CVE_data_meta/ID')
-            if cves_to_find.include?(cve_id)
-              return_value.push(doc.fetch)
-              cves_to_find.delete(cve_id)
-            elsif cves_to_find.empty?
-              break
-            end
-          end
-          raise "#{cves_to_find.join(', ')} are unexisting CVEs in this feed" unless cves_to_find.empty?
-        else
-          raise "the provided argument (#{arg_cve[0]}) is nor a String or an Array"
-        end
-      else
-        # Overloading a list of arguments as one array argument
-        return_value = cve(arg_cve)
-      end
-      return return_value
-    end
-
-    # Return a list with the name of all available CVEs in the feed.
-    # Can only be called after {#json_pull}.
-    # @return [Array<String>] List with the name of all available CVEs. May return thousands CVEs.
-    def available_cves
-      raise 'json_file is nil, it needs to be populated with json_pull' if @json_file.nil?
-      raise "json_file (#{@json_file}) doesn't exist" unless File.file?(@json_file)
-      doc = Oj::Doc.open(File.read(@json_file))
-      # Quicker than doc.fetch('/CVE_Items').size
-      doc_size = doc.fetch('/CVE_data_numberOfCVEs').to_i
-      cve_names = []
-      (1..doc_size).each do |i|
-        doc.move("/CVE_Items/#{i}")
-        cve_names.push(doc.fetch('cve/CVE_data_meta/ID'))
-      end
-      doc.close
-      return cve_names
-    end
-
-    protected
-
-    # @param arg_name [String] the new name of the feed.
-    # @return [String] the new name of the feed.
-    # @example
-    #   'CVE-2007'
-    def name=(arg_name)
-      raise "name (#{arg_name}) is not a string" unless arg_name.is_a(String)
-      @name = arg_name
-    end
-
-    # @param arg_updated [String] the last update date of the feed information on the NVD website.
-    # @return [String] the new date.
-    # @example
-    #   '10/19/2017 3:27:02 AM -04:00'
-    def updated=(arg_updated)
-      raise "updated date (#{arg_updated}) is not a string" unless arg_updated.is_a(String)
-      @updated = arg_updated
-    end
-
-    # @param arg_meta_url [String] the new URL of the metadata file of the feed.
-    # @return [String] the new URL of the metadata file of the feed.
-    # @example
-    #   'https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-2007.meta'
-    def meta_url=(arg_meta_url)
-      raise "meta_url (#{arg_meta_url}) is not a string" unless arg_meta_url.is_a(String)
-      @meta_url = arg_meta_url
-    end
-
-    # @param arg_gz_url [String] the new URL of the gz archive of the feed.
-    # @return [String] the new URL of the gz archive of the feed.
-    # @example
-    #   'https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-2007.json.gz'
-    def gz_url=(arg_gz_url)
-      raise "gz_url (#{arg_gz_url}) is not a string" unless arg_gz_url.is_a(String)
-      @gz_url = arg_gz_url
-    end
-
-    # @param arg_zip_url [String] the new URL of the zip archive of the feed.
-    # @return [String] the new URL of the zip archive of the feed.
-    # @example
-    #   'https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-2007.json.zip'
-    def zip_url=(arg_zip_url)
-      raise "zip_url (#{arg_zip_url}) is not a string" unless arg_zip_url.is_a(String)
-      @zip_url = arg_zip_url
-    end
-
-    # Download a file.
-    # @param file_url [String] the URL of the file.
-    # @param opts [Hash] the optional downlaod parameters.
-    # @option opts [String] :destination_path the destination path (may
-    #   overwrite existing file).
-    #   Default use {Feed#default_storage_location}.
-    # @option opts [String] :sha256 the SHA256 hash to check, if the file
-    #   already exist and the hash matches then the download will be skipped.
-    # @return [String] the saved file path.
-    # @example
-    #   download_file('https://example.org/example.zip') # => '/tmp/example.zip'
-    #   download_file('https://example.org/example.zip', destination_path: '/srv/save/') # => '/srv/save/example.zip'
-    #   download_file('https://example.org/example.zip', {destination_path: '/srv/save/', sha256: '70d6ea136d5036b6ce771921a949357216866c6442f44cea8497f0528c54642d'}) # => '/srv/save/example.zip'
-    def download_file(file_url, opts = {})
-      opts[:destination_path] ||= Feed.default_storage_location
-      opts[:sha256] ||= nil
-
-      destination_path = opts[:destination_path]
-      destination_path += '/' unless destination_path[-1] == '/'
-      skip_download = false
-      uri = URI(file_url)
-      filename = uri.path.split('/').last
-      destination_file = destination_path + filename
-      unless opts[:sha256].nil?
-        if File.file?(destination_file)
-          # Verify hash to see if it is the latest
-          computed_h = Digest::SHA256.file(destination_file)
-          skip_download = true if opts[:sha256].casecmp(computed_h.hexdigest).zero?
-        end
-      end
-      unless skip_download
-        res = Net::HTTP.get_response(uri)
-        raise "#{file_url} ended with #{res.code} #{res.message}" unless res.is_a?(Net::HTTPSuccess)
-        open(destination_file, 'wb') do |file|
-          file.write(res.body)
-        end
-      end
-      return destination_file
-    end
-  end
-
   # Initialize the scraper
   def initialize
     @url = URL
@@ -563,18 +240,7 @@ class NVDFeedScraper
       if arg_feed[0].is_a?(Feed)
         new_feed = feeds(arg_feed[0].name)
         # update attributes
-        if arg_feed[0].updated != new_feed.updated
-          arg_feed[0].name = new_feed.name
-          arg_feed[0].updated = new_feed.updated
-          arg_feed[0].meta_url = new_feed.meta_url
-          arg_feed[0].gz_url = new_feed.gz_url
-          arg_feed[0].zip_url = new_feed.zip_url
-          # update if @meta was set
-          arg_feed[0].meta_pull unless feed.meta.nil?
-          # update if @json_file was set
-          arg_feed[0].json_pull unless feed.json_file.nil?
-          return_value = true
-        end
+        return_value = arg_feed[0].update!(new_feed)
       elsif arg_feed[0].is_a?(Array)
         return_value = []
         arg_feed[0].each do |f|
@@ -608,115 +274,4 @@ class NVDFeedScraper
     end
     return cve_names
   end
-
-  # Manage the meta file from a feed.
-  #
-  # == Usage
-  #
-  # @example
-  #   s = NVDFeedScraper.new
-  #   s.scrap
-  #   metaUrl = s.feeds("CVE-2014").meta_url
-  #   m = NVDFeedScraper::Meta.new
-  #   m.url = metaUrl
-  #   m.parse
-  #   m.sha256
-  #
-  # Several ways to set the url:
-  #
-  #   m = NVDFeedScraper::Meta.new(metaUrl)
-  #   m.parse
-  #   # or
-  #   m = NVDFeedScraper::Meta.new
-  #   m.url = metaUrl
-  #   m.parse
-  #   # or
-  #   m = NVDFeedScraper::Meta.new
-  #   m.parse(metaUrl)
-  class Meta
-    # {Meta} last modified date getter
-    # @return [String] the last modified date and time.
-    # @example
-    #   '2017-10-19T03:27:02-04:00'
-    attr_reader :last_modified_date
-
-    # {Meta} JSON size getter
-    # @return [String] the size of the JSON file uncompressed.
-    # @example
-    #   '29443314'
-    attr_reader :size
-
-    # {Meta} zip size getter
-    # @return [String] the size of the zip file.
-    # @example
-    #   '2008493'
-    attr_reader :zip_size
-
-    # {Meta} gz size getter
-    # @return [String] the size of the gz file.
-    # @example
-    #   '2008357'
-    attr_reader :gz_size
-
-    # {Meta} JSON sha256 getter
-    # @return [String] the SHA256 value of the uncompressed JSON file.
-    # @example
-    #   '33ED52D451692596D644F23742ED42B4E350258B11ACB900F969F148FCE3777B'
-    attr_reader :sha256
-
-    # @param url [String, nil] see {Feed#meta_url}.
-    def initialize(url = nil)
-      @url = url
-    end
-
-    # {Meta} URL getter.
-    # @return [String] The URL of the meta file of the feed.
-    attr_reader :url
-
-    # {Meta} URL setter.
-    # @param url [String] see {Feed#meta_url}.
-    def url=(url)
-      @url = url
-      @last_modified_date = @size = @zip_size = @gz_size = @sha256 = nil
-    end
-
-    # Parse the meta file from the URL and set the attributes.
-    # @overload parse
-    #   Parse the meta file from the URL and set the attributes.
-    #   @return [Integer] Returns +0+ when there is no error.
-    # @overload parse(url)
-    #   Set the URL of the meta file of the feed and
-    #   parse the meta file from the URL and set the attributes.
-    #   @param url [String] see {Feed.meta_url}
-    #   @return [Integer] Returns +0+ when there is no error.
-    def parse(*arg)
-      if arg.empty?
-      elsif arg.length == 1 # arg = url
-        self.url = arg[0]
-      else
-        raise 'Too much arguments'
-      end
-
-      raise "Can't parse if the URL is empty" if @url.nil?
-      uri = URI(@url)
-
-      meta = Net::HTTP.get(uri)
-
-      meta = Hash[meta.split.map { |x| x.split(':', 2) }]
-
-      raise 'no lastModifiedDate attribute found' unless meta['lastModifiedDate']
-      raise 'no valid size attribute found' unless /[0-9]+/.match?(meta['size'])
-      raise 'no valid zipSize attribute found' unless /[0-9]+/.match?(meta['zipSize'])
-      raise 'no valid gzSize attribute found' unless /[0-9]+/.match?(meta['gzSize'])
-      raise 'no valid sha256 attribute found' unless /[0-9A-F]{64}/.match?(meta['sha256'])
-
-      @last_modified_date = meta['lastModifiedDate']
-      @size = meta['size']
-      @zip_size = meta['zipSize']
-      @gz_size = meta['gzSize']
-      @sha256 = meta['sha256']
-
-      0
-    end
-  end
 end

data/lib/nvd_feed_api/feed.rb

@@ -0,0 +1,397 @@
+# Ruby internal
+require 'digest'
+require 'net/https'
+require 'date'
+# External
+require 'archive/zip'
+require 'oj'
+# Project internal
+require 'nvd_feed_api/meta'
+
+class NVDFeedScraper
+  # Feed object.
+  class Feed
+    class << self
+      # Get / set default feed storage location, where will be stored JSON feeds and archives by default.
+      # @return [String] default feed storage location. Default to +/tmp/+.
+      # @example
+      #   NVDFeedScraper::Feed.default_storage_location = '/srv/downloads/'
+      attr_accessor :default_storage_location
+    end
+    @default_storage_location = '/tmp/'
+
+    # @return [String] the name of the feed.
+    # @example
+    #   'CVE-2007'
+    attr_reader :name
+
+    # @return [String] the last update date of the feed information on the NVD website.
+    # @example
+    #   '10/19/2017 3:27:02 AM -04:00'
+    attr_reader :updated
+
+    # @return [String] the URL of the metadata file of the feed.
+    # @example
+    #   'https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-2007.meta'
+    attr_reader :meta_url
+
+    # @return [String] the URL of the gz archive of the feed.
+    # @example
+    #   'https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-2007.json.gz'
+    attr_reader :gz_url
+
+    # @return [String] the URL of the zip archive of the feed.
+    # @example
+    #   'https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-2007.json.zip'
+    attr_reader :zip_url
+
+    # @return [Meta] the {Meta} object of the feed.
+    # @note
+    #   Return nil if not previously loaded by {#meta_pull}.
+    #   Note that {#json_pull} also calls {#meta_pull}.
+    # @example
+    #   s = NVDFeedScraper.new
+    #   s.scrap
+    #   f = s.feeds("CVE-2014")
+    #   f.meta # => nil
+    #   f.meta_pull
+    #   f.meta # => #<NVDFeedScraper::Meta:0x00555b53027570 ... >
+    attr_reader :meta
+
+    # @return [String] the path of the saved JSON file.
+    # @note Return nil if not previously loaded by {#json_pull}.
+    # @example
+    #   s = NVDFeedScraper.new
+    #   s.scrap
+    #   f = s.feeds("CVE-2014")
+    #   f.json_file # => nil
+    #   f.json_pull
+    #   f.json_file # => "/tmp/nvdcve-1.0-2014.json"
+    attr_reader :json_file
+
+    # @return [String] the type of the feed, should always be +CVE+.
+    # @note Return nil if not previously loaded by {#json_pull}.
+    attr_reader :data_type
+
+    # @return [String] the format of the feed, should always be +MITRE+.
+    # @note Return nil if not previously loaded by {#json_pull}.
+    attr_reader :data_format
+
+    # @return [Float] the version of the JSON schema of the feed.
+    # @note Return nil if not previously loaded by {#json_pull}.
+    attr_reader :data_version
+
+    # @return [Integer] the number of CVEs of in the feed.
+    # @note Return nil if not previously loaded by {#json_pull}.
+    attr_reader :data_number_of_cves
+
+    # @return [Date] the date of the last update of the feed by the NVD.
+    # @note Return nil if not previously loaded by {#json_pull}.
+    attr_reader :data_timestamp
+
+    # A new instance of Feed.
+    # @param name [String] see {#name}.
+    # @param updated [String] see {#updated}.
+    # @param meta_url [String] see {#meta_url}.
+    # @param gz_url [String] see {#gz_url}.
+    # @param zip_url [String] see {#zip_url}.
+    def initialize(name, updated, meta_url, gz_url, zip_url)
+      # Frome meta file
+      @name = name
+      @updated = updated
+      @meta_url = meta_url
+      @gz_url = gz_url
+      @zip_url = zip_url
+      # do not pull meta and json automatically for speed and memory footprint
+      @meta = nil
+      @json_file = nil
+      # feed data
+      @data_type = nil
+      @data_format = nil
+      @data_version = nil
+      @data_number_of_cves = nil
+      @data_timestamp = nil
+    end
+
+    # Create or update the {Meta} object (fill the attribute).
+    # @return [Meta] the updated {Meta} object of the feed.
+    # @see #meta
+    def meta_pull
+      meta_content = NVDFeedScraper::Meta.new(@meta_url)
+      meta_content.parse
+      # update @meta
+      @meta = meta_content
+    end
+
+    # Download the gz archive of the feed.
+    # @param opts [Hash] see {#download_file}.
+    # @return [String] the saved gz file path.
+    # @example
+    #   afeed.download_gz
+    #   afeed.download_gz(destination_path: '/srv/save/')
+    def download_gz(opts = {})
+      download_file(@gz_url, opts)
+    end
+
+    # Download the zip archive of the feed.
+    # @param opts [Hash] see {#download_file}.
+    # @return [String] the saved zip file path.
+    # @example
+    #   afeed.download_zip
+    #   afeed.download_zip(destination_path: '/srv/save/')
+    def download_zip(opts = {})
+      download_file(@zip_url, opts)
+    end
+
+    # Download the JSON feed and fill the attribute.
+    # @param opts [Hash] see {#download_file}.
+    # @return [String] the path of the saved JSON file. Default use {Feed#default_storage_location}.
+    # @note Will downlaod and save the zip of the JSON file, unzip and save it. This massively consume time.
+    # @see #json_file
+    def json_pull(opts = {})
+      opts[:destination_path] ||= Feed.default_storage_location
+
+      skip_download = false
+      destination_path = opts[:destination_path]
+      destination_path += '/' unless destination_path[-1] == '/'
+      filename = URI(@zip_url).path.split('/').last.chomp('.zip')
+      # do not use @json_file for destination_file because of offline loading
+      destination_file = destination_path + filename
+      meta_pull
+      if File.file?(destination_file)
+        # Verify hash to see if it is the latest
+        computed_h = Digest::SHA256.file(destination_file)
+        skip_download = true if meta.sha256.casecmp(computed_h.hexdigest).zero?
+      end
+      if skip_download
+        @json_file = destination_file
+        # Set data
+        if @data_type.nil?
+          doc = Oj::Doc.open(File.read(@json_file))
+          @data_type = doc.fetch('/CVE_data_type')
+          @data_format = doc.fetch('/CVE_data_format')
+          @data_version = doc.fetch('/CVE_data_version').to_f
+          @data_number_of_cves = doc.fetch('/CVE_data_numberOfCVEs').to_i
+          @data_timestamp = Date.strptime(doc.fetch('/CVE_data_timestamp'), '%FT%RZ')
+          doc.close
+        end
+      else
+        zip_path = download_zip(opts)
+        Archive::Zip.open(zip_path) do |z|
+          z.extract(destination_path, flatten: true)
+        end
+        @json_file = zip_path.chomp('.zip')
+        # Verify hash integrity
+        computed_h = Digest::SHA256.file(@json_file)
+        raise "File corruption: #{@json_file}" unless meta.sha256.casecmp(computed_h.hexdigest).zero?
+        # update data
+        doc = Oj::Doc.open(File.read(@json_file))
+        @data_type = doc.fetch('/CVE_data_type')
+        @data_format = doc.fetch('/CVE_data_format')
+        @data_version = doc.fetch('/CVE_data_version').to_f
+        @data_number_of_cves = doc.fetch('/CVE_data_numberOfCVEs').to_i
+        @data_timestamp = Date.strptime(doc.fetch('/CVE_data_timestamp'), '%FT%RZ')
+        doc.close
+      end
+      return @json_file
+    end
+
+    # Search for CVE in the feed.
+    # @overload cve(cve)
+    #   One CVE.
+    #   @param cve [String] CVE ID, case insensitive.
+    #   @return [Hash] a Ruby Hash corresponding to the CVE.
+    # @overload cve(cve_arr)
+    #   An array of CVEs.
+    #   @param cve_arr [Array<String>] Array of CVE ID, case insensitive.
+    #   @return [Array] an Array of CVE, each CVE is a Ruby Hash. May not be in the same order as provided.
+    # @overload cve(cve, *)
+    #   Multiple CVEs.
+    #   @param cve [String] CVE ID, case insensitive.
+    #   @param * [String] As many CVE ID as you want.
+    #   @return [Array] an Array of CVE, each CVE is a Ruby Hash. May not be in the same order as provided.
+    # @note {#json_pull} is needed before using this method. Remember you're searching only in the current feed.
+    # @todo implement a CVE Class instead of returning a Hash.
+    # @see https://scap.nist.gov/schema/nvd/feed/0.1/nvd_cve_feed_json_0.1_beta.schema
+    # @see https://scap.nist.gov/schema/nvd/feed/0.1/CVE_JSON_4.0_min.schema
+    # @example
+    #   s = NVDFeedScraper.new
+    #   s.scrap
+    #   f = s.feeds("CVE-2014")
+    #   f.json_pull
+    #   f.cve("CVE-2014-0002", "cve-2014-0001")
+    def cve(*arg_cve)
+      raise 'json_file is nil, it needs to be populated with json_pull' if @json_file.nil?
+      raise "json_file (#{@json_file}) doesn't exist" unless File.file?(@json_file)
+      return_value = nil
+      raise 'no argument provided, 1 or more expected' if arg_cve.empty?
+      if arg_cve.length == 1
+        if arg_cve[0].is_a?(String)
+          raise "bad CVE name (#{arg_cve[0]})" unless /^CVE-[0-9]{4}-[0-9]{4,}$/i.match?(arg_cve[0])
+          doc = Oj::Doc.open(File.read(@json_file))
+          # Quicker than doc.fetch('/CVE_Items').size
+          (1..@data_number_of_cves).each do |i|
+            if arg_cve[0].upcase == doc.fetch("/CVE_Items/#{i}/cve/CVE_data_meta/ID")
+              return_value = doc.fetch("/CVE_Items/#{i}")
+              break
+            end
+          end
+          doc.close
+        elsif arg_cve[0].is_a?(Array)
+          return_value = []
+          # Sorting CVE can allow us to parse quicker
+          # Upcase to be sure include? works
+          cves_to_find = arg_cve[0].map(&:upcase).sort
+          raise 'one of the provided arguments is not a String' unless cves_to_find.all? { |x| x.is_a?(String) }
+          raise 'bad CVE name' unless cves_to_find.all? { |x| /^CVE-[0-9]{4}-[0-9]{4,}$/i.match?(x) }
+          doc = Oj::Doc.open(File.read(@json_file))
+          # Quicker than doc.fetch('/CVE_Items').size
+          (1..@data_number_of_cves).each do |i|
+            doc.move("/CVE_Items/#{i}")
+            cve_id = doc.fetch('cve/CVE_data_meta/ID')
+            if cves_to_find.include?(cve_id)
+              return_value.push(doc.fetch)
+              cves_to_find.delete(cve_id)
+            elsif cves_to_find.empty?
+              break
+            end
+          end
+          raise "#{cves_to_find.join(', ')} are unexisting CVEs in this feed" unless cves_to_find.empty?
+        else
+          raise "the provided argument (#{arg_cve[0]}) is nor a String or an Array"
+        end
+      else
+        # Overloading a list of arguments as one array argument
+        return_value = cve(arg_cve)
+      end
+      return return_value
+    end
+
+    # Return a list with the name of all available CVEs in the feed.
+    # Can only be called after {#json_pull}.
+    # @return [Array<String>] List with the name of all available CVEs. May return thousands CVEs.
+    def available_cves
+      raise 'json_file is nil, it needs to be populated with json_pull' if @json_file.nil?
+      raise "json_file (#{@json_file}) doesn't exist" unless File.file?(@json_file)
+      doc = Oj::Doc.open(File.read(@json_file))
+      # Quicker than doc.fetch('/CVE_Items').size
+      cve_names = []
+      (1..@data_number_of_cves).each do |i|
+        doc.move("/CVE_Items/#{i}")
+        cve_names.push(doc.fetch('cve/CVE_data_meta/ID'))
+      end
+      doc.close
+      return cve_names
+    end
+
+    # @param arg_name [String] the new name of the feed.
+    # @return [String] the new name of the feed.
+    # @example
+    #   'CVE-2007'
+    def name=(arg_name)
+      raise "name (#{arg_name}) is not a string" unless arg_name.is_a?(String)
+      @name = arg_name
+    end
+
+    # @param arg_updated [String] the last update date of the feed information on the NVD website.
+    # @return [String] the new date.
+    # @example
+    #   '10/19/2017 3:27:02 AM -04:00'
+    def updated=(arg_updated)
+      raise "updated date (#{arg_updated}) is not a string" unless arg_updated.is_a?(String)
+      @updated = arg_updated
+    end
+
+    # @param arg_meta_url [String] the new URL of the metadata file of the feed.
+    # @return [String] the new URL of the metadata file of the feed.
+    # @example
+    #   'https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-2007.meta'
+    def meta_url=(arg_meta_url)
+      raise "meta_url (#{arg_meta_url}) is not a string" unless arg_meta_url.is_a?(String)
+      @meta_url = arg_meta_url
+    end
+
+    # @param arg_gz_url [String] the new URL of the gz archive of the feed.
+    # @return [String] the new URL of the gz archive of the feed.
+    # @example
+    #   'https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-2007.json.gz'
+    def gz_url=(arg_gz_url)
+      raise "gz_url (#{arg_gz_url}) is not a string" unless arg_gz_url.is_a?(String)
+      @gz_url = arg_gz_url
+    end
+
+    # @param arg_zip_url [String] the new URL of the zip archive of the feed.
+    # @return [String] the new URL of the zip archive of the feed.
+    # @example
+    #   'https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-2007.json.zip'
+    def zip_url=(arg_zip_url)
+      raise "zip_url (#{arg_zip_url}) is not a string" unless arg_zip_url.is_a?(String)
+      @zip_url = arg_zip_url
+    end
+
+    # Download a file.
+    # @param file_url [String] the URL of the file.
+    # @param opts [Hash] the optional downlaod parameters.
+    # @option opts [String] :destination_path the destination path (may
+    #   overwrite existing file).
+    #   Default use {Feed#default_storage_location}.
+    # @option opts [String] :sha256 the SHA256 hash to check, if the file
+    #   already exist and the hash matches then the download will be skipped.
+    # @return [String] the saved file path.
+    # @example
+    #   download_file('https://example.org/example.zip') # => '/tmp/example.zip'
+    #   download_file('https://example.org/example.zip', destination_path: '/srv/save/') # => '/srv/save/example.zip'
+    #   download_file('https://example.org/example.zip', {destination_path: '/srv/save/', sha256: '70d6ea136d5036b6ce771921a949357216866c6442f44cea8497f0528c54642d'}) # => '/srv/save/example.zip'
+    def download_file(file_url, opts = {})
+      opts[:destination_path] ||= Feed.default_storage_location
+      opts[:sha256] ||= nil
+
+      destination_path = opts[:destination_path]
+      destination_path += '/' unless destination_path[-1] == '/'
+      skip_download = false
+      uri = URI(file_url)
+      filename = uri.path.split('/').last
+      destination_file = destination_path + filename
+      unless opts[:sha256].nil?
+        if File.file?(destination_file)
+          # Verify hash to see if it is the latest
+          computed_h = Digest::SHA256.file(destination_file)
+          skip_download = true if opts[:sha256].casecmp(computed_h.hexdigest).zero?
+        end
+      end
+      unless skip_download
+        res = Net::HTTP.get_response(uri)
+        raise "#{file_url} ended with #{res.code} #{res.message}" unless res.is_a?(Net::HTTPSuccess)
+        open(destination_file, 'wb') do |file|
+          file.write(res.body)
+        end
+      end
+      return destination_file
+    end
+
+    # Update the feed
+    # @param fresh_feed [Feed] the fresh feed from which the feed will be updated.
+    # @return [Boolean] +true+ if the feed was updated, +false+ if it wasn't.
+    # @note Is not intended to be used directly, use {NVDFeedScraper#update_feeds} instead.
+    def update!(fresh_feed)
+      return_value = false
+      raise "#{fresh_feed} is not a Feed" unless fresh_feed.is_a?(Feed)
+      # update attributes
+      if updated != fresh_feed.updated
+        self.name = fresh_feed.name
+        self.updated = fresh_feed.updated
+        self.meta_url = fresh_feed.meta_url
+        self.gz_url = fresh_feed.gz_url
+        self.zip_url = fresh_feed.zip_url
+        # update if @meta was set
+        meta_pull unless @meta.nil?
+        # update if @json_file was set, this will also update @data_*
+        json_pull unless @json_file.nil?
+        return_value = true
+      end
+      return return_value
+    end
+
+    protected :name=, :updated=, :meta_url=, :gz_url=, :zip_url=, :download_file
+  end
+end

data/lib/nvd_feed_api/meta.rb

@@ -0,0 +1,115 @@
+# Ruby internal
+require 'net/https'
+
+class NVDFeedScraper
+  # Manage the meta file from a feed.
+  #
+  # == Usage
+  #
+  # @example
+  #   s = NVDFeedScraper.new
+  #   s.scrap
+  #   metaUrl = s.feeds("CVE-2014").meta_url
+  #   m = NVDFeedScraper::Meta.new
+  #   m.url = metaUrl
+  #   m.parse
+  #   m.sha256
+  #
+  # Several ways to set the url:
+  #
+  #   m = NVDFeedScraper::Meta.new(metaUrl)
+  #   m.parse
+  #   # or
+  #   m = NVDFeedScraper::Meta.new
+  #   m.url = metaUrl
+  #   m.parse
+  #   # or
+  #   m = NVDFeedScraper::Meta.new
+  #   m.parse(metaUrl)
+  class Meta
+    # {Meta} last modified date getter
+    # @return [String] the last modified date and time.
+    # @example
+    #   '2017-10-19T03:27:02-04:00'
+    attr_reader :last_modified_date
+
+    # {Meta} JSON size getter
+    # @return [String] the size of the JSON file uncompressed.
+    # @example
+    #   '29443314'
+    attr_reader :size
+
+    # {Meta} zip size getter
+    # @return [String] the size of the zip file.
+    # @example
+    #   '2008493'
+    attr_reader :zip_size
+
+    # {Meta} gz size getter
+    # @return [String] the size of the gz file.
+    # @example
+    #   '2008357'
+    attr_reader :gz_size
+
+    # {Meta} JSON sha256 getter
+    # @return [String] the SHA256 value of the uncompressed JSON file.
+    # @example
+    #   '33ED52D451692596D644F23742ED42B4E350258B11ACB900F969F148FCE3777B'
+    attr_reader :sha256
+
+    # @param url [String, nil] see {Feed#meta_url}.
+    def initialize(url = nil)
+      @url = url
+    end
+
+    # {Meta} URL getter.
+    # @return [String] The URL of the meta file of the feed.
+    attr_reader :url
+
+    # {Meta} URL setter.
+    # @param url [String] see {Feed#meta_url}.
+    def url=(url)
+      @url = url
+      @last_modified_date = @size = @zip_size = @gz_size = @sha256 = nil
+    end
+
+    # Parse the meta file from the URL and set the attributes.
+    # @overload parse
+    #   Parse the meta file from the URL and set the attributes.
+    #   @return [Integer] Returns +0+ when there is no error.
+    # @overload parse(url)
+    #   Set the URL of the meta file of the feed and
+    #   parse the meta file from the URL and set the attributes.
+    #   @param url [String] see {Feed.meta_url}
+    #   @return [Integer] Returns +0+ when there is no error.
+    def parse(*arg)
+      if arg.empty?
+      elsif arg.length == 1 # arg = url
+        self.url = arg[0]
+      else
+        raise 'Too much arguments'
+      end
+
+      raise "Can't parse if the URL is empty" if @url.nil?
+      uri = URI(@url)
+
+      meta = Net::HTTP.get(uri)
+
+      meta = Hash[meta.split.map { |x| x.split(':', 2) }]
+
+      raise 'no lastModifiedDate attribute found' unless meta['lastModifiedDate']
+      raise 'no valid size attribute found' unless /[0-9]+/.match?(meta['size'])
+      raise 'no valid zipSize attribute found' unless /[0-9]+/.match?(meta['zipSize'])
+      raise 'no valid gzSize attribute found' unless /[0-9]+/.match?(meta['gzSize'])
+      raise 'no valid sha256 attribute found' unless /[0-9A-F]{64}/.match?(meta['sha256'])
+
+      @last_modified_date = meta['lastModifiedDate']
+      @size = meta['size']
+      @zip_size = meta['zipSize']
+      @gz_size = meta['gzSize']
+      @sha256 = meta['sha256']
+
+      0
+    end
+  end
+end

data/lib/nvd_feed_api/version.rb

@@ -1,3 +1,3 @@
 module NvdFeedApi
-  VERSION = '0.1.0'.freeze
+  VERSION = '0.2.0'.freeze
 end

data/pages/CHANGELOG.md

@@ -1,3 +1,17 @@
+# [0.2.0] - 20 January 2018
+
+[0.2.0]: https://gitlab.com/noraj/nvd_api/tags/v0.2.0
+
+- new attributes for the Feed class:
+  + `data_type`
+  + `data_format`
+  + `data_version`
+  + `data_number_of_cves`
+  + `data_timestamp`
+- fix `update_feeds` method by using the new `update!` method from the Feed class
+- split source code in several files, one by class
+- improve tests and documentation
+
 # [0.1.0] - 17 January 2018
 
 [0.1.0]: https://gitlab.com/noraj/nvd_api/tags/v0.1.0

data/test/test_nvd_feed_api.rb

@@ -120,10 +120,6 @@ class NVDAPITest < Minitest::Test
     # Test updated
     assert_instance_of(String, f.updated, "updated doesn't return a string")
     refute_empty(f.updated, 'updated is empty')
-    # Test meta
-    assert_nil(f.meta)
-    # Test json_file
-    assert_nil(f.json_file)
     # Test gz_url
     assert_instance_of(String, f.gz_url, "gz_url doesn't return a string")
     refute_empty(f.gz_url, 'gz_url is empty')
@@ -136,6 +132,30 @@ class NVDAPITest < Minitest::Test
     assert_instance_of(String, f.meta_url, "meta_url doesn't return a string")
     refute_empty(f.meta_url, 'meta_url is empty')
     assert_equal(meta_url, f.meta_url, 'The meta_url url of the feed was modified')
+    # Test meta (before json_pull)
+    assert_nil(f.meta)
+    # Test json_file
+    assert_nil(f.json_file)
+    f.json_pull
+    assert_instance_of(String, f.json_file, "json_file doesn't return a string")
+    refute_empty(f.json_file, 'json_file is empty')
+    # Test meta (after json_pull)
+    f.meta_pull
+    assert_instance_of(NVDFeedScraper::Meta, f.meta, "meta doesn't return a Meta object")
+
+    # Test data (require json_pull)
+    # Test data_type
+    assert_instance_of(String, f.data_type, "data_type doesn't return a String")
+    refute_empty(f.data_type, 'data_type is empty')
+    # Test data_format
+    assert_instance_of(String, f.data_format, "data_format doesn't return a String")
+    refute_empty(f.data_format, 'data_format is empty')
+    # Test data_version
+    assert_instance_of(Float, f.data_version, "data_version doesn't return a Float")
+    # Test data_number_of_cves
+    assert_instance_of(Integer, f.data_number_of_cves, "data_number_of_cves doesn't return an Integer")
+    # Test data_timestamp
+    assert_instance_of(Date, f.data_timestamp, "data_timestamp doesn't return a Date")
   end
 
   def test_feed_available_cves
@@ -202,8 +222,21 @@ class NVDAPITest < Minitest::Test
 
   def test_feed_meta_pull
     f = @s.feeds('CVE-2005')
-    return_value = f.meta_pull
-    assert_instance_of(NVDFeedScraper::Meta, return_value, "meta_pull doesn't return a Meta object")
+    assert_instance_of(NVDFeedScraper::Meta, f.meta_pull, "meta_pull doesn't return a Meta object")
+  end
+
+  def test_feed_update!
+    f = @s.feeds('CVE-2006')
+    @s.scrap
+    f_new = @s.feeds('CVE-2006')
+    # Right arg
+    # can't use assert_instance_of because there is no boolean class
+    assert(%w[TrueClass FalseClass].include?(f.update!(f_new).class.to_s), "update! doesn't return a boolean")
+    # Bad arg
+    err = assert_raises(RuntimeError) do
+      f.update!('bad_arg')
+    end
+    assert_equal('bad_arg is not a Feed', err.message)
   end
 
   def test_meta_parse_noarg

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: nvd_feed_api
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Alexandre ZANNI
@@ -189,6 +189,8 @@ files:
 - bin/nvd_feed_api_console
 - bin/nvd_feed_api_setup
 - lib/nvd_feed_api.rb
+- lib/nvd_feed_api/feed.rb
+- lib/nvd_feed_api/meta.rb
 - lib/nvd_feed_api/version.rb
 - nvd_feed_api.gemspec
 - pages/CHANGELOG.md