ecosystems-bibliothecary 14.3.0 → 15.0.0

data/lib/bibliothecary/multi_parsers/cyclonedx.rb

@@ -1,170 +0,0 @@
-# frozen_string_literal: true
-
-require "json"
-require "ox"
-
-# packageurl-ruby uses pattern-matching (https://docs.ruby-lang.org/en/2.7.0/NEWS.html#label-Pattern+matching)
-# which warns a whole bunch in Ruby 2.7 as being an experimental feature, but has
-# been accepted in Ruby 3.0 (https://rubyreferences.github.io/rubychanges/3.0.html#pattern-matching).
-Warning[:experimental] = false
-require "package_url"
-Warning[:experimental] = true
-
-module Bibliothecary
-  module MultiParsers
-    module CycloneDX
-      include Bibliothecary::Analyser
-      include Bibliothecary::Analyser::TryCache
-
-      NoComponents = Class.new(StandardError)
-
-      class ManifestEntries
-        # If a purl type (key) exists, it will be used in a manifest for
-        # the key's value. If not, it's ignored.
-        #
-        # https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst
-        PURL_TYPE_MAPPING = {
-          "brew" => :homebrew,
-          "cargo" => :cargo,
-          "carthage" => :carthage, 
-          "clojars" => :clojars,
-          "cocoapods" => :cocoapods,
-          "composer" => :packagist,
-          "conda" => :conda,
-          "cpan" => :cpan,
-          "cran" => :cran,
-          "docker" => :docker,
-          "dub" => :dub,
-          "elm" => :elm,
-          "gem" => :rubygems,
-          "golang" => :go,
-          "hackage" => :hackage,
-          "haxe" => :haxelib,
-          "hex" => :hex,
-          "julia" => :julia,
-          "maven" => :maven,
-          "meteor" => :meteor,
-          "npm" => :npm,
-          "nuget" => :nuget,
-          "pub" => :pub,
-          "pypi" => :pypi,
-          "swift" => :swift_pm,          
-        }
-
-        attr_reader :manifests
-
-        def initialize(parse_queue:)
-          @manifests = {}
-
-          # Instead of recursing, we'll work through a queue of components
-          # to process, letting the different parser add components to the
-          # queue however they need to  pull them from the source document.
-          @parse_queue = parse_queue.dup
-        end
-
-        def add(purl, source = nil)
-          mapping = PurlUtil::PURL_TYPE_MAPPING[purl.type]
-          return unless mapping
-
-          @manifests[mapping] ||= Set.new
-          @manifests[mapping] << Dependency.new(
-            name: PurlUtil.full_name(purl),
-            requirement: purl.version,
-            platform: mapping.to_s,
-            type: "lockfile",
-            source: source
-          )
-        end
-
-        # Iterates over each manifest entry in the parse_queue, and accepts a block which will
-        # be called on each component. The block has two jobs: 1) add more sub-components
-        # to parse (if they exist), and 2) return the components purl.
-        def parse!(source = nil, &block)
-          until @parse_queue.empty?
-            component = @parse_queue.shift
-
-            purl_text = block.call(component, @parse_queue)
-
-            next unless purl_text
-
-            purl = PackageURL.parse(purl_text)
-
-            add(purl, source)
-          end
-        end
-
-        def [](key)
-          @manifests[key]&.to_a
-        end
-      end
-
-      def self.mapping
-        {
-          match_filename("cyclonedx.json") => {
-            kind: "lockfile",
-            parser: :parse_cyclonedx_json,
-            ungroupable: true,
-          },
-          match_extension("cdx.json") => {
-            kind: "lockfile",
-            parser: :parse_cyclonedx_json,
-            ungroupable: true,
-          },
-          match_filename("cyclonedx.xml") => {
-            kind: "lockfile",
-            parser: :parse_cyclonedx_xml,
-            ungroupable: true,
-          },
-          match_extension(".cdx.xml") => {
-            kind: "lockfile",
-            parser: :parse_cyclonedx_xml,
-            ungroupable: true,
-          },
-        }
-      end
-
-      def parse_cyclonedx_json(file_contents, options: {})
-        manifest = try_cache(options, options[:filename]) do
-          JSON.parse(file_contents)
-        end
-
-        raise NoComponents unless manifest["components"]
-
-        entries = ManifestEntries.new(parse_queue: manifest["components"])
-
-        entries.parse!(options.fetch(:filename, nil)) do |component, parse_queue|
-          parse_queue.concat(component["components"]) if component["components"]
-
-          component["purl"]
-        end
-
-        ParserResult.new(dependencies: entries[platform_name.to_sym] || [])
-      end
-
-      def parse_cyclonedx_xml(file_contents, options: {})
-        manifest = try_cache(options, options[:filename]) do
-          Ox.parse(file_contents)
-        end
-
-        root = manifest
-        if root.respond_to?(:bom)
-          root = root.bom
-        end
-
-        raise NoComponents unless root.locate("components").first
-
-        entries = ManifestEntries.new(parse_queue: root.locate("components/*"))
-
-        entries.parse!(options.fetch(:filename, nil)) do |component, parse_queue|
-          # #locate returns an empty array if nothing is found, so we can
-          # always safely concatenate it to the parse queue.
-          parse_queue.concat(component.locate("components/*"))
-
-          component.locate("purl").first&.text
-        end
-
-        ParserResult.new(dependencies: entries[platform_name.to_sym] || [])
-      end
-    end
-  end
-end

data/lib/bibliothecary/multi_parsers/dependencies_csv.rb

@@ -1,155 +0,0 @@
-# frozen_string_literal: true
-
-require "csv"
-
-module Bibliothecary
-  module MultiParsers
-    module DependenciesCSV
-      include Bibliothecary::Analyser
-      include Bibliothecary::Analyser::TryCache
-
-      def self.mapping
-        {
-          match_filename("dependencies.csv") => {
-            kind: "lockfile",
-            ungroupable: true,
-            parser: :parse_dependencies_csv,
-          },
-        }
-      end
-
-      # Processing a CSV file isn't as exact as using a real manifest file,
-      # but you can get pretty close as long as the data you're importing
-      # is simple.
-      class CSVFile
-        # Header structures are:
-        #
-        # <field to fill in for dependency> => {
-        #   match: [<regexp of incoming column name to match in priority order, highest priority first>...],
-        #   [default]: <optional default value for this field>
-        # }
-        HEADERS = {
-          "platform" => {
-            match: [
-              /^platform$/i,
-            ],
-          },
-          "name" => {
-            match: [
-              /^name$/i,
-            ],
-          },
-          # Manifests have versions that can have operators.
-          # However, since Bibliothecary only currently supports analyzing a
-          # single file as a single thing (either manifest or lockfile)
-          # we can't return manifest-y data. Only take the lockfile requirement
-          # when processing dependencies.csv for now.
-          "requirement" => {
-            match: [
-              /^(lockfile |)requirement$/i,
-              /^version$/i,
-            ],
-          },
-          "type" => {
-            default: "runtime",
-            match: [
-              /^(lockfile |)type$/i,
-              /^(manifest |)type$/i,
-            ],
-          },
-        }.freeze
-
-        attr_reader :result
-
-        def initialize(file_contents)
-          @file_contents = file_contents
-
-          @result = nil
-
-          # A Hash of "our field name" => ["header in CSV file", "lower priority header in CSV file"]
-          @header_mappings = {}
-        end
-
-        def parse!
-          table = parse_and_validate_csv_file
-
-          @result = table.map.with_index do |row, idx|
-            HEADERS.each_with_object({}) do |(header, info), obj|
-              # find the first non-empty field in the row for this header, or nil if not found
-              row_data = row[@header_mappings[header]]
-
-              # some column have default data to fall back on
-              if row_data
-                obj[header.to_sym] = row_data
-              elsif info.key?(:default)
-                # if the default is nil, don't even add the key to the hash
-                obj[header.to_sym] = info[:default] if info[:default]
-              else
-                # use 1-based index just like the 'csv' std lib, and count the headers as first row.
-                raise "Missing required field '#{header}' on line #{idx + 2}."
-              end
-            end
-          end
-        end
-
-        private
-
-        def parse_and_validate_csv_file
-          table = CSV.parse(@file_contents, headers: true)
-
-          header_examination_results = map_table_headers_to_local_lookups(table, HEADERS)
-          unless header_examination_results[:missing].empty?
-            raise "Missing required headers #{header_examination_results[:missing].join(', ')} in CSV. Check to make sure header names are all lowercase."
-          end
-
-          @header_mappings = header_examination_results[:found]
-
-          table
-        end
-
-        def map_table_headers_to_local_lookups(table, local_lookups)
-          result = local_lookups.each_with_object({ found: {}, missing: [] }) do |(header, info), obj|
-            results = table.headers.each_with_object([]) do |table_header, matches|
-              info[:match].each_with_index do |match_regexp, index|
-                matches << [table_header, index] if table_header[match_regexp]
-              end
-            end
-
-            if results.empty?
-              # if a header has a default value it's optional
-              obj[:missing] << header unless info.key?(:default)
-            else
-              # select the highest priority header possible
-              obj[:found][header] ||= nil
-              obj[:found][header] = ([obj[:found][header]] + results).compact.min_by(&:last)
-            end
-          end
-
-          # strip off the priorities. only one mapping should remain.
-          result[:found].transform_values!(&:first)
-
-          result
-        end
-      end
-
-      def parse_dependencies_csv(file_contents, options: {})
-        csv_file = try_cache(options, options[:filename]) do
-          raw_csv_file = CSVFile.new(file_contents)
-          raw_csv_file.parse!
-          raw_csv_file
-        end
-
-        dependencies = csv_file
-          .result
-          .find_all { |dependency| dependency[:platform] == platform_name.to_s }
-          .map do |dep_kvs|
-            Dependency.new(
-              **dep_kvs, source: options.fetch(:filename, nil)
-            )
-          end
-
-        ParserResult.new(dependencies: dependencies)
-      end
-    end
-  end
-end

data/lib/bibliothecary/multi_parsers/json_runtime.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-module Bibliothecary
-  module MultiParsers
-    # Provide JSON Runtime Manifest parsing
-    module JSONRuntime
-      def parse_json_runtime_manifest(file_contents, options: {})
-        dependencies = JSON.parse(file_contents).fetch("dependencies", []).map do |name, requirement|
-          Dependency.new(
-            platform: platform_name,
-            name: name,
-            requirement: requirement,
-            type: "runtime",
-            source: options.fetch(:filename, nil)
-          )
-        end
-
-        ParserResult.new(dependencies: dependencies)
-      end
-    end
-  end
-end

data/lib/bibliothecary/multi_parsers/spdx.rb

@@ -1,149 +0,0 @@
-# frozen_string_literal: true
-
-# packageurl-ruby uses pattern-matching (https://docs.ruby-lang.org/en/2.7.0/NEWS.html#label-Pattern+matching)
-# which warns a whole bunch in Ruby 2.7 as being an experimental feature, but has
-# been accepted in Ruby 3.0 (https://rubyreferences.github.io/rubychanges/3.0.html#pattern-matching).
-Warning[:experimental] = false
-require "package_url"
-Warning[:experimental] = true
-
-module Bibliothecary
-  module MultiParsers
-    module Spdx
-      include Bibliothecary::Analyser
-      include Bibliothecary::Analyser::TryCache
-
-      # e.g. 'SomeText:' (allowing for leading whitespace)
-      WELLFORMED_LINE_REGEXP = /^\s*[a-zA-Z]+:/
-
-      # e.g. 'PackageName: (allowing for excessive whitespace)
-      PACKAGE_NAME_REGEXP = /^\s*PackageName:\s*(.*)/
-
-      # e.g. 'PackageVersion:' (allowing for excessive whitespace)
-      PACKAGE_VERSION_REGEXP = /^\s*PackageVersion:\s*(.*)/
-
-      # e.g. "ExternalRef: PACKAGE-MANAGER purl (allowing for excessive whitespace)
-      PURL_REGEXP = /^\s*ExternalRef:\s*PACKAGE[-|_]MANAGER\s*purl\s*(.*)/
-
-      NoEntries = Class.new(StandardError)
-      MalformedFile = Class.new(StandardError)
-
-      def self.mapping
-        {
-          match_extension(".spdx") => {
-            kind: "lockfile",
-            parser: :parse_spdx_tag_value,
-            ungroupable: true,
-          },
-          match_extension(".spdx.json") => {
-            kind: "lockfile",
-            parser: :parse_spdx_json,
-            ungroupable: true,
-          },
-        }
-      end
-
-      def parse_spdx_tag_value(file_contents, options: {})
-        entries = try_cache(options, options[:filename]) do
-          parse_spdx_tag_value_file_contents(file_contents, options.fetch(:filename, nil))
-        end
-
-        raise NoEntries if entries.empty?
-
-        Bibliothecary::ParserResult.new(dependencies: entries[platform_name.to_sym] || [])
-      end
-
-      def parse_spdx_tag_value_file_contents(file_contents, source = nil)
-        entries = {}
-        spdx_name = spdx_version = platform = purl_name = purl_version = nil
-
-        file_contents.each_line do |line|
-          stripped_line = line.strip
-          next if skip_tag_value_line?(stripped_line)
-
-          raise MalformedFile unless stripped_line.match?(WELLFORMED_LINE_REGEXP)
-
-          if (match = stripped_line.match(PACKAGE_NAME_REGEXP))
-            # Per the spec:
-            # > A new package Information section is denoted by the package name (7.1) field.
-            add_entry(entries: entries, platform: platform, purl_name: purl_name,
-                      spdx_name: spdx_name, purl_version: purl_version, spdx_version: spdx_version,
-                      source: source)
-
-            # reset for this new package
-            spdx_name = spdx_version = platform = purl_name = purl_version = nil
-
-            # capture the new package's name
-            spdx_name = match[1]
-          elsif (match = stripped_line.match(PACKAGE_VERSION_REGEXP))
-            spdx_version = match[1]
-          elsif (match = stripped_line.match(PURL_REGEXP))
-            purl = PackageURL.parse(match[1])
-            platform ||= PurlUtil::PURL_TYPE_MAPPING[purl.type]
-            purl_name ||= PurlUtil.full_name(purl)
-            purl_version ||= purl.version
-          end
-        end
-
-        add_entry(entries: entries, platform: platform, purl_name: purl_name,
-                  spdx_name: spdx_name, purl_version: purl_version, spdx_version: spdx_version,
-                  source: source)
-
-        entries
-      end
-
-      def skip_tag_value_line?(stripped_line)
-        # Ignore blank lines and comments
-        stripped_line.empty? || stripped_line.start_with?("#")
-      end
-
-      def parse_spdx_json(file_contents, options: {})
-        entries = try_cache(options, options[:filename]) do
-          parse_spdx_json_file_contents(file_contents, options.fetch(:filename, nil))
-        end
-
-        raise NoEntries if entries.empty?
-
-        Bibliothecary::ParserResult.new(dependencies: entries[platform_name.to_sym] || [])
-      end
-
-      def parse_spdx_json_file_contents(file_contents, source = nil)
-        entries = {}
-        manifest = JSON.parse(file_contents)
-
-        manifest["packages"]&.each do |package|
-          spdx_name = package["name"]
-          spdx_version = package["versionInfo"]
-
-          first_purl_string = package["externalRefs"]&.find { |ref| ref["referenceType"] == "purl" }&.dig("referenceLocator")
-          purl = first_purl_string && PackageURL.parse(first_purl_string)
-          platform = PurlUtil::PURL_TYPE_MAPPING[purl&.type]
-          purl_name = PurlUtil.full_name(purl)
-          purl_version = purl&.version
-
-          add_entry(entries: entries, platform: platform, purl_name: purl_name,
-                    spdx_name: spdx_name, purl_version: purl_version, spdx_version: spdx_version,
-                    source: source)
-        end
-
-        entries
-      end
-
-      def add_entry(entries:, platform:, purl_name:, spdx_name:, purl_version:, spdx_version:, source: nil)
-        package_name = purl_name || spdx_name
-        package_version = purl_version || spdx_version
-
-        return unless platform && package_name && package_version
-
-        entries[platform.to_sym] ||= []
-        entries[platform.to_sym] << Dependency.new(
-          platform: platform.to_s,
-          name: package_name,
-          requirement: package_version,
-          type: "lockfile",
-          source: source
-        )
-      end
-    end
-  end
-end

data/lib/bibliothecary/purl_util.rb

@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-module Bibliothecary
-  class PurlUtil
-    # If a purl type (key) exists, it will be used in a manifest for
-    # the key's value. If not, it's ignored.
-    #
-    # https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst
-    PURL_TYPE_MAPPING = {
-      "golang" => :go,
-      "maven" => :maven,
-      "npm" => :npm,
-      "cargo" => :cargo,
-      "composer" => :packagist,
-      "conda" => :conda,
-      "cran" => :cran,
-      "gem" => :rubygems,
-      "nuget" => :nuget,
-      "pypi" => :pypi,
-    }.freeze
-
-    # @param purl [PackageURL]
-    # @return [String] The properly namespaced package name
-    def self.full_name(purl)
-      return nil if purl.nil?
-
-      parts = [purl.namespace, purl.name].compact
-
-      case purl.type
-      when "maven"
-        parts.join(":")
-      else
-        parts.join("/")
-      end
-    end
-  end
-end

data/lib/bibliothecary/runner/multi_manifest_filter.rb

@@ -1,92 +0,0 @@
-# frozen_string_literal: true
-
-module Bibliothecary
-  class Runner
-    class MultiManifestFilter
-      # Wrap up a file analysis for easier validity testing
-      class FileAnalysis
-        def initialize(file_analysis)
-          @file_analysis = file_analysis
-        end
-
-        # Determine if we should skip this file analysis when processing
-        # @return [Boolean] True if we should skip processing
-        def skip?
-          !@file_analysis ||
-            !@file_analysis[:dependencies] ||
-            @file_analysis[:dependencies].empty?
-        end
-      end
-
-      def initialize(path:, related_files_info_entries:, runner:)
-        @path = path
-        @related_files_info_entries = related_files_info_entries
-        @runner = runner
-      end
-
-      # Standalone multi manifest files should *always* be treated as lockfiles,
-      # since there's no human-written manifest file to go with them.
-      def files_to_check
-        @files_to_check ||= @related_files_info_entries.each_with_object({}) do |files_info, all|
-          files_info.lockfiles.each do |file|
-            all[file] ||= 0
-            all[file] += 1
-          end
-        end
-      end
-
-      def results
-        partition_file_entries!
-
-        (no_lockfile_results + single_file_results + multiple_file_results).uniq
-      end
-
-      def no_lockfile_results
-        @no_lockfile_results ||= @related_files_info_entries.find_all { |rfi| rfi.lockfiles.empty? }
-      end
-
-      def single_file_results
-        @single_file_results ||= @single_file_entries.map do |file|
-          @related_files_info_entries.find { |rfi| rfi.lockfiles.include?(file) }
-        end
-      end
-
-      def multiple_file_results
-        return @multiple_file_results if @multiple_file_results
-
-        @multiple_file_results = []
-
-        each_analysis_and_rfis do |analysis, rfis_for_file|
-          rfis_for_file.each do |rfi|
-            file_analysis = FileAnalysis.new(
-              analysis.find { |a| a[:platform] == rfi.platform }
-            )
-
-            next if file_analysis.skip?
-
-            @multiple_file_results << rfi
-          end
-        end
-
-        @multiple_file_results
-      end
-
-      def each_analysis_and_rfis
-        @multiple_file_entries.each do |file|
-          contents = Bibliothecary.utf8_string(File.read(File.join(@path, file)))
-          analysis = @runner.analyse_file(file, contents)
-          rfis_for_file = @related_files_info_entries.find_all { |rfi| rfi.lockfiles.include?(file) }
-
-          yield analysis, rfis_for_file
-        end
-      end
-
-      def partition_file_entries!
-        @single_file_entries, @multiple_file_entries = files_to_check.partition { |_file, count| count == 1 }
-
-        @single_file_entries = @single_file_entries.map(&:first)
-        @multiple_file_entries = @multiple_file_entries.map(&:first)
-      end
-    end
-  end
-end

data/lib/sdl_parser.rb

@@ -1,30 +0,0 @@
-# frozen_string_literal: true
-
-require "sdl4r"
-
-class SdlParser
-  attr_reader :contents, :type
-
-  def initialize(type, contents, platform, source = nil)
-    @contents = contents
-    @type = type || "runtime"
-    @platform = platform
-    @source = source
-  end
-
-  def dependencies
-    parse.children("dependency").inject([]) do |deps, dep|
-      deps.push(Bibliothecary::Dependency.new(
-                  platform: @platform,
-                  name: dep.value,
-                  requirement: dep.attribute("version") || ">= 0",
-                  type: type,
-                  source: @source
-                ))
-    end.uniq
-  end
-
-  def parse
-    SDL4R.read(contents)
-  end
-end