bibliothecary 11.0.1 → 12.1.0

data/lib/bibliothecary/parsers/npm.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "json"
 
 module Bibliothecary
@@ -37,29 +39,29 @@ module Bibliothecary
       add_multi_parser(Bibliothecary::MultiParsers::Spdx)
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
-      def self.parse_package_lock(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_package_lock(file_contents, options: {})
         manifest = JSON.parse(file_contents)
         # https://docs.npmjs.com/cli/v9/configuring-npm/package-lock-json#lockfileversion
         if manifest["lockfileVersion"].to_i <= 1
           # lockfileVersion 1 uses the "dependencies" object
-          parse_package_lock_v1(manifest)
+          parse_package_lock_v1(manifest, options.fetch(:filename, nil))
         else
           # lockfileVersion 2 has backwards-compatability by including both "packages" and the legacy "dependencies" object
           # lockfileVersion 3 has no backwards-compatibility and only includes the "packages" object
-          parse_package_lock_v2(manifest)
+          parse_package_lock_v2(manifest, options.fetch(:filename, nil))
         end
       end
 
       class << self
         # "package-lock.json" and "npm-shrinkwrap.json" have same format, so use same parsing logic
-        alias_method :parse_shrinkwrap, :parse_package_lock
+        alias parse_shrinkwrap parse_package_lock
       end
 
-      def self.parse_package_lock_v1(manifest)
-        parse_package_lock_deps_recursively(manifest.fetch("dependencies", []))
+      def self.parse_package_lock_v1(manifest, source = nil)
+        parse_package_lock_deps_recursively(manifest.fetch("dependencies", []), source)
       end
 
-      def self.parse_package_lock_v2(manifest)
+      def self.parse_package_lock_v2(manifest, source = nil)
         # "packages" is a flat object where each key is the installed location of the dep, e.g. node_modules/foo/node_modules/bar.
         manifest
           .fetch("packages")
@@ -73,33 +75,39 @@ module Bibliothecary
             Dependency.new(
               name: name.split("node_modules/").last,
               requirement: dep["version"],
-              type: dep.fetch("dev", false) || dep.fetch("devOptional", false)  ? "development" : "runtime",
+              type: dep.fetch("dev", false) || dep.fetch("devOptional", false) ? "development" : "runtime",
               local: dep.fetch("link", false),
+              source: source
             )
           end
       end
 
-      def self.parse_package_lock_deps_recursively(dependencies, depth=1)
+      def self.parse_package_lock_deps_recursively(dependencies, source = nil, depth = 1)
         dependencies.flat_map do |name, requirement|
           type = requirement.fetch("dev", false) ? "development" : "runtime"
           version = requirement.key?("from") ? requirement["from"][/#(?:semver:)?v?(.*)/, 1] : nil
           version ||= requirement["version"].split("#").last
           child_dependencies = if depth >= PACKAGE_LOCK_JSON_MAX_DEPTH
-            []
-          else
-            parse_package_lock_deps_recursively(requirement.fetch("dependencies", []), depth + 1)
+                                 []
+                               else
+                                 parse_package_lock_deps_recursively(requirement.fetch("dependencies", []), source, depth + 1)
                                end
 
           [Dependency.new(
             name: name,
             requirement: version,
             type: type,
+            source: source
           )] + child_dependencies
         end
       end
 
-      def self.parse_manifest(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_manifest(file_contents, options: {})
+        # on ruby 3.2 we suddenly get this JSON error, so detect and return early: "package.json: unexpected token at ''"
+        return [] if file_contents.empty?
+
         manifest = JSON.parse(file_contents)
+
         raise "appears to be a lockfile rather than manifest format" if manifest.key?("lockfileVersion")
 
         dependencies = manifest.fetch("dependencies", [])
@@ -109,7 +117,8 @@ module Bibliothecary
               name: name,
               requirement: requirement,
               type: "runtime",
-              local: requirement.start_with?("file:")
+              local: requirement.start_with?("file:"),
+              source: options.fetch(:filename, nil)
             )
           end
 
@@ -120,38 +129,91 @@ module Bibliothecary
               name: name,
               requirement: requirement,
               type: "development",
-              local: requirement.start_with?("file:")
+              local: requirement.start_with?("file:"),
+              source: options.fetch(:filename, nil)
             )
           end
 
         dependencies
       end
 
-      def self.parse_yarn_lock(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
-        response = Typhoeus.post("#{Bibliothecary.configuration.yarn_parser_host}/parse", body: file_contents)
+      def self.parse_yarn_lock(file_contents, options: {})
+        dep_hash = if file_contents.match(/__metadata:/)
+                     parse_v2_yarn_lock(file_contents, options.fetch(:filename, nil))
+                   else
+                     parse_v1_yarn_lock(file_contents, options.fetch(:filename, nil))
+                   end
+
+        dep_hash.map do |dep|
+          Dependency.new(
+            name: dep[:name],
+            requirement: dep[:version],
+            type: "runtime", # lockfile doesn't tell us more about the type of dep
+            local: dep[:requirements]&.first&.start_with?("file:"),
+            source: options.fetch(:filename, nil)
+          )
+        end
+      end
 
-        raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.yarn_parser_host}/parse", response.response_code) unless response.success?
+      # Returns a hash representation of the deps in yarn.lock, eg:
+      # [{
+      #   name: "foo",
+      #   requirements: [["foo", "^1.0.0"], ["foo", "^1.0.1"]],
+      #   version: "1.2.0",
+      # }, ...]
+      def self.parse_v1_yarn_lock(contents, source = nil)
+        contents
+          .gsub(/^#.*/, "")
+          .strip
+          .split("\n\n")
+          .map do |chunk|
+            requirements = chunk
+              .lines
+              .find { |l| !l.start_with?(" ") && l.strip.end_with?(":") } # first line, eg: '"@bar/foo@1.0.0", "@bar/foo@^1.0.1":'
+              .strip
+              .gsub(/"|:$/, "") # don't need quotes or trailing colon
+              .split(",") # split the list of requirements
+              .map { |d| d.strip.split(/(?<!^)@/, 2) } # split each requirement on name/version "@"", not on leading namespace "@"
+            version = chunk.match(/version "?([^"]*)"?/)[1]
+
+            {
+              name: requirements.first.first,
+              requirements: requirements.map { |x| x[1] },
+              version: version,
+              source: source,
+            }
+          end
+      end
 
-        json = JSON.parse(response.body, symbolize_names: true)
-        json
-          .uniq
-          .reject do |dep|
-            dep[:requirement]&.include?("workspace") && dep[:version].include?("use.local")
+      def self.parse_v2_yarn_lock(contents, source = nil)
+        parsed = YAML.load(contents)
+        parsed = parsed.except("__metadata")
+        parsed
+          .reject do |packages, info|
+            # yarn v4+ creates a lockfile entry: "myproject@workspace" with a "use.local" version
+            #   this lockfile entry is a reference to the project to which the lockfile belongs
+            # skip this self-referential package
+            info["version"].to_s.include?("use.local") && packages.include?("workspace")
           end
-          .map do |dep|
-            Dependency.new(
-              name: dep[:name],
-              requirement: dep[:version],
-              type: dep[:type],
-              local: dep[:requirement]&.start_with?("file:"),
-            )
+          .map do |packages, info|
+            packages = packages.split(", ")
+            # use first requirement's name, assuming that deps will always resolve from deps of the same name
+            name = packages.first.rpartition("@").first
+            requirements = packages.map { |p| p.rpartition("@").last.gsub(/^.*:/, "") }
+
+            {
+              name: name,
+              requirements: requirements,
+              version: info["version"].to_s,
+              source: source,
+            }
           end
       end
 
-      def self.parse_ls(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_ls(file_contents, options: {})
         manifest = JSON.parse(file_contents)
 
-        transform_tree_to_array(manifest.fetch("dependencies", {}))
+        transform_tree_to_array(manifest.fetch("dependencies", {}), options.fetch(:filename, nil))
       end
 
       def self.lockfile_preference_order(file_infos)
@@ -166,15 +228,16 @@ module Bibliothecary
         end
       end
 
-      private_class_method def self.transform_tree_to_array(deps_by_name)
+      private_class_method def self.transform_tree_to_array(deps_by_name, source = nil)
         deps_by_name.map do |name, metadata|
           [
             Dependency.new(
               name: name,
               requirement: metadata["version"],
               type: "runtime",
+              source: source
             ),
-          ] + transform_tree_to_array(metadata.fetch("dependencies", {}))
+          ] + transform_tree_to_array(metadata.fetch("dependencies", {}), source)
         end.flatten(1)
       end
     end

data/lib/bibliothecary/parsers/nuget.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "ox"
 require "json"
 
@@ -48,23 +50,24 @@ module Bibliothecary
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
       add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
-      def self.parse_project_lock_json(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_project_lock_json(file_contents, options: {})
         manifest = JSON.parse file_contents
-        manifest.fetch("libraries",[]).map do |name, _requirement|
+        manifest.fetch("libraries", []).map do |name, _requirement|
           dep = name.split("/")
           Dependency.new(
             name: dep[0],
             requirement: dep[1],
             type: "runtime",
+            source: options.fetch(:filename, nil)
           )
         end
       end
 
-      def self.parse_packages_lock_json(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_packages_lock_json(file_contents, options: {})
         manifest = JSON.parse file_contents
 
         frameworks = {}
-        manifest.fetch("dependencies",[]).each do |framework, deps|
+        manifest.fetch("dependencies", []).each do |framework, deps|
           frameworks[framework] = deps
             .reject { |_name, details| details["type"] == "Project" } # Projects do not have versions
             .map do |name, details|
@@ -74,44 +77,46 @@ module Bibliothecary
                 # so fallback to requested is pure paranoia
                 requirement: details.fetch("resolved", details.fetch("requested", "*")),
                 type: "runtime",
+                source: options.fetch(:filename, nil)
               )
             end
         end
 
-        if frameworks.size > 0
+        unless frameworks.empty?
           # we should really return multiple manifests, but bibliothecary doesn't
           # do that yet so at least pick deterministically.
 
           # Note, frameworks can be empty, so remove empty ones and then return the last sorted item if any
           frameworks = frameworks.delete_if { |_k, v| v.empty? }
-          return frameworks[frameworks.keys.sort.last] unless frameworks.empty?
+          return frameworks[frameworks.keys.max] unless frameworks.empty?
         end
         []
       end
 
-      def self.parse_packages_config(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_packages_config(file_contents, options: {})
         manifest = Ox.parse file_contents
         manifest.packages.locate("package").map do |dependency|
           Dependency.new(
             name: dependency.id,
             requirement: (dependency.version if dependency.respond_to? "version"),
             type: dependency.respond_to?("developmentDependency") && dependency.developmentDependency == "true" ? "development" : "runtime",
+            source: options.fetch(:filename, nil)
           )
         end
-      rescue
+      rescue StandardError
         []
       end
 
-      def self.parse_csproj(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_csproj(file_contents, options: {})
         manifest = Ox.parse file_contents
 
-        packages = manifest.locate("ItemGroup/PackageReference").select{ |dep| dep.respond_to? "Include" }.map do |dependency|
+        packages = manifest.locate("ItemGroup/PackageReference").select { |dep| dep.respond_to? "Include" }.map do |dependency|
           requirement = (dependency.Version if dependency.respond_to? "Version")
           if requirement.is_a?(Ox::Element)
-            requirement = dependency.nodes.detect{ |n| n.value == "Version" }&.text
+            requirement = dependency.nodes.detect { |n| n.value == "Version" }&.text
           end
 
-          type = if dependency.nodes.first && dependency.nodes.first.nodes.include?("all") && dependency.nodes.first.value.include?("PrivateAssets") || dependency.attributes[:PrivateAssets] == "All"
+          type = if (dependency.nodes.first&.nodes&.include?("all") && dependency.nodes.first.value.include?("PrivateAssets")) || dependency.attributes[:PrivateAssets] == "All"
                    "development"
                  else
                    "runtime"
@@ -121,27 +126,29 @@ module Bibliothecary
             name: dependency.Include,
             requirement: requirement,
             type: type,
+            source: options.fetch(:filename, nil)
           )
         end
-        packages.uniq {|package| package.name }
-      rescue
+        packages.uniq(&:name)
+      rescue StandardError
         []
       end
 
-      def self.parse_nuspec(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_nuspec(file_contents, options: {})
         manifest = Ox.parse file_contents
         manifest.package.metadata.dependencies.locate("dependency").map do |dependency|
           Dependency.new(
             name: dependency.id,
             requirement: dependency.attributes[:version],
             type: dependency.respond_to?("developmentDependency") && dependency.developmentDependency == "true" ? "development" : "runtime",
+            source: options.fetch(:filename, nil)
           )
         end
-      rescue
+      rescue StandardError
         []
       end
 
-      def self.parse_paket_lock(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_paket_lock(file_contents, options: {})
         lines = file_contents.split("\n")
         package_version_re = /\s+(?<name>\S+)\s\((?<version>\d+\.\d+[\.\d+[\.\d+]*]*)\)/
         packages = lines.select { |line| package_version_re.match(line) }.map { |line| package_version_re.match(line) }.map do |match|
@@ -149,36 +156,38 @@ module Bibliothecary
             name: match[:name].strip,
             requirement: match[:version],
             type: "runtime",
+            source: options.fetch(:filename, nil)
           )
         end
         # we only have to enforce uniqueness by name because paket ensures that there is only the single version globally in the project
-        packages.uniq {|package| package.name }
+        packages.uniq(&:name)
       end
 
-      def self.parse_project_assets_json(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_project_assets_json(file_contents, options: {})
         manifest = JSON.parse file_contents
 
         frameworks = {}
-        manifest.fetch("targets",[]).each do |framework, deps|
+        manifest.fetch("targets", []).each do |framework, deps|
           frameworks[framework] = deps
             .select { |_name, details| details["type"] == "package" }
             .map do |name, _details|
-            name_split = name.split("/")
-            Dependency.new(
-              name: name_split[0],
-              requirement: name_split[1],
-              type: "runtime",
-            )
+              name_split = name.split("/")
+              Dependency.new(
+                name: name_split[0],
+                requirement: name_split[1],
+                type: "runtime",
+                source: options.fetch(:filename, nil)
+              )
             end
         end
 
-        if frameworks.size > 0
+        unless frameworks.empty?
           # we should really return multiple manifests, but bibliothecary doesn't
           # do that yet so at least pick deterministically.
 
           # Note, frameworks can be empty, so remove empty ones and then return the last sorted item if any
           frameworks = frameworks.delete_if { |_k, v| v.empty? }
-          return frameworks[frameworks.keys.sort.last] unless frameworks.empty?
+          return frameworks[frameworks.keys.max] unless frameworks.empty?
         end
         []
       end

data/lib/bibliothecary/parsers/packagist.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "json"
 
 module Bibliothecary
@@ -22,39 +24,47 @@ module Bibliothecary
       add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
       add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
-      def self.parse_lockfile(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_lockfile(file_contents, options: {})
         manifest = JSON.parse file_contents
-        manifest.fetch("packages",[]).map do |dependency|
+        manifest.fetch("packages", []).map do |dependency|
           requirement = dependency["version"]
 
           # Store Drupal version if Drupal, but include the original manifest version for reference
-          original_requirement, requirement = requirement, dependency.dig("source", "reference") if is_drupal_module(dependency)
+          if drupal_module?(dependency)
+            original_requirement = requirement
+            requirement = dependency.dig("source", "reference")
+          end
 
           Dependency.new(
             name: dependency["name"],
             requirement: requirement,
             type: "runtime",
-            original_requirement: original_requirement
+            original_requirement: original_requirement,
+            source: options.fetch(:filename, nil)
           )
-        end + manifest.fetch("packages-dev",[]).map do |dependency|
+        end + manifest.fetch("packages-dev", []).map do |dependency|
           requirement = dependency["version"]
 
           # Store Drupal version if Drupal, but include the original manifest version for reference
-          original_requirement, requirement = requirement, dependency.dig("source", "reference") if is_drupal_module(dependency)
+          if drupal_module?(dependency)
+            original_requirement = requirement
+            requirement = dependency.dig("source", "reference")
+          end
 
           Dependency.new(
             name: dependency["name"],
             requirement: requirement,
             type: "development",
-            original_requirement: original_requirement
+            original_requirement: original_requirement,
+            source: options.fetch(:filename, nil)
           )
         end
       end
 
-      def self.parse_manifest(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
+      def self.parse_manifest(file_contents, options: {})
         manifest = JSON.parse file_contents
-        map_dependencies(manifest, "require", "runtime") +
-        map_dependencies(manifest, "require-dev", "development")
+        map_dependencies(manifest, "require", "runtime", options.fetch(:filename, nil)) +
+          map_dependencies(manifest, "require-dev", "development", options.fetch(:filename, nil))
       end
 
       # Drupal hosts its own Composer repository, where its "modules" are indexed and searchable. The best way to
@@ -65,7 +75,7 @@ module Bibliothecary
       # (https://www.drupal.org/project/project_composer/issues/2622450),
       # so we return the Drupal requirement instead of semver requirement if it's here
       # (https://www.drupal.org/docs/develop/using-composer/using-composer-to-install-drupal-and-manage-dependencies#s-about-semantic-versioning)
-      private_class_method def self.is_drupal_module(dependency)
+      private_class_method def self.drupal_module?(dependency)
         dependency["type"] =~ /drupal/ && dependency.dig("source", "reference")
       end
     end

data/lib/bibliothecary/parsers/pub.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "yaml"
 
 module Bibliothecary
@@ -23,7 +25,7 @@ module Bibliothecary
       def self.parse_yaml_manifest(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
         manifest = YAML.load file_contents
         map_dependencies(manifest, "dependencies", "runtime") +
-        map_dependencies(manifest, "dev_dependencies", "development")
+          map_dependencies(manifest, "dev_dependencies", "development")
       end
 
       def self.parse_yaml_lockfile(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
@@ -32,7 +34,7 @@ module Bibliothecary
           Dependency.new(
             name: name,
             requirement: dep["version"],
-            type: "runtime",
+            type: "runtime"
           )
         end
       end