bibliothecary 12.1.2 → 12.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1c70a38503e81b9c6d0abd51195077a225e09835ebedd66babbb55dd6742c0b4
-  data.tar.gz: f6c02ba982f9661216fdc1b2e4ebe80d879d278d5583188db9e41664b7f35ae8
+  metadata.gz: bbe2270af80c93aaa2613434faa3c801337ec316a68bdd25d4b6d92d9979398a
+  data.tar.gz: ee84bbf8cb2bd2beae91c080e480fe5fc3f0c2773a305743388cf3f225355eb2
 SHA512:
-  metadata.gz: 7b09e212ea2c5fe442ed23e1008d5e9c4abb50aab2cee4fd8314097566b6b5238af49bf71c83720b9b4ed4ceeb7e014a16b2ee891da5f22e1a8c2ff9f9c20f85
-  data.tar.gz: b2ae2d6f40eb17538b7f0241db8008db6bfd9cb61f52651562458c036dfc004d6356f833015b12ca6471f18a3777f2838c37ea26c64c1ee7059b039d970f8b2a
+  metadata.gz: 0ac214d1af05ecf0f156a81dc751421cbbe1d94c6306420d0aeb87c7dabf96eca819a0b199705d40d408b3a22b73227a523bb06e121c1e60c7dfbbb2f74dff10
+  data.tar.gz: 52d620bcd946c7197f129ebd59f2ba68676e67152dbeac297a4cdd5d72f74e9f3dc32c6c11b11f513443040721ccaa18b2c322e7b77162fb775fa7827fe3edd6

data/CHANGELOG.md

@@ -13,6 +13,51 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Removed
 
+## [12.1.4] - 2025-03-14
+
+### Added
+
+- Add support for PNPM lockfiles (lockfile versions 5, 6, and 9).
+- Add 'parser_options' arg to Bilbiothecary::Runner constructor.
+
+### Changed
+
+### Removed
+
+## [12.1.3] - 2025-02-26
+
+### Added
+
+- Add 'local' property to dependencies from Pipfile and Pipfile.lock
+
+### Changed
+
+- Handle aliases and NPM and Yarn, and ignore patched dependencies.
+- Fix a PyPI parser's regex to exclude false positive "require" names.
+- Drop all sub-projects from list of deps in a Maven maven-dependency-tree.txt.
+
+### Removed
+
+## [12.1.2] - 2025-02-26
+
+### Added
+
+- Add 'local' property to dependencies from Pipfile and Pipfile.lock
+
+### Changed
+
+### Removed
+
+## [12.1.1] - 2025-02-21
+
+### Added
+
+- Add test coverage for Go 1.24's new "tool" directive.
+
+### Changed
+
+### Removed
+
 ## [12.1.0] - 2025-01-30
 
 ### Added

data/Gemfile

@@ -20,5 +20,6 @@ group :test do
   gem "codeclimate-test-reporter", "~> 1.0.0"
   gem "rspec", "~> 3.0"
   gem "simplecov"
+  gem "super_diff", "~> 0.15.0"
   gem "webmock"
 end

data/lib/bibliothecary/parsers/maven.rb

@@ -254,34 +254,89 @@ module Bibliothecary
           .uniq
       end
 
-      def self.parse_maven_tree(file_contents, options: {})
-        captures = file_contents
+      # Return each item in the ascii art tree with a depth of that item,
+      # like [[0, "groupId:artifactId:jar:version:scope"], [1, "..."], ...]
+      # The depth-0 items are the (sub)project names
+      # These are in the original order, with no de-duplication.
+      def self.parse_maven_tree_items_with_depths(file_contents)
+        file_contents
           .gsub(ANSI_MATCHER, "")
           .gsub(/\r\n?/, "\n")
-          .scan(/^\[INFO\](?:(?:\+-)|\||(?:\\-)|\s)+((?:[\w\.-]+:)+[\w\.\-${}]+)/)
-          .flatten
-          .uniq
+          # capture two groups; one is the ASCII art telling us the tree depth,
+          # and two is the actual dependency
+          .scan(/^\[INFO\]\s((?:[-+|\\]|\s)*)((?:[\w\.-]+:)+[\w\.\-${}]+)/)
+          # lines that start with "-" aren't part of the tree, example: "[INFO] --- dependency:3.8.1:tree"
+          .reject { |(tree_ascii_art, _dep_info)| tree_ascii_art.start_with?("-") }
+          .map do |(tree_ascii_art, dep_info)|
+            child_marker_index = tree_ascii_art.index(/(\+-)|(\\-)/)
+            depth = if child_marker_index.nil?
+                      0
+                    else
+                      # There are three characters present in the line for each level of depth
+                      (child_marker_index / 3) + 1
+                    end
+            [depth, dep_info]
+          end
+      end
+
+      # split "org.yaml:snakeyaml:jar:2.2:compile" into
+      # ["org.yaml:snakeyaml", "2.2", "compile"]
+      def self.parse_maven_tree_dependency(item)
+        parts = item.split(":")
+        case parts.count
+        when 4
+          version = parts[-1]
+          type = parts[-2]
+        when 5..6
+          version, type = parts[-2..]
+        end
+
+        name = parts[0..1].join(":")
+
+        [name, version, type]
+      end
+
+      def self.parse_maven_tree(file_contents, options: {})
+        keep_subprojects = options.fetch(:keep_subprojects_in_maven_tree, false)
+
+        items = parse_maven_tree_items_with_depths(file_contents)
+
+        raise "found no lines with deps in maven-dependency-tree.txt" if items.empty?
 
-        deps = captures.map do |item|
-          parts = item.split(":")
-          case parts.count
-          when 4
-            version = parts[-1]
-            type = parts[-2]
-          when 5..6
-            version, type = parts[-2..]
+        projects = {}
+
+        if keep_subprojects
+          # traditional behavior: we only exclude the root project, and only if we parsed multiple lines
+          (root_name, root_version, _root_type) = parse_maven_tree_dependency(items.shift[1])
+          unless items.empty?
+            projects[root_name] = Set.new
+            projects[root_name].add(root_version)
           end
-          Dependency.new(
-            name: parts[0..1].join(":"),
-            requirement: version,
-            type: type,
-            source: options.fetch(:filename, nil)
-          )
         end
 
-        # First dep line will be the package itself (unless we're only analyzing a single line)
-        package = deps[0]
-        deps.size < 2 ? deps : deps[1..].reject { |d| d.name == package.name && d.requirement == package.requirement }
+        unique_items = items.map do |(depth, item)|
+          (name, version, type) = parse_maven_tree_dependency(item)
+          if depth == 0 && !keep_subprojects
+            # record and then remove the depth 0
+            projects[name] ||= Set.new
+            projects[name].add(version)
+            nil
+          else
+            [name, version, type]
+          end
+        end.compact.uniq
+
+        unique_items
+          # drop the projects and subprojects
+          .reject { |(name, version, _type)| projects[name]&.include?(version) }
+          .map do |(name, version, type)|
+            Bibliothecary::Dependency.new(
+              name: name,
+              requirement: version,
+              type: type,
+              source: options.fetch(:filename, nil)
+            )
+          end
       end
 
       def self.parse_resolved_dep_line(line, options: {})

data/lib/bibliothecary/parsers/npm.rb

@@ -16,10 +16,6 @@ module Bibliothecary
             kind: "manifest",
             parser: :parse_manifest,
           },
-          match_filename("npm-shrinkwrap.json") => {
-            kind: "lockfile",
-            parser: :parse_shrinkwrap,
-          },
           match_filename("yarn.lock") => {
             kind: "lockfile",
             parser: :parse_yarn_lock,
@@ -28,10 +24,18 @@ module Bibliothecary
             kind: "lockfile",
             parser: :parse_package_lock,
           },
+          match_filename("pnpm-lock.yaml") => {
+            kind: "lockfile",
+            parser: :parse_pnpm_lock,
+          },
           match_filename("npm-ls.json") => {
             kind: "lockfile",
             parser: :parse_ls,
           },
+          match_filename("npm-shrinkwrap.json") => {
+            kind: "lockfile",
+            parser: :parse_shrinkwrap,
+          },
         }
       end
 
@@ -72,9 +76,22 @@ module Bibliothecary
           #      * The other occurrence's name is the path to the local dependency (which has less information, and is duplicative, so we discard)
           .select { |name, _dep| name.start_with?("node_modules") }
           .map do |name, dep|
+            # check if the name property is available and differs from the node modules location
+            # this indicates that the package has been aliased
+            node_module_name = name.split("node_modules/").last
+            name_property = dep["name"]
+            if !name_property.nil? && node_module_name != name_property
+              name = name_property
+              original_name = node_module_name
+            else
+              name = node_module_name
+            end
+
             Dependency.new(
-              name: name.split("node_modules/").last,
+              name: name,
+              original_name: original_name,
               requirement: dep["version"],
+              original_requirement: original_name.nil? ? nil : dep["version"],
               type: dep.fetch("dev", false) || dep.fetch("devOptional", false) ? "development" : "runtime",
               local: dep.fetch("link", false),
               source: source
@@ -113,9 +130,20 @@ module Bibliothecary
         dependencies = manifest.fetch("dependencies", [])
           .reject { |name, _requirement| name.start_with?("//") } # Omit comment keys. They are valid in package.json: https://groups.google.com/g/nodejs/c/NmL7jdeuw0M/m/yTqI05DRQrIJ
           .map do |name, requirement|
+            # check to see if this is an aliased package name
+            # example: "alias-package-name": "npm:actual-package@^1.1.3"
+            if requirement.include?("npm:")
+              # the name of the real dependency is contained in the requirement with the version
+              requirement.gsub!("npm:", "")
+              original_name = name
+              name, _, requirement = requirement.rpartition("@")
+            end
+
             Dependency.new(
               name: name,
+              original_name: original_name,
               requirement: requirement,
+              original_requirement: original_name.nil? ? nil : requirement,
               type: "runtime",
               local: requirement.start_with?("file:"),
               source: options.fetch(:filename, nil)
@@ -147,7 +175,9 @@ module Bibliothecary
         dep_hash.map do |dep|
           Dependency.new(
             name: dep[:name],
+            original_name: dep[:original_name],
             requirement: dep[:version],
+            original_requirement: dep[:original_requirement],
             type: "runtime", # lockfile doesn't tell us more about the type of dep
             local: dep[:requirements]&.first&.start_with?("file:"),
             source: options.fetch(:filename, nil)
@@ -173,12 +203,17 @@ module Bibliothecary
               .strip
               .gsub(/"|:$/, "") # don't need quotes or trailing colon
               .split(",") # split the list of requirements
-              .map { |d| d.strip.split(/(?<!^)@/, 2) } # split each requirement on name/version "@"", not on leading namespace "@"
+
+            name, alias_name = yarn_strip_npm_protocol(requirements.first) # if a package is aliased, strip the alias and return the real package name
+            name = name.strip.split(/(?<!^)@/).first
+            requirements = requirements.map { |d| d.strip.split(/(?<!^)@/, 2) } # split each requirement on name/version "@"", not on leading namespace "@"
             version = chunk.match(/version "?([^"]*)"?/)[1]
 
             {
-              name: requirements.first.first,
+              name: name,
+              original_name: alias_name,
               requirements: requirements.map { |x| x[1] },
+              original_requirement: alias_name.nil? ? nil : version,
               version: version,
               source: source,
             }
@@ -193,23 +228,89 @@ module Bibliothecary
             # yarn v4+ creates a lockfile entry: "myproject@workspace" with a "use.local" version
             #   this lockfile entry is a reference to the project to which the lockfile belongs
             # skip this self-referential package
-            info["version"].to_s.include?("use.local") && packages.include?("workspace")
+            (info["version"].to_s.include?("use.local") && packages.include?("workspace")) ||
+              # yarn allows users to insert patches to their dependencies from within their project
+              # these patches are marked as a separate entry in the lock file but do not represent a new dependency
+              # and should be skipped here
+              # https://yarnpkg.com/protocol/patch
+              packages.include?("@patch:")
           end
           .map do |packages, info|
             packages = packages.split(", ")
             # use first requirement's name, assuming that deps will always resolve from deps of the same name
-            name = packages.first.rpartition("@").first
+            name, alias_name = yarn_strip_npm_protocol(packages.first.rpartition("@").first)
             requirements = packages.map { |p| p.rpartition("@").last.gsub(/^.*:/, "") }
 
             {
               name: name,
+              original_name: alias_name,
               requirements: requirements,
+              original_requirement: alias_name.nil? ? nil : info["version"].to_s,
               version: info["version"].to_s,
               source: source,
             }
           end
       end
 
+      # This method currently has been tested to support:
+      #   lockfileVersion: '9.0'
+      #   lockfileVersion: '6.0'
+      #   lockfileVersion: '5.4'
+      def self.parse_pnpm_lock(contents, _source = nil)
+        parsed = YAML.load(contents)
+        lockfile_version = parsed["lockfileVersion"].to_i
+
+        dev_dependencies = parsed.dig("importers", ".", "devDependencies") # <= v9
+        dev_dependencies ||= parsed["devDependencies"] # <v9
+
+        # "dependencies" is in "packages" for < v9 and in "snapshots" for >= v9
+        # as of https://github.com/pnpm/pnpm/pull/7700.
+        (parsed["snapshots"] || parsed["packages"])
+          .map do |name_version, details|
+            name, version = case lockfile_version
+                            when 5
+                              # e.g. '/debug/2.6.9:'
+                              n, v = name_version.sub(/^\//, "").split("/", 2)
+                              # e.g. '/debug/2.2.0_supports-color@1.2.0:'
+                              v = v.split("_", 2)[0]
+                              [n, v] # rubocop:disable Style/IdenticalConditionalBranches
+                            when 6
+                              # e.g. '/debug@2.6.9:'
+                              n, v = name_version.sub(/^\//, "").split("@", 2)
+                              # e.g. "debug@2.2.0(supports-color@1.2.0)"
+                              v = v.split("(", 2).first
+                              [n, v] # rubocop:disable Style/IdenticalConditionalBranches
+                            else
+                              # e.g. 'debug@2.6.9:'
+                              n, v = name_version.split("@", 2)
+                              # e.g. "debug@2.2.0(supports-color@1.2.0)"
+                              v = v.split("(", 2).first
+                              [n, v] # rubocop:disable Style/IdenticalConditionalBranches
+                            end
+
+            # TODO: the "dev" field was removed in v9 lockfiles (https://github.com/pnpm/pnpm/pull/7808)
+            # so this will only exist in v6 and below and might be unreliable.
+            # The proper way to set this for v9+ is to build a lookup of deps to
+            # their "dependencies", and then recurse through each package's
+            # parents. If the direct dep(s) that required them are all
+            # "devDependencies" then we can consider them "dev == true". This
+            # should be done using a DAG data structure, though, to be efficient
+            # and avoid cycles.
+            is_dev = details["dev"] == true
+
+            # Fallback for v9+: this only detects dev deps that are direct.
+            is_dev ||= dev_dependencies.any? do |dev_name, dev_details|
+              dev_name == name && dev_details["version"] == version
+            end
+
+            Dependency.new(
+              name: name,
+              requirement: version,
+              type: is_dev ? "development" : "runtime"
+            )
+          end
+      end
+
       def self.parse_ls(file_contents, options: {})
         manifest = JSON.parse(file_contents)
 
@@ -240,6 +341,20 @@ module Bibliothecary
           ] + transform_tree_to_array(metadata.fetch("dependencies", {}), source)
         end.flatten(1)
       end
+
+      # Yarn package names can be aliased by using the NPM protocol. If a package name includes
+      # the NPM protocol then the name following the @npm: protocol identifier is the name of the
+      # actual package being imported into the project under a different alias.
+      # https://classic.yarnpkg.com/lang/en/docs/cli/add/#toc-yarn-add-alias
+      private_class_method def self.yarn_strip_npm_protocol(dep_name)
+        if dep_name.include?("@npm:")
+          partitions = dep_name.rpartition("@npm:")
+          alias_name = partitions.first
+          dep_name = partitions.last
+        end
+
+        [dep_name, alias_name]
+      end
     end
   end
 end

data/lib/bibliothecary/parsers/nuget.rb

@@ -87,7 +87,7 @@ module Bibliothecary
           # do that yet so at least pick deterministically.
 
           # Note, frameworks can be empty, so remove empty ones and then return the last sorted item if any
-          frameworks = frameworks.delete_if { |_k, v| v.empty? }
+          frameworks.delete_if { |_k, v| v.empty? }
           return frameworks[frameworks.keys.max] unless frameworks.empty?
         end
         []
@@ -186,7 +186,7 @@ module Bibliothecary
           # do that yet so at least pick deterministically.
 
           # Note, frameworks can be empty, so remove empty ones and then return the last sorted item if any
-          frameworks = frameworks.delete_if { |_k, v| v.empty? }
+          frameworks.delete_if { |_k, v| v.empty? }
           return frameworks[frameworks.keys.max] unless frameworks.empty?
         end
         []

data/lib/bibliothecary/parsers/pypi.rb

@@ -13,7 +13,7 @@ module Bibliothecary
       REQUIRE_REGEXP = /([a-zA-Z0-9]+[a-zA-Z0-9\-_\.]+)(?:\[.*?\])*([><=\w\.,]+)?/
       REQUIREMENTS_REGEXP = /^#{REQUIRE_REGEXP}/
 
-      MANIFEST_REGEXP = /.*require[^\/]*(\/)?[^\/]*\.(txt|pip|in)$/
+      MANIFEST_REGEXP = /.*require[^\/]*\.(txt|pip|in)$/
       # TODO: can this be a more specific regexp so it doesn't match something like ".yarn/cache/create-require-npm-1.0.0.zip"?
       PIP_COMPILE_REGEXP = /.*require.*$/
 

data/lib/bibliothecary/runner.rb

@@ -5,11 +5,11 @@ module Bibliothecary
   # A runner is created every time a file is targeted to be parsed. Don't call
   # parse methods directory! Use a Runner.
   class Runner
-    def initialize(configuration)
+    def initialize(configuration, parser_options: {})
       @configuration = configuration
       @options = {
         cache: {},
-      }
+      }.merge(parser_options)
     end
 
     def analyse(path, ignore_unparseable_files: true)

data/lib/bibliothecary/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bibliothecary
-  VERSION = "12.1.2"
+  VERSION = "12.1.4"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: bibliothecary
 version: !ruby/object:Gem::Version
-  version: 12.1.2
+  version: 12.1.4
 platform: ruby
 authors:
 - Andrew Nesbitt
 bindir: bin
 cert_chain: []
-date: 2025-02-26 00:00:00.000000000 Z
+date: 2025-03-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: commander