bibliothecary 11.0.1 → 12.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 78844f8813583a84df76e0ad0c72f9b8136b47d82aec8194e3e6766049ba721b
-  data.tar.gz: bb84e383f0bbded7361d7efbf4550f100d08281a294def960c95ddb1dca4f97c
+  metadata.gz: 12b592879fc7f4fbe3622f495e8ef989b541c81487915b693d02907cedeb62a0
+  data.tar.gz: fd266f3924f202a864017c52ff1c2eb84fd68c4451b16b95e6aa44d80b850598
 SHA512:
-  metadata.gz: f537a3fd368046789baf28c758b8f60d8832b7601f2ae78211a78d4d583ee7ddc4d4729f1ca3387e92958756586a6c1608099a4849db70c9e0a2a3b7405ff564
-  data.tar.gz: 8b9d7e93affe8df9d4e898bf4c128f912f16dd2e1c61ba7dfde0163690df91c173ed0a75c48717a702ddb45dfff2053527b5730c3d76738e7bf51b2344d09cff
+  metadata.gz: e4622e4be1424fd679fac201929bcdf77ed02f0f5782799cf506c93af1d21b803a7e8319e8f471b4747eb5d1a02dba3c0bf8914bae3c4a7a9c95c1b49c37c7b8
+  data.tar.gz: 16950d0fdfcf98cbef1b0faeb61fe1e398816b5499a2fe7af02ea3380975fce4bf85e81a64bfc2d9bb382753f2bdd538e2a46896e8a7e08476716a777adc32aa

data/.circleci/config.yml

@@ -5,7 +5,7 @@ orbs:
 executors:
   bibliothecary:
     docker:
-      - image: cimg/ruby:3.0.7
+      - image: cimg/ruby:3.2.4
     working_directory: ~/bibliothecary
 
 

data/.ruby-version

@@ -1 +1 @@
-3.2.5
+3.2.4

data/CHANGELOG.md

@@ -13,6 +13,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Removed
 
+## [12.0.0] - 2025-01-27
+
+### Removed
+
+- This is a MAJOR release in that it removes support for hackage, carthage, hex, clojar, and swiftpm
+  from Bibliothecary. We are no longer doing any network calls when using Bibliothecary and reimplementing
+  parsing for those file types natively is non-trivial. Patches welcome :-)
+
+### Changed
+
+- Rewrote conda and yarn parsers to be in process vs calling out over the network
 
 ## [11.0.1] - 2024-12-20
 

data/bibliothecary.gemspec

@@ -31,7 +31,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "webmock"
-  spec.add_development_dependency "vcr"
   spec.add_development_dependency "rubocop"
   spec.add_development_dependency "rubocop-rails"
 end

data/lib/bibliothecary/configuration.rb

@@ -5,7 +5,6 @@ module Bibliothecary
     attr_accessor :carthage_parser_host
     attr_accessor :clojars_parser_host
     attr_accessor :mix_parser_host
-    attr_accessor :yarn_parser_host
     attr_accessor :conda_parser_host
     attr_accessor :swift_parser_host
     attr_accessor :cabal_parser_host
@@ -16,8 +15,6 @@ module Bibliothecary
       @carthage_parser_host = "https://carthage.libraries.io"
       @clojars_parser_host  = "https://clojars.libraries.io"
       @mix_parser_host      = "https://mix.libraries.io"
-      @yarn_parser_host     = "https://yarn-parser.libraries.io"
-      @conda_parser_host    = "https://conda-parser.libraries.io"
       @swift_parser_host    = "http://swift.libraries.io"
       @cabal_parser_host    = "http://cabal.libraries.io"
     end

data/lib/bibliothecary/parsers/conda.rb

@@ -1,4 +1,4 @@
-require "json"
+require "yaml"
 
 module Bibliothecary
   module Parsers
@@ -15,14 +15,6 @@ module Bibliothecary
             parser: :parse_conda,
             kind: "manifest",
           },
-          match_filename("environment.yml.lock") => {
-            parser: :parse_conda_lockfile,
-            kind: "lockfile",
-          },
-          match_filename("environment.yaml.lock") => {
-            parser: :parse_conda_lockfile,
-            kind: "lockfile",
-          },
         }
       end
 
@@ -31,34 +23,63 @@ module Bibliothecary
       add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
       def self.parse_conda(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
-        parse_conda_with_kind(file_contents, "manifest")
-      end
+        manifest = YAML.load(file_contents)
+        deps = manifest.dig("dependencies")
+        deps.map do |dep|
+          next unless dep.is_a? String # only deal with strings to skip parsing pip stuff
 
-      def self.parse_conda_lockfile(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
-        parse_conda_with_kind(file_contents, "lockfile")
+          parsed = parse_name_requirement_from_matchspec(dep)
+          Dependency.new(**parsed.merge(type: "runtime"))
+        end.compact
       end
 
-      def self.parse_conda_with_kind(info, kind)
-        dependencies = call_conda_parser_web(info, kind)[kind.to_sym]
-        dependencies.map { |dep_kv| Dependency.new(**dep_kv.merge(type: "runtime")) }
-      end
+      def self.parse_name_requirement_from_matchspec(ms)
+        # simplified version of the implementation in conda to handle what we care about
+        # https://github.com/conda/conda/blob/main/conda/models/match_spec.py#L598
+        # (channel(/subdir):(namespace):)name(version(build))[key1=value1,key2=value2]
+        return if ms.end_with?("@")
 
-      private_class_method def self.call_conda_parser_web(file_contents, kind)
-        host = Bibliothecary.configuration.conda_parser_host
-        response = Typhoeus.post(
-          "#{host}/parse",
-          headers: {
-            ContentType: "multipart/form-data",
-          },
-          body: {
-            file: file_contents,
-            # Unfortunately we do not get the filename in the mapping parsers, so hardcoding the file name depending on the kind
-            filename: kind == "manifest" ? "environment.yml" : "environment.yml.lock",
-          }
-        )
-        raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{host}/parse", response.response_code) unless response.success?
+        # strip off comments and optional features
+        ms = ms.split(/#/, 2).first
+        ms = ms.split(/ if /, 2).first
+
+        # strip off brackets
+        ms = ms.match(/^(.*)(?:\[(.*)\])?$/)[1]
 
-        JSON.parse(response.body, symbolize_names: true)
+        # strip off any parens
+        ms = ms.match(/^(.*)(?:(\(.*\)))?$/)[1]
+
+        # deal with channel and namespace, I wish there was rsplit in ruby
+        split = ms.reverse.split(":", 2)
+        ms = split.last.reverse
+
+        # split the name from the version/build combo
+        matches = ms.match(/([^ =<>!~]+)?([><!=~ ].+)?/)
+        name = matches[1]
+        version_build = matches[2]
+
+        version = nil
+        if matches && matches[2]
+          version_build = matches[2]
+          # and now deal with getting the version from version/build
+          matches = version_build.match(/((?:.+?)[^><!,|]?)(?:(?<![=!|,<>~])(?:[ =])([^-=,|<>~]+?))?$/)
+          version = if matches
+            matches[1].strip
+          else
+            version_build.strip
+          end
+        end
+        # if it's an exact requirement, lose the =
+        if version&.start_with?("==")
+          version = version[2..]
+        elsif version&.start_with?("=")
+            version = version[1..]
+        end
+
+        return {
+          name: name,
+          requirement: version || "", # NOTE: this ignores build info
+        }
       end
     end
   end

data/lib/bibliothecary/parsers/go.rb

@@ -105,7 +105,7 @@ module Bibliothecary
       end
 
       def self.parse_glide_lockfile(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
-        manifest = YAML.load file_contents
+        manifest = YAML.load(file_contents, permitted_classes: [Time])
         map_dependencies(manifest, "imports", "name", "version", "runtime")
       end
 

data/lib/bibliothecary/parsers/npm.rb

@@ -128,25 +128,74 @@ module Bibliothecary
       end
 
       def self.parse_yarn_lock(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
-        response = Typhoeus.post("#{Bibliothecary.configuration.yarn_parser_host}/parse", body: file_contents)
+        dep_hash = if file_contents.match(/__metadata:/)
+          parse_v2_yarn_lock(file_contents)
+        else
+          parse_v1_yarn_lock(file_contents)
+        end
 
-        raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.yarn_parser_host}/parse", response.response_code) unless response.success?
+        dep_hash.map do |dep|
+          Dependency.new(
+            name: dep[:name],
+            requirement: dep[:version],
+            type: "runtime", # lockfile doesn't tell us more about the type of dep
+            local: dep[:requirements]&.first&.start_with?("file:"),
+          )
+        end
+    end
 
-        json = JSON.parse(response.body, symbolize_names: true)
-        json
-          .uniq
-          .reject do |dep|
-            dep[:requirement]&.include?("workspace") && dep[:version].include?("use.local")
-          end
-          .map do |dep|
-            Dependency.new(
-              name: dep[:name],
-              requirement: dep[:version],
-              type: dep[:type],
-              local: dep[:requirement]&.start_with?("file:"),
-            )
-          end
-      end
+    # Returns a hash representation of the deps in yarn.lock, eg:
+    # [{
+    #   name: "foo",
+    #   requirements: [["foo", "^1.0.0"], ["foo", "^1.0.1"]],
+    #   version: "1.2.0",
+    # }, ...]
+    def self.parse_v1_yarn_lock(contents)
+      contents
+        .gsub(/^#.*/, "")
+        .strip
+        .split("\n\n")
+        .map do |chunk|
+          requirements = chunk
+            .lines
+            .find { |l| !l.start_with?(" ") && l.strip.end_with?(":") } # first line, eg: '"@bar/foo@1.0.0", "@bar/foo@^1.0.1":'
+            .strip
+            .gsub(/"|:$/, "") # don't need quotes or trailing colon
+            .split(",") # split the list of requirements
+            .map { |d| d.strip.split(/(?<!^)@/, 2) } # split each requirement on name/version "@"", not on leading namespace "@"
+          version = chunk.match(/version "?([^"]*)"?/)[1]
+
+          {
+            name: requirements.first.first,
+            requirements: requirements.map { |x| x[1] },
+            version: version,
+          }
+        end
+    end
+
+    def self.parse_v2_yarn_lock(contents)
+      parsed = YAML.load(contents)
+      parsed = parsed.except("__metadata")
+      parsed
+        .reject do |packages, info|
+          # yarn v4+ creates a lockfile entry: "myproject@workspace" with a "use.local" version
+          #   this lockfile entry is a reference to the project to which the lockfile belongs
+          # skip this self-referential package
+          info["version"].to_s.include?("use.local") && packages.include?("workspace")
+        end
+        .map do |packages, info|
+          packages = packages.split(", ")
+          # use first requirement's name, assuming that deps will always resolve from deps of the same name
+          name = packages.first.rpartition("@").first
+          requirements = packages.map { |p| p.rpartition("@").last.gsub(/^.*:/, "") }
+
+          {
+            name: name,
+            requirements: requirements,
+            version: info["version"].to_s,
+          }
+        end
+    end      
 
       def self.parse_ls(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
         manifest = JSON.parse(file_contents)

data/lib/bibliothecary/parsers/pypi.rb

@@ -79,14 +79,6 @@ module Bibliothecary
             parser: :parse_conda,
             kind: "manifest",
           },
-          match_filename("environment.yml.lock") => {
-            parser: :parse_conda,
-            kind: "lockfile",
-          },
-          match_filename("environment.yaml.lock") => {
-            parser: :parse_conda,
-            kind: "lockfile",
-          },
         }
       end
 

data/lib/bibliothecary/purl_util.rb

@@ -13,11 +13,8 @@ module Bibliothecary
       "conda" => :conda,
       "cran" => :cran,
       "gem" => :rubygems,
-      "hackage" => :hackage,
-      "hex" => :hex,
       "nuget" => :nuget,
       "pypi" => :pypi,
-      "swift" => :swift_pm,
     }.freeze
 
 

data/lib/bibliothecary/version.rb

@@ -1,3 +1,3 @@
 module Bibliothecary
-  VERSION = "11.0.1"
+  VERSION = "12.0.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bibliothecary
 version: !ruby/object:Gem::Version
-  version: 11.0.1
+  version: 12.0.0
 platform: ruby
 authors:
 - Andrew Nesbitt
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-12-20 00:00:00.000000000 Z
+date: 2025-01-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: tomlrb
@@ -178,20 +178,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: vcr
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
@@ -265,8 +251,6 @@ files:
 - lib/bibliothecary/multi_parsers/spdx.rb
 - lib/bibliothecary/parsers/bower.rb
 - lib/bibliothecary/parsers/cargo.rb
-- lib/bibliothecary/parsers/carthage.rb
-- lib/bibliothecary/parsers/clojars.rb
 - lib/bibliothecary/parsers/cocoapods.rb
 - lib/bibliothecary/parsers/conda.rb
 - lib/bibliothecary/parsers/cpan.rb
@@ -274,9 +258,7 @@ files:
 - lib/bibliothecary/parsers/dub.rb
 - lib/bibliothecary/parsers/elm.rb
 - lib/bibliothecary/parsers/go.rb
-- lib/bibliothecary/parsers/hackage.rb
 - lib/bibliothecary/parsers/haxelib.rb
-- lib/bibliothecary/parsers/hex.rb
 - lib/bibliothecary/parsers/julia.rb
 - lib/bibliothecary/parsers/maven.rb
 - lib/bibliothecary/parsers/meteor.rb
@@ -287,7 +269,6 @@ files:
 - lib/bibliothecary/parsers/pypi.rb
 - lib/bibliothecary/parsers/rubygems.rb
 - lib/bibliothecary/parsers/shard.rb
-- lib/bibliothecary/parsers/swift_pm.rb
 - lib/bibliothecary/purl_util.rb
 - lib/bibliothecary/related_files_info.rb
 - lib/bibliothecary/runner.rb

data/lib/bibliothecary/parsers/carthage.rb

@@ -1,52 +0,0 @@
-module Bibliothecary
-  module Parsers
-    class Carthage
-      include Bibliothecary::Analyser
-
-      def self.mapping
-        {
-          match_filename("Cartfile") => {
-            kind: "manifest",
-            parser: :parse_cartfile,
-          },
-          match_filename("Cartfile.private") => {
-            kind: "manifest",
-            parser: :parse_cartfile_private,
-          },
-          match_filename("Cartfile.resolved") => {
-            kind: "lockfile",
-            parser: :parse_cartfile_resolved,
-          },
-        }
-      end
-
-      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
-
-      def self.parse_cartfile(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
-        map_dependencies(file_contents, "cartfile")
-      end
-
-      def self.parse_cartfile_private(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
-        map_dependencies(file_contents, "cartfile.private")
-      end
-
-      def self.parse_cartfile_resolved(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
-        map_dependencies(file_contents, "cartfile.resolved")
-      end
-
-      def self.map_dependencies(manifest, path)
-        response = Typhoeus.post("#{Bibliothecary.configuration.carthage_parser_host}/#{path}", params: { body: manifest })
-        raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.carthage_parser_host}/#{path}", response.response_code) unless response.success?
-        json = JSON.parse(response.body)
-
-        json.map do |dependency|
-          Dependency.new(
-            name: dependency["name"],
-            requirement: dependency["version"],
-            type: dependency["type"],
-          )
-        end
-      end
-    end
-  end
-end

data/lib/bibliothecary/parsers/clojars.rb

@@ -1,38 +0,0 @@
-require "json"
-require "typhoeus"
-
-module Bibliothecary
-  module Parsers
-    class Clojars
-      include Bibliothecary::Analyser
-
-      def self.mapping
-        {
-          match_filename("project.clj") => {
-            kind: "manifest",
-            parser: :parse_manifest,
-          },
-        }
-      end
-
-      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
-
-      def self.parse_manifest(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
-        response = Typhoeus.post("#{Bibliothecary.configuration.clojars_parser_host}/project.clj", body: file_contents)
-        raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.clojars_parser_host}/project.clj", response.response_code) unless response.success?
-        json = JSON.parse response.body
-        index = json.index("dependencies")
-
-        return [] unless index;
-        dependencies = json[index + 1]
-        dependencies.map do |dependency|
-          Dependency.new(
-            name: dependency[0],
-            requirement: dependency[1],
-            type: "runtime",
-          )
-        end
-      end
-    end
-  end
-end

data/lib/bibliothecary/parsers/hackage.rb

@@ -1,53 +0,0 @@
-require "json"
-require "deb_control"
-
-module Bibliothecary
-  module Parsers
-    class Hackage
-      include Bibliothecary::Analyser
-
-      def self.mapping
-        {
-          match_extension(".cabal") => {
-            kind: "manifest",
-            parser: :parse_cabal,
-          },
-          match_extension("cabal.config") => {
-            kind: "lockfile",
-            parser: :parse_cabal_config,
-          },
-        }
-      end
-
-      add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
-      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
-      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
-
-      def self.parse_cabal(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
-        headers = {
-          "Content-Type" => "text/plain;charset=utf-8",
-        }
-
-        response = Typhoeus.post("#{Bibliothecary.configuration.cabal_parser_host}/parse", headers: headers, body: file_contents)
-
-        raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.cabal_parser_host}/parse", response.response_code) unless response.success?
-        JSON
-          .parse(response.body, symbolize_names: true)
-          .map { |dep_kvs| Dependency.new(**dep_kvs) }
-      end
-
-      def self.parse_cabal_config(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
-        manifest = DebControl::ControlFileBase.parse(file_contents)
-        deps = manifest.first["constraints"].delete("\n").split(",").map(&:strip)
-        deps.map do |dependency|
-          dep = dependency.delete("==").split(" ")
-          Dependency.new(
-            name: dep[0],
-            requirement: dep[1],
-            type: "runtime",
-          )
-        end
-      end
-    end
-  end
-end

data/lib/bibliothecary/parsers/hex.rb

@@ -1,54 +0,0 @@
-require "json"
-
-module Bibliothecary
-  module Parsers
-    class Hex
-      include Bibliothecary::Analyser
-
-      def self.mapping
-        {
-          match_filename("mix.exs") => {
-            kind: "manifest",
-            parser: :parse_mix,
-          },
-          match_filename("mix.lock") => {
-            kind: "lockfile",
-            parser: :parse_mix_lock,
-          },
-        }
-      end
-
-      add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
-      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
-      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
-
-      def self.parse_mix(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
-        response = Typhoeus.post("#{Bibliothecary.configuration.mix_parser_host}/", body: file_contents)
-        raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.mix_parser_host}/", response.response_code) unless response.success?
-        json = JSON.parse response.body
-
-        json.map do |name, version|
-          Dependency.new(
-            name: name,
-            requirement: version,
-            type: "runtime",
-          )
-        end
-      end
-
-      def self.parse_mix_lock(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
-        response = Typhoeus.post("#{Bibliothecary.configuration.mix_parser_host}/lock", body: file_contents)
-        raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.mix_parser_host}/", response.response_code) unless response.success?
-        json = JSON.parse response.body
-
-        json.map do |name, info|
-          Dependency.new(
-            name: name,
-            requirement: info["version"],
-            type: "runtime",
-          )
-        end
-      end
-    end
-  end
-end

data/lib/bibliothecary/parsers/swift_pm.rb

@@ -1,35 +0,0 @@
-module Bibliothecary
-  module Parsers
-    class SwiftPM
-      include Bibliothecary::Analyser
-
-      def self.mapping
-        {
-          match_filename("Package.swift", case_insensitive: true) => {
-            kind: "manifest",
-            parser: :parse_package_swift,
-          },
-        }
-      end
-
-      add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
-      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
-      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
-
-      def self.parse_package_swift(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
-        response = Typhoeus.post("#{Bibliothecary.configuration.swift_parser_host}/to-json", body: file_contents)
-        raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.swift_parser_host}/to-json", response.response_code) unless response.success?
-        json = JSON.parse(response.body)
-        json["dependencies"].map do |dependency|
-          name = dependency["url"].gsub(/^https?:\/\//, "").gsub(/\.git$/,"")
-          version = "#{dependency['version']['lowerBound']} - #{dependency['version']['upperBound']}"
-          Dependency.new(
-            name: name,
-            requirement: version,
-            type: "runtime",
-          )
-        end
-      end
-    end
-  end
-end