ecosystems-bibliothecary 14.3.0 → 15.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c529c2cc8f2f35098f5bbca1294b7a36249580d6ef44a735b00257422dfe568c
-  data.tar.gz: 2680b1d61e665b2bef8ef5b987f3b58b29559266549e3cd68ccdaa8a65d2c4b0
+  metadata.gz: 508531974284b48df7e084f6d0c7c3a4762f7cf5f1af7ed624c86f469a413bfb
+  data.tar.gz: 2442b323ba7169decede7da3c68284402440613dd1091fec7ff1dee8c5bdbe56
 SHA512:
-  metadata.gz: 7518c86a4817297d41c9bbab194901de59a0ca6f07dcad90a9b7d7a78584042d08f9765fc21080475767d399fc6e59f274f8e0132f6673d5e6f7dc2e7bc24908
-  data.tar.gz: e916a1d6f2c4fba1f908c8abc0717013ca21d870f7ddea1c0257c579c3de619037e6cf01ab0eb6f57be2c8191482655267b0765020d19cf8d5da7597dbf56ba8
+  metadata.gz: bf3b5d65d9d02cbafa1e86f04652cdb7f00ffcefd04f55410850bd904e4e1f1e09bf95cd4893270cd93b5219af6fe14c1cc4e7954fe21bd1deb0979b450a37b4
+  data.tar.gz: 24d881e490fcd2dc28647a3936015d552ded40a0567a11f28fea160e277a3b70f736d9a640b3f3773d7aec5a6c8622d8a10349b2d0c820a0338d31cce9d0ba07

data/CHANGELOG.md

@@ -13,6 +13,44 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Removed
 
+## [15.0.1]
+
+### Changed
+
+- Fixed Gemfile parser incorrectly prepending "=" to version requirements that already have operators (e.g., `~> 7.0` was becoming `= ~> 7.0`)
+
+## [15.0.0]
+
+### Added
+
+- Conan parser: conanfile.py, conanfile.txt, conan.lock
+- vcpkg lockfile support: _generated-vcpkg-list.json
+- vcpkg improvements: overrides support, dev dependency detection (host: true)
+
+### Changed
+
+- NuGet packages.lock.json now returns dependencies from all target frameworks instead of arbitrarily picking one
+- Optimized Maven text parsers: lazy ANSI stripping, skip newline normalization when not needed (10-20% faster)
+- Optimized yarn.lock v1 parser with lazy newline normalization (16% faster)
+- Optimized requirements.txt parser with each_line iteration and cached source lookup (10% faster)
+
+### Removed
+
+- SPDX parser and support for *.spdx, *.spdx.json files
+- CycloneDX parser and support for cyclonedx.xml, cyclonedx.json, *.cdx.xml, *.cdx.json files
+- DependenciesCSV multi_parser and support for dependencies.csv files
+- packageurl-ruby dependency
+- Multi-parser infrastructure (add_multi_parser, MultiManifestFilter)
+
+## [14.4.0]
+
+### Changed
+
+- Switched Cargo.lock, poetry.lock, uv.lock, Gopkg.lock, and pylock.toml parsers from full TOML parsing to regex-based parsing for 50-250x faster lockfile parsing on these formats.
+- Switched Gemfile.lock parser from Bundler::LockfileParser to regex-based parsing for 6x faster parsing.
+- Switched Podfile.lock parser from YAML to regex-based parsing for 5x faster parsing.
+- Switched yarn.lock v2+ parser from YAML to regex-based parsing for 14x faster parsing.
+
 ## [14.3.0]
 
 ### Added

data/README.md

@@ -4,8 +4,6 @@ Dependency manifest parsing library for https://github.com/ecosyste-ms
 
 This is a maintained fork of the original [Bibliothecary](https://github.com/librariesio/bibliothecary) gem, with support for additional manifest formats and bug fixes.
 
-[![license](https://img.shields.io/github/license/ecosyste-ms/bibliothecary.svg)](https://github.com/ecosyste-ms/bibliothecary/blob/master/LICENSE.txt)
-
 ## Installation
 
 Requires Ruby 3.4 or above.
@@ -18,7 +16,9 @@ gem "ecosystems-bibliothecary", git: "https://github.com/ecosyste-ms/bibliotheca
 
 And then execute:
 
-    $ bundle install
+```shell
+bundle install
+```
 
 ## Usage
 
@@ -40,14 +40,6 @@ Search a directory for manifest files and parse the contents:
 Bibliothecary.analyse('./')
 ```
 
-There are a number of parsers that rely on web services to parse the file formats, those urls can be configured like so:
-
-```ruby
-Bibliothecary.configure do |config|
-  config.carthage_parser_host = 'http://my-carthage-parsing-service.com'
-end
-```
-
 All available config options are in: https://github.com/ecosyste-ms/bibliothecary/blob/master/lib/bibliothecary/configuration.rb
 
 ## Supported package manager file formats
@@ -103,18 +95,6 @@ All available config options are in: https://github.com/ecosyste-ms/bibliothecar
   - paket.lock
   - *.csproj
   - project.assets.json
-- CycloneDX
-  - cyclonedx.xml
-  - cyclonedx.json
-  - *.cdx.xml
-  - *.cdx.json
-  - Note that CycloneDX manifests can contain information on multiple
-    package manager's packages!
-- SPDX
-  - tag:value as *.spdx
-  - JSON as *.spdx.json
-  - Note that SPDX manifests can contain information on multiple
-    package manager's packages!
 - Bower
   - bower.json
 - BentoML
@@ -134,6 +114,10 @@ All available config options are in: https://github.com/ecosyste-ms/bibliothecar
   - project.clj
 - Cog
   - cog.yaml
+- Conan
+  - conanfile.py
+  - conanfile.txt
+  - conan.lock
 - Meteor
   - versions.json
 - MLflow
@@ -198,6 +182,7 @@ All available config options are in: https://github.com/ecosyste-ms/bibliothecar
   - dvc.yaml
 - Vcpkg
   - vcpkg.json
+  - _generated-vcpkg-list.json
 - Homebrew
   - Brewfile
   - Brewfile.lock.json

data/bibliothecary.gemspec

@@ -16,23 +16,19 @@ Gem::Specification.new do |spec|
   spec.homepage      = "https://github.com/ecosyste-ms/bibliothecary"
   spec.license       = "AGPL-3.0"
 
-  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features|\.github)/|^bin/(benchmark|console|setup)|^\.|^(Gemfile|Rakefile|CODE_OF_CONDUCT)})
+  end
   spec.bindir        = "bin"
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.executables   = %w[bibliothecary]
   spec.require_paths = ["lib"]
 
   spec.add_dependency "bundler"
-  spec.add_dependency "commander"
   spec.add_dependency "csv"
-  spec.add_dependency "deb_control"
   spec.add_dependency "json", "~> 2.8"
-  spec.add_dependency "librariesio-gem-parser"
   spec.add_dependency "ox", ">= 2.8.1"
-  spec.add_dependency "packageurl-ruby"
-  spec.add_dependency "racc"
-  spec.add_dependency "sdl4r"
+  spec.add_dependency "racc" # required by tomlrb but not declared as a dependency
   spec.add_dependency "tomlrb", "~> 2.0"
-  spec.add_dependency "typhoeus"
 
   spec.metadata["rubygems_mfa_required"] = "true"
 end

data/lib/bibliothecary/analyser.rb

@@ -38,18 +38,6 @@ module Bibliothecary
       base.extend(Bibliothecary::Analyser::Analysis)
     end
 
-    module TryCache
-      def try_cache(options, key)
-        if options[:cache]
-          options[:cache][key] ||= yield
-
-          options[:cache][key]
-        else
-          yield
-        end
-      end
-    end
-
     module ClassMethods
       def platform_name
         @platform_name ||= name.to_s.split("::").last.downcase.freeze
@@ -66,25 +54,6 @@ module Bibliothecary
           )
         end
       end
-
-      # Add a MultiParser module to a Parser class. This extends the
-      # self.mapping method on the parser to include the multi parser's
-      # files to watch for, and it extends the Parser class with
-      # the multi parser for you.
-      #
-      # @param klass [Class] A Bibliothecary::MultiParsers class
-      def add_multi_parser(klass)
-        raise "No mapping found! You should place the add_multi_parser call below def self.mapping." unless respond_to?(:mapping)
-
-        original_mapping = mapping
-
-        singleton_class.remove_method(:mapping)
-        define_singleton_method(:mapping) do
-          original_mapping.merge(klass.mapping)
-        end
-
-        send(:extend, klass)
-      end
     end
   end
 end

data/lib/bibliothecary/cli.rb

@@ -2,39 +2,48 @@
 
 require "bibliothecary/version"
 require "bibliothecary"
-require "commander"
+require "optparse"
 
 module Bibliothecary
   class CLI
-    include Commander::Methods
-
     def run
-      program :name, "Bibliothecary"
-      program :version, Bibliothecary::VERSION
-      program :description, "Parse dependency information from a file or folder of code"
-
-      command(:list) do |c|
-        c.syntax = "bibliothecary list"
-        c.description = "List dependencies"
-        c.option("--path FILENAME", String, "Path to file/folder to analyse")
-        c.action do |_args, options|
-          options.default path: "./"
-          output = Bibliothecary.analyse(options.path)
-          output.each do |file_contents|
-            puts "#{file_contents[:path]} (#{file_contents[:platform]})"
-            file_contents[:dependencies].group_by { |d| d[:type] }.each do |type, deps|
-              puts "  #{type}"
-              deps.each do |dep|
-                puts "    #{dep[:name]} #{dep[:requirement]}"
-              end
-              puts
-            end
-            puts
-          end
+      options = { path: "./" }
+
+      parser = OptionParser.new do |opts|
+        opts.banner = "Usage: bibliothecary [options]"
+        opts.separator ""
+        opts.separator "Parse dependency information from a file or folder of code"
+        opts.separator ""
+
+        opts.on("-p", "--path PATH", "Path to file/folder to analyse (default: ./)") do |path|
+          options[:path] = path
+        end
+
+        opts.on("-v", "--version", "Show version") do
+          puts Bibliothecary::VERSION
+          exit
+        end
+
+        opts.on("-h", "--help", "Show this help") do
+          puts opts
+          exit
         end
       end
 
-      run!
+      parser.parse!
+
+      output = Bibliothecary.analyse(options[:path])
+      output.each do |file_contents|
+        puts "#{file_contents[:path]} (#{file_contents[:platform]})"
+        file_contents[:dependencies].group_by { |d| d[:type] }.each do |type, deps|
+          puts "  #{type}"
+          deps.each do |dep|
+            puts "    #{dep[:name]} #{dep[:requirement]}"
+          end
+          puts
+        end
+        puts
+      end
     end
   end
 end

data/lib/bibliothecary/configuration.rb

@@ -2,16 +2,11 @@
 
 module Bibliothecary
   class Configuration
-    attr_accessor :ignored_dirs, :ignored_files, :carthage_parser_host, :clojars_parser_host, :mix_parser_host, :conda_parser_host, :swift_parser_host, :cabal_parser_host
+    attr_accessor :ignored_dirs, :ignored_files
 
     def initialize
       @ignored_dirs = [".git", "node_modules", "bower_components", "vendor", "dist"]
       @ignored_files = []
-      @carthage_parser_host = "https://carthage.libraries.io"
-      @clojars_parser_host  = "https://clojars.libraries.io"
-      @mix_parser_host      = "https://mix.libraries.io"
-      @swift_parser_host    = "http://swift.libraries.io"
-      @cabal_parser_host    = "http://cabal.libraries.io"
     end
   end
 end

data/lib/bibliothecary/dependency.rb

@@ -5,10 +5,7 @@ module Bibliothecary
   #
   # @attr_reader [String] name The name of the package, e.g. "ansi-string-colors"
   # @attr_reader [String] requirement The version requirement of the release, e.g. "1.0.0" or "^1.0.0"
-  # @attr_reader [String] platform The platform of the package, e.g. "maven". This is optional because
-  #   it's implicit in most parser results, and the analyzer returns the platform name itself. One
-  #   exception are multi-parsers like DependenciesCSV, because they may return deps from multiple platforms.
-  #   Bibliothecary could start returning this field for *all* deps in future, and make it required. (default: nil)
+  # @attr_reader [String] platform The platform of the package, e.g. "maven".
   # @attr_reader [String] type The type or scope of dependency, e.g. "runtime" or "test". In some ecosystems a
   #   default may be set and in other ecosystems it may make sense to return nil when not found.
   # @attr_reader [Boolean] direct Is this dependency a direct dependency (vs transitive dependency)? (default: nil)

data/lib/bibliothecary/parsers/bentoml.rb

@@ -15,8 +15,6 @@ module Bibliothecary
         }
       end
 
-      add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
-      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_bentofile(file_contents, options: {})
         source = options.fetch(:filename, 'bentofile.yaml')

data/lib/bibliothecary/parsers/bower.rb

@@ -16,7 +16,6 @@ module Bibliothecary
         }
       end
 
-      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_manifest(file_contents, options: {})
         json = JSON.parse(file_contents)

data/lib/bibliothecary/parsers/cargo.rb

@@ -18,9 +18,6 @@ module Bibliothecary
         }
       end
 
-      add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
-      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
-      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_manifest(file_contents, options: {})
         manifest = Tomlrb.parse(file_contents)
@@ -48,19 +45,24 @@ module Bibliothecary
       end
 
       def self.parse_lockfile(file_contents, options: {})
-        manifest = Tomlrb.parse(file_contents)
-        dependencies = manifest.fetch("package", []).map do |dependency|
-          next if !dependency["source"] || !dependency["source"].start_with?("registry+")
+        dependencies = []
+        # Split into [[package]] blocks and extract fields from each
+        file_contents.split(/\[\[package\]\]/).drop(1).each do |block|
+          name = block[/name\s*=\s*"([^"]+)"/, 1]
+          version = block[/version\s*=\s*"([^"]+)"/, 1]
+          source = block[/source\s*=\s*"([^"]+)"/, 1]
+
+          # Skip packages without a registry source (local/workspace packages)
+          next unless source&.start_with?("registry+")
 
-          Dependency.new(
-            name: dependency["name"],
-            requirement: dependency["version"],
+          dependencies << Dependency.new(
+            name: name,
+            requirement: version,
             type: "runtime",
             source: options.fetch(:filename, nil),
             platform: platform_name
           )
         end
-          .compact
         ParserResult.new(dependencies: dependencies)
       end
     end

data/lib/bibliothecary/parsers/carthage.rb

@@ -3,6 +3,18 @@ module Bibliothecary
     class Carthage
       include Bibliothecary::Analyser
 
+      # Matches Cartfile entries:
+      # github "owner/repo" >= 1.0
+      # github "owner/repo" "branch"
+      # github "owner/repo"
+      # git "url" "ref"
+      # binary "url" >= 1.0
+      # Group 1: source type (github, git, binary)
+      # Group 2: identifier (owner/repo or URL)
+      # Group 3: quoted version/branch
+      # Group 4: unquoted requirement (e.g., >= 1.0, ~> 2.0)
+      CARTFILE_REGEXP = /^(github|git|binary)\s+"([^"]+)"(?:\s+(?:"([^"]+)"|((?:>=|<=|~>|==|>|<)\s*[\d.]+)))?/
+
       def self.mapping
         {
           match_filename("Cartfile") => {
@@ -20,36 +32,60 @@ module Bibliothecary
         }
       end
 
-      add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
-      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_cartfile(file_contents, options: {})
-        map_dependencies(file_contents, "cartfile", options.fetch(:filename, "Cartfile"))
+        parse_cartfile_contents(file_contents, options.fetch(:filename, "Cartfile"), "runtime")
       end
 
       def self.parse_cartfile_private(file_contents, options: {})
-        map_dependencies(file_contents, "cartfile.private", options.fetch(:filename, "Cartfile.private"))
+        parse_cartfile_contents(file_contents, options.fetch(:filename, "Cartfile.private"), "development")
       end
 
       def self.parse_cartfile_resolved(file_contents, options: {})
-        map_dependencies(file_contents, "cartfile.resolved", options.fetch(:filename, "Cartfile.resolved"))
+        parse_cartfile_contents(file_contents, options.fetch(:filename, "Cartfile.resolved"), "runtime")
       end
 
-      def self.map_dependencies(manifest, path, source)
-        response = Typhoeus.post("#{Bibliothecary.configuration.carthage_parser_host}/#{path}", params: {body: manifest}, timeout: 60)
-        raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.carthage_parser_host}/#{path}", response.response_code) unless response.success?
-        json = JSON.parse(response.body)
+      def self.parse_cartfile_contents(contents, source, type)
+        deps = []
+
+        contents.each_line do |line|
+          # Remove inline comments
+          line = line.sub(/#.*$/, "").strip
+          next if line.empty?
+
+          match = line.match(CARTFILE_REGEXP)
+          next unless match
 
-        deps = json.map do |dependency|
-          Bibliothecary::Dependency.new(
+          source_type = match[1]  # github, git, or binary
+          identifier = match[2]   # owner/repo or URL
+          # match[3] is quoted version/branch, match[4] is unquoted requirement
+          version = match[3] || match[4] || "*"
+
+          # For github sources, use identifier as-is (could be owner/repo or full URL)
+          # For git/binary sources, extract repo name from URL
+          name = case source_type
+                 when "github"
+                   # Could be "owner/repo" or a full URL like "https://enterprise.local/..."
+                   if identifier.include?("://")
+                     identifier.split("/").last&.sub(/\.git$/, "") || identifier
+                   else
+                     identifier
+                   end
+                 else
+                   # Extract name from URL (last path component without .git)
+                   identifier.split("/").last&.sub(/\.git$/, "") || identifier
+                 end
+
+          deps << Dependency.new(
             platform: platform_name,
-            name: dependency["name"],
-            requirement: dependency["version"],
-            type: dependency["type"],
+            name: name,
+            requirement: version,
+            type: type,
             source: source
           )
         end
-        Bibliothecary::ParserResult.new(dependencies: deps)
+
+        ParserResult.new(dependencies: deps)
       end
     end
   end

data/lib/bibliothecary/parsers/clojars.rb

@@ -1,11 +1,12 @@
-require "json"
-require "typhoeus"
-
 module Bibliothecary
   module Parsers
     class Clojars
       include Bibliothecary::Analyser
 
+      # Matches individual dependency: [name "version"]
+      # Name can be like: org.clojure/clojure, cheshire, ring/ring-defaults
+      DEPENDENCY_REGEXP = %r{\[([a-zA-Z0-9_./\-]+)\s+"([^"]+)"\]}
+
       def self.mapping
         {
           match_filename("project.clj") => {
@@ -15,31 +16,26 @@ module Bibliothecary
         }
       end
 
-      add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
-      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_manifest(file_contents, options: {})
         source = options.fetch(:filename, "project.clj")
-        response = Typhoeus.post("#{Bibliothecary.configuration.clojars_parser_host}/project.clj", body: file_contents, timeout: 60)
-        raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.clojars_parser_host}/project.clj", response.response_code) unless response.success?
-        json = JSON.parse response.body
-        index = json.index("dependencies")
+        deps = []
 
-        deps = if index
-          dependencies = json[index + 1]
-          dependencies.map do |dependency|
-            Bibliothecary::Dependency.new(
+        # Find the :dependencies section and extract deps
+        # Look for :dependencies followed by a vector of vectors
+        if (deps_section = file_contents[/:dependencies\s*\[.*?\]\]/m])
+          deps_section.scan(DEPENDENCY_REGEXP) do |name, version|
+            deps << Dependency.new(
               platform: platform_name,
-              name: dependency[0],
-              requirement: dependency[1],
+              name: name,
+              requirement: version,
               type: "runtime",
               source: source
             )
           end
-        else
-          []
         end
-        Bibliothecary::ParserResult.new(dependencies: deps)
+
+        ParserResult.new(dependencies: deps)
       end
     end
   end