ecosystems-bibliothecary 14.3.0 → 15.0.0

data/lib/bibliothecary/parsers/hackage.rb

@@ -1,11 +1,17 @@
 require "json"
-require "deb_control"
 
 module Bibliothecary
   module Parsers
     class Hackage
       include Bibliothecary::Analyser
 
+      # Matches dependency lines like: aeson == 1.1.* or base >= 4.9 && < 4.11
+      # Package names can contain letters, numbers, and hyphens
+      DEPENDENCY_REGEXP = /^\s*([a-zA-Z][a-zA-Z0-9-]*)\s*((?:[<>=!]+\s*[\d.*]+(?:\s*&&\s*[<>=!]+\s*[\d.*]+)*)?)/
+
+      # Matches build-tool-depends format: package:tool == version
+      BUILD_TOOL_REGEXP = /^\s*([a-zA-Z][a-zA-Z0-9-]*):[a-zA-Z][a-zA-Z0-9-]*\s*((?:[<>=!]+\s*[\d.*]+(?:\s*&&\s*[<>=!]+\s*[\d.*]+)*)?)/
+
       def self.mapping
         {
           match_extension(".cabal") => {
@@ -19,39 +25,140 @@ module Bibliothecary
         }
       end
 
-      add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
-      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
-      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
       def self.parse_cabal(file_contents, options: {})
-        source = options.fetch(:filename, 'package.cabal')
-        headers = {
-          "Content-Type" => "text/plain;charset=utf-8",
-        }
+        source = options.fetch(:filename, "package.cabal")
+        deps = []
+
+        # Track current section type
+        current_section = nil
+        in_build_depends = false
+        in_build_tool_depends = false
+        current_deps_buffer = []
+
+        file_contents.each_line do |line|
+          # Check for section headers (library, executable, test-suite, benchmark)
+          if line =~ /^(library|executable|test-suite|benchmark)\b/i
+            current_section = $1.downcase
+            in_build_depends = false
+            in_build_tool_depends = false
+            next
+          end
+
+          # Check for build-depends: or build-tool-depends: (can be at any indentation level)
+          if line =~ /^\s*build-depends\s*:/i
+            in_build_depends = true
+            in_build_tool_depends = false
+            # Extract deps from same line after colon
+            deps_part = line.sub(/^\s*build-depends\s*:/i, "")
+            parse_deps_line(deps_part, deps, current_section, "build-depends", source)
+            next
+          end
+
+          if line =~ /^\s*build-tool-depends\s*:/i
+            in_build_tool_depends = true
+            in_build_depends = false
+            # Extract deps from same line after colon
+            deps_part = line.sub(/^\s*build-tool-depends\s*:/i, "")
+            parse_deps_line(deps_part, deps, current_section, "build-tool-depends", source)
+            next
+          end
+
+          # Check for other field headers that end depends section
+          # Field headers are like "field-name:" but NOT "package:tool" (build-tool-depends format)
+          # Build-tool-depends entries have format: package:tool version-constraint
+          if line =~ /^\s*([a-z][a-z0-9-]*)\s*:/i
+            field_name = $1
+            # If this looks like a field header (not package:tool), end the depends section
+            unless line =~ /^\s*[a-z][a-z0-9-]*:[a-z][a-z0-9-]*\s+/i
+              in_build_depends = false
+              in_build_tool_depends = false
+              next
+            end
+          end
+
+          # Continue parsing dependencies if in a depends section and line is indented
+          if (in_build_depends || in_build_tool_depends) && line =~ /^\s+/
+            dep_type = in_build_tool_depends ? "build-tool-depends" : "build-depends"
+            parse_deps_line(line, deps, current_section, dep_type, source)
+          elsif line !~ /^\s/
+            # Non-indented line that's not a section header ends depends
+            in_build_depends = false
+            in_build_tool_depends = false
+          end
+        end
+
+        ParserResult.new(dependencies: deps)
+      end
 
-        response = Typhoeus.post("#{Bibliothecary.configuration.cabal_parser_host}/parse", headers: headers, body: file_contents, timeout: 60)
+      def self.parse_deps_line(line, deps, section, dep_type, source)
+        # Split by comma and parse each dep
+        line.split(",").each do |dep_str|
+          dep_str = dep_str.strip
+          next if dep_str.empty?
 
-        raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.cabal_parser_host}/parse", response.response_code) unless response.success?
-        raw_deps = JSON.parse(response.body, symbolize_names: true)
-        deps = raw_deps.map do |dep|
-          Bibliothecary::Dependency.new(
+          # Use different regex for build-tool-depends (package:tool format)
+          regex = dep_type == "build-tool-depends" ? BUILD_TOOL_REGEXP : DEPENDENCY_REGEXP
+          match = dep_str.match(regex)
+          next unless match
+
+          name = match[1]
+          requirement = match[2]&.strip
+          requirement = "*" if requirement.nil? || requirement.empty?
+          # Normalize spacing: "== 1.1.*" -> "==1.1.*", ">= 4.9 && < 4.11" -> ">=4.9 && <4.11"
+          requirement = requirement.gsub(/([<>=!]+)\s+/, '\1').gsub(/\s+(&&)\s+/, ' \1 ')
+
+          # Determine type based on section and dep_type
+          type = determine_dep_type(section, dep_type)
+
+          deps << Dependency.new(
             platform: platform_name,
-            name: dep[:name],
-            requirement: dep[:requirement],
-            type: dep[:type],
+            name: name,
+            requirement: requirement,
+            type: type,
             source: source
           )
         end
-        Bibliothecary::ParserResult.new(dependencies: deps)
+      end
+
+      def self.determine_dep_type(section, dep_type)
+        if dep_type == "build-tool-depends"
+          "build"
+        elsif section == "test-suite"
+          "test"
+        elsif section == "benchmark"
+          "benchmark"
+        else
+          "runtime"
+        end
       end
 
       def self.parse_cabal_config(file_contents, options: {})
-        source = options.fetch(:filename, 'cabal.config')
-        manifest = DebControl::ControlFileBase.parse(file_contents)
-        deps_raw = manifest.first["constraints"].delete("\n").split(",").map(&:strip)
-        deps = deps_raw.map do |dependency|
-          dep = dependency.delete("==").split(" ")
-          Bibliothecary::Dependency.new(
+        source = options.fetch(:filename, "cabal.config")
+        deps = []
+
+        # Parse RFC822-style format: constraints: pkg1 ==1.0, pkg2 ==2.0, ...
+        # Values can span multiple lines (continuation lines start with whitespace)
+        constraints = nil
+        file_contents.each_line do |line|
+          if line =~ /^constraints:\s*(.*)/i
+            constraints = $1.strip
+          elsif line =~ /^\s+(.*)/ && constraints
+            constraints += " " + $1.strip
+          elsif line =~ /^[a-z]/i && constraints
+            break
+          end
+        end
+
+        return ParserResult.new(dependencies: []) unless constraints
+
+        constraints.split(",").each do |dep_str|
+          dep_str = dep_str.strip
+          next if dep_str.empty?
+
+          # Format: package ==version or package ==version.*
+          dep = dep_str.delete("==").split(/\s+/)
+          deps << Dependency.new(
             platform: platform_name,
             name: dep[0],
             requirement: dep[1] || "*",
@@ -59,7 +166,8 @@ module Bibliothecary
             source: source
           )
         end
-        Bibliothecary::ParserResult.new(dependencies: deps)
+
+        ParserResult.new(dependencies: deps)
       end
     end
   end

data/lib/bibliothecary/parsers/haxelib.rb

@@ -6,19 +6,29 @@ module Bibliothecary
   module Parsers
     class Haxelib
       include Bibliothecary::Analyser
-      extend Bibliothecary::MultiParsers::JSONRuntime
 
       def self.mapping
         {
           match_filename("haxelib.json") => {
             kind: "manifest",
-            parser: :parse_json_runtime_manifest,
+            parser: :parse_manifest,
           },
         }
       end
 
-      add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
-      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+      def self.parse_manifest(file_contents, options: {})
+        manifest = JSON.parse(file_contents)
+        dependencies = manifest.fetch("dependencies", {}).map do |name, requirement|
+          Dependency.new(
+            name: name,
+            requirement: requirement,
+            type: "runtime",
+            source: options.fetch(:filename, nil),
+            platform: platform_name
+          )
+        end
+        ParserResult.new(dependencies: dependencies)
+      end
     end
   end
 end

data/lib/bibliothecary/parsers/hex.rb

@@ -5,6 +5,11 @@ module Bibliothecary
     class Hex
       include Bibliothecary::Analyser
 
+      # Matches mix.lock entries: "name": {:hex, :name, "version", ...
+      # or "name": {:git, "url", "ref", ...
+      HEX_LOCK_REGEXP = /"([^"]+)":\s*\{:hex,\s*:[^,]+,\s*"([^"]+)"/
+      GIT_LOCK_REGEXP = /"([^"]+)":\s*\{:git,\s*"([^"]+)",\s*"([^"]+)"/
+
       def self.mapping
         {
           match_filename("mix.exs") => {
@@ -18,44 +23,56 @@ module Bibliothecary
         }
       end
 
-      add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
-      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
-      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
       def self.parse_mix(file_contents, options: {})
-        source = options.fetch(:filename, 'mix.exs')
-        response = Typhoeus.post("#{Bibliothecary.configuration.mix_parser_host}/", body: file_contents, timeout: 60)
-        raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.mix_parser_host}/", response.response_code) unless response.success?
-        json = JSON.parse response.body
+        source = options.fetch(:filename, "mix.exs")
+        deps = []
+
+        # Remove comments before parsing
+        content = file_contents.gsub(/#.*$/, "")
 
-        deps = json.map do |name, version|
-          Bibliothecary::Dependency.new(
+        # Match deps in the dependencies list: {:name, "~> version"} or {:name, ">= version"}
+        # Format: {:dep_name, "requirement"} or {:dep_name, "requirement", opts}
+        content.scan(/\{:(\w+),\s*"([^"]+)"/) do |name, requirement|
+          deps << Dependency.new(
             platform: platform_name,
-            name: name,
-            requirement: version,
+            name: name.to_s,
+            requirement: requirement,
             type: "runtime",
             source: source
           )
         end
-        Bibliothecary::ParserResult.new(dependencies: deps)
+
+        ParserResult.new(dependencies: deps)
       end
 
       def self.parse_mix_lock(file_contents, options: {})
-        source = options.fetch(:filename, 'mix.lock')
-        response = Typhoeus.post("#{Bibliothecary.configuration.mix_parser_host}/lock", body: file_contents, timeout: 60)
-        raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.mix_parser_host}/", response.response_code) unless response.success?
-        json = JSON.parse response.body
+        source = options.fetch(:filename, "mix.lock")
+        deps = []
+
+        # Match hex packages: "name": {:hex, :name, "version", ...
+        file_contents.scan(HEX_LOCK_REGEXP) do |name, version|
+          deps << Dependency.new(
+            platform: platform_name,
+            name: name,
+            requirement: version,
+            type: "runtime",
+            source: source
+          )
+        end
 
-        deps = json.map do |name, info|
-          Bibliothecary::Dependency.new(
+        # Match git packages: "name": {:git, "url", "ref", ...
+        file_contents.scan(GIT_LOCK_REGEXP) do |name, _url, ref|
+          deps << Dependency.new(
             platform: platform_name,
             name: name,
-            requirement: info["version"],
+            requirement: ref,
             type: "runtime",
             source: source
           )
         end
-        Bibliothecary::ParserResult.new(dependencies: deps)
+
+        ParserResult.new(dependencies: deps)
       end
     end
   end

data/lib/bibliothecary/parsers/homebrew.rb

@@ -20,8 +20,6 @@ module Bibliothecary
         }
       end
 
-      add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
-      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_brewfile(file_contents, options: {})
         source = options.fetch(:filename, 'Brewfile')

data/lib/bibliothecary/parsers/julia.rb

@@ -14,8 +14,6 @@ module Bibliothecary
         }
       end
 
-      add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
-      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_require(file_contents, options: {})
         deps = []

data/lib/bibliothecary/parsers/maven.rb

@@ -133,9 +133,6 @@ module Bibliothecary
         }
       end
 
-      add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
-      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
-      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_ivy_manifest(file_contents, options: {})
         manifest = Ox.parse file_contents
@@ -195,15 +192,22 @@ module Bibliothecary
 
       def self.parse_gradle_resolved(file_contents, options: {})
         current_type = nil
+        source = options.fetch(:filename, nil)
+        dependencies = []
 
-        dependencies = file_contents.split("\n").map do |line|
-          current_type_match = GRADLE_TYPE_REGEXP.match(line)
-          current_type = current_type_match.captures[0] if current_type_match
+        file_contents.each_line do |line|
+          line = line.chomp
 
-          gradle_dep_match = GRADLE_DEP_REGEXP.match(line)
-          next unless gradle_dep_match
+          # Check if this is a type header line (starts with word character)
+          # e.g. "annotationProcessor - Annotation processors..."
+          if line[0] =~ /\w/
+            current_type = line[GRADLE_TYPE_REGEXP, 1]
+            next
+          end
 
-          split = gradle_dep_match.captures[0]
+          # Check for dependency line (contains +--- or \---)
+          split_match = line[GRADLE_DEP_REGEXP, 1]
+          next unless split_match
 
           # gradle can import on-disk projects and deps will be listed under them, e.g. `+--- project :test:integration`,
           # so we treat these projects as "internal" deps with requirement of "1.0.0"
@@ -217,7 +221,7 @@ module Bibliothecary
           end
 
           dep = line
-            .split(split)[1]
+            .split(split_match)[1]
             .sub(GRADLE_LINE_ENDING_REGEXP, "")
             .sub(/ FAILED$/, "") # dependency could not be resolved (but still may have a version)
             .sub(" -> ", ":") # handle version arrow syntax
@@ -230,45 +234,50 @@ module Bibliothecary
 
           if dep.count == 6
             # get name from renamed package resolution "org:name:version -> renamed_org:name:version"
-            Dependency.new(
+            dependencies << Dependency.new(
               original_name: dep[0, 2].join(":"),
               original_requirement: dep[2],
               name: dep[-3..-2].join(":"),
               requirement: dep[-1],
               type: current_type,
-              source: options.fetch(:filename, nil),
+              source: source,
               platform: platform_name
             )
           elsif dep.count == 5
             # get name from renamed package resolution "org:name -> renamed_org:name:version"
-            Dependency.new(
+            dependencies << Dependency.new(
               original_name: dep[0, 2].join(":"),
               original_requirement: "*",
               name: dep[-3..-2].join(":"),
               requirement: dep[-1],
               type: current_type,
-              source: options.fetch(:filename, nil),
+              source: source,
               platform: platform_name
             )
           else
             # get name from version conflict resolution ("org:name:version -> version") and no-resolution ("org:name:version")
-            Dependency.new(
+            dependencies << Dependency.new(
               name: dep[0..1].join(":"),
               requirement: dep[-1],
               type: current_type,
-              source: options.fetch(:filename, nil),
+              source: source,
               platform: platform_name
             )
           end
         end
-          .compact
-          .uniq { |item| [item.name, item.requirement, item.type, item.original_name, item.original_requirement] }
+
+        dependencies.uniq! { |item| [item.name, item.requirement, item.type, item.original_name, item.original_requirement] }
         ParserResult.new(dependencies: dependencies)
       end
 
+      def self.strip_ansi(string)
+        return string unless string.include?("\033")
+
+        string.gsub(ANSI_MATCHER, "")
+      end
+
       def self.parse_maven_resolved(file_contents, options: {})
-        dependencies = file_contents
-          .gsub(ANSI_MATCHER, "")
+        dependencies = strip_ansi(file_contents)
           .split("\n")
           .map { |line| parse_resolved_dep_line(line, options: options) }
           .compact
@@ -281,11 +290,12 @@ module Bibliothecary
       # The depth-0 items are the (sub)project names
       # These are in the original order, with no de-duplication.
       def self.parse_maven_tree_items_with_depths(file_contents)
-        file_contents
-          .gsub(ANSI_MATCHER, "")
-          .encode(universal_newline: true)
-          # capture two groups; one is the ASCII art telling us the tree depth,
-          # and two is the actual dependency
+        content = strip_ansi(file_contents)
+        content = content.gsub("\r\n", "\n").gsub("\r", "\n") if content.include?("\r")
+
+        # capture two groups; one is the ASCII art telling us the tree depth,
+        # and two is the actual dependency
+        content
           .scan(/^\[INFO\]\s((?:[-+|\\]|\s)*)((?:[\w.-]+:)+[\w.\-${}]+)/)
           # lines that start with "-" aren't part of the tree, example: "[INFO] --- dependency:3.8.1:tree"
           .reject { |(tree_ascii_art, _dep_info)| tree_ascii_art.start_with?("-") }

data/lib/bibliothecary/parsers/meteor.rb

@@ -6,19 +6,29 @@ module Bibliothecary
   module Parsers
     class Meteor
       include Bibliothecary::Analyser
-      extend Bibliothecary::MultiParsers::JSONRuntime
 
       def self.mapping
         {
           match_filename("versions.json") => {
             kind: "manifest",
-            parser: :parse_json_runtime_manifest,
+            parser: :parse_manifest,
           },
         }
       end
 
-      add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
-      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
+      def self.parse_manifest(file_contents, options: {})
+        manifest = JSON.parse(file_contents)
+        dependencies = manifest.fetch("dependencies", {}).map do |name, requirement|
+          Dependency.new(
+            name: name,
+            requirement: requirement,
+            type: "runtime",
+            source: options.fetch(:filename, nil),
+            platform: platform_name
+          )
+        end
+        ParserResult.new(dependencies: dependencies)
+      end
     end
   end
 end

data/lib/bibliothecary/parsers/mlflow.rb

@@ -15,8 +15,6 @@ module Bibliothecary
         }
       end
 
-      add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
-      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_mlmodel(file_contents, options: {})
         source = options.fetch(:filename, 'MLmodel')

data/lib/bibliothecary/parsers/npm.rb

@@ -43,9 +43,6 @@ module Bibliothecary
         }
       end
 
-      add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
-      add_multi_parser(Bibliothecary::MultiParsers::Spdx)
-      add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
 
       def self.parse_package_lock(file_contents, options: {})
         manifest = JSON.parse(file_contents)
@@ -203,65 +200,56 @@ module Bibliothecary
       #   version: "1.2.0",
       # }, ...]
       def self.parse_v1_yarn_lock(contents, source = nil)
-        contents
-          .encode(universal_newline: true)
-          .gsub(/^#.*/, "")
-          .strip
-          .split("\n\n")
-          .map do |chunk|
-            requirements = chunk
-              .lines
-              .find { |l| !l.start_with?(" ") && l.strip.end_with?(":") } # first line, eg: '"@bar/foo@1.0.0", "@bar/foo@^1.0.1":'
-              .strip
-              .gsub(/"|:$/, "") # don't need quotes or trailing colon
-              .split(",") # split the list of requirements
-
-            name, alias_name = yarn_strip_npm_protocol(requirements.first) # if a package is aliased, strip the alias and return the real package name
-            name = name.strip.split(/(?<!^)@/).first
-            requirements = requirements.map { |d| d.strip.split(/(?<!^)@/, 2) } # split each requirement on name/version "@"", not on leading namespace "@"
-            version = chunk.match(/version "?([^"]*)"?/)[1]
-
-            {
-              name: name,
-              original_name: alias_name,
-              requirements: requirements.map { |x| x[1] },
-              original_requirement: alias_name.nil? ? nil : version,
-              version: version,
-              source: source,
-            }
-          end
+        deps = []
+        # Normalize line endings only if needed
+        contents = contents.gsub("\r\n", "\n").gsub("\r", "\n") if contents.include?("\r")
+
+        # Match package blocks: header line(s) ending with ":" followed by version line
+        # Header examples: 'package@version:' or '"package@version", "package@version2":'
+        contents.scan(/^([^\s#][^\n]*?):\s*\r?\n\s+version "?([^"\n]+)"?/m) do |header, version|
+          # Parse requirements from header (remove quotes and trailing colon)
+          requirements = header.gsub(/"/, "").split(",").map(&:strip)
+
+          name, alias_name = yarn_strip_npm_protocol(requirements.first)
+          name = name.strip.split(/(?<!^)@/).first
+          req_versions = requirements.map { |d| d.strip.split(/(?<!^)@/, 2) }
+
+          deps << {
+            name: name,
+            original_name: alias_name,
+            requirements: req_versions.map { |x| x[1] },
+            original_requirement: alias_name.nil? ? nil : version,
+            version: version,
+            source: source,
+          }
+        end
+        deps
       end
 
       def self.parse_v2_yarn_lock(contents, source = nil)
-        parsed = YAML.load(contents)
-        parsed = parsed.except("__metadata")
-        parsed
-          .reject do |packages, info|
-            # yarn v4+ creates a lockfile entry: "myproject@workspace" with a "use.local" version
-            #   this lockfile entry is a reference to the project to which the lockfile belongs
-            # skip this self-referential package
-            (info["version"].to_s.include?("use.local") && packages.include?("workspace")) ||
-              # yarn allows users to insert patches to their dependencies from within their project
-              # these patches are marked as a separate entry in the lock file but do not represent a new dependency
-              # and should be skipped here
-              # https://yarnpkg.com/protocol/patch
-              packages.include?("@patch:")
-          end
-          .map do |packages, info|
-            packages = packages.split(", ")
-            # use first requirement's name, assuming that deps will always resolve from deps of the same name
-            name, alias_name = yarn_strip_npm_protocol(packages.first.rpartition("@").first)
-            requirements = packages.map { |p| p.rpartition("@").last.gsub(/^.*:/, "") }
-
-            {
-              name: name,
-              original_name: alias_name,
-              requirements: requirements,
-              original_requirement: alias_name.nil? ? nil : info["version"].to_s,
-              version: info["version"].to_s,
-              source: source,
-            }
-          end
+        deps = []
+        # Match package blocks: "package@npm:req": followed by version: x.y.z
+        # Examples: "js-tokens@npm:^3.0.0 || ^4.0.0": or "pkg1@npm:req1, pkg2@npm:req2":
+        contents.scan(/^"([^"]+)":\s*\n\s+version:\s*([^\n]+)/m) do |packages_str, version|
+          # Skip workspace/local packages and patches
+          next if version.include?("use.local") && packages_str.include?("workspace")
+          next if packages_str.include?("@patch:")
+
+          packages = packages_str.split(", ")
+          # use first requirement's name, assuming that deps will always resolve from deps of the same name
+          name, alias_name = yarn_strip_npm_protocol(packages.first.rpartition("@").first)
+          requirements = packages.map { |p| p.rpartition("@").last.gsub(/^.*:/, "") }
+
+          deps << {
+            name: name,
+            original_name: alias_name,
+            requirements: requirements,
+            original_requirement: alias_name.nil? ? nil : version.to_s,
+            version: version.to_s,
+            source: source,
+          }
+        end
+        deps
       end
 
       def self.parse_v5_pnpm_lock(parsed_contents, source = nil)