ecosystems-bibliothecary 14.2.0 → 15.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +48 -0
- data/README.md +9 -24
- data/bibliothecary.gemspec +5 -9
- data/lib/bibliothecary/analyser/analysis.rb +10 -5
- data/lib/bibliothecary/analyser/matchers.rb +7 -5
- data/lib/bibliothecary/analyser.rb +0 -30
- data/lib/bibliothecary/cli.rb +35 -26
- data/lib/bibliothecary/configuration.rb +1 -6
- data/lib/bibliothecary/dependency.rb +1 -4
- data/lib/bibliothecary/file_info.rb +7 -0
- data/lib/bibliothecary/parsers/bentoml.rb +0 -2
- data/lib/bibliothecary/parsers/bower.rb +0 -1
- data/lib/bibliothecary/parsers/cargo.rb +12 -10
- data/lib/bibliothecary/parsers/carthage.rb +51 -15
- data/lib/bibliothecary/parsers/clojars.rb +14 -18
- data/lib/bibliothecary/parsers/cocoapods.rb +100 -19
- data/lib/bibliothecary/parsers/cog.rb +0 -2
- data/lib/bibliothecary/parsers/conan.rb +156 -0
- data/lib/bibliothecary/parsers/conda.rb +0 -3
- data/lib/bibliothecary/parsers/cpan.rb +0 -2
- data/lib/bibliothecary/parsers/cran.rb +40 -19
- data/lib/bibliothecary/parsers/docker.rb +0 -2
- data/lib/bibliothecary/parsers/dub.rb +33 -8
- data/lib/bibliothecary/parsers/dvc.rb +0 -2
- data/lib/bibliothecary/parsers/elm.rb +13 -3
- data/lib/bibliothecary/parsers/go.rb +14 -5
- data/lib/bibliothecary/parsers/hackage.rb +132 -24
- data/lib/bibliothecary/parsers/haxelib.rb +14 -4
- data/lib/bibliothecary/parsers/hex.rb +37 -20
- data/lib/bibliothecary/parsers/homebrew.rb +0 -2
- data/lib/bibliothecary/parsers/julia.rb +0 -2
- data/lib/bibliothecary/parsers/maven.rb +35 -25
- data/lib/bibliothecary/parsers/meteor.rb +14 -4
- data/lib/bibliothecary/parsers/mlflow.rb +0 -2
- data/lib/bibliothecary/parsers/npm.rb +47 -59
- data/lib/bibliothecary/parsers/nuget.rb +23 -22
- data/lib/bibliothecary/parsers/ollama.rb +0 -2
- data/lib/bibliothecary/parsers/packagist.rb +0 -3
- data/lib/bibliothecary/parsers/pub.rb +0 -2
- data/lib/bibliothecary/parsers/pypi.rb +54 -35
- data/lib/bibliothecary/parsers/rubygems.rb +92 -27
- data/lib/bibliothecary/parsers/shard.rb +0 -1
- data/lib/bibliothecary/parsers/swift_pm.rb +77 -29
- data/lib/bibliothecary/parsers/vcpkg.rb +68 -17
- data/lib/bibliothecary/runner.rb +169 -22
- data/lib/bibliothecary/version.rb +1 -1
- data/lib/bibliothecary.rb +3 -10
- data/lib/dockerfile_parser.rb +1 -1
- data/lib/modelfile_parser.rb +8 -8
- metadata +2 -108
- data/.codeclimate.yml +0 -25
- data/.github/CONTRIBUTING.md +0 -195
- data/.github/workflows/ci.yml +0 -25
- data/.gitignore +0 -10
- data/.rspec +0 -2
- data/.rubocop.yml +0 -69
- data/.ruby-version +0 -1
- data/.tidelift +0 -1
- data/CODE_OF_CONDUCT.md +0 -74
- data/Gemfile +0 -34
- data/Rakefile +0 -18
- data/bin/console +0 -15
- data/bin/setup +0 -8
- data/lib/bibliothecary/multi_parsers/bundler_like_manifest.rb +0 -26
- data/lib/bibliothecary/multi_parsers/cyclonedx.rb +0 -170
- data/lib/bibliothecary/multi_parsers/dependencies_csv.rb +0 -155
- data/lib/bibliothecary/multi_parsers/json_runtime.rb +0 -22
- data/lib/bibliothecary/multi_parsers/spdx.rb +0 -149
- data/lib/bibliothecary/purl_util.rb +0 -37
- data/lib/bibliothecary/runner/multi_manifest_filter.rb +0 -92
- data/lib/sdl_parser.rb +0 -30
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b18779ed0610462aee8f4fc8c9df7989b42ae2ba1e87f1d7592d408b35fbb606
|
|
4
|
+
data.tar.gz: 004c0db1d58aefbb9cf5a599bb9f7b5790f3c5fd31f1f96b6e3159ba4d646711
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 466de11f118fe2097167ed318baf5d41289eadf61b68d34329ded86c43bf9efab5ff75bb6e1af7d96f8acc41d7ef2c2815c7838517704c9f6d62d443e491c49d
|
|
7
|
+
data.tar.gz: 5c9c5baef7c35f6de525449cc3b71331d607095a9491b56fbcc69b0d3e6683186ad7d066cf6d28b537960750cebdc533fb7065da51847b03069bd51175a57169
|
data/CHANGELOG.md
CHANGED
|
@@ -13,6 +13,54 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
13
13
|
|
|
14
14
|
### Removed
|
|
15
15
|
|
|
16
|
+
## [15.0.0]
|
|
17
|
+
|
|
18
|
+
### Added
|
|
19
|
+
|
|
20
|
+
- Conan parser: conanfile.py, conanfile.txt, conan.lock
|
|
21
|
+
- vcpkg lockfile support: _generated-vcpkg-list.json
|
|
22
|
+
- vcpkg improvements: overrides support, dev dependency detection (host: true)
|
|
23
|
+
|
|
24
|
+
### Changed
|
|
25
|
+
|
|
26
|
+
- NuGet packages.lock.json now returns dependencies from all target frameworks instead of arbitrarily picking one
|
|
27
|
+
- Optimized Maven text parsers: lazy ANSI stripping, skip newline normalization when not needed (10-20% faster)
|
|
28
|
+
- Optimized yarn.lock v1 parser with lazy newline normalization (16% faster)
|
|
29
|
+
- Optimized requirements.txt parser with each_line iteration and cached source lookup (10% faster)
|
|
30
|
+
|
|
31
|
+
### Removed
|
|
32
|
+
|
|
33
|
+
- SPDX parser and support for *.spdx, *.spdx.json files
|
|
34
|
+
- CycloneDX parser and support for cyclonedx.xml, cyclonedx.json, *.cdx.xml, *.cdx.json files
|
|
35
|
+
- DependenciesCSV multi_parser and support for dependencies.csv files
|
|
36
|
+
- packageurl-ruby dependency
|
|
37
|
+
- Multi-parser infrastructure (add_multi_parser, MultiManifestFilter)
|
|
38
|
+
|
|
39
|
+
## [14.4.0]
|
|
40
|
+
|
|
41
|
+
### Changed
|
|
42
|
+
|
|
43
|
+
- Switched Cargo.lock, poetry.lock, uv.lock, Gopkg.lock, and pylock.toml parsers from full TOML parsing to regex-based parsing for 50-250x faster lockfile parsing on these formats.
|
|
44
|
+
- Switched Gemfile.lock parser from Bundler::LockfileParser to regex-based parsing for 6x faster parsing.
|
|
45
|
+
- Switched Podfile.lock parser from YAML to regex-based parsing for 5x faster parsing.
|
|
46
|
+
- Switched yarn.lock v2+ parser from YAML to regex-based parsing for 14x faster parsing.
|
|
47
|
+
|
|
48
|
+
## [14.3.0]
|
|
49
|
+
|
|
50
|
+
### Added
|
|
51
|
+
|
|
52
|
+
- Added `bin/benchmark` script for performance testing.
|
|
53
|
+
|
|
54
|
+
### Changed
|
|
55
|
+
|
|
56
|
+
- Fixed bug where Runner was recreated on every Bibliothecary method call, causing repeated index rebuilding.
|
|
57
|
+
- Memoized package_managers array in Runner.
|
|
58
|
+
- Added filename/extension index for O(1) parser lookup instead of O(n) linear scan through all parsers.
|
|
59
|
+
- Optimized `identify_manifests` to use filename index directly (~139x faster).
|
|
60
|
+
- Optimized `analyse_file` to use filename index for candidate filtering (~16x faster).
|
|
61
|
+
- Added per-file caching of mapping details in FileInfo to avoid repeated lookups.
|
|
62
|
+
- Added `parse_file_info` method to reuse FileInfo objects during parsing.
|
|
63
|
+
|
|
16
64
|
## [14.2.0]
|
|
17
65
|
|
|
18
66
|
### Added
|
data/README.md
CHANGED
|
@@ -4,8 +4,6 @@ Dependency manifest parsing library for https://github.com/ecosyste-ms
|
|
|
4
4
|
|
|
5
5
|
This is a maintained fork of the original [Bibliothecary](https://github.com/librariesio/bibliothecary) gem, with support for additional manifest formats and bug fixes.
|
|
6
6
|
|
|
7
|
-
[](https://github.com/ecosyste-ms/bibliothecary/blob/master/LICENSE.txt)
|
|
8
|
-
|
|
9
7
|
## Installation
|
|
10
8
|
|
|
11
9
|
Requires Ruby 3.4 or above.
|
|
@@ -13,12 +11,14 @@ Requires Ruby 3.4 or above.
|
|
|
13
11
|
Add this line to your application's Gemfile:
|
|
14
12
|
|
|
15
13
|
```ruby
|
|
16
|
-
gem "bibliothecary", git: "https://github.com/ecosyste-ms/bibliothecary.git"
|
|
14
|
+
gem "ecosystems-bibliothecary", git: "https://github.com/ecosyste-ms/bibliothecary.git", require: "bibliothecary"
|
|
17
15
|
```
|
|
18
16
|
|
|
19
17
|
And then execute:
|
|
20
18
|
|
|
21
|
-
|
|
19
|
+
```shell
|
|
20
|
+
bundle install
|
|
21
|
+
```
|
|
22
22
|
|
|
23
23
|
## Usage
|
|
24
24
|
|
|
@@ -40,14 +40,6 @@ Search a directory for manifest files and parse the contents:
|
|
|
40
40
|
Bibliothecary.analyse('./')
|
|
41
41
|
```
|
|
42
42
|
|
|
43
|
-
There are a number of parsers that rely on web services to parse the file formats, those urls can be configured like so:
|
|
44
|
-
|
|
45
|
-
```ruby
|
|
46
|
-
Bibliothecary.configure do |config|
|
|
47
|
-
config.carthage_parser_host = 'http://my-carthage-parsing-service.com'
|
|
48
|
-
end
|
|
49
|
-
```
|
|
50
|
-
|
|
51
43
|
All available config options are in: https://github.com/ecosyste-ms/bibliothecary/blob/master/lib/bibliothecary/configuration.rb
|
|
52
44
|
|
|
53
45
|
## Supported package manager file formats
|
|
@@ -103,18 +95,6 @@ All available config options are in: https://github.com/ecosyste-ms/bibliothecar
|
|
|
103
95
|
- paket.lock
|
|
104
96
|
- *.csproj
|
|
105
97
|
- project.assets.json
|
|
106
|
-
- CycloneDX
|
|
107
|
-
- cyclonedx.xml
|
|
108
|
-
- cyclonedx.json
|
|
109
|
-
- *.cdx.xml
|
|
110
|
-
- *.cdx.json
|
|
111
|
-
- Note that CycloneDX manifests can contain information on multiple
|
|
112
|
-
package manager's packages!
|
|
113
|
-
- SPDX
|
|
114
|
-
- tag:value as *.spdx
|
|
115
|
-
- JSON as *.spdx.json
|
|
116
|
-
- Note that SPDX manifests can contain information on multiple
|
|
117
|
-
package manager's packages!
|
|
118
98
|
- Bower
|
|
119
99
|
- bower.json
|
|
120
100
|
- BentoML
|
|
@@ -134,6 +114,10 @@ All available config options are in: https://github.com/ecosyste-ms/bibliothecar
|
|
|
134
114
|
- project.clj
|
|
135
115
|
- Cog
|
|
136
116
|
- cog.yaml
|
|
117
|
+
- Conan
|
|
118
|
+
- conanfile.py
|
|
119
|
+
- conanfile.txt
|
|
120
|
+
- conan.lock
|
|
137
121
|
- Meteor
|
|
138
122
|
- versions.json
|
|
139
123
|
- MLflow
|
|
@@ -198,6 +182,7 @@ All available config options are in: https://github.com/ecosyste-ms/bibliothecar
|
|
|
198
182
|
- dvc.yaml
|
|
199
183
|
- Vcpkg
|
|
200
184
|
- vcpkg.json
|
|
185
|
+
- _generated-vcpkg-list.json
|
|
201
186
|
- Homebrew
|
|
202
187
|
- Brewfile
|
|
203
188
|
- Brewfile.lock.json
|
data/bibliothecary.gemspec
CHANGED
|
@@ -16,23 +16,19 @@ Gem::Specification.new do |spec|
|
|
|
16
16
|
spec.homepage = "https://github.com/ecosyste-ms/bibliothecary"
|
|
17
17
|
spec.license = "AGPL-3.0"
|
|
18
18
|
|
|
19
|
-
spec.files = `git ls-files -z`.split("\x0").reject
|
|
19
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
|
20
|
+
f.match(%r{^(test|spec|features|\.github)/|^bin/(benchmark|console|setup)|^\.|^(Gemfile|Rakefile|CODE_OF_CONDUCT)})
|
|
21
|
+
end
|
|
20
22
|
spec.bindir = "bin"
|
|
21
|
-
spec.executables =
|
|
23
|
+
spec.executables = %w[bibliothecary]
|
|
22
24
|
spec.require_paths = ["lib"]
|
|
23
25
|
|
|
24
26
|
spec.add_dependency "bundler"
|
|
25
|
-
spec.add_dependency "commander"
|
|
26
27
|
spec.add_dependency "csv"
|
|
27
|
-
spec.add_dependency "deb_control"
|
|
28
28
|
spec.add_dependency "json", "~> 2.8"
|
|
29
|
-
spec.add_dependency "librariesio-gem-parser"
|
|
30
29
|
spec.add_dependency "ox", ">= 2.8.1"
|
|
31
|
-
spec.add_dependency "
|
|
32
|
-
spec.add_dependency "racc"
|
|
33
|
-
spec.add_dependency "sdl4r"
|
|
30
|
+
spec.add_dependency "racc" # required by tomlrb but not declared as a dependency
|
|
34
31
|
spec.add_dependency "tomlrb", "~> 2.0"
|
|
35
|
-
spec.add_dependency "typhoeus"
|
|
36
32
|
|
|
37
33
|
spec.metadata["rubygems_mfa_required"] = "true"
|
|
38
34
|
end
|
|
@@ -40,7 +40,7 @@ module Bibliothecary
|
|
|
40
40
|
# If your Parser needs to return multiple responses for one file, please override this method
|
|
41
41
|
# For example see conda.rb
|
|
42
42
|
kind = determine_kind_from_info(info)
|
|
43
|
-
parser_result =
|
|
43
|
+
parser_result = parse_file_info(info, options: options)
|
|
44
44
|
parser_result = ParserResult.new(dependencies: []) if parser_result.nil? # work around any legacy parsers that return nil
|
|
45
45
|
|
|
46
46
|
Bibliothecary::Analyser.create_analysis(platform_name, info.relative_path, kind, parser_result)
|
|
@@ -52,26 +52,31 @@ module Bibliothecary
|
|
|
52
52
|
# Call the matching parse class method for this file with
|
|
53
53
|
# these contents
|
|
54
54
|
def parse_file(filename, contents, options: {})
|
|
55
|
-
|
|
55
|
+
parse_file_info(FileInfo.new(nil, filename, contents), options: options)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Parse a file using its FileInfo object, reusing cached mapping details.
|
|
59
|
+
def parse_file_info(info, options: {})
|
|
60
|
+
details = first_matching_mapping_details(info)
|
|
56
61
|
|
|
57
62
|
# this can be raised if we don't check match?/match_info?,
|
|
58
63
|
# OR don't have the file contents when we check them, so
|
|
59
64
|
# it turns out for example that a .xml file isn't a
|
|
60
65
|
# manifest after all.
|
|
61
|
-
raise Bibliothecary::FileParsingError.new("No parser for this file type",
|
|
66
|
+
raise Bibliothecary::FileParsingError.new("No parser for this file type", info.relative_path) unless details[:parser]
|
|
62
67
|
|
|
63
68
|
# The `parser` method should raise an exception if the file is malformed,
|
|
64
69
|
# should return empty [] if the file is fine but simply doesn't contain
|
|
65
70
|
# any dependencies, and should never return nil. At the time of writing
|
|
66
71
|
# this comment, some of the parsers return [] or nil to mean an error
|
|
67
72
|
# which is confusing to users.
|
|
68
|
-
send(details[:parser], contents, options: options.merge(filename:
|
|
73
|
+
send(details[:parser], info.contents, options: options.merge(filename: info.relative_path))
|
|
69
74
|
rescue Exception => e # default is StandardError but C bindings throw Exceptions # rubocop:disable Lint/RescueException
|
|
70
75
|
# the C xml parser also puts a newline at the end of the message
|
|
71
76
|
location = e.backtrace_locations[0]
|
|
72
77
|
.to_s
|
|
73
78
|
.then { |l| l =~ /bibliothecary\// ? l.split("bibliothecary/").last : l.split("gems/").last }
|
|
74
|
-
raise Bibliothecary::FileParsingError.new(e.message.strip,
|
|
79
|
+
raise Bibliothecary::FileParsingError.new(e.message.strip, info.relative_path, location)
|
|
75
80
|
end
|
|
76
81
|
|
|
77
82
|
private
|
|
@@ -52,12 +52,14 @@ module Bibliothecary
|
|
|
52
52
|
first_matching_mapping_details(info).any?
|
|
53
53
|
end
|
|
54
54
|
|
|
55
|
-
|
|
56
|
-
|
|
55
|
+
# Get mapping details for this file, using cache if available.
|
|
56
|
+
# The cache is stored on the FileInfo object to avoid repeated lookups.
|
|
57
57
|
def first_matching_mapping_details(info)
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
58
|
+
info.cached_mapping_details(self) do
|
|
59
|
+
mapping
|
|
60
|
+
.find { |matcher, details| mapping_entry_match?(matcher, details, info) }
|
|
61
|
+
&.last || {}
|
|
62
|
+
end
|
|
61
63
|
end
|
|
62
64
|
end
|
|
63
65
|
end
|
|
@@ -38,18 +38,6 @@ module Bibliothecary
|
|
|
38
38
|
base.extend(Bibliothecary::Analyser::Analysis)
|
|
39
39
|
end
|
|
40
40
|
|
|
41
|
-
module TryCache
|
|
42
|
-
def try_cache(options, key)
|
|
43
|
-
if options[:cache]
|
|
44
|
-
options[:cache][key] ||= yield
|
|
45
|
-
|
|
46
|
-
options[:cache][key]
|
|
47
|
-
else
|
|
48
|
-
yield
|
|
49
|
-
end
|
|
50
|
-
end
|
|
51
|
-
end
|
|
52
|
-
|
|
53
41
|
module ClassMethods
|
|
54
42
|
def platform_name
|
|
55
43
|
@platform_name ||= name.to_s.split("::").last.downcase.freeze
|
|
@@ -66,24 +54,6 @@ module Bibliothecary
|
|
|
66
54
|
)
|
|
67
55
|
end
|
|
68
56
|
end
|
|
69
|
-
|
|
70
|
-
# Add a MultiParser module to a Parser class. This extends the
|
|
71
|
-
# self.mapping method on the parser to include the multi parser's
|
|
72
|
-
# files to watch for, and it extends the Parser class with
|
|
73
|
-
# the multi parser for you.
|
|
74
|
-
#
|
|
75
|
-
# @param klass [Class] A Bibliothecary::MultiParsers class
|
|
76
|
-
def add_multi_parser(klass)
|
|
77
|
-
raise "No mapping found! You should place the add_multi_parser call below def self.mapping." unless respond_to?(:mapping)
|
|
78
|
-
|
|
79
|
-
original_mapping = mapping
|
|
80
|
-
|
|
81
|
-
define_singleton_method(:mapping) do
|
|
82
|
-
original_mapping.merge(klass.mapping)
|
|
83
|
-
end
|
|
84
|
-
|
|
85
|
-
send(:extend, klass)
|
|
86
|
-
end
|
|
87
57
|
end
|
|
88
58
|
end
|
|
89
59
|
end
|
data/lib/bibliothecary/cli.rb
CHANGED
|
@@ -2,39 +2,48 @@
|
|
|
2
2
|
|
|
3
3
|
require "bibliothecary/version"
|
|
4
4
|
require "bibliothecary"
|
|
5
|
-
require "
|
|
5
|
+
require "optparse"
|
|
6
6
|
|
|
7
7
|
module Bibliothecary
|
|
8
8
|
class CLI
|
|
9
|
-
include Commander::Methods
|
|
10
|
-
|
|
11
9
|
def run
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
options
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
puts
|
|
33
|
-
end
|
|
10
|
+
options = { path: "./" }
|
|
11
|
+
|
|
12
|
+
parser = OptionParser.new do |opts|
|
|
13
|
+
opts.banner = "Usage: bibliothecary [options]"
|
|
14
|
+
opts.separator ""
|
|
15
|
+
opts.separator "Parse dependency information from a file or folder of code"
|
|
16
|
+
opts.separator ""
|
|
17
|
+
|
|
18
|
+
opts.on("-p", "--path PATH", "Path to file/folder to analyse (default: ./)") do |path|
|
|
19
|
+
options[:path] = path
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
opts.on("-v", "--version", "Show version") do
|
|
23
|
+
puts Bibliothecary::VERSION
|
|
24
|
+
exit
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
opts.on("-h", "--help", "Show this help") do
|
|
28
|
+
puts opts
|
|
29
|
+
exit
|
|
34
30
|
end
|
|
35
31
|
end
|
|
36
32
|
|
|
37
|
-
|
|
33
|
+
parser.parse!
|
|
34
|
+
|
|
35
|
+
output = Bibliothecary.analyse(options[:path])
|
|
36
|
+
output.each do |file_contents|
|
|
37
|
+
puts "#{file_contents[:path]} (#{file_contents[:platform]})"
|
|
38
|
+
file_contents[:dependencies].group_by { |d| d[:type] }.each do |type, deps|
|
|
39
|
+
puts " #{type}"
|
|
40
|
+
deps.each do |dep|
|
|
41
|
+
puts " #{dep[:name]} #{dep[:requirement]}"
|
|
42
|
+
end
|
|
43
|
+
puts
|
|
44
|
+
end
|
|
45
|
+
puts
|
|
46
|
+
end
|
|
38
47
|
end
|
|
39
48
|
end
|
|
40
49
|
end
|
|
@@ -2,16 +2,11 @@
|
|
|
2
2
|
|
|
3
3
|
module Bibliothecary
|
|
4
4
|
class Configuration
|
|
5
|
-
attr_accessor :ignored_dirs, :ignored_files
|
|
5
|
+
attr_accessor :ignored_dirs, :ignored_files
|
|
6
6
|
|
|
7
7
|
def initialize
|
|
8
8
|
@ignored_dirs = [".git", "node_modules", "bower_components", "vendor", "dist"]
|
|
9
9
|
@ignored_files = []
|
|
10
|
-
@carthage_parser_host = "https://carthage.libraries.io"
|
|
11
|
-
@clojars_parser_host = "https://clojars.libraries.io"
|
|
12
|
-
@mix_parser_host = "https://mix.libraries.io"
|
|
13
|
-
@swift_parser_host = "http://swift.libraries.io"
|
|
14
|
-
@cabal_parser_host = "http://cabal.libraries.io"
|
|
15
10
|
end
|
|
16
11
|
end
|
|
17
12
|
end
|
|
@@ -5,10 +5,7 @@ module Bibliothecary
|
|
|
5
5
|
#
|
|
6
6
|
# @attr_reader [String] name The name of the package, e.g. "ansi-string-colors"
|
|
7
7
|
# @attr_reader [String] requirement The version requirement of the release, e.g. "1.0.0" or "^1.0.0"
|
|
8
|
-
# @attr_reader [String] platform The platform of the package, e.g. "maven".
|
|
9
|
-
# it's implicit in most parser results, and the analyzer returns the platform name itself. One
|
|
10
|
-
# exception are multi-parsers like DependenciesCSV, because they may return deps from multiple platforms.
|
|
11
|
-
# Bibliothecary could start returning this field for *all* deps in future, and make it required. (default: nil)
|
|
8
|
+
# @attr_reader [String] platform The platform of the package, e.g. "maven".
|
|
12
9
|
# @attr_reader [String] type The type or scope of dependency, e.g. "runtime" or "test". In some ecosystems a
|
|
13
10
|
# default may be set and in other ecosystems it may make sense to return nil when not found.
|
|
14
11
|
# @attr_reader [Boolean] direct Is this dependency a direct dependency (vs transitive dependency)? (default: nil)
|
|
@@ -46,10 +46,17 @@ module Bibliothecary
|
|
|
46
46
|
@contents = contents
|
|
47
47
|
|
|
48
48
|
@package_manager = nil
|
|
49
|
+
@mapping_cache = {}
|
|
49
50
|
end
|
|
50
51
|
|
|
51
52
|
def groupable?
|
|
52
53
|
@package_manager&.groupable?(self)
|
|
53
54
|
end
|
|
55
|
+
|
|
56
|
+
# Cache and retrieve mapping details for a given package manager class.
|
|
57
|
+
# This avoids repeatedly calling first_matching_mapping_details.
|
|
58
|
+
def cached_mapping_details(package_manager_class)
|
|
59
|
+
@mapping_cache[package_manager_class] ||= yield
|
|
60
|
+
end
|
|
54
61
|
end
|
|
55
62
|
end
|
|
@@ -15,8 +15,6 @@ module Bibliothecary
|
|
|
15
15
|
}
|
|
16
16
|
end
|
|
17
17
|
|
|
18
|
-
add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
|
|
19
|
-
add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
|
|
20
18
|
|
|
21
19
|
def self.parse_bentofile(file_contents, options: {})
|
|
22
20
|
source = options.fetch(:filename, 'bentofile.yaml')
|
|
@@ -18,9 +18,6 @@ module Bibliothecary
|
|
|
18
18
|
}
|
|
19
19
|
end
|
|
20
20
|
|
|
21
|
-
add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
|
|
22
|
-
add_multi_parser(Bibliothecary::MultiParsers::Spdx)
|
|
23
|
-
add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
|
|
24
21
|
|
|
25
22
|
def self.parse_manifest(file_contents, options: {})
|
|
26
23
|
manifest = Tomlrb.parse(file_contents)
|
|
@@ -48,19 +45,24 @@ module Bibliothecary
|
|
|
48
45
|
end
|
|
49
46
|
|
|
50
47
|
def self.parse_lockfile(file_contents, options: {})
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
48
|
+
dependencies = []
|
|
49
|
+
# Split into [[package]] blocks and extract fields from each
|
|
50
|
+
file_contents.split(/\[\[package\]\]/).drop(1).each do |block|
|
|
51
|
+
name = block[/name\s*=\s*"([^"]+)"/, 1]
|
|
52
|
+
version = block[/version\s*=\s*"([^"]+)"/, 1]
|
|
53
|
+
source = block[/source\s*=\s*"([^"]+)"/, 1]
|
|
54
|
+
|
|
55
|
+
# Skip packages without a registry source (local/workspace packages)
|
|
56
|
+
next unless source&.start_with?("registry+")
|
|
54
57
|
|
|
55
|
-
Dependency.new(
|
|
56
|
-
name:
|
|
57
|
-
requirement:
|
|
58
|
+
dependencies << Dependency.new(
|
|
59
|
+
name: name,
|
|
60
|
+
requirement: version,
|
|
58
61
|
type: "runtime",
|
|
59
62
|
source: options.fetch(:filename, nil),
|
|
60
63
|
platform: platform_name
|
|
61
64
|
)
|
|
62
65
|
end
|
|
63
|
-
.compact
|
|
64
66
|
ParserResult.new(dependencies: dependencies)
|
|
65
67
|
end
|
|
66
68
|
end
|
|
@@ -3,6 +3,18 @@ module Bibliothecary
|
|
|
3
3
|
class Carthage
|
|
4
4
|
include Bibliothecary::Analyser
|
|
5
5
|
|
|
6
|
+
# Matches Cartfile entries:
|
|
7
|
+
# github "owner/repo" >= 1.0
|
|
8
|
+
# github "owner/repo" "branch"
|
|
9
|
+
# github "owner/repo"
|
|
10
|
+
# git "url" "ref"
|
|
11
|
+
# binary "url" >= 1.0
|
|
12
|
+
# Group 1: source type (github, git, binary)
|
|
13
|
+
# Group 2: identifier (owner/repo or URL)
|
|
14
|
+
# Group 3: quoted version/branch
|
|
15
|
+
# Group 4: unquoted requirement (e.g., >= 1.0, ~> 2.0)
|
|
16
|
+
CARTFILE_REGEXP = /^(github|git|binary)\s+"([^"]+)"(?:\s+(?:"([^"]+)"|((?:>=|<=|~>|==|>|<)\s*[\d.]+)))?/
|
|
17
|
+
|
|
6
18
|
def self.mapping
|
|
7
19
|
{
|
|
8
20
|
match_filename("Cartfile") => {
|
|
@@ -20,36 +32,60 @@ module Bibliothecary
|
|
|
20
32
|
}
|
|
21
33
|
end
|
|
22
34
|
|
|
23
|
-
add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
|
|
24
|
-
add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
|
|
25
35
|
|
|
26
36
|
def self.parse_cartfile(file_contents, options: {})
|
|
27
|
-
|
|
37
|
+
parse_cartfile_contents(file_contents, options.fetch(:filename, "Cartfile"), "runtime")
|
|
28
38
|
end
|
|
29
39
|
|
|
30
40
|
def self.parse_cartfile_private(file_contents, options: {})
|
|
31
|
-
|
|
41
|
+
parse_cartfile_contents(file_contents, options.fetch(:filename, "Cartfile.private"), "development")
|
|
32
42
|
end
|
|
33
43
|
|
|
34
44
|
def self.parse_cartfile_resolved(file_contents, options: {})
|
|
35
|
-
|
|
45
|
+
parse_cartfile_contents(file_contents, options.fetch(:filename, "Cartfile.resolved"), "runtime")
|
|
36
46
|
end
|
|
37
47
|
|
|
38
|
-
def self.
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
48
|
+
def self.parse_cartfile_contents(contents, source, type)
|
|
49
|
+
deps = []
|
|
50
|
+
|
|
51
|
+
contents.each_line do |line|
|
|
52
|
+
# Remove inline comments
|
|
53
|
+
line = line.sub(/#.*$/, "").strip
|
|
54
|
+
next if line.empty?
|
|
55
|
+
|
|
56
|
+
match = line.match(CARTFILE_REGEXP)
|
|
57
|
+
next unless match
|
|
42
58
|
|
|
43
|
-
|
|
44
|
-
|
|
59
|
+
source_type = match[1] # github, git, or binary
|
|
60
|
+
identifier = match[2] # owner/repo or URL
|
|
61
|
+
# match[3] is quoted version/branch, match[4] is unquoted requirement
|
|
62
|
+
version = match[3] || match[4] || "*"
|
|
63
|
+
|
|
64
|
+
# For github sources, use identifier as-is (could be owner/repo or full URL)
|
|
65
|
+
# For git/binary sources, extract repo name from URL
|
|
66
|
+
name = case source_type
|
|
67
|
+
when "github"
|
|
68
|
+
# Could be "owner/repo" or a full URL like "https://enterprise.local/..."
|
|
69
|
+
if identifier.include?("://")
|
|
70
|
+
identifier.split("/").last&.sub(/\.git$/, "") || identifier
|
|
71
|
+
else
|
|
72
|
+
identifier
|
|
73
|
+
end
|
|
74
|
+
else
|
|
75
|
+
# Extract name from URL (last path component without .git)
|
|
76
|
+
identifier.split("/").last&.sub(/\.git$/, "") || identifier
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
deps << Dependency.new(
|
|
45
80
|
platform: platform_name,
|
|
46
|
-
name:
|
|
47
|
-
requirement:
|
|
48
|
-
type:
|
|
81
|
+
name: name,
|
|
82
|
+
requirement: version,
|
|
83
|
+
type: type,
|
|
49
84
|
source: source
|
|
50
85
|
)
|
|
51
86
|
end
|
|
52
|
-
|
|
87
|
+
|
|
88
|
+
ParserResult.new(dependencies: deps)
|
|
53
89
|
end
|
|
54
90
|
end
|
|
55
91
|
end
|
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
require "json"
|
|
2
|
-
require "typhoeus"
|
|
3
|
-
|
|
4
1
|
module Bibliothecary
|
|
5
2
|
module Parsers
|
|
6
3
|
class Clojars
|
|
7
4
|
include Bibliothecary::Analyser
|
|
8
5
|
|
|
6
|
+
# Matches individual dependency: [name "version"]
|
|
7
|
+
# Name can be like: org.clojure/clojure, cheshire, ring/ring-defaults
|
|
8
|
+
DEPENDENCY_REGEXP = %r{\[([a-zA-Z0-9_./\-]+)\s+"([^"]+)"\]}
|
|
9
|
+
|
|
9
10
|
def self.mapping
|
|
10
11
|
{
|
|
11
12
|
match_filename("project.clj") => {
|
|
@@ -15,31 +16,26 @@ module Bibliothecary
|
|
|
15
16
|
}
|
|
16
17
|
end
|
|
17
18
|
|
|
18
|
-
add_multi_parser(Bibliothecary::MultiParsers::CycloneDX)
|
|
19
|
-
add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
|
|
20
19
|
|
|
21
20
|
def self.parse_manifest(file_contents, options: {})
|
|
22
21
|
source = options.fetch(:filename, "project.clj")
|
|
23
|
-
|
|
24
|
-
raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{Bibliothecary.configuration.clojars_parser_host}/project.clj", response.response_code) unless response.success?
|
|
25
|
-
json = JSON.parse response.body
|
|
26
|
-
index = json.index("dependencies")
|
|
22
|
+
deps = []
|
|
27
23
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
24
|
+
# Find the :dependencies section and extract deps
|
|
25
|
+
# Look for :dependencies followed by a vector of vectors
|
|
26
|
+
if (deps_section = file_contents[/:dependencies\s*\[.*?\]\]/m])
|
|
27
|
+
deps_section.scan(DEPENDENCY_REGEXP) do |name, version|
|
|
28
|
+
deps << Dependency.new(
|
|
32
29
|
platform: platform_name,
|
|
33
|
-
name:
|
|
34
|
-
requirement:
|
|
30
|
+
name: name,
|
|
31
|
+
requirement: version,
|
|
35
32
|
type: "runtime",
|
|
36
33
|
source: source
|
|
37
34
|
)
|
|
38
35
|
end
|
|
39
|
-
else
|
|
40
|
-
[]
|
|
41
36
|
end
|
|
42
|
-
|
|
37
|
+
|
|
38
|
+
ParserResult.new(dependencies: deps)
|
|
43
39
|
end
|
|
44
40
|
end
|
|
45
41
|
end
|