bibliothecary 12.0.0 → 12.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +2 -1
- data/.rubocop.yml +10 -2
- data/.ruby-version +1 -1
- data/CHANGELOG.md +13 -0
- data/Gemfile +16 -1
- data/Rakefile +2 -0
- data/bibliothecary.gemspec +11 -13
- data/bin/bibliothecary +2 -1
- data/bin/console +1 -0
- data/lib/bibliothecary/analyser/analysis.rb +13 -8
- data/lib/bibliothecary/analyser/determinations.rb +2 -0
- data/lib/bibliothecary/analyser/matchers.rb +17 -17
- data/lib/bibliothecary/analyser.rb +11 -8
- data/lib/bibliothecary/cli.rb +3 -1
- data/lib/bibliothecary/configuration.rb +3 -8
- data/lib/bibliothecary/dependency.rb +17 -15
- data/lib/bibliothecary/exceptions.rb +6 -2
- data/lib/bibliothecary/file_info.rb +9 -11
- data/lib/bibliothecary/multi_parsers/bundler_like_manifest.rb +13 -10
- data/lib/bibliothecary/multi_parsers/cyclonedx.rb +10 -8
- data/lib/bibliothecary/multi_parsers/dependencies_csv.rb +11 -4
- data/lib/bibliothecary/multi_parsers/json_runtime.rb +5 -2
- data/lib/bibliothecary/multi_parsers/spdx.rb +24 -19
- data/lib/bibliothecary/parsers/bower.rb +5 -3
- data/lib/bibliothecary/parsers/cargo.rb +10 -4
- data/lib/bibliothecary/parsers/cocoapods.rb +15 -11
- data/lib/bibliothecary/parsers/conda.rb +20 -18
- data/lib/bibliothecary/parsers/cpan.rb +6 -4
- data/lib/bibliothecary/parsers/cran.rb +10 -6
- data/lib/bibliothecary/parsers/dub.rb +4 -2
- data/lib/bibliothecary/parsers/elm.rb +4 -1
- data/lib/bibliothecary/parsers/go.rb +51 -43
- data/lib/bibliothecary/parsers/haxelib.rb +2 -1
- data/lib/bibliothecary/parsers/julia.rb +5 -1
- data/lib/bibliothecary/parsers/maven.rb +93 -77
- data/lib/bibliothecary/parsers/meteor.rb +2 -0
- data/lib/bibliothecary/parsers/npm.rb +89 -75
- data/lib/bibliothecary/parsers/nuget.rb +37 -28
- data/lib/bibliothecary/parsers/packagist.rb +21 -11
- data/lib/bibliothecary/parsers/pub.rb +4 -2
- data/lib/bibliothecary/parsers/pypi.rb +48 -29
- data/lib/bibliothecary/parsers/rubygems.rb +16 -12
- data/lib/bibliothecary/parsers/shard.rb +10 -7
- data/lib/bibliothecary/purl_util.rb +2 -1
- data/lib/bibliothecary/related_files_info.rb +7 -8
- data/lib/bibliothecary/runner/multi_manifest_filter.rb +5 -4
- data/lib/bibliothecary/runner.rb +13 -10
- data/lib/bibliothecary/version.rb +3 -1
- data/lib/bibliothecary.rb +7 -4
- data/lib/sdl_parser.rb +11 -6
- metadata +19 -106
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Bibliothecary
|
2
4
|
module Parsers
|
3
5
|
class Pypi
|
@@ -24,15 +26,15 @@ module Bibliothecary
|
|
24
26
|
"requirements-docs.txt", "requirements/docs.txt",
|
25
27
|
"requirements-test.txt", "requirements/test.txt",
|
26
28
|
"requirements-tools.txt", "requirements/tools.txt") => {
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
29
|
+
kind: "manifest",
|
30
|
+
parser: :parse_requirements_txt,
|
31
|
+
},
|
32
|
+
->(p) { PIP_COMPILE_REGEXP.match(p) } => {
|
31
33
|
content_matcher: :pip_compile?,
|
32
34
|
kind: "lockfile",
|
33
35
|
parser: :parse_requirements_txt,
|
34
36
|
},
|
35
|
-
|
37
|
+
->(p) { MANIFEST_REGEXP.match(p) } => {
|
36
38
|
kind: "manifest",
|
37
39
|
parser: :parse_requirements_txt,
|
38
40
|
can_have_lockfile: false,
|
@@ -86,33 +88,34 @@ module Bibliothecary
|
|
86
88
|
add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
|
87
89
|
add_multi_parser(Bibliothecary::MultiParsers::Spdx)
|
88
90
|
|
89
|
-
def self.parse_pipfile(file_contents, options: {})
|
91
|
+
def self.parse_pipfile(file_contents, options: {})
|
90
92
|
manifest = Tomlrb.parse(file_contents)
|
91
|
-
map_dependencies(manifest["packages"], "runtime"
|
93
|
+
map_dependencies(manifest["packages"], "runtime", options.fetch(:filename, nil)) +
|
94
|
+
map_dependencies(manifest["dev-packages"], "develop", options.fetch(:filename, nil))
|
92
95
|
end
|
93
96
|
|
94
|
-
def self.parse_pyproject(file_contents, options: {})
|
97
|
+
def self.parse_pyproject(file_contents, options: {})
|
95
98
|
deps = []
|
96
99
|
|
97
100
|
file_contents = Tomlrb.parse(file_contents)
|
98
101
|
|
99
102
|
# Parse poetry [tool.poetry] deps
|
100
103
|
poetry_manifest = file_contents.fetch("tool", {}).fetch("poetry", {})
|
101
|
-
deps += map_dependencies(poetry_manifest["dependencies"], "runtime")
|
104
|
+
deps += map_dependencies(poetry_manifest["dependencies"], "runtime", options.fetch(:filename, nil))
|
102
105
|
# Poetry 1.0.0-1.2.0 way of defining dev deps
|
103
|
-
deps += map_dependencies(poetry_manifest["dev-dependencies"], "develop")
|
106
|
+
deps += map_dependencies(poetry_manifest["dev-dependencies"], "develop", options.fetch(:filename, nil))
|
104
107
|
# Poetry's 1.2.0+ of defining dev deps
|
105
108
|
poetry_manifest
|
106
109
|
.fetch("group", {})
|
107
110
|
.each_pair do |group_name, obj|
|
108
111
|
group_name = "develop" if group_name == "dev"
|
109
|
-
deps += map_dependencies(obj.fetch("dependencies", {}), group_name)
|
112
|
+
deps += map_dependencies(obj.fetch("dependencies", {}), group_name, options.fetch(:filename, nil))
|
110
113
|
end
|
111
114
|
|
112
115
|
# Parse PEP621 [project] deps
|
113
116
|
pep621_manifest = file_contents.fetch("project", {})
|
114
117
|
pep621_deps = pep621_manifest.fetch("dependencies", []).map { |d| parse_pep_508_dep_spec(d) }
|
115
|
-
deps += map_dependencies(pep621_deps, "runtime")
|
118
|
+
deps += map_dependencies(pep621_deps, "runtime", options.fetch(:filename, nil))
|
116
119
|
|
117
120
|
# We're combining both poetry+PEP621 deps instead of making them mutually exclusive, until we
|
118
121
|
# find a reason not to ingest them both.
|
@@ -125,24 +128,26 @@ module Bibliothecary
|
|
125
128
|
parse_pyproject(file_contents, options)
|
126
129
|
end
|
127
130
|
|
128
|
-
def self.parse_conda(file_contents, options: {})
|
131
|
+
def self.parse_conda(file_contents, options: {})
|
129
132
|
contents = YAML.safe_load(file_contents)
|
130
133
|
return [] unless contents
|
131
134
|
|
132
135
|
dependencies = contents["dependencies"]
|
133
|
-
pip = dependencies.find { |dep| dep.is_a?(Hash) && dep["pip"]}
|
136
|
+
pip = dependencies.find { |dep| dep.is_a?(Hash) && dep["pip"] }
|
134
137
|
return [] unless pip
|
135
138
|
|
136
|
-
Pypi.parse_requirements_txt(pip["pip"].join("\n"))
|
139
|
+
Pypi.parse_requirements_txt(pip["pip"].join("\n"), options:)
|
137
140
|
end
|
138
141
|
|
139
|
-
def self.map_dependencies(packages, type)
|
142
|
+
def self.map_dependencies(packages, type, source = nil)
|
140
143
|
return [] unless packages
|
144
|
+
|
141
145
|
packages.map do |name, info|
|
142
146
|
Dependency.new(
|
143
147
|
name: name,
|
144
148
|
requirement: map_requirements(info),
|
145
149
|
type: type,
|
150
|
+
source: source
|
146
151
|
)
|
147
152
|
end
|
148
153
|
end
|
@@ -152,7 +157,7 @@ module Bibliothecary
|
|
152
157
|
if info["version"]
|
153
158
|
info["version"]
|
154
159
|
elsif info["git"]
|
155
|
-
info[
|
160
|
+
"#{info['git']}##{info['ref']}"
|
156
161
|
else
|
157
162
|
"*"
|
158
163
|
end
|
@@ -161,24 +166,26 @@ module Bibliothecary
|
|
161
166
|
end
|
162
167
|
end
|
163
168
|
|
164
|
-
def self.parse_pipfile_lock(file_contents, options: {})
|
169
|
+
def self.parse_pipfile_lock(file_contents, options: {})
|
165
170
|
manifest = JSON.parse(file_contents)
|
166
171
|
deps = []
|
167
172
|
manifest.each do |group, dependencies|
|
168
173
|
next if group == "_meta"
|
174
|
+
|
169
175
|
group = "runtime" if group == "default"
|
170
176
|
dependencies.each do |name, info|
|
171
177
|
deps << Dependency.new(
|
172
178
|
name: name,
|
173
179
|
requirement: map_requirements(info),
|
174
180
|
type: group,
|
181
|
+
source: options.fetch(:filename, nil)
|
175
182
|
)
|
176
183
|
end
|
177
184
|
end
|
178
185
|
deps
|
179
186
|
end
|
180
187
|
|
181
|
-
def self.parse_poetry_lock(file_contents, options: {})
|
188
|
+
def self.parse_poetry_lock(file_contents, options: {})
|
182
189
|
manifest = Tomlrb.parse(file_contents)
|
183
190
|
deps = []
|
184
191
|
manifest["package"].each do |package|
|
@@ -194,23 +201,28 @@ module Bibliothecary
|
|
194
201
|
name: package["name"],
|
195
202
|
requirement: map_requirements(package),
|
196
203
|
type: group,
|
204
|
+
source: options.fetch(:filename, nil)
|
197
205
|
)
|
198
206
|
end
|
199
207
|
deps
|
200
208
|
end
|
201
209
|
|
202
|
-
def self.parse_setup_py(file_contents, options: {})
|
210
|
+
def self.parse_setup_py(file_contents, options: {})
|
203
211
|
match = file_contents.match(INSTALL_REGEXP)
|
204
212
|
return [] unless match
|
213
|
+
|
205
214
|
deps = []
|
206
215
|
match[1].gsub(/',(\s)?'/, "\n").split("\n").each do |line|
|
207
216
|
next if line.match(/^#/)
|
217
|
+
|
208
218
|
match = line.match(REQUIRE_REGEXP)
|
209
219
|
next unless match
|
220
|
+
|
210
221
|
deps << Dependency.new(
|
211
222
|
name: match[1],
|
212
223
|
requirement: match[-1],
|
213
224
|
type: "runtime",
|
225
|
+
source: options.fetch(:filename, nil)
|
214
226
|
)
|
215
227
|
end
|
216
228
|
deps
|
@@ -226,9 +238,10 @@ module Bibliothecary
|
|
226
238
|
JSON.parse(file_contents)
|
227
239
|
.map do |pkg|
|
228
240
|
Dependency.new(
|
229
|
-
|
230
|
-
|
231
|
-
|
241
|
+
name: pkg.dig("package", "package_name"),
|
242
|
+
requirement: pkg.dig("package", "installed_version"),
|
243
|
+
type: "runtime",
|
244
|
+
source: options.fetch(:filename, nil)
|
232
245
|
)
|
233
246
|
end
|
234
247
|
.uniq
|
@@ -238,7 +251,7 @@ module Bibliothecary
|
|
238
251
|
# https://pip.pypa.io/en/stable/cli/pip_install/#requirement-specifiers
|
239
252
|
# and https://pip.pypa.io/en/stable/topics/vcs-support/#git.
|
240
253
|
# Invalid lines in requirements.txt are skipped.
|
241
|
-
def self.parse_requirements_txt(file_contents, options: {})
|
254
|
+
def self.parse_requirements_txt(file_contents, options: {})
|
242
255
|
deps = []
|
243
256
|
type = case options[:filename]
|
244
257
|
when /dev/ || /docs/ || /tools/
|
@@ -252,7 +265,7 @@ module Bibliothecary
|
|
252
265
|
file_contents.split("\n").each do |line|
|
253
266
|
if line["://"]
|
254
267
|
begin
|
255
|
-
result = parse_requirements_txt_url(line, type)
|
268
|
+
result = parse_requirements_txt_url(line, type, options.fetch(:filename, nil))
|
256
269
|
rescue URI::Error, NoEggSpecified
|
257
270
|
next
|
258
271
|
end
|
@@ -263,6 +276,7 @@ module Bibliothecary
|
|
263
276
|
name: match[1],
|
264
277
|
requirement: match[-1],
|
265
278
|
type: type,
|
279
|
+
source: options.fetch(:filename, nil)
|
266
280
|
)
|
267
281
|
end
|
268
282
|
end
|
@@ -270,7 +284,7 @@ module Bibliothecary
|
|
270
284
|
deps.uniq
|
271
285
|
end
|
272
286
|
|
273
|
-
def self.parse_requirements_txt_url(url, type=nil)
|
287
|
+
def self.parse_requirements_txt_url(url, type = nil, source = nil)
|
274
288
|
uri = URI.parse(url)
|
275
289
|
raise NoEggSpecified, "No egg specified in #{url}" unless uri.fragment
|
276
290
|
|
@@ -279,11 +293,16 @@ module Bibliothecary
|
|
279
293
|
|
280
294
|
requirement = uri.path[/@(.+)$/, 1]
|
281
295
|
|
282
|
-
Dependency.new(
|
296
|
+
Dependency.new(
|
297
|
+
name: name,
|
298
|
+
requirement: requirement,
|
299
|
+
type: type,
|
300
|
+
source: source
|
301
|
+
)
|
283
302
|
end
|
284
303
|
|
285
304
|
def self.pip_compile?(file_contents)
|
286
|
-
|
305
|
+
file_contents.include?("This file is autogenerated by pip-compile")
|
287
306
|
rescue Exception # rubocop:disable Lint/RescueException
|
288
307
|
# We rescue exception here since native libs can throw a non-StandardError
|
289
308
|
# We don't want to throw errors during the matching phase, only during
|
@@ -297,7 +316,7 @@ module Bibliothecary
|
|
297
316
|
name, requirement = dep.split(PEP_508_NAME_REGEXP, 2).last(2).map(&:strip)
|
298
317
|
requirement = requirement.sub(/^[\s;]*/, "")
|
299
318
|
requirement = "*" if requirement == ""
|
300
|
-
|
319
|
+
[name, requirement]
|
301
320
|
end
|
302
321
|
end
|
303
322
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "gemnasium/parser"
|
2
4
|
|
3
5
|
module Bibliothecary
|
@@ -6,7 +8,7 @@ module Bibliothecary
|
|
6
8
|
include Bibliothecary::Analyser
|
7
9
|
extend Bibliothecary::MultiParsers::BundlerLikeManifest
|
8
10
|
|
9
|
-
NAME_VERSION = '(?! )(.*?)(?: \(([^-]*)(?:-(.*))?\))?'
|
11
|
+
NAME_VERSION = '(?! )(.*?)(?: \(([^-]*)(?:-(.*))?\))?'
|
10
12
|
NAME_VERSION_4 = /^ {4}#{NAME_VERSION}$/
|
11
13
|
BUNDLED_WITH = /BUNDLED WITH/
|
12
14
|
|
@@ -15,17 +17,17 @@ module Bibliothecary
|
|
15
17
|
match_filenames("Gemfile", "gems.rb") => {
|
16
18
|
kind: "manifest",
|
17
19
|
parser: :parse_gemfile,
|
18
|
-
related_to: [
|
20
|
+
related_to: %w[manifest lockfile],
|
19
21
|
},
|
20
22
|
match_extension(".gemspec") => {
|
21
23
|
kind: "manifest",
|
22
24
|
parser: :parse_gemspec,
|
23
|
-
related_to: [
|
25
|
+
related_to: %w[manifest lockfile],
|
24
26
|
},
|
25
27
|
match_filenames("Gemfile.lock", "gems.locked") => {
|
26
28
|
kind: "lockfile",
|
27
29
|
parser: :parse_gemfile_lock,
|
28
|
-
related_to: [
|
30
|
+
related_to: %w[manifest lockfile],
|
29
31
|
},
|
30
32
|
}
|
31
33
|
end
|
@@ -34,7 +36,7 @@ module Bibliothecary
|
|
34
36
|
add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
|
35
37
|
add_multi_parser(Bibliothecary::MultiParsers::Spdx)
|
36
38
|
|
37
|
-
def self.parse_gemfile_lock(file_contents, options: {})
|
39
|
+
def self.parse_gemfile_lock(file_contents, options: {})
|
38
40
|
file_contents.lines(chomp: true).map do |line|
|
39
41
|
match = line.match(NAME_VERSION_4)
|
40
42
|
bundler_match = line.match(BUNDLED_WITH)
|
@@ -42,29 +44,30 @@ module Bibliothecary
|
|
42
44
|
|
43
45
|
if match
|
44
46
|
name = match[1]
|
45
|
-
version = match[2].gsub(/\(|\)/,"")
|
47
|
+
version = match[2].gsub(/\(|\)/, "")
|
46
48
|
Dependency.new(
|
47
49
|
name: name,
|
48
50
|
requirement: version,
|
49
51
|
type: "runtime",
|
52
|
+
source: options.fetch(:filename, nil)
|
50
53
|
)
|
51
54
|
else
|
52
|
-
parse_bundler(file_contents)
|
55
|
+
parse_bundler(file_contents, options.fetch(:filename, nil))
|
53
56
|
end
|
54
57
|
end.compact
|
55
58
|
end
|
56
59
|
|
57
|
-
def self.parse_gemfile(file_contents, options: {})
|
60
|
+
def self.parse_gemfile(file_contents, options: {})
|
58
61
|
manifest = Gemnasium::Parser.send(:gemfile, file_contents)
|
59
|
-
parse_ruby_manifest(manifest)
|
62
|
+
parse_ruby_manifest(manifest, options.fetch(:filename, nil))
|
60
63
|
end
|
61
64
|
|
62
|
-
def self.parse_gemspec(file_contents, options: {})
|
65
|
+
def self.parse_gemspec(file_contents, options: {})
|
63
66
|
manifest = Gemnasium::Parser.send(:gemspec, file_contents)
|
64
|
-
parse_ruby_manifest(manifest)
|
67
|
+
parse_ruby_manifest(manifest, options.fetch(:filename, nil))
|
65
68
|
end
|
66
69
|
|
67
|
-
def self.parse_bundler(file_contents)
|
70
|
+
def self.parse_bundler(file_contents, source = nil)
|
68
71
|
bundled_with_index = file_contents.lines(chomp: true).find_index { |line| line.match(BUNDLED_WITH) }
|
69
72
|
version = file_contents.lines(chomp: true).fetch(bundled_with_index + 1)&.strip
|
70
73
|
|
@@ -74,6 +77,7 @@ module Bibliothecary
|
|
74
77
|
name: "bundler",
|
75
78
|
requirement: version,
|
76
79
|
type: "runtime",
|
80
|
+
source: source
|
77
81
|
)
|
78
82
|
end
|
79
83
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "yaml"
|
2
4
|
|
3
5
|
module Bibliothecary
|
@@ -20,23 +22,24 @@ module Bibliothecary
|
|
20
22
|
|
21
23
|
add_multi_parser(Bibliothecary::MultiParsers::DependenciesCSV)
|
22
24
|
|
23
|
-
def self.parse_yaml_lockfile(file_contents, options: {})
|
25
|
+
def self.parse_yaml_lockfile(file_contents, options: {})
|
24
26
|
manifest = YAML.load file_contents
|
25
|
-
map_dependencies(manifest, "shards", "runtime")
|
27
|
+
map_dependencies(manifest, "shards", "runtime", options.fetch(:filename, nil))
|
26
28
|
end
|
27
29
|
|
28
|
-
def self.parse_yaml_manifest(file_contents, options: {})
|
30
|
+
def self.parse_yaml_manifest(file_contents, options: {})
|
29
31
|
manifest = YAML.load file_contents
|
30
|
-
map_dependencies(manifest, "dependencies", "runtime") +
|
31
|
-
|
32
|
+
map_dependencies(manifest, "dependencies", "runtime", options.fetch(:filename, nil)) +
|
33
|
+
map_dependencies(manifest, "development_dependencies", "runtime", options.fetch(:filename, nil))
|
32
34
|
end
|
33
35
|
|
34
|
-
def self.map_dependencies(hash, key, type)
|
35
|
-
hash.fetch(key,[]).map do |name, requirement|
|
36
|
+
def self.map_dependencies(hash, key, type, source = nil)
|
37
|
+
hash.fetch(key, []).map do |name, requirement|
|
36
38
|
Dependency.new(
|
37
39
|
name: name,
|
38
40
|
requirement: requirement["version"],
|
39
41
|
type: type,
|
42
|
+
source: source
|
40
43
|
)
|
41
44
|
end
|
42
45
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Bibliothecary
|
2
4
|
class PurlUtil
|
3
5
|
# If a purl type (key) exists, it will be used in a manifest for
|
@@ -17,7 +19,6 @@ module Bibliothecary
|
|
17
19
|
"pypi" => :pypi,
|
18
20
|
}.freeze
|
19
21
|
|
20
|
-
|
21
22
|
# @param purl [PackageURL]
|
22
23
|
# @return [String] The properly namespaced package name
|
23
24
|
def self.full_name(purl)
|
@@ -1,17 +1,16 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Bibliothecary
|
2
4
|
class RelatedFilesInfo
|
3
|
-
attr_reader :path
|
4
|
-
attr_reader :platform
|
5
|
-
attr_reader :manifests
|
6
|
-
attr_reader :lockfiles
|
5
|
+
attr_reader :path, :platform, :manifests, :lockfiles
|
7
6
|
|
8
7
|
# Create a set of RelatedFilesInfo for the provided file_infos,
|
9
|
-
# where each RelatedFilesInfo contains all the file_infos
|
8
|
+
# where each RelatedFilesInfo contains all the file_infos
|
10
9
|
def self.create_from_file_infos(file_infos)
|
11
10
|
returns = []
|
12
11
|
|
13
12
|
file_infos_by_directory = file_infos.group_by { |info| File.dirname(info.relative_path) }
|
14
|
-
file_infos_by_directory.
|
13
|
+
file_infos_by_directory.each_value do |file_infos_for_path|
|
15
14
|
groupable, ungroupable = file_infos_for_path.partition(&:groupable?)
|
16
15
|
|
17
16
|
# add ungroupable ones as separate RFIs
|
@@ -19,9 +18,9 @@ module Bibliothecary
|
|
19
18
|
returns.append(RelatedFilesInfo.new([file_info]))
|
20
19
|
end
|
21
20
|
|
22
|
-
file_infos_by_directory_by_package_manager = groupable.group_by
|
21
|
+
file_infos_by_directory_by_package_manager = groupable.group_by(&:package_manager)
|
23
22
|
|
24
|
-
file_infos_by_directory_by_package_manager.
|
23
|
+
file_infos_by_directory_by_package_manager.each_value do |file_infos_in_directory_for_package_manager|
|
25
24
|
returns.append(RelatedFilesInfo.new(file_infos_in_directory_for_package_manager))
|
26
25
|
end
|
27
26
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Bibliothecary
|
2
4
|
class Runner
|
3
5
|
class MultiManifestFilter
|
@@ -11,8 +13,8 @@ module Bibliothecary
|
|
11
13
|
# @return [Boolean] True if we should skip processing
|
12
14
|
def skip?
|
13
15
|
!@file_analysis ||
|
14
|
-
|
15
|
-
|
16
|
+
!@file_analysis[:dependencies] ||
|
17
|
+
@file_analysis[:dependencies].empty?
|
16
18
|
end
|
17
19
|
end
|
18
20
|
|
@@ -80,7 +82,7 @@ module Bibliothecary
|
|
80
82
|
end
|
81
83
|
|
82
84
|
def partition_file_entries!
|
83
|
-
@single_file_entries, @multiple_file_entries = files_to_check.partition { |_file, count| count == 1
|
85
|
+
@single_file_entries, @multiple_file_entries = files_to_check.partition { |_file, count| count == 1 }
|
84
86
|
|
85
87
|
@single_file_entries = @single_file_entries.map(&:first)
|
86
88
|
@multiple_file_entries = @multiple_file_entries.map(&:first)
|
@@ -88,4 +90,3 @@ module Bibliothecary
|
|
88
90
|
end
|
89
91
|
end
|
90
92
|
end
|
91
|
-
|
data/lib/bibliothecary/runner.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Bibliothecary
|
2
4
|
# A class that allows bibliothecary to run with multiple configurations at once, rather than with one global.
|
3
5
|
# A runner is created every time a file is targeted to be parsed. Don't call
|
@@ -24,8 +26,8 @@ module Bibliothecary
|
|
24
26
|
analyses = analyses.flatten.compact
|
25
27
|
|
26
28
|
info_list.select { |info| info.package_manager.nil? }.each do |info|
|
27
|
-
analyses.push(Bibliothecary::Analyser
|
28
|
-
|
29
|
+
analyses.push(Bibliothecary::Analyser.create_error_analysis("unknown", info.relative_path, "unknown",
|
30
|
+
"No parser for this file type"))
|
29
31
|
end
|
30
32
|
|
31
33
|
analyses
|
@@ -34,16 +36,16 @@ module Bibliothecary
|
|
34
36
|
|
35
37
|
# deprecated; use load_file_info_list.
|
36
38
|
def load_file_list(path)
|
37
|
-
load_file_info_list(path).map
|
39
|
+
load_file_info_list(path).map(&:full_path)
|
38
40
|
end
|
39
41
|
|
40
42
|
def applicable_package_managers(info)
|
41
43
|
managers = package_managers.select { |pm| pm.match_info?(info) }
|
42
|
-
managers.
|
44
|
+
managers.empty? ? [nil] : managers
|
43
45
|
end
|
44
46
|
|
45
47
|
def package_managers
|
46
|
-
Bibliothecary::Parsers.constants.map{|c| Bibliothecary::Parsers.const_get(c) }.sort_by{|c| c.to_s.downcase }
|
48
|
+
Bibliothecary::Parsers.constants.map { |c| Bibliothecary::Parsers.const_get(c) }.sort_by { |c| c.to_s.downcase }
|
47
49
|
end
|
48
50
|
|
49
51
|
# Parses an array of format [{file_path: "", contents: ""},] to match
|
@@ -130,13 +132,14 @@ module Bibliothecary
|
|
130
132
|
# because this API is used by libraries.io and we don't want to
|
131
133
|
# download all .xml files from GitHub.
|
132
134
|
def identify_manifests(file_list)
|
133
|
-
ignored_dirs_with_slash = ignored_dirs.map { |d|
|
135
|
+
ignored_dirs_with_slash = ignored_dirs.map { |d| d.end_with?("/") ? d : "#{d}/" }
|
134
136
|
allowed_file_list = file_list.reject do |f|
|
135
137
|
ignored_dirs.include?(f) || f.start_with?(*ignored_dirs_with_slash)
|
136
138
|
end
|
137
|
-
allowed_file_list = allowed_file_list.reject{|f| ignored_files.include?(f)}
|
139
|
+
allowed_file_list = allowed_file_list.reject { |f| ignored_files.include?(f) }
|
138
140
|
package_managers.map do |pm|
|
139
|
-
|
141
|
+
# (skip rubocop false positive, since match? is a custom method)
|
142
|
+
allowed_file_list.select do |file_path| # rubocop:disable Style/SelectByRegexp
|
140
143
|
# this is a call to match? without file contents, which will skip
|
141
144
|
# ambiguous filenames that are only possibly a manifest
|
142
145
|
pm.match?(file_path)
|
@@ -159,7 +162,7 @@ module Bibliothecary
|
|
159
162
|
# This means we're likely analyzing these files twice in processing,
|
160
163
|
# but we need that accurate package manager information.
|
161
164
|
def filter_multi_manifest_entries(path, related_files_info_entries)
|
162
|
-
MultiManifestFilter.new(path: path, related_files_info_entries: related_files_info_entries
|
165
|
+
MultiManifestFilter.new(path: path, related_files_info_entries: related_files_info_entries, runner: self).results
|
163
166
|
end
|
164
167
|
|
165
168
|
private
|
@@ -178,4 +181,4 @@ module Bibliothecary
|
|
178
181
|
end
|
179
182
|
end
|
180
183
|
|
181
|
-
require_relative "
|
184
|
+
require_relative "runner/multi_manifest_filter"
|
data/lib/bibliothecary.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "bibliothecary/version"
|
2
4
|
require "bibliothecary/dependency"
|
3
5
|
require "bibliothecary/analyser"
|
@@ -10,16 +12,16 @@ require "bibliothecary/purl_util"
|
|
10
12
|
require "find"
|
11
13
|
require "tomlrb"
|
12
14
|
|
13
|
-
Dir[File.expand_path("
|
15
|
+
Dir[File.expand_path("bibliothecary/multi_parsers/*.rb", __dir__)].each do |file|
|
14
16
|
require file
|
15
17
|
end
|
16
|
-
Dir[File.expand_path("
|
18
|
+
Dir[File.expand_path("bibliothecary/parsers/*.rb", __dir__)].each do |file|
|
17
19
|
require file
|
18
20
|
end
|
19
21
|
|
20
22
|
module Bibliothecary
|
21
|
-
VERSION_OPERATORS = /[~^<>*"]
|
22
|
-
INVALID_UTF8_ERROR_REGEXP = /invalid byte sequence
|
23
|
+
VERSION_OPERATORS = /[~^<>*"]/
|
24
|
+
INVALID_UTF8_ERROR_REGEXP = /invalid byte sequence/
|
23
25
|
|
24
26
|
def self.analyse(path, ignore_unparseable_files: true)
|
25
27
|
runner.analyse(path, ignore_unparseable_files: ignore_unparseable_files)
|
@@ -86,6 +88,7 @@ module Bibliothecary
|
|
86
88
|
rescue ArgumentError => e
|
87
89
|
# Bibliothecary doesn't need to analyze non-UTF8 files like binary files, so just return blank.
|
88
90
|
return "" if e.message.match?(INVALID_UTF8_ERROR_REGEXP)
|
91
|
+
|
89
92
|
raise e
|
90
93
|
end
|
91
94
|
|
data/lib/sdl_parser.rb
CHANGED
@@ -1,23 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "sdl4r"
|
2
4
|
|
3
5
|
class SdlParser
|
4
6
|
attr_reader :contents, :type
|
5
|
-
|
7
|
+
|
8
|
+
def initialize(type, contents, source = nil)
|
6
9
|
@contents = contents
|
7
10
|
@type = type || "runtime"
|
11
|
+
@source = source
|
8
12
|
end
|
9
13
|
|
10
14
|
def dependencies
|
11
15
|
parse.children("dependency").inject([]) do |deps, dep|
|
12
16
|
deps.push(Bibliothecary::Dependency.new(
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
+
name: dep.value,
|
18
|
+
requirement: dep.attribute("version") || ">= 0",
|
19
|
+
type: type,
|
20
|
+
source: @source
|
21
|
+
))
|
17
22
|
end.uniq
|
18
23
|
end
|
19
24
|
|
20
25
|
def parse
|
21
|
-
SDL4R
|
26
|
+
SDL4R.read(contents)
|
22
27
|
end
|
23
28
|
end
|