bibliothecary 12.1.2 → 12.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 1c70a38503e81b9c6d0abd51195077a225e09835ebedd66babbb55dd6742c0b4
4
- data.tar.gz: f6c02ba982f9661216fdc1b2e4ebe80d879d278d5583188db9e41664b7f35ae8
3
+ metadata.gz: bbe2270af80c93aaa2613434faa3c801337ec316a68bdd25d4b6d92d9979398a
4
+ data.tar.gz: ee84bbf8cb2bd2beae91c080e480fe5fc3f0c2773a305743388cf3f225355eb2
5
5
  SHA512:
6
- metadata.gz: 7b09e212ea2c5fe442ed23e1008d5e9c4abb50aab2cee4fd8314097566b6b5238af49bf71c83720b9b4ed4ceeb7e014a16b2ee891da5f22e1a8c2ff9f9c20f85
7
- data.tar.gz: b2ae2d6f40eb17538b7f0241db8008db6bfd9cb61f52651562458c036dfc004d6356f833015b12ca6471f18a3777f2838c37ea26c64c1ee7059b039d970f8b2a
6
+ metadata.gz: 0ac214d1af05ecf0f156a81dc751421cbbe1d94c6306420d0aeb87c7dabf96eca819a0b199705d40d408b3a22b73227a523bb06e121c1e60c7dfbbb2f74dff10
7
+ data.tar.gz: 52d620bcd946c7197f129ebd59f2ba68676e67152dbeac297a4cdd5d72f74e9f3dc32c6c11b11f513443040721ccaa18b2c322e7b77162fb775fa7827fe3edd6
data/CHANGELOG.md CHANGED
@@ -13,6 +13,51 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
13
13
 
14
14
  ### Removed
15
15
 
16
+ ## [12.1.4] - 2025-03-14
17
+
18
+ ### Added
19
+
20
+ - Add support for PNPM lockfiles (lockfile versions 5, 6, and 9).
21
+ - Add 'parser_options' arg to Bilbiothecary::Runner constructor.
22
+
23
+ ### Changed
24
+
25
+ ### Removed
26
+
27
+ ## [12.1.3] - 2025-02-26
28
+
29
+ ### Added
30
+
31
+ - Add 'local' property to dependencies from Pipfile and Pipfile.lock
32
+
33
+ ### Changed
34
+
35
+ - Handle aliases and NPM and Yarn, and ignore patched dependencies.
36
+ - Fix a PyPI parser's regex to exclude false positive "require" names.
37
+ - Drop all sub-projects from list of deps in a Maven maven-dependency-tree.txt.
38
+
39
+ ### Removed
40
+
41
+ ## [12.1.2] - 2025-02-26
42
+
43
+ ### Added
44
+
45
+ - Add 'local' property to dependencies from Pipfile and Pipfile.lock
46
+
47
+ ### Changed
48
+
49
+ ### Removed
50
+
51
+ ## [12.1.1] - 2025-02-21
52
+
53
+ ### Added
54
+
55
+ - Add test coverage for Go 1.24's new "tool" directive.
56
+
57
+ ### Changed
58
+
59
+ ### Removed
60
+
16
61
  ## [12.1.0] - 2025-01-30
17
62
 
18
63
  ### Added
data/Gemfile CHANGED
@@ -20,5 +20,6 @@ group :test do
20
20
  gem "codeclimate-test-reporter", "~> 1.0.0"
21
21
  gem "rspec", "~> 3.0"
22
22
  gem "simplecov"
23
+ gem "super_diff", "~> 0.15.0"
23
24
  gem "webmock"
24
25
  end
@@ -254,34 +254,89 @@ module Bibliothecary
254
254
  .uniq
255
255
  end
256
256
 
257
- def self.parse_maven_tree(file_contents, options: {})
258
- captures = file_contents
257
+ # Return each item in the ascii art tree with a depth of that item,
258
+ # like [[0, "groupId:artifactId:jar:version:scope"], [1, "..."], ...]
259
+ # The depth-0 items are the (sub)project names
260
+ # These are in the original order, with no de-duplication.
261
+ def self.parse_maven_tree_items_with_depths(file_contents)
262
+ file_contents
259
263
  .gsub(ANSI_MATCHER, "")
260
264
  .gsub(/\r\n?/, "\n")
261
- .scan(/^\[INFO\](?:(?:\+-)|\||(?:\\-)|\s)+((?:[\w\.-]+:)+[\w\.\-${}]+)/)
262
- .flatten
263
- .uniq
265
+ # capture two groups; one is the ASCII art telling us the tree depth,
266
+ # and two is the actual dependency
267
+ .scan(/^\[INFO\]\s((?:[-+|\\]|\s)*)((?:[\w\.-]+:)+[\w\.\-${}]+)/)
268
+ # lines that start with "-" aren't part of the tree, example: "[INFO] --- dependency:3.8.1:tree"
269
+ .reject { |(tree_ascii_art, _dep_info)| tree_ascii_art.start_with?("-") }
270
+ .map do |(tree_ascii_art, dep_info)|
271
+ child_marker_index = tree_ascii_art.index(/(\+-)|(\\-)/)
272
+ depth = if child_marker_index.nil?
273
+ 0
274
+ else
275
+ # There are three characters present in the line for each level of depth
276
+ (child_marker_index / 3) + 1
277
+ end
278
+ [depth, dep_info]
279
+ end
280
+ end
281
+
282
+ # split "org.yaml:snakeyaml:jar:2.2:compile" into
283
+ # ["org.yaml:snakeyaml", "2.2", "compile"]
284
+ def self.parse_maven_tree_dependency(item)
285
+ parts = item.split(":")
286
+ case parts.count
287
+ when 4
288
+ version = parts[-1]
289
+ type = parts[-2]
290
+ when 5..6
291
+ version, type = parts[-2..]
292
+ end
293
+
294
+ name = parts[0..1].join(":")
295
+
296
+ [name, version, type]
297
+ end
298
+
299
+ def self.parse_maven_tree(file_contents, options: {})
300
+ keep_subprojects = options.fetch(:keep_subprojects_in_maven_tree, false)
301
+
302
+ items = parse_maven_tree_items_with_depths(file_contents)
303
+
304
+ raise "found no lines with deps in maven-dependency-tree.txt" if items.empty?
264
305
 
265
- deps = captures.map do |item|
266
- parts = item.split(":")
267
- case parts.count
268
- when 4
269
- version = parts[-1]
270
- type = parts[-2]
271
- when 5..6
272
- version, type = parts[-2..]
306
+ projects = {}
307
+
308
+ if keep_subprojects
309
+ # traditional behavior: we only exclude the root project, and only if we parsed multiple lines
310
+ (root_name, root_version, _root_type) = parse_maven_tree_dependency(items.shift[1])
311
+ unless items.empty?
312
+ projects[root_name] = Set.new
313
+ projects[root_name].add(root_version)
273
314
  end
274
- Dependency.new(
275
- name: parts[0..1].join(":"),
276
- requirement: version,
277
- type: type,
278
- source: options.fetch(:filename, nil)
279
- )
280
315
  end
281
316
 
282
- # First dep line will be the package itself (unless we're only analyzing a single line)
283
- package = deps[0]
284
- deps.size < 2 ? deps : deps[1..].reject { |d| d.name == package.name && d.requirement == package.requirement }
317
+ unique_items = items.map do |(depth, item)|
318
+ (name, version, type) = parse_maven_tree_dependency(item)
319
+ if depth == 0 && !keep_subprojects
320
+ # record and then remove the depth 0
321
+ projects[name] ||= Set.new
322
+ projects[name].add(version)
323
+ nil
324
+ else
325
+ [name, version, type]
326
+ end
327
+ end.compact.uniq
328
+
329
+ unique_items
330
+ # drop the projects and subprojects
331
+ .reject { |(name, version, _type)| projects[name]&.include?(version) }
332
+ .map do |(name, version, type)|
333
+ Bibliothecary::Dependency.new(
334
+ name: name,
335
+ requirement: version,
336
+ type: type,
337
+ source: options.fetch(:filename, nil)
338
+ )
339
+ end
285
340
  end
286
341
 
287
342
  def self.parse_resolved_dep_line(line, options: {})
@@ -16,10 +16,6 @@ module Bibliothecary
16
16
  kind: "manifest",
17
17
  parser: :parse_manifest,
18
18
  },
19
- match_filename("npm-shrinkwrap.json") => {
20
- kind: "lockfile",
21
- parser: :parse_shrinkwrap,
22
- },
23
19
  match_filename("yarn.lock") => {
24
20
  kind: "lockfile",
25
21
  parser: :parse_yarn_lock,
@@ -28,10 +24,18 @@ module Bibliothecary
28
24
  kind: "lockfile",
29
25
  parser: :parse_package_lock,
30
26
  },
27
+ match_filename("pnpm-lock.yaml") => {
28
+ kind: "lockfile",
29
+ parser: :parse_pnpm_lock,
30
+ },
31
31
  match_filename("npm-ls.json") => {
32
32
  kind: "lockfile",
33
33
  parser: :parse_ls,
34
34
  },
35
+ match_filename("npm-shrinkwrap.json") => {
36
+ kind: "lockfile",
37
+ parser: :parse_shrinkwrap,
38
+ },
35
39
  }
36
40
  end
37
41
 
@@ -72,9 +76,22 @@ module Bibliothecary
72
76
  # * The other occurrence's name is the path to the local dependency (which has less information, and is duplicative, so we discard)
73
77
  .select { |name, _dep| name.start_with?("node_modules") }
74
78
  .map do |name, dep|
79
+ # check if the name property is available and differs from the node modules location
80
+ # this indicates that the package has been aliased
81
+ node_module_name = name.split("node_modules/").last
82
+ name_property = dep["name"]
83
+ if !name_property.nil? && node_module_name != name_property
84
+ name = name_property
85
+ original_name = node_module_name
86
+ else
87
+ name = node_module_name
88
+ end
89
+
75
90
  Dependency.new(
76
- name: name.split("node_modules/").last,
91
+ name: name,
92
+ original_name: original_name,
77
93
  requirement: dep["version"],
94
+ original_requirement: original_name.nil? ? nil : dep["version"],
78
95
  type: dep.fetch("dev", false) || dep.fetch("devOptional", false) ? "development" : "runtime",
79
96
  local: dep.fetch("link", false),
80
97
  source: source
@@ -113,9 +130,20 @@ module Bibliothecary
113
130
  dependencies = manifest.fetch("dependencies", [])
114
131
  .reject { |name, _requirement| name.start_with?("//") } # Omit comment keys. They are valid in package.json: https://groups.google.com/g/nodejs/c/NmL7jdeuw0M/m/yTqI05DRQrIJ
115
132
  .map do |name, requirement|
133
+ # check to see if this is an aliased package name
134
+ # example: "alias-package-name": "npm:actual-package@^1.1.3"
135
+ if requirement.include?("npm:")
136
+ # the name of the real dependency is contained in the requirement with the version
137
+ requirement.gsub!("npm:", "")
138
+ original_name = name
139
+ name, _, requirement = requirement.rpartition("@")
140
+ end
141
+
116
142
  Dependency.new(
117
143
  name: name,
144
+ original_name: original_name,
118
145
  requirement: requirement,
146
+ original_requirement: original_name.nil? ? nil : requirement,
119
147
  type: "runtime",
120
148
  local: requirement.start_with?("file:"),
121
149
  source: options.fetch(:filename, nil)
@@ -147,7 +175,9 @@ module Bibliothecary
147
175
  dep_hash.map do |dep|
148
176
  Dependency.new(
149
177
  name: dep[:name],
178
+ original_name: dep[:original_name],
150
179
  requirement: dep[:version],
180
+ original_requirement: dep[:original_requirement],
151
181
  type: "runtime", # lockfile doesn't tell us more about the type of dep
152
182
  local: dep[:requirements]&.first&.start_with?("file:"),
153
183
  source: options.fetch(:filename, nil)
@@ -173,12 +203,17 @@ module Bibliothecary
173
203
  .strip
174
204
  .gsub(/"|:$/, "") # don't need quotes or trailing colon
175
205
  .split(",") # split the list of requirements
176
- .map { |d| d.strip.split(/(?<!^)@/, 2) } # split each requirement on name/version "@"", not on leading namespace "@"
206
+
207
+ name, alias_name = yarn_strip_npm_protocol(requirements.first) # if a package is aliased, strip the alias and return the real package name
208
+ name = name.strip.split(/(?<!^)@/).first
209
+ requirements = requirements.map { |d| d.strip.split(/(?<!^)@/, 2) } # split each requirement on name/version "@"", not on leading namespace "@"
177
210
  version = chunk.match(/version "?([^"]*)"?/)[1]
178
211
 
179
212
  {
180
- name: requirements.first.first,
213
+ name: name,
214
+ original_name: alias_name,
181
215
  requirements: requirements.map { |x| x[1] },
216
+ original_requirement: alias_name.nil? ? nil : version,
182
217
  version: version,
183
218
  source: source,
184
219
  }
@@ -193,23 +228,89 @@ module Bibliothecary
193
228
  # yarn v4+ creates a lockfile entry: "myproject@workspace" with a "use.local" version
194
229
  # this lockfile entry is a reference to the project to which the lockfile belongs
195
230
  # skip this self-referential package
196
- info["version"].to_s.include?("use.local") && packages.include?("workspace")
231
+ (info["version"].to_s.include?("use.local") && packages.include?("workspace")) ||
232
+ # yarn allows users to insert patches to their dependencies from within their project
233
+ # these patches are marked as a separate entry in the lock file but do not represent a new dependency
234
+ # and should be skipped here
235
+ # https://yarnpkg.com/protocol/patch
236
+ packages.include?("@patch:")
197
237
  end
198
238
  .map do |packages, info|
199
239
  packages = packages.split(", ")
200
240
  # use first requirement's name, assuming that deps will always resolve from deps of the same name
201
- name = packages.first.rpartition("@").first
241
+ name, alias_name = yarn_strip_npm_protocol(packages.first.rpartition("@").first)
202
242
  requirements = packages.map { |p| p.rpartition("@").last.gsub(/^.*:/, "") }
203
243
 
204
244
  {
205
245
  name: name,
246
+ original_name: alias_name,
206
247
  requirements: requirements,
248
+ original_requirement: alias_name.nil? ? nil : info["version"].to_s,
207
249
  version: info["version"].to_s,
208
250
  source: source,
209
251
  }
210
252
  end
211
253
  end
212
254
 
255
+ # This method currently has been tested to support:
256
+ # lockfileVersion: '9.0'
257
+ # lockfileVersion: '6.0'
258
+ # lockfileVersion: '5.4'
259
+ def self.parse_pnpm_lock(contents, _source = nil)
260
+ parsed = YAML.load(contents)
261
+ lockfile_version = parsed["lockfileVersion"].to_i
262
+
263
+ dev_dependencies = parsed.dig("importers", ".", "devDependencies") # <= v9
264
+ dev_dependencies ||= parsed["devDependencies"] # <v9
265
+
266
+ # "dependencies" is in "packages" for < v9 and in "snapshots" for >= v9
267
+ # as of https://github.com/pnpm/pnpm/pull/7700.
268
+ (parsed["snapshots"] || parsed["packages"])
269
+ .map do |name_version, details|
270
+ name, version = case lockfile_version
271
+ when 5
272
+ # e.g. '/debug/2.6.9:'
273
+ n, v = name_version.sub(/^\//, "").split("/", 2)
274
+ # e.g. '/debug/2.2.0_supports-color@1.2.0:'
275
+ v = v.split("_", 2)[0]
276
+ [n, v] # rubocop:disable Style/IdenticalConditionalBranches
277
+ when 6
278
+ # e.g. '/debug@2.6.9:'
279
+ n, v = name_version.sub(/^\//, "").split("@", 2)
280
+ # e.g. "debug@2.2.0(supports-color@1.2.0)"
281
+ v = v.split("(", 2).first
282
+ [n, v] # rubocop:disable Style/IdenticalConditionalBranches
283
+ else
284
+ # e.g. 'debug@2.6.9:'
285
+ n, v = name_version.split("@", 2)
286
+ # e.g. "debug@2.2.0(supports-color@1.2.0)"
287
+ v = v.split("(", 2).first
288
+ [n, v] # rubocop:disable Style/IdenticalConditionalBranches
289
+ end
290
+
291
+ # TODO: the "dev" field was removed in v9 lockfiles (https://github.com/pnpm/pnpm/pull/7808)
292
+ # so this will only exist in v6 and below and might be unreliable.
293
+ # The proper way to set this for v9+ is to build a lookup of deps to
294
+ # their "dependencies", and then recurse through each package's
295
+ # parents. If the direct dep(s) that required them are all
296
+ # "devDependencies" then we can consider them "dev == true". This
297
+ # should be done using a DAG data structure, though, to be efficient
298
+ # and avoid cycles.
299
+ is_dev = details["dev"] == true
300
+
301
+ # Fallback for v9+: this only detects dev deps that are direct.
302
+ is_dev ||= dev_dependencies.any? do |dev_name, dev_details|
303
+ dev_name == name && dev_details["version"] == version
304
+ end
305
+
306
+ Dependency.new(
307
+ name: name,
308
+ requirement: version,
309
+ type: is_dev ? "development" : "runtime"
310
+ )
311
+ end
312
+ end
313
+
213
314
  def self.parse_ls(file_contents, options: {})
214
315
  manifest = JSON.parse(file_contents)
215
316
 
@@ -240,6 +341,20 @@ module Bibliothecary
240
341
  ] + transform_tree_to_array(metadata.fetch("dependencies", {}), source)
241
342
  end.flatten(1)
242
343
  end
344
+
345
+ # Yarn package names can be aliased by using the NPM protocol. If a package name includes
346
+ # the NPM protocol then the name following the @npm: protocol identifier is the name of the
347
+ # actual package being imported into the project under a different alias.
348
+ # https://classic.yarnpkg.com/lang/en/docs/cli/add/#toc-yarn-add-alias
349
+ private_class_method def self.yarn_strip_npm_protocol(dep_name)
350
+ if dep_name.include?("@npm:")
351
+ partitions = dep_name.rpartition("@npm:")
352
+ alias_name = partitions.first
353
+ dep_name = partitions.last
354
+ end
355
+
356
+ [dep_name, alias_name]
357
+ end
243
358
  end
244
359
  end
245
360
  end
@@ -87,7 +87,7 @@ module Bibliothecary
87
87
  # do that yet so at least pick deterministically.
88
88
 
89
89
  # Note, frameworks can be empty, so remove empty ones and then return the last sorted item if any
90
- frameworks = frameworks.delete_if { |_k, v| v.empty? }
90
+ frameworks.delete_if { |_k, v| v.empty? }
91
91
  return frameworks[frameworks.keys.max] unless frameworks.empty?
92
92
  end
93
93
  []
@@ -186,7 +186,7 @@ module Bibliothecary
186
186
  # do that yet so at least pick deterministically.
187
187
 
188
188
  # Note, frameworks can be empty, so remove empty ones and then return the last sorted item if any
189
- frameworks = frameworks.delete_if { |_k, v| v.empty? }
189
+ frameworks.delete_if { |_k, v| v.empty? }
190
190
  return frameworks[frameworks.keys.max] unless frameworks.empty?
191
191
  end
192
192
  []
@@ -13,7 +13,7 @@ module Bibliothecary
13
13
  REQUIRE_REGEXP = /([a-zA-Z0-9]+[a-zA-Z0-9\-_\.]+)(?:\[.*?\])*([><=\w\.,]+)?/
14
14
  REQUIREMENTS_REGEXP = /^#{REQUIRE_REGEXP}/
15
15
 
16
- MANIFEST_REGEXP = /.*require[^\/]*(\/)?[^\/]*\.(txt|pip|in)$/
16
+ MANIFEST_REGEXP = /.*require[^\/]*\.(txt|pip|in)$/
17
17
  # TODO: can this be a more specific regexp so it doesn't match something like ".yarn/cache/create-require-npm-1.0.0.zip"?
18
18
  PIP_COMPILE_REGEXP = /.*require.*$/
19
19
 
@@ -5,11 +5,11 @@ module Bibliothecary
5
5
  # A runner is created every time a file is targeted to be parsed. Don't call
6
6
  # parse methods directory! Use a Runner.
7
7
  class Runner
8
- def initialize(configuration)
8
+ def initialize(configuration, parser_options: {})
9
9
  @configuration = configuration
10
10
  @options = {
11
11
  cache: {},
12
- }
12
+ }.merge(parser_options)
13
13
  end
14
14
 
15
15
  def analyse(path, ignore_unparseable_files: true)
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Bibliothecary
4
- VERSION = "12.1.2"
4
+ VERSION = "12.1.4"
5
5
  end
metadata CHANGED
@@ -1,13 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: bibliothecary
3
3
  version: !ruby/object:Gem::Version
4
- version: 12.1.2
4
+ version: 12.1.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Andrew Nesbitt
8
8
  bindir: bin
9
9
  cert_chain: []
10
- date: 2025-02-26 00:00:00.000000000 Z
10
+ date: 2025-03-14 00:00:00.000000000 Z
11
11
  dependencies:
12
12
  - !ruby/object:Gem::Dependency
13
13
  name: commander