spandx 0.12.3 → 0.13.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +74 -25
  3. data/README.md +11 -7
  4. data/exe/spandx +1 -2
  5. data/ext/spandx/extconf.rb +5 -0
  6. data/ext/spandx/spandx.c +55 -0
  7. data/ext/spandx/spandx.h +6 -0
  8. data/lib/spandx.rb +6 -3
  9. data/lib/spandx/cli.rb +2 -0
  10. data/lib/spandx/cli/commands/build.rb +13 -2
  11. data/lib/spandx/cli/commands/scan.rb +11 -20
  12. data/lib/spandx/cli/main.rb +3 -2
  13. data/lib/spandx/core/cache.rb +38 -51
  14. data/lib/spandx/core/content.rb +5 -23
  15. data/lib/spandx/core/data_file.rb +66 -0
  16. data/lib/spandx/core/dependency.rb +47 -13
  17. data/lib/spandx/core/git.rb +8 -32
  18. data/lib/spandx/core/guess.rb +48 -40
  19. data/lib/spandx/core/http.rb +7 -2
  20. data/lib/spandx/core/index_file.rb +103 -0
  21. data/lib/spandx/core/license_plugin.rb +15 -4
  22. data/lib/spandx/core/parser.rb +10 -3
  23. data/lib/spandx/core/path_traversal.rb +35 -0
  24. data/lib/spandx/core/relation.rb +38 -0
  25. data/lib/spandx/core/report.rb +6 -12
  26. data/lib/spandx/core/spinner.rb +51 -0
  27. data/lib/spandx/dotnet/index.rb +21 -79
  28. data/lib/spandx/dotnet/parsers/csproj.rb +7 -7
  29. data/lib/spandx/dotnet/parsers/packages_config.rb +7 -7
  30. data/lib/spandx/dotnet/parsers/sln.rb +10 -13
  31. data/lib/spandx/dotnet/project_file.rb +3 -3
  32. data/lib/spandx/java/index.rb +5 -2
  33. data/lib/spandx/java/parsers/maven.rb +7 -7
  34. data/lib/spandx/js/parsers/npm.rb +6 -6
  35. data/lib/spandx/js/parsers/yarn.rb +7 -7
  36. data/lib/spandx/php/parsers/composer.rb +7 -7
  37. data/lib/spandx/python/index.rb +4 -33
  38. data/lib/spandx/python/parsers/pipfile_lock.rb +4 -4
  39. data/lib/spandx/python/pypi.rb +0 -2
  40. data/lib/spandx/python/source.rb +12 -0
  41. data/lib/spandx/ruby/parsers/gemfile_lock.rb +10 -9
  42. data/lib/spandx/spdx/catalogue.rb +5 -1
  43. data/lib/spandx/spdx/composite_license.rb +60 -0
  44. data/lib/spandx/spdx/expression.rb +114 -0
  45. data/lib/spandx/spdx/license.rb +4 -14
  46. data/lib/spandx/version.rb +1 -1
  47. data/spandx.gemspec +16 -10
  48. metadata +100 -30
  49. data/lib/spandx/core/null_gateway.rb +0 -11
  50. data/lib/spandx/core/table.rb +0 -29
  51. data/lib/spandx/core/thread_pool.rb +0 -38
@@ -3,12 +3,13 @@
3
3
  module Spandx
4
4
  module Core
5
5
  class LicensePlugin < Spandx::Core::Plugin
6
- def initialize(catalogue: Spdx::Catalogue.from_git)
6
+ def initialize(catalogue: Spdx::Catalogue.default)
7
7
  @guess = Guess.new(catalogue)
8
8
  end
9
9
 
10
10
  def enhance(dependency)
11
- return dependency unless known?(dependency.package_manager)
11
+ package_manager = package_manager_for(dependency)
12
+ return dependency unless known?(package_manager)
12
13
  return enhance_from_metadata(dependency) if available_in?(dependency.meta)
13
14
 
14
15
  licenses_for(dependency).each do |text|
@@ -25,8 +26,10 @@ module Spandx
25
26
  end
26
27
 
27
28
  def cache_for(dependency, git: Spandx.git)
28
- db = git[dependency.package_manager.to_sym] || git[:cache]
29
- Spandx::Core::Cache.new(dependency.package_manager, db: db)
29
+ package_manager = package_manager_for(dependency)
30
+ git = git[package_manager.to_sym] || git[:cache]
31
+ key = key_for(package_manager)
32
+ Spandx::Core::Cache.new(key, root: "#{git.root}/.index")
30
33
  end
31
34
 
32
35
  def known?(package_manager)
@@ -49,6 +52,14 @@ module Spandx
49
52
  end
50
53
  dependency
51
54
  end
55
+
56
+ def key_for(package_manager)
57
+ package_manager == :yarn ? :npm : package_manager
58
+ end
59
+
60
+ def package_manager_for(dependency)
61
+ dependency.package_manager
62
+ end
52
63
  end
53
64
  end
54
65
  end
@@ -9,8 +9,8 @@ module Spandx
9
9
  end
10
10
  end
11
11
 
12
- def matches?(_filename)
13
- raise ::Spandx::Error, :matches?
12
+ def match?(_path)
13
+ raise ::Spandx::Error, :match?
14
14
  end
15
15
 
16
16
  def parse(_dependency)
@@ -20,8 +20,15 @@ module Spandx
20
20
  class << self
21
21
  include Registerable
22
22
 
23
+ def parse(path)
24
+ self.for(path).parse(path)
25
+ end
26
+
23
27
  def for(path)
24
- find { |x| x.matches?(File.basename(path)) } || UNKNOWN
28
+ path = Pathname.new(path)
29
+ return UNKNOWN if !path.exist? || path.zero?
30
+
31
+ find { |x| x.match?(path) } || UNKNOWN
25
32
  end
26
33
  end
27
34
  end
@@ -0,0 +1,35 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Spandx
4
+ module Core
5
+ class PathTraversal
6
+ attr_reader :root
7
+
8
+ def initialize(root, recursive: true)
9
+ @root = Pathname.new(root)
10
+ @recursive = recursive
11
+ end
12
+
13
+ def each(&block)
14
+ each_file_in(root, &block)
15
+ end
16
+
17
+ private
18
+
19
+ def recursive?
20
+ @recursive
21
+ end
22
+
23
+ def each_file_in(path, &block)
24
+ files = path.directory? ? path.children : [path]
25
+ files.each do |file|
26
+ if file.directory?
27
+ each_file_in(file, &block) if recursive?
28
+ else
29
+ block.call(file)
30
+ end
31
+ end
32
+ end
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Spandx
4
+ module Core
5
+ class Relation
6
+ attr_reader :io, :index
7
+
8
+ def initialize(io, index)
9
+ @io = io
10
+ @index = index
11
+ end
12
+
13
+ def each
14
+ size.times do |n|
15
+ yield row(n)
16
+ end
17
+ end
18
+
19
+ def size
20
+ index.size
21
+ end
22
+
23
+ def row(number)
24
+ offset = number.zero? ? 0 : index.position_for(number)
25
+ return unless offset
26
+
27
+ io.seek(offset)
28
+ parse_row(io.gets)
29
+ end
30
+
31
+ private
32
+
33
+ def parse_row(line)
34
+ CsvParser.parse(line)
35
+ end
36
+ end
37
+ end
38
+ end
@@ -3,7 +3,7 @@
3
3
  module Spandx
4
4
  module Core
5
5
  class Report
6
- include Enumerable
6
+ attr_reader :dependencies
7
7
 
8
8
  FORMATS = {
9
9
  csv: :to_csv,
@@ -20,27 +20,21 @@ module Spandx
20
20
  @dependencies << dependency
21
21
  end
22
22
 
23
- def each
24
- @dependencies.each do |dependency|
25
- yield dependency
26
- end
27
- end
28
-
29
23
  def to(format, formats: FORMATS)
30
24
  public_send(formats.fetch(format&.to_sym, :to_json))
31
25
  end
32
26
 
33
27
  def to_table
34
- Table.new do |table|
35
- map do |dependency|
36
- table << dependency
28
+ Terminal::Table.new(headings: ['Name', 'Version', 'Licenses', 'Location']) do |t|
29
+ dependencies.each do |d|
30
+ t.add_row d.to_a
37
31
  end
38
32
  end
39
33
  end
40
34
 
41
35
  def to_h
42
36
  { version: '1.0', dependencies: [] }.tap do |report|
43
- each do |dependency|
37
+ dependencies.each do |dependency|
44
38
  report[:dependencies].push(dependency.to_h)
45
39
  end
46
40
  end
@@ -51,7 +45,7 @@ module Spandx
51
45
  end
52
46
 
53
47
  def to_csv
54
- map do |dependency|
48
+ dependencies.map do |dependency|
55
49
  CSV.generate_line(dependency.to_a)
56
50
  end
57
51
  end
@@ -0,0 +1,51 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Spandx
4
+ module Core
5
+ class Spinner
6
+ NULL = Class.new do
7
+ def self.spin(*args); end
8
+
9
+ def self.stop(*args); end
10
+ end
11
+
12
+ attr_reader :columns, :spinner
13
+
14
+ def initialize(columns: TTY::Screen.columns, output: $stderr)
15
+ @columns = columns
16
+ @spinner = Nanospinner.new(output)
17
+ @queue = Queue.new
18
+ @thread = Thread.new { work }
19
+ end
20
+
21
+ def spin(message)
22
+ @queue.enq(justify(message))
23
+ yield if block_given?
24
+ end
25
+
26
+ def stop
27
+ @queue.clear
28
+ @queue.enq(:stop)
29
+ @thread.join
30
+ end
31
+
32
+ private
33
+
34
+ def justify(message)
35
+ message.to_s.ljust(columns - 3)
36
+ end
37
+
38
+ def work
39
+ last_message = justify('')
40
+ loop do
41
+ message = @queue.empty? ? last_message : @queue.deq
42
+ break if message == :stop
43
+
44
+ spinner.spin(message)
45
+ last_message = message
46
+ sleep 0.1
47
+ end
48
+ end
49
+ end
50
+ end
51
+ end
@@ -4,98 +4,40 @@ module Spandx
4
4
  module Dotnet
5
5
  class Index
6
6
  DEFAULT_DIR = File.expand_path(File.join(Dir.home, '.local', 'share', 'spandx'))
7
- attr_reader :directory, :name
7
+ attr_reader :cache, :directory, :name, :gateway
8
8
 
9
- def initialize(directory: DEFAULT_DIR)
9
+ def initialize(directory: DEFAULT_DIR, gateway: Spandx::Dotnet::NugetGateway.new)
10
10
  @directory = directory ? File.expand_path(directory) : DEFAULT_DIR
11
11
  @name = 'nuget'
12
+ @gateway = gateway
13
+ @cache = Spandx::Core::Cache.new(@name, root: directory)
12
14
  end
13
15
 
14
- def licenses_for(name:, version:)
15
- search_key = [name, version].join
16
- CSV.open(data_file_for(name), 'r') do |io|
17
- found = io.readlines.bsearch { |x| search_key <=> [x[0], x[1]].join }
18
- found ? found[2].split('-|-') : []
19
- end
20
- end
21
-
22
- def update!(catalogue:, output: StringIO.new)
23
- catalogue.version
24
- insert_latest(Spandx::Dotnet::NugetGateway.new) do |page|
25
- output.puts "Checkpoint #{page}"
26
- checkpoint!(page)
27
- end
28
- sort_index!
16
+ def update!(*)
17
+ queue = Queue.new
18
+ [fetch(queue), save(queue)].each(&:join)
19
+ cache.rebuild_index
29
20
  end
30
21
 
31
22
  private
32
23
 
33
- def files(pattern)
34
- Dir.glob(File.join(directory, pattern)).sort.each do |file|
35
- fullpath = File.join(directory, file)
36
- next if File.directory?(fullpath)
37
- next unless File.exist?(fullpath)
38
-
39
- yield fullpath
24
+ def fetch(queue)
25
+ Thread.new do
26
+ gateway.each do |item|
27
+ queue.enq(item)
28
+ end
29
+ queue.enq(:stop)
40
30
  end
41
31
  end
42
32
 
43
- def sort_index!
44
- files('**/nuget') do |path|
45
- next if File.extname(path) == '.checkpoints'
46
-
47
- IO.write(path, IO.readlines(path).sort.join)
48
- end
49
- end
50
-
51
- def digest_for(components)
52
- Digest::SHA1.hexdigest(Array(components).join('/'))
53
- end
54
-
55
- def data_dir_for(name)
56
- digest = digest_for(name)
57
- File.join(directory, digest[0...2].downcase)
58
- end
59
-
60
- def data_file_for(name)
61
- File.join(data_dir_for(name), 'nuget')
62
- end
63
-
64
- def checkpoints_filepath
65
- @checkpoints_filepath ||= File.join(directory, 'nuget.checkpoints')
66
- end
67
-
68
- def checkpoints
69
- @checkpoints ||= File.exist?(checkpoints_filepath) ? JSON.parse(IO.read(checkpoints_filepath)) : {}
70
- end
71
-
72
- def checkpoint!(page)
73
- checkpoints[page.to_s] = Time.now.utc
74
- IO.write(checkpoints_filepath, JSON.pretty_generate(checkpoints))
75
- end
76
-
77
- def insert(name, version, license)
78
- path = data_file_for(name)
79
- FileUtils.mkdir_p(File.dirname(path))
80
- IO.write(
81
- path,
82
- CSV.generate_line([name, version, license], force_quotes: true),
83
- mode: 'a'
84
- )
85
- end
86
-
87
- def completed_pages
88
- checkpoints.keys.map(&:to_i)
89
- end
90
-
91
- def insert_latest(gateway)
92
- current_page = completed_pages.max || 0
93
- gateway.each(start_page: current_page) do |spec, page|
94
- break if checkpoints[page.to_s]
33
+ def save(queue)
34
+ Thread.new do
35
+ loop do
36
+ item = queue.deq
37
+ break if item == :stop
95
38
 
96
- yield current_page if current_page && page != current_page
97
- current_page = page
98
- insert(spec['id'], spec['version'], spec['licenseExpression'])
39
+ cache.insert(item['id'], item['version'], [item['licenseExpression']])
40
+ end
99
41
  end
100
42
  end
101
43
  end
@@ -4,22 +4,22 @@ module Spandx
4
4
  module Dotnet
5
5
  module Parsers
6
6
  class Csproj < ::Spandx::Core::Parser
7
- def matches?(filename)
8
- ['.csproj', '.props'].include?(File.extname(filename))
7
+ def match?(path)
8
+ ['.csproj', '.props'].include?(path.extname)
9
9
  end
10
10
 
11
- def parse(lockfile)
11
+ def parse(path)
12
12
  ProjectFile
13
- .new(lockfile)
13
+ .new(path)
14
14
  .package_references
15
- .map { |x| map_from(x) }
15
+ .map { |x| map_from(path, x) }
16
16
  end
17
17
 
18
18
  private
19
19
 
20
- def map_from(package_reference)
20
+ def map_from(path, package_reference)
21
21
  ::Spandx::Core::Dependency.new(
22
- package_manager: :nuget,
22
+ path: path,
23
23
  name: package_reference.name,
24
24
  version: package_reference.version,
25
25
  meta: package_reference
@@ -4,22 +4,22 @@ module Spandx
4
4
  module Dotnet
5
5
  module Parsers
6
6
  class PackagesConfig < ::Spandx::Core::Parser
7
- def matches?(filename)
8
- filename.match?(/packages\.config/)
7
+ def match?(path)
8
+ path.basename.fnmatch?('packages.config')
9
9
  end
10
10
 
11
- def parse(lockfile)
12
- Nokogiri::XML(IO.read(lockfile))
11
+ def parse(path)
12
+ Nokogiri::XML(path.read)
13
13
  .search('//package')
14
- .map { |node| map_from(node) }
14
+ .map { |node| map_from(path, node) }
15
15
  end
16
16
 
17
17
  private
18
18
 
19
- def map_from(node)
19
+ def map_from(path, node)
20
20
  name = attribute_for('id', node)
21
21
  version = attribute_for('version', node)
22
- ::Spandx::Core::Dependency.new(package_manager: :nuget, name: name, version: version)
22
+ ::Spandx::Core::Dependency.new(name: name, version: version, path: path)
23
23
  end
24
24
 
25
25
  def attribute_for(key, node)
@@ -4,29 +4,26 @@ module Spandx
4
4
  module Dotnet
5
5
  module Parsers
6
6
  class Sln < ::Spandx::Core::Parser
7
- def matches?(filename)
8
- filename.match?(/.*\.sln/)
7
+ def match?(path)
8
+ path.extname == '.sln'
9
9
  end
10
10
 
11
- def parse(file_path)
12
- project_paths_from(file_path).map do |path|
13
- ::Spandx::Core::Parser
14
- .for(path)
15
- .parse(path)
11
+ def parse(path)
12
+ project_paths_from(path).map do |project_path|
13
+ ::Spandx::Core::Parser.parse(project_path)
16
14
  end.flatten
17
15
  end
18
16
 
19
17
  private
20
18
 
21
- def project_paths_from(file_path)
22
- IO.readlines(file_path).map do |line|
19
+ def project_paths_from(path)
20
+ path.each_line.map do |line|
23
21
  next unless project_line?(line)
24
22
 
25
- path = project_path_from(line)
26
- next unless path
23
+ project_path = project_path_from(line)
24
+ next unless project_path
27
25
 
28
- path = File.join(File.dirname(file_path), path)
29
- Pathname.new(path).cleanpath.to_path
26
+ path.dirname.join(project_path).cleanpath.to_path
30
27
  end.compact
31
28
  end
32
29