spandx 0.12.3 → 0.13.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +74 -25
- data/README.md +11 -7
- data/exe/spandx +1 -2
- data/ext/spandx/extconf.rb +5 -0
- data/ext/spandx/spandx.c +55 -0
- data/ext/spandx/spandx.h +6 -0
- data/lib/spandx.rb +6 -3
- data/lib/spandx/cli.rb +2 -0
- data/lib/spandx/cli/commands/build.rb +13 -2
- data/lib/spandx/cli/commands/scan.rb +11 -20
- data/lib/spandx/cli/main.rb +3 -2
- data/lib/spandx/core/cache.rb +38 -51
- data/lib/spandx/core/content.rb +5 -23
- data/lib/spandx/core/data_file.rb +66 -0
- data/lib/spandx/core/dependency.rb +47 -13
- data/lib/spandx/core/git.rb +8 -32
- data/lib/spandx/core/guess.rb +48 -40
- data/lib/spandx/core/http.rb +7 -2
- data/lib/spandx/core/index_file.rb +103 -0
- data/lib/spandx/core/license_plugin.rb +15 -4
- data/lib/spandx/core/parser.rb +10 -3
- data/lib/spandx/core/path_traversal.rb +35 -0
- data/lib/spandx/core/relation.rb +38 -0
- data/lib/spandx/core/report.rb +6 -12
- data/lib/spandx/core/spinner.rb +51 -0
- data/lib/spandx/dotnet/index.rb +21 -79
- data/lib/spandx/dotnet/parsers/csproj.rb +7 -7
- data/lib/spandx/dotnet/parsers/packages_config.rb +7 -7
- data/lib/spandx/dotnet/parsers/sln.rb +10 -13
- data/lib/spandx/dotnet/project_file.rb +3 -3
- data/lib/spandx/java/index.rb +5 -2
- data/lib/spandx/java/parsers/maven.rb +7 -7
- data/lib/spandx/js/parsers/npm.rb +6 -6
- data/lib/spandx/js/parsers/yarn.rb +7 -7
- data/lib/spandx/php/parsers/composer.rb +7 -7
- data/lib/spandx/python/index.rb +4 -33
- data/lib/spandx/python/parsers/pipfile_lock.rb +4 -4
- data/lib/spandx/python/pypi.rb +0 -2
- data/lib/spandx/python/source.rb +12 -0
- data/lib/spandx/ruby/parsers/gemfile_lock.rb +10 -9
- data/lib/spandx/spdx/catalogue.rb +5 -1
- data/lib/spandx/spdx/composite_license.rb +60 -0
- data/lib/spandx/spdx/expression.rb +114 -0
- data/lib/spandx/spdx/license.rb +4 -14
- data/lib/spandx/version.rb +1 -1
- data/spandx.gemspec +16 -10
- metadata +100 -30
- data/lib/spandx/core/null_gateway.rb +0 -11
- data/lib/spandx/core/table.rb +0 -29
- data/lib/spandx/core/thread_pool.rb +0 -38
@@ -3,12 +3,13 @@
|
|
3
3
|
module Spandx
|
4
4
|
module Core
|
5
5
|
class LicensePlugin < Spandx::Core::Plugin
|
6
|
-
def initialize(catalogue: Spdx::Catalogue.
|
6
|
+
def initialize(catalogue: Spdx::Catalogue.default)
|
7
7
|
@guess = Guess.new(catalogue)
|
8
8
|
end
|
9
9
|
|
10
10
|
def enhance(dependency)
|
11
|
-
|
11
|
+
package_manager = package_manager_for(dependency)
|
12
|
+
return dependency unless known?(package_manager)
|
12
13
|
return enhance_from_metadata(dependency) if available_in?(dependency.meta)
|
13
14
|
|
14
15
|
licenses_for(dependency).each do |text|
|
@@ -25,8 +26,10 @@ module Spandx
|
|
25
26
|
end
|
26
27
|
|
27
28
|
def cache_for(dependency, git: Spandx.git)
|
28
|
-
|
29
|
-
|
29
|
+
package_manager = package_manager_for(dependency)
|
30
|
+
git = git[package_manager.to_sym] || git[:cache]
|
31
|
+
key = key_for(package_manager)
|
32
|
+
Spandx::Core::Cache.new(key, root: "#{git.root}/.index")
|
30
33
|
end
|
31
34
|
|
32
35
|
def known?(package_manager)
|
@@ -49,6 +52,14 @@ module Spandx
|
|
49
52
|
end
|
50
53
|
dependency
|
51
54
|
end
|
55
|
+
|
56
|
+
def key_for(package_manager)
|
57
|
+
package_manager == :yarn ? :npm : package_manager
|
58
|
+
end
|
59
|
+
|
60
|
+
def package_manager_for(dependency)
|
61
|
+
dependency.package_manager
|
62
|
+
end
|
52
63
|
end
|
53
64
|
end
|
54
65
|
end
|
data/lib/spandx/core/parser.rb
CHANGED
@@ -9,8 +9,8 @@ module Spandx
|
|
9
9
|
end
|
10
10
|
end
|
11
11
|
|
12
|
-
def
|
13
|
-
raise ::Spandx::Error, :
|
12
|
+
def match?(_path)
|
13
|
+
raise ::Spandx::Error, :match?
|
14
14
|
end
|
15
15
|
|
16
16
|
def parse(_dependency)
|
@@ -20,8 +20,15 @@ module Spandx
|
|
20
20
|
class << self
|
21
21
|
include Registerable
|
22
22
|
|
23
|
+
def parse(path)
|
24
|
+
self.for(path).parse(path)
|
25
|
+
end
|
26
|
+
|
23
27
|
def for(path)
|
24
|
-
|
28
|
+
path = Pathname.new(path)
|
29
|
+
return UNKNOWN if !path.exist? || path.zero?
|
30
|
+
|
31
|
+
find { |x| x.match?(path) } || UNKNOWN
|
25
32
|
end
|
26
33
|
end
|
27
34
|
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Spandx
|
4
|
+
module Core
|
5
|
+
class PathTraversal
|
6
|
+
attr_reader :root
|
7
|
+
|
8
|
+
def initialize(root, recursive: true)
|
9
|
+
@root = Pathname.new(root)
|
10
|
+
@recursive = recursive
|
11
|
+
end
|
12
|
+
|
13
|
+
def each(&block)
|
14
|
+
each_file_in(root, &block)
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
def recursive?
|
20
|
+
@recursive
|
21
|
+
end
|
22
|
+
|
23
|
+
def each_file_in(path, &block)
|
24
|
+
files = path.directory? ? path.children : [path]
|
25
|
+
files.each do |file|
|
26
|
+
if file.directory?
|
27
|
+
each_file_in(file, &block) if recursive?
|
28
|
+
else
|
29
|
+
block.call(file)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Spandx
|
4
|
+
module Core
|
5
|
+
class Relation
|
6
|
+
attr_reader :io, :index
|
7
|
+
|
8
|
+
def initialize(io, index)
|
9
|
+
@io = io
|
10
|
+
@index = index
|
11
|
+
end
|
12
|
+
|
13
|
+
def each
|
14
|
+
size.times do |n|
|
15
|
+
yield row(n)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def size
|
20
|
+
index.size
|
21
|
+
end
|
22
|
+
|
23
|
+
def row(number)
|
24
|
+
offset = number.zero? ? 0 : index.position_for(number)
|
25
|
+
return unless offset
|
26
|
+
|
27
|
+
io.seek(offset)
|
28
|
+
parse_row(io.gets)
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def parse_row(line)
|
34
|
+
CsvParser.parse(line)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
data/lib/spandx/core/report.rb
CHANGED
@@ -3,7 +3,7 @@
|
|
3
3
|
module Spandx
|
4
4
|
module Core
|
5
5
|
class Report
|
6
|
-
|
6
|
+
attr_reader :dependencies
|
7
7
|
|
8
8
|
FORMATS = {
|
9
9
|
csv: :to_csv,
|
@@ -20,27 +20,21 @@ module Spandx
|
|
20
20
|
@dependencies << dependency
|
21
21
|
end
|
22
22
|
|
23
|
-
def each
|
24
|
-
@dependencies.each do |dependency|
|
25
|
-
yield dependency
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
23
|
def to(format, formats: FORMATS)
|
30
24
|
public_send(formats.fetch(format&.to_sym, :to_json))
|
31
25
|
end
|
32
26
|
|
33
27
|
def to_table
|
34
|
-
Table.new do |
|
35
|
-
|
36
|
-
|
28
|
+
Terminal::Table.new(headings: ['Name', 'Version', 'Licenses', 'Location']) do |t|
|
29
|
+
dependencies.each do |d|
|
30
|
+
t.add_row d.to_a
|
37
31
|
end
|
38
32
|
end
|
39
33
|
end
|
40
34
|
|
41
35
|
def to_h
|
42
36
|
{ version: '1.0', dependencies: [] }.tap do |report|
|
43
|
-
each do |dependency|
|
37
|
+
dependencies.each do |dependency|
|
44
38
|
report[:dependencies].push(dependency.to_h)
|
45
39
|
end
|
46
40
|
end
|
@@ -51,7 +45,7 @@ module Spandx
|
|
51
45
|
end
|
52
46
|
|
53
47
|
def to_csv
|
54
|
-
map do |dependency|
|
48
|
+
dependencies.map do |dependency|
|
55
49
|
CSV.generate_line(dependency.to_a)
|
56
50
|
end
|
57
51
|
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Spandx
|
4
|
+
module Core
|
5
|
+
class Spinner
|
6
|
+
NULL = Class.new do
|
7
|
+
def self.spin(*args); end
|
8
|
+
|
9
|
+
def self.stop(*args); end
|
10
|
+
end
|
11
|
+
|
12
|
+
attr_reader :columns, :spinner
|
13
|
+
|
14
|
+
def initialize(columns: TTY::Screen.columns, output: $stderr)
|
15
|
+
@columns = columns
|
16
|
+
@spinner = Nanospinner.new(output)
|
17
|
+
@queue = Queue.new
|
18
|
+
@thread = Thread.new { work }
|
19
|
+
end
|
20
|
+
|
21
|
+
def spin(message)
|
22
|
+
@queue.enq(justify(message))
|
23
|
+
yield if block_given?
|
24
|
+
end
|
25
|
+
|
26
|
+
def stop
|
27
|
+
@queue.clear
|
28
|
+
@queue.enq(:stop)
|
29
|
+
@thread.join
|
30
|
+
end
|
31
|
+
|
32
|
+
private
|
33
|
+
|
34
|
+
def justify(message)
|
35
|
+
message.to_s.ljust(columns - 3)
|
36
|
+
end
|
37
|
+
|
38
|
+
def work
|
39
|
+
last_message = justify('')
|
40
|
+
loop do
|
41
|
+
message = @queue.empty? ? last_message : @queue.deq
|
42
|
+
break if message == :stop
|
43
|
+
|
44
|
+
spinner.spin(message)
|
45
|
+
last_message = message
|
46
|
+
sleep 0.1
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
data/lib/spandx/dotnet/index.rb
CHANGED
@@ -4,98 +4,40 @@ module Spandx
|
|
4
4
|
module Dotnet
|
5
5
|
class Index
|
6
6
|
DEFAULT_DIR = File.expand_path(File.join(Dir.home, '.local', 'share', 'spandx'))
|
7
|
-
attr_reader :directory, :name
|
7
|
+
attr_reader :cache, :directory, :name, :gateway
|
8
8
|
|
9
|
-
def initialize(directory: DEFAULT_DIR)
|
9
|
+
def initialize(directory: DEFAULT_DIR, gateway: Spandx::Dotnet::NugetGateway.new)
|
10
10
|
@directory = directory ? File.expand_path(directory) : DEFAULT_DIR
|
11
11
|
@name = 'nuget'
|
12
|
+
@gateway = gateway
|
13
|
+
@cache = Spandx::Core::Cache.new(@name, root: directory)
|
12
14
|
end
|
13
15
|
|
14
|
-
def
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
found ? found[2].split('-|-') : []
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
def update!(catalogue:, output: StringIO.new)
|
23
|
-
catalogue.version
|
24
|
-
insert_latest(Spandx::Dotnet::NugetGateway.new) do |page|
|
25
|
-
output.puts "Checkpoint #{page}"
|
26
|
-
checkpoint!(page)
|
27
|
-
end
|
28
|
-
sort_index!
|
16
|
+
def update!(*)
|
17
|
+
queue = Queue.new
|
18
|
+
[fetch(queue), save(queue)].each(&:join)
|
19
|
+
cache.rebuild_index
|
29
20
|
end
|
30
21
|
|
31
22
|
private
|
32
23
|
|
33
|
-
def
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
yield fullpath
|
24
|
+
def fetch(queue)
|
25
|
+
Thread.new do
|
26
|
+
gateway.each do |item|
|
27
|
+
queue.enq(item)
|
28
|
+
end
|
29
|
+
queue.enq(:stop)
|
40
30
|
end
|
41
31
|
end
|
42
32
|
|
43
|
-
def
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
def digest_for(components)
|
52
|
-
Digest::SHA1.hexdigest(Array(components).join('/'))
|
53
|
-
end
|
54
|
-
|
55
|
-
def data_dir_for(name)
|
56
|
-
digest = digest_for(name)
|
57
|
-
File.join(directory, digest[0...2].downcase)
|
58
|
-
end
|
59
|
-
|
60
|
-
def data_file_for(name)
|
61
|
-
File.join(data_dir_for(name), 'nuget')
|
62
|
-
end
|
63
|
-
|
64
|
-
def checkpoints_filepath
|
65
|
-
@checkpoints_filepath ||= File.join(directory, 'nuget.checkpoints')
|
66
|
-
end
|
67
|
-
|
68
|
-
def checkpoints
|
69
|
-
@checkpoints ||= File.exist?(checkpoints_filepath) ? JSON.parse(IO.read(checkpoints_filepath)) : {}
|
70
|
-
end
|
71
|
-
|
72
|
-
def checkpoint!(page)
|
73
|
-
checkpoints[page.to_s] = Time.now.utc
|
74
|
-
IO.write(checkpoints_filepath, JSON.pretty_generate(checkpoints))
|
75
|
-
end
|
76
|
-
|
77
|
-
def insert(name, version, license)
|
78
|
-
path = data_file_for(name)
|
79
|
-
FileUtils.mkdir_p(File.dirname(path))
|
80
|
-
IO.write(
|
81
|
-
path,
|
82
|
-
CSV.generate_line([name, version, license], force_quotes: true),
|
83
|
-
mode: 'a'
|
84
|
-
)
|
85
|
-
end
|
86
|
-
|
87
|
-
def completed_pages
|
88
|
-
checkpoints.keys.map(&:to_i)
|
89
|
-
end
|
90
|
-
|
91
|
-
def insert_latest(gateway)
|
92
|
-
current_page = completed_pages.max || 0
|
93
|
-
gateway.each(start_page: current_page) do |spec, page|
|
94
|
-
break if checkpoints[page.to_s]
|
33
|
+
def save(queue)
|
34
|
+
Thread.new do
|
35
|
+
loop do
|
36
|
+
item = queue.deq
|
37
|
+
break if item == :stop
|
95
38
|
|
96
|
-
|
97
|
-
|
98
|
-
insert(spec['id'], spec['version'], spec['licenseExpression'])
|
39
|
+
cache.insert(item['id'], item['version'], [item['licenseExpression']])
|
40
|
+
end
|
99
41
|
end
|
100
42
|
end
|
101
43
|
end
|
@@ -4,22 +4,22 @@ module Spandx
|
|
4
4
|
module Dotnet
|
5
5
|
module Parsers
|
6
6
|
class Csproj < ::Spandx::Core::Parser
|
7
|
-
def
|
8
|
-
['.csproj', '.props'].include?(
|
7
|
+
def match?(path)
|
8
|
+
['.csproj', '.props'].include?(path.extname)
|
9
9
|
end
|
10
10
|
|
11
|
-
def parse(
|
11
|
+
def parse(path)
|
12
12
|
ProjectFile
|
13
|
-
.new(
|
13
|
+
.new(path)
|
14
14
|
.package_references
|
15
|
-
.map { |x| map_from(x) }
|
15
|
+
.map { |x| map_from(path, x) }
|
16
16
|
end
|
17
17
|
|
18
18
|
private
|
19
19
|
|
20
|
-
def map_from(package_reference)
|
20
|
+
def map_from(path, package_reference)
|
21
21
|
::Spandx::Core::Dependency.new(
|
22
|
-
|
22
|
+
path: path,
|
23
23
|
name: package_reference.name,
|
24
24
|
version: package_reference.version,
|
25
25
|
meta: package_reference
|
@@ -4,22 +4,22 @@ module Spandx
|
|
4
4
|
module Dotnet
|
5
5
|
module Parsers
|
6
6
|
class PackagesConfig < ::Spandx::Core::Parser
|
7
|
-
def
|
8
|
-
|
7
|
+
def match?(path)
|
8
|
+
path.basename.fnmatch?('packages.config')
|
9
9
|
end
|
10
10
|
|
11
|
-
def parse(
|
12
|
-
Nokogiri::XML(
|
11
|
+
def parse(path)
|
12
|
+
Nokogiri::XML(path.read)
|
13
13
|
.search('//package')
|
14
|
-
.map { |node| map_from(node) }
|
14
|
+
.map { |node| map_from(path, node) }
|
15
15
|
end
|
16
16
|
|
17
17
|
private
|
18
18
|
|
19
|
-
def map_from(node)
|
19
|
+
def map_from(path, node)
|
20
20
|
name = attribute_for('id', node)
|
21
21
|
version = attribute_for('version', node)
|
22
|
-
::Spandx::Core::Dependency.new(
|
22
|
+
::Spandx::Core::Dependency.new(name: name, version: version, path: path)
|
23
23
|
end
|
24
24
|
|
25
25
|
def attribute_for(key, node)
|
@@ -4,29 +4,26 @@ module Spandx
|
|
4
4
|
module Dotnet
|
5
5
|
module Parsers
|
6
6
|
class Sln < ::Spandx::Core::Parser
|
7
|
-
def
|
8
|
-
|
7
|
+
def match?(path)
|
8
|
+
path.extname == '.sln'
|
9
9
|
end
|
10
10
|
|
11
|
-
def parse(
|
12
|
-
project_paths_from(
|
13
|
-
::Spandx::Core::Parser
|
14
|
-
.for(path)
|
15
|
-
.parse(path)
|
11
|
+
def parse(path)
|
12
|
+
project_paths_from(path).map do |project_path|
|
13
|
+
::Spandx::Core::Parser.parse(project_path)
|
16
14
|
end.flatten
|
17
15
|
end
|
18
16
|
|
19
17
|
private
|
20
18
|
|
21
|
-
def project_paths_from(
|
22
|
-
|
19
|
+
def project_paths_from(path)
|
20
|
+
path.each_line.map do |line|
|
23
21
|
next unless project_line?(line)
|
24
22
|
|
25
|
-
|
26
|
-
next unless
|
23
|
+
project_path = project_path_from(line)
|
24
|
+
next unless project_path
|
27
25
|
|
28
|
-
path
|
29
|
-
Pathname.new(path).cleanpath.to_path
|
26
|
+
path.dirname.join(project_path).cleanpath.to_path
|
30
27
|
end.compact
|
31
28
|
end
|
32
29
|
|