spandx 0.7.0 → 0.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -2
- data/lib/spandx/cli/commands/scan.rb +31 -9
- data/lib/spandx/cli.rb +3 -3
- data/lib/spandx/core/parser.rb +1 -0
- data/lib/spandx/dotnet/index.rb +21 -16
- data/lib/spandx/dotnet/nuget_gateway.rb +4 -6
- data/lib/spandx/gateways/http.rb +1 -1
- data/lib/spandx/version.rb +1 -1
- data/lib/spandx.rb +7 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c8d1e5a7326e983e0bfdacf9b2e30b65f91357bec5a0a6f6f1ea617e53b20e69
|
4
|
+
data.tar.gz: e1c386c30848bda74c9728fdae88898e430b3048005e39a0d6ba64bc8146fddb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2e65997919e6ea2e23cabc18b2bdaceebc469553472241a45e87e65e701d9435e594faa188c5f94e5ada7d959de4e51260b2bc68903f8f7bcbc95ec6ecf3d12f
|
7
|
+
data.tar.gz: a2754d35a2a863634ef67b153bb4d54c72151ab083f9fe608836ee33aa3ff80b76d018bb8438af55f907c67dad6218c55b821a94ec45a6782bf1e5ffb8339cf9
|
data/CHANGELOG.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
Version 0.
|
1
|
+
Version 0.8.0
|
2
2
|
|
3
3
|
# Changelog
|
4
4
|
|
@@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
9
9
|
|
10
10
|
## [Unreleased]
|
11
11
|
|
12
|
+
## [0.8.0] - 2020-03-11
|
13
|
+
### Added
|
14
|
+
- Allow scanning a directory.
|
15
|
+
- Allow recursive scanning of a directory.
|
16
|
+
|
12
17
|
## [0.7.0] - 2020-03-11
|
13
18
|
### Changed
|
14
19
|
- Improve how the `nuget` index is built.
|
@@ -88,7 +93,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
88
93
|
### Added
|
89
94
|
- Provide ruby API to the latest SPDX catalogue.
|
90
95
|
|
91
|
-
[Unreleased]: https://github.com/mokhan/spandx/compare/v0.
|
96
|
+
[Unreleased]: https://github.com/mokhan/spandx/compare/v0.8.0...HEAD
|
97
|
+
[0.8.0]: https://github.com/mokhan/spandx/compare/v0.7.0...v0.8.0
|
98
|
+
[0.7.0]: https://github.com/mokhan/spandx/compare/v0.6.0...v0.7.0
|
92
99
|
[0.6.0]: https://github.com/mokhan/spandx/compare/v0.5.0...v0.6.0
|
93
100
|
[0.5.0]: https://github.com/mokhan/spandx/compare/v0.4.1...v0.5.0
|
94
101
|
[0.4.1]: https://github.com/mokhan/spandx/compare/v0.4.0...v0.4.1
|
@@ -4,23 +4,45 @@ module Spandx
|
|
4
4
|
module Cli
|
5
5
|
module Commands
|
6
6
|
class Scan < Spandx::Cli::Command
|
7
|
-
attr_reader :
|
7
|
+
attr_reader :scan_path
|
8
8
|
|
9
|
-
def initialize(
|
10
|
-
@
|
9
|
+
def initialize(scan_path, options)
|
10
|
+
@scan_path = ::Pathname.new(scan_path)
|
11
11
|
@options = options
|
12
12
|
end
|
13
13
|
|
14
14
|
def execute(output: $stdout)
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
report = ::Spandx::Core::Report.new
|
19
|
-
::Spandx::Core::Parser.for(lockfile).parse(lockfile).each do |dependency|
|
15
|
+
report = ::Spandx::Core::Report.new
|
16
|
+
each_file_in(scan_path) do |file|
|
17
|
+
each_dependency_from(file) do |dependency|
|
20
18
|
report.add(dependency)
|
21
19
|
end
|
22
|
-
output.puts report.to_json
|
23
20
|
end
|
21
|
+
output.puts report.to_json
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def recursive?
|
27
|
+
@options['recursive']
|
28
|
+
end
|
29
|
+
|
30
|
+
def each_file_in(dir, &block)
|
31
|
+
files = File.directory?(dir) ? Dir.glob(File.join(dir, '*')) : [dir]
|
32
|
+
files.each do |file|
|
33
|
+
if File.directory?(file)
|
34
|
+
each_file_in(file, &block) if recursive?
|
35
|
+
else
|
36
|
+
block.call(file)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def each_dependency_from(file)
|
42
|
+
::Spandx::Core::Parser
|
43
|
+
.for(file)
|
44
|
+
.parse(file)
|
45
|
+
.each { |dependency| yield dependency }
|
24
46
|
end
|
25
47
|
end
|
26
48
|
end
|
data/lib/spandx/cli.rb
CHANGED
@@ -19,9 +19,9 @@ module Spandx
|
|
19
19
|
register Spandx::Cli::Commands::Index, 'index', 'index [SUBCOMMAND]', 'Command description...'
|
20
20
|
|
21
21
|
desc 'scan LOCKFILE', 'Scan a lockfile and list dependencies/licenses'
|
22
|
-
method_option :help, aliases: '-h', type: :boolean,
|
23
|
-
|
24
|
-
def scan(lockfile
|
22
|
+
method_option :help, aliases: '-h', type: :boolean, desc: 'Display usage information'
|
23
|
+
method_option :recursive, aliases: '-r', type: :boolean, desc: 'Perform recursive scan', default: false
|
24
|
+
def scan(lockfile)
|
25
25
|
if options[:help]
|
26
26
|
invoke :help, ['scan']
|
27
27
|
else
|
data/lib/spandx/core/parser.rb
CHANGED
data/lib/spandx/dotnet/index.rb
CHANGED
@@ -12,7 +12,7 @@ module Spandx
|
|
12
12
|
|
13
13
|
def licenses_for(name:, version:)
|
14
14
|
search_key = [name, version].join
|
15
|
-
|
15
|
+
CSV.open(data_file_for(name), 'r') do |io|
|
16
16
|
found = io.readlines.bsearch { |x| search_key <=> [x[0], x[1]].join }
|
17
17
|
found ? found[2].split('-|-') : []
|
18
18
|
end
|
@@ -37,6 +37,8 @@ module Spandx
|
|
37
37
|
|
38
38
|
def sort_index!
|
39
39
|
files('**/*') do |path|
|
40
|
+
next if File.extname(path) == '.checkpoints'
|
41
|
+
|
40
42
|
IO.write(path, IO.readlines(path).sort.join)
|
41
43
|
end
|
42
44
|
end
|
@@ -45,14 +47,6 @@ module Spandx
|
|
45
47
|
Digest::SHA1.hexdigest(Array(components).join('/'))
|
46
48
|
end
|
47
49
|
|
48
|
-
def open_data(name, mode: 'a')
|
49
|
-
data_dir = data_dir_for(name)
|
50
|
-
FileUtils.mkdir_p(data_dir)
|
51
|
-
CSV.open(data_file_for(name), mode, force_quotes: true) do |csv|
|
52
|
-
yield csv
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
50
|
def data_dir_for(name)
|
57
51
|
digest = digest_for(name)
|
58
52
|
File.join(directory, digest[0...2].downcase)
|
@@ -75,16 +69,27 @@ module Spandx
|
|
75
69
|
IO.write(checkpoints_filepath, JSON.pretty_generate(checkpoints))
|
76
70
|
end
|
77
71
|
|
78
|
-
def insert(
|
79
|
-
|
80
|
-
|
81
|
-
|
72
|
+
def insert(name, version, license)
|
73
|
+
path = license ? data_file_for(name) : dead_letter_path
|
74
|
+
FileUtils.mkdir_p(File.dirname(path))
|
75
|
+
IO.write(
|
76
|
+
path,
|
77
|
+
CSV.generate_line([name, version, license], force_quotes: true),
|
78
|
+
mode: 'a'
|
79
|
+
)
|
80
|
+
end
|
81
|
+
|
82
|
+
def completed_pages
|
83
|
+
checkpoints.keys.map(&:to_i)
|
84
|
+
end
|
85
|
+
|
86
|
+
def dead_letter_path
|
87
|
+
@dead_letter_path ||= File.join(directory, 'nuget.unknown')
|
82
88
|
end
|
83
89
|
|
84
90
|
def insert_latest(gateway)
|
85
|
-
current_page =
|
86
|
-
gateway.each do |spec, page|
|
87
|
-
next unless spec['licenseExpression']
|
91
|
+
current_page = completed_pages.max || 0
|
92
|
+
gateway.each(start_page: current_page) do |spec, page|
|
88
93
|
break if checkpoints[page.to_s]
|
89
94
|
|
90
95
|
yield current_page if current_page && page != current_page
|
@@ -21,8 +21,8 @@ module Spandx
|
|
21
21
|
guess_licenses_from(document)
|
22
22
|
end
|
23
23
|
|
24
|
-
def each(
|
25
|
-
each_page(start_page:
|
24
|
+
def each(start_page: 0)
|
25
|
+
each_page(start_page: start_page) do |page_json|
|
26
26
|
items_from(page_json).each do |item|
|
27
27
|
yield(fetch_json(item['@id']), page_number_from(page_json['@id']))
|
28
28
|
end
|
@@ -36,7 +36,7 @@ module Spandx
|
|
36
36
|
def each_page(start_page:)
|
37
37
|
url = "https://#{host}/v3/catalog0/index.json"
|
38
38
|
items_from(fetch_json(url))
|
39
|
-
.find_all { |page| page_number_from(page['@id'])
|
39
|
+
.find_all { |page| page_number_from(page['@id']) >= start_page }
|
40
40
|
.each { |page| yield fetch_json(page['@id']) }
|
41
41
|
end
|
42
42
|
|
@@ -82,9 +82,7 @@ module Spandx
|
|
82
82
|
end
|
83
83
|
|
84
84
|
def items_from(page)
|
85
|
-
page['items']
|
86
|
-
.sort_by { |x| x['commitTimeStamp'] }
|
87
|
-
.reverse
|
85
|
+
page['items'].sort_by { |x| x['commitTimeStamp'] }
|
88
86
|
end
|
89
87
|
|
90
88
|
def page_number_from(url)
|
data/lib/spandx/gateways/http.rb
CHANGED
data/lib/spandx/version.rb
CHANGED
data/lib/spandx.rb
CHANGED
@@ -5,6 +5,7 @@ require 'bundler'
|
|
5
5
|
require 'csv'
|
6
6
|
require 'forwardable'
|
7
7
|
require 'json'
|
8
|
+
require 'logger'
|
8
9
|
require 'net/hippie'
|
9
10
|
require 'nokogiri'
|
10
11
|
require 'pathname'
|
@@ -40,6 +41,8 @@ module Spandx
|
|
40
41
|
class Error < StandardError; end
|
41
42
|
|
42
43
|
class << self
|
44
|
+
attr_writer :logger
|
45
|
+
|
43
46
|
def root
|
44
47
|
Pathname.new(File.dirname(__FILE__)).join('../..')
|
45
48
|
end
|
@@ -48,6 +51,10 @@ module Spandx
|
|
48
51
|
@http ||= Spandx::Gateways::Http.new
|
49
52
|
end
|
50
53
|
|
54
|
+
def logger
|
55
|
+
@logger ||= Logger.new('/dev/null')
|
56
|
+
end
|
57
|
+
|
51
58
|
def spdx_db
|
52
59
|
@spdx_db ||= Spandx::Core::Database
|
53
60
|
.new(url: 'https://github.com/spdx/license-list-data.git')
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: spandx
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- mo khan
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-03-
|
11
|
+
date: 2020-03-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: addressable
|