aspera-cli 4.24.1 → 4.25.0.pre
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of aspera-cli might be problematic. Click here for more details.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +1064 -745
- data/CONTRIBUTING.md +43 -100
- data/README.md +1281 -720
- data/bin/ascli +20 -1
- data/bin/asession +23 -27
- data/lib/aspera/agent/base.rb +10 -21
- data/lib/aspera/agent/connect.rb +2 -3
- data/lib/aspera/agent/desktop.rb +2 -2
- data/lib/aspera/agent/direct.rb +49 -32
- data/lib/aspera/agent/factory.rb +31 -0
- data/lib/aspera/api/aoc.rb +134 -76
- data/lib/aspera/api/cos_node.rb +3 -2
- data/lib/aspera/api/faspex.rb +213 -0
- data/lib/aspera/api/node.rb +107 -94
- data/lib/aspera/ascmd.rb +1 -2
- data/lib/aspera/ascp/installation.rb +73 -58
- data/lib/aspera/ascp/management.rb +119 -23
- data/lib/aspera/assert.rb +39 -11
- data/lib/aspera/cli/error.rb +4 -2
- data/lib/aspera/cli/extended_value.rb +91 -67
- data/lib/aspera/cli/formatter.rb +62 -27
- data/lib/aspera/cli/hints.rb +8 -0
- data/lib/aspera/cli/info.rb +4 -4
- data/lib/aspera/cli/main.rb +76 -84
- data/lib/aspera/cli/manager.rb +352 -248
- data/lib/aspera/cli/plugins/alee.rb +5 -4
- data/lib/aspera/cli/plugins/aoc.rb +175 -195
- data/lib/aspera/cli/plugins/ats.rb +4 -4
- data/lib/aspera/cli/plugins/base.rb +343 -0
- data/lib/aspera/cli/plugins/basic_auth.rb +45 -0
- data/lib/aspera/cli/plugins/config.rb +283 -269
- data/lib/aspera/cli/plugins/console.rb +27 -22
- data/lib/aspera/cli/plugins/cos.rb +3 -3
- data/lib/aspera/cli/plugins/factory.rb +78 -0
- data/lib/aspera/cli/plugins/faspex.rb +49 -46
- data/lib/aspera/cli/plugins/faspex5.rb +113 -225
- data/lib/aspera/cli/plugins/faspio.rb +19 -18
- data/lib/aspera/cli/plugins/httpgw.rb +14 -13
- data/lib/aspera/cli/plugins/node.rb +162 -149
- data/lib/aspera/cli/plugins/oauth.rb +48 -0
- data/lib/aspera/cli/plugins/orchestrator.rb +129 -45
- data/lib/aspera/cli/plugins/preview.rb +30 -50
- data/lib/aspera/cli/plugins/server.rb +21 -21
- data/lib/aspera/cli/plugins/shares.rb +45 -47
- data/lib/aspera/cli/sync_actions.rb +50 -39
- data/lib/aspera/cli/transfer_agent.rb +35 -49
- data/lib/aspera/cli/transfer_progress.rb +6 -6
- data/lib/aspera/cli/version.rb +3 -3
- data/lib/aspera/cli/wizard.rb +70 -55
- data/lib/aspera/colors.rb +6 -0
- data/lib/aspera/command_line_builder.rb +59 -61
- data/lib/aspera/command_line_converter.rb +2 -1
- data/lib/aspera/coverage.rb +2 -2
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +51 -41
- data/lib/aspera/faspex_gw.rb +7 -5
- data/lib/aspera/faspex_postproc.rb +1 -1
- data/lib/aspera/keychain/factory.rb +1 -2
- data/lib/aspera/keychain/macos_security.rb +1 -1
- data/lib/aspera/log.rb +37 -9
- data/lib/aspera/markdown.rb +31 -0
- data/lib/aspera/nagios.rb +7 -6
- data/lib/aspera/oauth/base.rb +25 -28
- data/lib/aspera/oauth/factory.rb +9 -9
- data/lib/aspera/oauth/url_json.rb +2 -1
- data/lib/aspera/oauth/web.rb +2 -2
- data/lib/aspera/preview/file_types.rb +23 -37
- data/lib/aspera/products/connect.rb +7 -6
- data/lib/aspera/products/desktop.rb +1 -4
- data/lib/aspera/products/other.rb +9 -1
- data/lib/aspera/products/transferd.rb +0 -1
- data/lib/aspera/rest.rb +168 -113
- data/lib/aspera/rest_error_analyzer.rb +4 -4
- data/lib/aspera/ssh.rb +7 -4
- data/lib/aspera/ssl.rb +41 -0
- data/lib/aspera/sync/args.schema.yaml +46 -3
- data/lib/aspera/sync/conf.schema.yaml +307 -123
- data/lib/aspera/sync/database.rb +2 -1
- data/lib/aspera/sync/operations.rb +135 -79
- data/lib/aspera/temp_file_manager.rb +17 -5
- data/lib/aspera/transfer/error.rb +16 -7
- data/lib/aspera/transfer/parameters.rb +35 -22
- data/lib/aspera/transfer/resumer.rb +74 -0
- data/lib/aspera/transfer/spec.rb +5 -5
- data/lib/aspera/transfer/spec.schema.yaml +170 -59
- data/lib/aspera/transfer/spec_doc.rb +49 -43
- data/lib/aspera/uri_reader.rb +2 -2
- data/lib/aspera/web_auth.rb +6 -6
- data/lib/transferd_pb.rb +2 -2
- data.tar.gz.sig +0 -0
- metadata +26 -11
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/basic_auth_plugin.rb +0 -43
- data/lib/aspera/cli/plugin.rb +0 -333
- data/lib/aspera/cli/plugin_factory.rb +0 -81
- data/lib/aspera/resumer.rb +0 -77
- data/lib/aspera/transfer/error_info.rb +0 -91
|
@@ -8,7 +8,7 @@ require 'aspera/log'
|
|
|
8
8
|
require 'aspera/assert'
|
|
9
9
|
|
|
10
10
|
module Aspera
|
|
11
|
-
#
|
|
11
|
+
# Start a Faspex-4 style post-processing script using Faspex-5 webhook
|
|
12
12
|
class Faspex4PostProcServlet < WEBrick::HTTPServlet::AbstractServlet
|
|
13
13
|
ALLOWED_PARAMETERS = %i[root script_folder fail_on_error timeout_seconds].freeze
|
|
14
14
|
def initialize(server, parameters)
|
|
@@ -12,8 +12,7 @@ module Aspera
|
|
|
12
12
|
# @param folder [String] folder to store the vault (if needed)
|
|
13
13
|
# @param password [String] password to open the vault
|
|
14
14
|
def create(info, name, folder, password)
|
|
15
|
-
Aspera.
|
|
16
|
-
Aspera.assert(info.values.all?(String)){'vault info shall have only string values'}
|
|
15
|
+
Aspera.assert_hash_all(info, Symbol, String){'vault info shall have only string values'}
|
|
17
16
|
info = info.symbolize_keys
|
|
18
17
|
vault_type = info.delete(:type)
|
|
19
18
|
Aspera.assert_values(vault_type, LIST.map(&:to_s)){'vault.type'}
|
|
@@ -73,7 +73,7 @@ module Aspera
|
|
|
73
73
|
end
|
|
74
74
|
|
|
75
75
|
def list(options = {})
|
|
76
|
-
Aspera.assert_values(options[:domain], DOMAINS, type:
|
|
76
|
+
Aspera.assert_values(options[:domain], DOMAINS, type: ParameterError){'domain'} unless options[:domain].nil?
|
|
77
77
|
key_chains(execute('list-keychains', options, LIST_OPTIONS))
|
|
78
78
|
end
|
|
79
79
|
|
data/lib/aspera/log.rb
CHANGED
|
@@ -15,7 +15,7 @@ $VERBOSE = nil
|
|
|
15
15
|
|
|
16
16
|
# Extend Ruby logger with trace levels
|
|
17
17
|
class Logger
|
|
18
|
-
# Two
|
|
18
|
+
# Two additional trace levels
|
|
19
19
|
TRACE_MAX = 2
|
|
20
20
|
|
|
21
21
|
# Add custom level to logger severity, below debug level
|
|
@@ -23,9 +23,12 @@ class Logger
|
|
|
23
23
|
1.upto(TRACE_MAX).each{ |level| const_set(:"TRACE#{level}", - level)}
|
|
24
24
|
end
|
|
25
25
|
|
|
26
|
-
# Hash
|
|
26
|
+
# Hash
|
|
27
|
+
# key [Integer] Log level (e.g. 0 for DEBUG)
|
|
28
|
+
# value [Symbol] Uppercase log level label (e.g. :DEBUG)
|
|
27
29
|
SEVERITY_LABEL = Severity.constants.each_with_object({}){ |name, hash| hash[Severity.const_get(name)] = name}
|
|
28
30
|
|
|
31
|
+
# Override
|
|
29
32
|
# @param severity [Integer] Log severity as int
|
|
30
33
|
# @return [String] Log severity upper case label
|
|
31
34
|
def format_severity(severity)
|
|
@@ -106,6 +109,31 @@ module Aspera
|
|
|
106
109
|
ensure
|
|
107
110
|
$stderr = real_stderr
|
|
108
111
|
end
|
|
112
|
+
|
|
113
|
+
# Returns the last 2 containers (module/class) and method caller
|
|
114
|
+
def caller_method
|
|
115
|
+
stack = caller
|
|
116
|
+
i = stack.rindex{ |line| line.include?('Logger')}
|
|
117
|
+
frame = stack[i + 1] if i && stack[i + 1]
|
|
118
|
+
return '???' unless frame
|
|
119
|
+
# Extract the "Class::Module::Method" or "Class#method" part
|
|
120
|
+
full = frame[/'([^']+)'/, 1]
|
|
121
|
+
return '???' unless full
|
|
122
|
+
# Split into class/module and method parts
|
|
123
|
+
parts = full.split(/(::|#)/)
|
|
124
|
+
# Reconstruct keeping only last two class/module names + separator + method
|
|
125
|
+
if parts.include?('#')
|
|
126
|
+
sep_index = parts.index('#')
|
|
127
|
+
classes = parts[0...sep_index].join
|
|
128
|
+
method = parts[sep_index + 1]
|
|
129
|
+
else
|
|
130
|
+
classes = parts[0..-2].join
|
|
131
|
+
method = parts.last
|
|
132
|
+
end
|
|
133
|
+
class_parts = classes.split('::')
|
|
134
|
+
selected_classes = class_parts.last(2).join('::')
|
|
135
|
+
"#{selected_classes}.#{method}"
|
|
136
|
+
end
|
|
109
137
|
end
|
|
110
138
|
|
|
111
139
|
attr_reader :logger_type, :logger
|
|
@@ -119,6 +147,7 @@ module Aspera
|
|
|
119
147
|
# Set log level of underlying logger given symbol level
|
|
120
148
|
# @param new_level [Symbol] One of LEVELS
|
|
121
149
|
def level=(new_level)
|
|
150
|
+
Aspera.assert_values(new_level, LEVELS)
|
|
122
151
|
@logger.level = Logger::Severity.const_get(new_level.to_sym.upcase)
|
|
123
152
|
end
|
|
124
153
|
|
|
@@ -140,10 +169,8 @@ module Aspera
|
|
|
140
169
|
# Get symbol of debug level of underlying logger
|
|
141
170
|
# @return [Symbol] One of LEVELS
|
|
142
171
|
def level
|
|
143
|
-
Logger::
|
|
144
|
-
|
|
145
|
-
end
|
|
146
|
-
Aspera.error_unexpected_value(@logger.level){'log level'}
|
|
172
|
+
Aspera.assert(Logger::SEVERITY_LABEL.key?(@logger.level))
|
|
173
|
+
Logger::SEVERITY_LABEL[@logger.level].downcase
|
|
147
174
|
end
|
|
148
175
|
|
|
149
176
|
# Change underlying logger, but keep log level
|
|
@@ -158,8 +185,8 @@ module Aspera
|
|
|
158
185
|
@logger = Logger.new($stdout, progname: @program_name, formatter: DEFAULT_FORMATTER)
|
|
159
186
|
when :syslog
|
|
160
187
|
require 'syslog/logger'
|
|
161
|
-
#
|
|
162
|
-
#
|
|
188
|
+
# The syslog class automatically creates methods from the severity names.
|
|
189
|
+
# We just need to add the mapping (but syslog lowest is DEBUG)
|
|
163
190
|
1.upto(Logger::TRACE_MAX).each do |level|
|
|
164
191
|
Syslog::Logger.const_get(:LEVEL_MAP)[Logger.const_get("TRACE#{level}")] = Syslog::LOG_DEBUG
|
|
165
192
|
end
|
|
@@ -207,7 +234,8 @@ module Aspera
|
|
|
207
234
|
# pre-defined formatters
|
|
208
235
|
FORMATTERS = {
|
|
209
236
|
standard: Logger::Formatter.new,
|
|
210
|
-
default: DEFAULT_FORMATTER
|
|
237
|
+
default: DEFAULT_FORMATTER,
|
|
238
|
+
caller: ->(s, _d, _p, m){"#{LVL_COLOR[s]} #{Log.caller_method}\n#{m}\n"}
|
|
211
239
|
}.freeze
|
|
212
240
|
|
|
213
241
|
private_constant :LVL_DECO, :LVL_COLOR, :DEFAULT_FORMATTER, :FORMATTERS
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Aspera
|
|
4
|
+
# Formatting for Markdown
|
|
5
|
+
class Markdown
|
|
6
|
+
# Matches: **bold**, `code`, or an HTML entity (&, ©, 💩)
|
|
7
|
+
FORMATS = /(?:\*\*(?<bold>[^*]+?)\*\*)|(?:`(?<code>[^`]+)`)|&(?<entity>(?:[A-Za-z][A-Za-z0-9]{1,31}|#\d{1,7}|#x[0-9A-Fa-f]{1,6}));/m
|
|
8
|
+
HTML_BREAK = '<br/>'
|
|
9
|
+
|
|
10
|
+
class << self
|
|
11
|
+
# Generate markdown from the provided 2D table
|
|
12
|
+
def table(table)
|
|
13
|
+
# get max width of each columns
|
|
14
|
+
col_widths = table.transpose.map do |col|
|
|
15
|
+
[col.flat_map{ |c| c.to_s.delete('`').split(HTML_BREAK).map(&:size)}.max, 80].min
|
|
16
|
+
end
|
|
17
|
+
headings = table.shift
|
|
18
|
+
table.unshift(col_widths.map{ |col_width| '-' * col_width})
|
|
19
|
+
table.unshift(headings)
|
|
20
|
+
lines = table.map{ |line| "| #{line.map{ |i| i.to_s.gsub('\\', '\\\\').gsub('|', '\|')}.join(' | ')} |\n"}
|
|
21
|
+
lines[1] = lines[1].tr(' ', '-')
|
|
22
|
+
return lines.join.chomp
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Generate markdown list from the provided list
|
|
26
|
+
def list(items)
|
|
27
|
+
items.map{ |i| "- #{i}"}.join("\n")
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
data/lib/aspera/nagios.rb
CHANGED
|
@@ -21,7 +21,7 @@ module Aspera
|
|
|
21
21
|
end
|
|
22
22
|
|
|
23
23
|
class << self
|
|
24
|
-
#
|
|
24
|
+
# Process results of a analysis and display status and exit with code
|
|
25
25
|
def process(data)
|
|
26
26
|
Aspera.assert_type(data, Array)
|
|
27
27
|
Aspera.assert(!data.empty?){'data is empty'}
|
|
@@ -54,10 +54,10 @@ module Aspera
|
|
|
54
54
|
@data = []
|
|
55
55
|
end
|
|
56
56
|
|
|
57
|
-
#
|
|
57
|
+
# Compare remote time with local time
|
|
58
58
|
def check_time_offset(remote_date, component)
|
|
59
59
|
# check date if specified : 2015-10-13T07:32:01Z
|
|
60
|
-
remote_time = Time.
|
|
60
|
+
remote_time = Time.parse(remote_date)
|
|
61
61
|
diff_time = (remote_time - Time.now).abs
|
|
62
62
|
diff_rounded = diff_time.round(-2)
|
|
63
63
|
Log.log.debug{"DATE: #{remote_date} #{remote_time} diff=#{diff_rounded}"}
|
|
@@ -76,10 +76,11 @@ module Aspera
|
|
|
76
76
|
# TODO: check on database if latest version
|
|
77
77
|
end
|
|
78
78
|
|
|
79
|
-
#
|
|
80
|
-
|
|
79
|
+
# Readable status list
|
|
80
|
+
# @return [Array] of Hash
|
|
81
|
+
def status_list
|
|
81
82
|
Aspera.assert(!@data.empty?){'missing result'}
|
|
82
|
-
|
|
83
|
+
@data.map{ |i| {'status' => LEVELS[i[:code]].to_s, 'component' => i[:comp], 'message' => i[:msg]}}
|
|
83
84
|
end
|
|
84
85
|
end
|
|
85
86
|
end
|
data/lib/aspera/oauth/base.rb
CHANGED
|
@@ -13,6 +13,7 @@ module Aspera
|
|
|
13
13
|
# OAuth 2.0 Authorization Framework: https://tools.ietf.org/html/rfc6749
|
|
14
14
|
# Bearer Token Usage: https://tools.ietf.org/html/rfc6750
|
|
15
15
|
class Base
|
|
16
|
+
Aspera.require_method!(:create_token)
|
|
16
17
|
# @param ** Parameters for REST
|
|
17
18
|
# @param client_id [String, nil]
|
|
18
19
|
# @param client_secret [String, nil]
|
|
@@ -31,8 +32,7 @@ module Aspera
|
|
|
31
32
|
cache_ids: nil,
|
|
32
33
|
**rest_params
|
|
33
34
|
)
|
|
34
|
-
|
|
35
|
-
# this is the OAuth API
|
|
35
|
+
# This is the OAuth API
|
|
36
36
|
@api = Rest.new(**rest_params)
|
|
37
37
|
@scope = nil
|
|
38
38
|
@token_cache_id = nil
|
|
@@ -43,6 +43,7 @@ module Aspera
|
|
|
43
43
|
@use_query = use_query
|
|
44
44
|
@base_cache_ids = cache_ids.nil? ? [] : cache_ids.clone
|
|
45
45
|
Aspera.assert_type(@base_cache_ids, Array)
|
|
46
|
+
# TODO: this shall be done in class, using cache_ids
|
|
46
47
|
@base_cache_ids.push(@api.auth_params[:username]) if @api.auth_params.key?(:username)
|
|
47
48
|
@base_cache_ids.compact!
|
|
48
49
|
@base_cache_ids.freeze
|
|
@@ -56,31 +57,24 @@ module Aspera
|
|
|
56
57
|
@token_cache_id = Factory.cache_id(@api.base_url, self.class, @base_cache_ids, @scope)
|
|
57
58
|
end
|
|
58
59
|
|
|
59
|
-
attr_reader :scope, :api, :path_token
|
|
60
|
+
attr_reader :scope, :api, :path_token, :client_id
|
|
60
61
|
|
|
61
|
-
#
|
|
62
|
+
# Helper method to create token as per RFC
|
|
63
|
+
# @return [HTTPResponse]
|
|
64
|
+
# @raise RestError if not 2XX code
|
|
62
65
|
def create_token_call(creation_params)
|
|
63
66
|
Log.log.debug{'Generating a new token'.bg_green}
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
payload[:query] = creation_params
|
|
67
|
-
else
|
|
68
|
-
payload[:body] = creation_params
|
|
69
|
-
end
|
|
70
|
-
return @api.call(
|
|
71
|
-
operation: 'POST',
|
|
72
|
-
subpath: @path_token,
|
|
73
|
-
headers: {'Accept' => Rest::MIME_JSON},
|
|
74
|
-
**payload
|
|
75
|
-
)
|
|
67
|
+
return @api.create(@path_token, nil, query: creation_params, ret: :resp) if @use_query
|
|
68
|
+
return @api.create(@path_token, creation_params, content_type: Rest::MIME_WWW, ret: :resp)
|
|
76
69
|
end
|
|
77
70
|
|
|
78
|
-
# @
|
|
71
|
+
# @param add_secret [Boolean] Add secret in default call parameters
|
|
72
|
+
# @return [Hash] Optional general parameters
|
|
79
73
|
def optional_scope_client_id(add_secret: false)
|
|
80
74
|
call_params = {}
|
|
81
75
|
call_params[:scope] = @scope unless @scope.nil?
|
|
82
76
|
call_params[:client_id] = @client_id unless @client_id.nil?
|
|
83
|
-
call_params[:client_secret] = @client_secret
|
|
77
|
+
call_params[:client_secret] = @client_secret unless !add_secret || @client_id.nil? || @client_secret.nil?
|
|
84
78
|
return call_params
|
|
85
79
|
end
|
|
86
80
|
|
|
@@ -106,6 +100,7 @@ module Aspera
|
|
|
106
100
|
# `direct` agent is equipped with refresh code
|
|
107
101
|
# an API was already called, but failed, we need to regenerate or refresh
|
|
108
102
|
if refresh || token_info[:expired]
|
|
103
|
+
Log.log.trace1{"refresh: #{refresh} expired: #{token_info[:expired]}"}
|
|
109
104
|
refresh_token = nil
|
|
110
105
|
if token_data.key?('refresh_token') && !token_data['refresh_token'].eql?('not_supported')
|
|
111
106
|
# save possible refresh token, before deleting the cache
|
|
@@ -115,17 +110,18 @@ module Aspera
|
|
|
115
110
|
Factory.instance.persist_mgr.delete(@token_cache_id)
|
|
116
111
|
token_data = nil
|
|
117
112
|
# lets try the existing refresh token
|
|
113
|
+
# NOTE: AoC admin token has no refresh, and lives by default 1800secs
|
|
118
114
|
if !refresh_token.nil?
|
|
119
|
-
Log.log.info{"refresh=[#{refresh_token}]".bg_green}
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
json_data = resp[:http].body
|
|
115
|
+
Log.log.info{"refresh token=[#{refresh_token}]".bg_green}
|
|
116
|
+
begin
|
|
117
|
+
http = create_token_call(optional_scope_client_id(add_secret: true).merge(grant_type: 'refresh_token', refresh_token: refresh_token))
|
|
118
|
+
# Save only if success
|
|
119
|
+
json_data = http.body
|
|
125
120
|
token_data = JSON.parse(json_data)
|
|
126
121
|
Factory.instance.persist_mgr.put(@token_cache_id, json_data)
|
|
127
|
-
|
|
128
|
-
|
|
122
|
+
rescue => e
|
|
123
|
+
# Refresh token can fail.
|
|
124
|
+
Log.log.warn{"Refresh failed: #{e}"}
|
|
129
125
|
end
|
|
130
126
|
end
|
|
131
127
|
end
|
|
@@ -133,8 +129,9 @@ module Aspera
|
|
|
133
129
|
|
|
134
130
|
# no cache, nor refresh: generate a token
|
|
135
131
|
if token_data.nil?
|
|
136
|
-
|
|
137
|
-
|
|
132
|
+
# Call the method-specific token creation
|
|
133
|
+
# which returns the result of create_token_call
|
|
134
|
+
json_data = create_token.body
|
|
138
135
|
token_data = JSON.parse(json_data)
|
|
139
136
|
Factory.instance.persist_mgr.put(@token_cache_id, json_data)
|
|
140
137
|
end
|
data/lib/aspera/oauth/factory.rb
CHANGED
|
@@ -63,10 +63,10 @@ module Aspera
|
|
|
63
63
|
@decoders = []
|
|
64
64
|
# default parameters, others can be added by handlers
|
|
65
65
|
@parameters = {
|
|
66
|
-
# tokens older than
|
|
67
|
-
|
|
68
|
-
# tokens valid for less than this duration will be regenerated
|
|
69
|
-
|
|
66
|
+
# tokens older than this duration in sec. will be discarded from cache
|
|
67
|
+
token_cache_max_age: 1800,
|
|
68
|
+
# tokens valid for less than this duration in sec. will be regenerated
|
|
69
|
+
token_refresh_threshold: 120
|
|
70
70
|
}
|
|
71
71
|
end
|
|
72
72
|
|
|
@@ -77,7 +77,7 @@ module Aspera
|
|
|
77
77
|
def persist_mgr=(manager)
|
|
78
78
|
@persist = manager
|
|
79
79
|
# cleanup expired tokens
|
|
80
|
-
@persist.garbage_collect(PERSIST_CATEGORY_TOKEN, @parameters[:
|
|
80
|
+
@persist.garbage_collect(PERSIST_CATEGORY_TOKEN, @parameters[:token_cache_max_age])
|
|
81
81
|
end
|
|
82
82
|
|
|
83
83
|
def persist_mgr
|
|
@@ -108,7 +108,7 @@ module Aspera
|
|
|
108
108
|
end
|
|
109
109
|
end
|
|
110
110
|
|
|
111
|
-
#
|
|
111
|
+
# Get token information from cache
|
|
112
112
|
# @param id [String] identifier of token
|
|
113
113
|
# @return [Hash] token internal information , including Date object for `expiration_date`
|
|
114
114
|
def get_token_info(id)
|
|
@@ -118,7 +118,6 @@ module Aspera
|
|
|
118
118
|
Aspera.assert_type(token_data, Hash)
|
|
119
119
|
decoded_token = decode_token(token_data[TOKEN_FIELD])
|
|
120
120
|
info = {data: token_data}
|
|
121
|
-
Log.dump(:decoded_token, decoded_token)
|
|
122
121
|
if decoded_token.is_a?(Hash)
|
|
123
122
|
info[:decoded] = decoded_token
|
|
124
123
|
# TODO: move date decoding to token decoder ?
|
|
@@ -129,9 +128,10 @@ module Aspera
|
|
|
129
128
|
unless expiration_date.nil?
|
|
130
129
|
info[:expiration] = expiration_date
|
|
131
130
|
info[:ttl_sec] = expiration_date - Time.now
|
|
132
|
-
info[:expired] = info[:ttl_sec] < @parameters[:
|
|
131
|
+
info[:expired] = info[:ttl_sec] < @parameters[:token_refresh_threshold]
|
|
133
132
|
end
|
|
134
133
|
end
|
|
134
|
+
Log.dump(:token_info, info)
|
|
135
135
|
return info
|
|
136
136
|
end
|
|
137
137
|
|
|
@@ -156,7 +156,7 @@ module Aspera
|
|
|
156
156
|
def register_token_creator(creator_class)
|
|
157
157
|
Aspera.assert_type(creator_class, Class)
|
|
158
158
|
id = Factory.class_to_id(creator_class)
|
|
159
|
-
Log.log.debug{"registering
|
|
159
|
+
Log.log.debug{"registering creator for #{id}"}
|
|
160
160
|
@token_type_classes[id] = creator_class
|
|
161
161
|
end
|
|
162
162
|
|
|
@@ -25,7 +25,8 @@ module Aspera
|
|
|
25
25
|
query: @query.merge(scope: scope), # scope is here because it may change over time (node)
|
|
26
26
|
content_type: Rest::MIME_JSON,
|
|
27
27
|
body: @body,
|
|
28
|
-
headers: {'Accept' => Rest::MIME_JSON}
|
|
28
|
+
headers: {'Accept' => Rest::MIME_JSON},
|
|
29
|
+
ret: :resp
|
|
29
30
|
)
|
|
30
31
|
end
|
|
31
32
|
end
|
data/lib/aspera/oauth/web.rb
CHANGED
|
@@ -9,7 +9,7 @@ module Aspera
|
|
|
9
9
|
# Authentication using Web browser
|
|
10
10
|
class Web < Base
|
|
11
11
|
class << self
|
|
12
|
-
attr_accessor :
|
|
12
|
+
attr_accessor :additional_info
|
|
13
13
|
end
|
|
14
14
|
# @param redirect_uri url to receive the code after auth (to be exchanged for token)
|
|
15
15
|
# @param path_authorize path to login page on web app
|
|
@@ -37,7 +37,7 @@ module Aspera
|
|
|
37
37
|
# here, we need a human to authorize on a web page
|
|
38
38
|
Log.log.info{"login_page_url=#{login_page_url}".bg_red.gray}
|
|
39
39
|
# start a web server to receive request code
|
|
40
|
-
web_server = WebAuth.new(@redirect_uri, self.class.
|
|
40
|
+
web_server = WebAuth.new(@redirect_uri, self.class.additional_info)
|
|
41
41
|
# start browser on login page
|
|
42
42
|
Environment.instance.open_uri(login_page_url)
|
|
43
43
|
# wait for code in request
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
require 'aspera/log'
|
|
4
4
|
require 'aspera/assert'
|
|
5
5
|
require 'singleton'
|
|
6
|
-
require '
|
|
6
|
+
require 'marcel'
|
|
7
7
|
|
|
8
8
|
module Aspera
|
|
9
9
|
module Preview
|
|
@@ -61,7 +61,7 @@ module Aspera
|
|
|
61
61
|
end
|
|
62
62
|
|
|
63
63
|
# @param mimetype [String] mime type
|
|
64
|
-
# @return file type, one of enum CONVERSION_TYPES, or nil if not found
|
|
64
|
+
# @return [NilClass,Symbol] file type, one of enum CONVERSION_TYPES, or nil if not found
|
|
65
65
|
def mime_to_type(mimetype)
|
|
66
66
|
Aspera.assert_type(mimetype, String)
|
|
67
67
|
return SUPPORTED_MIME_TYPES[mimetype] if SUPPORTED_MIME_TYPES.key?(mimetype)
|
|
@@ -72,19 +72,17 @@ module Aspera
|
|
|
72
72
|
return
|
|
73
73
|
end
|
|
74
74
|
|
|
75
|
-
# @param filepath [String]
|
|
76
|
-
# @param mimetype [String] provided by node API
|
|
75
|
+
# @param filepath [String] Full path to file
|
|
76
|
+
# @param mimetype [String] MIME typre provided by node API
|
|
77
77
|
# @return file type, one of enum CONVERSION_TYPES
|
|
78
78
|
# @raise [RuntimeError] if no conversion type found
|
|
79
79
|
def conversion_type(filepath, mimetype)
|
|
80
80
|
Log.log.debug{"conversion_type(#{filepath},mime=#{mimetype},magic=#{@use_mimemagic})"}
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
mimetype
|
|
84
|
-
mimetype
|
|
85
|
-
|
|
86
|
-
mimetype ||= MIME::Types.of(File.basename(filepath)).first
|
|
87
|
-
raise "no MIME type found for #{File.basename(filepath)}" if mimetype.nil?
|
|
81
|
+
# Default type or empty means no type
|
|
82
|
+
mimetype = TYPE_NOT_FOUND if mimetype.nil? || (mimetype.is_a?(String) && mimetype.empty?)
|
|
83
|
+
mimetype = Marcel::MimeType.for(Pathname.new(filepath), name: File.basename(filepath), declared_type: mimetype)
|
|
84
|
+
mimetype = 'text/plain' if mimetype.eql?(TYPE_NOT_FOUND) && ascii_text_file?(filepath)
|
|
85
|
+
raise "no MIME type found for #{File.basename(filepath)}" if mimetype.eql?(TYPE_NOT_FOUND)
|
|
88
86
|
conversion_type = mime_to_type(mimetype)
|
|
89
87
|
raise "no conversion type found for #{File.basename(filepath)}" if conversion_type.nil?
|
|
90
88
|
Log.log.trace1{"conversion_type(#{File.basename(filepath)}): #{conversion_type.class.name} [#{conversion_type}]"}
|
|
@@ -93,33 +91,21 @@ module Aspera
|
|
|
93
91
|
|
|
94
92
|
private
|
|
95
93
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
# @return [String] mime type, or nil if not found
|
|
99
|
-
def mime_using_mimemagic(filepath)
|
|
100
|
-
return unless @use_mimemagic
|
|
101
|
-
# moved here, as `mimemagic` can cause installation issues
|
|
102
|
-
require 'mimemagic'
|
|
103
|
-
require 'mimemagic/version'
|
|
104
|
-
require 'mimemagic/overlay' if MimeMagic::VERSION.start_with?('0.3.')
|
|
105
|
-
# check magic number inside file (empty string if not found)
|
|
106
|
-
detected_mime = MimeMagic.by_magic(File.open(filepath)).to_s
|
|
107
|
-
# check extension only
|
|
108
|
-
if mime_to_type(detected_mime).nil?
|
|
109
|
-
Log.log.debug{"no conversion for #{detected_mime}, trying extension"}
|
|
110
|
-
detected_mime = MimeMagic.by_extension(File.extname(filepath)).to_s
|
|
111
|
-
end
|
|
112
|
-
detected_mime = nil if detected_mime.empty?
|
|
113
|
-
Log.log.debug{"mimemagic: #{detected_mime.class.name} [#{detected_mime}]"}
|
|
114
|
-
return detected_mime
|
|
115
|
-
end
|
|
94
|
+
TYPE_NOT_FOUND = 'application/octet-stream'
|
|
95
|
+
ACCEPT_CTRL_CHARS = [9, 10, 13]
|
|
116
96
|
|
|
117
|
-
#
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
97
|
+
# Returns true if the file looks like ASCII text (printable ASCII + \t, \r, \n, space).
|
|
98
|
+
# It reads only a small prefix (default: 64KB) and fails fast on the first bad byte.
|
|
99
|
+
def ascii_text_file?(path, sample_size: 64 * 1024)
|
|
100
|
+
File.open(path, 'rb') do |f|
|
|
101
|
+
sample = f.read(sample_size) || ''.b
|
|
102
|
+
sample.each_byte do |b|
|
|
103
|
+
next if b.between?(32, 126) || ACCEPT_CTRL_CHARS.include?(b)
|
|
104
|
+
# Any other control character => not ASCII text
|
|
105
|
+
return false
|
|
106
|
+
end
|
|
107
|
+
true
|
|
108
|
+
end
|
|
123
109
|
end
|
|
124
110
|
end
|
|
125
111
|
end
|
|
@@ -48,17 +48,18 @@ module Aspera
|
|
|
48
48
|
end
|
|
49
49
|
end
|
|
50
50
|
|
|
51
|
+
# Base URL for CDN of Connect
|
|
51
52
|
def cdn_api
|
|
52
53
|
Rest.new(base_url: CDN_BASE_URL)
|
|
53
54
|
end
|
|
54
55
|
|
|
55
|
-
#
|
|
56
|
+
# Retrieve structure from cloud (CDN) with all versions available
|
|
56
57
|
def versions
|
|
57
58
|
if @connect_versions.nil?
|
|
58
|
-
|
|
59
|
+
http = cdn_api.read(VERSION_INFO_FILE, ret: :resp)
|
|
59
60
|
# get result on one line
|
|
60
|
-
connect_versions_javascript =
|
|
61
|
-
Log.
|
|
61
|
+
connect_versions_javascript = http.body.gsub(/\r?\n\s*/, '')
|
|
62
|
+
Log.dump(:javascript, connect_versions_javascript)
|
|
62
63
|
# get javascript object only
|
|
63
64
|
found = connect_versions_javascript.match(/^.*? = (.*);/)
|
|
64
65
|
raise Cli::Error, 'Problem when getting connect versions from internet' if found.nil?
|
|
@@ -74,10 +75,10 @@ module Aspera
|
|
|
74
75
|
@connect_versions = nil
|
|
75
76
|
end
|
|
76
77
|
|
|
77
|
-
VERSION_INFO_FILE = 'connectversions.js' # cspell: disable-line
|
|
78
78
|
CDN_BASE_URL = 'https://d3gcli72yxqn2z.cloudfront.net/connect'
|
|
79
|
+
VERSION_INFO_FILE = 'connectversions.js' # cspell: disable-line
|
|
79
80
|
|
|
80
|
-
private_constant :
|
|
81
|
+
private_constant :CDN_BASE_URL, :VERSION_INFO_FILE
|
|
81
82
|
end
|
|
82
83
|
end
|
|
83
84
|
end
|
|
@@ -8,6 +8,7 @@ module Aspera
|
|
|
8
8
|
class Desktop
|
|
9
9
|
APP_NAME = 'IBM Aspera for Desktop'
|
|
10
10
|
APP_IDENTIFIER = 'com.ibm.software.aspera.desktop'
|
|
11
|
+
LOG_FILENAME = 'ibm-aspera-desktop.log'
|
|
11
12
|
class << self
|
|
12
13
|
# standard folder locations
|
|
13
14
|
def locations
|
|
@@ -20,10 +21,6 @@ module Aspera
|
|
|
20
21
|
else []
|
|
21
22
|
end.map{ |i| i.merge({expected: APP_NAME})}
|
|
22
23
|
end
|
|
23
|
-
|
|
24
|
-
def log_file
|
|
25
|
-
File.join(Dir.home, 'Library', 'Logs', APP_IDENTIFIER, 'ibm-aspera-desktop.log')
|
|
26
|
-
end
|
|
27
24
|
end
|
|
28
25
|
end
|
|
29
26
|
end
|
|
@@ -54,8 +54,14 @@ module Aspera
|
|
|
54
54
|
}]
|
|
55
55
|
end
|
|
56
56
|
class << self
|
|
57
|
+
# Find installed products and provide paths for it.
|
|
58
|
+
# @param scan_locations [Array] Array of Hash with keys: expected, app_root, sub_bin, ascp_path, name, version
|
|
59
|
+
# @return [Array] of products found, with filled missing fields
|
|
60
|
+
# @raise Exception if no installed product found
|
|
57
61
|
def find(scan_locations)
|
|
58
|
-
|
|
62
|
+
product_names = []
|
|
63
|
+
found = scan_locations.select do |item|
|
|
64
|
+
product_names.push(item[:expected]) unless product_names.include?(item[:expected])
|
|
59
65
|
# skip if not main folder
|
|
60
66
|
Log.log.trace1{"Checking #{item[:app_root]}"}
|
|
61
67
|
next false unless Dir.exist?(item[:app_root])
|
|
@@ -75,6 +81,8 @@ module Aspera
|
|
|
75
81
|
end
|
|
76
82
|
true # select this version
|
|
77
83
|
end
|
|
84
|
+
raise "Product: #{product_names.join(', ')} not found, please install." if found.empty?
|
|
85
|
+
found
|
|
78
86
|
end
|
|
79
87
|
end
|
|
80
88
|
end
|