aspera-cli 4.4.0 → 4.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2095 -1503
- data/bin/ascli +2 -1
- data/bin/asession +4 -5
- data/docs/test_env.conf +3 -0
- data/examples/aoc.rb +4 -3
- data/examples/faspex4.rb +25 -25
- data/examples/proxy.pac +1 -1
- data/examples/transfer.rb +17 -17
- data/lib/aspera/aoc.rb +238 -185
- data/lib/aspera/ascmd.rb +93 -83
- data/lib/aspera/ats_api.rb +11 -10
- data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
- data/lib/aspera/cli/extended_value.rb +42 -33
- data/lib/aspera/cli/formater.rb +142 -108
- data/lib/aspera/cli/info.rb +17 -0
- data/lib/aspera/cli/listener/line_dump.rb +3 -2
- data/lib/aspera/cli/listener/logger.rb +2 -1
- data/lib/aspera/cli/listener/progress.rb +16 -18
- data/lib/aspera/cli/listener/progress_multi.rb +18 -21
- data/lib/aspera/cli/main.rb +173 -149
- data/lib/aspera/cli/manager.rb +163 -168
- data/lib/aspera/cli/plugin.rb +43 -31
- data/lib/aspera/cli/plugins/alee.rb +6 -6
- data/lib/aspera/cli/plugins/aoc.rb +405 -370
- data/lib/aspera/cli/plugins/ats.rb +86 -79
- data/lib/aspera/cli/plugins/bss.rb +14 -16
- data/lib/aspera/cli/plugins/config.rb +580 -362
- data/lib/aspera/cli/plugins/console.rb +23 -19
- data/lib/aspera/cli/plugins/cos.rb +18 -18
- data/lib/aspera/cli/plugins/faspex.rb +201 -158
- data/lib/aspera/cli/plugins/faspex5.rb +80 -57
- data/lib/aspera/cli/plugins/node.rb +183 -166
- data/lib/aspera/cli/plugins/orchestrator.rb +71 -67
- data/lib/aspera/cli/plugins/preview.rb +92 -96
- data/lib/aspera/cli/plugins/server.rb +79 -75
- data/lib/aspera/cli/plugins/shares.rb +35 -19
- data/lib/aspera/cli/plugins/sync.rb +20 -22
- data/lib/aspera/cli/transfer_agent.rb +76 -113
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +35 -27
- data/lib/aspera/command_line_builder.rb +48 -34
- data/lib/aspera/cos_node.rb +29 -21
- data/lib/aspera/data_repository.rb +3 -2
- data/lib/aspera/environment.rb +50 -45
- data/lib/aspera/fasp/{manager.rb → agent_base.rb} +28 -25
- data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +52 -43
- data/lib/aspera/fasp/{local.rb → agent_direct.rb} +58 -72
- data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +37 -43
- data/lib/aspera/fasp/{node.rb → agent_node.rb} +35 -16
- data/lib/aspera/fasp/agent_trsdk.rb +104 -0
- data/lib/aspera/fasp/error.rb +2 -1
- data/lib/aspera/fasp/error_info.rb +68 -52
- data/lib/aspera/fasp/installation.rb +152 -124
- data/lib/aspera/fasp/listener.rb +1 -0
- data/lib/aspera/fasp/parameters.rb +87 -92
- data/lib/aspera/fasp/parameters.yaml +305 -249
- data/lib/aspera/fasp/resume_policy.rb +11 -14
- data/lib/aspera/fasp/transfer_spec.rb +26 -0
- data/lib/aspera/fasp/uri.rb +22 -21
- data/lib/aspera/faspex_gw.rb +55 -89
- data/lib/aspera/hash_ext.rb +4 -3
- data/lib/aspera/id_generator.rb +8 -7
- data/lib/aspera/keychain/encrypted_hash.rb +121 -0
- data/lib/aspera/keychain/macos_security.rb +90 -0
- data/lib/aspera/log.rb +55 -37
- data/lib/aspera/nagios.rb +13 -12
- data/lib/aspera/node.rb +30 -25
- data/lib/aspera/oauth.rb +175 -226
- data/lib/aspera/open_application.rb +4 -3
- data/lib/aspera/persistency_action_once.rb +6 -6
- data/lib/aspera/persistency_folder.rb +5 -9
- data/lib/aspera/preview/file_types.rb +6 -5
- data/lib/aspera/preview/generator.rb +25 -24
- data/lib/aspera/preview/options.rb +16 -14
- data/lib/aspera/preview/utils.rb +98 -98
- data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
- data/lib/aspera/proxy_auto_config.rb +111 -20
- data/lib/aspera/rest.rb +154 -135
- data/lib/aspera/rest_call_error.rb +2 -2
- data/lib/aspera/rest_error_analyzer.rb +23 -25
- data/lib/aspera/rest_errors_aspera.rb +15 -14
- data/lib/aspera/ssh.rb +12 -10
- data/lib/aspera/sync.rb +42 -41
- data/lib/aspera/temp_file_manager.rb +18 -14
- data/lib/aspera/timer_limiter.rb +2 -1
- data/lib/aspera/uri_reader.rb +7 -5
- data/lib/aspera/web_auth.rb +79 -76
- metadata +116 -29
- data/docs/Makefile +0 -66
- data/docs/README.erb.md +0 -3973
- data/docs/README.md +0 -13
- data/docs/diagrams.txt +0 -49
- data/docs/doc_tools.rb +0 -58
- data/lib/aspera/api_detector.rb +0 -60
- data/lib/aspera/cli/plugins/shares2.rb +0 -114
- data/lib/aspera/secrets.rb +0 -20
|
@@ -1,8 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
require 'aspera/fasp/
|
|
3
|
-
require 'aspera/fasp/connect'
|
|
4
|
-
require 'aspera/fasp/node'
|
|
5
|
-
require 'aspera/fasp/http_gw'
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require 'aspera/fasp/transfer_spec'
|
|
6
3
|
require 'aspera/cli/listener/logger'
|
|
7
4
|
require 'aspera/cli/listener/progress_multi'
|
|
8
5
|
|
|
@@ -16,21 +13,23 @@ module Aspera
|
|
|
16
13
|
FILE_LIST_FROM_ARGS='@args'
|
|
17
14
|
# special value for --sources : read file list from transfer spec (--ts)
|
|
18
15
|
FILE_LIST_FROM_TRANSFER_SPEC='@ts'
|
|
19
|
-
DEFAULT_TRANSFER_NOTIF_TMPL
|
|
20
|
-
From: <%=from_name%> <<%=from_email%>>
|
|
21
|
-
To: <<%=to%>>
|
|
22
|
-
Subject: <%=subject%>
|
|
16
|
+
DEFAULT_TRANSFER_NOTIF_TMPL=<<~END_OF_TEMPLATE
|
|
17
|
+
From: <%=from_name%> <<%=from_email%>>
|
|
18
|
+
To: <<%=to%>>
|
|
19
|
+
Subject: <%=subject%>
|
|
23
20
|
|
|
24
|
-
Transfer is: <%=global_transfer_status%>
|
|
21
|
+
Transfer is: <%=global_transfer_status%>
|
|
25
22
|
|
|
26
|
-
<%=ts.to_yaml%>
|
|
27
|
-
END_OF_TEMPLATE
|
|
28
|
-
#%
|
|
23
|
+
<%=ts.to_yaml%>
|
|
24
|
+
END_OF_TEMPLATE
|
|
25
|
+
#% (formating bug in eclipse)
|
|
29
26
|
private_constant :FILE_LIST_FROM_ARGS,:FILE_LIST_FROM_TRANSFER_SPEC,:DEFAULT_TRANSFER_NOTIF_TMPL
|
|
27
|
+
TRANSFER_AGENTS=[:direct,:node,:connect,:httpgw,:trsdk]
|
|
28
|
+
|
|
30
29
|
# @param env external objects: option manager, config file manager
|
|
31
|
-
def initialize(
|
|
32
|
-
|
|
33
|
-
@
|
|
30
|
+
def initialize(opt_mgr,config)
|
|
31
|
+
@opt_mgr=opt_mgr
|
|
32
|
+
@config=config
|
|
34
33
|
# command line can override transfer spec
|
|
35
34
|
@transfer_spec_cmdline={'create_dir'=>true}
|
|
36
35
|
# the currently selected transfer agent
|
|
@@ -38,25 +37,21 @@ END_OF_TEMPLATE
|
|
|
38
37
|
@progress_listener=Listener::ProgressMulti.new
|
|
39
38
|
# source/destination pair, like "paths" of transfer spec
|
|
40
39
|
@transfer_paths=nil
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
40
|
+
@opt_mgr.set_obj_attr(:ts,self,:option_transfer_spec)
|
|
41
|
+
@opt_mgr.add_opt_simple(:ts,"override transfer spec values (Hash, use @json: prefix), current=#{@opt_mgr.get_option(:ts,:optional)}")
|
|
42
|
+
@opt_mgr.add_opt_simple(:local_resume,"set resume policy (Hash, use @json: prefix), current=#{@opt_mgr.get_option(:local_resume,:optional)}")
|
|
43
|
+
@opt_mgr.add_opt_simple(:to_folder,'destination folder for downloaded files')
|
|
44
|
+
@opt_mgr.add_opt_simple(:sources,'list of source files (see doc)')
|
|
45
|
+
@opt_mgr.add_opt_simple(:transfer_info,'parameters for transfer agent')
|
|
46
|
+
@opt_mgr.add_opt_list(:src_type,[:list,:pair],'type of file list')
|
|
47
|
+
@opt_mgr.add_opt_list(:transfer,TRANSFER_AGENTS,'type of transfer agent')
|
|
48
|
+
@opt_mgr.add_opt_list(:progress,[:none,:native,:multi],'type of progress bar')
|
|
49
|
+
@opt_mgr.set_option(:transfer,:direct)
|
|
50
|
+
@opt_mgr.set_option(:src_type,:list)
|
|
51
|
+
@opt_mgr.set_option(:progress,:native) # use native ascp progress bar as it is more reliable
|
|
52
|
+
@opt_mgr.parse_options!
|
|
54
53
|
end
|
|
55
54
|
|
|
56
|
-
def options; @env[:options];end
|
|
57
|
-
|
|
58
|
-
def config; @env[:config];end
|
|
59
|
-
|
|
60
55
|
def option_transfer_spec; @transfer_spec_cmdline; end
|
|
61
56
|
|
|
62
57
|
# multiple option are merged
|
|
@@ -64,12 +59,12 @@ END_OF_TEMPLATE
|
|
|
64
59
|
|
|
65
60
|
def option_transfer_spec_deep_merge(ts); @transfer_spec_cmdline.deep_merge!(ts); end
|
|
66
61
|
|
|
67
|
-
def
|
|
62
|
+
def agent_instance=(instance)
|
|
68
63
|
@agent=instance
|
|
69
64
|
@agent.add_listener(Listener::Logger.new)
|
|
70
65
|
# use local progress bar if asked so, or if native and non local ascp (because only local ascp has native progress bar)
|
|
71
|
-
if
|
|
72
|
-
(
|
|
66
|
+
if @opt_mgr.get_option(:progress,:mandatory).eql?(:multi) ||
|
|
67
|
+
(@opt_mgr.get_option(:progress,:mandatory).eql?(:native) && !instance.class.to_s.eql?('Aspera::Fasp::AgentDirect'))
|
|
73
68
|
@agent.add_listener(@progress_listener)
|
|
74
69
|
end
|
|
75
70
|
end
|
|
@@ -77,57 +72,25 @@ END_OF_TEMPLATE
|
|
|
77
72
|
# analyze options and create new agent if not already created or set
|
|
78
73
|
def set_agent_by_options
|
|
79
74
|
return nil unless @agent.nil?
|
|
80
|
-
agent_type
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
# support: @preset:<name>
|
|
95
|
-
# support extended values
|
|
96
|
-
node_config=options.get_option(:transfer_info,:optional)
|
|
97
|
-
# if not specified: use default node
|
|
98
|
-
if node_config.nil?
|
|
99
|
-
param_set_name=config.get_plugin_default_config_name(:node)
|
|
100
|
-
raise CliBadArgument,"No default node configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
|
|
101
|
-
node_config=config.preset_by_name(param_set_name)
|
|
102
|
-
end
|
|
103
|
-
Log.log.debug("node=#{node_config}")
|
|
104
|
-
raise CliBadArgument,"the node configuration shall be Hash, not #{node_config.class} (#{node_config}), use either @json:<json> or @preset:<parameter set name>" unless node_config.is_a?(Hash)
|
|
105
|
-
# here, node_config is a Hash
|
|
106
|
-
node_config=node_config.symbolize_keys
|
|
107
|
-
# Check mandatory params
|
|
108
|
-
[:url,:username,:password].each { |k| raise CliBadArgument,"missing parameter [#{k}] in node specification: #{node_config}" unless node_config.has_key?(k) }
|
|
109
|
-
if node_config[:password].match(/^Bearer /)
|
|
110
|
-
node_api=Rest.new({
|
|
111
|
-
base_url: node_config[:url],
|
|
112
|
-
headers: {
|
|
113
|
-
'X-Aspera-AccessKey'=>node_config[:username],
|
|
114
|
-
'Authorization' =>node_config[:password]}})
|
|
115
|
-
else
|
|
116
|
-
node_api=Rest.new({
|
|
117
|
-
base_url: node_config[:url],
|
|
118
|
-
auth: {
|
|
119
|
-
type: :basic,
|
|
120
|
-
username: node_config[:username],
|
|
121
|
-
password: node_config[:password]
|
|
122
|
-
}})
|
|
123
|
-
end
|
|
124
|
-
new_agent=Fasp::Node.new(node_api)
|
|
125
|
-
# add root id if it's an access key
|
|
126
|
-
new_agent.options={root_id: node_config[:root_id]} if node_config.has_key?(:root_id)
|
|
127
|
-
else
|
|
128
|
-
raise "Unexpected transfer agent type: #{agent_type}"
|
|
75
|
+
agent_type=@opt_mgr.get_option(:transfer,:mandatory)
|
|
76
|
+
require "aspera/fasp/agent_#{agent_type}"
|
|
77
|
+
agent_options=@opt_mgr.get_option(:transfer_info,:optional)
|
|
78
|
+
raise CliBadArgument,"the transfer agent configuration shall be Hash, not #{agent_options.class} (#{agent_options}), use either @json:<json> or @preset:<parameter set name>" unless [Hash,NilClass].include?(agent_options.class)
|
|
79
|
+
# special case
|
|
80
|
+
if agent_type.eql?(:node) && agent_options.nil?
|
|
81
|
+
param_set_name=@config.get_plugin_default_config_name(:node)
|
|
82
|
+
raise CliBadArgument,"No default node configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
|
|
83
|
+
agent_options=@config.preset_by_name(param_set_name)
|
|
84
|
+
end
|
|
85
|
+
# special case
|
|
86
|
+
if agent_type.eql?(:direct) && @opt_mgr.get_option(:progress,:mandatory).eql?(:native)
|
|
87
|
+
agent_options={} if agent_options.nil?
|
|
88
|
+
agent_options[:quiet]=false
|
|
129
89
|
end
|
|
130
|
-
|
|
90
|
+
agent_options=agent_options.symbolize_keys if agent_options.is_a?(Hash)
|
|
91
|
+
# get agent instance
|
|
92
|
+
new_agent=Kernel.const_get("Aspera::Fasp::Agent#{agent_type.capitalize}").new(agent_options)
|
|
93
|
+
self.agent_instance=new_agent
|
|
131
94
|
return nil
|
|
132
95
|
end
|
|
133
96
|
|
|
@@ -135,14 +98,14 @@ END_OF_TEMPLATE
|
|
|
135
98
|
# sets default if needed
|
|
136
99
|
# param: 'send' or 'receive'
|
|
137
100
|
def destination_folder(direction)
|
|
138
|
-
dest_folder
|
|
139
|
-
return dest_folder unless dest_folder.nil?
|
|
101
|
+
dest_folder=@opt_mgr.get_option(:to_folder,:optional)
|
|
102
|
+
return File.expand_path(dest_folder) unless dest_folder.nil?
|
|
140
103
|
dest_folder=@transfer_spec_cmdline['destination_root']
|
|
141
104
|
return dest_folder unless dest_folder.nil?
|
|
142
105
|
# default: / on remote, . on local
|
|
143
106
|
case direction.to_s
|
|
144
|
-
when
|
|
145
|
-
when
|
|
107
|
+
when Fasp::TransferSpec::DIRECTION_SEND then dest_folder='/'
|
|
108
|
+
when Fasp::TransferSpec::DIRECTION_RECEIVE then dest_folder='.'
|
|
146
109
|
else raise "wrong direction: #{direction}"
|
|
147
110
|
end
|
|
148
111
|
return dest_folder
|
|
@@ -150,7 +113,7 @@ END_OF_TEMPLATE
|
|
|
150
113
|
|
|
151
114
|
# This is how the list of files to be transfered is specified
|
|
152
115
|
# get paths suitable for transfer spec from command line
|
|
153
|
-
# @return {:
|
|
116
|
+
# @return {source: (mandatory), destination: (optional)}
|
|
154
117
|
# computation is done only once, cache is kept in @transfer_paths
|
|
155
118
|
def ts_source_paths
|
|
156
119
|
# return cache if set
|
|
@@ -158,37 +121,37 @@ END_OF_TEMPLATE
|
|
|
158
121
|
# start with lower priority : get paths from transfer spec on command line
|
|
159
122
|
@transfer_paths=@transfer_spec_cmdline['paths'] if @transfer_spec_cmdline.has_key?('paths')
|
|
160
123
|
# is there a source list option ?
|
|
161
|
-
file_list
|
|
124
|
+
file_list=@opt_mgr.get_option(:sources,:optional)
|
|
162
125
|
case file_list
|
|
163
126
|
when nil,FILE_LIST_FROM_ARGS
|
|
164
|
-
Log.log.debug(
|
|
127
|
+
Log.log.debug('getting file list as parameters')
|
|
165
128
|
# get remaining arguments
|
|
166
|
-
file_list
|
|
167
|
-
raise CliBadArgument,"specify at least one file on command line or use --sources=#{FILE_LIST_FROM_TRANSFER_SPEC} to use transfer spec" if !file_list.is_a?(Array)
|
|
129
|
+
file_list=@opt_mgr.get_next_argument('source file list',:multiple)
|
|
130
|
+
raise CliBadArgument,"specify at least one file on command line or use --sources=#{FILE_LIST_FROM_TRANSFER_SPEC} to use transfer spec" if !file_list.is_a?(Array) || file_list.empty?
|
|
168
131
|
when FILE_LIST_FROM_TRANSFER_SPEC
|
|
169
|
-
Log.log.debug(
|
|
170
|
-
special_case_direct_with_list
|
|
171
|
-
raise CliBadArgument,
|
|
132
|
+
Log.log.debug('assume list provided in transfer spec')
|
|
133
|
+
special_case_direct_with_list=@opt_mgr.get_option(:transfer,:mandatory).eql?(:direct) and Fasp::Parameters.ts_has_file_list(@transfer_spec_cmdline)
|
|
134
|
+
raise CliBadArgument,'transfer spec on command line must have sources' if @transfer_paths.nil? && !special_case_direct_with_list
|
|
172
135
|
# here we assume check of sources is made in transfer agent
|
|
173
136
|
return @transfer_paths
|
|
174
137
|
when Array
|
|
175
|
-
Log.log.debug(
|
|
176
|
-
raise CliBadArgument,
|
|
138
|
+
Log.log.debug('getting file list as extended value')
|
|
139
|
+
raise CliBadArgument,'sources must be a Array of String' if !file_list.reject{|f|f.is_a?(String)}.empty?
|
|
177
140
|
else
|
|
178
141
|
raise CliBadArgument,"sources must be a Array, not #{file_list.class}"
|
|
179
142
|
end
|
|
180
143
|
# here, file_list is an Array or String
|
|
181
144
|
if !@transfer_paths.nil?
|
|
182
|
-
Log.log.warn(
|
|
145
|
+
Log.log.warn('--sources overrides paths from --ts')
|
|
183
146
|
end
|
|
184
|
-
case
|
|
147
|
+
case @opt_mgr.get_option(:src_type,:mandatory)
|
|
185
148
|
when :list
|
|
186
149
|
# when providing a list, just specify source
|
|
187
150
|
@transfer_paths=file_list.map{|i|{'source'=>i}}
|
|
188
151
|
when :pair
|
|
189
152
|
raise CliBadArgument,"When using pair, provide an even number of paths: #{file_list.length}" unless file_list.length.even?
|
|
190
153
|
@transfer_paths=file_list.each_slice(2).to_a.map{|s,d|{'source'=>s,'destination'=>d}}
|
|
191
|
-
else raise
|
|
154
|
+
else raise 'Unsupported src_type'
|
|
192
155
|
end
|
|
193
156
|
Log.log.debug("paths=#{@transfer_paths}")
|
|
194
157
|
return @transfer_paths
|
|
@@ -201,14 +164,14 @@ END_OF_TEMPLATE
|
|
|
201
164
|
# other options are carried to specific agent
|
|
202
165
|
def start(transfer_spec,tr_opts)
|
|
203
166
|
# check parameters
|
|
204
|
-
raise
|
|
205
|
-
raise
|
|
167
|
+
raise 'transfer_spec must be hash' unless transfer_spec.is_a?(Hash)
|
|
168
|
+
raise 'tr_opts must be hash' unless tr_opts.is_a?(Hash)
|
|
206
169
|
# process :src option
|
|
207
170
|
case transfer_spec['direction']
|
|
208
|
-
when
|
|
171
|
+
when Fasp::TransferSpec::DIRECTION_RECEIVE
|
|
209
172
|
# init default if required in any case
|
|
210
173
|
@transfer_spec_cmdline['destination_root']||=destination_folder(transfer_spec['direction'])
|
|
211
|
-
when
|
|
174
|
+
when Fasp::TransferSpec::DIRECTION_SEND
|
|
212
175
|
case tr_opts[:src]
|
|
213
176
|
when :direct
|
|
214
177
|
# init default if required
|
|
@@ -232,18 +195,19 @@ END_OF_TEMPLATE
|
|
|
232
195
|
|
|
233
196
|
transfer_spec.merge!(@transfer_spec_cmdline)
|
|
234
197
|
# create transfer agent
|
|
235
|
-
|
|
198
|
+
set_agent_by_options
|
|
236
199
|
Log.log.debug("transfer agent is a #{@agent.class}")
|
|
237
200
|
@agent.start_transfer(transfer_spec,tr_opts)
|
|
201
|
+
# list of : :success or error message
|
|
238
202
|
result=@agent.wait_for_transfers_completion
|
|
239
203
|
@progress_listener.reset
|
|
240
|
-
Fasp::
|
|
204
|
+
Fasp::AgentBase.validate_status_list(result)
|
|
241
205
|
send_email_transfer_notification(transfer_spec,result)
|
|
242
206
|
return result
|
|
243
207
|
end
|
|
244
208
|
|
|
245
209
|
def send_email_transfer_notification(transfer_spec,statuses)
|
|
246
|
-
return if
|
|
210
|
+
return if @opt_mgr.get_option(:notif_to,:optional).nil?
|
|
247
211
|
global_status=self.class.session_status(statuses)
|
|
248
212
|
email_vars={
|
|
249
213
|
global_transfer_status: global_status,
|
|
@@ -251,13 +215,13 @@ END_OF_TEMPLATE
|
|
|
251
215
|
body: "Transfer is: #{global_status}",
|
|
252
216
|
ts: transfer_spec
|
|
253
217
|
}
|
|
254
|
-
@
|
|
218
|
+
@config.send_email_template(email_vars,DEFAULT_TRANSFER_NOTIF_TMPL)
|
|
255
219
|
end
|
|
256
220
|
|
|
257
221
|
# @return :success if all sessions statuses returned by "start" are success
|
|
258
222
|
# else return the first error exception object
|
|
259
223
|
def self.session_status(statuses)
|
|
260
|
-
error_statuses=statuses.
|
|
224
|
+
error_statuses=statuses.reject{|i|i.eql?(:success)}
|
|
261
225
|
return :success if error_statuses.empty?
|
|
262
226
|
return error_statuses.first
|
|
263
227
|
end
|
|
@@ -266,7 +230,6 @@ END_OF_TEMPLATE
|
|
|
266
230
|
def shutdown
|
|
267
231
|
@agent.shutdown if @agent.respond_to?(:shutdown)
|
|
268
232
|
end
|
|
269
|
-
|
|
270
233
|
end
|
|
271
234
|
end
|
|
272
235
|
end
|
data/lib/aspera/cli/version.rb
CHANGED
data/lib/aspera/colors.rb
CHANGED
|
@@ -1,43 +1,51 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
# simple vt100 colors
|
|
2
3
|
class String
|
|
3
|
-
|
|
4
|
-
|
|
4
|
+
class<<self
|
|
5
|
+
private
|
|
6
|
+
|
|
7
|
+
def vtcmd(code);"\e[#{code}m";end
|
|
8
|
+
end
|
|
5
9
|
# see https://en.wikipedia.org/wiki/ANSI_escape_code
|
|
6
10
|
# symbol is the method name added to String
|
|
7
11
|
# it adds control chars to set color (and reset at the end).
|
|
8
12
|
VTSTYLES = {
|
|
9
|
-
:
|
|
10
|
-
:
|
|
11
|
-
:
|
|
12
|
-
:
|
|
13
|
-
:
|
|
14
|
-
:
|
|
15
|
-
:
|
|
16
|
-
:
|
|
17
|
-
:
|
|
18
|
-
:
|
|
19
|
-
:
|
|
20
|
-
:
|
|
21
|
-
:
|
|
22
|
-
:
|
|
23
|
-
:
|
|
24
|
-
:
|
|
25
|
-
:
|
|
26
|
-
:
|
|
27
|
-
:
|
|
28
|
-
:
|
|
29
|
-
:
|
|
13
|
+
bold: 1,
|
|
14
|
+
italic: 3,
|
|
15
|
+
underline: 4,
|
|
16
|
+
blink: 5,
|
|
17
|
+
reverse_color: 7,
|
|
18
|
+
black: 30,
|
|
19
|
+
red: 31,
|
|
20
|
+
green: 32,
|
|
21
|
+
brown: 33,
|
|
22
|
+
blue: 34,
|
|
23
|
+
magenta: 35,
|
|
24
|
+
cyan: 36,
|
|
25
|
+
gray: 37,
|
|
26
|
+
bg_black: 40,
|
|
27
|
+
bg_red: 41,
|
|
28
|
+
bg_green: 42,
|
|
29
|
+
bg_brown: 43,
|
|
30
|
+
bg_blue: 44,
|
|
31
|
+
bg_magenta: 45,
|
|
32
|
+
bg_cyan: 46,
|
|
33
|
+
bg_gray: 47
|
|
30
34
|
}
|
|
31
35
|
private_constant :VTSTYLES
|
|
32
36
|
# defines methods to String, one per entry in VTSTYLES
|
|
33
37
|
VTSTYLES.each do |name,code|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
38
|
+
if $stderr.tty?
|
|
39
|
+
begin_seq=vtcmd(code)
|
|
40
|
+
end_code=
|
|
41
|
+
if code >= 10 then 0
|
|
42
|
+
elsif code.eql?(1) then 22
|
|
43
|
+
else code+20
|
|
44
|
+
end
|
|
45
|
+
end_seq=vtcmd(end_code)
|
|
37
46
|
define_method(name){"#{begin_seq}#{self}#{end_seq}"}
|
|
38
47
|
else
|
|
39
48
|
define_method(name){self}
|
|
40
49
|
end
|
|
41
|
-
public name
|
|
42
50
|
end
|
|
43
51
|
end
|
|
@@ -1,14 +1,15 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
module Aspera
|
|
2
3
|
# helper class to build command line from a parameter list (key-value hash)
|
|
3
4
|
# constructor takes hash: { 'param1':'value1', ...}
|
|
4
5
|
# process_param is called repeatedly with all known parameters
|
|
5
6
|
# add_env_args is called to get resulting param list and env var (also checks that all params were used)
|
|
6
7
|
class CommandLineBuilder
|
|
7
|
-
# transform yes/no to
|
|
8
|
+
# transform yes/no to true/false
|
|
8
9
|
def self.yes_to_true(value)
|
|
9
10
|
case value
|
|
10
|
-
when 'yes'
|
|
11
|
-
when 'no'
|
|
11
|
+
when 'yes' then return true
|
|
12
|
+
when 'no' then return false
|
|
12
13
|
end
|
|
13
14
|
raise "unsupported value: #{value}"
|
|
14
15
|
end
|
|
@@ -22,13 +23,13 @@ module Aspera
|
|
|
22
23
|
options[:mandatory]||=false
|
|
23
24
|
options[:desc]||=''
|
|
24
25
|
# by default : string, unless it's without arg
|
|
25
|
-
if !
|
|
26
|
+
if !options.has_key?(:accepted_types)
|
|
26
27
|
options[:accepted_types]=options[:cltype].eql?(:opt_without_arg) ? :bool : :string
|
|
27
28
|
end
|
|
28
29
|
# single type is placed in array
|
|
29
30
|
options[:accepted_types]=[options[:accepted_types]] unless options[:accepted_types].is_a?(Array)
|
|
30
|
-
if !options.has_key?(:
|
|
31
|
-
options[:
|
|
31
|
+
if !options.has_key?(:clswitch) && options.has_key?(:cltype) && [:opt_without_arg,:opt_with_arg].include?(options[:cltype])
|
|
32
|
+
options[:clswitch]='--'+param_name.to_s.gsub('_','-')
|
|
32
33
|
end
|
|
33
34
|
end
|
|
34
35
|
end
|
|
@@ -36,7 +37,7 @@ module Aspera
|
|
|
36
37
|
private
|
|
37
38
|
|
|
38
39
|
# clvarname : command line variable name
|
|
39
|
-
def env_name(
|
|
40
|
+
def env_name(_param_name,options)
|
|
40
41
|
return options[:clvarname]
|
|
41
42
|
end
|
|
42
43
|
|
|
@@ -46,7 +47,7 @@ module Aspera
|
|
|
46
47
|
|
|
47
48
|
# @param param_hash
|
|
48
49
|
def initialize(param_hash,params_definition)
|
|
49
|
-
@param_hash=param_hash
|
|
50
|
+
@param_hash=param_hash # keep reference so that it can be modified by caller before calling `process_params`
|
|
50
51
|
@params_definition=params_definition
|
|
51
52
|
@result_env={}
|
|
52
53
|
@result_args=[]
|
|
@@ -86,42 +87,53 @@ module Aspera
|
|
|
86
87
|
# @param options : options for type
|
|
87
88
|
def process_param(param_name,action=nil)
|
|
88
89
|
options=@params_definition[param_name]
|
|
89
|
-
action=options[:cltype] if action.nil?
|
|
90
90
|
# should not happen
|
|
91
|
-
|
|
91
|
+
if options.nil?
|
|
92
|
+
Log.log.warn("Unknown parameter #{param_name}")
|
|
93
|
+
return
|
|
94
|
+
end
|
|
95
|
+
action=options[:cltype] if action.nil?
|
|
92
96
|
# check mandatory parameter (nil is valid value)
|
|
93
|
-
raise Fasp::Error
|
|
97
|
+
raise Fasp::Error, "Missing mandatory parameter: #{param_name}" if options[:mandatory] && !@param_hash.has_key?(param_name)
|
|
94
98
|
parameter_value=@param_hash[param_name]
|
|
99
|
+
|
|
95
100
|
#parameter_value=options[:default] if parameter_value.nil? and options.has_key?(:default)
|
|
101
|
+
|
|
102
|
+
# Check parameter type
|
|
96
103
|
expected_classes=options[:accepted_types].map do |s|
|
|
97
104
|
case s
|
|
98
|
-
when :string
|
|
99
|
-
when :array
|
|
100
|
-
when :hash
|
|
101
|
-
when :int
|
|
102
|
-
when :bool
|
|
105
|
+
when :string then String
|
|
106
|
+
when :array then Array
|
|
107
|
+
when :hash then Hash
|
|
108
|
+
when :int then Integer
|
|
109
|
+
when :bool then [TrueClass,FalseClass]
|
|
103
110
|
else raise "INTERNAL: unexpected value: #{s}"
|
|
104
111
|
end
|
|
105
112
|
end.flatten
|
|
106
|
-
#
|
|
107
|
-
raise Fasp::Error.new("#{param_name} is : #{parameter_value.class} (#{parameter_value}), shall be #{options[:accepted_types]}, ") unless parameter_value.nil? or expected_classes.include?(parameter_value.class)
|
|
113
|
+
raise Fasp::Error,"#{param_name} is : #{parameter_value.class} (#{parameter_value}), shall be #{options[:accepted_types]}, " unless parameter_value.nil? || expected_classes.include?(parameter_value.class)
|
|
108
114
|
@used_param_names.push(param_name) unless action.eql?(:defer)
|
|
109
115
|
|
|
110
116
|
# process only non-nil values
|
|
111
117
|
return nil if parameter_value.nil?
|
|
112
118
|
|
|
113
|
-
|
|
119
|
+
# check that value is of an accepted type (string, int bool)
|
|
120
|
+
raise "Value #{parameter_value} is not allowed for #{param_name}" if options.has_key?(:enum) && !options[:enum].include?(parameter_value)
|
|
121
|
+
|
|
122
|
+
# convert some values if value on command line needs processing from value in structure
|
|
123
|
+
case options[:clconvert]
|
|
124
|
+
when Hash
|
|
114
125
|
# translate using conversion table
|
|
115
|
-
new_value=options[:
|
|
116
|
-
raise "unsupported value: #{parameter_value}" if new_value.nil?
|
|
126
|
+
new_value=options[:clconvert][parameter_value]
|
|
127
|
+
raise "unsupported value: #{parameter_value}, expect: #{options[:clconvert].keys.join(', ')}" if new_value.nil?
|
|
117
128
|
parameter_value=new_value
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
raise Fasp::Error.new("unsupported #{param_name}: #{parameter_value}") if newvalue.nil?
|
|
129
|
+
when String
|
|
130
|
+
# :clconvert has name of class and encoding method
|
|
131
|
+
convclass,convmethod=options[:clconvert].split('.')
|
|
132
|
+
newvalue=Kernel.const_get(convclass).send(convmethod,parameter_value)
|
|
133
|
+
raise Fasp::Error, "unsupported #{param_name}: #{parameter_value}" if newvalue.nil?
|
|
124
134
|
parameter_value=newvalue
|
|
135
|
+
when NilClass
|
|
136
|
+
else raise "not expected type for clconvert #{options[:clconvert].class} for #{param_name}"
|
|
125
137
|
end
|
|
126
138
|
|
|
127
139
|
case action
|
|
@@ -135,19 +147,21 @@ module Aspera
|
|
|
135
147
|
when :opt_without_arg # if present and true : just add option without value
|
|
136
148
|
add_param=false
|
|
137
149
|
case parameter_value
|
|
138
|
-
when false# nothing to put on command line, no creation by default
|
|
139
|
-
when true
|
|
140
|
-
else raise Fasp::Error
|
|
150
|
+
when false then nil # nothing to put on command line, no creation by default
|
|
151
|
+
when true then add_param=true
|
|
152
|
+
else raise Fasp::Error, "unsupported #{param_name}: #{parameter_value}"
|
|
141
153
|
end
|
|
142
|
-
add_param
|
|
143
|
-
add_command_line_options([options[:
|
|
154
|
+
add_param= !add_param if options[:add_on_false]
|
|
155
|
+
add_command_line_options([options[:clswitch]]) if add_param
|
|
144
156
|
when :opt_with_arg # transform into command line option with value
|
|
145
157
|
#parameter_value=parameter_value.to_s if parameter_value.is_a?(Integer)
|
|
146
158
|
parameter_value=[parameter_value] unless parameter_value.is_a?(Array)
|
|
147
159
|
# if transfer_spec value is an array, applies option many times
|
|
148
|
-
parameter_value.each{|v|add_command_line_options([options[:
|
|
160
|
+
parameter_value.each{|v|add_command_line_options([options[:clswitch],v])}
|
|
161
|
+
when NilClass
|
|
162
|
+
Log.log.debug("Ignoring parameter: #{param_name}")
|
|
149
163
|
else
|
|
150
|
-
raise "
|
|
164
|
+
raise "ERROR: unknown action: #{action}/#{action.class}"
|
|
151
165
|
end
|
|
152
166
|
end
|
|
153
167
|
end
|
data/lib/aspera/cos_node.rb
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
require 'aspera/log'
|
|
2
3
|
require 'aspera/rest'
|
|
3
4
|
require 'xmlsimple'
|
|
@@ -11,25 +12,28 @@ module Aspera
|
|
|
11
12
|
@auth_url=auth_url
|
|
12
13
|
@api_key=api_key
|
|
13
14
|
s3_api=Aspera::Rest.new({
|
|
14
|
-
:
|
|
15
|
-
:
|
|
16
|
-
:
|
|
17
|
-
:
|
|
18
|
-
|
|
19
|
-
:
|
|
20
|
-
:
|
|
21
|
-
:
|
|
22
|
-
|
|
15
|
+
base_url: storage_endpoint,
|
|
16
|
+
not_auth_codes: ['401','403'], # error codes when not authorized
|
|
17
|
+
headers: {'ibm-service-instance-id' => instance_id},
|
|
18
|
+
auth: {
|
|
19
|
+
type: :oauth2,
|
|
20
|
+
base_url: @auth_url,
|
|
21
|
+
crtype: :generic,
|
|
22
|
+
generic: {
|
|
23
|
+
grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
|
|
24
|
+
response_type: 'cloud_iam',
|
|
25
|
+
apikey: @api_key
|
|
26
|
+
}}})
|
|
23
27
|
# read FASP connection information for bucket
|
|
24
|
-
xml_result_text=s3_api.call({:
|
|
28
|
+
xml_result_text=s3_api.call({operation: 'GET',subpath: bucket_name,headers: {'Accept'=>'application/xml'},url_params: {'faspConnectionInfo'=>nil}})[:http].body
|
|
25
29
|
ats_info=XmlSimple.xml_in(xml_result_text, {'ForceArray' => false})
|
|
26
30
|
Aspera::Log.dump('ats_info',ats_info)
|
|
27
31
|
super({
|
|
28
|
-
:
|
|
29
|
-
:
|
|
30
|
-
|
|
31
|
-
:
|
|
32
|
-
:
|
|
32
|
+
base_url: ats_info['ATSEndpoint'],
|
|
33
|
+
auth: {
|
|
34
|
+
type: :basic,
|
|
35
|
+
username: ats_info['AccessKey']['Id'],
|
|
36
|
+
password: ats_info['AccessKey']['Secret']}})
|
|
33
37
|
# prepare transfer spec addition
|
|
34
38
|
@add_ts={'tags'=>{'aspera'=>{'node'=>{'storage_credentials'=>{
|
|
35
39
|
'type' => 'token',
|
|
@@ -42,12 +46,16 @@ module Aspera
|
|
|
42
46
|
def generate_token
|
|
43
47
|
# OAuth API to get delegated token
|
|
44
48
|
delegated_oauth=Oauth.new({
|
|
45
|
-
|
|
46
|
-
:
|
|
47
|
-
:
|
|
48
|
-
:
|
|
49
|
-
:
|
|
50
|
-
|
|
49
|
+
type: :oauth2,
|
|
50
|
+
base_url: @auth_url,
|
|
51
|
+
token_field: TOKEN_FIELD,
|
|
52
|
+
crtype: :generic,
|
|
53
|
+
generic: {
|
|
54
|
+
grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
|
|
55
|
+
response_type: 'delegated_refresh_token',
|
|
56
|
+
apikey: @api_key,
|
|
57
|
+
receiver_client_ids: 'aspera_ats'
|
|
58
|
+
}})
|
|
51
59
|
# get delagated token to be placed in rest call header and in transfer tags
|
|
52
60
|
@add_ts['tags']['aspera']['node']['storage_credentials']['token'][TOKEN_FIELD]=delegated_oauth.get_authorization().gsub(/^Bearer /,'')
|
|
53
61
|
@params[:headers]={'X-Aspera-Storage-Credentials'=>JSON.generate(@add_ts['tags']['aspera']['node']['storage_credentials'])}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
require 'aspera/log'
|
|
2
3
|
require 'singleton'
|
|
3
4
|
|
|
@@ -6,8 +7,8 @@ module Aspera
|
|
|
6
7
|
class DataRepository
|
|
7
8
|
include Singleton
|
|
8
9
|
# get binary value from data repository
|
|
9
|
-
def
|
|
10
|
-
File.read(File.join(
|
|
10
|
+
def data(id)
|
|
11
|
+
File.read(File.join(__dir__,'data',id.to_s),mode: 'rb')
|
|
11
12
|
end
|
|
12
13
|
end
|
|
13
14
|
end
|