aspera-cli 4.4.0 → 4.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2095 -1503
  3. data/bin/ascli +2 -1
  4. data/bin/asession +4 -5
  5. data/docs/test_env.conf +3 -0
  6. data/examples/aoc.rb +4 -3
  7. data/examples/faspex4.rb +25 -25
  8. data/examples/proxy.pac +1 -1
  9. data/examples/transfer.rb +17 -17
  10. data/lib/aspera/aoc.rb +238 -185
  11. data/lib/aspera/ascmd.rb +93 -83
  12. data/lib/aspera/ats_api.rb +11 -10
  13. data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
  14. data/lib/aspera/cli/extended_value.rb +42 -33
  15. data/lib/aspera/cli/formater.rb +142 -108
  16. data/lib/aspera/cli/info.rb +17 -0
  17. data/lib/aspera/cli/listener/line_dump.rb +3 -2
  18. data/lib/aspera/cli/listener/logger.rb +2 -1
  19. data/lib/aspera/cli/listener/progress.rb +16 -18
  20. data/lib/aspera/cli/listener/progress_multi.rb +18 -21
  21. data/lib/aspera/cli/main.rb +173 -149
  22. data/lib/aspera/cli/manager.rb +163 -168
  23. data/lib/aspera/cli/plugin.rb +43 -31
  24. data/lib/aspera/cli/plugins/alee.rb +6 -6
  25. data/lib/aspera/cli/plugins/aoc.rb +405 -370
  26. data/lib/aspera/cli/plugins/ats.rb +86 -79
  27. data/lib/aspera/cli/plugins/bss.rb +14 -16
  28. data/lib/aspera/cli/plugins/config.rb +580 -362
  29. data/lib/aspera/cli/plugins/console.rb +23 -19
  30. data/lib/aspera/cli/plugins/cos.rb +18 -18
  31. data/lib/aspera/cli/plugins/faspex.rb +201 -158
  32. data/lib/aspera/cli/plugins/faspex5.rb +80 -57
  33. data/lib/aspera/cli/plugins/node.rb +183 -166
  34. data/lib/aspera/cli/plugins/orchestrator.rb +71 -67
  35. data/lib/aspera/cli/plugins/preview.rb +92 -96
  36. data/lib/aspera/cli/plugins/server.rb +79 -75
  37. data/lib/aspera/cli/plugins/shares.rb +35 -19
  38. data/lib/aspera/cli/plugins/sync.rb +20 -22
  39. data/lib/aspera/cli/transfer_agent.rb +76 -113
  40. data/lib/aspera/cli/version.rb +2 -1
  41. data/lib/aspera/colors.rb +35 -27
  42. data/lib/aspera/command_line_builder.rb +48 -34
  43. data/lib/aspera/cos_node.rb +29 -21
  44. data/lib/aspera/data_repository.rb +3 -2
  45. data/lib/aspera/environment.rb +50 -45
  46. data/lib/aspera/fasp/{manager.rb → agent_base.rb} +28 -25
  47. data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +52 -43
  48. data/lib/aspera/fasp/{local.rb → agent_direct.rb} +58 -72
  49. data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +37 -43
  50. data/lib/aspera/fasp/{node.rb → agent_node.rb} +35 -16
  51. data/lib/aspera/fasp/agent_trsdk.rb +104 -0
  52. data/lib/aspera/fasp/error.rb +2 -1
  53. data/lib/aspera/fasp/error_info.rb +68 -52
  54. data/lib/aspera/fasp/installation.rb +152 -124
  55. data/lib/aspera/fasp/listener.rb +1 -0
  56. data/lib/aspera/fasp/parameters.rb +87 -92
  57. data/lib/aspera/fasp/parameters.yaml +305 -249
  58. data/lib/aspera/fasp/resume_policy.rb +11 -14
  59. data/lib/aspera/fasp/transfer_spec.rb +26 -0
  60. data/lib/aspera/fasp/uri.rb +22 -21
  61. data/lib/aspera/faspex_gw.rb +55 -89
  62. data/lib/aspera/hash_ext.rb +4 -3
  63. data/lib/aspera/id_generator.rb +8 -7
  64. data/lib/aspera/keychain/encrypted_hash.rb +121 -0
  65. data/lib/aspera/keychain/macos_security.rb +90 -0
  66. data/lib/aspera/log.rb +55 -37
  67. data/lib/aspera/nagios.rb +13 -12
  68. data/lib/aspera/node.rb +30 -25
  69. data/lib/aspera/oauth.rb +175 -226
  70. data/lib/aspera/open_application.rb +4 -3
  71. data/lib/aspera/persistency_action_once.rb +6 -6
  72. data/lib/aspera/persistency_folder.rb +5 -9
  73. data/lib/aspera/preview/file_types.rb +6 -5
  74. data/lib/aspera/preview/generator.rb +25 -24
  75. data/lib/aspera/preview/options.rb +16 -14
  76. data/lib/aspera/preview/utils.rb +98 -98
  77. data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
  78. data/lib/aspera/proxy_auto_config.rb +111 -20
  79. data/lib/aspera/rest.rb +154 -135
  80. data/lib/aspera/rest_call_error.rb +2 -2
  81. data/lib/aspera/rest_error_analyzer.rb +23 -25
  82. data/lib/aspera/rest_errors_aspera.rb +15 -14
  83. data/lib/aspera/ssh.rb +12 -10
  84. data/lib/aspera/sync.rb +42 -41
  85. data/lib/aspera/temp_file_manager.rb +18 -14
  86. data/lib/aspera/timer_limiter.rb +2 -1
  87. data/lib/aspera/uri_reader.rb +7 -5
  88. data/lib/aspera/web_auth.rb +79 -76
  89. metadata +116 -29
  90. data/docs/Makefile +0 -66
  91. data/docs/README.erb.md +0 -3973
  92. data/docs/README.md +0 -13
  93. data/docs/diagrams.txt +0 -49
  94. data/docs/doc_tools.rb +0 -58
  95. data/lib/aspera/api_detector.rb +0 -60
  96. data/lib/aspera/cli/plugins/shares2.rb +0 -114
  97. data/lib/aspera/secrets.rb +0 -20
@@ -1,8 +1,5 @@
1
- require 'aspera/fasp/local'
2
- require 'aspera/fasp/parameters'
3
- require 'aspera/fasp/connect'
4
- require 'aspera/fasp/node'
5
- require 'aspera/fasp/http_gw'
1
+ # frozen_string_literal: true
2
+ require 'aspera/fasp/transfer_spec'
6
3
  require 'aspera/cli/listener/logger'
7
4
  require 'aspera/cli/listener/progress_multi'
8
5
 
@@ -16,21 +13,23 @@ module Aspera
16
13
  FILE_LIST_FROM_ARGS='@args'
17
14
  # special value for --sources : read file list from transfer spec (--ts)
18
15
  FILE_LIST_FROM_TRANSFER_SPEC='@ts'
19
- DEFAULT_TRANSFER_NOTIF_TMPL=<<END_OF_TEMPLATE
20
- From: <%=from_name%> <<%=from_email%>>
21
- To: <<%=to%>>
22
- Subject: <%=subject%>
16
+ DEFAULT_TRANSFER_NOTIF_TMPL=<<~END_OF_TEMPLATE
17
+ From: <%=from_name%> <<%=from_email%>>
18
+ To: <<%=to%>>
19
+ Subject: <%=subject%>
23
20
 
24
- Transfer is: <%=global_transfer_status%>
21
+ Transfer is: <%=global_transfer_status%>
25
22
 
26
- <%=ts.to_yaml%>
27
- END_OF_TEMPLATE
28
- #%
23
+ <%=ts.to_yaml%>
24
+ END_OF_TEMPLATE
25
+ #% (formating bug in eclipse)
29
26
  private_constant :FILE_LIST_FROM_ARGS,:FILE_LIST_FROM_TRANSFER_SPEC,:DEFAULT_TRANSFER_NOTIF_TMPL
27
+ TRANSFER_AGENTS=[:direct,:node,:connect,:httpgw,:trsdk]
28
+
30
29
  # @param env external objects: option manager, config file manager
31
- def initialize(env)
32
- # same as plugin environment
33
- @env=env
30
+ def initialize(opt_mgr,config)
31
+ @opt_mgr=opt_mgr
32
+ @config=config
34
33
  # command line can override transfer spec
35
34
  @transfer_spec_cmdline={'create_dir'=>true}
36
35
  # the currently selected transfer agent
@@ -38,25 +37,21 @@ END_OF_TEMPLATE
38
37
  @progress_listener=Listener::ProgressMulti.new
39
38
  # source/destination pair, like "paths" of transfer spec
40
39
  @transfer_paths=nil
41
- options.set_obj_attr(:ts,self,:option_transfer_spec)
42
- options.add_opt_simple(:ts,"override transfer spec values (Hash, use @json: prefix), current=#{options.get_option(:ts,:optional)}")
43
- options.add_opt_simple(:local_resume,"set resume policy (Hash, use @json: prefix), current=#{options.get_option(:local_resume,:optional)}")
44
- options.add_opt_simple(:to_folder,"destination folder for downloaded files")
45
- options.add_opt_simple(:sources,"list of source files (see doc)")
46
- options.add_opt_simple(:transfer_info,"parameters for transfer agent")
47
- options.add_opt_list(:src_type,[:list,:pair],"type of file list")
48
- options.add_opt_list(:transfer,[:direct,:httpgw,:connect,:node],"type of transfer agent")
49
- options.add_opt_list(:progress,[:none,:native,:multi],"type of progress bar")
50
- options.set_option(:transfer,:direct)
51
- options.set_option(:src_type,:list)
52
- options.set_option(:progress,:native) # use native ascp progress bar as it is more reliable
53
- options.parse_options!
40
+ @opt_mgr.set_obj_attr(:ts,self,:option_transfer_spec)
41
+ @opt_mgr.add_opt_simple(:ts,"override transfer spec values (Hash, use @json: prefix), current=#{@opt_mgr.get_option(:ts,:optional)}")
42
+ @opt_mgr.add_opt_simple(:local_resume,"set resume policy (Hash, use @json: prefix), current=#{@opt_mgr.get_option(:local_resume,:optional)}")
43
+ @opt_mgr.add_opt_simple(:to_folder,'destination folder for downloaded files')
44
+ @opt_mgr.add_opt_simple(:sources,'list of source files (see doc)')
45
+ @opt_mgr.add_opt_simple(:transfer_info,'parameters for transfer agent')
46
+ @opt_mgr.add_opt_list(:src_type,[:list,:pair],'type of file list')
47
+ @opt_mgr.add_opt_list(:transfer,TRANSFER_AGENTS,'type of transfer agent')
48
+ @opt_mgr.add_opt_list(:progress,[:none,:native,:multi],'type of progress bar')
49
+ @opt_mgr.set_option(:transfer,:direct)
50
+ @opt_mgr.set_option(:src_type,:list)
51
+ @opt_mgr.set_option(:progress,:native) # use native ascp progress bar as it is more reliable
52
+ @opt_mgr.parse_options!
54
53
  end
55
54
 
56
- def options; @env[:options];end
57
-
58
- def config; @env[:config];end
59
-
60
55
  def option_transfer_spec; @transfer_spec_cmdline; end
61
56
 
62
57
  # multiple option are merged
@@ -64,12 +59,12 @@ END_OF_TEMPLATE
64
59
 
65
60
  def option_transfer_spec_deep_merge(ts); @transfer_spec_cmdline.deep_merge!(ts); end
66
61
 
67
- def set_agent_instance(instance)
62
+ def agent_instance=(instance)
68
63
  @agent=instance
69
64
  @agent.add_listener(Listener::Logger.new)
70
65
  # use local progress bar if asked so, or if native and non local ascp (because only local ascp has native progress bar)
71
- if options.get_option(:progress,:mandatory).eql?(:multi) or
72
- (options.get_option(:progress,:mandatory).eql?(:native) and !options.get_option(:transfer,:mandatory).eql?(:direct))
66
+ if @opt_mgr.get_option(:progress,:mandatory).eql?(:multi) ||
67
+ (@opt_mgr.get_option(:progress,:mandatory).eql?(:native) && !instance.class.to_s.eql?('Aspera::Fasp::AgentDirect'))
73
68
  @agent.add_listener(@progress_listener)
74
69
  end
75
70
  end
@@ -77,57 +72,25 @@ END_OF_TEMPLATE
77
72
  # analyze options and create new agent if not already created or set
78
73
  def set_agent_by_options
79
74
  return nil unless @agent.nil?
80
- agent_type=options.get_option(:transfer,:mandatory)
81
- case agent_type
82
- when :direct
83
- agent_options=options.get_option(:transfer_info,:optional)
84
- agent_options=agent_options.symbolize_keys if agent_options.is_a?(Hash)
85
- new_agent=Fasp::Local.new(agent_options)
86
- new_agent.quiet=false if options.get_option(:progress,:mandatory).eql?(:native)
87
- when :httpgw
88
- httpgw_config=options.get_option(:transfer_info,:mandatory)
89
- new_agent=Fasp::HttpGW.new(httpgw_config)
90
- when :connect
91
- new_agent=Fasp::Connect.new
92
- when :node
93
- # way for code to setup alternate node api in advance
94
- # support: @preset:<name>
95
- # support extended values
96
- node_config=options.get_option(:transfer_info,:optional)
97
- # if not specified: use default node
98
- if node_config.nil?
99
- param_set_name=config.get_plugin_default_config_name(:node)
100
- raise CliBadArgument,"No default node configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
101
- node_config=config.preset_by_name(param_set_name)
102
- end
103
- Log.log.debug("node=#{node_config}")
104
- raise CliBadArgument,"the node configuration shall be Hash, not #{node_config.class} (#{node_config}), use either @json:<json> or @preset:<parameter set name>" unless node_config.is_a?(Hash)
105
- # here, node_config is a Hash
106
- node_config=node_config.symbolize_keys
107
- # Check mandatory params
108
- [:url,:username,:password].each { |k| raise CliBadArgument,"missing parameter [#{k}] in node specification: #{node_config}" unless node_config.has_key?(k) }
109
- if node_config[:password].match(/^Bearer /)
110
- node_api=Rest.new({
111
- base_url: node_config[:url],
112
- headers: {
113
- 'X-Aspera-AccessKey'=>node_config[:username],
114
- 'Authorization' =>node_config[:password]}})
115
- else
116
- node_api=Rest.new({
117
- base_url: node_config[:url],
118
- auth: {
119
- type: :basic,
120
- username: node_config[:username],
121
- password: node_config[:password]
122
- }})
123
- end
124
- new_agent=Fasp::Node.new(node_api)
125
- # add root id if it's an access key
126
- new_agent.options={root_id: node_config[:root_id]} if node_config.has_key?(:root_id)
127
- else
128
- raise "Unexpected transfer agent type: #{agent_type}"
75
+ agent_type=@opt_mgr.get_option(:transfer,:mandatory)
76
+ require "aspera/fasp/agent_#{agent_type}"
77
+ agent_options=@opt_mgr.get_option(:transfer_info,:optional)
78
+ raise CliBadArgument,"the transfer agent configuration shall be Hash, not #{agent_options.class} (#{agent_options}), use either @json:<json> or @preset:<parameter set name>" unless [Hash,NilClass].include?(agent_options.class)
79
+ # special case
80
+ if agent_type.eql?(:node) && agent_options.nil?
81
+ param_set_name=@config.get_plugin_default_config_name(:node)
82
+ raise CliBadArgument,"No default node configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
83
+ agent_options=@config.preset_by_name(param_set_name)
84
+ end
85
+ # special case
86
+ if agent_type.eql?(:direct) && @opt_mgr.get_option(:progress,:mandatory).eql?(:native)
87
+ agent_options={} if agent_options.nil?
88
+ agent_options[:quiet]=false
129
89
  end
130
- set_agent_instance(new_agent)
90
+ agent_options=agent_options.symbolize_keys if agent_options.is_a?(Hash)
91
+ # get agent instance
92
+ new_agent=Kernel.const_get("Aspera::Fasp::Agent#{agent_type.capitalize}").new(agent_options)
93
+ self.agent_instance=new_agent
131
94
  return nil
132
95
  end
133
96
 
@@ -135,14 +98,14 @@ END_OF_TEMPLATE
135
98
  # sets default if needed
136
99
  # param: 'send' or 'receive'
137
100
  def destination_folder(direction)
138
- dest_folder=options.get_option(:to_folder,:optional)
139
- return dest_folder unless dest_folder.nil?
101
+ dest_folder=@opt_mgr.get_option(:to_folder,:optional)
102
+ return File.expand_path(dest_folder) unless dest_folder.nil?
140
103
  dest_folder=@transfer_spec_cmdline['destination_root']
141
104
  return dest_folder unless dest_folder.nil?
142
105
  # default: / on remote, . on local
143
106
  case direction.to_s
144
- when 'send';dest_folder='/'
145
- when 'receive';dest_folder='.'
107
+ when Fasp::TransferSpec::DIRECTION_SEND then dest_folder='/'
108
+ when Fasp::TransferSpec::DIRECTION_RECEIVE then dest_folder='.'
146
109
  else raise "wrong direction: #{direction}"
147
110
  end
148
111
  return dest_folder
@@ -150,7 +113,7 @@ END_OF_TEMPLATE
150
113
 
151
114
  # This is how the list of files to be transfered is specified
152
115
  # get paths suitable for transfer spec from command line
153
- # @return {:source=>(mandatory), :destination=>(optional)}
116
+ # @return {source: (mandatory), destination: (optional)}
154
117
  # computation is done only once, cache is kept in @transfer_paths
155
118
  def ts_source_paths
156
119
  # return cache if set
@@ -158,37 +121,37 @@ END_OF_TEMPLATE
158
121
  # start with lower priority : get paths from transfer spec on command line
159
122
  @transfer_paths=@transfer_spec_cmdline['paths'] if @transfer_spec_cmdline.has_key?('paths')
160
123
  # is there a source list option ?
161
- file_list=options.get_option(:sources,:optional)
124
+ file_list=@opt_mgr.get_option(:sources,:optional)
162
125
  case file_list
163
126
  when nil,FILE_LIST_FROM_ARGS
164
- Log.log.debug("getting file list as parameters")
127
+ Log.log.debug('getting file list as parameters')
165
128
  # get remaining arguments
166
- file_list=options.get_next_argument("source file list",:multiple)
167
- raise CliBadArgument,"specify at least one file on command line or use --sources=#{FILE_LIST_FROM_TRANSFER_SPEC} to use transfer spec" if !file_list.is_a?(Array) or file_list.empty?
129
+ file_list=@opt_mgr.get_next_argument('source file list',:multiple)
130
+ raise CliBadArgument,"specify at least one file on command line or use --sources=#{FILE_LIST_FROM_TRANSFER_SPEC} to use transfer spec" if !file_list.is_a?(Array) || file_list.empty?
168
131
  when FILE_LIST_FROM_TRANSFER_SPEC
169
- Log.log.debug("assume list provided in transfer spec")
170
- special_case_direct_with_list=options.get_option(:transfer,:mandatory).eql?(:direct) and Fasp::Parameters.ts_has_file_list(@transfer_spec_cmdline)
171
- raise CliBadArgument,"transfer spec on command line must have sources" if @transfer_paths.nil? and !special_case_direct_with_list
132
+ Log.log.debug('assume list provided in transfer spec')
133
+ special_case_direct_with_list=@opt_mgr.get_option(:transfer,:mandatory).eql?(:direct) and Fasp::Parameters.ts_has_file_list(@transfer_spec_cmdline)
134
+ raise CliBadArgument,'transfer spec on command line must have sources' if @transfer_paths.nil? && !special_case_direct_with_list
172
135
  # here we assume check of sources is made in transfer agent
173
136
  return @transfer_paths
174
137
  when Array
175
- Log.log.debug("getting file list as extended value")
176
- raise CliBadArgument,"sources must be a Array of String" if !file_list.select{|f|!f.is_a?(String)}.empty?
138
+ Log.log.debug('getting file list as extended value')
139
+ raise CliBadArgument,'sources must be a Array of String' if !file_list.reject{|f|f.is_a?(String)}.empty?
177
140
  else
178
141
  raise CliBadArgument,"sources must be a Array, not #{file_list.class}"
179
142
  end
180
143
  # here, file_list is an Array or String
181
144
  if !@transfer_paths.nil?
182
- Log.log.warn("--sources overrides paths from --ts")
145
+ Log.log.warn('--sources overrides paths from --ts')
183
146
  end
184
- case options.get_option(:src_type,:mandatory)
147
+ case @opt_mgr.get_option(:src_type,:mandatory)
185
148
  when :list
186
149
  # when providing a list, just specify source
187
150
  @transfer_paths=file_list.map{|i|{'source'=>i}}
188
151
  when :pair
189
152
  raise CliBadArgument,"When using pair, provide an even number of paths: #{file_list.length}" unless file_list.length.even?
190
153
  @transfer_paths=file_list.each_slice(2).to_a.map{|s,d|{'source'=>s,'destination'=>d}}
191
- else raise "Unsupported src_type"
154
+ else raise 'Unsupported src_type'
192
155
  end
193
156
  Log.log.debug("paths=#{@transfer_paths}")
194
157
  return @transfer_paths
@@ -201,14 +164,14 @@ END_OF_TEMPLATE
201
164
  # other options are carried to specific agent
202
165
  def start(transfer_spec,tr_opts)
203
166
  # check parameters
204
- raise "transfer_spec must be hash" unless transfer_spec.is_a?(Hash)
205
- raise "tr_opts must be hash" unless tr_opts.is_a?(Hash)
167
+ raise 'transfer_spec must be hash' unless transfer_spec.is_a?(Hash)
168
+ raise 'tr_opts must be hash' unless tr_opts.is_a?(Hash)
206
169
  # process :src option
207
170
  case transfer_spec['direction']
208
- when 'receive'
171
+ when Fasp::TransferSpec::DIRECTION_RECEIVE
209
172
  # init default if required in any case
210
173
  @transfer_spec_cmdline['destination_root']||=destination_folder(transfer_spec['direction'])
211
- when 'send'
174
+ when Fasp::TransferSpec::DIRECTION_SEND
212
175
  case tr_opts[:src]
213
176
  when :direct
214
177
  # init default if required
@@ -232,18 +195,19 @@ END_OF_TEMPLATE
232
195
 
233
196
  transfer_spec.merge!(@transfer_spec_cmdline)
234
197
  # create transfer agent
235
- self.set_agent_by_options
198
+ set_agent_by_options
236
199
  Log.log.debug("transfer agent is a #{@agent.class}")
237
200
  @agent.start_transfer(transfer_spec,tr_opts)
201
+ # list of : :success or error message
238
202
  result=@agent.wait_for_transfers_completion
239
203
  @progress_listener.reset
240
- Fasp::Manager.validate_status_list(result)
204
+ Fasp::AgentBase.validate_status_list(result)
241
205
  send_email_transfer_notification(transfer_spec,result)
242
206
  return result
243
207
  end
244
208
 
245
209
  def send_email_transfer_notification(transfer_spec,statuses)
246
- return if options.get_option(:notif_to,:optional).nil?
210
+ return if @opt_mgr.get_option(:notif_to,:optional).nil?
247
211
  global_status=self.class.session_status(statuses)
248
212
  email_vars={
249
213
  global_transfer_status: global_status,
@@ -251,13 +215,13 @@ END_OF_TEMPLATE
251
215
  body: "Transfer is: #{global_status}",
252
216
  ts: transfer_spec
253
217
  }
254
- @env[:config].send_email_template(email_vars,DEFAULT_TRANSFER_NOTIF_TMPL)
218
+ @config.send_email_template(email_vars,DEFAULT_TRANSFER_NOTIF_TMPL)
255
219
  end
256
220
 
257
221
  # @return :success if all sessions statuses returned by "start" are success
258
222
  # else return the first error exception object
259
223
  def self.session_status(statuses)
260
- error_statuses=statuses.select{|i|!i.eql?(:success)}
224
+ error_statuses=statuses.reject{|i|i.eql?(:success)}
261
225
  return :success if error_statuses.empty?
262
226
  return error_statuses.first
263
227
  end
@@ -266,7 +230,6 @@ END_OF_TEMPLATE
266
230
  def shutdown
267
231
  @agent.shutdown if @agent.respond_to?(:shutdown)
268
232
  end
269
-
270
233
  end
271
234
  end
272
235
  end
@@ -1,5 +1,6 @@
1
+ # frozen_string_literal: true
1
2
  module Aspera
2
3
  module Cli
3
- VERSION = "4.4.0"
4
+ VERSION = '4.7.0'
4
5
  end
5
6
  end
data/lib/aspera/colors.rb CHANGED
@@ -1,43 +1,51 @@
1
+ # frozen_string_literal: true
1
2
  # simple vt100 colors
2
3
  class String
3
- private
4
- def self.vtcmd(code);"\e[#{code}m";end
4
+ class<<self
5
+ private
6
+
7
+ def vtcmd(code);"\e[#{code}m";end
8
+ end
5
9
  # see https://en.wikipedia.org/wiki/ANSI_escape_code
6
10
  # symbol is the method name added to String
7
11
  # it adds control chars to set color (and reset at the end).
8
12
  VTSTYLES = {
9
- :bold=>1,
10
- :italic=>3,
11
- :underline=>4,
12
- :blink=>5,
13
- :reverse_color=>7,
14
- :black=>30,
15
- :red=>31,
16
- :green=>32,
17
- :brown=>33,
18
- :blue=>34,
19
- :magenta=>35,
20
- :cyan=>36,
21
- :gray=>37,
22
- :bg_black=>40,
23
- :bg_red=>41,
24
- :bg_green=>42,
25
- :bg_brown=>43,
26
- :bg_blue=>44,
27
- :bg_magenta=>45,
28
- :bg_cyan=>46,
29
- :bg_gray=>47,
13
+ bold: 1,
14
+ italic: 3,
15
+ underline: 4,
16
+ blink: 5,
17
+ reverse_color: 7,
18
+ black: 30,
19
+ red: 31,
20
+ green: 32,
21
+ brown: 33,
22
+ blue: 34,
23
+ magenta: 35,
24
+ cyan: 36,
25
+ gray: 37,
26
+ bg_black: 40,
27
+ bg_red: 41,
28
+ bg_green: 42,
29
+ bg_brown: 43,
30
+ bg_blue: 44,
31
+ bg_magenta: 45,
32
+ bg_cyan: 46,
33
+ bg_gray: 47
30
34
  }
31
35
  private_constant :VTSTYLES
32
36
  # defines methods to String, one per entry in VTSTYLES
33
37
  VTSTYLES.each do |name,code|
34
- begin_seq=vtcmd(code)
35
- end_seq=vtcmd((code >= 10) ? 0 : code+20+(code.eql?(1)?1:0))
36
- if STDERR.tty?
38
+ if $stderr.tty?
39
+ begin_seq=vtcmd(code)
40
+ end_code=
41
+ if code >= 10 then 0
42
+ elsif code.eql?(1) then 22
43
+ else code+20
44
+ end
45
+ end_seq=vtcmd(end_code)
37
46
  define_method(name){"#{begin_seq}#{self}#{end_seq}"}
38
47
  else
39
48
  define_method(name){self}
40
49
  end
41
- public name
42
50
  end
43
51
  end
@@ -1,14 +1,15 @@
1
+ # frozen_string_literal: true
1
2
  module Aspera
2
3
  # helper class to build command line from a parameter list (key-value hash)
3
4
  # constructor takes hash: { 'param1':'value1', ...}
4
5
  # process_param is called repeatedly with all known parameters
5
6
  # add_env_args is called to get resulting param list and env var (also checks that all params were used)
6
7
  class CommandLineBuilder
7
- # transform yes/no to trye/false
8
+ # transform yes/no to true/false
8
9
  def self.yes_to_true(value)
9
10
  case value
10
- when 'yes'; return true
11
- when 'no'; return false
11
+ when 'yes' then return true
12
+ when 'no' then return false
12
13
  end
13
14
  raise "unsupported value: #{value}"
14
15
  end
@@ -22,13 +23,13 @@ module Aspera
22
23
  options[:mandatory]||=false
23
24
  options[:desc]||=''
24
25
  # by default : string, unless it's without arg
25
- if ! options.has_key?(:accepted_types)
26
+ if !options.has_key?(:accepted_types)
26
27
  options[:accepted_types]=options[:cltype].eql?(:opt_without_arg) ? :bool : :string
27
28
  end
28
29
  # single type is placed in array
29
30
  options[:accepted_types]=[options[:accepted_types]] unless options[:accepted_types].is_a?(Array)
30
- if !options.has_key?(:option_switch) and options.has_key?(:cltype) and [:opt_without_arg,:opt_with_arg].include?(options[:cltype])
31
- options[:option_switch]='--'+param_name.to_s.gsub('_','-')
31
+ if !options.has_key?(:clswitch) && options.has_key?(:cltype) && [:opt_without_arg,:opt_with_arg].include?(options[:cltype])
32
+ options[:clswitch]='--'+param_name.to_s.gsub('_','-')
32
33
  end
33
34
  end
34
35
  end
@@ -36,7 +37,7 @@ module Aspera
36
37
  private
37
38
 
38
39
  # clvarname : command line variable name
39
- def env_name(param_name,options)
40
+ def env_name(_param_name,options)
40
41
  return options[:clvarname]
41
42
  end
42
43
 
@@ -46,7 +47,7 @@ module Aspera
46
47
 
47
48
  # @param param_hash
48
49
  def initialize(param_hash,params_definition)
49
- @param_hash=param_hash # keep reference so that it can be modified by caller before calling `process_params`
50
+ @param_hash=param_hash # keep reference so that it can be modified by caller before calling `process_params`
50
51
  @params_definition=params_definition
51
52
  @result_env={}
52
53
  @result_args=[]
@@ -86,42 +87,53 @@ module Aspera
86
87
  # @param options : options for type
87
88
  def process_param(param_name,action=nil)
88
89
  options=@params_definition[param_name]
89
- action=options[:cltype] if action.nil?
90
90
  # should not happen
91
- raise "Internal error: ask processing of param #{param_name}" if options.nil?
91
+ if options.nil?
92
+ Log.log.warn("Unknown parameter #{param_name}")
93
+ return
94
+ end
95
+ action=options[:cltype] if action.nil?
92
96
  # check mandatory parameter (nil is valid value)
93
- raise Fasp::Error.new("mandatory parameter: #{param_name}") if options[:mandatory] and !@param_hash.has_key?(param_name)
97
+ raise Fasp::Error, "Missing mandatory parameter: #{param_name}" if options[:mandatory] && !@param_hash.has_key?(param_name)
94
98
  parameter_value=@param_hash[param_name]
99
+
95
100
  #parameter_value=options[:default] if parameter_value.nil? and options.has_key?(:default)
101
+
102
+ # Check parameter type
96
103
  expected_classes=options[:accepted_types].map do |s|
97
104
  case s
98
- when :string; String
99
- when :array; Array
100
- when :hash; Hash
101
- when :int; Integer
102
- when :bool; [TrueClass,FalseClass]
105
+ when :string then String
106
+ when :array then Array
107
+ when :hash then Hash
108
+ when :int then Integer
109
+ when :bool then [TrueClass,FalseClass]
103
110
  else raise "INTERNAL: unexpected value: #{s}"
104
111
  end
105
112
  end.flatten
106
- # check provided type
107
- raise Fasp::Error.new("#{param_name} is : #{parameter_value.class} (#{parameter_value}), shall be #{options[:accepted_types]}, ") unless parameter_value.nil? or expected_classes.include?(parameter_value.class)
113
+ raise Fasp::Error,"#{param_name} is : #{parameter_value.class} (#{parameter_value}), shall be #{options[:accepted_types]}, " unless parameter_value.nil? || expected_classes.include?(parameter_value.class)
108
114
  @used_param_names.push(param_name) unless action.eql?(:defer)
109
115
 
110
116
  # process only non-nil values
111
117
  return nil if parameter_value.nil?
112
118
 
113
- if options.has_key?(:translate_values)
119
+ # check that value is of an accepted type (string, int bool)
120
+ raise "Value #{parameter_value} is not allowed for #{param_name}" if options.has_key?(:enum) && !options[:enum].include?(parameter_value)
121
+
122
+ # convert some values if value on command line needs processing from value in structure
123
+ case options[:clconvert]
124
+ when Hash
114
125
  # translate using conversion table
115
- new_value=options[:translate_values][parameter_value]
116
- raise "unsupported value: #{parameter_value}" if new_value.nil?
126
+ new_value=options[:clconvert][parameter_value]
127
+ raise "unsupported value: #{parameter_value}, expect: #{options[:clconvert].keys.join(', ')}" if new_value.nil?
117
128
  parameter_value=new_value
118
- end
119
- raise "unsupported value: #{parameter_value}" unless options[:accepted_values].nil? or options[:accepted_values].include?(parameter_value)
120
- if options[:encode]
121
- # :encode has name of class with encoding method
122
- newvalue=Kernel.const_get(options[:encode]).send("encode_#{param_name}",parameter_value)
123
- raise Fasp::Error.new("unsupported #{param_name}: #{parameter_value}") if newvalue.nil?
129
+ when String
130
+ # :clconvert has name of class and encoding method
131
+ convclass,convmethod=options[:clconvert].split('.')
132
+ newvalue=Kernel.const_get(convclass).send(convmethod,parameter_value)
133
+ raise Fasp::Error, "unsupported #{param_name}: #{parameter_value}" if newvalue.nil?
124
134
  parameter_value=newvalue
135
+ when NilClass
136
+ else raise "not expected type for clconvert #{options[:clconvert].class} for #{param_name}"
125
137
  end
126
138
 
127
139
  case action
@@ -135,19 +147,21 @@ module Aspera
135
147
  when :opt_without_arg # if present and true : just add option without value
136
148
  add_param=false
137
149
  case parameter_value
138
- when false# nothing to put on command line, no creation by default
139
- when true; add_param=true
140
- else raise Fasp::Error.new("unsupported #{param_name}: #{parameter_value}")
150
+ when false then nil # nothing to put on command line, no creation by default
151
+ when true then add_param=true
152
+ else raise Fasp::Error, "unsupported #{param_name}: #{parameter_value}"
141
153
  end
142
- add_param=!add_param if options[:add_on_false]
143
- add_command_line_options([options[:option_switch]]) if add_param
154
+ add_param= !add_param if options[:add_on_false]
155
+ add_command_line_options([options[:clswitch]]) if add_param
144
156
  when :opt_with_arg # transform into command line option with value
145
157
  #parameter_value=parameter_value.to_s if parameter_value.is_a?(Integer)
146
158
  parameter_value=[parameter_value] unless parameter_value.is_a?(Array)
147
159
  # if transfer_spec value is an array, applies option many times
148
- parameter_value.each{|v|add_command_line_options([options[:option_switch],v])}
160
+ parameter_value.each{|v|add_command_line_options([options[:clswitch],v])}
161
+ when NilClass
162
+ Log.log.debug("Ignoring parameter: #{param_name}")
149
163
  else
150
- raise "Error"
164
+ raise "ERROR: unknown action: #{action}/#{action.class}"
151
165
  end
152
166
  end
153
167
  end
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  require 'aspera/log'
2
3
  require 'aspera/rest'
3
4
  require 'xmlsimple'
@@ -11,25 +12,28 @@ module Aspera
11
12
  @auth_url=auth_url
12
13
  @api_key=api_key
13
14
  s3_api=Aspera::Rest.new({
14
- :base_url => storage_endpoint,
15
- :not_auth_codes => ['401','403'], # error codes when not authorized
16
- :headers => {'ibm-service-instance-id' => instance_id},
17
- :auth => {
18
- :type => :oauth2,
19
- :base_url => @auth_url,
20
- :grant => :ibm_apikey,
21
- :api_key => @api_key
22
- }})
15
+ base_url: storage_endpoint,
16
+ not_auth_codes: ['401','403'], # error codes when not authorized
17
+ headers: {'ibm-service-instance-id' => instance_id},
18
+ auth: {
19
+ type: :oauth2,
20
+ base_url: @auth_url,
21
+ crtype: :generic,
22
+ generic: {
23
+ grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
24
+ response_type: 'cloud_iam',
25
+ apikey: @api_key
26
+ }}})
23
27
  # read FASP connection information for bucket
24
- xml_result_text=s3_api.call({:operation=>'GET',:subpath=>bucket_name,:headers=>{'Accept'=>'application/xml'},:url_params=>{'faspConnectionInfo'=>nil}})[:http].body
28
+ xml_result_text=s3_api.call({operation: 'GET',subpath: bucket_name,headers: {'Accept'=>'application/xml'},url_params: {'faspConnectionInfo'=>nil}})[:http].body
25
29
  ats_info=XmlSimple.xml_in(xml_result_text, {'ForceArray' => false})
26
30
  Aspera::Log.dump('ats_info',ats_info)
27
31
  super({
28
- :base_url => ats_info['ATSEndpoint'],
29
- :auth => {
30
- :type => :basic,
31
- :username => ats_info['AccessKey']['Id'],
32
- :password => ats_info['AccessKey']['Secret']}})
32
+ base_url: ats_info['ATSEndpoint'],
33
+ auth: {
34
+ type: :basic,
35
+ username: ats_info['AccessKey']['Id'],
36
+ password: ats_info['AccessKey']['Secret']}})
33
37
  # prepare transfer spec addition
34
38
  @add_ts={'tags'=>{'aspera'=>{'node'=>{'storage_credentials'=>{
35
39
  'type' => 'token',
@@ -42,12 +46,16 @@ module Aspera
42
46
  def generate_token
43
47
  # OAuth API to get delegated token
44
48
  delegated_oauth=Oauth.new({
45
- :type => :oauth2,
46
- :base_url => @auth_url,
47
- :grant => :delegated_refresh,
48
- :api_key => @api_key,
49
- :token_field=> TOKEN_FIELD
50
- })
49
+ type: :oauth2,
50
+ base_url: @auth_url,
51
+ token_field: TOKEN_FIELD,
52
+ crtype: :generic,
53
+ generic: {
54
+ grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
55
+ response_type: 'delegated_refresh_token',
56
+ apikey: @api_key,
57
+ receiver_client_ids: 'aspera_ats'
58
+ }})
51
59
  # get delagated token to be placed in rest call header and in transfer tags
52
60
  @add_ts['tags']['aspera']['node']['storage_credentials']['token'][TOKEN_FIELD]=delegated_oauth.get_authorization().gsub(/^Bearer /,'')
53
61
  @params[:headers]={'X-Aspera-Storage-Credentials'=>JSON.generate(@add_ts['tags']['aspera']['node']['storage_credentials'])}
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  require 'aspera/log'
2
3
  require 'singleton'
3
4
 
@@ -6,8 +7,8 @@ module Aspera
6
7
  class DataRepository
7
8
  include Singleton
8
9
  # get binary value from data repository
9
- def get_bin(id)
10
- File.read(File.join(File.expand_path(File.dirname(__FILE__)),'data',id.to_s),mode: 'rb')
10
+ def data(id)
11
+ File.read(File.join(__dir__,'data',id.to_s),mode: 'rb')
11
12
  end
12
13
  end
13
14
  end