aspera-cli 4.0.0.pre2 → 4.0.0.pre3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,6 +2,7 @@ require 'aspera/colors'
2
2
  require 'aspera/log'
3
3
  require 'aspera/cli/extended_value'
4
4
  require 'optparse'
5
+ require 'io/console'
5
6
 
6
7
  module Aspera
7
8
  module Cli
@@ -135,6 +136,12 @@ module Aspera
135
136
  Log.log.debug("add_cmd_line_options:commands/args=#{@unprocessed_cmd_line_arguments},options=#{@unprocessed_cmd_line_options}".red)
136
137
  end
137
138
 
139
+ def prompt_user_input(prompt,sensitive)
140
+ return STDIN.getpass("#{prompt}> ") if sensitive
141
+ print "#{prompt}> "
142
+ return STDIN.gets.chomp
143
+ end
144
+
138
145
  def get_interactive(type,descr,expected=:single)
139
146
  if !@ask_missing_mandatory
140
147
  if expected.is_a?(Array)
@@ -143,23 +150,23 @@ module Aspera
143
150
  raise CliBadArgument,"missing argument (#{expected}): #{descr}"
144
151
  end
145
152
  result=nil
153
+ # Note: mandatory parenthesis here !
154
+ sensitive = (type.eql?(:option) and @declared_options[descr.to_sym][:sensitive].eql?(true))
155
+ default_prompt="#{type}: #{descr}"
146
156
  # ask interactively
147
157
  case expected
148
158
  when :multiple
149
159
  result=[]
150
160
  puts " (one per line, end with empty line)"
151
161
  loop do
152
- print "#{type}: #{descr}> "
153
- entry=STDIN.gets.chomp
162
+ entry=prompt_user_input(default_prompt,sensitive)
154
163
  break if entry.empty?
155
164
  result.push(ExtendedValue.instance.evaluate(entry))
156
165
  end
157
166
  when :single
158
- print "#{type}: #{descr}> "
159
- result=ExtendedValue.instance.evaluate(STDIN.gets.chomp)
167
+ result=ExtendedValue.instance.evaluate(prompt_user_input(default_prompt,sensitive))
160
168
  else # one fixed
161
- print "#{expected.join(' ')}\n#{type}: #{descr}> "
162
- result=self.class.get_from_list(STDIN.gets.chomp,descr,expected)
169
+ result=self.class.get_from_list(prompt_user_input("#{expected.join(' ')}\n#{default_prompt}",sensitive),descr,expected)
163
170
  end
164
171
  return result
165
172
  end
@@ -206,6 +213,12 @@ module Aspera
206
213
  return
207
214
  end
208
215
  @declared_options[option_symbol]={:type=>type}
216
+ # by default passwords and secrets are sensitive, else specify when declaring the option
217
+ set_is_sensitive(option_symbol) if !%w{password secret key}.select{|i| option_symbol.to_s.end_with?(i)}.empty?
218
+ end
219
+
220
+ def set_is_sensitive(option_symbol)
221
+ @declared_options[option_symbol][:sensitive]=true
209
222
  end
210
223
 
211
224
  # define option with handler
@@ -11,7 +11,7 @@ require 'date'
11
11
  module Aspera
12
12
  module Cli
13
13
  module Plugins
14
- class Oncloud < BasicAuthPlugin
14
+ class Aoc < BasicAuthPlugin
15
15
  VAL_ALL='ALL'
16
16
  private_constant :VAL_ALL
17
17
  attr_reader :api_aoc
@@ -32,10 +32,8 @@ module Aspera
32
32
  PROGRAM_NAME_V2 = 'mlia'
33
33
  # default redirect for AoC web auth
34
34
  DEFAULT_REDIRECT='http://localhost:12345'
35
- # folder containing custom plugins in `main_folder`
35
+ # folder containing custom plugins in user's config folder
36
36
  ASPERA_PLUGINS_FOLDERNAME='plugins'
37
- # folder containing plugins in the gem's main folder
38
- GEM_PLUGINS_FOLDER='aspera/cli/plugins'
39
37
  RUBY_FILE_EXT='.rb'
40
38
  AOC_COMMAND_V1='files'
41
39
  AOC_COMMAND_V2='aspera'
@@ -50,7 +48,7 @@ module Aspera
50
48
  self.options.add_option_preset(preset_by_name(value))
51
49
  end
52
50
 
53
- private_constant :ASPERA_HOME_FOLDER_NAME,:DEFAULT_CONFIG_FILENAME,:CONF_PRESET_CONFIG,:CONF_PRESET_VERSION,:CONF_PRESET_DEFAULT,:PROGRAM_NAME_V1,:PROGRAM_NAME_V2,:DEFAULT_REDIRECT,:ASPERA_PLUGINS_FOLDERNAME,:GEM_PLUGINS_FOLDER,:RUBY_FILE_EXT,:AOC_COMMAND_V1,:AOC_COMMAND_V2,:AOC_COMMAND_V3,:AOC_COMMAND_CURRENT,:DEMO
51
+ private_constant :ASPERA_HOME_FOLDER_NAME,:DEFAULT_CONFIG_FILENAME,:CONF_PRESET_CONFIG,:CONF_PRESET_VERSION,:CONF_PRESET_DEFAULT,:PROGRAM_NAME_V1,:PROGRAM_NAME_V2,:DEFAULT_REDIRECT,:ASPERA_PLUGINS_FOLDERNAME,:RUBY_FILE_EXT,:AOC_COMMAND_V1,:AOC_COMMAND_V2,:AOC_COMMAND_V3,:AOC_COMMAND_CURRENT,:DEMO
54
52
  attr_accessor :option_ak_secret,:option_secrets
55
53
 
56
54
  def initialize(env,tool_name,help_url,version)
@@ -61,18 +59,25 @@ module Aspera
61
59
  @plugin_lookup_folders=[]
62
60
  @use_plugin_defaults=true
63
61
  @config_presets=nil
62
+ @connect_versions=nil
64
63
  @program_version=version
65
64
  @tool_name=tool_name
66
65
  @help_url=help_url
67
- @main_folder=File.join(Dir.home,ASPERA_HOME_FOLDER_NAME,tool_name)
66
+ tool_main_env_var="#{tool_name.upcase}_HOME"
67
+ if ENV.has_key?(tool_main_env_var)
68
+ @main_folder=ENV[tool_main_env_var]
69
+ else
70
+ user_home_folder=Dir.home
71
+ raise CliError,"Home folder does not exist: #{user_home_folder}. Check your user environment or use #{tool_main_env_var}." unless Dir.exist?(user_home_folder)
72
+ @main_folder=File.join(user_home_folder,ASPERA_HOME_FOLDER_NAME,tool_name)
73
+ end
68
74
  @conf_file_default=File.join(@main_folder,DEFAULT_CONFIG_FILENAME)
69
75
  @option_config_file=@conf_file_default
70
- @connect_versions=nil
71
- # set folder where generated FASP files are
76
+ Log.log.debug("#{tool_name} folder: #{@main_folder}")
77
+ # set folder for FASP SDK
72
78
  Fasp::Installation.instance.folder=File.join(@main_folder,'sdk')
73
- FileUtils.mkdir_p(Fasp::Installation.instance.folder)
74
79
  add_plugin_lookup_folder(File.join(@main_folder,ASPERA_PLUGINS_FOLDERNAME))
75
- add_plugin_lookup_folder(File.join(Main.gem_root,GEM_PLUGINS_FOLDER))
80
+ add_plugin_lookup_folder(self.class.gem_plugins_folder)
76
81
  # do file parameter first
77
82
  self.options.set_obj_attr(:config_file,self,:option_config_file)
78
83
  self.options.add_opt_simple(:config_file,"read parameters from file in YAML format, current=#{@option_config_file}")
@@ -154,6 +159,24 @@ module Aspera
154
159
  nil
155
160
  end
156
161
 
162
+ # folder containing plugins in the gem's main folder
163
+ def self.gem_plugins_folder
164
+ File.dirname(File.expand_path(__FILE__))
165
+ end
166
+
167
+ # find the root folder of gem where this class is
168
+ # go up as many times as englobing modules (not counting class, as it is a file)
169
+ def self.gem_root
170
+ File.expand_path(Module.nesting[1].to_s.gsub('::','/').gsub(%r([^/]+),'..'),File.dirname(__FILE__))
171
+ end
172
+
173
+ # instanciate a plugin
174
+ # plugins must be Capitalized
175
+ def self.plugin_new(plugin_name_sym,env)
176
+ # Module.nesting[2] is Aspera::Cli
177
+ return Object::const_get("#{Module.nesting[2].to_s}::Plugins::#{plugin_name_sym.to_s.capitalize}").new(env)
178
+ end
179
+
157
180
  def self.flatten_all_config(t)
158
181
  r=[]
159
182
  t.each do |k,v|
@@ -219,7 +242,7 @@ module Aspera
219
242
  end
220
243
 
221
244
  def option_ascp_path
222
- Fasp::Installation.instance.ascp_path
245
+ Fasp::Installation.instance.path(:ascp)
223
246
  end
224
247
 
225
248
  def option_use_product=(value)
@@ -450,7 +473,7 @@ module Aspera
450
473
  when :use
451
474
  default_product=self.options.get_next_argument('product name')
452
475
  Fasp::Installation.instance.use_ascp_from_product(default_product)
453
- preset_name=set_global_default(:ascp_path,Fasp::Installation.instance.ascp_path)
476
+ preset_name=set_global_default(:ascp_path,Fasp::Installation.instance.path(:ascp))
454
477
  save_presets_to_config_file
455
478
  return {:type=>:status, :data=>"saved to default global preset #{preset_name}"}
456
479
  end
@@ -578,7 +601,7 @@ module Aspera
578
601
  # init defaults if necessary
579
602
  @config_presets[CONF_PRESET_DEFAULT]||=Hash.new
580
603
  if !option_override
581
- raise CliError,"a default configuration already exists for plugin '#{AOC_COMMAND_V2}' (use --override=yes)" if @config_presets[CONF_PRESET_DEFAULT].has_key?(AOC_COMMAND_V2)
604
+ raise CliError,"a default configuration already exists for plugin '#{AOC_COMMAND_CURRENT}' (use --override=yes)" if @config_presets[CONF_PRESET_DEFAULT].has_key?(AOC_COMMAND_CURRENT)
582
605
  raise CliError,"preset already exists: #{aspera_preset_name} (use --override=yes)" if @config_presets.has_key?(aspera_preset_name)
583
606
  end
584
607
  # lets see if path to priv key is provided
@@ -605,11 +628,11 @@ module Aspera
605
628
  require 'aspera/cli/plugins/aoc'
606
629
  # make username mandatory for jwt, this triggers interactive input
607
630
  self.options.get_option(:username,:mandatory)
608
- files_plugin=Plugins::Oncloud.new(@agents.merge({skip_basic_auth_options: true, private_key_path: private_key_path}))
631
+ # instanciate AoC plugin
632
+ files_plugin=self.class.plugin_new(AOC_COMMAND_CURRENT,@agents.merge({skip_basic_auth_options: true, private_key_path: private_key_path}))
609
633
  auto_set_pub_key=false
610
634
  auto_set_jwt=false
611
635
  use_browser_authentication=false
612
-
613
636
  if self.options.get_option(:use_generic_client)
614
637
  self.format.display_status("Using global client_id.")
615
638
  self.format.display_status("Please Login to your Aspera on Cloud instance.".red)
@@ -669,8 +692,8 @@ module Aspera
669
692
  o=self.options.get_option(s)
670
693
  @config_presets[s.to_s] = o unless o.nil?
671
694
  end
672
- self.format.display_status("Setting config preset as default for #{AOC_COMMAND_V2}")
673
- @config_presets[CONF_PRESET_DEFAULT][AOC_COMMAND_V2]=aspera_preset_name
695
+ self.format.display_status("Setting config preset as default for #{AOC_COMMAND_CURRENT}")
696
+ @config_presets[CONF_PRESET_DEFAULT][AOC_COMMAND_CURRENT]=aspera_preset_name
674
697
  self.format.display_status("saving config file")
675
698
  save_presets_to_config_file
676
699
  return Main.result_status("Done.\nYou can test with:\n#{@tool_name} #{AOC_COMMAND_CURRENT} user info show")
@@ -682,7 +705,8 @@ module Aspera
682
705
  require 'aspera/cli/plugins/aoc'
683
706
  # need url / username
684
707
  add_plugin_default_preset(AOC_COMMAND_V3.to_sym)
685
- files_plugin=Plugins::Oncloud.new(@agents) # TODO: is this line needed ?
708
+ # instanciate AoC plugin
709
+ files_plugin=self.class.plugin_new(AOC_COMMAND_CURRENT,@agents) # TODO: is this line needed ?
686
710
  url=self.options.get_option(:url,:mandatory)
687
711
  cli_conf_file=Fasp::Installation.instance.cli_conf_file
688
712
  data=JSON.parse(File.read(cli_conf_file))
@@ -721,7 +745,7 @@ module Aspera
721
745
  when :ascp
722
746
  execute_action_ascp
723
747
  when :gem_path
724
- return Main.result_status(Main.gem_root)
748
+ return Main.result_status(self.class.gem_root)
725
749
  when :folder
726
750
  return Main.result_status(@main_folder)
727
751
  when :file
@@ -1,5 +1,5 @@
1
1
  module Aspera
2
2
  module Cli
3
- VERSION = "4.0.0.pre2"
3
+ VERSION = "4.0.0.pre3"
4
4
  end
5
5
  end
@@ -33,7 +33,11 @@ class String
33
33
  VTSTYLES.each do |name,code|
34
34
  begin_seq=vtcmd(code)
35
35
  end_seq=vtcmd((code >= 10) ? 0 : code+20+(code.eql?(1)?1:0))
36
- define_method(name){"#{begin_seq}#{self}#{end_seq}"}
36
+ if STDERR.tty?
37
+ define_method(name){"#{begin_seq}#{self}#{end_seq}"}
38
+ else
39
+ define_method(name){self}
40
+ end
37
41
  public name
38
42
  end
39
43
  end
@@ -1,4 +1,5 @@
1
1
  require 'aspera/log'
2
+ require 'rbconfig'
2
3
 
3
4
  module Aspera
4
5
  # a simple binary data repository
@@ -20,7 +21,7 @@ module Aspera
20
21
  when /aix/
21
22
  return OS_AIX
22
23
  else
23
- raise "Unknown: #{RbConfig::CONFIG['host_os']}"
24
+ raise "Unknown OS: #{RbConfig::CONFIG['host_os']}"
24
25
  end
25
26
  end
26
27
  CPU_X86_64=:x86_64
@@ -31,7 +32,7 @@ module Aspera
31
32
 
32
33
  def self.cpu
33
34
  case RbConfig::CONFIG['host_cpu']
34
- when /x86_64/
35
+ when /x86_64/,/x64/
35
36
  return :x86_64
36
37
  when /powerpc/
37
38
  return :ppc64le if os.eql?(OS_LINUX)
@@ -39,7 +40,7 @@ module Aspera
39
40
  when /s390/
40
41
  return :s390
41
42
  else # other
42
- raise "Unknown: #{RbConfig::CONFIG['host_cpu']}"
43
+ raise "Unknown CPU: #{RbConfig::CONFIG['host_cpu']}"
43
44
  end
44
45
  end
45
46
 
@@ -51,5 +52,15 @@ module Aspera
51
52
  return '.exe' if os.eql?(OS_WINDOWS)
52
53
  return ''
53
54
  end
55
+
56
+ # on Windows, the env var %USERPROFILE% provides the path to user's home more reliably then %HOMEDRIVE%%HOMEPATH%
57
+ def self.fix_home
58
+ if os.eql?(OS_WINDOWS)
59
+ if ENV.has_key?('USERPROFILE') and Dir.exist?(ENV['USERPROFILE'])
60
+ ENV['HOME']=ENV['USERPROFILE']
61
+ Log.log.debug("Windows: set home to USERPROFILE: #{ENV['HOME']}")
62
+ end
63
+ end
64
+ end
54
65
  end
55
66
  end
@@ -5,6 +5,7 @@ require 'aspera/data_repository'
5
5
  require 'xmlsimple'
6
6
  require 'zlib'
7
7
  require 'base64'
8
+ require 'fileutils'
8
9
 
9
10
  module Aspera
10
11
  module Fasp
@@ -17,14 +18,21 @@ module Aspera
17
18
  # Installation.instance.ascp_path=""
18
19
  class Installation
19
20
  include Singleton
20
- # currently used ascp executable
21
- attr_accessor :ascp_path
22
- # location of SDK files
23
- attr_accessor :folder
24
21
  PRODUCT_CONNECT='Aspera Connect'
25
22
  PRODUCT_CLI_V1='Aspera CLI'
26
23
  PRODUCT_DRIVE='Aspera Drive'
27
24
  PRODUCT_ENTSRV='Enterprise Server'
25
+ # currently used ascp executable
26
+ def ascp_path=(v)
27
+ @path_to_ascp=v
28
+ end
29
+
30
+ # location of SDK files
31
+ def folder=(v)
32
+ @sdk_folder=v
33
+ folder_path
34
+ end
35
+
28
36
  # find ascp in named product (use value : FIRST_FOUND='FIRST' to just use first one)
29
37
  # or select one from installed_products()
30
38
  def use_ascp_from_product(product_name)
@@ -35,8 +43,8 @@ module Aspera
35
43
  pl=installed_products.select{|i|i[:name].eql?(product_name)}.first
36
44
  raise "no such product installed: #{product_name}" if pl.nil?
37
45
  end
38
- @ascp_path=pl[:ascp_path]
39
- Log.log.debug("ascp_path=#{@ascp_path}")
46
+ self.ascp_path=pl[:ascp_path]
47
+ Log.log.debug("ascp_path=#{@path_to_ascp}")
40
48
  end
41
49
 
42
50
  # @return the list of installed products in format of product_locations
@@ -46,7 +54,7 @@ module Aspera
46
54
  # add sdk as first search path
47
55
  @found_products.unshift({# SDK
48
56
  :expected =>'SDK',
49
- :app_root =>@folder,
57
+ :app_root =>self.folder_path,
50
58
  :sub_bin =>''
51
59
  })
52
60
  @found_products.select! do |pl|
@@ -76,24 +84,24 @@ module Aspera
76
84
  def path(k)
77
85
  case k
78
86
  when :ascp,:ascp4
79
- use_ascp_from_product(FIRST_FOUND) if @ascp_path.nil?
80
- file=@ascp_path
87
+ use_ascp_from_product(FIRST_FOUND) if @path_to_ascp.nil?
88
+ file=@path_to_ascp
81
89
  # note that there might be a .exe at the end
82
90
  file=file.gsub('ascp','ascp4') if k.eql?(:ascp4)
83
91
  when :ssh_bypass_key_dsa
84
- file=File.join(@folder,'aspera_bypass_dsa.pem')
92
+ file=File.join(self.folder_path,'aspera_bypass_dsa.pem')
85
93
  File.write(file,get_key('dsa',1)) unless File.exist?(file)
86
94
  File.chmod(0400,file)
87
95
  when :ssh_bypass_key_rsa
88
- file=File.join(@folder,'aspera_bypass_rsa.pem')
96
+ file=File.join(self.folder_path,'aspera_bypass_rsa.pem')
89
97
  File.write(file,get_key('rsa',2)) unless File.exist?(file)
90
98
  File.chmod(0400,file)
91
99
  when :aspera_license
92
- file=File.join(@folder,'aspera-license')
100
+ file=File.join(self.folder_path,'aspera-license')
93
101
  File.write(file,Base64.strict_encode64("#{Zlib::Inflate.inflate(DataRepository.instance.get_bin(6))}==SIGNATURE==\n#{Base64.strict_encode64(DataRepository.instance.get_bin(7))}")) unless File.exist?(file)
94
102
  File.chmod(0400,file)
95
103
  when :aspera_conf
96
- file=File.join(@folder,'aspera.conf')
104
+ file=File.join(self.folder_path,'aspera.conf')
97
105
  File.write(file,%Q{<?xml version='1.0' encoding='UTF-8'?>
98
106
  <CONF version="2">
99
107
  <default>
@@ -112,8 +120,8 @@ module Aspera
112
120
  }) unless File.exist?(file)
113
121
  File.chmod(0400,file)
114
122
  when :fallback_cert,:fallback_key
115
- file_key=File.join(@folder,'aspera_fallback_key.pem')
116
- file_cert=File.join(@folder,'aspera_fallback_cert.pem')
123
+ file_key=File.join(self.folder_path,'aspera_fallback_key.pem')
124
+ file_cert=File.join(self.folder_path,'aspera_fallback_cert.pem')
117
125
  if !File.exist?(file_key) or !File.exist?(file_cert)
118
126
  require 'openssl'
119
127
  # create new self signed certificate for http fallback
@@ -180,7 +188,7 @@ module Aspera
180
188
  Zip::File.open(sdk_zip_path) do |zip_file|
181
189
  zip_file.each do |entry|
182
190
  if entry.name.include?(filter) and !entry.name.end_with?('/')
183
- archive_file=File.join(@folder,File.basename(entry.name))
191
+ archive_file=File.join(self.folder_path,File.basename(entry.name))
184
192
  File.open(archive_file, 'wb') do |output_stream|
185
193
  IO.copy_stream(entry.get_input_stream, output_stream)
186
194
  end
@@ -197,7 +205,7 @@ module Aspera
197
205
  # get version from ascp, only after full extract, as windows requires SSL/TLS DLLs
198
206
  m=%x{#{ascp_path} -A}.match(/ascp version (.*)/)
199
207
  ascp_version=m[1] unless m.nil?
200
- File.write(File.join(@folder,PRODUCT_INFO),"<product><name>IBM Aspera SDK</name><version>#{ascp_version}</version></product>")
208
+ File.write(File.join(self.folder_path,PRODUCT_INFO),"<product><name>IBM Aspera SDK</name><version>#{ascp_version}</version></product>")
201
209
  end
202
210
  return ascp_version
203
211
  end
@@ -223,11 +231,17 @@ module Aspera
223
231
  end
224
232
 
225
233
  def initialize
226
- @ascp_path=nil
227
- @folder='.'
234
+ @path_to_ascp=nil
235
+ @sdk_folder=nil
228
236
  @found_products=nil
229
237
  end
230
238
 
239
+ def folder_path
240
+ raise "undefined path to SDK" if @sdk_folder.nil?
241
+ FileUtils.mkdir_p(@sdk_folder) unless Dir.exist?(@sdk_folder)
242
+ @sdk_folder
243
+ end
244
+
231
245
  # returns product folders depending on OS
232
246
  # fields
233
247
  # :expected M app name is taken from the manifest if present, else defaults to this value
@@ -17,17 +17,25 @@ require 'securerandom'
17
17
 
18
18
  module Aspera
19
19
  module Fasp
20
- # default transfer username for access key based transfers
20
+ # (public) default transfer username for access key based transfers
21
21
  ACCESS_KEY_TRANSFER_USER='xfer'
22
22
  # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
23
23
  class Local < Manager
24
+ ASCP_SPAWN_TIMEOUT_SEC = 3
25
+ private_constant :ASCP_SPAWN_TIMEOUT_SEC
24
26
  # set to false to keep ascp progress bar display (basically: removes ascp's option -q)
25
27
  attr_accessor :quiet
26
- # start FASP transfer based on transfer spec (hash table)
27
- # note that it is asynchronous
28
+ # start ascp transfer (non blocking), single or multi-session
29
+ # job information added to @jobs
30
+ # @param transfer_spec [Hash] aspera transfer specification
31
+ # @param options [Hash] :resumer, :regenerate_token
28
32
  def start_transfer(transfer_spec,options={})
29
33
  raise "option: must be hash (or nil)" unless options.is_a?(Hash)
30
- job_id=options[:job_id] || SecureRandom.uuid
34
+ job_options = options.clone
35
+ job_options[:resumer] ||= @resume_policy
36
+ job_options[:job_id] ||= SecureRandom.uuid
37
+ # clone transfer spec because we modify it (first level keys)
38
+ transfer_spec=transfer_spec.clone
31
39
  # if there is aspera tags
32
40
  if transfer_spec['tags'].is_a?(Hash) and transfer_spec['tags']['aspera'].is_a?(Hash)
33
41
  # TODO: what is this for ? only on local ascp ?
@@ -39,6 +47,7 @@ module Aspera
39
47
  transfer_spec['tags']['aspera']['xfer_retry']||=3600
40
48
  end
41
49
  Log.dump('ts',transfer_spec)
50
+
42
51
  # add bypass keys when authentication is token and no auth is provided
43
52
  if transfer_spec.has_key?('token') and
44
53
  !transfer_spec.has_key?('remote_password') and
@@ -47,6 +56,21 @@ module Aspera
47
56
  transfer_spec['EX_ssh_key_paths'] = Installation.instance.bypass_keys
48
57
  end
49
58
 
59
+ # TODO: check if changing fasp(UDP) port is really necessary, not clear from doc
60
+ # compute this before using transfer spec, even if the var is not used in single session
61
+ multi_session_udp_port_base=33001
62
+ multi_session_number=nil
63
+ if transfer_spec.has_key?('multi_session')
64
+ multi_session_number=transfer_spec['multi_session'].to_i
65
+ raise "multi_session(#{transfer_spec['multi_session']}) shall be integer > 1" unless multi_session_number >= 1
66
+ # managed here, so delete from transfer spec
67
+ transfer_spec.delete('multi_session')
68
+ if transfer_spec.has_key?('fasp_port')
69
+ multi_session_udp_port_base=transfer_spec['fasp_port']
70
+ transfer_spec.delete('fasp_port')
71
+ end
72
+ end
73
+
50
74
  # compute known args
51
75
  env_args=Parameters.ts_to_env_args(transfer_spec,wss: @enable_wss)
52
76
 
@@ -60,41 +84,32 @@ module Aspera
60
84
 
61
85
  # transfer job can be multi session
62
86
  xfer_job={
63
- :id => job_id,
64
- :sessions => []
87
+ :id => job_options[:job_id],
88
+ :sessions => [] # all sessions as below
65
89
  }
66
90
 
67
91
  # generic session information
68
92
  session={
69
- :state => :initial, # :initial, :started, :success, :failed
70
- :env_args => env_args,
71
- :resumer => options['resume_policy'] || @resume_policy,
72
- :options => options
93
+ :thread => nil, # Thread object monitoring management port, not nil when pushed to :sessions
94
+ :error => nil, # exception if failed
95
+ :io => nil, # management port server socket
96
+ :id => nil, # SessionId from INIT message in mgt port
97
+ :env_args => env_args, # env vars and args to ascp (from transfer spec)
98
+ :options => job_options # [Hash]
73
99
  }
74
100
 
75
101
  Log.log.debug("starting session thread(s)")
76
- if !transfer_spec.has_key?('multi_session')
102
+ if !multi_session_number
77
103
  # single session for transfer : simple
78
104
  session[:thread] = Thread.new(session) {|s|transfer_thread_entry(s)}
79
105
  xfer_job[:sessions].push(session)
80
106
  else
81
- # default value overriden by fasp_port
82
- multi_session_udp_port_base=33001
83
- multi_session=transfer_spec['multi_session'].to_i
84
- raise "multi_session(#{transfer_spec['multi_session']}) shall be integer > 1" unless multi_session >= 1
85
- # managed here, so delete from transfer spec
86
- transfer_spec.delete('multi_session')
87
- # TODO: check if changing fasp(UDP) port is really necessary, not clear from doc
88
- if transfer_spec.has_key?('fasp_port')
89
- multi_session_udp_port_base=transfer_spec['fasp_port']
90
- transfer_spec.delete('fasp_port')
91
- end
92
- 1.upto(multi_session) do |i|
107
+ 1.upto(multi_session_number) do |i|
93
108
  # do deep copy (each thread has its own copy because it is modified here below and in thread)
94
109
  this_session=session.clone()
95
110
  this_session[:env_args]=this_session[:env_args].clone()
96
111
  this_session[:env_args][:args]=this_session[:env_args][:args].clone()
97
- this_session[:env_args][:args].unshift("-C#{i}:#{multi_session}")
112
+ this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_number}")
98
113
  # necessary only if server is not linux, i.e. server does not support port re-use
99
114
  this_session[:env_args][:args].unshift("-O","#{multi_session_udp_port_base+i-1}")
100
115
  this_session[:thread] = Thread.new(this_session) {|s|transfer_thread_entry(s)}
@@ -104,74 +119,59 @@ module Aspera
104
119
  Log.log.debug("started session thread(s)")
105
120
 
106
121
  # add job to list of jobs
107
- @jobs[job_id]=xfer_job
108
-
122
+ @jobs[job_options[:job_id]]=xfer_job
109
123
  Log.log.debug("jobs: #{@jobs.keys.count}")
110
- return job_id
124
+
125
+ return job_options[:job_id]
111
126
  end # start_transfer
112
127
 
113
128
  # wait for completion of all jobs started
114
129
  # @return list of :success or error message
115
130
  def wait_for_transfers_completion
116
- Log.log.debug("wait_for_sessions: #{@jobs.values.inject(0){|m,j|m+j[:sessions].count}}")
117
- @mutex.synchronize do
118
- loop do
119
- running=0
120
- result=[]
121
- @jobs.each do |id,job|
122
- job[:sessions].each do |session|
123
- case session[:state]
124
- when :failed; result.push(session[:error])
125
- when :success; result.push(:success)
126
- else running+=1
127
- end
128
- end
129
- end
130
- if running.eql?(0)
131
- # since all are finished and we return the result, clear statuses
132
- @jobs.clear
133
- return result
134
- end
135
- Log.log.debug("wait for completed: running: #{running}")
136
- # wait for session termination
137
- @cond_var.wait(@mutex)
138
- end # loop
139
- end # mutex
140
- # never reach here
141
- raise "internal error"
131
+ Log.log.debug("wait_for_transfers_completion")
132
+ # set to non-nil to exit loop
133
+ result=[]
134
+ @jobs.each do |id,job|
135
+ job[:sessions].each do |session|
136
+ Log.log.debug("join #{session[:thread]}")
137
+ session[:thread].join
138
+ result.push(session[:error] ? session[:error] : :success)
139
+ end
140
+ end
141
+ Log.log.debug("all transfers joined")
142
+ # since all are finished and we return the result, clear statuses
143
+ @jobs.clear
144
+ return result
142
145
  end
143
146
 
144
- # terminates monitor thread
147
+ # used by asession (to be removed ?)
145
148
  def shutdown
146
149
  Log.log.debug("fasp local shutdown")
147
- Log.log.debug("send signal to monitor")
148
- # tell monitor to stop
149
- @mutex.synchronize do
150
- @monitor_stop=true
151
- @cond_var.broadcast
152
- end
153
- # wait for thread termination
154
- @monitor_thread.join
155
- @monitor_thread=nil
156
- Log.log.debug("joined monitor")
157
150
  end
158
151
 
159
- # This is the low level method to start FASP
152
+ # This is the low level method to start the "ascp" process
160
153
  # currently, relies on command line arguments
161
154
  # start ascp with management port.
162
155
  # raises FaspError on error
163
156
  # if there is a thread info: set and broadcast session id
164
157
  # @param env_args a hash containing :args :env :ascp_version
165
- # cloud be private method
166
- def start_transfer_with_args_env(env_args,session=nil)
158
+ # @param session this session information
159
+ # could be private method
160
+ def start_transfer_with_args_env(env_args,session)
161
+ raise "env_args must be Hash" unless env_args.is_a?(Hash)
162
+ raise "session must be Hash" unless session.is_a?(Hash)
167
163
  begin
168
164
  Log.log.debug("env_args=#{env_args.inspect}")
169
- ascp_path=Fasp::Installation.instance.path(env_args[:ascp_version])
165
+ # get location of ascp executable
166
+ ascp_path=@mutex.synchronize do
167
+ Fasp::Installation.instance.path(env_args[:ascp_version])
168
+ end
169
+ # (optional) check it exists
170
170
  raise Fasp::Error.new("no such file: #{ascp_path}") unless File.exist?(ascp_path)
171
- ascp_pid=nil
171
+ # open random local TCP port for listening for ascp management
172
+ mgt_sock = TCPServer.new('127.0.0.1',0)
173
+ # clone arguments as we eed to modify with mgt port
172
174
  ascp_arguments=env_args[:args].clone
173
- # open random local TCP port listening
174
- mgt_sock = TCPServer.new('127.0.0.1',0 )
175
175
  # add management port
176
176
  ascp_arguments.unshift('-M', mgt_sock.addr[1].to_s)
177
177
  # start ascp in sub process
@@ -180,8 +180,9 @@ module Aspera
180
180
  ascp_pid = Process.spawn(env_args[:env],[ascp_path,ascp_path],*ascp_arguments)
181
181
  # in parent, wait for connection to socket max 3 seconds
182
182
  Log.log.debug("before accept for pid (#{ascp_pid})")
183
+ # init management socket
183
184
  ascp_mgt_io=nil
184
- Timeout.timeout( 3 ) do
185
+ Timeout.timeout(ASCP_SPAWN_TIMEOUT_SEC) do
185
186
  ascp_mgt_io = mgt_sock.accept
186
187
  # management messages include file names which may be utf8
187
188
  # by default socket is US-ASCII
@@ -189,21 +190,13 @@ module Aspera
189
190
  ascp_mgt_io.set_encoding(Encoding::UTF_8)
190
191
  end
191
192
  Log.log.debug("after accept (#{ascp_mgt_io})")
192
-
193
- unless session.nil?
194
- @mutex.synchronize do
195
- session[:io]=ascp_mgt_io
196
- @cond_var.broadcast
197
- end
198
- end
193
+ session[:io]=ascp_mgt_io
199
194
  # exact text for event, with \n
200
195
  current_event_text=''
201
196
  # parsed event (hash)
202
197
  current_event_data=nil
203
-
204
198
  # this is the last full status
205
199
  last_status_event=nil
206
-
207
200
  # read management port
208
201
  loop do
209
202
  # TODO: timeout here ?
@@ -222,45 +215,40 @@ module Aspera
222
215
  # event field
223
216
  current_event_data[$1] = $2
224
217
  when ''
225
- # end event
218
+ # empty line is separator to end event information
226
219
  raise "unexpected empty line" if current_event_data.nil?
227
220
  current_event_data[Manager::LISTENER_SESSION_ID_B]=ascp_pid
228
221
  notify_listeners(current_event_text,current_event_data)
229
- # TODO: check if this is always the last event
230
222
  case current_event_data['Type']
223
+ when 'INIT'
224
+ session[:id]=current_event_data['SessionId']
225
+ Log.log.debug("session id: #{session[:id]}")
231
226
  when 'DONE','ERROR'
227
+ # TODO: check if this is always the last event
232
228
  last_status_event = current_event_data
233
- when 'INIT'
234
- unless session.nil?
235
- @mutex.synchronize do
236
- session[:state]=:started
237
- session[:id]=current_event_data['SessionId']
238
- Log.log.debug("session id: #{session[:id]}")
239
- @cond_var.broadcast
240
- end
241
- end
242
229
  end # event type
243
230
  else
244
231
  raise "unexpected line:[#{line}]"
245
232
  end # case
246
- end # loop
233
+ end # loop (process mgt port lines)
247
234
  # check that last status was received before process exit
248
235
  raise "INTERNAL: nil last status" if last_status_event.nil?
249
236
  case last_status_event['Type']
250
237
  when 'DONE'
238
+ # return method (or just don't do anything)
251
239
  return
252
240
  when 'ERROR'
253
241
  Log.log.error("code: #{last_status_event['Code']}")
254
242
  if last_status_event['Description'] =~ /bearer token/i
255
243
  Log.log.error("need to regenerate token".red)
256
- if !session.nil? and session[:options].is_a?(Hash) and session[:options].has_key?(:regenerate_token)
244
+ if session[:options].is_a?(Hash) and session[:options].has_key?(:regenerate_token)
257
245
  # regenerate token here, expired, or error on it
258
246
  env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
259
247
  end
260
248
  end
261
249
  raise Fasp::Error.new(last_status_event['Description'],last_status_event['Code'].to_i)
262
250
  else
263
- raise "INTERNAL ERROR: unexpected last event"
251
+ raise "unexpected last event type: #{last_status_event['Type']}"
264
252
  end
265
253
  rescue SystemCallError => e
266
254
  # Process.spawn
@@ -270,13 +258,9 @@ module Aspera
270
258
  rescue Interrupt => e
271
259
  raise Fasp::Error.new('transfer interrupted by user')
272
260
  ensure
273
- # ensure there is no ascp left running
261
+ # if ascp was successfully started
274
262
  unless ascp_pid.nil?
275
- begin
276
- Process.kill('INT',ascp_pid)
277
- rescue
278
- end
279
- # avoid zombie
263
+ # "wait" for process to avoid zombie
280
264
  Process.wait(ascp_pid)
281
265
  ascp_pid=nil
282
266
  session.delete(:io)
@@ -284,24 +268,26 @@ module Aspera
284
268
  end # begin-ensure
285
269
  end # start_transfer_with_args_env
286
270
 
287
- # send command on mgt port, examples:
271
+ # send command of management port to ascp session
272
+ # @param job_id identified transfer process
273
+ # @param session_index index of session (for multi session)
274
+ # @param data command on mgt port, examples:
288
275
  # {'type'=>'START','source'=>_path_,'destination'=>_path_}
289
276
  # {'type'=>'DONE'}
290
277
  def send_command(job_id,session_index,data)
291
- @mutex.synchronize do
292
- job=@jobs[job_id]
293
- raise "no such job" if job.nil?
294
- session=job[:sessions][session_index]
295
- raise "no such session" if session.nil?
296
- Log.log.debug("command: #{data}")
297
- command=data.
298
- keys.
299
- map{|k|"#{k.capitalize}: #{data[k]}"}.
300
- unshift('FASPMGR 2').
301
- push('','').
302
- join("\n")
303
- session[:io].puts(command)
304
- end
278
+ job=@jobs[job_id]
279
+ raise "no such job" if job.nil?
280
+ session=job[:sessions][session_index]
281
+ raise "no such session" if session.nil?
282
+ Log.log.debug("command: #{data}")
283
+ # build command
284
+ command=data.
285
+ keys.
286
+ map{|k|"#{k.capitalize}: #{data[k]}"}.
287
+ unshift('FASPMGR 2').
288
+ push('','').
289
+ join("\n")
290
+ session[:io].puts(command)
305
291
  end
306
292
 
307
293
  private
@@ -311,66 +297,33 @@ module Aspera
311
297
  super()
312
298
  # by default no interactive progress bar
313
299
  @quiet=true
314
- # shared data between transfer threads and others: protected by mutex, CV on change
300
+ # all transfer jobs, key = SecureRandom.uuid, protected by mutex, condvar on change
315
301
  @jobs={}
316
- # mutex protects jobs data
302
+ # mutex protects global data accessed by threads
317
303
  @mutex=Mutex.new
318
- # cond var is waited or broadcast on jobs data change
319
- @cond_var=ConditionVariable.new
320
- # must be set before starting monitor, set to false to stop thread. also shared and protected by mutex
321
- @monitor_stop=false
322
- @monitor_thread=Thread.new{monitor_thread_entry}
323
304
  @resume_policy=ResumePolicy.new(agent_options)
324
305
  @enable_wss = agent_options[:wss] || false
325
306
  end
326
307
 
327
308
  # transfer thread entry
328
- # implements resumable transfer
329
- # TODO: extract resume algorithm in a specific object
309
+ # @param session information
330
310
  def transfer_thread_entry(session)
331
311
  begin
332
312
  # set name for logging
333
313
  Thread.current[:name]="transfer"
334
- # update state once in thread
335
- session[:state]=:started
336
314
  Log.log.debug("ENTER (#{Thread.current[:name]})")
337
315
  # start transfer with selected resumer policy
338
- session[:resumer].process do
316
+ session[:options][:resumer].process do
339
317
  start_transfer_with_args_env(session[:env_args],session)
340
318
  end
341
319
  Log.log.debug('transfer ok'.bg_green)
342
- session[:state]=:success
343
320
  rescue => e
344
- session[:state]=:failed
345
321
  session[:error]=e
346
- Log.log.error("#{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red) if Log.instance.level.eql?(:debug)
347
- ensure
348
- @mutex.synchronize do
349
- # ensure id is set to unblock start procedure
350
- session[:id]||=nil
351
- @cond_var.broadcast
352
- end
322
+ Log.log.error("Transfer thread error: #{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red) if Log.instance.level.eql?(:debug)
353
323
  end
354
324
  Log.log.debug("EXIT (#{Thread.current[:name]})")
355
325
  end
356
326
 
357
- # main thread method for monitor
358
- # currently: just joins started threads
359
- def monitor_thread_entry
360
- Thread.current[:name]="monitor"
361
- @mutex.synchronize do
362
- until @monitor_stop do
363
- # wait for session termination
364
- @cond_var.wait(@mutex)
365
- @jobs.values do |job|
366
- job[:sessions].each do |session|
367
- session[:thread].join if [:success,:failed].include?(session[:state])
368
- end # sessions
369
- end # jobs
370
- end # monitor run
371
- end # sync
372
- Log.log.debug("EXIT (#{Thread.current[:name]})")
373
- end # monitor_thread_entry
374
327
  end # Local
375
328
  end
376
329
  end