aspera-cli 4.0.0.pre2 → 4.0.0.pre3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +113 -35
- data/bin/ascli +2 -0
- data/docs/Makefile +2 -1
- data/docs/README.erb.md +36 -10
- data/docs/test_env.conf +5 -5
- data/lib/aspera/cli/main.rb +2 -11
- data/lib/aspera/cli/manager.rb +19 -6
- data/lib/aspera/cli/plugins/aoc.rb +1 -1
- data/lib/aspera/cli/plugins/config.rb +42 -18
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +5 -1
- data/lib/aspera/environment.rb +14 -3
- data/lib/aspera/fasp/installation.rb +33 -19
- data/lib/aspera/fasp/local.rb +107 -154
- data/lib/aspera/log.rb +9 -1
- metadata +2 -2
data/lib/aspera/cli/manager.rb
CHANGED
@@ -2,6 +2,7 @@ require 'aspera/colors'
|
|
2
2
|
require 'aspera/log'
|
3
3
|
require 'aspera/cli/extended_value'
|
4
4
|
require 'optparse'
|
5
|
+
require 'io/console'
|
5
6
|
|
6
7
|
module Aspera
|
7
8
|
module Cli
|
@@ -135,6 +136,12 @@ module Aspera
|
|
135
136
|
Log.log.debug("add_cmd_line_options:commands/args=#{@unprocessed_cmd_line_arguments},options=#{@unprocessed_cmd_line_options}".red)
|
136
137
|
end
|
137
138
|
|
139
|
+
def prompt_user_input(prompt,sensitive)
|
140
|
+
return STDIN.getpass("#{prompt}> ") if sensitive
|
141
|
+
print "#{prompt}> "
|
142
|
+
return STDIN.gets.chomp
|
143
|
+
end
|
144
|
+
|
138
145
|
def get_interactive(type,descr,expected=:single)
|
139
146
|
if !@ask_missing_mandatory
|
140
147
|
if expected.is_a?(Array)
|
@@ -143,23 +150,23 @@ module Aspera
|
|
143
150
|
raise CliBadArgument,"missing argument (#{expected}): #{descr}"
|
144
151
|
end
|
145
152
|
result=nil
|
153
|
+
# Note: mandatory parenthesis here !
|
154
|
+
sensitive = (type.eql?(:option) and @declared_options[descr.to_sym][:sensitive].eql?(true))
|
155
|
+
default_prompt="#{type}: #{descr}"
|
146
156
|
# ask interactively
|
147
157
|
case expected
|
148
158
|
when :multiple
|
149
159
|
result=[]
|
150
160
|
puts " (one per line, end with empty line)"
|
151
161
|
loop do
|
152
|
-
|
153
|
-
entry=STDIN.gets.chomp
|
162
|
+
entry=prompt_user_input(default_prompt,sensitive)
|
154
163
|
break if entry.empty?
|
155
164
|
result.push(ExtendedValue.instance.evaluate(entry))
|
156
165
|
end
|
157
166
|
when :single
|
158
|
-
|
159
|
-
result=ExtendedValue.instance.evaluate(STDIN.gets.chomp)
|
167
|
+
result=ExtendedValue.instance.evaluate(prompt_user_input(default_prompt,sensitive))
|
160
168
|
else # one fixed
|
161
|
-
|
162
|
-
result=self.class.get_from_list(STDIN.gets.chomp,descr,expected)
|
169
|
+
result=self.class.get_from_list(prompt_user_input("#{expected.join(' ')}\n#{default_prompt}",sensitive),descr,expected)
|
163
170
|
end
|
164
171
|
return result
|
165
172
|
end
|
@@ -206,6 +213,12 @@ module Aspera
|
|
206
213
|
return
|
207
214
|
end
|
208
215
|
@declared_options[option_symbol]={:type=>type}
|
216
|
+
# by default passwords and secrets are sensitive, else specify when declaring the option
|
217
|
+
set_is_sensitive(option_symbol) if !%w{password secret key}.select{|i| option_symbol.to_s.end_with?(i)}.empty?
|
218
|
+
end
|
219
|
+
|
220
|
+
def set_is_sensitive(option_symbol)
|
221
|
+
@declared_options[option_symbol][:sensitive]=true
|
209
222
|
end
|
210
223
|
|
211
224
|
# define option with handler
|
@@ -32,10 +32,8 @@ module Aspera
|
|
32
32
|
PROGRAM_NAME_V2 = 'mlia'
|
33
33
|
# default redirect for AoC web auth
|
34
34
|
DEFAULT_REDIRECT='http://localhost:12345'
|
35
|
-
# folder containing custom plugins in
|
35
|
+
# folder containing custom plugins in user's config folder
|
36
36
|
ASPERA_PLUGINS_FOLDERNAME='plugins'
|
37
|
-
# folder containing plugins in the gem's main folder
|
38
|
-
GEM_PLUGINS_FOLDER='aspera/cli/plugins'
|
39
37
|
RUBY_FILE_EXT='.rb'
|
40
38
|
AOC_COMMAND_V1='files'
|
41
39
|
AOC_COMMAND_V2='aspera'
|
@@ -50,7 +48,7 @@ module Aspera
|
|
50
48
|
self.options.add_option_preset(preset_by_name(value))
|
51
49
|
end
|
52
50
|
|
53
|
-
private_constant :ASPERA_HOME_FOLDER_NAME,:DEFAULT_CONFIG_FILENAME,:CONF_PRESET_CONFIG,:CONF_PRESET_VERSION,:CONF_PRESET_DEFAULT,:PROGRAM_NAME_V1,:PROGRAM_NAME_V2,:DEFAULT_REDIRECT,:ASPERA_PLUGINS_FOLDERNAME,:
|
51
|
+
private_constant :ASPERA_HOME_FOLDER_NAME,:DEFAULT_CONFIG_FILENAME,:CONF_PRESET_CONFIG,:CONF_PRESET_VERSION,:CONF_PRESET_DEFAULT,:PROGRAM_NAME_V1,:PROGRAM_NAME_V2,:DEFAULT_REDIRECT,:ASPERA_PLUGINS_FOLDERNAME,:RUBY_FILE_EXT,:AOC_COMMAND_V1,:AOC_COMMAND_V2,:AOC_COMMAND_V3,:AOC_COMMAND_CURRENT,:DEMO
|
54
52
|
attr_accessor :option_ak_secret,:option_secrets
|
55
53
|
|
56
54
|
def initialize(env,tool_name,help_url,version)
|
@@ -61,18 +59,25 @@ module Aspera
|
|
61
59
|
@plugin_lookup_folders=[]
|
62
60
|
@use_plugin_defaults=true
|
63
61
|
@config_presets=nil
|
62
|
+
@connect_versions=nil
|
64
63
|
@program_version=version
|
65
64
|
@tool_name=tool_name
|
66
65
|
@help_url=help_url
|
67
|
-
|
66
|
+
tool_main_env_var="#{tool_name.upcase}_HOME"
|
67
|
+
if ENV.has_key?(tool_main_env_var)
|
68
|
+
@main_folder=ENV[tool_main_env_var]
|
69
|
+
else
|
70
|
+
user_home_folder=Dir.home
|
71
|
+
raise CliError,"Home folder does not exist: #{user_home_folder}. Check your user environment or use #{tool_main_env_var}." unless Dir.exist?(user_home_folder)
|
72
|
+
@main_folder=File.join(user_home_folder,ASPERA_HOME_FOLDER_NAME,tool_name)
|
73
|
+
end
|
68
74
|
@conf_file_default=File.join(@main_folder,DEFAULT_CONFIG_FILENAME)
|
69
75
|
@option_config_file=@conf_file_default
|
70
|
-
@
|
71
|
-
# set folder
|
76
|
+
Log.log.debug("#{tool_name} folder: #{@main_folder}")
|
77
|
+
# set folder for FASP SDK
|
72
78
|
Fasp::Installation.instance.folder=File.join(@main_folder,'sdk')
|
73
|
-
FileUtils.mkdir_p(Fasp::Installation.instance.folder)
|
74
79
|
add_plugin_lookup_folder(File.join(@main_folder,ASPERA_PLUGINS_FOLDERNAME))
|
75
|
-
add_plugin_lookup_folder(
|
80
|
+
add_plugin_lookup_folder(self.class.gem_plugins_folder)
|
76
81
|
# do file parameter first
|
77
82
|
self.options.set_obj_attr(:config_file,self,:option_config_file)
|
78
83
|
self.options.add_opt_simple(:config_file,"read parameters from file in YAML format, current=#{@option_config_file}")
|
@@ -154,6 +159,24 @@ module Aspera
|
|
154
159
|
nil
|
155
160
|
end
|
156
161
|
|
162
|
+
# folder containing plugins in the gem's main folder
|
163
|
+
def self.gem_plugins_folder
|
164
|
+
File.dirname(File.expand_path(__FILE__))
|
165
|
+
end
|
166
|
+
|
167
|
+
# find the root folder of gem where this class is
|
168
|
+
# go up as many times as englobing modules (not counting class, as it is a file)
|
169
|
+
def self.gem_root
|
170
|
+
File.expand_path(Module.nesting[1].to_s.gsub('::','/').gsub(%r([^/]+),'..'),File.dirname(__FILE__))
|
171
|
+
end
|
172
|
+
|
173
|
+
# instanciate a plugin
|
174
|
+
# plugins must be Capitalized
|
175
|
+
def self.plugin_new(plugin_name_sym,env)
|
176
|
+
# Module.nesting[2] is Aspera::Cli
|
177
|
+
return Object::const_get("#{Module.nesting[2].to_s}::Plugins::#{plugin_name_sym.to_s.capitalize}").new(env)
|
178
|
+
end
|
179
|
+
|
157
180
|
def self.flatten_all_config(t)
|
158
181
|
r=[]
|
159
182
|
t.each do |k,v|
|
@@ -219,7 +242,7 @@ module Aspera
|
|
219
242
|
end
|
220
243
|
|
221
244
|
def option_ascp_path
|
222
|
-
Fasp::Installation.instance.
|
245
|
+
Fasp::Installation.instance.path(:ascp)
|
223
246
|
end
|
224
247
|
|
225
248
|
def option_use_product=(value)
|
@@ -450,7 +473,7 @@ module Aspera
|
|
450
473
|
when :use
|
451
474
|
default_product=self.options.get_next_argument('product name')
|
452
475
|
Fasp::Installation.instance.use_ascp_from_product(default_product)
|
453
|
-
preset_name=set_global_default(:ascp_path,Fasp::Installation.instance.
|
476
|
+
preset_name=set_global_default(:ascp_path,Fasp::Installation.instance.path(:ascp))
|
454
477
|
save_presets_to_config_file
|
455
478
|
return {:type=>:status, :data=>"saved to default global preset #{preset_name}"}
|
456
479
|
end
|
@@ -578,7 +601,7 @@ module Aspera
|
|
578
601
|
# init defaults if necessary
|
579
602
|
@config_presets[CONF_PRESET_DEFAULT]||=Hash.new
|
580
603
|
if !option_override
|
581
|
-
raise CliError,"a default configuration already exists for plugin '#{
|
604
|
+
raise CliError,"a default configuration already exists for plugin '#{AOC_COMMAND_CURRENT}' (use --override=yes)" if @config_presets[CONF_PRESET_DEFAULT].has_key?(AOC_COMMAND_CURRENT)
|
582
605
|
raise CliError,"preset already exists: #{aspera_preset_name} (use --override=yes)" if @config_presets.has_key?(aspera_preset_name)
|
583
606
|
end
|
584
607
|
# lets see if path to priv key is provided
|
@@ -605,11 +628,11 @@ module Aspera
|
|
605
628
|
require 'aspera/cli/plugins/aoc'
|
606
629
|
# make username mandatory for jwt, this triggers interactive input
|
607
630
|
self.options.get_option(:username,:mandatory)
|
608
|
-
|
631
|
+
# instanciate AoC plugin
|
632
|
+
files_plugin=self.class.plugin_new(AOC_COMMAND_CURRENT,@agents.merge({skip_basic_auth_options: true, private_key_path: private_key_path}))
|
609
633
|
auto_set_pub_key=false
|
610
634
|
auto_set_jwt=false
|
611
635
|
use_browser_authentication=false
|
612
|
-
|
613
636
|
if self.options.get_option(:use_generic_client)
|
614
637
|
self.format.display_status("Using global client_id.")
|
615
638
|
self.format.display_status("Please Login to your Aspera on Cloud instance.".red)
|
@@ -669,8 +692,8 @@ module Aspera
|
|
669
692
|
o=self.options.get_option(s)
|
670
693
|
@config_presets[s.to_s] = o unless o.nil?
|
671
694
|
end
|
672
|
-
self.format.display_status("Setting config preset as default for #{
|
673
|
-
@config_presets[CONF_PRESET_DEFAULT][
|
695
|
+
self.format.display_status("Setting config preset as default for #{AOC_COMMAND_CURRENT}")
|
696
|
+
@config_presets[CONF_PRESET_DEFAULT][AOC_COMMAND_CURRENT]=aspera_preset_name
|
674
697
|
self.format.display_status("saving config file")
|
675
698
|
save_presets_to_config_file
|
676
699
|
return Main.result_status("Done.\nYou can test with:\n#{@tool_name} #{AOC_COMMAND_CURRENT} user info show")
|
@@ -682,7 +705,8 @@ module Aspera
|
|
682
705
|
require 'aspera/cli/plugins/aoc'
|
683
706
|
# need url / username
|
684
707
|
add_plugin_default_preset(AOC_COMMAND_V3.to_sym)
|
685
|
-
|
708
|
+
# instanciate AoC plugin
|
709
|
+
files_plugin=self.class.plugin_new(AOC_COMMAND_CURRENT,@agents) # TODO: is this line needed ?
|
686
710
|
url=self.options.get_option(:url,:mandatory)
|
687
711
|
cli_conf_file=Fasp::Installation.instance.cli_conf_file
|
688
712
|
data=JSON.parse(File.read(cli_conf_file))
|
@@ -721,7 +745,7 @@ module Aspera
|
|
721
745
|
when :ascp
|
722
746
|
execute_action_ascp
|
723
747
|
when :gem_path
|
724
|
-
return Main.result_status(
|
748
|
+
return Main.result_status(self.class.gem_root)
|
725
749
|
when :folder
|
726
750
|
return Main.result_status(@main_folder)
|
727
751
|
when :file
|
data/lib/aspera/cli/version.rb
CHANGED
data/lib/aspera/colors.rb
CHANGED
@@ -33,7 +33,11 @@ class String
|
|
33
33
|
VTSTYLES.each do |name,code|
|
34
34
|
begin_seq=vtcmd(code)
|
35
35
|
end_seq=vtcmd((code >= 10) ? 0 : code+20+(code.eql?(1)?1:0))
|
36
|
-
|
36
|
+
if STDERR.tty?
|
37
|
+
define_method(name){"#{begin_seq}#{self}#{end_seq}"}
|
38
|
+
else
|
39
|
+
define_method(name){self}
|
40
|
+
end
|
37
41
|
public name
|
38
42
|
end
|
39
43
|
end
|
data/lib/aspera/environment.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'aspera/log'
|
2
|
+
require 'rbconfig'
|
2
3
|
|
3
4
|
module Aspera
|
4
5
|
# a simple binary data repository
|
@@ -20,7 +21,7 @@ module Aspera
|
|
20
21
|
when /aix/
|
21
22
|
return OS_AIX
|
22
23
|
else
|
23
|
-
raise "Unknown: #{RbConfig::CONFIG['host_os']}"
|
24
|
+
raise "Unknown OS: #{RbConfig::CONFIG['host_os']}"
|
24
25
|
end
|
25
26
|
end
|
26
27
|
CPU_X86_64=:x86_64
|
@@ -31,7 +32,7 @@ module Aspera
|
|
31
32
|
|
32
33
|
def self.cpu
|
33
34
|
case RbConfig::CONFIG['host_cpu']
|
34
|
-
when /x86_64/
|
35
|
+
when /x86_64/,/x64/
|
35
36
|
return :x86_64
|
36
37
|
when /powerpc/
|
37
38
|
return :ppc64le if os.eql?(OS_LINUX)
|
@@ -39,7 +40,7 @@ module Aspera
|
|
39
40
|
when /s390/
|
40
41
|
return :s390
|
41
42
|
else # other
|
42
|
-
raise "Unknown: #{RbConfig::CONFIG['host_cpu']}"
|
43
|
+
raise "Unknown CPU: #{RbConfig::CONFIG['host_cpu']}"
|
43
44
|
end
|
44
45
|
end
|
45
46
|
|
@@ -51,5 +52,15 @@ module Aspera
|
|
51
52
|
return '.exe' if os.eql?(OS_WINDOWS)
|
52
53
|
return ''
|
53
54
|
end
|
55
|
+
|
56
|
+
# on Windows, the env var %USERPROFILE% provides the path to user's home more reliably then %HOMEDRIVE%%HOMEPATH%
|
57
|
+
def self.fix_home
|
58
|
+
if os.eql?(OS_WINDOWS)
|
59
|
+
if ENV.has_key?('USERPROFILE') and Dir.exist?(ENV['USERPROFILE'])
|
60
|
+
ENV['HOME']=ENV['USERPROFILE']
|
61
|
+
Log.log.debug("Windows: set home to USERPROFILE: #{ENV['HOME']}")
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
54
65
|
end
|
55
66
|
end
|
@@ -5,6 +5,7 @@ require 'aspera/data_repository'
|
|
5
5
|
require 'xmlsimple'
|
6
6
|
require 'zlib'
|
7
7
|
require 'base64'
|
8
|
+
require 'fileutils'
|
8
9
|
|
9
10
|
module Aspera
|
10
11
|
module Fasp
|
@@ -17,14 +18,21 @@ module Aspera
|
|
17
18
|
# Installation.instance.ascp_path=""
|
18
19
|
class Installation
|
19
20
|
include Singleton
|
20
|
-
# currently used ascp executable
|
21
|
-
attr_accessor :ascp_path
|
22
|
-
# location of SDK files
|
23
|
-
attr_accessor :folder
|
24
21
|
PRODUCT_CONNECT='Aspera Connect'
|
25
22
|
PRODUCT_CLI_V1='Aspera CLI'
|
26
23
|
PRODUCT_DRIVE='Aspera Drive'
|
27
24
|
PRODUCT_ENTSRV='Enterprise Server'
|
25
|
+
# currently used ascp executable
|
26
|
+
def ascp_path=(v)
|
27
|
+
@path_to_ascp=v
|
28
|
+
end
|
29
|
+
|
30
|
+
# location of SDK files
|
31
|
+
def folder=(v)
|
32
|
+
@sdk_folder=v
|
33
|
+
folder_path
|
34
|
+
end
|
35
|
+
|
28
36
|
# find ascp in named product (use value : FIRST_FOUND='FIRST' to just use first one)
|
29
37
|
# or select one from installed_products()
|
30
38
|
def use_ascp_from_product(product_name)
|
@@ -35,8 +43,8 @@ module Aspera
|
|
35
43
|
pl=installed_products.select{|i|i[:name].eql?(product_name)}.first
|
36
44
|
raise "no such product installed: #{product_name}" if pl.nil?
|
37
45
|
end
|
38
|
-
|
39
|
-
Log.log.debug("ascp_path=#{@
|
46
|
+
self.ascp_path=pl[:ascp_path]
|
47
|
+
Log.log.debug("ascp_path=#{@path_to_ascp}")
|
40
48
|
end
|
41
49
|
|
42
50
|
# @return the list of installed products in format of product_locations
|
@@ -46,7 +54,7 @@ module Aspera
|
|
46
54
|
# add sdk as first search path
|
47
55
|
@found_products.unshift({# SDK
|
48
56
|
:expected =>'SDK',
|
49
|
-
:app_root
|
57
|
+
:app_root =>self.folder_path,
|
50
58
|
:sub_bin =>''
|
51
59
|
})
|
52
60
|
@found_products.select! do |pl|
|
@@ -76,24 +84,24 @@ module Aspera
|
|
76
84
|
def path(k)
|
77
85
|
case k
|
78
86
|
when :ascp,:ascp4
|
79
|
-
use_ascp_from_product(FIRST_FOUND) if @
|
80
|
-
file=@
|
87
|
+
use_ascp_from_product(FIRST_FOUND) if @path_to_ascp.nil?
|
88
|
+
file=@path_to_ascp
|
81
89
|
# note that there might be a .exe at the end
|
82
90
|
file=file.gsub('ascp','ascp4') if k.eql?(:ascp4)
|
83
91
|
when :ssh_bypass_key_dsa
|
84
|
-
file=File.join(
|
92
|
+
file=File.join(self.folder_path,'aspera_bypass_dsa.pem')
|
85
93
|
File.write(file,get_key('dsa',1)) unless File.exist?(file)
|
86
94
|
File.chmod(0400,file)
|
87
95
|
when :ssh_bypass_key_rsa
|
88
|
-
file=File.join(
|
96
|
+
file=File.join(self.folder_path,'aspera_bypass_rsa.pem')
|
89
97
|
File.write(file,get_key('rsa',2)) unless File.exist?(file)
|
90
98
|
File.chmod(0400,file)
|
91
99
|
when :aspera_license
|
92
|
-
file=File.join(
|
100
|
+
file=File.join(self.folder_path,'aspera-license')
|
93
101
|
File.write(file,Base64.strict_encode64("#{Zlib::Inflate.inflate(DataRepository.instance.get_bin(6))}==SIGNATURE==\n#{Base64.strict_encode64(DataRepository.instance.get_bin(7))}")) unless File.exist?(file)
|
94
102
|
File.chmod(0400,file)
|
95
103
|
when :aspera_conf
|
96
|
-
file=File.join(
|
104
|
+
file=File.join(self.folder_path,'aspera.conf')
|
97
105
|
File.write(file,%Q{<?xml version='1.0' encoding='UTF-8'?>
|
98
106
|
<CONF version="2">
|
99
107
|
<default>
|
@@ -112,8 +120,8 @@ module Aspera
|
|
112
120
|
}) unless File.exist?(file)
|
113
121
|
File.chmod(0400,file)
|
114
122
|
when :fallback_cert,:fallback_key
|
115
|
-
file_key=File.join(
|
116
|
-
file_cert=File.join(
|
123
|
+
file_key=File.join(self.folder_path,'aspera_fallback_key.pem')
|
124
|
+
file_cert=File.join(self.folder_path,'aspera_fallback_cert.pem')
|
117
125
|
if !File.exist?(file_key) or !File.exist?(file_cert)
|
118
126
|
require 'openssl'
|
119
127
|
# create new self signed certificate for http fallback
|
@@ -180,7 +188,7 @@ module Aspera
|
|
180
188
|
Zip::File.open(sdk_zip_path) do |zip_file|
|
181
189
|
zip_file.each do |entry|
|
182
190
|
if entry.name.include?(filter) and !entry.name.end_with?('/')
|
183
|
-
archive_file=File.join(
|
191
|
+
archive_file=File.join(self.folder_path,File.basename(entry.name))
|
184
192
|
File.open(archive_file, 'wb') do |output_stream|
|
185
193
|
IO.copy_stream(entry.get_input_stream, output_stream)
|
186
194
|
end
|
@@ -197,7 +205,7 @@ module Aspera
|
|
197
205
|
# get version from ascp, only after full extract, as windows requires SSL/TLS DLLs
|
198
206
|
m=%x{#{ascp_path} -A}.match(/ascp version (.*)/)
|
199
207
|
ascp_version=m[1] unless m.nil?
|
200
|
-
File.write(File.join(
|
208
|
+
File.write(File.join(self.folder_path,PRODUCT_INFO),"<product><name>IBM Aspera SDK</name><version>#{ascp_version}</version></product>")
|
201
209
|
end
|
202
210
|
return ascp_version
|
203
211
|
end
|
@@ -223,11 +231,17 @@ module Aspera
|
|
223
231
|
end
|
224
232
|
|
225
233
|
def initialize
|
226
|
-
@
|
227
|
-
@
|
234
|
+
@path_to_ascp=nil
|
235
|
+
@sdk_folder=nil
|
228
236
|
@found_products=nil
|
229
237
|
end
|
230
238
|
|
239
|
+
def folder_path
|
240
|
+
raise "undefined path to SDK" if @sdk_folder.nil?
|
241
|
+
FileUtils.mkdir_p(@sdk_folder) unless Dir.exist?(@sdk_folder)
|
242
|
+
@sdk_folder
|
243
|
+
end
|
244
|
+
|
231
245
|
# returns product folders depending on OS
|
232
246
|
# fields
|
233
247
|
# :expected M app name is taken from the manifest if present, else defaults to this value
|
data/lib/aspera/fasp/local.rb
CHANGED
@@ -17,17 +17,25 @@ require 'securerandom'
|
|
17
17
|
|
18
18
|
module Aspera
|
19
19
|
module Fasp
|
20
|
-
# default transfer username for access key based transfers
|
20
|
+
# (public) default transfer username for access key based transfers
|
21
21
|
ACCESS_KEY_TRANSFER_USER='xfer'
|
22
22
|
# executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
|
23
23
|
class Local < Manager
|
24
|
+
ASCP_SPAWN_TIMEOUT_SEC = 3
|
25
|
+
private_constant :ASCP_SPAWN_TIMEOUT_SEC
|
24
26
|
# set to false to keep ascp progress bar display (basically: removes ascp's option -q)
|
25
27
|
attr_accessor :quiet
|
26
|
-
# start
|
27
|
-
#
|
28
|
+
# start ascp transfer (non blocking), single or multi-session
|
29
|
+
# job information added to @jobs
|
30
|
+
# @param transfer_spec [Hash] aspera transfer specification
|
31
|
+
# @param options [Hash] :resumer, :regenerate_token
|
28
32
|
def start_transfer(transfer_spec,options={})
|
29
33
|
raise "option: must be hash (or nil)" unless options.is_a?(Hash)
|
30
|
-
|
34
|
+
job_options = options.clone
|
35
|
+
job_options[:resumer] ||= @resume_policy
|
36
|
+
job_options[:job_id] ||= SecureRandom.uuid
|
37
|
+
# clone transfer spec because we modify it (first level keys)
|
38
|
+
transfer_spec=transfer_spec.clone
|
31
39
|
# if there is aspera tags
|
32
40
|
if transfer_spec['tags'].is_a?(Hash) and transfer_spec['tags']['aspera'].is_a?(Hash)
|
33
41
|
# TODO: what is this for ? only on local ascp ?
|
@@ -39,6 +47,7 @@ module Aspera
|
|
39
47
|
transfer_spec['tags']['aspera']['xfer_retry']||=3600
|
40
48
|
end
|
41
49
|
Log.dump('ts',transfer_spec)
|
50
|
+
|
42
51
|
# add bypass keys when authentication is token and no auth is provided
|
43
52
|
if transfer_spec.has_key?('token') and
|
44
53
|
!transfer_spec.has_key?('remote_password') and
|
@@ -47,6 +56,21 @@ module Aspera
|
|
47
56
|
transfer_spec['EX_ssh_key_paths'] = Installation.instance.bypass_keys
|
48
57
|
end
|
49
58
|
|
59
|
+
# TODO: check if changing fasp(UDP) port is really necessary, not clear from doc
|
60
|
+
# compute this before using transfer spec, even if the var is not used in single session
|
61
|
+
multi_session_udp_port_base=33001
|
62
|
+
multi_session_number=nil
|
63
|
+
if transfer_spec.has_key?('multi_session')
|
64
|
+
multi_session_number=transfer_spec['multi_session'].to_i
|
65
|
+
raise "multi_session(#{transfer_spec['multi_session']}) shall be integer > 1" unless multi_session_number >= 1
|
66
|
+
# managed here, so delete from transfer spec
|
67
|
+
transfer_spec.delete('multi_session')
|
68
|
+
if transfer_spec.has_key?('fasp_port')
|
69
|
+
multi_session_udp_port_base=transfer_spec['fasp_port']
|
70
|
+
transfer_spec.delete('fasp_port')
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
50
74
|
# compute known args
|
51
75
|
env_args=Parameters.ts_to_env_args(transfer_spec,wss: @enable_wss)
|
52
76
|
|
@@ -60,41 +84,32 @@ module Aspera
|
|
60
84
|
|
61
85
|
# transfer job can be multi session
|
62
86
|
xfer_job={
|
63
|
-
:id => job_id,
|
64
|
-
:sessions => []
|
87
|
+
:id => job_options[:job_id],
|
88
|
+
:sessions => [] # all sessions as below
|
65
89
|
}
|
66
90
|
|
67
91
|
# generic session information
|
68
92
|
session={
|
69
|
-
:
|
70
|
-
:
|
71
|
-
:
|
72
|
-
:
|
93
|
+
:thread => nil, # Thread object monitoring management port, not nil when pushed to :sessions
|
94
|
+
:error => nil, # exception if failed
|
95
|
+
:io => nil, # management port server socket
|
96
|
+
:id => nil, # SessionId from INIT message in mgt port
|
97
|
+
:env_args => env_args, # env vars and args to ascp (from transfer spec)
|
98
|
+
:options => job_options # [Hash]
|
73
99
|
}
|
74
100
|
|
75
101
|
Log.log.debug("starting session thread(s)")
|
76
|
-
if !
|
102
|
+
if !multi_session_number
|
77
103
|
# single session for transfer : simple
|
78
104
|
session[:thread] = Thread.new(session) {|s|transfer_thread_entry(s)}
|
79
105
|
xfer_job[:sessions].push(session)
|
80
106
|
else
|
81
|
-
|
82
|
-
multi_session_udp_port_base=33001
|
83
|
-
multi_session=transfer_spec['multi_session'].to_i
|
84
|
-
raise "multi_session(#{transfer_spec['multi_session']}) shall be integer > 1" unless multi_session >= 1
|
85
|
-
# managed here, so delete from transfer spec
|
86
|
-
transfer_spec.delete('multi_session')
|
87
|
-
# TODO: check if changing fasp(UDP) port is really necessary, not clear from doc
|
88
|
-
if transfer_spec.has_key?('fasp_port')
|
89
|
-
multi_session_udp_port_base=transfer_spec['fasp_port']
|
90
|
-
transfer_spec.delete('fasp_port')
|
91
|
-
end
|
92
|
-
1.upto(multi_session) do |i|
|
107
|
+
1.upto(multi_session_number) do |i|
|
93
108
|
# do deep copy (each thread has its own copy because it is modified here below and in thread)
|
94
109
|
this_session=session.clone()
|
95
110
|
this_session[:env_args]=this_session[:env_args].clone()
|
96
111
|
this_session[:env_args][:args]=this_session[:env_args][:args].clone()
|
97
|
-
this_session[:env_args][:args].unshift("-C#{i}:#{
|
112
|
+
this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_number}")
|
98
113
|
# necessary only if server is not linux, i.e. server does not support port re-use
|
99
114
|
this_session[:env_args][:args].unshift("-O","#{multi_session_udp_port_base+i-1}")
|
100
115
|
this_session[:thread] = Thread.new(this_session) {|s|transfer_thread_entry(s)}
|
@@ -104,74 +119,59 @@ module Aspera
|
|
104
119
|
Log.log.debug("started session thread(s)")
|
105
120
|
|
106
121
|
# add job to list of jobs
|
107
|
-
@jobs[job_id]=xfer_job
|
108
|
-
|
122
|
+
@jobs[job_options[:job_id]]=xfer_job
|
109
123
|
Log.log.debug("jobs: #{@jobs.keys.count}")
|
110
|
-
|
124
|
+
|
125
|
+
return job_options[:job_id]
|
111
126
|
end # start_transfer
|
112
127
|
|
113
128
|
# wait for completion of all jobs started
|
114
129
|
# @return list of :success or error message
|
115
130
|
def wait_for_transfers_completion
|
116
|
-
Log.log.debug("
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
if running.eql?(0)
|
131
|
-
# since all are finished and we return the result, clear statuses
|
132
|
-
@jobs.clear
|
133
|
-
return result
|
134
|
-
end
|
135
|
-
Log.log.debug("wait for completed: running: #{running}")
|
136
|
-
# wait for session termination
|
137
|
-
@cond_var.wait(@mutex)
|
138
|
-
end # loop
|
139
|
-
end # mutex
|
140
|
-
# never reach here
|
141
|
-
raise "internal error"
|
131
|
+
Log.log.debug("wait_for_transfers_completion")
|
132
|
+
# set to non-nil to exit loop
|
133
|
+
result=[]
|
134
|
+
@jobs.each do |id,job|
|
135
|
+
job[:sessions].each do |session|
|
136
|
+
Log.log.debug("join #{session[:thread]}")
|
137
|
+
session[:thread].join
|
138
|
+
result.push(session[:error] ? session[:error] : :success)
|
139
|
+
end
|
140
|
+
end
|
141
|
+
Log.log.debug("all transfers joined")
|
142
|
+
# since all are finished and we return the result, clear statuses
|
143
|
+
@jobs.clear
|
144
|
+
return result
|
142
145
|
end
|
143
146
|
|
144
|
-
#
|
147
|
+
# used by asession (to be removed ?)
|
145
148
|
def shutdown
|
146
149
|
Log.log.debug("fasp local shutdown")
|
147
|
-
Log.log.debug("send signal to monitor")
|
148
|
-
# tell monitor to stop
|
149
|
-
@mutex.synchronize do
|
150
|
-
@monitor_stop=true
|
151
|
-
@cond_var.broadcast
|
152
|
-
end
|
153
|
-
# wait for thread termination
|
154
|
-
@monitor_thread.join
|
155
|
-
@monitor_thread=nil
|
156
|
-
Log.log.debug("joined monitor")
|
157
150
|
end
|
158
151
|
|
159
|
-
# This is the low level method to start
|
152
|
+
# This is the low level method to start the "ascp" process
|
160
153
|
# currently, relies on command line arguments
|
161
154
|
# start ascp with management port.
|
162
155
|
# raises FaspError on error
|
163
156
|
# if there is a thread info: set and broadcast session id
|
164
157
|
# @param env_args a hash containing :args :env :ascp_version
|
165
|
-
#
|
166
|
-
|
158
|
+
# @param session this session information
|
159
|
+
# could be private method
|
160
|
+
def start_transfer_with_args_env(env_args,session)
|
161
|
+
raise "env_args must be Hash" unless env_args.is_a?(Hash)
|
162
|
+
raise "session must be Hash" unless session.is_a?(Hash)
|
167
163
|
begin
|
168
164
|
Log.log.debug("env_args=#{env_args.inspect}")
|
169
|
-
|
165
|
+
# get location of ascp executable
|
166
|
+
ascp_path=@mutex.synchronize do
|
167
|
+
Fasp::Installation.instance.path(env_args[:ascp_version])
|
168
|
+
end
|
169
|
+
# (optional) check it exists
|
170
170
|
raise Fasp::Error.new("no such file: #{ascp_path}") unless File.exist?(ascp_path)
|
171
|
-
|
171
|
+
# open random local TCP port for listening for ascp management
|
172
|
+
mgt_sock = TCPServer.new('127.0.0.1',0)
|
173
|
+
# clone arguments as we eed to modify with mgt port
|
172
174
|
ascp_arguments=env_args[:args].clone
|
173
|
-
# open random local TCP port listening
|
174
|
-
mgt_sock = TCPServer.new('127.0.0.1',0 )
|
175
175
|
# add management port
|
176
176
|
ascp_arguments.unshift('-M', mgt_sock.addr[1].to_s)
|
177
177
|
# start ascp in sub process
|
@@ -180,8 +180,9 @@ module Aspera
|
|
180
180
|
ascp_pid = Process.spawn(env_args[:env],[ascp_path,ascp_path],*ascp_arguments)
|
181
181
|
# in parent, wait for connection to socket max 3 seconds
|
182
182
|
Log.log.debug("before accept for pid (#{ascp_pid})")
|
183
|
+
# init management socket
|
183
184
|
ascp_mgt_io=nil
|
184
|
-
Timeout.timeout(
|
185
|
+
Timeout.timeout(ASCP_SPAWN_TIMEOUT_SEC) do
|
185
186
|
ascp_mgt_io = mgt_sock.accept
|
186
187
|
# management messages include file names which may be utf8
|
187
188
|
# by default socket is US-ASCII
|
@@ -189,21 +190,13 @@ module Aspera
|
|
189
190
|
ascp_mgt_io.set_encoding(Encoding::UTF_8)
|
190
191
|
end
|
191
192
|
Log.log.debug("after accept (#{ascp_mgt_io})")
|
192
|
-
|
193
|
-
unless session.nil?
|
194
|
-
@mutex.synchronize do
|
195
|
-
session[:io]=ascp_mgt_io
|
196
|
-
@cond_var.broadcast
|
197
|
-
end
|
198
|
-
end
|
193
|
+
session[:io]=ascp_mgt_io
|
199
194
|
# exact text for event, with \n
|
200
195
|
current_event_text=''
|
201
196
|
# parsed event (hash)
|
202
197
|
current_event_data=nil
|
203
|
-
|
204
198
|
# this is the last full status
|
205
199
|
last_status_event=nil
|
206
|
-
|
207
200
|
# read management port
|
208
201
|
loop do
|
209
202
|
# TODO: timeout here ?
|
@@ -222,45 +215,40 @@ module Aspera
|
|
222
215
|
# event field
|
223
216
|
current_event_data[$1] = $2
|
224
217
|
when ''
|
225
|
-
# end event
|
218
|
+
# empty line is separator to end event information
|
226
219
|
raise "unexpected empty line" if current_event_data.nil?
|
227
220
|
current_event_data[Manager::LISTENER_SESSION_ID_B]=ascp_pid
|
228
221
|
notify_listeners(current_event_text,current_event_data)
|
229
|
-
# TODO: check if this is always the last event
|
230
222
|
case current_event_data['Type']
|
223
|
+
when 'INIT'
|
224
|
+
session[:id]=current_event_data['SessionId']
|
225
|
+
Log.log.debug("session id: #{session[:id]}")
|
231
226
|
when 'DONE','ERROR'
|
227
|
+
# TODO: check if this is always the last event
|
232
228
|
last_status_event = current_event_data
|
233
|
-
when 'INIT'
|
234
|
-
unless session.nil?
|
235
|
-
@mutex.synchronize do
|
236
|
-
session[:state]=:started
|
237
|
-
session[:id]=current_event_data['SessionId']
|
238
|
-
Log.log.debug("session id: #{session[:id]}")
|
239
|
-
@cond_var.broadcast
|
240
|
-
end
|
241
|
-
end
|
242
229
|
end # event type
|
243
230
|
else
|
244
231
|
raise "unexpected line:[#{line}]"
|
245
232
|
end # case
|
246
|
-
end # loop
|
233
|
+
end # loop (process mgt port lines)
|
247
234
|
# check that last status was received before process exit
|
248
235
|
raise "INTERNAL: nil last status" if last_status_event.nil?
|
249
236
|
case last_status_event['Type']
|
250
237
|
when 'DONE'
|
238
|
+
# return method (or just don't do anything)
|
251
239
|
return
|
252
240
|
when 'ERROR'
|
253
241
|
Log.log.error("code: #{last_status_event['Code']}")
|
254
242
|
if last_status_event['Description'] =~ /bearer token/i
|
255
243
|
Log.log.error("need to regenerate token".red)
|
256
|
-
if
|
244
|
+
if session[:options].is_a?(Hash) and session[:options].has_key?(:regenerate_token)
|
257
245
|
# regenerate token here, expired, or error on it
|
258
246
|
env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
|
259
247
|
end
|
260
248
|
end
|
261
249
|
raise Fasp::Error.new(last_status_event['Description'],last_status_event['Code'].to_i)
|
262
250
|
else
|
263
|
-
raise "
|
251
|
+
raise "unexpected last event type: #{last_status_event['Type']}"
|
264
252
|
end
|
265
253
|
rescue SystemCallError => e
|
266
254
|
# Process.spawn
|
@@ -270,13 +258,9 @@ module Aspera
|
|
270
258
|
rescue Interrupt => e
|
271
259
|
raise Fasp::Error.new('transfer interrupted by user')
|
272
260
|
ensure
|
273
|
-
#
|
261
|
+
# if ascp was successfully started
|
274
262
|
unless ascp_pid.nil?
|
275
|
-
|
276
|
-
Process.kill('INT',ascp_pid)
|
277
|
-
rescue
|
278
|
-
end
|
279
|
-
# avoid zombie
|
263
|
+
# "wait" for process to avoid zombie
|
280
264
|
Process.wait(ascp_pid)
|
281
265
|
ascp_pid=nil
|
282
266
|
session.delete(:io)
|
@@ -284,24 +268,26 @@ module Aspera
|
|
284
268
|
end # begin-ensure
|
285
269
|
end # start_transfer_with_args_env
|
286
270
|
|
287
|
-
# send command
|
271
|
+
# send command of management port to ascp session
|
272
|
+
# @param job_id identified transfer process
|
273
|
+
# @param session_index index of session (for multi session)
|
274
|
+
# @param data command on mgt port, examples:
|
288
275
|
# {'type'=>'START','source'=>_path_,'destination'=>_path_}
|
289
276
|
# {'type'=>'DONE'}
|
290
277
|
def send_command(job_id,session_index,data)
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
|
304
|
-
end
|
278
|
+
job=@jobs[job_id]
|
279
|
+
raise "no such job" if job.nil?
|
280
|
+
session=job[:sessions][session_index]
|
281
|
+
raise "no such session" if session.nil?
|
282
|
+
Log.log.debug("command: #{data}")
|
283
|
+
# build command
|
284
|
+
command=data.
|
285
|
+
keys.
|
286
|
+
map{|k|"#{k.capitalize}: #{data[k]}"}.
|
287
|
+
unshift('FASPMGR 2').
|
288
|
+
push('','').
|
289
|
+
join("\n")
|
290
|
+
session[:io].puts(command)
|
305
291
|
end
|
306
292
|
|
307
293
|
private
|
@@ -311,66 +297,33 @@ module Aspera
|
|
311
297
|
super()
|
312
298
|
# by default no interactive progress bar
|
313
299
|
@quiet=true
|
314
|
-
#
|
300
|
+
# all transfer jobs, key = SecureRandom.uuid, protected by mutex, condvar on change
|
315
301
|
@jobs={}
|
316
|
-
# mutex protects
|
302
|
+
# mutex protects global data accessed by threads
|
317
303
|
@mutex=Mutex.new
|
318
|
-
# cond var is waited or broadcast on jobs data change
|
319
|
-
@cond_var=ConditionVariable.new
|
320
|
-
# must be set before starting monitor, set to false to stop thread. also shared and protected by mutex
|
321
|
-
@monitor_stop=false
|
322
|
-
@monitor_thread=Thread.new{monitor_thread_entry}
|
323
304
|
@resume_policy=ResumePolicy.new(agent_options)
|
324
305
|
@enable_wss = agent_options[:wss] || false
|
325
306
|
end
|
326
307
|
|
327
308
|
# transfer thread entry
|
328
|
-
#
|
329
|
-
# TODO: extract resume algorithm in a specific object
|
309
|
+
# @param session information
|
330
310
|
def transfer_thread_entry(session)
|
331
311
|
begin
|
332
312
|
# set name for logging
|
333
313
|
Thread.current[:name]="transfer"
|
334
|
-
# update state once in thread
|
335
|
-
session[:state]=:started
|
336
314
|
Log.log.debug("ENTER (#{Thread.current[:name]})")
|
337
315
|
# start transfer with selected resumer policy
|
338
|
-
session[:resumer].process do
|
316
|
+
session[:options][:resumer].process do
|
339
317
|
start_transfer_with_args_env(session[:env_args],session)
|
340
318
|
end
|
341
319
|
Log.log.debug('transfer ok'.bg_green)
|
342
|
-
session[:state]=:success
|
343
320
|
rescue => e
|
344
|
-
session[:state]=:failed
|
345
321
|
session[:error]=e
|
346
|
-
Log.log.error("#{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red) if Log.instance.level.eql?(:debug)
|
347
|
-
ensure
|
348
|
-
@mutex.synchronize do
|
349
|
-
# ensure id is set to unblock start procedure
|
350
|
-
session[:id]||=nil
|
351
|
-
@cond_var.broadcast
|
352
|
-
end
|
322
|
+
Log.log.error("Transfer thread error: #{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red) if Log.instance.level.eql?(:debug)
|
353
323
|
end
|
354
324
|
Log.log.debug("EXIT (#{Thread.current[:name]})")
|
355
325
|
end
|
356
326
|
|
357
|
-
# main thread method for monitor
|
358
|
-
# currently: just joins started threads
|
359
|
-
def monitor_thread_entry
|
360
|
-
Thread.current[:name]="monitor"
|
361
|
-
@mutex.synchronize do
|
362
|
-
until @monitor_stop do
|
363
|
-
# wait for session termination
|
364
|
-
@cond_var.wait(@mutex)
|
365
|
-
@jobs.values do |job|
|
366
|
-
job[:sessions].each do |session|
|
367
|
-
session[:thread].join if [:success,:failed].include?(session[:state])
|
368
|
-
end # sessions
|
369
|
-
end # jobs
|
370
|
-
end # monitor run
|
371
|
-
end # sync
|
372
|
-
Log.log.debug("EXIT (#{Thread.current[:name]})")
|
373
|
-
end # monitor_thread_entry
|
374
327
|
end # Local
|
375
328
|
end
|
376
329
|
end
|