aspera-cli 4.4.0 → 4.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +2095 -1503
- data/bin/ascli +2 -1
- data/bin/asession +4 -5
- data/docs/test_env.conf +3 -0
- data/examples/aoc.rb +4 -3
- data/examples/faspex4.rb +25 -25
- data/examples/proxy.pac +1 -1
- data/examples/transfer.rb +17 -17
- data/lib/aspera/aoc.rb +238 -185
- data/lib/aspera/ascmd.rb +93 -83
- data/lib/aspera/ats_api.rb +11 -10
- data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
- data/lib/aspera/cli/extended_value.rb +42 -33
- data/lib/aspera/cli/formater.rb +142 -108
- data/lib/aspera/cli/info.rb +17 -0
- data/lib/aspera/cli/listener/line_dump.rb +3 -2
- data/lib/aspera/cli/listener/logger.rb +2 -1
- data/lib/aspera/cli/listener/progress.rb +16 -18
- data/lib/aspera/cli/listener/progress_multi.rb +18 -21
- data/lib/aspera/cli/main.rb +173 -149
- data/lib/aspera/cli/manager.rb +163 -168
- data/lib/aspera/cli/plugin.rb +43 -31
- data/lib/aspera/cli/plugins/alee.rb +6 -6
- data/lib/aspera/cli/plugins/aoc.rb +405 -370
- data/lib/aspera/cli/plugins/ats.rb +86 -79
- data/lib/aspera/cli/plugins/bss.rb +14 -16
- data/lib/aspera/cli/plugins/config.rb +580 -362
- data/lib/aspera/cli/plugins/console.rb +23 -19
- data/lib/aspera/cli/plugins/cos.rb +18 -18
- data/lib/aspera/cli/plugins/faspex.rb +201 -158
- data/lib/aspera/cli/plugins/faspex5.rb +80 -57
- data/lib/aspera/cli/plugins/node.rb +183 -166
- data/lib/aspera/cli/plugins/orchestrator.rb +71 -67
- data/lib/aspera/cli/plugins/preview.rb +92 -96
- data/lib/aspera/cli/plugins/server.rb +79 -75
- data/lib/aspera/cli/plugins/shares.rb +35 -19
- data/lib/aspera/cli/plugins/sync.rb +20 -22
- data/lib/aspera/cli/transfer_agent.rb +76 -113
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +35 -27
- data/lib/aspera/command_line_builder.rb +48 -34
- data/lib/aspera/cos_node.rb +29 -21
- data/lib/aspera/data_repository.rb +3 -2
- data/lib/aspera/environment.rb +50 -45
- data/lib/aspera/fasp/{manager.rb → agent_base.rb} +28 -25
- data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +52 -43
- data/lib/aspera/fasp/{local.rb → agent_direct.rb} +58 -72
- data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +37 -43
- data/lib/aspera/fasp/{node.rb → agent_node.rb} +35 -16
- data/lib/aspera/fasp/agent_trsdk.rb +104 -0
- data/lib/aspera/fasp/error.rb +2 -1
- data/lib/aspera/fasp/error_info.rb +68 -52
- data/lib/aspera/fasp/installation.rb +152 -124
- data/lib/aspera/fasp/listener.rb +1 -0
- data/lib/aspera/fasp/parameters.rb +87 -92
- data/lib/aspera/fasp/parameters.yaml +305 -249
- data/lib/aspera/fasp/resume_policy.rb +11 -14
- data/lib/aspera/fasp/transfer_spec.rb +26 -0
- data/lib/aspera/fasp/uri.rb +22 -21
- data/lib/aspera/faspex_gw.rb +55 -89
- data/lib/aspera/hash_ext.rb +4 -3
- data/lib/aspera/id_generator.rb +8 -7
- data/lib/aspera/keychain/encrypted_hash.rb +121 -0
- data/lib/aspera/keychain/macos_security.rb +90 -0
- data/lib/aspera/log.rb +55 -37
- data/lib/aspera/nagios.rb +13 -12
- data/lib/aspera/node.rb +30 -25
- data/lib/aspera/oauth.rb +175 -226
- data/lib/aspera/open_application.rb +4 -3
- data/lib/aspera/persistency_action_once.rb +6 -6
- data/lib/aspera/persistency_folder.rb +5 -9
- data/lib/aspera/preview/file_types.rb +6 -5
- data/lib/aspera/preview/generator.rb +25 -24
- data/lib/aspera/preview/options.rb +16 -14
- data/lib/aspera/preview/utils.rb +98 -98
- data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
- data/lib/aspera/proxy_auto_config.rb +111 -20
- data/lib/aspera/rest.rb +154 -135
- data/lib/aspera/rest_call_error.rb +2 -2
- data/lib/aspera/rest_error_analyzer.rb +23 -25
- data/lib/aspera/rest_errors_aspera.rb +15 -14
- data/lib/aspera/ssh.rb +12 -10
- data/lib/aspera/sync.rb +42 -41
- data/lib/aspera/temp_file_manager.rb +18 -14
- data/lib/aspera/timer_limiter.rb +2 -1
- data/lib/aspera/uri_reader.rb +7 -5
- data/lib/aspera/web_auth.rb +79 -76
- metadata +116 -29
- data/docs/Makefile +0 -66
- data/docs/README.erb.md +0 -3973
- data/docs/README.md +0 -13
- data/docs/diagrams.txt +0 -49
- data/docs/doc_tools.rb +0 -58
- data/lib/aspera/api_detector.rb +0 -60
- data/lib/aspera/cli/plugins/shares2.rb +0 -114
- data/lib/aspera/secrets.rb +0 -20
data/bin/ascli
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
2
3
|
require 'rubygems'
|
3
|
-
$LOAD_PATH.unshift(File.dirname(__FILE__)+
|
4
|
+
$LOAD_PATH.unshift(File.realpath(File.dirname(File.realpath(__FILE__))+'/../lib'))
|
4
5
|
require 'aspera/cli/main'
|
5
6
|
require 'aspera/environment'
|
6
7
|
Encoding.default_internal = Encoding::UTF_8
|
data/bin/asession
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
2
3
|
# Laurent Martin/2017
|
3
4
|
$LOAD_PATH.unshift(File.dirname(__FILE__)+"/../lib")
|
4
|
-
require 'aspera/fasp/
|
5
|
+
require 'aspera/fasp/agent_direct'
|
5
6
|
require 'aspera/cli/listener/line_dump'
|
6
7
|
require 'aspera/cli/extended_value'
|
7
8
|
require 'aspera/log'
|
@@ -65,10 +66,8 @@ if transfer_spec.has_key?(TS_TMP_FILELIST_FOLDER)
|
|
65
66
|
Aspera::Fasp::Parameters.file_list_folder=transfer_spec[TS_TMP_FILELIST_FOLDER]
|
66
67
|
transfer_spec.delete(TS_TMP_FILELIST_FOLDER)
|
67
68
|
end
|
68
|
-
# get local agent (ascp)
|
69
|
-
client=Aspera::Fasp::
|
70
|
-
# disable ascp output on stdout to not mix with JSON events
|
71
|
-
client.quiet=true
|
69
|
+
# get local agent (ascp), disable ascp output on stdout to not mix with JSON events
|
70
|
+
client=Aspera::Fasp::AgentDirect.new({quiet: true})
|
72
71
|
# display JSON instead of legacy Lines
|
73
72
|
client.add_listener(Aspera::Cli::Listener::LineDump.new)
|
74
73
|
# start transfer (asynchronous)
|
data/docs/test_env.conf
CHANGED
@@ -124,6 +124,8 @@ misc:
|
|
124
124
|
faspex_publink_recv_from_fxuser: your value here
|
125
125
|
faspex_publink_send_to_fxuser: your value here
|
126
126
|
faspex_publink_send_to_dropbox: your value here
|
127
|
+
faspex_dbx: your value here
|
128
|
+
faspex_wkg: your value here
|
127
129
|
shares_upload: your value here
|
128
130
|
console_smart_id: your value here
|
129
131
|
console_smart_file: your value here
|
@@ -143,6 +145,7 @@ misc:
|
|
143
145
|
aoc_publink_folder: your value here
|
144
146
|
aoc_shbx_ws: your value here
|
145
147
|
aoc_shbx_name: your value here
|
148
|
+
aoc_shbx_meta: your value here
|
146
149
|
aoc_node1_name: your value here
|
147
150
|
aoc_node1_secret: your value here
|
148
151
|
icos_bucket_key: your value here
|
data/examples/aoc.rb
CHANGED
@@ -1,13 +1,14 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
2
3
|
require 'aspera/aoc'
|
3
4
|
require 'aspera/log'
|
4
5
|
|
5
6
|
Aspera::Log.instance.level=:debug
|
6
7
|
|
7
|
-
if !
|
8
|
+
if !ARGV.length.eql?(3)
|
8
9
|
Aspera::Log.log.error("wrong number of args: #{ARGV.length}")
|
9
|
-
Aspera::Log.log.error("Usage: #{$
|
10
|
-
Aspera::Log.log.error("Example: #{$
|
10
|
+
Aspera::Log.log.error("Usage: #{$PROGRAM_NAME} <aoc URL> <aoc username> <aoc private key content>")
|
11
|
+
Aspera::Log.log.error("Example: #{$PROGRAM_NAME} https://myorg.ibmaspera.com john@example.com $(cat /home/john/my_key.pem)")
|
11
12
|
Process.exit(1)
|
12
13
|
end
|
13
14
|
|
data/examples/faspex4.rb
CHANGED
@@ -1,11 +1,12 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
2
3
|
# find Faspex API here: https://developer.ibm.com/apis/catalog/?search=faspex
|
3
4
|
# this example makes use of class Aspera::Rest for REST calls, alternatively class RestClient of gem rest-client could be used
|
4
|
-
# this example makes use of class Aspera::Fasp::
|
5
|
+
# this example makes use of class Aspera::Fasp::AgentDirect for transfers, alternatively the official "Transfer SDK" could be used
|
5
6
|
# Aspera SDK can be downloaded with: `ascli conf ascp install` , it installs in $HOME/.aspera/ascli/sdk
|
6
7
|
require 'aspera/rest'
|
7
8
|
require 'aspera/log'
|
8
|
-
require 'aspera/fasp/
|
9
|
+
require 'aspera/fasp/agent_direct'
|
9
10
|
|
10
11
|
tmpdir=ENV['tmp']||Dir.tmpdir || '.'
|
11
12
|
|
@@ -17,10 +18,10 @@ Aspera::Log.instance.level=:debug
|
|
17
18
|
# (if data files are not there, they will be created)
|
18
19
|
Aspera::Fasp::Installation.instance.folder = tmpdir
|
19
20
|
|
20
|
-
if !
|
21
|
+
if !ARGV.length.eql?(3)
|
21
22
|
Aspera::Log.log.error("Wrong number of args: #{ARGV.length}")
|
22
|
-
Aspera::Log.log.error("Usage: #{$
|
23
|
-
Aspera::Log.log.error("Example: #{$
|
23
|
+
Aspera::Log.log.error("Usage: #{$PROGRAM_NAME} <faspex URL> <faspex username> <faspex password>")
|
24
|
+
Aspera::Log.log.error("Example: #{$PROGRAM_NAME} https://faspex.com/aspera/faspex john p@sSw0rd")
|
24
25
|
Process.exit(1)
|
25
26
|
end
|
26
27
|
|
@@ -36,11 +37,11 @@ Aspera::Rest.insecure=true
|
|
36
37
|
|
37
38
|
# create REST API object
|
38
39
|
api_v3=Aspera::Rest.new({
|
39
|
-
:
|
40
|
-
:
|
41
|
-
:
|
42
|
-
:
|
43
|
-
:
|
40
|
+
base_url: faspex_url,
|
41
|
+
auth: {
|
42
|
+
type: :basic,
|
43
|
+
username: faspex_user,
|
44
|
+
password: faspex_pass
|
44
45
|
}})
|
45
46
|
|
46
47
|
# very simple api call
|
@@ -51,7 +52,7 @@ api_v3.read('me')
|
|
51
52
|
|
52
53
|
# create a sample file to send
|
53
54
|
file_to_send=File.join(tmpdir,'myfile.bin')
|
54
|
-
File.
|
55
|
+
File.write(file_to_send, 'sample data')
|
55
56
|
# package creation parameters
|
56
57
|
package_create_params={'delivery'=>{'title'=>'test package','recipients'=>['aspera.user1@gmail.com'],'sources'=>[{'paths'=>[file_to_send]}]}}
|
57
58
|
pkg_created=api_v3.create('send',package_create_params)[:data]
|
@@ -59,31 +60,30 @@ pkg_created=api_v3.create('send',package_create_params)[:data]
|
|
59
60
|
transfer_spec=pkg_created['xfer_sessions'].first
|
60
61
|
# set paths of files to send
|
61
62
|
transfer_spec['paths']=[{'source'=>file_to_send}]
|
62
|
-
# get
|
63
|
-
transfer_client=Aspera::Fasp::
|
64
|
-
# disable ascp output on stdout (optional)
|
65
|
-
transfer_client.quiet=true
|
63
|
+
# get local agent (ascp), disable ascp output on stdout to not mix with JSON events
|
64
|
+
transfer_client=Aspera::Fasp::AgentDirect.new({quiet: true})
|
66
65
|
# start transfer (asynchronous)
|
67
66
|
job_id=transfer_client.start_transfer(transfer_spec)
|
67
|
+
Aspera::Log.dump('job_id',job_id)
|
68
68
|
# wait for all transfer completion (for the example)
|
69
69
|
result=transfer_client.wait_for_transfers_completion
|
70
70
|
# notify of any transfer error
|
71
|
-
result.
|
71
|
+
result.reject{|i|i.eql?(:success)}.each do |e|
|
72
72
|
Aspera::Log.log.error("A transfer error occured: #{e.message}")
|
73
73
|
end
|
74
74
|
|
75
75
|
# 3: Faspex 4 API v4
|
76
76
|
#---------------
|
77
77
|
api_v4=Aspera::Rest.new({
|
78
|
-
:
|
79
|
-
:
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
78
|
+
base_url: faspex_url+'/api',
|
79
|
+
auth: {
|
80
|
+
type: :oauth2,
|
81
|
+
base_url: faspex_url+'/auth/oauth2',
|
82
|
+
auth: {type: :basic, username: faspex_user, password: faspex_pass},
|
83
|
+
crtype: :generic,
|
84
|
+
generic: {grant_type: 'password'},
|
85
|
+
scope: 'admin'
|
86
|
+
}})
|
87
87
|
|
88
88
|
# Use it. Note that Faspex 4 API v4 is totally different from Faspex 4 v3 APIs, see ref on line 2
|
89
89
|
Aspera::Log.dump('users',api_v4.read('users')[:data])
|
data/examples/proxy.pac
CHANGED
data/examples/transfer.rb
CHANGED
@@ -1,8 +1,9 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
2
3
|
# Example: transfer a file using one of the provided transfer agents
|
3
4
|
# location of ascp can be specified with env var "ascp"
|
4
5
|
# temp folder can be specified with env var "tmp"
|
5
|
-
require 'aspera/fasp/
|
6
|
+
require 'aspera/fasp/agent_direct'
|
6
7
|
require 'aspera/fasp/listener'
|
7
8
|
require 'aspera/fasp/installation'
|
8
9
|
require 'aspera/log'
|
@@ -16,7 +17,7 @@ tmpdir=ENV['tmp']||Dir.tmpdir || '.'
|
|
16
17
|
DEMO_CONFIG=[
|
17
18
|
'ssh://asperaweb@eudemo.asperademo.com:33001',
|
18
19
|
'https://node_asperaweb@eudemo.asperademo.com:9092',
|
19
|
-
'demoaspera'
|
20
|
+
'demoaspera'
|
20
21
|
]
|
21
22
|
|
22
23
|
##############################################################
|
@@ -26,7 +27,7 @@ DEMO_CONFIG=[
|
|
26
27
|
Aspera::Log.instance.level=:debug
|
27
28
|
|
28
29
|
# register aspera REST call error handlers
|
29
|
-
Aspera::RestErrorsAspera.
|
30
|
+
Aspera::RestErrorsAspera.register_handlers
|
30
31
|
|
31
32
|
# some required files are generated here (keys, certs)
|
32
33
|
Aspera::Fasp::Installation.instance.folder = tmpdir
|
@@ -36,10 +37,10 @@ Aspera::Fasp::Installation.instance.ascp_path=ENV['ascp'] if ENV.has_key?('ascp'
|
|
36
37
|
#Aspera::Fasp::Installation.instance.installed_products.each{|p|puts("found: #{p[:name]}")}
|
37
38
|
#Aspera::Fasp::Installation.instance.use_ascp_from_product('Aspera Connect')
|
38
39
|
# or install:
|
39
|
-
#
|
40
|
+
#
|
40
41
|
|
41
42
|
# get FASP Manager singleton based on above ascp location
|
42
|
-
fasp_manager=Aspera::Fasp::
|
43
|
+
fasp_manager=Aspera::Fasp::AgentDirect.new
|
43
44
|
|
44
45
|
# Note that it would also be possible to start transfers using other agents
|
45
46
|
#require 'aspera/fasp/connect'
|
@@ -54,7 +55,7 @@ fasp_manager=Aspera::Fasp::Local.new
|
|
54
55
|
class MyListener < Aspera::Fasp::Listener
|
55
56
|
# this is the callback called during transfers, here we only display the received information
|
56
57
|
# but it could be used to get detailed error information, check "type" field is "ERROR"
|
57
|
-
def event_enhanced(data)
|
58
|
+
def event_enhanced(data);$stdout.puts(JSON.generate(data));$stdout.flush;end
|
58
59
|
end
|
59
60
|
|
60
61
|
# register the sample listener to display events
|
@@ -84,9 +85,9 @@ fasp_manager.start_transfer(transfer_spec)
|
|
84
85
|
# get array of status, one for each session (so, a single value array)
|
85
86
|
# each status is either :success or "error message"
|
86
87
|
transfer_result=fasp_manager.wait_for_transfers_completion
|
87
|
-
|
88
|
+
$stdout.puts(JSON.generate(transfer_result))
|
88
89
|
# get list of errors only
|
89
|
-
errors=transfer_result.
|
90
|
+
errors=transfer_result.reject{|i|i.eql?(:success)}
|
90
91
|
# the transfer was not success, as there is at least one error
|
91
92
|
raise "Error(s) occured: #{errors.join(',')}" if !errors.empty?
|
92
93
|
|
@@ -95,19 +96,19 @@ raise "Error(s) occured: #{errors.join(',')}" if !errors.empty?
|
|
95
96
|
|
96
97
|
# create rest client for Node API on a public demo system, using public demo credentials
|
97
98
|
node_api=Aspera::Rest.new({
|
98
|
-
:
|
99
|
-
:
|
100
|
-
:
|
101
|
-
:
|
102
|
-
:
|
99
|
+
base_url: DEMO_CONFIG[1],
|
100
|
+
auth: {
|
101
|
+
type: :basic,
|
102
|
+
username: URI.parse(DEMO_CONFIG[1]).user,
|
103
|
+
password: DEMO_CONFIG[2]
|
103
104
|
}})
|
104
105
|
# define sample file(s) and destination folder
|
105
106
|
sources=["#{tmpdir}/sample_file.txt"]
|
106
107
|
destination='/Upload'
|
107
108
|
# create sample file(s)
|
108
|
-
sources.each{|p|File.write(p,
|
109
|
+
sources.each{|p|File.write(p,'Hello World!')}
|
109
110
|
# request transfer authorization to node for a single transfer (This is a node api v3 call)
|
110
|
-
send_result=node_api.create('files/upload_setup',{ :
|
111
|
+
send_result=node_api.create('files/upload_setup',{ transfer_requests: [{ transfer_request: { paths: [{ destination: destination }] } }] })[:data]
|
111
112
|
# we normally have only one transfer spec in list, so just get the first transfer_spec
|
112
113
|
transfer_spec=send_result['transfer_specs'].first['transfer_spec']
|
113
114
|
# add list of files to upload
|
@@ -118,7 +119,6 @@ transfer_spec['authentication']='token'
|
|
118
119
|
fasp_manager.start_transfer(transfer_spec)
|
119
120
|
# optional: wait for transfer completion helper function to get events
|
120
121
|
transfer_result=fasp_manager.wait_for_transfers_completion
|
121
|
-
errors=transfer_result.
|
122
|
+
errors=transfer_result.reject{|i|i.eql?(:success)}
|
122
123
|
# the transfer was not success, as there is at least one error
|
123
124
|
raise "Error(s) occured: #{errors.join(',')}" if !errors.empty?
|
124
|
-
|