aspera-cli 4.4.0 → 4.7.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2095 -1503
  3. data/bin/ascli +2 -1
  4. data/bin/asession +4 -5
  5. data/docs/test_env.conf +3 -0
  6. data/examples/aoc.rb +4 -3
  7. data/examples/faspex4.rb +25 -25
  8. data/examples/proxy.pac +1 -1
  9. data/examples/transfer.rb +17 -17
  10. data/lib/aspera/aoc.rb +238 -185
  11. data/lib/aspera/ascmd.rb +93 -83
  12. data/lib/aspera/ats_api.rb +11 -10
  13. data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
  14. data/lib/aspera/cli/extended_value.rb +42 -33
  15. data/lib/aspera/cli/formater.rb +142 -108
  16. data/lib/aspera/cli/info.rb +17 -0
  17. data/lib/aspera/cli/listener/line_dump.rb +3 -2
  18. data/lib/aspera/cli/listener/logger.rb +2 -1
  19. data/lib/aspera/cli/listener/progress.rb +16 -18
  20. data/lib/aspera/cli/listener/progress_multi.rb +18 -21
  21. data/lib/aspera/cli/main.rb +173 -149
  22. data/lib/aspera/cli/manager.rb +163 -168
  23. data/lib/aspera/cli/plugin.rb +43 -31
  24. data/lib/aspera/cli/plugins/alee.rb +6 -6
  25. data/lib/aspera/cli/plugins/aoc.rb +405 -370
  26. data/lib/aspera/cli/plugins/ats.rb +86 -79
  27. data/lib/aspera/cli/plugins/bss.rb +14 -16
  28. data/lib/aspera/cli/plugins/config.rb +580 -362
  29. data/lib/aspera/cli/plugins/console.rb +23 -19
  30. data/lib/aspera/cli/plugins/cos.rb +18 -18
  31. data/lib/aspera/cli/plugins/faspex.rb +201 -158
  32. data/lib/aspera/cli/plugins/faspex5.rb +80 -57
  33. data/lib/aspera/cli/plugins/node.rb +183 -166
  34. data/lib/aspera/cli/plugins/orchestrator.rb +71 -67
  35. data/lib/aspera/cli/plugins/preview.rb +92 -96
  36. data/lib/aspera/cli/plugins/server.rb +79 -75
  37. data/lib/aspera/cli/plugins/shares.rb +35 -19
  38. data/lib/aspera/cli/plugins/sync.rb +20 -22
  39. data/lib/aspera/cli/transfer_agent.rb +76 -113
  40. data/lib/aspera/cli/version.rb +2 -1
  41. data/lib/aspera/colors.rb +35 -27
  42. data/lib/aspera/command_line_builder.rb +48 -34
  43. data/lib/aspera/cos_node.rb +29 -21
  44. data/lib/aspera/data_repository.rb +3 -2
  45. data/lib/aspera/environment.rb +50 -45
  46. data/lib/aspera/fasp/{manager.rb → agent_base.rb} +28 -25
  47. data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +52 -43
  48. data/lib/aspera/fasp/{local.rb → agent_direct.rb} +58 -72
  49. data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +37 -43
  50. data/lib/aspera/fasp/{node.rb → agent_node.rb} +35 -16
  51. data/lib/aspera/fasp/agent_trsdk.rb +104 -0
  52. data/lib/aspera/fasp/error.rb +2 -1
  53. data/lib/aspera/fasp/error_info.rb +68 -52
  54. data/lib/aspera/fasp/installation.rb +152 -124
  55. data/lib/aspera/fasp/listener.rb +1 -0
  56. data/lib/aspera/fasp/parameters.rb +87 -92
  57. data/lib/aspera/fasp/parameters.yaml +305 -249
  58. data/lib/aspera/fasp/resume_policy.rb +11 -14
  59. data/lib/aspera/fasp/transfer_spec.rb +26 -0
  60. data/lib/aspera/fasp/uri.rb +22 -21
  61. data/lib/aspera/faspex_gw.rb +55 -89
  62. data/lib/aspera/hash_ext.rb +4 -3
  63. data/lib/aspera/id_generator.rb +8 -7
  64. data/lib/aspera/keychain/encrypted_hash.rb +121 -0
  65. data/lib/aspera/keychain/macos_security.rb +90 -0
  66. data/lib/aspera/log.rb +55 -37
  67. data/lib/aspera/nagios.rb +13 -12
  68. data/lib/aspera/node.rb +30 -25
  69. data/lib/aspera/oauth.rb +175 -226
  70. data/lib/aspera/open_application.rb +4 -3
  71. data/lib/aspera/persistency_action_once.rb +6 -6
  72. data/lib/aspera/persistency_folder.rb +5 -9
  73. data/lib/aspera/preview/file_types.rb +6 -5
  74. data/lib/aspera/preview/generator.rb +25 -24
  75. data/lib/aspera/preview/options.rb +16 -14
  76. data/lib/aspera/preview/utils.rb +98 -98
  77. data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
  78. data/lib/aspera/proxy_auto_config.rb +111 -20
  79. data/lib/aspera/rest.rb +154 -135
  80. data/lib/aspera/rest_call_error.rb +2 -2
  81. data/lib/aspera/rest_error_analyzer.rb +23 -25
  82. data/lib/aspera/rest_errors_aspera.rb +15 -14
  83. data/lib/aspera/ssh.rb +12 -10
  84. data/lib/aspera/sync.rb +42 -41
  85. data/lib/aspera/temp_file_manager.rb +18 -14
  86. data/lib/aspera/timer_limiter.rb +2 -1
  87. data/lib/aspera/uri_reader.rb +7 -5
  88. data/lib/aspera/web_auth.rb +79 -76
  89. metadata +116 -29
  90. data/docs/Makefile +0 -66
  91. data/docs/README.erb.md +0 -3973
  92. data/docs/README.md +0 -13
  93. data/docs/diagrams.txt +0 -49
  94. data/docs/doc_tools.rb +0 -58
  95. data/lib/aspera/api_detector.rb +0 -60
  96. data/lib/aspera/cli/plugins/shares2.rb +0 -114
  97. data/lib/aspera/secrets.rb +0 -20
data/bin/ascli CHANGED
@@ -1,6 +1,7 @@
1
1
  #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
2
3
  require 'rubygems'
3
- $LOAD_PATH.unshift(File.dirname(__FILE__)+"/../lib")
4
+ $LOAD_PATH.unshift(File.realpath(File.dirname(File.realpath(__FILE__))+'/../lib'))
4
5
  require 'aspera/cli/main'
5
6
  require 'aspera/environment'
6
7
  Encoding.default_internal = Encoding::UTF_8
data/bin/asession CHANGED
@@ -1,7 +1,8 @@
1
1
  #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
2
3
  # Laurent Martin/2017
3
4
  $LOAD_PATH.unshift(File.dirname(__FILE__)+"/../lib")
4
- require 'aspera/fasp/local'
5
+ require 'aspera/fasp/agent_direct'
5
6
  require 'aspera/cli/listener/line_dump'
6
7
  require 'aspera/cli/extended_value'
7
8
  require 'aspera/log'
@@ -65,10 +66,8 @@ if transfer_spec.has_key?(TS_TMP_FILELIST_FOLDER)
65
66
  Aspera::Fasp::Parameters.file_list_folder=transfer_spec[TS_TMP_FILELIST_FOLDER]
66
67
  transfer_spec.delete(TS_TMP_FILELIST_FOLDER)
67
68
  end
68
- # get local agent (ascp)
69
- client=Aspera::Fasp::Local.new
70
- # disable ascp output on stdout to not mix with JSON events
71
- client.quiet=true
69
+ # get local agent (ascp), disable ascp output on stdout to not mix with JSON events
70
+ client=Aspera::Fasp::AgentDirect.new({quiet: true})
72
71
  # display JSON instead of legacy Lines
73
72
  client.add_listener(Aspera::Cli::Listener::LineDump.new)
74
73
  # start transfer (asynchronous)
data/docs/test_env.conf CHANGED
@@ -124,6 +124,8 @@ misc:
124
124
  faspex_publink_recv_from_fxuser: your value here
125
125
  faspex_publink_send_to_fxuser: your value here
126
126
  faspex_publink_send_to_dropbox: your value here
127
+ faspex_dbx: your value here
128
+ faspex_wkg: your value here
127
129
  shares_upload: your value here
128
130
  console_smart_id: your value here
129
131
  console_smart_file: your value here
@@ -143,6 +145,7 @@ misc:
143
145
  aoc_publink_folder: your value here
144
146
  aoc_shbx_ws: your value here
145
147
  aoc_shbx_name: your value here
148
+ aoc_shbx_meta: your value here
146
149
  aoc_node1_name: your value here
147
150
  aoc_node1_secret: your value here
148
151
  icos_bucket_key: your value here
data/examples/aoc.rb CHANGED
@@ -1,13 +1,14 @@
1
1
  #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
2
3
  require 'aspera/aoc'
3
4
  require 'aspera/log'
4
5
 
5
6
  Aspera::Log.instance.level=:debug
6
7
 
7
- if ! ARGV.length.eql?(3)
8
+ if !ARGV.length.eql?(3)
8
9
  Aspera::Log.log.error("wrong number of args: #{ARGV.length}")
9
- Aspera::Log.log.error("Usage: #{$0} <aoc URL> <aoc username> <aoc private key content>")
10
- Aspera::Log.log.error("Example: #{$0} https://myorg.ibmaspera.com john@example.com $(cat /home/john/my_key.pem)")
10
+ Aspera::Log.log.error("Usage: #{$PROGRAM_NAME} <aoc URL> <aoc username> <aoc private key content>")
11
+ Aspera::Log.log.error("Example: #{$PROGRAM_NAME} https://myorg.ibmaspera.com john@example.com $(cat /home/john/my_key.pem)")
11
12
  Process.exit(1)
12
13
  end
13
14
 
data/examples/faspex4.rb CHANGED
@@ -1,11 +1,12 @@
1
1
  #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
2
3
  # find Faspex API here: https://developer.ibm.com/apis/catalog/?search=faspex
3
4
  # this example makes use of class Aspera::Rest for REST calls, alternatively class RestClient of gem rest-client could be used
4
- # this example makes use of class Aspera::Fasp::Local for transfers, alternatively the official "Transfer SDK" could be used
5
+ # this example makes use of class Aspera::Fasp::AgentDirect for transfers, alternatively the official "Transfer SDK" could be used
5
6
  # Aspera SDK can be downloaded with: `ascli conf ascp install` , it installs in $HOME/.aspera/ascli/sdk
6
7
  require 'aspera/rest'
7
8
  require 'aspera/log'
8
- require 'aspera/fasp/local'
9
+ require 'aspera/fasp/agent_direct'
9
10
 
10
11
  tmpdir=ENV['tmp']||Dir.tmpdir || '.'
11
12
 
@@ -17,10 +18,10 @@ Aspera::Log.instance.level=:debug
17
18
  # (if data files are not there, they will be created)
18
19
  Aspera::Fasp::Installation.instance.folder = tmpdir
19
20
 
20
- if ! ARGV.length.eql?(3)
21
+ if !ARGV.length.eql?(3)
21
22
  Aspera::Log.log.error("Wrong number of args: #{ARGV.length}")
22
- Aspera::Log.log.error("Usage: #{$0} <faspex URL> <faspex username> <faspex password>")
23
- Aspera::Log.log.error("Example: #{$0} https://faspex.com/aspera/faspex john p@sSw0rd")
23
+ Aspera::Log.log.error("Usage: #{$PROGRAM_NAME} <faspex URL> <faspex username> <faspex password>")
24
+ Aspera::Log.log.error("Example: #{$PROGRAM_NAME} https://faspex.com/aspera/faspex john p@sSw0rd")
24
25
  Process.exit(1)
25
26
  end
26
27
 
@@ -36,11 +37,11 @@ Aspera::Rest.insecure=true
36
37
 
37
38
  # create REST API object
38
39
  api_v3=Aspera::Rest.new({
39
- :base_url => faspex_url,
40
- :auth => {
41
- :type => :basic,
42
- :username => faspex_user,
43
- :password => faspex_pass
40
+ base_url: faspex_url,
41
+ auth: {
42
+ type: :basic,
43
+ username: faspex_user,
44
+ password: faspex_pass
44
45
  }})
45
46
 
46
47
  # very simple api call
@@ -51,7 +52,7 @@ api_v3.read('me')
51
52
 
52
53
  # create a sample file to send
53
54
  file_to_send=File.join(tmpdir,'myfile.bin')
54
- File.open(file_to_send, "w") {|f| f.write("sample data") }
55
+ File.write(file_to_send, 'sample data')
55
56
  # package creation parameters
56
57
  package_create_params={'delivery'=>{'title'=>'test package','recipients'=>['aspera.user1@gmail.com'],'sources'=>[{'paths'=>[file_to_send]}]}}
57
58
  pkg_created=api_v3.create('send',package_create_params)[:data]
@@ -59,31 +60,30 @@ pkg_created=api_v3.create('send',package_create_params)[:data]
59
60
  transfer_spec=pkg_created['xfer_sessions'].first
60
61
  # set paths of files to send
61
62
  transfer_spec['paths']=[{'source'=>file_to_send}]
62
- # get the local agent (i.e. ascp)
63
- transfer_client=Aspera::Fasp::Local.new
64
- # disable ascp output on stdout (optional)
65
- transfer_client.quiet=true
63
+ # get local agent (ascp), disable ascp output on stdout to not mix with JSON events
64
+ transfer_client=Aspera::Fasp::AgentDirect.new({quiet: true})
66
65
  # start transfer (asynchronous)
67
66
  job_id=transfer_client.start_transfer(transfer_spec)
67
+ Aspera::Log.dump('job_id',job_id)
68
68
  # wait for all transfer completion (for the example)
69
69
  result=transfer_client.wait_for_transfers_completion
70
70
  # notify of any transfer error
71
- result.select{|i|!i.eql?(:success)}.each do |e|
71
+ result.reject{|i|i.eql?(:success)}.each do |e|
72
72
  Aspera::Log.log.error("A transfer error occured: #{e.message}")
73
73
  end
74
74
 
75
75
  # 3: Faspex 4 API v4
76
76
  #---------------
77
77
  api_v4=Aspera::Rest.new({
78
- :base_url => faspex_url+'/api',
79
- :auth => {
80
- :type => :oauth2,
81
- :base_url => faspex_url+'/auth/oauth2',
82
- :grant => :header_userpass,
83
- :user_name => faspex_user,
84
- :user_pass => faspex_pass,
85
- :scope => 'admin'
86
- }})
78
+ base_url: faspex_url+'/api',
79
+ auth: {
80
+ type: :oauth2,
81
+ base_url: faspex_url+'/auth/oauth2',
82
+ auth: {type: :basic, username: faspex_user, password: faspex_pass},
83
+ crtype: :generic,
84
+ generic: {grant_type: 'password'},
85
+ scope: 'admin'
86
+ }})
87
87
 
88
88
  # Use it. Note that Faspex 4 API v4 is totally different from Faspex 4 v3 APIs, see ref on line 2
89
89
  Aspera::Log.dump('users',api_v4.read('users')[:data])
data/examples/proxy.pac CHANGED
@@ -1,4 +1,4 @@
1
- /* demo proxy pac for Amelia */
1
+ /* demo proxy pac for ascli */
2
2
  function FindProxyForURL(url, host) {
3
3
  /* Normalize the URL for pattern matching */
4
4
  url = url.toLowerCase();
data/examples/transfer.rb CHANGED
@@ -1,8 +1,9 @@
1
1
  #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
2
3
  # Example: transfer a file using one of the provided transfer agents
3
4
  # location of ascp can be specified with env var "ascp"
4
5
  # temp folder can be specified with env var "tmp"
5
- require 'aspera/fasp/local'
6
+ require 'aspera/fasp/agent_direct'
6
7
  require 'aspera/fasp/listener'
7
8
  require 'aspera/fasp/installation'
8
9
  require 'aspera/log'
@@ -16,7 +17,7 @@ tmpdir=ENV['tmp']||Dir.tmpdir || '.'
16
17
  DEMO_CONFIG=[
17
18
  'ssh://asperaweb@eudemo.asperademo.com:33001',
18
19
  'https://node_asperaweb@eudemo.asperademo.com:9092',
19
- 'demoaspera',
20
+ 'demoaspera'
20
21
  ]
21
22
 
22
23
  ##############################################################
@@ -26,7 +27,7 @@ DEMO_CONFIG=[
26
27
  Aspera::Log.instance.level=:debug
27
28
 
28
29
  # register aspera REST call error handlers
29
- Aspera::RestErrorsAspera.registerHandlers
30
+ Aspera::RestErrorsAspera.register_handlers
30
31
 
31
32
  # some required files are generated here (keys, certs)
32
33
  Aspera::Fasp::Installation.instance.folder = tmpdir
@@ -36,10 +37,10 @@ Aspera::Fasp::Installation.instance.ascp_path=ENV['ascp'] if ENV.has_key?('ascp'
36
37
  #Aspera::Fasp::Installation.instance.installed_products.each{|p|puts("found: #{p[:name]}")}
37
38
  #Aspera::Fasp::Installation.instance.use_ascp_from_product('Aspera Connect')
38
39
  # or install:
39
- #
40
+ #
40
41
 
41
42
  # get FASP Manager singleton based on above ascp location
42
- fasp_manager=Aspera::Fasp::Local.new
43
+ fasp_manager=Aspera::Fasp::AgentDirect.new
43
44
 
44
45
  # Note that it would also be possible to start transfers using other agents
45
46
  #require 'aspera/fasp/connect'
@@ -54,7 +55,7 @@ fasp_manager=Aspera::Fasp::Local.new
54
55
  class MyListener < Aspera::Fasp::Listener
55
56
  # this is the callback called during transfers, here we only display the received information
56
57
  # but it could be used to get detailed error information, check "type" field is "ERROR"
57
- def event_enhanced(data);STDOUT.puts(JSON.generate(data));STDOUT.flush;end
58
+ def event_enhanced(data);$stdout.puts(JSON.generate(data));$stdout.flush;end
58
59
  end
59
60
 
60
61
  # register the sample listener to display events
@@ -84,9 +85,9 @@ fasp_manager.start_transfer(transfer_spec)
84
85
  # get array of status, one for each session (so, a single value array)
85
86
  # each status is either :success or "error message"
86
87
  transfer_result=fasp_manager.wait_for_transfers_completion
87
- STDOUT.puts(JSON.generate(transfer_result))
88
+ $stdout.puts(JSON.generate(transfer_result))
88
89
  # get list of errors only
89
- errors=transfer_result.select{|i|!i.eql?(:success)}
90
+ errors=transfer_result.reject{|i|i.eql?(:success)}
90
91
  # the transfer was not success, as there is at least one error
91
92
  raise "Error(s) occured: #{errors.join(',')}" if !errors.empty?
92
93
 
@@ -95,19 +96,19 @@ raise "Error(s) occured: #{errors.join(',')}" if !errors.empty?
95
96
 
96
97
  # create rest client for Node API on a public demo system, using public demo credentials
97
98
  node_api=Aspera::Rest.new({
98
- :base_url => DEMO_CONFIG[1],
99
- :auth => {
100
- :type => :basic,
101
- :username => URI.parse(DEMO_CONFIG[1]).user,
102
- :password => DEMO_CONFIG[2]
99
+ base_url: DEMO_CONFIG[1],
100
+ auth: {
101
+ type: :basic,
102
+ username: URI.parse(DEMO_CONFIG[1]).user,
103
+ password: DEMO_CONFIG[2]
103
104
  }})
104
105
  # define sample file(s) and destination folder
105
106
  sources=["#{tmpdir}/sample_file.txt"]
106
107
  destination='/Upload'
107
108
  # create sample file(s)
108
- sources.each{|p|File.write(p,"Hello World!")}
109
+ sources.each{|p|File.write(p,'Hello World!')}
109
110
  # request transfer authorization to node for a single transfer (This is a node api v3 call)
110
- send_result=node_api.create('files/upload_setup',{ :transfer_requests => [ { :transfer_request => { :paths => [ { :destination => destination } ] } } ] } )[:data]
111
+ send_result=node_api.create('files/upload_setup',{ transfer_requests: [{ transfer_request: { paths: [{ destination: destination }] } }] })[:data]
111
112
  # we normally have only one transfer spec in list, so just get the first transfer_spec
112
113
  transfer_spec=send_result['transfer_specs'].first['transfer_spec']
113
114
  # add list of files to upload
@@ -118,7 +119,6 @@ transfer_spec['authentication']='token'
118
119
  fasp_manager.start_transfer(transfer_spec)
119
120
  # optional: wait for transfer completion helper function to get events
120
121
  transfer_result=fasp_manager.wait_for_transfers_completion
121
- errors=transfer_result.select{|i|!i.eql?(:success)}
122
+ errors=transfer_result.reject{|i|i.eql?(:success)}
122
123
  # the transfer was not success, as there is at least one error
123
124
  raise "Error(s) occured: #{errors.join(',')}" if !errors.empty?
124
-