aspera-cli 4.10.0 → 4.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +19 -0
  4. data/CHANGELOG.md +528 -0
  5. data/CONTRIBUTING.md +143 -0
  6. data/README.md +977 -589
  7. data/bin/ascli +4 -4
  8. data/bin/asession +12 -12
  9. data/docs/test_env.conf +29 -19
  10. data/examples/aoc.rb +6 -6
  11. data/examples/dascli +18 -16
  12. data/examples/faspex4.rb +15 -15
  13. data/examples/node.rb +12 -12
  14. data/examples/proxy.pac +2 -2
  15. data/examples/server.rb +12 -12
  16. data/lib/aspera/aoc.rb +344 -272
  17. data/lib/aspera/ascmd.rb +56 -54
  18. data/lib/aspera/ats_api.rb +4 -4
  19. data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
  20. data/lib/aspera/cli/extended_value.rb +9 -9
  21. data/lib/aspera/cli/{formater.rb → formatter.rb} +69 -69
  22. data/lib/aspera/cli/listener/line_dump.rb +1 -1
  23. data/lib/aspera/cli/listener/logger.rb +1 -1
  24. data/lib/aspera/cli/listener/progress.rb +5 -6
  25. data/lib/aspera/cli/listener/progress_multi.rb +16 -21
  26. data/lib/aspera/cli/main.rb +72 -73
  27. data/lib/aspera/cli/manager.rb +112 -112
  28. data/lib/aspera/cli/plugin.rb +68 -48
  29. data/lib/aspera/cli/plugins/alee.rb +4 -4
  30. data/lib/aspera/cli/plugins/aoc.rb +322 -720
  31. data/lib/aspera/cli/plugins/ats.rb +50 -52
  32. data/lib/aspera/cli/plugins/bss.rb +10 -10
  33. data/lib/aspera/cli/plugins/config.rb +514 -410
  34. data/lib/aspera/cli/plugins/console.rb +12 -12
  35. data/lib/aspera/cli/plugins/cos.rb +18 -20
  36. data/lib/aspera/cli/plugins/faspex.rb +134 -136
  37. data/lib/aspera/cli/plugins/faspex5.rb +235 -70
  38. data/lib/aspera/cli/plugins/node.rb +378 -309
  39. data/lib/aspera/cli/plugins/orchestrator.rb +52 -49
  40. data/lib/aspera/cli/plugins/preview.rb +129 -120
  41. data/lib/aspera/cli/plugins/server.rb +137 -83
  42. data/lib/aspera/cli/plugins/shares.rb +77 -52
  43. data/lib/aspera/cli/plugins/sync.rb +13 -33
  44. data/lib/aspera/cli/transfer_agent.rb +61 -61
  45. data/lib/aspera/cli/version.rb +2 -1
  46. data/lib/aspera/colors.rb +3 -3
  47. data/lib/aspera/command_line_builder.rb +78 -74
  48. data/lib/aspera/cos_node.rb +31 -29
  49. data/lib/aspera/data_repository.rb +1 -1
  50. data/lib/aspera/environment.rb +30 -28
  51. data/lib/aspera/fasp/agent_base.rb +17 -15
  52. data/lib/aspera/fasp/agent_connect.rb +34 -32
  53. data/lib/aspera/fasp/agent_direct.rb +70 -73
  54. data/lib/aspera/fasp/agent_httpgw.rb +79 -74
  55. data/lib/aspera/fasp/agent_node.rb +26 -26
  56. data/lib/aspera/fasp/agent_trsdk.rb +20 -20
  57. data/lib/aspera/fasp/error.rb +3 -2
  58. data/lib/aspera/fasp/error_info.rb +11 -8
  59. data/lib/aspera/fasp/installation.rb +80 -80
  60. data/lib/aspera/fasp/listener.rb +2 -2
  61. data/lib/aspera/fasp/parameters.rb +103 -92
  62. data/lib/aspera/fasp/parameters.yaml +313 -214
  63. data/lib/aspera/fasp/resume_policy.rb +10 -10
  64. data/lib/aspera/fasp/transfer_spec.rb +22 -2
  65. data/lib/aspera/fasp/uri.rb +7 -7
  66. data/lib/aspera/faspex_gw.rb +80 -159
  67. data/lib/aspera/faspex_postproc.rb +77 -0
  68. data/lib/aspera/hash_ext.rb +3 -3
  69. data/lib/aspera/id_generator.rb +5 -5
  70. data/lib/aspera/keychain/encrypted_hash.rb +23 -28
  71. data/lib/aspera/keychain/macos_security.rb +21 -20
  72. data/lib/aspera/log.rb +13 -13
  73. data/lib/aspera/nagios.rb +24 -23
  74. data/lib/aspera/node.rb +217 -38
  75. data/lib/aspera/oauth.rb +78 -74
  76. data/lib/aspera/open_application.rb +19 -11
  77. data/lib/aspera/persistency_action_once.rb +4 -4
  78. data/lib/aspera/persistency_folder.rb +13 -13
  79. data/lib/aspera/preview/file_types.rb +8 -8
  80. data/lib/aspera/preview/generator.rb +67 -67
  81. data/lib/aspera/preview/utils.rb +27 -27
  82. data/lib/aspera/proxy_auto_config.js +63 -63
  83. data/lib/aspera/proxy_auto_config.rb +19 -19
  84. data/lib/aspera/rest.rb +65 -67
  85. data/lib/aspera/rest_call_error.rb +2 -1
  86. data/lib/aspera/rest_error_analyzer.rb +22 -21
  87. data/lib/aspera/rest_errors_aspera.rb +16 -16
  88. data/lib/aspera/secret_hider.rb +17 -14
  89. data/lib/aspera/ssh.rb +15 -14
  90. data/lib/aspera/sync.rb +177 -62
  91. data/lib/aspera/temp_file_manager.rb +2 -2
  92. data/lib/aspera/uri_reader.rb +4 -4
  93. data/lib/aspera/web_auth.rb +13 -64
  94. data/lib/aspera/web_server_simple.rb +76 -0
  95. data.tar.gz.sig +0 -0
  96. metadata +11 -6
  97. metadata.gz.sig +0 -0
@@ -20,36 +20,36 @@ module Aspera
20
20
  @parameters = DEFAULTS.dup
21
21
  if !params.nil?
22
22
  raise "expecting Hash (or nil), but have #{params.class}" unless params.is_a?(Hash)
23
- params.each do |k,v|
24
- raise "unknown resume parameter: #{k}, expect one of #{DEFAULTS.keys.map(&:to_s).join(',')}" unless DEFAULTS.has_key?(k)
23
+ params.each do |k, v|
24
+ raise "unknown resume parameter: #{k}, expect one of #{DEFAULTS.keys.map(&:to_s).join(',')}" unless DEFAULTS.key?(k)
25
25
  raise "#{k} must be Integer" unless v.is_a?(Integer)
26
26
  @parameters[k] = v
27
27
  end
28
28
  end
29
- Log.log.debug("resume params=#{@parameters}")
29
+ Log.log.debug{"resume params=#{@parameters}"}
30
30
  end
31
31
 
32
32
  # calls block a number of times (resumes) until success or limit reached
33
33
  # this is re-entrant, one resumer can handle multiple transfers in //
34
34
  def execute_with_resume
35
- raise 'block manndatory' unless block_given?
35
+ raise 'block mandatory' unless block_given?
36
36
  # maximum of retry
37
37
  remaining_resumes = @parameters[:iter_max]
38
38
  sleep_seconds = @parameters[:sleep_initial]
39
- Log.log.debug("retries=#{remaining_resumes}")
40
- # try to send the file until ascp is succesful
39
+ Log.log.debug{"retries=#{remaining_resumes}"}
40
+ # try to send the file until ascp is successful
41
41
  loop do
42
- Log.log.debug('transfer starting');
42
+ Log.log.debug('transfer starting')
43
43
  begin
44
44
  # call provided block
45
45
  yield
46
46
  break
47
47
  rescue Fasp::Error => e
48
- Log.log.warn("An error occurred: #{e.message}");
48
+ Log.log.warn{"An error occurred: #{e.message}"}
49
49
  # failure in ascp
50
50
  if e.retryable?
51
51
  # exit if we exceed the max number of retry
52
- raise Fasp::Error,'Maximum number of retry reached' if remaining_resumes <= 0
52
+ raise Fasp::Error, 'Maximum number of retry reached' if remaining_resumes <= 0
53
53
  else
54
54
  # give one chance only to non retryable errors
55
55
  unless remaining_resumes.eql?(@parameters[:iter_max])
@@ -61,7 +61,7 @@ module Aspera
61
61
 
62
62
  # take this retry in account
63
63
  remaining_resumes -= 1
64
- Log.log.warn("resuming in #{sleep_seconds} seconds (retry left:#{remaining_resumes})");
64
+ Log.log.warn{"resuming in #{sleep_seconds} seconds (retry left:#{remaining_resumes})"}
65
65
 
66
66
  # wait a bit before retrying, maybe network condition will be better
67
67
  sleep(sleep_seconds)
@@ -16,10 +16,30 @@ module Aspera
16
16
  'fasp_port' => UDP_PORT
17
17
  }.freeze
18
18
  # define constants for enums of parameters: <paramater>_<enum>, e.g. CIPHER_AES_128
19
- Aspera::Fasp::Parameters.description.each do |k,v|
19
+ Aspera::Fasp::Parameters.description.each do |k, v|
20
20
  next unless v[:enum].is_a?(Array)
21
21
  v[:enum].each do |enum|
22
- TransferSpec.const_set("#{k.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/,'_')}", enum.freeze)
22
+ TransferSpec.const_set("#{k.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/, '_')}", enum.freeze)
23
+ end
24
+ end
25
+ class << self
26
+ def action_to_direction(tspec, command)
27
+ raise 'transfer spec must be a Hash' unless tspec.is_a?(Hash)
28
+ tspec['direction'] = case command.to_sym
29
+ when :upload then DIRECTION_SEND
30
+ when :download then DIRECTION_RECEIVE
31
+ else raise 'Error: upload or download only'
32
+ end
33
+ return tspec
34
+ end
35
+
36
+ def action(tspec)
37
+ raise 'transfer spec must be a Hash' unless tspec.is_a?(Hash)
38
+ return case tspec['direction']
39
+ when DIRECTION_SEND then :upload
40
+ when DIRECTION_RECEIVE then :download
41
+ else raise 'Error: upload or download only'
42
+ end
23
43
  end
24
44
  end
25
45
  end
@@ -5,10 +5,10 @@ require 'aspera/command_line_builder'
5
5
 
6
6
  module Aspera
7
7
  module Fasp
8
- # translates a "faspe:" URI (used in Faspex) into transfer spec hash
8
+ # translates a "faspe:" URI (used in Faspex 4) into transfer spec hash
9
9
  class Uri
10
10
  def initialize(fasplink)
11
- @fasp_uri = URI.parse(fasplink.gsub(' ','%20'))
11
+ @fasp_uri = URI.parse(fasplink.gsub(' ', '%20'))
12
12
  # TODO: check scheme is faspe
13
13
  end
14
14
 
@@ -34,16 +34,16 @@ module Aspera
34
34
  when 'minrate' then result_ts['min_rate_kbps'] = value.to_i
35
35
  when 'port' then result_ts['fasp_port'] = value.to_i
36
36
  when 'bwcap' then result_ts['target_rate_cap_kbps'] = value.to_i
37
- when 'enc' then result_ts['cipher'] = value.gsub(/^aes/,'aes-').gsub(/cfb$/,'-cfb').gsub(/gcm$/,'-gcm').gsub(/--/,'-')
37
+ when 'enc' then result_ts['cipher'] = value.gsub(/^aes/, 'aes-').gsub(/cfb$/, '-cfb').gsub(/gcm$/, '-gcm').gsub(/--/, '-')
38
38
  when 'tags64' then result_ts['tags'] = JSON.parse(Base64.strict_decode64(value))
39
39
  when 'createpath' then result_ts['create_dir'] = CommandLineBuilder.yes_to_true(value)
40
40
  when 'fallback' then result_ts['http_fallback'] = CommandLineBuilder.yes_to_true(value)
41
41
  when 'lockpolicy' then result_ts['lock_rate_policy'] = CommandLineBuilder.yes_to_true(value)
42
42
  when 'lockminrate' then result_ts['lock_min_rate'] = CommandLineBuilder.yes_to_true(value)
43
- when 'auth' then Log.log.debug("ignoring auth #{name}=#{value}") # TODO: translate into transfer spec ? yes/no
44
- when 'v' then Log.log.debug("ignoring v #{name}=#{value}") # TODO: translate into transfer spec ? 2
45
- when 'protect' then Log.log.debug("ignoring protect #{name}=#{value}") # TODO: translate into transfer spec ?
46
- else Log.log.warn("URI parameter ignored: #{name} = #{value}")
43
+ when 'auth' then Log.log.debug{"ignoring auth #{name}=#{value}"} # TODO: translate into transfer spec ? yes/no
44
+ when 'v' then Log.log.debug{"ignoring v #{name}=#{value}"} # TODO: translate into transfer spec ? 2
45
+ when 'protect' then Log.log.debug{"ignoring protect #{name}=#{value}"} # TODO: translate into transfer spec ?
46
+ else Log.log.warn{"URI parameter ignored: #{name} = #{value}"}
47
47
  end
48
48
  end
49
49
  return result_ts
@@ -1,174 +1,95 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'aspera/web_server_simple'
3
4
  require 'aspera/log'
4
- require 'aspera/aoc'
5
- require 'aspera/fasp/transfer_spec'
6
- require 'aspera/cli/main'
7
- require 'webrick'
8
- require 'webrick/https'
9
- require 'securerandom'
10
- require 'openssl'
11
5
  require 'json'
12
6
 
13
7
  module Aspera
14
8
  # this class answers the Faspex /send API and creates a package on Aspera on Cloud
15
- class FaspexGW
16
- class FxGwServlet < WEBrick::HTTPServlet::AbstractServlet
17
- def initialize(_server, a_aoc_api_user, a_workspace_id)
18
- super
19
- @aoc_api_user = a_aoc_api_user
20
- @aoc_workspace_id = a_workspace_id
21
- end
22
-
23
- # parameters from user to Faspex API call
24
- # {"delivery":{"use_encryption_at_rest":false,"note":"note","sources":[{"paths":["file1"]}],"title":"my title","recipients":["email1"],"send_upload_result":true}}
25
- # {
26
- # "delivery"=>{
27
- # "use_encryption_at_rest"=>false,
28
- # "note"=>"note",
29
- # "sources"=>[{"paths"=>["file1"]}],
30
- # "title"=>"my title",
31
- # "recipients"=>["email1"],
32
- # "send_upload_result"=>true
33
- # }
34
- # }
35
- def process_faspex_send(request, response)
36
- raise 'no payload' if request.body.nil?
37
-
38
- faspex_pkg_parameters = JSON.parse(request.body)
39
- faspex_pkg_delivery = faspex_pkg_parameters['delivery']
40
- Log.log.debug("faspex pkg create parameters=#{faspex_pkg_parameters}")
41
-
42
- # get recipient ids
43
- files_pkg_recipients = []
44
- faspex_pkg_delivery['recipients'].each do |recipient_email|
45
- user_lookup = @aoc_api_user.read('contacts',
46
- { 'current_workspace_id' => @aoc_workspace_id, 'q' => recipient_email })[:data]
47
- raise StandardError,
48
- "no such unique user: #{recipient_email} / #{user_lookup}" unless !user_lookup.nil? && user_lookup.length.eql?(1)
49
-
50
- recipient_user_info = user_lookup.first
51
- files_pkg_recipients.push({
52
- 'id' => recipient_user_info['source_id'],
53
- 'type' => recipient_user_info['source_type']
54
- })
55
- end
56
-
57
- # create a new package with one file
58
- the_package = @aoc_api_user.create('packages', {
59
- 'file_names' => faspex_pkg_delivery['sources'][0]['paths'],
60
- 'name' => faspex_pkg_delivery['title'],
61
- 'note' => faspex_pkg_delivery['note'],
62
- 'recipients' => files_pkg_recipients,
63
- 'workspace_id' => @aoc_workspace_id
64
- })[:data]
65
-
66
- # get node information for the node on which package must be created
67
- node_info = @aoc_api_user.read("nodes/#{the_package['node_id']}")[:data]
68
-
69
- # get transfer token (for node)
70
- node_auth_bearer_token = @aoc_api_user.oauth_token(scope: AoC.node_scope(node_info['access_key'],
71
- AoC::SCOPE_NODE_USER))
72
-
73
- # tell Files what to expect in package: 1 transfer (can also be done after transfer)
74
- @aoc_api_user.update("packages/#{the_package['id']}", { 'sent' => true, 'transfers_expected' => 1 })
75
-
76
- # to return an error:
77
- # response.status=400
78
- # return 'ERROR HERE'
79
-
80
- # TODO: check about xfer_*
81
- ts_tags = {
82
- 'aspera' => {
83
- 'files' => { 'package_id' => the_package['id'], 'package_operation' => 'upload' },
84
- 'node' => { 'access_key' => node_info['access_key'], 'file_id' => the_package['contents_file_id'] },
85
- 'xfer_id' => SecureRandom.uuid,
86
- 'xfer_retry' => 3600
87
- }
88
- }
89
- # this transfer spec is for transfer to AoC
90
- faspex_transfer_spec = {
91
- 'direction' => 'send',
92
- 'remote_host' => node_info['host'],
93
- 'remote_user' => Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER,
94
- 'ssh_port' => Fasp::TransferSpec::SSH_PORT,
95
- 'fasp_port' => Fasp::TransferSpec::UDP_PORT,
96
- 'tags' => ts_tags,
97
- 'token' => node_auth_bearer_token,
98
- 'paths' => [{ 'destination' => '/' }],
99
- 'cookie' => 'unused',
100
- 'create_dir' => true,
101
- 'rate_policy' => 'fair',
102
- 'rate_policy_allowed' => 'fixed',
103
- 'min_rate_cap_kbps' => nil,
104
- 'min_rate_kbps' => 0,
105
- 'target_rate_percentage' => nil,
106
- 'lock_target_rate' => nil,
107
- 'fasp_url' => 'unused',
108
- 'lock_min_rate' => true,
109
- 'lock_rate_policy' => true,
110
- 'source_root' => '',
111
- 'content_protection' => nil,
112
- 'target_rate_cap_kbps' => 20_000, # TODO: is this value useful ?
113
- 'target_rate_kbps' => 10_000, # TODO: get from where?
114
- 'cipher' => 'aes-128',
115
- 'cipher_allowed' => nil,
116
- 'http_fallback' => false,
117
- 'http_fallback_port' => nil,
118
- 'https_fallback_port' => nil,
119
- 'destination_root' => '/'
120
- }
121
- # but we place it in a Faspex package creation response
122
- faspex_package_create_result = {
123
- 'links' => { 'status' => 'unused' },
124
- 'xfer_sessions' => [faspex_transfer_spec]
125
- }
126
- Log.log.info("faspex_package_create_result=#{faspex_package_create_result}")
127
- response.status = 200
128
- response.content_type = 'application/json'
129
- response.body = JSON.generate(faspex_package_create_result)
130
- end
131
-
132
- def do_GET(request, response)
133
- case request.path
134
- when '/aspera/faspex/send'
135
- process_faspex_send(request, response)
136
- else
137
- response.status = 400
138
- 'ERROR HERE'
139
- end
140
- end
141
- end # FxGwServlet
9
+ class Faspex4GWServlet < WEBrick::HTTPServlet::AbstractServlet
10
+ # @param app_api [Aspera::AoC]
11
+ # @param app_context [String]
12
+ def initialize(server, app_api, app_context)
13
+ super(server)
14
+ # typed: Aspera::AoC
15
+ @app_api = app_api
16
+ @app_context = app_context
17
+ end
142
18
 
143
- class NewUserServlet < WEBrick::HTTPServlet::AbstractServlet
144
- def do_GET(request, response)
145
- case request.path
146
- when '/newuser'
147
- response.status = 200
148
- response.content_type = 'text/html'
149
- response.body = '<html><body>hello world</body></html>'
150
- else
151
- raise "unsupported path: [#{request.path}]"
152
- end
153
- end
19
+ # Map Faspex 4 /send API to AoC package create
20
+ # parameters from user to Faspex API call
21
+ # https://developer.ibm.com/apis/catalog/aspera--aspera-faspex-client-sdk/Sending%20Packages%20(API%20v.3)
22
+ def faspex4_send_to_aoc(faspex_pkg_parameters)
23
+ faspex_pkg_delivery = faspex_pkg_parameters['delivery']
24
+ package_data = {
25
+ # 'file_names' => faspex_pkg_delivery['sources'][0]['paths'],
26
+ 'name' => faspex_pkg_delivery['title'],
27
+ 'note' => faspex_pkg_delivery['note'],
28
+ 'recipients' => faspex_pkg_delivery['recipients'],
29
+ 'workspace_id' => @app_context
30
+ }
31
+ created_package = @app_api.create_package_simple(package_data, true, @new_user_option)
32
+ # but we place it in a Faspex package creation response
33
+ return {
34
+ 'links' => { 'status' => 'unused' },
35
+ 'xfer_sessions' => [created_package[:spec]]
36
+ }
154
37
  end
155
38
 
156
- def initialize(a_aoc_api_user, a_workspace_id)
157
- webrick_options = {
158
- Port: 9443,
159
- Logger: Log.log,
160
- SSLEnable: true,
161
- SSLCertName: [['CN', WEBrick::Utils.getservername]]
39
+ def faspex4_send_to_faspex5(faspex_pkg_parameters)
40
+ faspex_pkg_delivery = faspex_pkg_parameters['delivery']
41
+ package_data = {
42
+ 'title' => faspex_pkg_delivery['title'],
43
+ 'note' => faspex_pkg_delivery['note'],
44
+ 'recipients' => faspex_pkg_delivery['recipients'].map{|name|{'name'=>name}}
45
+ }
46
+ package = @app_api.create('packages', package_data)[:data]
47
+ # TODO: option to send from remote source or httpgw
48
+ transfer_spec = @app_api.call(
49
+ operation: 'POST',
50
+ subpath: "packages/#{package['id']}/transfer_spec/upload",
51
+ headers: {'Accept' => 'application/json'},
52
+ url_params: {transfer_type: Aspera::Cli::Plugins::Faspex5::TRANSFER_CONNECT},
53
+ json_params: {paths: [{'destination'=>'/'}]}
54
+ )[:data]
55
+ transfer_spec.delete('authentication')
56
+ # but we place it in a Faspex package creation response
57
+ return {
58
+ 'links' => { 'status' => 'unused' },
59
+ 'xfer_sessions' => [transfer_spec]
162
60
  }
163
- Log.log.info("Server started on port #{webrick_options[:Port]}")
164
- @server = WEBrick::HTTPServer.new(webrick_options)
165
- @server.mount('/aspera/faspex', FxGwServlet, a_aoc_api_user, a_workspace_id)
166
- @server.mount('/newuser', NewUserServlet)
167
- trap('INT') { @server.shutdown }
168
61
  end
169
62
 
170
- def start_server
171
- @server.start
63
+ def do_POST(request, response)
64
+ case request.path
65
+ when '/aspera/faspex/send'
66
+ begin
67
+ raise 'no payload' if request.body.nil?
68
+ faspex_pkg_parameters = JSON.parse(request.body)
69
+ Log.log.debug{"faspex pkg create parameters=#{faspex_pkg_parameters}"}
70
+ faspex_package_create_result =
71
+ if @app_api.is_a?(Aspera::AoC)
72
+ faspex4_send_to_aoc(faspex_pkg_parameters)
73
+ elsif @app_api.is_a?(Aspera::Rest)
74
+ faspex4_send_to_faspex5(faspex_pkg_parameters)
75
+ else
76
+ raise "No such adapter: #{@app_api.class}"
77
+ end
78
+ Log.log.info{"faspex_package_create_result=#{faspex_package_create_result}"}
79
+ response.status = 200
80
+ response.content_type = 'application/json'
81
+ response.body = JSON.generate(faspex_package_create_result)
82
+ rescue => e
83
+ response.status = 500
84
+ response['Content-Type'] = 'application/json'
85
+ response.body = {error: e.message}.to_json
86
+ Log.log.error(e.message)
87
+ end
88
+ else
89
+ response.status = 400
90
+ response['Content-Type'] = 'application/json'
91
+ response.body = {error: 'Bad request'}.to_json
92
+ end
172
93
  end
173
- end # FaspexGW
94
+ end # Faspex4GWServlet
174
95
  end # AsperaLm
@@ -0,0 +1,77 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'English'
4
+ require 'aspera/web_server_simple'
5
+ require 'aspera/log'
6
+ require 'json'
7
+ require 'timeout'
8
+
9
+ module Aspera
10
+ # this class answers the Faspex /send API and creates a package on Aspera on Cloud
11
+ class Faspex4PostProcServlet < WEBrick::HTTPServlet::AbstractServlet
12
+ ALLOWED_PARAMETERS = %i[root script_folder fail_on_error timeout_seconds].freeze
13
+ def initialize(server, parameters)
14
+ raise 'parameters must be Hash' unless parameters.is_a?(Hash)
15
+ @parameters = parameters.symbolize_keys
16
+ Log.dump(:postproc_parameters, @parameters)
17
+ raise "unexpected key in parameters config: only: #{ALLOWED_PARAMETERS.join(', ')}" if @parameters.keys.any?{|k|!ALLOWED_PARAMETERS.include?(k)}
18
+ @parameters[:script_folder] ||= '.'
19
+ @parameters[:fail_on_error] ||= false
20
+ @parameters[:timeout_seconds] ||= 60
21
+ super(server)
22
+ Log.log.debug{"Faspex4PostProcServlet initialized"}
23
+ end
24
+
25
+ def do_POST(request, response)
26
+ Log.log.debug{"request=#{request.path}"}
27
+ begin
28
+ # only accept requests on the root
29
+ if !request.path.start_with?(@parameters[:root])
30
+ response.status = 400
31
+ response['Content-Type'] = 'application/json'
32
+ response.body = {status: 'error', message: 'Request outside domain'}.to_json
33
+ return
34
+ end
35
+ if request.body.nil?
36
+ response.status = 400
37
+ response['Content-Type'] = 'application/json'
38
+ response.body = {status: 'error', message: 'Empty request'}.to_json
39
+ return
40
+ end
41
+ # build script path by removing domain, and adding script folder
42
+ script_file = request.path[@parameters[:root].size .. ]
43
+ Log.log.debug{"script file=#{script_file}"}
44
+ script_path = File.join(@parameters[:script_folder], script_file)
45
+ Log.log.debug{"script=#{script_path}"}
46
+ webhook_parameters = JSON.parse(request.body)
47
+ Log.dump(:webhook_parameters, webhook_parameters)
48
+ # env expects only strings
49
+ environment = webhook_parameters.each_with_object({}) { |(k, v), h| h[k] = v.to_s }
50
+ post_proc_pid = Process.spawn(environment, [script_path, script_path])
51
+ Log.log.debug{"pid=#{post_proc_pid}"}
52
+ raise 'no pid' if post_proc_pid.nil?
53
+ # "wait" for process to avoid zombie
54
+ Timeout.timeout(@parameters[:timeout_seconds]) do
55
+ Process.wait(post_proc_pid)
56
+ post_proc_pid = nil
57
+ end
58
+ process_status = $CHILD_STATUS
59
+ raise "script #{script_path} failed with code #{process_status.exitstatus}" if !process_status.success? && @parameters[:fail_on_error]
60
+ response.status = 200
61
+ response.content_type = 'application/json'
62
+ response.body = JSON.generate({status: 'success', script: script_path, exit_code: process_status.exitstatus})
63
+ Log.log.debug{'Script executed successfully'}
64
+ rescue => e
65
+ Log.log.error("Script failed: #{e.class}:#{e.message}")
66
+ if !post_proc_pid.nil?
67
+ Process.kill('SIGKILL', post_proc_pid)
68
+ Process.wait(post_proc_pid)
69
+ Log.log.error("Killed process: #{post_proc_pid}")
70
+ end
71
+ response.status = 500
72
+ response['Content-Type'] = 'application/json'
73
+ response.body = {status: 'error', script: script_path, message: e.message}.to_json
74
+ end
75
+ end
76
+ end # Faspex4PostProcServlet
77
+ end # AsperaLm
@@ -2,11 +2,11 @@
2
2
 
3
3
  class ::Hash
4
4
  def deep_merge(second)
5
- merge(second){|_key,v1,v2|Hash === v1 && Hash === v2 ? v1.deep_merge(v2) : v2}
5
+ merge(second){|_key, v1, v2|Hash === v1 && Hash === v2 ? v1.deep_merge(v2) : v2}
6
6
  end
7
7
 
8
8
  def deep_merge!(second)
9
- merge!(second){|_key,v1,v2|Hash === v1 && Hash === v2 ? v1.deep_merge!(v2) : v2}
9
+ merge!(second){|_key, v1, v2|Hash === v1 && Hash === v2 ? v1.deep_merge!(v2) : v2}
10
10
  end
11
11
  end
12
12
 
@@ -14,7 +14,7 @@ end
14
14
  unless Hash.method_defined?(:transform_keys)
15
15
  class Hash
16
16
  def transform_keys
17
- return each_with_object({}){|(k,v),memo|memo[yield(k)]=v} if block_given?
17
+ return each_with_object({}){|(k, v), memo|memo[yield(k)] = v} if block_given?
18
18
  raise 'missing block'
19
19
  end
20
20
  end
@@ -7,7 +7,7 @@ module Aspera
7
7
  ID_SEPARATOR = '_'
8
8
  WINDOWS_PROTECTED_CHAR = %r{[/:"<>\\*?]}.freeze
9
9
  PROTECTED_CHAR_REPLACE = '_'
10
- private_constant :ID_SEPARATOR,:PROTECTED_CHAR_REPLACE,:WINDOWS_PROTECTED_CHAR
10
+ private_constant :ID_SEPARATOR, :PROTECTED_CHAR_REPLACE, :WINDOWS_PROTECTED_CHAR
11
11
  class << self
12
12
  def from_list(object_id)
13
13
  if object_id.is_a?(Array)
@@ -16,10 +16,10 @@ module Aspera
16
16
  end.join(ID_SEPARATOR)
17
17
  end
18
18
  raise 'id must be a String' unless object_id.is_a?(String)
19
- return object_id.
20
- gsub(WINDOWS_PROTECTED_CHAR,PROTECTED_CHAR_REPLACE). # remove windows forbidden chars
21
- gsub('.',PROTECTED_CHAR_REPLACE). # keep dot for extension only (nicer)
22
- downcase
19
+ return object_id
20
+ .gsub(WINDOWS_PROTECTED_CHAR, PROTECTED_CHAR_REPLACE) # remove windows forbidden chars
21
+ .gsub('.', PROTECTED_CHAR_REPLACE) # keep dot for extension only (nicer)
22
+ .downcase
23
23
  end
24
24
  end
25
25
  end
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'aspera/hash_ext'
4
+ require 'aspera/environment'
4
5
  require 'symmetric_encryption/core'
5
6
  require 'yaml'
6
7
 
@@ -8,65 +9,59 @@ module Aspera
8
9
  module Keychain
9
10
  # Manage secrets in a simple Hash
10
11
  class EncryptedHash
11
- CIPHER_NAME='aes-256-cbc'
12
- ACCEPTED_KEYS = %i[label username password url description].freeze
13
- def initialize(path,current_password)
14
- @path=path
15
- self.password=current_password
12
+ CIPHER_NAME = 'aes-256-cbc'
13
+ CONTENT_KEYS = %i[label username password url description].freeze
14
+ def initialize(path, current_password)
15
+ @path = path
16
+ self.password = current_password
16
17
  raise 'path to vault file shall be String' unless @path.is_a?(String)
17
- @all_secrets=File.exist?(@path) ? YAML.load_stream(@cipher.decrypt(File.read(@path))).first : {}
18
+ @all_secrets = File.exist?(@path) ? YAML.load_stream(@cipher.decrypt(File.read(@path))).first : {}
18
19
  end
19
20
 
20
21
  def password=(new_password)
21
- key_bytes=CIPHER_NAME.split('-')[1].to_i/8
22
- # derive key from passphrase
23
- key="#{new_password}#{"\x0"*key_bytes}"[0..(key_bytes-1)]
24
- Log.log.debug("key=[#{key}],#{key.length}")
25
- SymmetricEncryption.cipher=@cipher = SymmetricEncryption::Cipher.new(cipher_name: CIPHER_NAME,key: key,encoding: :none)
22
+ # number of bits in second position
23
+ key_bytes = CIPHER_NAME.split('-')[1].to_i / Environment::BITS_PER_BYTE
24
+ # derive key from passphrase, add trailing zeros
25
+ key = "#{new_password}#{"\x0" * key_bytes}"[0..(key_bytes - 1)]
26
+ Log.log.debug{"key=[#{key}],#{key.length}"}
27
+ SymmetricEncryption.cipher = @cipher = SymmetricEncryption::Cipher.new(cipher_name: CIPHER_NAME, key: key, encoding: :none)
26
28
  end
27
29
 
28
30
  def save
29
- File.write(@path, @cipher.encrypt(YAML.dump(@all_secrets)),encoding: 'BINARY')
31
+ File.write(@path, @cipher.encrypt(YAML.dump(@all_secrets)), encoding: 'BINARY')
30
32
  end
31
33
 
32
34
  def set(options)
33
35
  raise 'options shall be Hash' unless options.is_a?(Hash)
34
- unsupported = options.keys - ACCEPTED_KEYS
36
+ unsupported = options.keys - CONTENT_KEYS
37
+ options.each_value {|v| raise 'value must be String' unless v.is_a?(String)}
35
38
  raise "unsupported options: #{unsupported}" unless unsupported.empty?
36
39
  label = options.delete(:label)
37
- raise "secret #{label} already exist, delete first" if @all_secrets.has_key?(label)
40
+ raise "secret #{label} already exist, delete first" if @all_secrets.key?(label)
38
41
  @all_secrets[label] = options.symbolize_keys
39
42
  save
40
43
  end
41
44
 
42
45
  def list
43
46
  result = []
44
- @all_secrets.each do |label,values|
47
+ @all_secrets.each do |label, values|
45
48
  normal = values.symbolize_keys
46
49
  normal[:label] = label
47
- ACCEPTED_KEYS.each{|k|normal[k] = '' unless normal.has_key?(k)}
50
+ CONTENT_KEYS.each{|k|normal[k] = '' unless normal.key?(k)}
48
51
  result.push(normal)
49
52
  end
50
53
  return result
51
54
  end
52
55
 
53
- def delete(options)
54
- raise 'options shall be Hash' unless options.is_a?(Hash)
55
- unsupported = options.keys - %i[label]
56
- raise "unsupported options: #{unsupported}" unless unsupported.empty?
57
- label=options[:label]
56
+ def delete(label:)
58
57
  @all_secrets.delete(label)
59
58
  save
60
59
  end
61
60
 
62
- def get(options)
63
- raise 'options shall be Hash' unless options.is_a?(Hash)
64
- unsupported = options.keys - %i[label]
65
- raise "unsupported options: #{unsupported}" unless unsupported.empty?
66
- label=options[:label]
61
+ def get(label:, exception: true)
62
+ raise "Label not found: #{label}" unless @all_secrets.key?(label) || !exception
67
63
  result = @all_secrets[label].clone
68
- raise "no such entry #{label}" if result.nil?
69
- result[:label]=label
64
+ result[:label] = label if result.is_a?(Hash)
70
65
  return result
71
66
  end
72
67
  end