aspera-cli 4.21.1 → 4.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +1 -1
  4. data/CHANGELOG.md +52 -22
  5. data/CONTRIBUTING.md +69 -148
  6. data/README.md +929 -668
  7. data/bin/ascli +5 -14
  8. data/bin/asession +1 -3
  9. data/examples/get_proto_file.rb +4 -3
  10. data/examples/proxy.pac +20 -20
  11. data/lib/aspera/agent/base.rb +11 -5
  12. data/lib/aspera/agent/connect.rb +30 -28
  13. data/lib/aspera/agent/{alpha.rb → desktop.rb} +35 -31
  14. data/lib/aspera/agent/direct.rb +141 -121
  15. data/lib/aspera/agent/httpgw.rb +22 -26
  16. data/lib/aspera/agent/node.rb +14 -11
  17. data/lib/aspera/agent/transferd.rb +30 -19
  18. data/lib/aspera/api/alee.rb +1 -1
  19. data/lib/aspera/api/aoc.rb +6 -6
  20. data/lib/aspera/api/cos_node.rb +2 -2
  21. data/lib/aspera/api/httpgw.rb +7 -3
  22. data/lib/aspera/api/node.rb +10 -8
  23. data/lib/aspera/ascmd.rb +3 -3
  24. data/lib/aspera/ascp/installation.rb +53 -72
  25. data/lib/aspera/ascp/management.rb +1 -1
  26. data/lib/aspera/assert.rb +11 -2
  27. data/lib/aspera/cli/error.rb +2 -2
  28. data/lib/aspera/cli/extended_value.rb +46 -21
  29. data/lib/aspera/cli/formatter.rb +55 -48
  30. data/lib/aspera/cli/hints.rb +1 -1
  31. data/lib/aspera/cli/info.rb +1 -0
  32. data/lib/aspera/cli/main.rb +192 -170
  33. data/lib/aspera/cli/manager.rb +18 -18
  34. data/lib/aspera/cli/plugin.rb +23 -20
  35. data/lib/aspera/cli/plugin_factory.rb +1 -1
  36. data/lib/aspera/cli/plugins/alee.rb +1 -1
  37. data/lib/aspera/cli/plugins/aoc.rb +247 -159
  38. data/lib/aspera/cli/plugins/ats.rb +19 -17
  39. data/lib/aspera/cli/plugins/config.rb +76 -113
  40. data/lib/aspera/cli/plugins/console.rb +5 -3
  41. data/lib/aspera/cli/plugins/faspex.rb +39 -35
  42. data/lib/aspera/cli/plugins/faspex5.rb +111 -84
  43. data/lib/aspera/cli/plugins/faspio.rb +13 -1
  44. data/lib/aspera/cli/plugins/httpgw.rb +13 -1
  45. data/lib/aspera/cli/plugins/node.rb +312 -182
  46. data/lib/aspera/cli/plugins/orchestrator.rb +34 -40
  47. data/lib/aspera/cli/plugins/preview.rb +3 -3
  48. data/lib/aspera/cli/plugins/server.rb +6 -6
  49. data/lib/aspera/cli/plugins/shares.rb +5 -5
  50. data/lib/aspera/cli/sync_actions.rb +19 -18
  51. data/lib/aspera/cli/transfer_agent.rb +5 -5
  52. data/lib/aspera/cli/transfer_progress.rb +2 -2
  53. data/lib/aspera/cli/version.rb +1 -1
  54. data/lib/aspera/command_line_builder.rb +116 -95
  55. data/lib/aspera/coverage.rb +8 -5
  56. data/lib/aspera/environment.rb +26 -17
  57. data/lib/aspera/faspex_gw.rb +14 -14
  58. data/lib/aspera/faspex_postproc.rb +10 -11
  59. data/lib/aspera/hash_ext.rb +4 -14
  60. data/lib/aspera/json_rpc.rb +1 -1
  61. data/lib/aspera/keychain/encrypted_hash.rb +47 -34
  62. data/lib/aspera/keychain/factory.rb +41 -0
  63. data/lib/aspera/keychain/hashicorp_vault.rb +71 -0
  64. data/lib/aspera/keychain/macos_security.rb +19 -11
  65. data/lib/aspera/log.rb +28 -34
  66. data/lib/aspera/nagios.rb +6 -6
  67. data/lib/aspera/node_simulator.rb +8 -8
  68. data/lib/aspera/oauth/base.rb +14 -7
  69. data/lib/aspera/oauth/factory.rb +5 -6
  70. data/lib/aspera/oauth/url_json.rb +6 -6
  71. data/lib/aspera/persistency_action_once.rb +6 -4
  72. data/lib/aspera/persistency_folder.rb +2 -2
  73. data/lib/aspera/preview/generator.rb +13 -10
  74. data/lib/aspera/preview/options.rb +16 -16
  75. data/lib/aspera/preview/terminal.rb +4 -4
  76. data/lib/aspera/preview/utils.rb +15 -17
  77. data/lib/aspera/products/connect.rb +35 -1
  78. data/lib/aspera/products/{alpha.rb → desktop.rb} +3 -3
  79. data/lib/aspera/products/transferd.rb +9 -2
  80. data/lib/aspera/proxy_auto_config.rb +2 -2
  81. data/lib/aspera/rest.rb +56 -47
  82. data/lib/aspera/rest_errors_aspera.rb +1 -1
  83. data/lib/aspera/secret_hider.rb +12 -5
  84. data/lib/aspera/ssh.rb +4 -4
  85. data/lib/aspera/temp_file_manager.rb +5 -4
  86. data/lib/aspera/transfer/convert.rb +29 -0
  87. data/lib/aspera/transfer/error_info.rb +66 -66
  88. data/lib/aspera/transfer/parameters.rb +13 -68
  89. data/lib/aspera/transfer/spec.rb +5 -6
  90. data/lib/aspera/transfer/spec.schema.yaml +753 -0
  91. data/lib/aspera/transfer/spec_doc.rb +62 -0
  92. data/lib/aspera/transfer/sync.rb +23 -72
  93. data/lib/aspera/transfer/sync_instance.schema.yaml +13 -0
  94. data/lib/aspera/transfer/sync_session.schema.yaml +79 -0
  95. data/lib/aspera/transfer/uri.rb +6 -6
  96. data/lib/aspera/uri_reader.rb +18 -1
  97. data/lib/aspera/web_auth.rb +1 -1
  98. data/lib/aspera/web_server_simple.rb +53 -44
  99. data.tar.gz.sig +0 -0
  100. metadata +28 -165
  101. metadata.gz.sig +0 -0
  102. data/examples/build_exec +0 -74
  103. data/examples/build_exec_rubyc +0 -40
  104. data/examples/build_package.sh +0 -28
  105. data/lib/aspera/transfer/spec.yaml +0 -718
@@ -2,79 +2,101 @@
2
2
 
3
3
  require 'aspera/log'
4
4
  require 'aspera/assert'
5
+ require 'yaml'
5
6
  module Aspera
6
7
  # helper class to build command line from a parameter list (key-value hash)
7
8
  # constructor takes hash: { 'param1':'value1', ...}
8
9
  # process_param is called repeatedly with all known parameters
9
10
  # add_env_args is called to get resulting param list and env var (also checks that all params were used)
10
11
  class CommandLineBuilder
11
- # parameter with one of those tags is a command line option with --
12
- CLI_OPTION_TYPE_SWITCH = %i[opt_without_arg opt_with_arg].freeze
13
- CLI_OPTION_TYPES = %i[special ignore envvar].concat(CLI_OPTION_TYPE_SWITCH).freeze
14
- OPTIONS_KEYS = %i[desc accepted_types default enum agents required cli ts deprecation].freeze
15
- CLI_KEYS = %i[type switch convert variable].freeze
16
-
17
- private_constant :CLI_OPTION_TYPE_SWITCH, :CLI_OPTION_TYPES, :OPTIONS_KEYS, :CLI_KEYS
12
+ # description [String] Description
13
+ # type [String,Array] Accepted type(s) for non-enum
14
+ # default [String] Default value if not specified
15
+ # enum [Array] Set with list of values for enum types accepted in transfer spec
16
+ # x-cli-envvar [String] Name of env var
17
+ # x-cli-option [String] Command line option (starts with "-")
18
+ # x-cli-switch [Bool] true if option has no arg, else by default option has a value
19
+ # x-cli-special [Bool] true if special handling (defered)
20
+ # x-cli-convert [String,Hash] Method name for Convert object or Conversion for enum ts to arg
21
+ # x-agents [Array] Supported agents (for doc only), if not specified: all
22
+ # x-tspec [Bool,String] (async) true if same name in transfer spec, else name in transfer spec, else ignored
23
+ # x-deprecation [String] Deprecation message for doc
24
+ SCHEMA_KEYS = %w[
25
+ description
26
+ type
27
+ default
28
+ enum
29
+ required
30
+ $comment
31
+ x-cli-envvar
32
+ x-cli-option
33
+ x-cli-switch
34
+ x-cli-special
35
+ x-cli-convert
36
+ x-agents
37
+ x-tspec
38
+ x-deprecation
39
+ ].freeze
40
+
41
+ CLI_AGENT = 'direct'
42
+
43
+ private_constant :SCHEMA_KEYS, :CLI_AGENT
18
44
 
19
45
  class << self
20
- # transform yes/no to true/false
21
- def yes_to_true(value)
22
- case value
23
- when 'yes' then return true
24
- when 'no' then return false
25
- else Aspera.error_unexpected_value(value){'only: yes or no: '}
26
- end
46
+ # @return true if given agent supports that field
47
+ def supported_by_agent(agent, properties)
48
+ !properties.key?('x-agents') || properties['x-agents'].include?(agent)
27
49
  end
28
50
 
29
- # Called by provider of definition before constructor of this class so that params_definition has all mandatory fields
30
- def normalize_description(full_description)
31
- full_description.each do |name, options|
32
- Aspera.assert_type(options, Hash){name}
33
- unsupported_keys = options.keys - OPTIONS_KEYS
51
+ # Called by provider of definition before constructor of this class so that schema has all mandatory fields
52
+ def read_schema(source_path, suffix=nil)
53
+ suffix = "_#{suffix}" unless suffix.nil?
54
+ schema = YAML.load_file("#{source_path[0..-4]}#{suffix}.schema.yaml")
55
+ schema['properties'].each do |name, properties|
56
+ Aspera.assert_type(properties, Hash){name}
57
+ unsupported_keys = properties.keys - SCHEMA_KEYS
34
58
  Aspera.assert(unsupported_keys.empty?){"Unsupported definition keys: #{unsupported_keys}"}
35
- Aspera.assert(options.key?(:cli)){"Missing key: cli for #{name}"}
36
- Aspera.assert_type(options[:cli], Hash){'Key: cli'}
37
- Aspera.assert(options[:cli].key?(:type)){'Missing key: cli.type'}
38
- Aspera.assert_values(options[:cli][:type], CLI_OPTION_TYPES){"Unsupported processing type for #{name}"}
39
- # by default : optional
40
- options[:mandatory] ||= false
41
- options[:desc] ||= ''
42
- options[:desc] = "DEPRECATED: #{options[:deprecation]}\n#{options[:desc]}" if options.key?(:deprecation)
43
- cli = options[:cli]
44
- unsupported_cli_keys = cli.keys - CLI_KEYS
45
- Aspera.assert(unsupported_cli_keys.empty?){"Unsupported cli keys: #{unsupported_cli_keys}"}
46
59
  # by default : string, unless it's without arg
47
- options[:accepted_types] ||= options[:cli][:type].eql?(:opt_without_arg) ? :bool : :string
48
- # single type is placed in array
49
- options[:accepted_types] = [options[:accepted_types]] unless options[:accepted_types].is_a?(Array)
50
- # add default switch name if not present
51
- if !cli.key?(:switch) && cli.key?(:type) && CLI_OPTION_TYPE_SWITCH.include?(cli[:type])
52
- cli[:switch] = '--' + name.to_s.tr('_', '-')
53
- end
60
+ properties['type'] ||= properties['x-cli-switch'] ? 'boolean' : 'string'
61
+ # add default cli option name if not present, and if supported in "direct".
62
+ properties['x-cli-option'] = '--' + name.to_s.tr('_', '-') if !properties.key?('x-cli-option') && !properties['x-cli-envvar'] && (properties.key?('x-cli-switch') || supported_by_agent(CLI_AGENT, properties))
63
+ properties.freeze
54
64
  end
65
+ schema['required'] = [] unless schema.key?('required')
66
+ schema.freeze
55
67
  end
56
68
  end
57
69
 
58
- attr_reader :params_definition
59
-
60
- # @param [Hash] param_hash with parameters
61
- # @param [Hash] params_definition with definition of parameters
62
- def initialize(param_hash, params_definition)
63
- @param_hash = param_hash # keep reference so that it can be modified by caller before calling `process_params`
64
- @params_definition = params_definition
70
+ # @param [Hash] object with parameters
71
+ # @param [Hash] schema JSON schema
72
+ def initialize(object, schema, convert)
73
+ @object = object # keep reference so that it can be modified by caller before calling `process_params`
74
+ @schema = schema
75
+ @convert = convert
65
76
  @result = {
66
77
  env: {},
67
78
  args: []
68
79
  }
69
- @used_param_names = []
80
+ @processed_parameters = []
81
+ end
82
+
83
+ # Change required-ness of property in schema
84
+ def required(name, required)
85
+ if required
86
+ @schema['required'].push(name) unless @schema['required'].include?(name)
87
+ else
88
+ @schema['required'].delete(name)
89
+ end
70
90
  end
71
91
 
72
- # add processed parameters to env and args, warns about unused parameters
92
+ # Add processed parameters to env and args, warns about unused parameters
73
93
  # @param [Hash] env_args with :env and :args
74
94
  def add_env_args(env_args)
75
95
  Log.log.debug{"add_env_args: ENV=#{@result[:env]}, ARGS=#{@result[:args]}"}
76
96
  # warn about non translated arguments
77
- @param_hash.each_pair{|key, val|Log.log.warn{"unrecognized parameter: #{key} = \"#{val}\""} if !@used_param_names.include?(key)}
97
+ @object.each_pair do |name, value|
98
+ Log.log.warn{raise "Unknown transfer spec parameter: #{name} = \"#{value}\""} unless @processed_parameters.include?(name)
99
+ end
78
100
  # set result
79
101
  env_args[:env].merge!(@result[:env])
80
102
  env_args[:args].concat(@result[:args])
@@ -84,11 +106,11 @@ module Aspera
84
106
  # add options directly to command line
85
107
  def add_command_line_options(options)
86
108
  return if options.nil?
87
- options.each{|o|@result[:args].push(o.to_s)}
109
+ options.each{ |o| @result[:args].push(o.to_s)}
88
110
  end
89
111
 
90
112
  def process_params
91
- @params_definition.each_key do |k|
113
+ @schema['properties'].each_key do |k|
92
114
  process_param(k)
93
115
  end
94
116
  end
@@ -103,81 +125,80 @@ module Aspera
103
125
  # @param name [String] of parameter
104
126
  # @param read [TrueClass,FalseClass] read and return value of parameter instead of normal processing (for special)
105
127
  def process_param(name, read: false)
106
- options = @params_definition[name]
128
+ properties = @schema['properties'][name]
107
129
  # should not happen
108
- if options.nil?
130
+ if properties.nil?
109
131
  Log.log.warn{"Unknown parameter #{name}"}
110
132
  return
111
133
  end
112
- processing_type = read ? :get_value : options[:cli][:type]
113
- # check mandatory parameter (nil is valid value)
114
- raise Transfer::Error, "Missing mandatory parameter: #{name}" if options[:mandatory] && !@param_hash.key?(name)
115
- parameter_value = @param_hash[name]
134
+ # check mandatory parameter (nil is valid value), TODO: change exception ?
135
+ raise Transfer::Error, "Missing mandatory parameter: #{name}" if @schema['required'].include?(name) && !properties['x-cli-special'] && !@object.key?(name)
136
+ parameter_value = @object[name]
116
137
  # no default setting
117
- # parameter_value=options[:default] if parameter_value.nil? and options.has_key?(:default)
138
+ # parameter_value=properties['default'] if parameter_value.nil? and properties.has_key?('default')
118
139
  # Check parameter type
119
- expected_classes = options[:accepted_types].map do |type_symbol|
120
- case type_symbol
121
- when :string then String
122
- when :array then Array
123
- when :hash then Hash
124
- when :int then Integer
125
- when :bool then [TrueClass, FalseClass]
126
- else Aspera.error_unexpected_value(type_symbol)
127
- end
128
- end.flatten
140
+ expected_classes =
141
+ [properties['type']].flatten.map do |type|
142
+ case type
143
+ when 'string' then [String]
144
+ when 'array' then [Array]
145
+ when 'object' then [Hash]
146
+ when 'integer' then [Integer]
147
+ when 'boolean' then [TrueClass, FalseClass]
148
+ else Aspera.error_unexpected_value(properties['type'])
149
+ end
150
+ end.flatten
129
151
  # check that value is of expected type
130
- raise Transfer::Error, "#{name} is : #{parameter_value.class} (#{parameter_value}), shall be #{options[:accepted_types]}, " \
152
+ raise Transfer::Error, "#{name} is : #{parameter_value.class} (#{parameter_value}), shall be #{properties['type']}, " \
131
153
  unless parameter_value.nil? || expected_classes.include?(parameter_value.class)
132
154
  # special processing will be requested with type get_value
133
- @used_param_names.push(name) unless processing_type.eql?(:special)
155
+ @processed_parameters.push(name) if !properties['x-cli-special'] || read
134
156
 
135
157
  # process only non-nil values
136
158
  return nil if parameter_value.nil?
137
159
 
138
- # check that value is of an accepted type (string, int bool)
139
- raise "Enum value #{parameter_value} is not allowed for #{name}" if options.key?(:enum) && !options[:enum].include?(parameter_value)
160
+ # check that value is of an accepted type (string, integer, boolean)
161
+ raise "Enum value #{parameter_value} is not allowed for #{name}" if properties.key?('enum') && !properties['enum'].include?(parameter_value)
140
162
 
141
163
  # convert some values if value on command line needs processing from value in structure
142
- case options[:cli][:convert]
143
- when Hash
144
- # translate using conversion table
145
- new_value = options[:cli][:convert][parameter_value]
146
- raise "unsupported value: #{parameter_value}, expect: #{options[:cli][:convert].keys.join(', ')}" if new_value.nil?
147
- parameter_value = new_value
148
- when String
149
- # :convert has name of class and encoding method
150
- conversion_class, conversion_method = options[:cli][:convert].split('.')
151
- converted_value = Kernel.const_get(conversion_class).send(conversion_method, parameter_value)
152
- raise Transfer::Error, "unsupported #{name}: #{parameter_value}" if converted_value.nil?
164
+ if (convert = properties['x-cli-convert'])
165
+ converted_value =
166
+ case convert
167
+ when Hash then convert[parameter_value]
168
+ when String then @convert.send(convert, parameter_value)
169
+ else Aspera.error_unexpected_value(convert){"Conversion type for #{name} is Hash or String only."}
170
+ end
171
+ raise "No conversion for: #{name}=#{parameter_value}" if converted_value.nil?
153
172
  parameter_value = converted_value
154
- when NilClass
155
- else Aspera.error_unexpected_value(options[:cli][:convert].class)
156
173
  end
157
174
 
158
- case processing_type
159
- when :get_value # just get value (deferred)
175
+ return unless self.class.supported_by_agent(CLI_AGENT, properties)
176
+
177
+ if read
178
+ # just get value (deferred)
160
179
  return parameter_value
161
- when :ignore, :special # ignore this parameter or process later
180
+ elsif properties['x-cli-special']
181
+ # process later
162
182
  return
163
- when :envvar # set in env var
164
- Aspera.assert(options[:cli].key?(:variable)){'missing key: cli.variable'}
165
- @result[:env][options[:cli][:variable]] = parameter_value
166
- when :opt_without_arg # if present and true : just add option without value
183
+ elsif properties.key?('x-cli-envvar')
184
+ # set in env var
185
+ @result[:env][properties['x-cli-envvar']] = parameter_value
186
+ elsif properties['x-cli-switch']
187
+ # if present and true : just add option without value
167
188
  add_param = false
168
189
  case parameter_value
169
190
  when false then nil # nothing to put on command line, no creation by default
170
191
  when true then add_param = true
171
192
  else Aspera.error_unexpected_value(parameter_value){name}
172
193
  end
173
- add_param = !add_param if options[:add_on_false]
174
- add_command_line_options([options[:cli][:switch]]) if add_param
175
- when :opt_with_arg # transform into command line option with value
194
+ # add_param = !add_param if properties[:add_on_false]
195
+ add_command_line_options([properties['x-cli-option']]) if add_param
196
+ else
197
+ # transform into command line option with value
176
198
  # parameter_value=parameter_value.to_s if parameter_value.is_a?(Integer)
177
199
  parameter_value = [parameter_value] unless parameter_value.is_a?(Array)
178
200
  # if transfer_spec value is an array, applies option many times
179
- parameter_value.each{|v|add_command_line_options([options[:cli][:switch], v])}
180
- else Aspera.error_unexpected_value(processing_type){processing_type.class.name}
201
+ parameter_value.each{ |v| add_command_line_options([properties['x-cli-option'], v])}
181
202
  end
182
203
  end
183
204
  end
@@ -5,7 +5,7 @@ if ENV.key?('ENABLE_COVERAGE')
5
5
  require 'simplecov'
6
6
  require 'securerandom'
7
7
  # compute development top folder based on this source location
8
- development_root = 3.times.inject(File.realpath(__FILE__)) { |p, _| File.dirname(p) }
8
+ development_root = 3.times.inject(File.realpath(__FILE__)){ |p, _| File.dirname(p)}
9
9
  SimpleCov.root(development_root)
10
10
  SimpleCov.enable_for_subprocesses if SimpleCov.respond_to?(:enable_for_subprocesses)
11
11
  # keep cache data for 1 day (must be longer that time to run the whole test suite)
@@ -20,14 +20,17 @@ if ENV.key?('ENABLE_COVERAGE')
20
20
  # lines with those words are ignored from coverage
21
21
  no_cov_functions = %w[error_unreachable_line error_unexpected_value Log.log.trace].freeze
22
22
  SimpleCov.start do
23
- add_filter 'lib/aspera/cli/plugins/faspex.rb'
24
- add_filter 'lib/aspera/node_simulator.rb'
25
- add_filter 'lib/aspera/keychain/macos_security.rb'
23
+ # assert usually do not trigger
26
24
  add_filter do |source_file|
27
25
  source_file.lines.each do |line|
28
- line.skipped! if no_cov_functions.any?{|i|line.src.include?(i)}
26
+ line.skipped! if no_cov_functions.any?{ |i| line.src.include?(i)}
29
27
  end
30
28
  false
31
29
  end
30
+ # no coverage in these files
31
+ add_filter 'lib/aspera/cli/plugins/faspex.rb'
32
+ add_filter 'lib/aspera/node_simulator.rb'
33
+ add_filter 'lib/aspera/keychain/macos_security.rb'
34
+ add_filter 'lib/aspera/assert.rb'
32
35
  end
33
36
  end
@@ -103,29 +103,34 @@ module Aspera
103
103
  def log_spawn(exec:, args: nil, env: nil)
104
104
  [
105
105
  'execute:'.red,
106
- env&.map{|k, v| "#{k}=#{Shellwords.shellescape(v)}"},
106
+ env&.map{ |k, v| "#{k}=#{Shellwords.shellescape(v)}"},
107
107
  Shellwords.shellescape(exec),
108
- args&.map{|a|Shellwords.shellescape(a)}
108
+ args&.map{ |a| Shellwords.shellescape(a)}
109
109
  ].compact.flatten.join(' ')
110
110
  end
111
111
 
112
- # start process in background, or raise exception
112
+ # Start process in background
113
113
  # caller can call Process.wait on returned value
114
- # @param env [Hash, nil] environment variables
115
- # @param exec [String] path to executable
116
- # @param args [Array, nil] arguments
117
- # @return [String] PID of process
118
- def secure_spawn(exec:, args: nil, env: nil)
114
+ # @param exec [String] path to executable
115
+ # @param args [Array, nil] arguments for executable
116
+ # @param env [Hash, nil] environment variables
117
+ # @param options [Hash, nil] spawn options
118
+ # @return [String] PID of process
119
+ # @raise [Exception] if problem
120
+ def secure_spawn(exec:, args: nil, env: nil, **options)
119
121
  Aspera.assert_type(exec, String)
120
122
  Aspera.assert_type(args, Array) unless args.nil?
121
123
  Aspera.assert_type(env, Hash) unless env.nil?
122
- Log.log.debug {log_spawn(exec: exec, args: args, env: env)}
124
+ Aspera.assert_type(options, Hash) unless options.nil?
125
+ Log.log.debug{log_spawn(exec: exec, args: args, env: env)}
123
126
  # start ascp in separate process
124
127
  spawn_args = []
125
128
  spawn_args.push(env) unless env.nil?
126
129
  spawn_args.push([exec, exec])
127
130
  spawn_args.concat(args) unless args.nil?
128
- ascp_pid = Process.spawn(*spawn_args, close_others: true)
131
+ opts = {close_others: true}
132
+ opts.merge!(options) unless options.nil?
133
+ ascp_pid = Process.spawn(*spawn_args, **opts)
129
134
  Log.log.debug{"pid: #{ascp_pid}"}
130
135
  return ascp_pid
131
136
  end
@@ -135,29 +140,33 @@ module Aspera
135
140
  # @param exec [String] path to executable
136
141
  # @param args [Array, nil] arguments
137
142
  # @return [String] PID of process
138
- def secure_execute(exec:, args: nil, env: nil)
143
+ def secure_execute(exec:, args: nil, env: nil, **system_args)
139
144
  Aspera.assert_type(exec, String)
140
145
  Aspera.assert_type(args, Array) unless args.nil?
141
146
  Aspera.assert_type(env, Hash) unless env.nil?
142
- Log.log.debug {log_spawn(exec: exec, args: args, env: env)}
143
- # start ascp in separate process
147
+ Log.log.debug{log_spawn(exec: exec, args: args, env: env)}
148
+ # start in separate process
144
149
  spawn_args = []
145
150
  spawn_args.push(env) unless env.nil?
151
+ # ensure no shell expansion
146
152
  spawn_args.push([exec, exec])
147
153
  spawn_args.concat(args) unless args.nil?
148
- Kernel.system(*spawn_args, exception: true)
154
+ kwargs = {exception: true}
155
+ kwargs.merge!(system_args)
156
+ Kernel.system(*spawn_args, **kwargs)
149
157
  nil
150
158
  end
151
159
 
160
+ # Execute process and capture stdout
152
161
  # @param exec [String] path to executable
153
162
  # @param args [Array] arguments to executable
154
163
  # @param opts [Hash] options to capture3
155
- # @return stdout of executable or raise expcetion
164
+ # @return stdout of executable or raise exception
156
165
  def secure_capture(exec:, args: [], **opts)
157
166
  Aspera.assert_type(exec, String)
158
167
  Aspera.assert_type(args, Array)
159
168
  Aspera.assert_type(opts, Hash)
160
- Log.log.debug {log_spawn(exec: exec, args: args)}
169
+ Log.log.debug{log_spawn(exec: exec, args: args)}
161
170
  stdout, stderr, status = Open3.capture3(exec, *args, **opts)
162
171
  Log.log.debug{"status=#{status}, stderr=#{stderr}"}
163
172
  Log.log.trace1{"stdout=#{stdout}"}
@@ -247,7 +256,7 @@ module Aspera
247
256
  # https://www.gnu.org/software/libc/manual/html_node/Locale-Categories.html
248
257
  # https://pubs.opengroup.org/onlinepubs/7908799/xbd/envvar.html
249
258
  def terminal_supports_unicode?
250
- @terminal_supports_unicode = self.class.terminal? && %w(LC_ALL LC_CTYPE LANG).any?{|var|ENV[var]&.include?('UTF-8')} if @terminal_supports_unicode.nil?
259
+ @terminal_supports_unicode = self.class.terminal? && %w(LC_ALL LC_CTYPE LANG).any?{ |var| ENV[var]&.include?('UTF-8')} if @terminal_supports_unicode.nil?
251
260
  return @terminal_supports_unicode
252
261
  end
253
262
 
@@ -8,8 +8,8 @@ require 'json'
8
8
  module Aspera
9
9
  # Simulate the Faspex 4 /send API and creates a package on Aspera on Cloud or Faspex 5
10
10
  class Faspex4GWServlet < WEBrick::HTTPServlet::AbstractServlet
11
- # @param app_api
12
- # @param app_context [String]
11
+ # @param app_api [Rest] API object
12
+ # @param app_context [String] workspace id (aoc only)
13
13
  def initialize(server, app_api, app_context)
14
14
  Aspera.assert_values(app_api.class.name, ['Aspera::Api::AoC', 'Aspera::Rest'])
15
15
  super(server)
@@ -32,7 +32,7 @@ module Aspera
32
32
  created_package = @app_api.create_package_simple(package_data, true, nil)
33
33
  # but we place it in a Faspex package creation response
34
34
  return {
35
- 'links' => { 'status' => 'unused' },
35
+ 'links' => {'status' => 'unused'},
36
36
  'xfer_sessions' => [created_package[:spec]]
37
37
  }
38
38
  end
@@ -42,22 +42,22 @@ module Aspera
42
42
  package_data = {
43
43
  'title' => faspex_pkg_delivery['title'],
44
44
  'note' => faspex_pkg_delivery['note'],
45
- 'recipients' => faspex_pkg_delivery['recipients'].map{|name|{'name'=>name}}
45
+ 'recipients' => faspex_pkg_delivery['recipients'].map{ |name| {'name'=>name}}
46
46
  }
47
47
  package = @app_api.create('packages', package_data)
48
48
  # TODO: option to send from remote source or httpgw
49
49
  transfer_spec = @app_api.call(
50
- operation: 'POST',
51
- subpath: "packages/#{package['id']}/transfer_spec/upload",
52
- headers: {'Accept' => 'application/json'},
53
- query: {transfer_type: Cli::Plugins::Faspex5::TRANSFER_CONNECT},
54
- body: {paths: [{'destination'=>'/'}]},
55
- body_type: :json
50
+ operation: 'POST',
51
+ subpath: "packages/#{package['id']}/transfer_spec/upload",
52
+ query: {transfer_type: Cli::Plugins::Faspex5::TRANSFER_CONNECT},
53
+ content_type: Rest::MIME_JSON,
54
+ body: {paths: [{'destination'=>'/'}]},
55
+ headers: {'Accept' => Rest::MIME_JSON}
56
56
  )[:data]
57
57
  transfer_spec.delete('authentication')
58
58
  # but we place it in a Faspex package creation response
59
59
  return {
60
- 'links' => { 'status' => 'unused' },
60
+ 'links' => {'status' => 'unused'},
61
61
  'xfer_sessions' => [transfer_spec]
62
62
  }
63
63
  end
@@ -80,18 +80,18 @@ module Aspera
80
80
  end
81
81
  Log.log.info{"faspex_package_create_result=#{faspex_package_create_result}"}
82
82
  response.status = 200
83
- response.content_type = 'application/json'
83
+ response.content_type = Rest::MIME_JSON
84
84
  response.body = JSON.generate(faspex_package_create_result)
85
85
  rescue => e
86
86
  response.status = 500
87
- response['Content-Type'] = 'application/json'
87
+ response['Content-Type'] = Rest::MIME_JSON
88
88
  response.body = {error: e.message, stacktrace: e.backtrace}.to_json
89
89
  Log.log.error(e.message)
90
90
  Log.log.debug{e.backtrace.join("\n")}
91
91
  end
92
92
  else
93
93
  response.status = 400
94
- response['Content-Type'] = 'application/json'
94
+ response['Content-Type'] = Rest::MIME_JSON
95
95
  response.body = {error: 'Unsupported endpoint'}.to_json
96
96
  end
97
97
  end
@@ -15,7 +15,8 @@ module Aspera
15
15
  Aspera.assert_type(parameters, Hash)
16
16
  @parameters = parameters.symbolize_keys
17
17
  Log.log.debug{Log.dump(:post_proc_parameters, @parameters)}
18
- raise "unexpected key in parameters config: only: #{ALLOWED_PARAMETERS.join(', ')}" if @parameters.keys.any?{|k|!ALLOWED_PARAMETERS.include?(k)}
18
+ not_allowed = @parameters.keys - ALLOWED_PARAMETERS
19
+ raise "unsupported parameters: #{not_allowed.join(', ')}" unless not_allowed.empty?
19
20
  @parameters[:script_folder] ||= '.'
20
21
  @parameters[:fail_on_error] ||= false
21
22
  @parameters[:timeout_seconds] ||= 60
@@ -29,17 +30,17 @@ module Aspera
29
30
  # only accept requests on the root
30
31
  if !request.path.start_with?(@parameters[:root])
31
32
  response.status = 400
32
- response['Content-Type'] = 'application/json'
33
+ response['Content-Type'] = Rest::MIME_JSON
33
34
  response.body = {status: 'error', message: 'Request outside domain'}.to_json
34
35
  return
35
36
  end
36
37
  if request.body.nil?
37
38
  response.status = 400
38
- response['Content-Type'] = 'application/json'
39
+ response['Content-Type'] = Rest::MIME_JSON
39
40
  response.body = {status: 'error', message: 'Empty request'}.to_json
40
41
  return
41
42
  end
42
- # build script path by removing domain, and adding script folder
43
+ # build script path by removing domain and adding script folder
43
44
  script_file = request.path[@parameters[:root].size..]
44
45
  Log.log.debug{"script file=#{script_file}"}
45
46
  script_path = File.join(@parameters[:script_folder], script_file)
@@ -47,19 +48,17 @@ module Aspera
47
48
  webhook_parameters = JSON.parse(request.body)
48
49
  Log.log.debug{Log.dump(:webhook_parameters, webhook_parameters)}
49
50
  # env expects only strings
50
- environment = webhook_parameters.each_with_object({}) { |(k, v), h| h[k] = v.to_s }
51
- post_proc_pid = Process.spawn(environment, [script_path, script_path])
52
- Log.log.debug{"pid=#{post_proc_pid}"}
53
- raise 'no pid' if post_proc_pid.nil?
54
- # "wait" for process to avoid zombie
51
+ environment = webhook_parameters.each_with_object({}){ |(k, v), h| h[k] = v.to_s}
52
+ post_proc_pid = Environment.secure_spawn(env: environment, exec: script_path)
55
53
  Timeout.timeout(@parameters[:timeout_seconds]) do
54
+ # "wait" for process to avoid zombie
56
55
  Process.wait(post_proc_pid)
57
56
  post_proc_pid = nil
58
57
  end
59
58
  process_status = $CHILD_STATUS
60
59
  raise "script #{script_path} failed with code #{process_status.exitstatus}" if !process_status.success? && @parameters[:fail_on_error]
61
60
  response.status = 200
62
- response.content_type = 'application/json'
61
+ response.content_type = Rest::MIME_JSON
63
62
  response.body = JSON.generate({status: 'success', script: script_path, exit_code: process_status.exitstatus})
64
63
  Log.log.debug{'Script executed successfully'}
65
64
  rescue => e
@@ -70,7 +69,7 @@ module Aspera
70
69
  Log.log.error("Killed process: #{post_proc_pid}")
71
70
  end
72
71
  response.status = 500
73
- response['Content-Type'] = 'application/json'
72
+ response['Content-Type'] = Rest::MIME_JSON
74
73
  response.body = {status: 'error', script: script_path, message: e.message}.to_json
75
74
  end
76
75
  end
@@ -2,11 +2,11 @@
2
2
 
3
3
  class ::Hash
4
4
  def deep_merge(second)
5
- merge(second){|_key, v1, v2|v1.is_a?(Hash) && v2.is_a?(Hash) ? v1.deep_merge(v2) : v2}
5
+ merge(second){ |_key, v1, v2| v1.is_a?(Hash) && v2.is_a?(Hash) ? v1.deep_merge(v2) : v2}
6
6
  end
7
7
 
8
8
  def deep_merge!(second)
9
- merge!(second){|_key, v1, v2|v1.is_a?(Hash) && v2.is_a?(Hash) ? v1.deep_merge!(v2) : v2}
9
+ merge!(second){ |_key, v1, v2| v1.is_a?(Hash) && v2.is_a?(Hash) ? v1.deep_merge!(v2) : v2}
10
10
  end
11
11
 
12
12
  def deep_do(memory=nil, &block)
@@ -20,17 +20,7 @@ class ::Hash
20
20
  end
21
21
  end
22
22
 
23
- # in 2.5
24
- unless Hash.method_defined?(:transform_keys)
25
- class Hash
26
- def transform_keys
27
- raise 'missing block' unless block_given?
28
- return each_with_object({}){|(k, v), memo|memo[yield(k)] = v}
29
- end
30
- end
31
- end
32
-
33
- # rails
23
+ # Exists in Rails
34
24
  unless Hash.method_defined?(:symbolize_keys)
35
25
  class Hash
36
26
  def symbolize_keys
@@ -39,7 +29,7 @@ unless Hash.method_defined?(:symbolize_keys)
39
29
  end
40
30
  end
41
31
 
42
- # rails
32
+ # Exists in Rails
43
33
  unless Hash.method_defined?(:stringify_keys)
44
34
  class Hash
45
35
  def stringify_keys
@@ -23,7 +23,7 @@ module Aspera
23
23
  @request_id = 0
24
24
  end
25
25
 
26
- def respond_to_missing?(sym, include_private = false)
26
+ def respond_to_missing?(_sym, _include_private = false)
27
27
  true
28
28
  end
29
29