aspera-cli 4.22.0 → 4.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +374 -364
  4. data/README.md +255 -155
  5. data/lib/aspera/agent/direct.rb +1 -1
  6. data/lib/aspera/api/aoc.rb +9 -12
  7. data/lib/aspera/api/httpgw.rb +8 -4
  8. data/lib/aspera/ascmd.rb +14 -6
  9. data/lib/aspera/ascp/installation.rb +6 -3
  10. data/lib/aspera/assert.rb +3 -3
  11. data/lib/aspera/cli/hints.rb +9 -1
  12. data/lib/aspera/cli/main.rb +1 -1
  13. data/lib/aspera/cli/manager.rb +1 -1
  14. data/lib/aspera/cli/plugin.rb +1 -1
  15. data/lib/aspera/cli/plugins/aoc.rb +33 -23
  16. data/lib/aspera/cli/plugins/config.rb +20 -15
  17. data/lib/aspera/cli/plugins/node.rb +96 -92
  18. data/lib/aspera/cli/plugins/server.rb +1 -0
  19. data/lib/aspera/cli/transfer_agent.rb +7 -11
  20. data/lib/aspera/cli/version.rb +1 -1
  21. data/lib/aspera/data_repository.rb +1 -0
  22. data/lib/aspera/environment.rb +1 -0
  23. data/lib/aspera/log.rb +1 -0
  24. data/lib/aspera/oauth/base.rb +2 -0
  25. data/lib/aspera/oauth/factory.rb +1 -0
  26. data/lib/aspera/preview/file_types.rb +40 -33
  27. data/lib/aspera/preview/generator.rb +1 -1
  28. data/lib/aspera/products/connect.rb +1 -0
  29. data/lib/aspera/rest.rb +18 -7
  30. data/lib/aspera/rest_error_analyzer.rb +1 -0
  31. data/lib/aspera/ssh.rb +1 -1
  32. data/lib/aspera/temp_file_manager.rb +1 -0
  33. data/lib/aspera/timer_limiter.rb +7 -5
  34. data/lib/aspera/transfer/async_conf.schema.yaml +716 -0
  35. data/lib/aspera/transfer/sync.rb +14 -4
  36. data/lib/aspera/transfer/sync_instance.schema.yaml +7 -0
  37. data/lib/aspera/transfer/sync_session.schema.yaml +7 -0
  38. data.tar.gz.sig +0 -0
  39. metadata +3 -5
  40. metadata.gz.sig +0 -0
  41. data/examples/dascli +0 -30
  42. data/examples/get_proto_file.rb +0 -8
  43. data/examples/proxy.pac +0 -60
@@ -21,6 +21,7 @@ module Aspera
21
21
  module Plugins
22
22
  class Node < Cli::BasicAuthPlugin
23
23
  include SyncActions
24
+
24
25
  class << self
25
26
  # directory: node, container: shares
26
27
  FOLDER_TYPES = %w[directory container].freeze
@@ -849,7 +850,6 @@ module Aspera
849
850
  case command
850
851
  when :list
851
852
  transfer_filter = query_read_delete(default: {})
852
- last_iteration_token = nil
853
853
  iteration_persistency = nil
854
854
  if options.get_option(:once_only, mandatory: true)
855
855
  iteration_persistency = PersistencyActionOnce.new(
@@ -865,35 +865,10 @@ module Aspera
865
865
  iteration_persistency.save
866
866
  return Main.result_status('Persistency reset')
867
867
  end
868
- last_iteration_token = iteration_persistency.data.first
869
868
  end
870
869
  raise Cli::BadArgument, 'reset only with once_only' if transfer_filter.key?('reset') && iteration_persistency.nil?
871
870
  max_items = transfer_filter.delete(MAX_ITEMS)
872
- transfers_data = []
873
- loop do
874
- transfer_filter['iteration_token'] = last_iteration_token unless last_iteration_token.nil?
875
- result = @api_node.call(operation: 'GET', subpath: 'ops/transfers', query: transfer_filter)
876
- # no data
877
- break if result[:data].empty?
878
- # get next iteration token from link
879
- next_iteration_token = nil
880
- link_info = result[:http]['Link']
881
- unless link_info.nil?
882
- m = link_info.match(/<([^>]+)>/)
883
- raise "Cannot parse iteration in Link: #{link_info}" if m.nil?
884
- next_iteration_token = CGI.parse(URI.parse(m[1]).query)['iteration_token']&.first
885
- end
886
- # same as last iteration: stop
887
- break if next_iteration_token&.eql?(last_iteration_token)
888
- last_iteration_token = next_iteration_token
889
- transfers_data.concat(result[:data])
890
- if max_items&.<=(transfers_data.length)
891
- transfers_data = transfers_data.slice(0, max_items)
892
- break
893
- end
894
- break if last_iteration_token.nil?
895
- end
896
- iteration_persistency&.data&.[]=(0, last_iteration_token)
871
+ transfers_data = call_with_iteration(api: @api_node, operation: 'GET', subpath: 'ops/transfers', max: max_items, query: transfer_filter, iteration: iteration_persistency&.data)
897
872
  iteration_persistency&.save
898
873
  return {
899
874
  type: :object_list,
@@ -1077,7 +1052,7 @@ module Aspera
1077
1052
  return Main.result_status('Simulator terminated')
1078
1053
  when :telemetry
1079
1054
  parameters = value_create_modify(command: command, default: {}).symbolize_keys
1080
- %i[url apikey].each do |psym|
1055
+ %i[url key].each do |psym|
1081
1056
  raise Cli::BadArgument, "Missing parameter: #{psym}" unless parameters.key?(psym)
1082
1057
  end
1083
1058
  require 'socket'
@@ -1085,79 +1060,69 @@ module Aspera
1085
1060
  parameters[:hostname] = Socket.gethostname unless parameters.key?(:hostname)
1086
1061
  interval = parameters[:interval].to_f
1087
1062
  raise Cli::BadArgument, 'Interval must be a positive number in seconds' if interval <= 0
1088
- backend_api = Rest.new(
1063
+ otel_api = Rest.new(
1089
1064
  base_url: "#{parameters[:url]}/v1",
1090
1065
  headers: {
1091
- # 'Authorization' => "apiToken #{parameters[:apikey]}",
1092
- 'x-instana-key' => parameters[:apikey],
1066
+ # 'Authorization' => "apiToken #{parameters[:key]}",
1067
+ 'x-instana-key' => parameters[:key],
1093
1068
  'x-instana-host' => parameters[:hostname]
1094
1069
  }
1095
1070
  )
1096
-
1097
- loop do
1098
- start_time = Time.now
1099
- transfer_filter = {active_only: true}
1100
- transfers_data = []
1101
- loop do
1102
- result = @api_node.call(operation: 'GET', subpath: 'ops/transfers', query: transfer_filter)
1103
- # no data
1104
- break if result[:data].empty?
1105
- # get next iteration token from link
1106
- next_iteration_token = nil
1107
- link_info = result[:http]['Link']
1108
- unless link_info.nil?
1109
- m = link_info.match(/<([^>]+)>/)
1110
- Aspera.assert(m){"Cannot parse iteration in Link: #{link_info}"}
1111
- next_iteration_token = CGI.parse(URI.parse(m[1]).query)['iteration_token']&.first
1112
- end
1113
- # same as last iteration: stop
1114
- break if next_iteration_token&.eql?(transfer_filter[:iteration_token])
1115
- transfer_filter[:iteration_token] = next_iteration_token
1116
- transfers_data.concat(result[:data])
1117
- break if next_iteration_token.nil?
1118
- end
1119
- puts("#{transfers_data.length} active transfers")
1120
- epoch_nsec = start_time.to_i * 1_000_000_000 + start_time.nsec
1121
- # https://www.ibm.com/docs/en/instana-observability/current?topic=instana-backend
1122
- backend_api.create('metrics', {
1123
- resourceMetrics: [
1124
- {
1125
- resource: {
1126
- attributes: [
1071
+ datapoint = {
1072
+ attributes: [
1073
+ {
1074
+ key: 'server.name',
1075
+ value: {
1076
+ stringValue: 'HSTS1'
1077
+ }
1078
+ }
1079
+ ],
1080
+ asInt: nil,
1081
+ timeUnixNano: nil
1082
+ }
1083
+ # https://opentelemetry.io/docs/specs/otel/metrics/data-model/#gauge
1084
+ metrics = {
1085
+ resourceMetrics: [
1086
+ {
1087
+ resource: {
1088
+ attributes: [
1089
+ {
1090
+ key: 'service.name',
1091
+ value: {
1092
+ stringValue: 'IBMAspera'
1093
+ }
1094
+ }
1095
+ ]
1096
+ },
1097
+ scopeMetrics: [
1098
+ {
1099
+ metrics: [
1127
1100
  {
1128
- key: 'service.name',
1129
- value: {
1130
- stringValue: 'mycurl5'
1101
+ name: 'active.transfers',
1102
+ description: 'Number of active transfers',
1103
+ unit: '1',
1104
+ gauge: {
1105
+ dataPoints: [
1106
+ datapoint
1107
+ ]
1131
1108
  }
1132
1109
  }
1133
1110
  ]
1134
- },
1135
- scopeMetrics: [
1136
- {
1137
- metrics: [
1138
- {
1139
- name: 'tutur2',
1140
- unit: '1',
1141
- description: '',
1142
- sum: {
1143
- aggregationTemporality: 1,
1144
- isMonotonic: true,
1145
- dataPoints: [
1146
- {
1147
- asDouble: 4,
1148
- startTimeUnixNano: epoch_nsec,
1149
- timeUnixNano: epoch_nsec
1150
- }
1151
- ]
1152
- }
1153
- }
1154
- ]
1155
- }
1156
- ]
1157
- }
1158
- ]
1159
- })
1160
- sleep([0, interval - (Time.now - start_time)].max)
1111
+ }
1112
+ ]
1113
+ }
1114
+ ]
1115
+ }
1116
+ loop do
1117
+ timestamp = Time.now
1118
+ transfers_data = call_with_iteration(api: @api_node, operation: 'GET', subpath: 'ops/transfers', query: {active_only: true})
1119
+ datapoint[:asInt] = transfers_data.length
1120
+ datapoint[:timeUnixNano] = timestamp.to_i * 1_000_000_000 + timestamp.nsec
1121
+ Log.log.info("#{datapoint[:asInt]} active transfers")
1122
+ # https://www.ibm.com/docs/en/instana-observability/current?topic=instana-backend
1123
+ otel_api.create('metrics', metrics)
1124
+ break if interval.eql?(0.0)
1125
+ sleep([0.0, interval - (Time.now - timestamp)].max)
1161
1126
  end
1162
1127
  end
1163
1128
  Aspera.error_unreachable_line
@@ -1178,6 +1143,45 @@ module Aspera
1178
1143
  return path_arg if @prefix_path.nil?
1179
1144
  return File.join(@prefix_path, path_arg)
1180
1145
  end
1146
+
1147
+ # Executes the provided API call in loop
1148
+ # @param api [Rest] the API to call
1149
+ # @param iteration [Array] a single element array with the iteration token or nil
1150
+ # @param max [Integer] maximum number of items to return, or nil for no limit
1151
+ # @param query [Hash] query parameters to use for the API call
1152
+ # @param call_args [Hash] additional arguments to pass to the API call
1153
+ # @return [Array] list of items returned by the API call
1154
+ def call_with_iteration(api:, iteration: nil, max: nil, query: nil, **call_args)
1155
+ query_token = query.clone || {}
1156
+ item_list = []
1157
+ query_token[:iteration_token] = iteration.first if iteration.is_a?(Array)
1158
+ loop do
1159
+ result = api.call(**call_args, query: query_token)
1160
+ Aspera.assert_type(result[:data], Array){"Expected data to be an Array, got: #{result[:data].class}"}
1161
+ # no data
1162
+ break if result[:data].empty?
1163
+ # get next iteration token from link
1164
+ next_iteration_token = nil
1165
+ link_info = result[:http]['Link']
1166
+ unless link_info.nil?
1167
+ m = link_info.match(/<([^>]+)>/)
1168
+ Aspera.assert(m){"Cannot parse iteration in Link: #{link_info}"}
1169
+ next_iteration_token = CGI.parse(URI.parse(m[1]).query)['iteration_token']&.first
1170
+ end
1171
+ # same as last iteration: stop
1172
+ break if next_iteration_token&.eql?(query_token[:iteration_token])
1173
+ query_token[:iteration_token] = next_iteration_token
1174
+ item_list.concat(result[:data])
1175
+ if max&.<=(item_list.length)
1176
+ item_list = item_list.slice(0, max)
1177
+ break
1178
+ end
1179
+ break if next_iteration_token.nil?
1180
+ end
1181
+ # save iteration token if needed
1182
+ iteration[0] = query_token[:iteration_token] unless iteration.nil?
1183
+ item_list
1184
+ end
1181
1185
  end
1182
1186
  end
1183
1187
  end
@@ -19,6 +19,7 @@ module Aspera
19
19
  # implement basic remote access with FASP/SSH
20
20
  class Server < Cli::BasicAuthPlugin
21
21
  include SyncActions
22
+
22
23
  SSH_SCHEME = 'ssh'
23
24
  LOCAL_SCHEME = 'local'
24
25
  HTTPS_SCHEME = 'https'
@@ -88,14 +88,6 @@ module Aspera
88
88
  # add other transfer spec parameters
89
89
  def option_transfer_spec_deep_merge(ts); @transfer_spec_command_line.deep_merge!(ts); end
90
90
 
91
- # @return [Hash] transfer spec with updated values from command line, including removed values
92
- def updated_ts(transfer_spec={})
93
- transfer_spec.deep_merge!(@transfer_spec_command_line)
94
- # recursively remove values that are nil (user wants to delete)
95
- transfer_spec.deep_do{ |hash, key, value, _unused| hash.delete(key) if value.nil?}
96
- return transfer_spec
97
- end
98
-
99
91
  attr_reader :transfer_info
100
92
 
101
93
  # multiple option are merged
@@ -173,9 +165,10 @@ module Aspera
173
165
 
174
166
  # This is how the list of files to be transferred is specified
175
167
  # get paths suitable for transfer spec from command line
168
+ # @param default [String] if set, used as default file for --sources=@args
176
169
  # @return [Hash] {source: (mandatory), destination: (optional)}
177
170
  # computation is done only once, cache is kept in @transfer_paths
178
- def ts_source_paths
171
+ def ts_source_paths(default: nil)
179
172
  # return cache if set
180
173
  return @transfer_paths unless @transfer_paths.nil?
181
174
  # start with lower priority : get paths from transfer spec on command line
@@ -185,8 +178,9 @@ module Aspera
185
178
  case file_list
186
179
  when nil, FILE_LIST_FROM_ARGS
187
180
  Log.log.debug('getting file list as parameters')
181
+ Aspera.assert_type(default, Array) unless default.nil?
188
182
  # get remaining arguments
189
- file_list = @opt_mgr.get_next_argument('source file list', multiple: true)
183
+ file_list = @opt_mgr.get_next_argument('source file list', multiple: true, default: default)
190
184
  raise Cli::BadArgument, 'specify at least one file on command line or use ' \
191
185
  "--sources=#{FILE_LIST_FROM_TRANSFER_SPEC} to use transfer spec" if !file_list.is_a?(Array) || file_list.empty?
192
186
  when FILE_LIST_FROM_TRANSFER_SPEC
@@ -252,7 +246,9 @@ module Aspera
252
246
  # update command line paths, unless destination already has one
253
247
  @transfer_spec_command_line['paths'] = transfer_spec['paths'] || ts_source_paths
254
248
  # updated transfer spec with command line
255
- updated_ts(transfer_spec)
249
+ transfer_spec.deep_merge!(@transfer_spec_command_line)
250
+ # recursively remove values that are nil (user wants to delete)
251
+ transfer_spec.deep_do{ |hash, key, value, _unused| hash.delete(key) if value.nil?}
256
252
  # if TS from app has content_protection (e.g. F5), that means content is protected: ask password if not provided
257
253
  if transfer_spec['content_protection'].eql?('decrypt') && !transfer_spec.key?('content_protection_password')
258
254
  transfer_spec['content_protection_password'] = @opt_mgr.prompt_user_input('content protection password', sensitive: true)
@@ -4,6 +4,6 @@ module Aspera
4
4
  module Cli
5
5
  # for beta add extension : .beta1
6
6
  # for dev version add extension : .pre
7
- VERSION = '4.22.0'
7
+ VERSION = '4.23.0'
8
8
  end
9
9
  end
@@ -8,6 +8,7 @@ module Aspera
8
8
  # a simple binary data repository
9
9
  class DataRepository
10
10
  include Singleton
11
+
11
12
  # in same order as elements in folder
12
13
  ELEMENTS = %i[dsa rsa uuid aspera.global-cli-client aspera.drive license]
13
14
  START_INDEX = 1
@@ -13,6 +13,7 @@ module Aspera
13
13
  # detect OS, architecture, and specific stuff
14
14
  class Environment
15
15
  include Singleton
16
+
16
17
  USER_INTERFACES = %i[text graphical].freeze
17
18
 
18
19
  OS_WINDOWS = :windows
data/lib/aspera/log.rb CHANGED
@@ -48,6 +48,7 @@ module Aspera
48
48
  # Singleton object for logging
49
49
  class Log
50
50
  include Singleton
51
+
51
52
  # Where logs are sent to
52
53
  LOG_TYPES = %i[stderr stdout syslog].freeze
53
54
  @@format = :json # rubocop:disable Style/ClassVars
@@ -54,6 +54,8 @@ module Aspera
54
54
  @token_cache_id = Factory.cache_id(@api.base_url, self.class, @base_cache_ids, @scope)
55
55
  end
56
56
 
57
+ attr_reader :scope
58
+
57
59
  # helper method to create token as per RFC
58
60
  def create_token_call(creation_params)
59
61
  Log.log.debug{'Generating a new token'.bg_green}
@@ -9,6 +9,7 @@ module Aspera
9
9
  # Factory to create tokens and manage their cache
10
10
  class Factory
11
11
  include Singleton
12
+
12
13
  # a prefix for persistency of tokens (simplify garbage collect)
13
14
  PERSIST_CATEGORY_TOKEN = 'token'
14
15
  # prefix for bearer token when in header
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'aspera/log'
4
+ require 'aspera/assert'
4
5
  require 'singleton'
5
6
  require 'mime/types'
6
7
 
@@ -9,6 +10,7 @@ module Aspera
9
10
  # function conversion_type returns one of the types: CONVERSION_TYPES
10
11
  class FileTypes
11
12
  include Singleton
13
+
12
14
  # values for conversion_type : input format
13
15
  CONVERSION_TYPES = %i[image office pdf plaintext video].freeze
14
16
 
@@ -22,6 +24,8 @@ module Aspera
22
24
  'application/mxf' => :video,
23
25
  'application/mac-binhex40' => :office,
24
26
  'application/msword' => :office,
27
+ 'application/vnd.ms-excel' => :office,
28
+ 'application/vnd.ms-powerpoint' => :office,
25
29
  'application/rtf' => :office,
26
30
  'application/x-abiword' => :office,
27
31
  'application/x-mspublisher' => :office,
@@ -56,17 +60,43 @@ module Aspera
56
60
  end
57
61
 
58
62
  # @param mimetype [String] mime type
59
- # @return file type, one of enum CONVERSION_TYPES
63
+ # @return file type, one of enum CONVERSION_TYPES, or nil if not found
60
64
  def mime_to_type(mimetype)
65
+ Aspera.assert_type(mimetype, String)
61
66
  return SUPPORTED_MIME_TYPES[mimetype] if SUPPORTED_MIME_TYPES.key?(mimetype)
62
- return :office if mimetype.start_with?('application/vnd.')
67
+ return :office if mimetype.start_with?('application/vnd.ms-')
68
+ return :office if mimetype.start_with?('application/vnd.openxmlformats-officedocument')
63
69
  return :video if mimetype.start_with?('video/')
64
70
  return :image if mimetype.start_with?('image/')
65
71
  return nil
66
72
  end
67
73
 
68
- # use mime magic to find mime type based on file content (magic numbers)
69
- def file_to_mime(filepath)
74
+ # @param filepath [String] full path to file
75
+ # @param mimetype [String] provided by node API
76
+ # @return file type, one of enum CONVERSION_TYPES
77
+ # @raise [RuntimeError] if no conversion type found
78
+ def conversion_type(filepath, mimetype)
79
+ Log.log.debug{"conversion_type(#{filepath},mime=#{mimetype},magic=#{@use_mimemagic})"}
80
+ mimetype = nil if mimetype.is_a?(String) && (mimetype == 'application/octet-stream' || mimetype.empty?)
81
+ # Use mimemagic if available
82
+ mimetype ||= mime_using_mimemagic(filepath)
83
+ mimetype ||= mime_using_file(filepath)
84
+ # from extensions, using local mapping
85
+ mimetype ||= MIME::Types.of(File.basename(filepath)).first
86
+ raise "no MIME type found for #{File.basename(filepath)}" if mimetype.nil?
87
+ conversion_type = mime_to_type(mimetype)
88
+ raise "no conversion type found for #{File.basename(filepath)}" if conversion_type.nil?
89
+ Log.log.trace1{"conversion_type(#{File.basename(filepath)}): #{conversion_type.class.name} [#{conversion_type}]"}
90
+ return conversion_type
91
+ end
92
+
93
+ private
94
+
95
+ # Use mime magic to find mime type based on file content (magic numbers)
96
+ # @param filepath [String] full path to file
97
+ # @return [String] mime type, or nil if not found
98
+ def mime_using_mimemagic(filepath)
99
+ return unless @use_mimemagic
70
100
  # moved here, as `mimemagic` can cause installation issues
71
101
  require 'mimemagic'
72
102
  require 'mimemagic/version'
@@ -83,35 +113,12 @@ module Aspera
83
113
  return detected_mime
84
114
  end
85
115
 
86
- # @param filepath [String] full path to file
87
- # @param mimetype [String] provided by node API
88
- # @return file type, one of enum CONVERSION_TYPES
89
- # @raise [RuntimeError] if no conversion type found
90
- def conversion_type(filepath, mimetype)
91
- Log.log.debug{"conversion_type(#{filepath},m=#{mimetype},t=#{@use_mimemagic})"}
92
- # 1- get type from provided mime type, using local mapping
93
- conversion_type = mime_to_type(mimetype) if !mimetype.nil?
94
- # 2- else, from computed mime type (if available)
95
- if conversion_type.nil? && @use_mimemagic
96
- detected_mime = file_to_mime(filepath)
97
- if !detected_mime.nil?
98
- conversion_type = mime_to_type(detected_mime)
99
- if !mimetype.nil?
100
- if mimetype.eql?(detected_mime)
101
- Log.log.debug('matching mime type per magic number')
102
- else
103
- # NOTE: detected can be nil
104
- Log.log.debug{"non matching mime types: node=[#{mimetype}], magic=[#{detected_mime}]"}
105
- end
106
- end
107
- end
108
- end
109
- # 3- else, from extensions, using local mapping
110
- mime_by_ext = MIME::Types.of(File.basename(filepath)).first
111
- conversion_type = mime_to_type(mime_by_ext.to_s) if conversion_type.nil? && !mime_by_ext.nil?
112
- raise "no conversion type found for #{File.basename(filepath)}" if conversion_type.nil?
113
- Log.log.trace1{"conversion_type(#{File.basename(filepath)}): #{conversion_type.class.name} [#{conversion_type}]"}
114
- return conversion_type
116
+ # Use 'file' command to find mime type based on file content (Unix)
117
+ def mime_using_file(filepath)
118
+ return Environment.secure_capture(exec: 'file', args: ['--mime-type', '--brief', filepath]).strip
119
+ rescue => e
120
+ Log.log.error{"error using 'file' command: #{e.message}"}
121
+ return nil
115
122
  end
116
123
  end
117
124
  end
@@ -240,7 +240,7 @@ module Aspera
240
240
  # text to png
241
241
  def convert_plaintext_to_png
242
242
  # get 100 first lines of text file
243
- first_lines = File.open(@source_file_path){ |f| Array.new(100){f.readline rescue ''}.join}
243
+ first_lines = File.foreach(@source_file_path).first(100).join
244
244
  Utils.external_command(:magick, [
245
245
  'convert',
246
246
  '-size', "#{@options.thumb_img_size}x#{@options.thumb_img_size}",
@@ -7,6 +7,7 @@ module Aspera
7
7
  module Products
8
8
  class Connect
9
9
  include Singleton
10
+
10
11
  APP_NAME = 'IBM Aspera Connect'
11
12
 
12
13
  class << self
data/lib/aspera/rest.rb CHANGED
@@ -6,12 +6,14 @@ require 'aspera/log'
6
6
  require 'aspera/assert'
7
7
  require 'aspera/oauth'
8
8
  require 'aspera/hash_ext'
9
+ require 'aspera/timer_limiter'
9
10
  require 'net/http'
10
11
  require 'net/https'
11
12
  require 'json'
12
13
  require 'base64'
13
14
  require 'singleton'
14
15
  require 'securerandom'
16
+ require 'fileutils'
15
17
 
16
18
  # Cancel method for HTTP
17
19
  class Net::HTTP::Cancel < Net::HTTPRequest # rubocop:disable Style/ClassAndModuleChildren
@@ -172,6 +174,10 @@ module Aspera
172
174
  return result
173
175
  end
174
176
 
177
+ # Parse a header string as returned by HTTP
178
+ # @param header [String] header string, e.g. "application/json; charset=utf-8"
179
+ # @return [Hash] parsed header with type and parameters
180
+ # {type: 'application/json', parameters: {charset: 'utf-8'}}
175
181
  def parse_header(header)
176
182
  type, *params = header.split(/;\s*/)
177
183
  parameters = params.map do |param|
@@ -358,18 +364,18 @@ module Aspera
358
364
  http_session.request(req) do |response|
359
365
  result[:http] = response
360
366
  result_mime = self.class.parse_header(result[:http]['Content-Type'] || MIME_TEXT)[:type]
367
+ Log.log.debug{"response: code=#{result[:http].code}, mime=#{result_mime}, mime2= #{response['Content-Type']}"}
361
368
  # JSON data needs to be parsed, in case it contains an error code
362
369
  if !save_to_file.nil? &&
363
370
  result[:http].code.to_s.start_with?('2') &&
364
- !result[:http]['Content-Length'].nil? &&
365
371
  !JSON_DECODE.include?(result_mime)
366
- total_size = result[:http]['Content-Length'].to_i
372
+ total_size = result[:http]['Content-Length']&.to_i
367
373
  Log.log.debug('before write file')
368
374
  target_file = save_to_file
369
375
  # override user's path to path in header
370
376
  if !response['Content-Disposition'].nil?
371
377
  disposition = self.class.parse_header(response['Content-Disposition'])
372
- if disposition[:parameters].key?(:filename)
378
+ if disposition[:parameters].key?(:filename) && !disposition[:parameters][:filename].eql?('.')
373
379
  target_file = File.join(File.dirname(target_file), disposition[:parameters][:filename])
374
380
  end
375
381
  end
@@ -379,12 +385,14 @@ module Aspera
379
385
  written_size = 0
380
386
  session_id = SecureRandom.uuid.freeze
381
387
  RestParameters.instance.progress_bar&.event(:session_start, session_id: session_id)
382
- RestParameters.instance.progress_bar&.event(:session_size, session_id: session_id, info: total_size)
388
+ RestParameters.instance.progress_bar&.event(:session_size, session_id: session_id, info: total_size) if total_size
389
+ FileUtils.mkdir_p(File.dirname(target_file_tmp))
390
+ limiter = TimerLimiter.new(0.5)
383
391
  File.open(target_file_tmp, 'wb') do |file|
384
392
  result[:http].read_body do |fragment|
385
393
  file.write(fragment)
386
394
  written_size += fragment.length
387
- RestParameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size)
395
+ RestParameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size) if limiter.trigger?
388
396
  end
389
397
  end
390
398
  RestParameters.instance.progress_bar&.event(:end, session_id: session_id)
@@ -405,7 +413,10 @@ module Aspera
405
413
  result[:data] = result[:http].body
406
414
  end
407
415
  RestErrorAnalyzer.instance.raise_on_error(req, result)
408
- File.write(save_to_file, result[:http].body, binmode: true) unless file_saved || save_to_file.nil?
416
+ unless file_saved || save_to_file.nil?
417
+ FileUtils.mkdir_p(File.dirname(save_to_file))
418
+ File.write(save_to_file, result[:http].body, binmode: true)
419
+ end
409
420
  rescue RestCallError => e
410
421
  do_retry = false
411
422
  # AoC have some timeout , like Connect to platform.bss.asperasoft.com:443 ...
@@ -451,7 +462,7 @@ module Aspera
451
462
  # raise exception if could not retry and not return error in result
452
463
  raise e unless return_error
453
464
  end
454
- Log.log.debug{"result=#{result}"}
465
+ Log.log.debug{"result=http:#{result[:http]}, data:#{result[:data].class}"}
455
466
  return result
456
467
  end
457
468
 
@@ -8,6 +8,7 @@ module Aspera
8
8
  # analyze error codes returned by REST calls and raise ruby exception
9
9
  class RestErrorAnalyzer
10
10
  include Singleton
11
+
11
12
  attr_accessor :log_file
12
13
 
13
14
  # the singleton object is registered with application specific handlers
data/lib/aspera/ssh.rb CHANGED
@@ -55,7 +55,7 @@ module Aspera
55
55
  error_message = "#{cmd}: [#{data.chomp}]"
56
56
  # Happens when windows user hasn't logged in and created home account.
57
57
  error_message += "\nHint: home not created in Windows?" if data.include?('Could not chdir to home directory')
58
- raise error_message
58
+ Log.log.debug(error_message)
59
59
  end
60
60
  # send command to SSH channel (execute) cspell: disable-next-line
61
61
  channel.send('cexe'.reverse, cmd){ |_ch, _success| channel.send_data(input) unless input.nil?}
@@ -14,6 +14,7 @@ module Aspera
14
14
  FILE_LIST_AGE_MAX_SEC = SEC_IN_DAY * 5
15
15
  private_constant :SEC_IN_DAY, :FILE_LIST_AGE_MAX_SEC
16
16
  include Singleton
17
+
17
18
  attr_accessor :cleanup_on_exit
18
19
 
19
20
  def initialize
@@ -1,20 +1,22 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Aspera
4
- # used to throttle logs
4
+ # trigger returns true only if the delay has passed since the last trigger
5
5
  class TimerLimiter
6
6
  # @param delay in seconds (float)
7
7
  def initialize(delay)
8
8
  @delay = delay
9
- @last_time = nil
9
+ @last_trigger_time = nil
10
10
  @count = 0
11
11
  end
12
12
 
13
+ # Check if the trigger condition is met
14
+ # @return [Boolean] true if the trigger condition is met, false otherwise
13
15
  def trigger?
14
- old_time = @last_time
15
- @last_time = Time.now.to_f
16
+ current_time = Time.now.to_f
16
17
  @count += 1
17
- if old_time.nil? || ((@last_time - old_time) > @delay)
18
+ if @last_trigger_time.nil? || ((current_time - @last_trigger_time) > @delay)
19
+ @last_trigger_time = current_time
18
20
  @count = 0
19
21
  return true
20
22
  end