aspera-cli 4.22.0 → 4.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +374 -364
- data/README.md +255 -155
- data/lib/aspera/agent/direct.rb +1 -1
- data/lib/aspera/api/aoc.rb +9 -12
- data/lib/aspera/api/httpgw.rb +8 -4
- data/lib/aspera/ascmd.rb +14 -6
- data/lib/aspera/ascp/installation.rb +6 -3
- data/lib/aspera/assert.rb +3 -3
- data/lib/aspera/cli/hints.rb +9 -1
- data/lib/aspera/cli/main.rb +1 -1
- data/lib/aspera/cli/manager.rb +1 -1
- data/lib/aspera/cli/plugin.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +33 -23
- data/lib/aspera/cli/plugins/config.rb +20 -15
- data/lib/aspera/cli/plugins/node.rb +96 -92
- data/lib/aspera/cli/plugins/server.rb +1 -0
- data/lib/aspera/cli/transfer_agent.rb +7 -11
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/data_repository.rb +1 -0
- data/lib/aspera/environment.rb +1 -0
- data/lib/aspera/log.rb +1 -0
- data/lib/aspera/oauth/base.rb +2 -0
- data/lib/aspera/oauth/factory.rb +1 -0
- data/lib/aspera/preview/file_types.rb +40 -33
- data/lib/aspera/preview/generator.rb +1 -1
- data/lib/aspera/products/connect.rb +1 -0
- data/lib/aspera/rest.rb +18 -7
- data/lib/aspera/rest_error_analyzer.rb +1 -0
- data/lib/aspera/ssh.rb +1 -1
- data/lib/aspera/temp_file_manager.rb +1 -0
- data/lib/aspera/timer_limiter.rb +7 -5
- data/lib/aspera/transfer/async_conf.schema.yaml +716 -0
- data/lib/aspera/transfer/sync.rb +14 -4
- data/lib/aspera/transfer/sync_instance.schema.yaml +7 -0
- data/lib/aspera/transfer/sync_session.schema.yaml +7 -0
- data.tar.gz.sig +0 -0
- metadata +3 -5
- metadata.gz.sig +0 -0
- data/examples/dascli +0 -30
- data/examples/get_proto_file.rb +0 -8
- data/examples/proxy.pac +0 -60
@@ -21,6 +21,7 @@ module Aspera
|
|
21
21
|
module Plugins
|
22
22
|
class Node < Cli::BasicAuthPlugin
|
23
23
|
include SyncActions
|
24
|
+
|
24
25
|
class << self
|
25
26
|
# directory: node, container: shares
|
26
27
|
FOLDER_TYPES = %w[directory container].freeze
|
@@ -849,7 +850,6 @@ module Aspera
|
|
849
850
|
case command
|
850
851
|
when :list
|
851
852
|
transfer_filter = query_read_delete(default: {})
|
852
|
-
last_iteration_token = nil
|
853
853
|
iteration_persistency = nil
|
854
854
|
if options.get_option(:once_only, mandatory: true)
|
855
855
|
iteration_persistency = PersistencyActionOnce.new(
|
@@ -865,35 +865,10 @@ module Aspera
|
|
865
865
|
iteration_persistency.save
|
866
866
|
return Main.result_status('Persistency reset')
|
867
867
|
end
|
868
|
-
last_iteration_token = iteration_persistency.data.first
|
869
868
|
end
|
870
869
|
raise Cli::BadArgument, 'reset only with once_only' if transfer_filter.key?('reset') && iteration_persistency.nil?
|
871
870
|
max_items = transfer_filter.delete(MAX_ITEMS)
|
872
|
-
transfers_data =
|
873
|
-
loop do
|
874
|
-
transfer_filter['iteration_token'] = last_iteration_token unless last_iteration_token.nil?
|
875
|
-
result = @api_node.call(operation: 'GET', subpath: 'ops/transfers', query: transfer_filter)
|
876
|
-
# no data
|
877
|
-
break if result[:data].empty?
|
878
|
-
# get next iteration token from link
|
879
|
-
next_iteration_token = nil
|
880
|
-
link_info = result[:http]['Link']
|
881
|
-
unless link_info.nil?
|
882
|
-
m = link_info.match(/<([^>]+)>/)
|
883
|
-
raise "Cannot parse iteration in Link: #{link_info}" if m.nil?
|
884
|
-
next_iteration_token = CGI.parse(URI.parse(m[1]).query)['iteration_token']&.first
|
885
|
-
end
|
886
|
-
# same as last iteration: stop
|
887
|
-
break if next_iteration_token&.eql?(last_iteration_token)
|
888
|
-
last_iteration_token = next_iteration_token
|
889
|
-
transfers_data.concat(result[:data])
|
890
|
-
if max_items&.<=(transfers_data.length)
|
891
|
-
transfers_data = transfers_data.slice(0, max_items)
|
892
|
-
break
|
893
|
-
end
|
894
|
-
break if last_iteration_token.nil?
|
895
|
-
end
|
896
|
-
iteration_persistency&.data&.[]=(0, last_iteration_token)
|
871
|
+
transfers_data = call_with_iteration(api: @api_node, operation: 'GET', subpath: 'ops/transfers', max: max_items, query: transfer_filter, iteration: iteration_persistency&.data)
|
897
872
|
iteration_persistency&.save
|
898
873
|
return {
|
899
874
|
type: :object_list,
|
@@ -1077,7 +1052,7 @@ module Aspera
|
|
1077
1052
|
return Main.result_status('Simulator terminated')
|
1078
1053
|
when :telemetry
|
1079
1054
|
parameters = value_create_modify(command: command, default: {}).symbolize_keys
|
1080
|
-
%i[url
|
1055
|
+
%i[url key].each do |psym|
|
1081
1056
|
raise Cli::BadArgument, "Missing parameter: #{psym}" unless parameters.key?(psym)
|
1082
1057
|
end
|
1083
1058
|
require 'socket'
|
@@ -1085,79 +1060,69 @@ module Aspera
|
|
1085
1060
|
parameters[:hostname] = Socket.gethostname unless parameters.key?(:hostname)
|
1086
1061
|
interval = parameters[:interval].to_f
|
1087
1062
|
raise Cli::BadArgument, 'Interval must be a positive number in seconds' if interval <= 0
|
1088
|
-
|
1063
|
+
otel_api = Rest.new(
|
1089
1064
|
base_url: "#{parameters[:url]}/v1",
|
1090
1065
|
headers: {
|
1091
|
-
# 'Authorization' => "apiToken #{parameters[:
|
1092
|
-
'x-instana-key' => parameters[:
|
1066
|
+
# 'Authorization' => "apiToken #{parameters[:key]}",
|
1067
|
+
'x-instana-key' => parameters[:key],
|
1093
1068
|
'x-instana-host' => parameters[:hostname]
|
1094
1069
|
}
|
1095
1070
|
)
|
1096
|
-
|
1097
|
-
|
1098
|
-
|
1099
|
-
|
1100
|
-
|
1101
|
-
|
1102
|
-
|
1103
|
-
|
1104
|
-
|
1105
|
-
|
1106
|
-
|
1107
|
-
|
1108
|
-
|
1109
|
-
|
1110
|
-
|
1111
|
-
|
1112
|
-
|
1113
|
-
|
1114
|
-
|
1115
|
-
|
1116
|
-
|
1117
|
-
|
1118
|
-
|
1119
|
-
|
1120
|
-
|
1121
|
-
|
1122
|
-
|
1123
|
-
|
1124
|
-
|
1125
|
-
resource: {
|
1126
|
-
attributes: [
|
1071
|
+
datapoint = {
|
1072
|
+
attributes: [
|
1073
|
+
{
|
1074
|
+
key: 'server.name',
|
1075
|
+
value: {
|
1076
|
+
stringValue: 'HSTS1'
|
1077
|
+
}
|
1078
|
+
}
|
1079
|
+
],
|
1080
|
+
asInt: nil,
|
1081
|
+
timeUnixNano: nil
|
1082
|
+
}
|
1083
|
+
# https://opentelemetry.io/docs/specs/otel/metrics/data-model/#gauge
|
1084
|
+
metrics = {
|
1085
|
+
resourceMetrics: [
|
1086
|
+
{
|
1087
|
+
resource: {
|
1088
|
+
attributes: [
|
1089
|
+
{
|
1090
|
+
key: 'service.name',
|
1091
|
+
value: {
|
1092
|
+
stringValue: 'IBMAspera'
|
1093
|
+
}
|
1094
|
+
}
|
1095
|
+
]
|
1096
|
+
},
|
1097
|
+
scopeMetrics: [
|
1098
|
+
{
|
1099
|
+
metrics: [
|
1127
1100
|
{
|
1128
|
-
|
1129
|
-
|
1130
|
-
|
1101
|
+
name: 'active.transfers',
|
1102
|
+
description: 'Number of active transfers',
|
1103
|
+
unit: '1',
|
1104
|
+
gauge: {
|
1105
|
+
dataPoints: [
|
1106
|
+
datapoint
|
1107
|
+
]
|
1131
1108
|
}
|
1132
1109
|
}
|
1133
1110
|
]
|
1134
|
-
}
|
1135
|
-
|
1136
|
-
|
1137
|
-
|
1138
|
-
|
1139
|
-
|
1140
|
-
|
1141
|
-
|
1142
|
-
|
1143
|
-
|
1144
|
-
|
1145
|
-
|
1146
|
-
|
1147
|
-
|
1148
|
-
|
1149
|
-
timeUnixNano: epoch_nsec
|
1150
|
-
}
|
1151
|
-
]
|
1152
|
-
}
|
1153
|
-
}
|
1154
|
-
]
|
1155
|
-
}
|
1156
|
-
]
|
1157
|
-
}
|
1158
|
-
]
|
1159
|
-
})
|
1160
|
-
sleep([0, interval - (Time.now - start_time)].max)
|
1111
|
+
}
|
1112
|
+
]
|
1113
|
+
}
|
1114
|
+
]
|
1115
|
+
}
|
1116
|
+
loop do
|
1117
|
+
timestamp = Time.now
|
1118
|
+
transfers_data = call_with_iteration(api: @api_node, operation: 'GET', subpath: 'ops/transfers', query: {active_only: true})
|
1119
|
+
datapoint[:asInt] = transfers_data.length
|
1120
|
+
datapoint[:timeUnixNano] = timestamp.to_i * 1_000_000_000 + timestamp.nsec
|
1121
|
+
Log.log.info("#{datapoint[:asInt]} active transfers")
|
1122
|
+
# https://www.ibm.com/docs/en/instana-observability/current?topic=instana-backend
|
1123
|
+
otel_api.create('metrics', metrics)
|
1124
|
+
break if interval.eql?(0.0)
|
1125
|
+
sleep([0.0, interval - (Time.now - timestamp)].max)
|
1161
1126
|
end
|
1162
1127
|
end
|
1163
1128
|
Aspera.error_unreachable_line
|
@@ -1178,6 +1143,45 @@ module Aspera
|
|
1178
1143
|
return path_arg if @prefix_path.nil?
|
1179
1144
|
return File.join(@prefix_path, path_arg)
|
1180
1145
|
end
|
1146
|
+
|
1147
|
+
# Executes the provided API call in loop
|
1148
|
+
# @param api [Rest] the API to call
|
1149
|
+
# @param iteration [Array] a single element array with the iteration token or nil
|
1150
|
+
# @param max [Integer] maximum number of items to return, or nil for no limit
|
1151
|
+
# @param query [Hash] query parameters to use for the API call
|
1152
|
+
# @param call_args [Hash] additional arguments to pass to the API call
|
1153
|
+
# @return [Array] list of items returned by the API call
|
1154
|
+
def call_with_iteration(api:, iteration: nil, max: nil, query: nil, **call_args)
|
1155
|
+
query_token = query.clone || {}
|
1156
|
+
item_list = []
|
1157
|
+
query_token[:iteration_token] = iteration.first if iteration.is_a?(Array)
|
1158
|
+
loop do
|
1159
|
+
result = api.call(**call_args, query: query_token)
|
1160
|
+
Aspera.assert_type(result[:data], Array){"Expected data to be an Array, got: #{result[:data].class}"}
|
1161
|
+
# no data
|
1162
|
+
break if result[:data].empty?
|
1163
|
+
# get next iteration token from link
|
1164
|
+
next_iteration_token = nil
|
1165
|
+
link_info = result[:http]['Link']
|
1166
|
+
unless link_info.nil?
|
1167
|
+
m = link_info.match(/<([^>]+)>/)
|
1168
|
+
Aspera.assert(m){"Cannot parse iteration in Link: #{link_info}"}
|
1169
|
+
next_iteration_token = CGI.parse(URI.parse(m[1]).query)['iteration_token']&.first
|
1170
|
+
end
|
1171
|
+
# same as last iteration: stop
|
1172
|
+
break if next_iteration_token&.eql?(query_token[:iteration_token])
|
1173
|
+
query_token[:iteration_token] = next_iteration_token
|
1174
|
+
item_list.concat(result[:data])
|
1175
|
+
if max&.<=(item_list.length)
|
1176
|
+
item_list = item_list.slice(0, max)
|
1177
|
+
break
|
1178
|
+
end
|
1179
|
+
break if next_iteration_token.nil?
|
1180
|
+
end
|
1181
|
+
# save iteration token if needed
|
1182
|
+
iteration[0] = query_token[:iteration_token] unless iteration.nil?
|
1183
|
+
item_list
|
1184
|
+
end
|
1181
1185
|
end
|
1182
1186
|
end
|
1183
1187
|
end
|
@@ -88,14 +88,6 @@ module Aspera
|
|
88
88
|
# add other transfer spec parameters
|
89
89
|
def option_transfer_spec_deep_merge(ts); @transfer_spec_command_line.deep_merge!(ts); end
|
90
90
|
|
91
|
-
# @return [Hash] transfer spec with updated values from command line, including removed values
|
92
|
-
def updated_ts(transfer_spec={})
|
93
|
-
transfer_spec.deep_merge!(@transfer_spec_command_line)
|
94
|
-
# recursively remove values that are nil (user wants to delete)
|
95
|
-
transfer_spec.deep_do{ |hash, key, value, _unused| hash.delete(key) if value.nil?}
|
96
|
-
return transfer_spec
|
97
|
-
end
|
98
|
-
|
99
91
|
attr_reader :transfer_info
|
100
92
|
|
101
93
|
# multiple option are merged
|
@@ -173,9 +165,10 @@ module Aspera
|
|
173
165
|
|
174
166
|
# This is how the list of files to be transferred is specified
|
175
167
|
# get paths suitable for transfer spec from command line
|
168
|
+
# @param default [String] if set, used as default file for --sources=@args
|
176
169
|
# @return [Hash] {source: (mandatory), destination: (optional)}
|
177
170
|
# computation is done only once, cache is kept in @transfer_paths
|
178
|
-
def ts_source_paths
|
171
|
+
def ts_source_paths(default: nil)
|
179
172
|
# return cache if set
|
180
173
|
return @transfer_paths unless @transfer_paths.nil?
|
181
174
|
# start with lower priority : get paths from transfer spec on command line
|
@@ -185,8 +178,9 @@ module Aspera
|
|
185
178
|
case file_list
|
186
179
|
when nil, FILE_LIST_FROM_ARGS
|
187
180
|
Log.log.debug('getting file list as parameters')
|
181
|
+
Aspera.assert_type(default, Array) unless default.nil?
|
188
182
|
# get remaining arguments
|
189
|
-
file_list = @opt_mgr.get_next_argument('source file list', multiple: true)
|
183
|
+
file_list = @opt_mgr.get_next_argument('source file list', multiple: true, default: default)
|
190
184
|
raise Cli::BadArgument, 'specify at least one file on command line or use ' \
|
191
185
|
"--sources=#{FILE_LIST_FROM_TRANSFER_SPEC} to use transfer spec" if !file_list.is_a?(Array) || file_list.empty?
|
192
186
|
when FILE_LIST_FROM_TRANSFER_SPEC
|
@@ -252,7 +246,9 @@ module Aspera
|
|
252
246
|
# update command line paths, unless destination already has one
|
253
247
|
@transfer_spec_command_line['paths'] = transfer_spec['paths'] || ts_source_paths
|
254
248
|
# updated transfer spec with command line
|
255
|
-
|
249
|
+
transfer_spec.deep_merge!(@transfer_spec_command_line)
|
250
|
+
# recursively remove values that are nil (user wants to delete)
|
251
|
+
transfer_spec.deep_do{ |hash, key, value, _unused| hash.delete(key) if value.nil?}
|
256
252
|
# if TS from app has content_protection (e.g. F5), that means content is protected: ask password if not provided
|
257
253
|
if transfer_spec['content_protection'].eql?('decrypt') && !transfer_spec.key?('content_protection_password')
|
258
254
|
transfer_spec['content_protection_password'] = @opt_mgr.prompt_user_input('content protection password', sensitive: true)
|
data/lib/aspera/cli/version.rb
CHANGED
data/lib/aspera/environment.rb
CHANGED
data/lib/aspera/log.rb
CHANGED
data/lib/aspera/oauth/base.rb
CHANGED
@@ -54,6 +54,8 @@ module Aspera
|
|
54
54
|
@token_cache_id = Factory.cache_id(@api.base_url, self.class, @base_cache_ids, @scope)
|
55
55
|
end
|
56
56
|
|
57
|
+
attr_reader :scope
|
58
|
+
|
57
59
|
# helper method to create token as per RFC
|
58
60
|
def create_token_call(creation_params)
|
59
61
|
Log.log.debug{'Generating a new token'.bg_green}
|
data/lib/aspera/oauth/factory.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'aspera/log'
|
4
|
+
require 'aspera/assert'
|
4
5
|
require 'singleton'
|
5
6
|
require 'mime/types'
|
6
7
|
|
@@ -9,6 +10,7 @@ module Aspera
|
|
9
10
|
# function conversion_type returns one of the types: CONVERSION_TYPES
|
10
11
|
class FileTypes
|
11
12
|
include Singleton
|
13
|
+
|
12
14
|
# values for conversion_type : input format
|
13
15
|
CONVERSION_TYPES = %i[image office pdf plaintext video].freeze
|
14
16
|
|
@@ -22,6 +24,8 @@ module Aspera
|
|
22
24
|
'application/mxf' => :video,
|
23
25
|
'application/mac-binhex40' => :office,
|
24
26
|
'application/msword' => :office,
|
27
|
+
'application/vnd.ms-excel' => :office,
|
28
|
+
'application/vnd.ms-powerpoint' => :office,
|
25
29
|
'application/rtf' => :office,
|
26
30
|
'application/x-abiword' => :office,
|
27
31
|
'application/x-mspublisher' => :office,
|
@@ -56,17 +60,43 @@ module Aspera
|
|
56
60
|
end
|
57
61
|
|
58
62
|
# @param mimetype [String] mime type
|
59
|
-
# @return file type, one of enum CONVERSION_TYPES
|
63
|
+
# @return file type, one of enum CONVERSION_TYPES, or nil if not found
|
60
64
|
def mime_to_type(mimetype)
|
65
|
+
Aspera.assert_type(mimetype, String)
|
61
66
|
return SUPPORTED_MIME_TYPES[mimetype] if SUPPORTED_MIME_TYPES.key?(mimetype)
|
62
|
-
return :office if mimetype.start_with?('application/vnd.')
|
67
|
+
return :office if mimetype.start_with?('application/vnd.ms-')
|
68
|
+
return :office if mimetype.start_with?('application/vnd.openxmlformats-officedocument')
|
63
69
|
return :video if mimetype.start_with?('video/')
|
64
70
|
return :image if mimetype.start_with?('image/')
|
65
71
|
return nil
|
66
72
|
end
|
67
73
|
|
68
|
-
#
|
69
|
-
|
74
|
+
# @param filepath [String] full path to file
|
75
|
+
# @param mimetype [String] provided by node API
|
76
|
+
# @return file type, one of enum CONVERSION_TYPES
|
77
|
+
# @raise [RuntimeError] if no conversion type found
|
78
|
+
def conversion_type(filepath, mimetype)
|
79
|
+
Log.log.debug{"conversion_type(#{filepath},mime=#{mimetype},magic=#{@use_mimemagic})"}
|
80
|
+
mimetype = nil if mimetype.is_a?(String) && (mimetype == 'application/octet-stream' || mimetype.empty?)
|
81
|
+
# Use mimemagic if available
|
82
|
+
mimetype ||= mime_using_mimemagic(filepath)
|
83
|
+
mimetype ||= mime_using_file(filepath)
|
84
|
+
# from extensions, using local mapping
|
85
|
+
mimetype ||= MIME::Types.of(File.basename(filepath)).first
|
86
|
+
raise "no MIME type found for #{File.basename(filepath)}" if mimetype.nil?
|
87
|
+
conversion_type = mime_to_type(mimetype)
|
88
|
+
raise "no conversion type found for #{File.basename(filepath)}" if conversion_type.nil?
|
89
|
+
Log.log.trace1{"conversion_type(#{File.basename(filepath)}): #{conversion_type.class.name} [#{conversion_type}]"}
|
90
|
+
return conversion_type
|
91
|
+
end
|
92
|
+
|
93
|
+
private
|
94
|
+
|
95
|
+
# Use mime magic to find mime type based on file content (magic numbers)
|
96
|
+
# @param filepath [String] full path to file
|
97
|
+
# @return [String] mime type, or nil if not found
|
98
|
+
def mime_using_mimemagic(filepath)
|
99
|
+
return unless @use_mimemagic
|
70
100
|
# moved here, as `mimemagic` can cause installation issues
|
71
101
|
require 'mimemagic'
|
72
102
|
require 'mimemagic/version'
|
@@ -83,35 +113,12 @@ module Aspera
|
|
83
113
|
return detected_mime
|
84
114
|
end
|
85
115
|
|
86
|
-
#
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
# 1- get type from provided mime type, using local mapping
|
93
|
-
conversion_type = mime_to_type(mimetype) if !mimetype.nil?
|
94
|
-
# 2- else, from computed mime type (if available)
|
95
|
-
if conversion_type.nil? && @use_mimemagic
|
96
|
-
detected_mime = file_to_mime(filepath)
|
97
|
-
if !detected_mime.nil?
|
98
|
-
conversion_type = mime_to_type(detected_mime)
|
99
|
-
if !mimetype.nil?
|
100
|
-
if mimetype.eql?(detected_mime)
|
101
|
-
Log.log.debug('matching mime type per magic number')
|
102
|
-
else
|
103
|
-
# NOTE: detected can be nil
|
104
|
-
Log.log.debug{"non matching mime types: node=[#{mimetype}], magic=[#{detected_mime}]"}
|
105
|
-
end
|
106
|
-
end
|
107
|
-
end
|
108
|
-
end
|
109
|
-
# 3- else, from extensions, using local mapping
|
110
|
-
mime_by_ext = MIME::Types.of(File.basename(filepath)).first
|
111
|
-
conversion_type = mime_to_type(mime_by_ext.to_s) if conversion_type.nil? && !mime_by_ext.nil?
|
112
|
-
raise "no conversion type found for #{File.basename(filepath)}" if conversion_type.nil?
|
113
|
-
Log.log.trace1{"conversion_type(#{File.basename(filepath)}): #{conversion_type.class.name} [#{conversion_type}]"}
|
114
|
-
return conversion_type
|
116
|
+
# Use 'file' command to find mime type based on file content (Unix)
|
117
|
+
def mime_using_file(filepath)
|
118
|
+
return Environment.secure_capture(exec: 'file', args: ['--mime-type', '--brief', filepath]).strip
|
119
|
+
rescue => e
|
120
|
+
Log.log.error{"error using 'file' command: #{e.message}"}
|
121
|
+
return nil
|
115
122
|
end
|
116
123
|
end
|
117
124
|
end
|
@@ -240,7 +240,7 @@ module Aspera
|
|
240
240
|
# text to png
|
241
241
|
def convert_plaintext_to_png
|
242
242
|
# get 100 first lines of text file
|
243
|
-
first_lines = File.
|
243
|
+
first_lines = File.foreach(@source_file_path).first(100).join
|
244
244
|
Utils.external_command(:magick, [
|
245
245
|
'convert',
|
246
246
|
'-size', "#{@options.thumb_img_size}x#{@options.thumb_img_size}",
|
data/lib/aspera/rest.rb
CHANGED
@@ -6,12 +6,14 @@ require 'aspera/log'
|
|
6
6
|
require 'aspera/assert'
|
7
7
|
require 'aspera/oauth'
|
8
8
|
require 'aspera/hash_ext'
|
9
|
+
require 'aspera/timer_limiter'
|
9
10
|
require 'net/http'
|
10
11
|
require 'net/https'
|
11
12
|
require 'json'
|
12
13
|
require 'base64'
|
13
14
|
require 'singleton'
|
14
15
|
require 'securerandom'
|
16
|
+
require 'fileutils'
|
15
17
|
|
16
18
|
# Cancel method for HTTP
|
17
19
|
class Net::HTTP::Cancel < Net::HTTPRequest # rubocop:disable Style/ClassAndModuleChildren
|
@@ -172,6 +174,10 @@ module Aspera
|
|
172
174
|
return result
|
173
175
|
end
|
174
176
|
|
177
|
+
# Parse a header string as returned by HTTP
|
178
|
+
# @param header [String] header string, e.g. "application/json; charset=utf-8"
|
179
|
+
# @return [Hash] parsed header with type and parameters
|
180
|
+
# {type: 'application/json', parameters: {charset: 'utf-8'}}
|
175
181
|
def parse_header(header)
|
176
182
|
type, *params = header.split(/;\s*/)
|
177
183
|
parameters = params.map do |param|
|
@@ -358,18 +364,18 @@ module Aspera
|
|
358
364
|
http_session.request(req) do |response|
|
359
365
|
result[:http] = response
|
360
366
|
result_mime = self.class.parse_header(result[:http]['Content-Type'] || MIME_TEXT)[:type]
|
367
|
+
Log.log.debug{"response: code=#{result[:http].code}, mime=#{result_mime}, mime2= #{response['Content-Type']}"}
|
361
368
|
# JSON data needs to be parsed, in case it contains an error code
|
362
369
|
if !save_to_file.nil? &&
|
363
370
|
result[:http].code.to_s.start_with?('2') &&
|
364
|
-
!result[:http]['Content-Length'].nil? &&
|
365
371
|
!JSON_DECODE.include?(result_mime)
|
366
|
-
total_size = result[:http]['Content-Length']
|
372
|
+
total_size = result[:http]['Content-Length']&.to_i
|
367
373
|
Log.log.debug('before write file')
|
368
374
|
target_file = save_to_file
|
369
375
|
# override user's path to path in header
|
370
376
|
if !response['Content-Disposition'].nil?
|
371
377
|
disposition = self.class.parse_header(response['Content-Disposition'])
|
372
|
-
if disposition[:parameters].key?(:filename)
|
378
|
+
if disposition[:parameters].key?(:filename) && !disposition[:parameters][:filename].eql?('.')
|
373
379
|
target_file = File.join(File.dirname(target_file), disposition[:parameters][:filename])
|
374
380
|
end
|
375
381
|
end
|
@@ -379,12 +385,14 @@ module Aspera
|
|
379
385
|
written_size = 0
|
380
386
|
session_id = SecureRandom.uuid.freeze
|
381
387
|
RestParameters.instance.progress_bar&.event(:session_start, session_id: session_id)
|
382
|
-
RestParameters.instance.progress_bar&.event(:session_size, session_id: session_id, info: total_size)
|
388
|
+
RestParameters.instance.progress_bar&.event(:session_size, session_id: session_id, info: total_size) if total_size
|
389
|
+
FileUtils.mkdir_p(File.dirname(target_file_tmp))
|
390
|
+
limiter = TimerLimiter.new(0.5)
|
383
391
|
File.open(target_file_tmp, 'wb') do |file|
|
384
392
|
result[:http].read_body do |fragment|
|
385
393
|
file.write(fragment)
|
386
394
|
written_size += fragment.length
|
387
|
-
RestParameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size)
|
395
|
+
RestParameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size) if limiter.trigger?
|
388
396
|
end
|
389
397
|
end
|
390
398
|
RestParameters.instance.progress_bar&.event(:end, session_id: session_id)
|
@@ -405,7 +413,10 @@ module Aspera
|
|
405
413
|
result[:data] = result[:http].body
|
406
414
|
end
|
407
415
|
RestErrorAnalyzer.instance.raise_on_error(req, result)
|
408
|
-
|
416
|
+
unless file_saved || save_to_file.nil?
|
417
|
+
FileUtils.mkdir_p(File.dirname(save_to_file))
|
418
|
+
File.write(save_to_file, result[:http].body, binmode: true)
|
419
|
+
end
|
409
420
|
rescue RestCallError => e
|
410
421
|
do_retry = false
|
411
422
|
# AoC have some timeout , like Connect to platform.bss.asperasoft.com:443 ...
|
@@ -451,7 +462,7 @@ module Aspera
|
|
451
462
|
# raise exception if could not retry and not return error in result
|
452
463
|
raise e unless return_error
|
453
464
|
end
|
454
|
-
Log.log.debug{"result
|
465
|
+
Log.log.debug{"result=http:#{result[:http]}, data:#{result[:data].class}"}
|
455
466
|
return result
|
456
467
|
end
|
457
468
|
|
data/lib/aspera/ssh.rb
CHANGED
@@ -55,7 +55,7 @@ module Aspera
|
|
55
55
|
error_message = "#{cmd}: [#{data.chomp}]"
|
56
56
|
# Happens when windows user hasn't logged in and created home account.
|
57
57
|
error_message += "\nHint: home not created in Windows?" if data.include?('Could not chdir to home directory')
|
58
|
-
|
58
|
+
Log.log.debug(error_message)
|
59
59
|
end
|
60
60
|
# send command to SSH channel (execute) cspell: disable-next-line
|
61
61
|
channel.send('cexe'.reverse, cmd){ |_ch, _success| channel.send_data(input) unless input.nil?}
|
data/lib/aspera/timer_limiter.rb
CHANGED
@@ -1,20 +1,22 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module Aspera
|
4
|
-
#
|
4
|
+
# trigger returns true only if the delay has passed since the last trigger
|
5
5
|
class TimerLimiter
|
6
6
|
# @param delay in seconds (float)
|
7
7
|
def initialize(delay)
|
8
8
|
@delay = delay
|
9
|
-
@
|
9
|
+
@last_trigger_time = nil
|
10
10
|
@count = 0
|
11
11
|
end
|
12
12
|
|
13
|
+
# Check if the trigger condition is met
|
14
|
+
# @return [Boolean] true if the trigger condition is met, false otherwise
|
13
15
|
def trigger?
|
14
|
-
|
15
|
-
@last_time = Time.now.to_f
|
16
|
+
current_time = Time.now.to_f
|
16
17
|
@count += 1
|
17
|
-
if
|
18
|
+
if @last_trigger_time.nil? || ((current_time - @last_trigger_time) > @delay)
|
19
|
+
@last_trigger_time = current_time
|
18
20
|
@count = 0
|
19
21
|
return true
|
20
22
|
end
|