aspera-cli 4.0.0 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +518 -137
- data/bin/dascli +13 -0
- data/docs/README.erb.md +473 -105
- data/docs/test_env.conf +4 -1
- data/docs/transfer_spec.html +1 -1
- data/lib/aspera/aoc.rb +68 -86
- data/lib/aspera/cli/formater.rb +2 -0
- data/lib/aspera/cli/main.rb +27 -19
- data/lib/aspera/cli/plugin.rb +9 -4
- data/lib/aspera/cli/plugins/alee.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +173 -136
- data/lib/aspera/cli/plugins/config.rb +80 -27
- data/lib/aspera/cli/plugins/console.rb +2 -2
- data/lib/aspera/cli/plugins/faspex.rb +13 -6
- data/lib/aspera/cli/plugins/faspex5.rb +93 -37
- data/lib/aspera/cli/plugins/node.rb +3 -3
- data/lib/aspera/cli/plugins/preview.rb +25 -24
- data/lib/aspera/cli/plugins/server.rb +23 -8
- data/lib/aspera/cli/transfer_agent.rb +1 -1
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/fasp/connect.rb +28 -21
- data/lib/aspera/fasp/http_gw.rb +140 -28
- data/lib/aspera/fasp/installation.rb +28 -4
- data/lib/aspera/fasp/local.rb +24 -16
- data/lib/aspera/fasp/manager.rb +12 -0
- data/lib/aspera/fasp/node.rb +4 -4
- data/lib/aspera/fasp/parameters.rb +3 -16
- data/lib/aspera/log.rb +1 -1
- data/lib/aspera/node.rb +48 -1
- data/lib/aspera/oauth.rb +24 -11
- data/lib/aspera/persistency_folder.rb +9 -4
- data/lib/aspera/preview/file_types.rb +53 -21
- data/lib/aspera/preview/generator.rb +3 -3
- data/lib/aspera/rest.rb +38 -18
- data/lib/aspera/temp_file_manager.rb +19 -0
- metadata +37 -20
@@ -15,7 +15,7 @@ module Aspera
|
|
15
15
|
|
16
16
|
# @param src source file path
|
17
17
|
# @param dst destination file path
|
18
|
-
# @param api_mime_type optional mime type as provided by node api
|
18
|
+
# @param api_mime_type optional mime type as provided by node api (or nil)
|
19
19
|
# node API mime types are from: http://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types
|
20
20
|
# supported preview type is one of Preview::PREVIEW_FORMATS
|
21
21
|
# the resulting preview file type is taken from destination file extension.
|
@@ -25,14 +25,14 @@ module Aspera
|
|
25
25
|
# -> preview_format is one of Generator::PREVIEW_FORMATS
|
26
26
|
# the conversion video->mp4 is implemented in methods: convert_video_to_mp4_using_<video_conversion>
|
27
27
|
# -> conversion method is one of Generator::VIDEO_CONVERSION_METHODS
|
28
|
-
def initialize(options,src,dst,main_temp_dir,api_mime_type
|
28
|
+
def initialize(options,src,dst,main_temp_dir,api_mime_type)
|
29
29
|
@options=options
|
30
30
|
@source_file_path=src
|
31
31
|
@destination_file_path=dst
|
32
32
|
@temp_folder=File.join(main_temp_dir,@source_file_path.split('/').last.gsub(/\s/, '_').gsub(/\W/, ''))
|
33
33
|
# extract preview format from extension of target file
|
34
34
|
@preview_format_symb=File.extname(@destination_file_path).gsub(/^\./,'').to_sym
|
35
|
-
@conversion_type=FileTypes.conversion_type(@source_file_path,api_mime_type
|
35
|
+
@conversion_type=FileTypes.instance.conversion_type(@source_file_path,api_mime_type)
|
36
36
|
end
|
37
37
|
|
38
38
|
# name of processing method in this object
|
data/lib/aspera/rest.rb
CHANGED
@@ -7,6 +7,7 @@ require 'net/http'
|
|
7
7
|
require 'net/https'
|
8
8
|
require 'json'
|
9
9
|
require 'base64'
|
10
|
+
require 'cgi'
|
10
11
|
require 'ruby-progressbar'
|
11
12
|
|
12
13
|
# add cancel method to http
|
@@ -33,6 +34,7 @@ module Aspera
|
|
33
34
|
# true if https ignore certificate
|
34
35
|
@@insecure=false
|
35
36
|
@@user_agent='Ruby'
|
37
|
+
@@download_partial_suffix='.http_partial'
|
36
38
|
|
37
39
|
public
|
38
40
|
def self.insecure=(v); @@insecure=v;Log.log.debug("insecure => #{@@insecure}".red);end
|
@@ -98,9 +100,10 @@ module Aspera
|
|
98
100
|
public
|
99
101
|
|
100
102
|
attr_reader :params
|
103
|
+
attr_reader :oauth
|
101
104
|
|
102
|
-
# @param a_rest_params default call parameters and authentication (:auth) :
|
103
|
-
# :type (:basic, :oauth2, :url)
|
105
|
+
# @param a_rest_params default call parameters (merged at call) and (+) authentication (:auth) :
|
106
|
+
# :type (:basic, :oauth2, :url, :none)
|
104
107
|
# :username [:basic]
|
105
108
|
# :password [:basic]
|
106
109
|
# :url_creds [:url]
|
@@ -117,16 +120,16 @@ module Aspera
|
|
117
120
|
# default is no auth
|
118
121
|
@params[:auth]||={:type=>:none}
|
119
122
|
@params[:not_auth_codes]||=['401']
|
120
|
-
# translate old auth parameters, remove prefix, place in auth
|
121
|
-
[:auth,:basic,:oauth].each do |p_sym|
|
122
|
-
p_str=p_sym.to_s+'_'
|
123
|
-
@params.keys.select{|k|k.to_s.start_with?(p_str)}.each do |k_sym|
|
124
|
-
name=k_sym.to_s[p_str.length..-1]
|
125
|
-
name='grant' if k_sym.eql?(:oauth_type)
|
126
|
-
@params[:auth][name.to_sym]=@params[k_sym]
|
127
|
-
@params.delete(k_sym)
|
128
|
-
end
|
129
|
-
end
|
123
|
+
# translate old auth parameters, remove prefix, place in auth (TODO: delete this)
|
124
|
+
# [:auth,:basic,:oauth].each do |p_sym|
|
125
|
+
# p_str=p_sym.to_s+'_'
|
126
|
+
# @params.keys.select{|k|k.to_s.start_with?(p_str)}.each do |k_sym|
|
127
|
+
# name=k_sym.to_s[p_str.length..-1]
|
128
|
+
# name='grant' if k_sym.eql?(:oauth_type)
|
129
|
+
# @params[:auth][name.to_sym]=@params[k_sym]
|
130
|
+
# @params.delete(k_sym)
|
131
|
+
# end
|
132
|
+
# end
|
130
133
|
@oauth=Oauth.new(@params[:auth]) if @params[:auth][:type].eql?(:oauth2)
|
131
134
|
Log.dump('REST params(2)',@params)
|
132
135
|
end
|
@@ -206,10 +209,13 @@ module Aspera
|
|
206
209
|
|
207
210
|
Log.log.debug("call_data = #{call_data}")
|
208
211
|
result={:http=>nil}
|
212
|
+
# start a block to be able to retry the actual HTTP request
|
209
213
|
begin
|
210
214
|
# we try the call, and will retry only if oauth, as we can, first with refresh, and then re-auth if refresh is bad
|
211
215
|
oauth_tries ||= 2
|
216
|
+
tries_remain_redirect||=4
|
212
217
|
Log.log.debug("send request")
|
218
|
+
# make http request (pipelined)
|
213
219
|
http_session.request(req) do |response|
|
214
220
|
result[:http] = response
|
215
221
|
if call_data.has_key?(:save_to_file)
|
@@ -223,12 +229,12 @@ module Aspera
|
|
223
229
|
target_file=call_data[:save_to_file]
|
224
230
|
# override user's path to path in header
|
225
231
|
if !response['Content-Disposition'].nil? and m=response['Content-Disposition'].match(/filename="([^"]+)"/)
|
226
|
-
target_file=m[1]
|
232
|
+
target_file=File.join(File.dirname(target_file),m[1])
|
227
233
|
end
|
228
234
|
# download with temp filename
|
229
|
-
target_file_tmp=target_file
|
235
|
+
target_file_tmp="#{target_file}#{@@download_partial_suffix}"
|
230
236
|
Log.log.debug("saving to: #{target_file}")
|
231
|
-
File.open(target_file_tmp,
|
237
|
+
File.open(target_file_tmp, 'wb') do |file|
|
232
238
|
result[:http].read_body do |fragment|
|
233
239
|
file.write(fragment)
|
234
240
|
new_process=progress.progress+fragment.length
|
@@ -239,7 +245,7 @@ module Aspera
|
|
239
245
|
# rename at the end
|
240
246
|
File.rename(target_file_tmp, target_file)
|
241
247
|
progress=nil
|
242
|
-
end
|
248
|
+
end # save_to_file
|
243
249
|
end
|
244
250
|
# sometimes there is a ITF8 char (e.g. (c) )
|
245
251
|
result[:http].body.force_encoding("UTF-8") if result[:http].body.is_a?(String)
|
@@ -267,10 +273,24 @@ module Aspera
|
|
267
273
|
end
|
268
274
|
Log.log.debug("using new token=#{call_data[:headers]['Authorization']}")
|
269
275
|
retry unless (oauth_tries -= 1).zero?
|
270
|
-
end # if
|
276
|
+
end # if oauth
|
277
|
+
# moved ?
|
278
|
+
if e.response.is_a?(Net::HTTPRedirection)
|
279
|
+
if tries_remain_redirect > 0
|
280
|
+
tries_remain_redirect-=1
|
281
|
+
Log.log.error("URL is moved, check your config: #{e.response['location']}")
|
282
|
+
raise e
|
283
|
+
# TODO: rebuild request with new location
|
284
|
+
#retry
|
285
|
+
else
|
286
|
+
raise "too many redirect"
|
287
|
+
end
|
288
|
+
else
|
289
|
+
raise e
|
290
|
+
end
|
271
291
|
# raise exception if could not retry and not return error in result
|
272
292
|
raise e unless call_data[:return_error]
|
273
|
-
end
|
293
|
+
end # begin request
|
274
294
|
Log.log.debug("result=#{result}")
|
275
295
|
return result
|
276
296
|
|
@@ -6,6 +6,11 @@ module Aspera
|
|
6
6
|
# create a temp file name for a given folder
|
7
7
|
# files can be deleted on process exit by calling cleanup
|
8
8
|
class TempFileManager
|
9
|
+
SEC_IN_DAY=86400
|
10
|
+
# assume no transfer last longer than this
|
11
|
+
# (garbage collect file list which were not deleted after transfer)
|
12
|
+
FILE_LIST_AGE_MAX_SEC=5*SEC_IN_DAY
|
13
|
+
private_constant :SEC_IN_DAY,:FILE_LIST_AGE_MAX_SEC
|
9
14
|
include Singleton
|
10
15
|
def initialize
|
11
16
|
@created_files=[]
|
@@ -33,5 +38,19 @@ module Aspera
|
|
33
38
|
username = Etc.getlogin || Etc.getpwuid(Process.uid).name || 'unknown_user' rescue 'unknown_user'
|
34
39
|
return new_file_path_in_folder(Etc.systmpdir,base_name+'_'+username+'_')
|
35
40
|
end
|
41
|
+
|
42
|
+
def cleanup_expired(temp_folder)
|
43
|
+
# garbage collect undeleted files
|
44
|
+
Dir.entries(temp_folder).each do |name|
|
45
|
+
file_path=File.join(temp_folder,name)
|
46
|
+
age_sec=(Time.now - File.stat(file_path).mtime).to_i
|
47
|
+
# check age of file, delete too old
|
48
|
+
if File.file?(file_path) and age_sec > FILE_LIST_AGE_MAX_SEC
|
49
|
+
Log.log.debug("garbage collecting #{name}")
|
50
|
+
File.delete(file_path)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
36
55
|
end
|
37
56
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: aspera-cli
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.
|
4
|
+
version: 4.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Laurent Martin
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-06-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: xml-simple
|
@@ -58,28 +58,14 @@ dependencies:
|
|
58
58
|
requirements:
|
59
59
|
- - "~>"
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: '
|
61
|
+
version: '6.0'
|
62
62
|
type: :runtime
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
66
|
- - "~>"
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version: '
|
69
|
-
- !ruby/object:Gem::Dependency
|
70
|
-
name: mimemagic
|
71
|
-
requirement: !ruby/object:Gem::Requirement
|
72
|
-
requirements:
|
73
|
-
- - "~>"
|
74
|
-
- !ruby/object:Gem::Version
|
75
|
-
version: '0.3'
|
76
|
-
type: :runtime
|
77
|
-
prerelease: false
|
78
|
-
version_requirements: !ruby/object:Gem::Requirement
|
79
|
-
requirements:
|
80
|
-
- - "~>"
|
81
|
-
- !ruby/object:Gem::Version
|
82
|
-
version: '0.3'
|
68
|
+
version: '6.0'
|
83
69
|
- !ruby/object:Gem::Dependency
|
84
70
|
name: execjs
|
85
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -136,6 +122,34 @@ dependencies:
|
|
136
122
|
- - "~>"
|
137
123
|
- !ruby/object:Gem::Version
|
138
124
|
version: '2.0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: websocket
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - "~>"
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '1.2'
|
132
|
+
type: :runtime
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - "~>"
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '1.2'
|
139
|
+
- !ruby/object:Gem::Dependency
|
140
|
+
name: websocket-client-simple
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
142
|
+
requirements:
|
143
|
+
- - "~>"
|
144
|
+
- !ruby/object:Gem::Version
|
145
|
+
version: '0.3'
|
146
|
+
type: :runtime
|
147
|
+
prerelease: false
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
149
|
+
requirements:
|
150
|
+
- - "~>"
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: '0.3'
|
139
153
|
- !ruby/object:Gem::Dependency
|
140
154
|
name: bundler
|
141
155
|
requirement: !ruby/object:Gem::Requirement
|
@@ -184,12 +198,14 @@ email:
|
|
184
198
|
executables:
|
185
199
|
- ascli
|
186
200
|
- asession
|
201
|
+
- dascli
|
187
202
|
extensions: []
|
188
203
|
extra_rdoc_files: []
|
189
204
|
files:
|
190
205
|
- README.md
|
191
206
|
- bin/ascli
|
192
207
|
- bin/asession
|
208
|
+
- bin/dascli
|
193
209
|
- docs/Makefile
|
194
210
|
- docs/README.erb.md
|
195
211
|
- docs/README.md
|
@@ -290,6 +306,7 @@ metadata:
|
|
290
306
|
source_code_uri: https://github.com/IBM/aspera-cli
|
291
307
|
changelog_uri: https://github.com/IBM/aspera-cli
|
292
308
|
rubygems_uri: https://rubygems.org/gems/aspera-cli
|
309
|
+
documentation_uri: https://www.rubydoc.info/gems/aspera-cli
|
293
310
|
post_install_message:
|
294
311
|
rdoc_options: []
|
295
312
|
require_paths:
|
@@ -305,8 +322,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
305
322
|
- !ruby/object:Gem::Version
|
306
323
|
version: '0'
|
307
324
|
requirements:
|
308
|
-
-
|
309
|
-
rubygems_version: 3.0.3
|
325
|
+
- Read the manual for any requirement
|
326
|
+
rubygems_version: 3.0.3.1
|
310
327
|
signing_key:
|
311
328
|
specification_version: 4
|
312
329
|
summary: 'Execute actions using command line on IBM Aspera Server products: Aspera
|