aspera-cli 4.0.0 → 4.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -15,7 +15,7 @@ module Aspera
15
15
 
16
16
  # @param src source file path
17
17
  # @param dst destination file path
18
- # @param api_mime_type optional mime type as provided by node api
18
+ # @param api_mime_type optional mime type as provided by node api (or nil)
19
19
  # node API mime types are from: http://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types
20
20
  # supported preview type is one of Preview::PREVIEW_FORMATS
21
21
  # the resulting preview file type is taken from destination file extension.
@@ -25,14 +25,14 @@ module Aspera
25
25
  # -> preview_format is one of Generator::PREVIEW_FORMATS
26
26
  # the conversion video->mp4 is implemented in methods: convert_video_to_mp4_using_<video_conversion>
27
27
  # -> conversion method is one of Generator::VIDEO_CONVERSION_METHODS
28
- def initialize(options,src,dst,main_temp_dir,api_mime_type=nil,try_local_mime=true)
28
+ def initialize(options,src,dst,main_temp_dir,api_mime_type)
29
29
  @options=options
30
30
  @source_file_path=src
31
31
  @destination_file_path=dst
32
32
  @temp_folder=File.join(main_temp_dir,@source_file_path.split('/').last.gsub(/\s/, '_').gsub(/\W/, ''))
33
33
  # extract preview format from extension of target file
34
34
  @preview_format_symb=File.extname(@destination_file_path).gsub(/^\./,'').to_sym
35
- @conversion_type=FileTypes.conversion_type(@source_file_path,api_mime_type,try_local_mime)
35
+ @conversion_type=FileTypes.instance.conversion_type(@source_file_path,api_mime_type)
36
36
  end
37
37
 
38
38
  # name of processing method in this object
data/lib/aspera/rest.rb CHANGED
@@ -7,6 +7,7 @@ require 'net/http'
7
7
  require 'net/https'
8
8
  require 'json'
9
9
  require 'base64'
10
+ require 'cgi'
10
11
  require 'ruby-progressbar'
11
12
 
12
13
  # add cancel method to http
@@ -33,6 +34,7 @@ module Aspera
33
34
  # true if https ignore certificate
34
35
  @@insecure=false
35
36
  @@user_agent='Ruby'
37
+ @@download_partial_suffix='.http_partial'
36
38
 
37
39
  public
38
40
  def self.insecure=(v); @@insecure=v;Log.log.debug("insecure => #{@@insecure}".red);end
@@ -98,9 +100,10 @@ module Aspera
98
100
  public
99
101
 
100
102
  attr_reader :params
103
+ attr_reader :oauth
101
104
 
102
- # @param a_rest_params default call parameters and authentication (:auth) :
103
- # :type (:basic, :oauth2, :url)
105
+ # @param a_rest_params default call parameters (merged at call) and (+) authentication (:auth) :
106
+ # :type (:basic, :oauth2, :url, :none)
104
107
  # :username [:basic]
105
108
  # :password [:basic]
106
109
  # :url_creds [:url]
@@ -117,16 +120,16 @@ module Aspera
117
120
  # default is no auth
118
121
  @params[:auth]||={:type=>:none}
119
122
  @params[:not_auth_codes]||=['401']
120
- # translate old auth parameters, remove prefix, place in auth
121
- [:auth,:basic,:oauth].each do |p_sym|
122
- p_str=p_sym.to_s+'_'
123
- @params.keys.select{|k|k.to_s.start_with?(p_str)}.each do |k_sym|
124
- name=k_sym.to_s[p_str.length..-1]
125
- name='grant' if k_sym.eql?(:oauth_type)
126
- @params[:auth][name.to_sym]=@params[k_sym]
127
- @params.delete(k_sym)
128
- end
129
- end
123
+ # translate old auth parameters, remove prefix, place in auth (TODO: delete this)
124
+ # [:auth,:basic,:oauth].each do |p_sym|
125
+ # p_str=p_sym.to_s+'_'
126
+ # @params.keys.select{|k|k.to_s.start_with?(p_str)}.each do |k_sym|
127
+ # name=k_sym.to_s[p_str.length..-1]
128
+ # name='grant' if k_sym.eql?(:oauth_type)
129
+ # @params[:auth][name.to_sym]=@params[k_sym]
130
+ # @params.delete(k_sym)
131
+ # end
132
+ # end
130
133
  @oauth=Oauth.new(@params[:auth]) if @params[:auth][:type].eql?(:oauth2)
131
134
  Log.dump('REST params(2)',@params)
132
135
  end
@@ -206,10 +209,13 @@ module Aspera
206
209
 
207
210
  Log.log.debug("call_data = #{call_data}")
208
211
  result={:http=>nil}
212
+ # start a block to be able to retry the actual HTTP request
209
213
  begin
210
214
  # we try the call, and will retry only if oauth, as we can, first with refresh, and then re-auth if refresh is bad
211
215
  oauth_tries ||= 2
216
+ tries_remain_redirect||=4
212
217
  Log.log.debug("send request")
218
+ # make http request (pipelined)
213
219
  http_session.request(req) do |response|
214
220
  result[:http] = response
215
221
  if call_data.has_key?(:save_to_file)
@@ -223,12 +229,12 @@ module Aspera
223
229
  target_file=call_data[:save_to_file]
224
230
  # override user's path to path in header
225
231
  if !response['Content-Disposition'].nil? and m=response['Content-Disposition'].match(/filename="([^"]+)"/)
226
- target_file=m[1]
232
+ target_file=File.join(File.dirname(target_file),m[1])
227
233
  end
228
234
  # download with temp filename
229
- target_file_tmp=target_file+'.http.partial'
235
+ target_file_tmp="#{target_file}#{@@download_partial_suffix}"
230
236
  Log.log.debug("saving to: #{target_file}")
231
- File.open(target_file_tmp, "wb") do |file|
237
+ File.open(target_file_tmp, 'wb') do |file|
232
238
  result[:http].read_body do |fragment|
233
239
  file.write(fragment)
234
240
  new_process=progress.progress+fragment.length
@@ -239,7 +245,7 @@ module Aspera
239
245
  # rename at the end
240
246
  File.rename(target_file_tmp, target_file)
241
247
  progress=nil
242
- end
248
+ end # save_to_file
243
249
  end
244
250
  # sometimes there is a ITF8 char (e.g. (c) )
245
251
  result[:http].body.force_encoding("UTF-8") if result[:http].body.is_a?(String)
@@ -267,10 +273,24 @@ module Aspera
267
273
  end
268
274
  Log.log.debug("using new token=#{call_data[:headers]['Authorization']}")
269
275
  retry unless (oauth_tries -= 1).zero?
270
- end # if
276
+ end # if oauth
277
+ # moved ?
278
+ if e.response.is_a?(Net::HTTPRedirection)
279
+ if tries_remain_redirect > 0
280
+ tries_remain_redirect-=1
281
+ Log.log.error("URL is moved, check your config: #{e.response['location']}")
282
+ raise e
283
+ # TODO: rebuild request with new location
284
+ #retry
285
+ else
286
+ raise "too many redirect"
287
+ end
288
+ else
289
+ raise e
290
+ end
271
291
  # raise exception if could not retry and not return error in result
272
292
  raise e unless call_data[:return_error]
273
- end
293
+ end # begin request
274
294
  Log.log.debug("result=#{result}")
275
295
  return result
276
296
 
@@ -6,6 +6,11 @@ module Aspera
6
6
  # create a temp file name for a given folder
7
7
  # files can be deleted on process exit by calling cleanup
8
8
  class TempFileManager
9
+ SEC_IN_DAY=86400
10
+ # assume no transfer last longer than this
11
+ # (garbage collect file list which were not deleted after transfer)
12
+ FILE_LIST_AGE_MAX_SEC=5*SEC_IN_DAY
13
+ private_constant :SEC_IN_DAY,:FILE_LIST_AGE_MAX_SEC
9
14
  include Singleton
10
15
  def initialize
11
16
  @created_files=[]
@@ -33,5 +38,19 @@ module Aspera
33
38
  username = Etc.getlogin || Etc.getpwuid(Process.uid).name || 'unknown_user' rescue 'unknown_user'
34
39
  return new_file_path_in_folder(Etc.systmpdir,base_name+'_'+username+'_')
35
40
  end
41
+
42
+ def cleanup_expired(temp_folder)
43
+ # garbage collect undeleted files
44
+ Dir.entries(temp_folder).each do |name|
45
+ file_path=File.join(temp_folder,name)
46
+ age_sec=(Time.now - File.stat(file_path).mtime).to_i
47
+ # check age of file, delete too old
48
+ if File.file?(file_path) and age_sec > FILE_LIST_AGE_MAX_SEC
49
+ Log.log.debug("garbage collecting #{name}")
50
+ File.delete(file_path)
51
+ end
52
+ end
53
+
54
+ end
36
55
  end
37
56
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: aspera-cli
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.0.0
4
+ version: 4.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Laurent Martin
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-02-03 00:00:00.000000000 Z
11
+ date: 2021-06-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: xml-simple
@@ -58,28 +58,14 @@ dependencies:
58
58
  requirements:
59
59
  - - "~>"
60
60
  - !ruby/object:Gem::Version
61
- version: '4.0'
61
+ version: '6.0'
62
62
  type: :runtime
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
66
  - - "~>"
67
67
  - !ruby/object:Gem::Version
68
- version: '4.0'
69
- - !ruby/object:Gem::Dependency
70
- name: mimemagic
71
- requirement: !ruby/object:Gem::Requirement
72
- requirements:
73
- - - "~>"
74
- - !ruby/object:Gem::Version
75
- version: '0.3'
76
- type: :runtime
77
- prerelease: false
78
- version_requirements: !ruby/object:Gem::Requirement
79
- requirements:
80
- - - "~>"
81
- - !ruby/object:Gem::Version
82
- version: '0.3'
68
+ version: '6.0'
83
69
  - !ruby/object:Gem::Dependency
84
70
  name: execjs
85
71
  requirement: !ruby/object:Gem::Requirement
@@ -136,6 +122,34 @@ dependencies:
136
122
  - - "~>"
137
123
  - !ruby/object:Gem::Version
138
124
  version: '2.0'
125
+ - !ruby/object:Gem::Dependency
126
+ name: websocket
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - "~>"
130
+ - !ruby/object:Gem::Version
131
+ version: '1.2'
132
+ type: :runtime
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - "~>"
137
+ - !ruby/object:Gem::Version
138
+ version: '1.2'
139
+ - !ruby/object:Gem::Dependency
140
+ name: websocket-client-simple
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - "~>"
144
+ - !ruby/object:Gem::Version
145
+ version: '0.3'
146
+ type: :runtime
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - "~>"
151
+ - !ruby/object:Gem::Version
152
+ version: '0.3'
139
153
  - !ruby/object:Gem::Dependency
140
154
  name: bundler
141
155
  requirement: !ruby/object:Gem::Requirement
@@ -184,12 +198,14 @@ email:
184
198
  executables:
185
199
  - ascli
186
200
  - asession
201
+ - dascli
187
202
  extensions: []
188
203
  extra_rdoc_files: []
189
204
  files:
190
205
  - README.md
191
206
  - bin/ascli
192
207
  - bin/asession
208
+ - bin/dascli
193
209
  - docs/Makefile
194
210
  - docs/README.erb.md
195
211
  - docs/README.md
@@ -290,6 +306,7 @@ metadata:
290
306
  source_code_uri: https://github.com/IBM/aspera-cli
291
307
  changelog_uri: https://github.com/IBM/aspera-cli
292
308
  rubygems_uri: https://rubygems.org/gems/aspera-cli
309
+ documentation_uri: https://www.rubydoc.info/gems/aspera-cli
293
310
  post_install_message:
294
311
  rdoc_options: []
295
312
  require_paths:
@@ -305,8 +322,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
305
322
  - !ruby/object:Gem::Version
306
323
  version: '0'
307
324
  requirements:
308
- - IBM Aspera ascp installed for the user
309
- rubygems_version: 3.0.3
325
+ - Read the manual for any requirement
326
+ rubygems_version: 3.0.3.1
310
327
  signing_key:
311
328
  specification_version: 4
312
329
  summary: 'Execute actions using command line on IBM Aspera Server products: Aspera