aspera-cli 4.0.0.pre3 → 4.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +695 -205
  3. data/bin/dascli +13 -0
  4. data/docs/README.erb.md +615 -157
  5. data/docs/test_env.conf +23 -5
  6. data/docs/transfer_spec.html +1 -1
  7. data/examples/aoc.rb +14 -3
  8. data/examples/faspex4.rb +78 -0
  9. data/lib/aspera/aoc.rb +87 -108
  10. data/lib/aspera/cli/formater.rb +2 -0
  11. data/lib/aspera/cli/main.rb +46 -34
  12. data/lib/aspera/cli/plugin.rb +9 -4
  13. data/lib/aspera/cli/plugins/alee.rb +1 -1
  14. data/lib/aspera/cli/plugins/aoc.rb +207 -182
  15. data/lib/aspera/cli/plugins/ats.rb +2 -2
  16. data/lib/aspera/cli/plugins/config.rb +173 -117
  17. data/lib/aspera/cli/plugins/console.rb +2 -2
  18. data/lib/aspera/cli/plugins/faspex.rb +51 -36
  19. data/lib/aspera/cli/plugins/faspex5.rb +82 -41
  20. data/lib/aspera/cli/plugins/node.rb +3 -3
  21. data/lib/aspera/cli/plugins/preview.rb +35 -25
  22. data/lib/aspera/cli/plugins/server.rb +23 -8
  23. data/lib/aspera/cli/transfer_agent.rb +7 -6
  24. data/lib/aspera/cli/version.rb +1 -1
  25. data/lib/aspera/cos_node.rb +33 -28
  26. data/lib/aspera/environment.rb +2 -2
  27. data/lib/aspera/fasp/connect.rb +28 -21
  28. data/lib/aspera/fasp/http_gw.rb +140 -28
  29. data/lib/aspera/fasp/installation.rb +101 -53
  30. data/lib/aspera/fasp/local.rb +88 -45
  31. data/lib/aspera/fasp/manager.rb +15 -0
  32. data/lib/aspera/fasp/node.rb +4 -4
  33. data/lib/aspera/fasp/parameters.rb +6 -18
  34. data/lib/aspera/fasp/resume_policy.rb +13 -12
  35. data/lib/aspera/log.rb +1 -1
  36. data/lib/aspera/node.rb +61 -1
  37. data/lib/aspera/oauth.rb +49 -46
  38. data/lib/aspera/persistency_folder.rb +9 -4
  39. data/lib/aspera/preview/file_types.rb +53 -21
  40. data/lib/aspera/preview/generator.rb +3 -3
  41. data/lib/aspera/rest.rb +29 -18
  42. data/lib/aspera/secrets.rb +20 -0
  43. data/lib/aspera/temp_file_manager.rb +19 -0
  44. data/lib/aspera/web_auth.rb +105 -0
  45. metadata +42 -22
@@ -14,13 +14,13 @@ module Aspera
14
14
  self.options.parse_options!
15
15
  end
16
16
 
17
- ACTIONS=[:transfer,:nagios_check]
17
+ ACTIONS=[:transfer,:health]
18
18
 
19
19
  def execute_action
20
20
  api_console=basic_auth_api('api')
21
21
  command=self.options.get_next_command(ACTIONS)
22
22
  case command
23
- when :nagios_check
23
+ when :health
24
24
  nagios=Nagios.new
25
25
  begin
26
26
  api_console.read('ssh_keys')
@@ -26,6 +26,7 @@ module Aspera
26
26
  self.options.add_opt_simple(:delivery_info,'package delivery information (extended value)')
27
27
  self.options.add_opt_simple(:source_name,'create package from remote source (by name)')
28
28
  self.options.add_opt_simple(:storage,'Faspex local storage definition')
29
+ self.options.add_opt_simple(:recipient,'use if recipient is a dropbox (with *)')
29
30
  self.options.add_opt_list(:box,[:inbox,:sent,:archive],'package box')
30
31
  self.options.set_option(:box,:inbox)
31
32
  self.options.parse_options!
@@ -44,7 +45,7 @@ module Aspera
44
45
  result={
45
46
  :base_url => "#{publink_uri.scheme}://#{publink_uri.host}#{port_add}#{base}",
46
47
  :subpath => subpath,
47
- :query => URI::decode_www_form(publink_uri.query).inject({}){|m,v|m[v.first]=v.last;m}
48
+ :query => URI::decode_www_form(publink_uri.query).inject({}){|h,v|h[v.first]=v.last;h}
48
49
  }
49
50
  Log.dump('publink',result)
50
51
  return result
@@ -52,7 +53,7 @@ module Aspera
52
53
 
53
54
  # get faspe: URI from entry in xml, and fix problems..
54
55
  def self.get_fasp_uri_from_entry(entry)
55
- raise CliBadArgument, 'package is empty' unless entry.has_key?('link')
56
+ raise CliBadArgument, 'package has no link (deleted?)' unless entry.has_key?('link')
56
57
  result=entry['link'].select{|e| e['rel'].eql?('package')}.first['href']
57
58
  # tags in the end of URL is not well % encoded... there are "=" that should be %3D
58
59
  # TODO: enter ticket to Faspex ?
@@ -72,7 +73,7 @@ module Aspera
72
73
  def self.get_source_id(source_list,source_name)
73
74
  source_ids=source_list.select { |i| i['name'].eql?(source_name) }
74
75
  if source_ids.empty?
75
- raise CliError,"No such Faspex source #{field_sym.to_s}: #{field_value} in [#{source_list.map{|i| i[field_sym.to_s]}.join(', ')}]"
76
+ raise CliError,%Q{No such Faspex source "#{source_name}" in [#{source_list.map{|i| %Q{"#{i['name']}"}}.join(', ')}]}
76
77
  end
77
78
  return source_ids.first['id']
78
79
  end
@@ -101,19 +102,26 @@ module Aspera
101
102
  return @api_v4
102
103
  end
103
104
 
104
- ACTIONS=[ :nagios_check,:package, :source, :me, :dropbox, :v4, :address_book, :login_methods ]
105
+ ACTIONS=[ :health,:package, :source, :me, :dropbox, :v4, :address_book, :login_methods ]
105
106
 
106
107
  # we match recv command on atom feed on this field
107
108
  PACKAGE_MATCH_FIELD='package_id'
108
109
 
109
110
  def mailbox_all_entries
110
- mailbox=self.options.get_option(:box,:mandatory).to_s
111
+ recipient_name=self.options.get_option(:recipient,:optional) || self.options.get_option(:username,:mandatory)
112
+ mailbox=self.options.get_option(:box,:mandatory)
111
113
  all_inbox_xml=api_v3.call({:operation=>'GET',:subpath=>"#{mailbox}.atom",:headers=>{'Accept'=>'application/xml'}})[:http].body
112
114
  all_inbox_data=XmlSimple.xml_in(all_inbox_xml, {'ForceArray' => true})
113
115
  Log.dump(:all_inbox_data,all_inbox_data)
114
116
  result=all_inbox_data.has_key?('entry') ? all_inbox_data['entry'] : []
115
117
  result.each do |e|
116
- e[PACKAGE_MATCH_FIELD]=e['to'].first['recipient_delivery_id'].first
118
+ case mailbox
119
+ when :inbox,:archive
120
+ recipient=e['to'].select{|i|i['name'].first.eql?(recipient_name)}.first
121
+ e[PACKAGE_MATCH_FIELD]=recipient.nil? ? 'n/a' : recipient['recipient_delivery_id'].first
122
+ when :sent
123
+ e[PACKAGE_MATCH_FIELD]=e['delivery_id'].first
124
+ end
117
125
  end
118
126
  # remove dropbox packages
119
127
  result.select!{|p|p['metadata'].first['field'].select{|j|j['name'].eql?('_dropbox_name')}.empty? rescue false}
@@ -155,7 +163,7 @@ module Aspera
155
163
  def execute_action
156
164
  command=self.options.get_next_command(ACTIONS)
157
165
  case command
158
- when :nagios_check
166
+ when :health
159
167
  nagios=Nagios.new
160
168
  begin
161
169
  api_v3.read('me')
@@ -201,9 +209,18 @@ module Aspera
201
209
  #Log.dump('transfer_spec',transfer_spec)
202
210
  return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
203
211
  when :recv
204
- public_link_url=self.options.get_option(:link,:optional)
205
- if !public_link_url.nil?
206
- link_data=self.class.get_link_data(public_link_url)
212
+ link_url=self.options.get_option(:link,:optional)
213
+ # list of faspex ID/URI to download
214
+ pkg_id_uri=nil
215
+ skip_ids_data=[]
216
+ skip_ids_persistency=nil
217
+ case link_url
218
+ when nil
219
+ # usual case: no link
220
+ when /^faspe:/
221
+ pkg_id_uri=[{:id=>'package',:uri=>link_url}]
222
+ else
223
+ link_data=self.class.get_link_data(link_url)
207
224
  if !link_data[:subpath].match(%r{external_deliveries/})
208
225
  raise CliBadArgument,"pub link is #{link_data[:subpath]}, expecting external_deliveries/"
209
226
  end
@@ -211,7 +228,7 @@ module Aspera
211
228
  api_public_link=Rest.new({:base_url=>link_data[:base_url]})
212
229
  pkgdatares=api_public_link.call({:operation=>'GET',:subpath=>link_data[:subpath],:url_params=>{:passcode=>link_data[:query]['passcode']},:headers=>{'Accept'=>'application/xml'}})
213
230
  if !pkgdatares[:http].body.start_with?('<?xml ')
214
- OpenApplication.instance.uri(public_link_url)
231
+ OpenApplication.instance.uri(link_url)
215
232
  raise CliError, 'no such package'
216
233
  end
217
234
  package_entry=XmlSimple.xml_in(pkgdatares[:http].body, {'ForceArray' => false})
@@ -220,32 +237,30 @@ module Aspera
220
237
  transfer_spec['direction']='receive'
221
238
  return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
222
239
  end # public link
223
- # get command line parameters
224
- delivid=self.options.get_option(:id,:mandatory)
225
- # list of faspex ID/URI to download
226
- pkg_id_uri=nil
227
- skip_ids_data=[]
228
- skip_ids_persistency=nil
229
- if self.options.get_option(:once_only,:mandatory)
230
- skip_ids_persistency=PersistencyActionOnce.new(
231
- manager: @agents[:persistency],
232
- data: skip_ids_data,
233
- ids: ['faspex_recv',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),self.options.get_option(:box,:mandatory).to_s])
234
- end
235
- if delivid.eql?(VAL_ALL)
236
- pkg_id_uri=mailbox_all_entries.map{|i|{:id=>i[PACKAGE_MATCH_FIELD],:uri=>self.class.get_fasp_uri_from_entry(i)}}
237
- # TODO : remove ids from skip not present in inbox
238
- # skip_ids_data.select!{|id|pkg_id_uri.select{|p|p[:id].eql?(id)}}
239
- pkg_id_uri.select!{|i|!skip_ids_data.include?(i[:id])}
240
- else
241
- # TODO: delivery id is the right one if package was receive by group
242
- endpoint=case self.options.get_option(:box,:mandatory)
243
- when :inbox,:archive;'received'
244
- when :sent;'sent'
240
+ if pkg_id_uri.nil?
241
+ # get command line parameters
242
+ delivid=self.options.get_option(:id,:mandatory)
243
+ if self.options.get_option(:once_only,:mandatory)
244
+ skip_ids_persistency=PersistencyActionOnce.new(
245
+ manager: @agents[:persistency],
246
+ data: skip_ids_data,
247
+ ids: ['faspex_recv',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),self.options.get_option(:box,:mandatory).to_s])
248
+ end
249
+ if delivid.eql?(VAL_ALL)
250
+ pkg_id_uri=mailbox_all_entries.map{|i|{:id=>i[PACKAGE_MATCH_FIELD],:uri=>self.class.get_fasp_uri_from_entry(i)}}
251
+ # TODO : remove ids from skip not present in inbox
252
+ # skip_ids_data.select!{|id|pkg_id_uri.select{|p|p[:id].eql?(id)}}
253
+ pkg_id_uri.select!{|i|!skip_ids_data.include?(i[:id])}
254
+ else
255
+ # TODO: delivery id is the right one if package was receive by group
256
+ endpoint=case self.options.get_option(:box,:mandatory)
257
+ when :inbox,:archive;'received'
258
+ when :sent; 'sent'
259
+ end
260
+ entry_xml=api_v3.call({:operation=>'GET',:subpath=>"#{endpoint}/#{delivid}",:headers=>{'Accept'=>'application/xml'}})[:http].body
261
+ package_entry=XmlSimple.xml_in(entry_xml, {'ForceArray' => true})
262
+ pkg_id_uri=[{:id=>delivid,:uri=>self.class.get_fasp_uri_from_entry(package_entry)}]
245
263
  end
246
- entry_xml=api_v3.call({:operation=>'GET',:subpath=>"#{endpoint}/#{delivid}",:headers=>{'Accept'=>'application/xml'}})[:http].body
247
- package_entry=XmlSimple.xml_in(entry_xml, {'ForceArray' => true})
248
- pkg_id_uri=[{:id=>delivid,:uri=>self.class.get_fasp_uri_from_entry(package_entry)}]
249
264
  end
250
265
  Log.dump(:pkg_id_uri,pkg_id_uri)
251
266
  return Main.result_status('no package') if pkg_id_uri.empty?
@@ -1,5 +1,6 @@
1
1
  require 'aspera/cli/basic_auth_plugin'
2
2
  require 'aspera/persistency_action_once'
3
+ require 'securerandom'
3
4
 
4
5
  module Aspera
5
6
  module Cli
@@ -8,74 +9,114 @@ module Aspera
8
9
  VAL_ALL='ALL'
9
10
  def initialize(env)
10
11
  super(env)
11
- #self.options.add_opt_simple(:delivery_info,'package delivery information (extended value)')
12
- #self.options.parse_options!
12
+ options.add_opt_simple(:client_id,'API client identifier in application')
13
+ options.add_opt_simple(:client_secret,'API client secret in application')
14
+ options.add_opt_simple(:redirect_uri,'API client redirect URI')
15
+ options.add_opt_list(:auth,Oauth.auth_types.clone.push(:boot),'type of Oauth authentication')
16
+ options.add_opt_simple(:private_key,'RSA private key PEM value for JWT (prefix file path with @val:@file:)')
17
+ options.set_option(:auth,:jwt)
18
+ options.parse_options!
13
19
  end
14
- ACTIONS=[ :node, :package ]
15
20
 
16
- # http://apie-next-ui-shell-dev.mybluemix.net/explorer/catalog/aspera/product/ibm-aspera/api/faspex5-api/spec/openapi
21
+ def set_api
22
+ faxpex5_api_base_url=options.get_option(:url,:mandatory)
23
+ faxpex5_api_v5_url="#{faxpex5_api_base_url}/api/v5"
24
+ faxpex5_api_auth_url="#{faxpex5_api_base_url}/auth"
25
+ case options.get_option(:auth,:mandatory)
26
+ when :boot
27
+ # the password here is the token copied directly from browser in developer mode
28
+ @api_v5=Rest.new({
29
+ :base_url => faxpex5_api_v5_url,
30
+ :headers => {'Authorization'=>options.get_option(:password,:mandatory)},
31
+ })
32
+ when :web
33
+ # opens a browser and ask user to auth using web
34
+ @api_v5=Rest.new({
35
+ :base_url => faxpex5_api_v5_url,
36
+ :auth => {
37
+ :type => :oauth2,
38
+ :base_url => faxpex5_api_auth_url,
39
+ :grant => :web,
40
+ :state => SecureRandom.uuid,
41
+ :client_id => options.get_option(:client_id,:mandatory),
42
+ :redirect_uri => options.get_option(:redirect_uri,:mandatory),
43
+ }})
44
+ when :jwt
45
+ # currently Faspex 5 beta 3 only supports non-user based apis (e.g. jobs)
46
+ app_client_id=options.get_option(:client_id,:mandatory)
47
+ @api_v5=Rest.new({
48
+ :base_url => faxpex5_api_v5_url,
49
+ :auth => {
50
+ :type => :oauth2,
51
+ :base_url => faxpex5_api_auth_url,
52
+ :grant => :jwt,
53
+ :client_id => app_client_id,
54
+ :client_secret => options.get_option(:client_secret,:mandatory),
55
+ :jwt_subject => "client:#{app_client_id}", # TODO Mmmm
56
+ :jwt_audience => app_client_id, # TODO Mmmm
57
+ :jwt_private_key_obj => OpenSSL::PKey::RSA.new(options.get_option(:private_key,:mandatory)),
58
+ :jwt_is_f5 => true, # TODO: remove when clarified
59
+ :jwt_headers => {typ: 'JWT'}
60
+ }})
61
+ end
62
+ end
63
+
64
+ ACTIONS=[ :node, :package, :auth_client, :jobs ]
65
+
66
+ #
17
67
  def execute_action
18
- # get parameters
19
- faxpex5_api_base_url=self.options.get_option(:url,:mandatory)
20
- faxpex5_username=self.options.get_option(:username,:mandatory)
21
- faxpex5_password=self.options.get_option(:password,:mandatory)
22
- faxpex5_api_base_url+='/api/v5'
23
- # create object for REST calls to Shares2
24
- api_v5=Rest.new({
25
- :base_url => faxpex5_api_base_url,
26
- :auth => {
27
- :type => :oauth2,
28
- :base_url => faxpex5_api_base_url,
29
- :grant => :body_data,
30
- :token_field =>'auth_token',
31
- :path_token => 'authenticate',
32
- :path_authorize => :unused,
33
- :userpass_body => {name: faxpex5_username,password: faxpex5_password}
34
- }})
35
- command=self.options.get_next_command(ACTIONS)
68
+ set_api
69
+ command=options.get_next_command(ACTIONS)
36
70
  case command
71
+ when :auth_client
72
+ api_auth=Rest.new(@api_v5.params.merge({base_url: @api_v5.params[:base_url].gsub(/api\/v5$/,'auth')}))
73
+ return self.entity_action(api_auth,'oauth_clients',nil,:id,nil,true)
37
74
  when :node
38
- return self.entity_action(api_v5,'nodes',nil,:id,nil,true)
75
+ return self.entity_action(@api_v5,'nodes',nil,:id,nil,true)
76
+ when :jobs
77
+ # to test JWT
78
+ return self.entity_action(@api_v5,'jobs',nil,:id,nil,true)
39
79
  when :package
40
- command=self.options.get_next_command([:list,:show,:send,:receive])
80
+ command=options.get_next_command([:list,:show,:send,:receive])
41
81
  case command
42
82
  when :list
43
- parameters=self.options.get_option(:value,:optional)
44
- return {:type => :object_list, :data=>api_v5.read('packages',parameters)[:data]['packages']}
83
+ parameters=options.get_option(:value,:optional)
84
+ return {:type => :object_list, :data=>@api_v5.read('packages',parameters)[:data]['packages']}
45
85
  when :show
46
- id=self.options.get_option(:id,:mandatory)
47
- return {:type => :single_object, :data=>api_v5.read("packages/#{id}")[:data]}
86
+ id=options.get_option(:id,:mandatory)
87
+ return {:type => :single_object, :data=>@api_v5.read("packages/#{id}")[:data]}
48
88
  when :send
49
- parameters=self.options.get_option(:value,:mandatory)
50
- raise CliBadArgument,'package value must be hash, refer to API' unless parameters.is_a?(Hash)
51
- package=api_v5.create('packages',parameters)[:data]
52
- transfer_spec=api_v5.create("packages/#{package['id']}/transfer_spec/upload",{transfer_type: 'Connect'})[:data]
89
+ parameters=options.get_option(:value,:mandatory)
90
+ raise CliBadArgument,'value must be hash, refer to API' unless parameters.is_a?(Hash)
91
+ package=@api_v5.create('packages',parameters)[:data]
92
+ transfer_spec=@api_v5.create("packages/#{package['id']}/transfer_spec/upload",{transfer_type: 'Connect'})[:data]
53
93
  transfer_spec.delete('authentication')
54
94
  return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
55
95
  when :receive
56
96
  pkg_type='received'
57
- pack_id=self.options.get_option(:id,:mandatory)
97
+ pack_id=options.get_option(:id,:mandatory)
58
98
  package_ids=[pack_id]
59
99
  skip_ids_data=[]
60
100
  skip_ids_persistency=nil
61
- if self.options.get_option(:once_only,:mandatory)
101
+ if options.get_option(:once_only,:mandatory)
102
+ # read ids from persistency
62
103
  skip_ids_persistency=PersistencyActionOnce.new(
63
104
  manager: @agents[:persistency],
64
105
  data: skip_ids_data,
65
- ids: ['faspex_recv',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),pkg_type])
106
+ ids: ['faspex_recv',options.get_option(:url,:mandatory),options.get_option(:username,:mandatory),pkg_type])
66
107
  end
67
108
  if pack_id.eql?(VAL_ALL)
68
- # todo: if packages have same name, they will overwrite
69
- parameters=self.options.get_option(:value,:optional)
70
- parameters||={"type"=>"received","subtype"=>"mypackages","limit"=>1000}
71
- raise CliBadArgument,'value filter must be hash (API GET)' unless parameters.is_a?(Hash)
72
- package_ids=api_v5.read('packages',parameters)[:data]['packages'].map{|p|p['id']}
109
+ # TODO: if packages have same name, they will overwrite
110
+ parameters=options.get_option(:value,:optional)
111
+ parameters||={'type'=>'received','subtype'=>'mypackages','limit'=>1000}
112
+ raise CliBadArgument,'value filter must be Hash (API GET)' unless parameters.is_a?(Hash)
113
+ package_ids=@api_v5.read('packages',parameters)[:data]['packages'].map{|p|p['id']}
73
114
  package_ids.select!{|i|!skip_ids_data.include?(i)}
74
115
  end
75
116
  result_transfer=[]
76
117
  package_ids.each do |id|
77
118
  # TODO: allow from sent as well ?
78
- transfer_spec=api_v5.create("packages/#{id}/transfer_spec/download",{transfer_type: 'Connect', type: pkg_type})[:data]
119
+ transfer_spec=@api_v5.create("packages/#{id}/transfer_spec/download",{transfer_type: 'Connect', type: pkg_type})[:data]
79
120
  transfer_spec.delete('authentication')
80
121
  statuses=self.transfer.start(transfer_spec,{:src=>:node_gen3})
81
122
  result_transfer.push({'package'=>id,'status'=>statuses.map{|i|i.to_s}.join(',')})
@@ -88,7 +88,7 @@ module Aspera
88
88
  raise StandardError,"expect: nil, String or Array"
89
89
  end
90
90
 
91
- SIMPLE_ACTIONS=[:nagios_check,:events, :space, :info, :license, :mkdir, :mklink, :mkfile, :rename, :delete, :search ]
91
+ SIMPLE_ACTIONS=[:health,:events, :space, :info, :license, :mkdir, :mklink, :mkfile, :rename, :delete, :search ]
92
92
 
93
93
  COMMON_ACTIONS=[:browse, :upload, :download, :api_details ].concat(SIMPLE_ACTIONS)
94
94
 
@@ -96,7 +96,7 @@ module Aspera
96
96
  # prefix_path is used to list remote sources in Faspex
97
97
  def execute_simple_common(command,prefix_path)
98
98
  case command
99
- when :nagios_check
99
+ when :health
100
100
  nagios=Nagios.new
101
101
  begin
102
102
  info=@api_node.read('info')[:data]
@@ -341,7 +341,7 @@ module Aspera
341
341
  raise "error"
342
342
  end
343
343
  when :access_key
344
- return self.entity_action(@api_node,'access_keys',['id','root_file_id','storage'],:id,'self')
344
+ return self.entity_action(@api_node,'access_keys',nil,:id,'self')
345
345
  when :service
346
346
  command=self.options.get_next_command([ :list, :create, :delete])
347
347
  if [:delete].include?(command)
@@ -4,6 +4,7 @@ require 'aspera/preview/options'
4
4
  require 'aspera/preview/utils'
5
5
  require 'aspera/preview/file_types'
6
6
  require 'aspera/persistency_action_once'
7
+ require 'aspera/node'
7
8
  require 'aspera/hash_ext'
8
9
  require 'date'
9
10
  require 'securerandom'
@@ -56,9 +57,11 @@ module Aspera
56
57
  self.options.add_opt_simple(:case,'basename of output for for test')
57
58
  self.options.add_opt_simple(:scan_path,'subpath in folder id to start scan in (default=/)')
58
59
  self.options.add_opt_simple(:scan_id,'forder id in storage to start scan in, default is access key main folder id')
60
+ self.options.add_opt_boolean(:mimemagic,'use Mime type detection of gem mimemagic')
59
61
  self.options.add_opt_list(:overwrite,[:always,:never,:mtime],'when to overwrite result file')
60
62
  self.options.add_opt_list(:file_access,[:local,:remote],'how to read and write files in repository')
61
63
  self.options.set_option(:temp_folder,Dir.tmpdir)
64
+ self.options.set_option(:mimemagic,:false)
62
65
 
63
66
  # add other options for generator (and set default values)
64
67
  Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
@@ -100,8 +103,6 @@ module Aspera
100
103
  return @preview_formats_to_generate.map{|i|i.to_s}.join(',')
101
104
  end
102
105
 
103
- ACTIONS=[:scan,:events,:trevents,:check,:test]
104
-
105
106
  # /files/id/files is normally cached in redis, but we can discard the cache
106
107
  # but /files/id is not cached
107
108
  def get_folder_entries(file_id,request_args=nil)
@@ -112,14 +113,23 @@ module Aspera
112
113
  end
113
114
 
114
115
  # old version based on folders
115
- def process_transfer_events(iteration_token)
116
+ def process_trevents(iteration_token)
116
117
  events_filter={
117
118
  'access_key'=>@access_key_self['id'],
118
119
  'type'=>'download.ended'
119
120
  }
120
121
  # optionally by iteration token
121
122
  events_filter['iteration_token']=iteration_token unless iteration_token.nil?
122
- events=@api_node.read('events',events_filter)[:data]
123
+ begin
124
+ events=@api_node.read('events',events_filter)[:data]
125
+ rescue RestCallError => e
126
+ if e.message.include?('Invalid iteration_token')
127
+ Log.log.warn("Retrying without iteration token: #{e}")
128
+ events_filter.delete('iteration_token')
129
+ retry
130
+ end
131
+ raise e
132
+ end
123
133
  return if events.empty?
124
134
  events.each do |event|
125
135
  next unless event['data']['direction'].eql?('receive')
@@ -137,7 +147,7 @@ module Aspera
137
147
  end
138
148
 
139
149
  # requests recent events on node api and process newly modified folders
140
- def process_file_events(iteration_token)
150
+ def process_events(iteration_token)
141
151
  # get new file creation by access key (TODO: what if file already existed?)
142
152
  events_filter={
143
153
  'access_key'=>@access_key_self['id'],
@@ -276,7 +286,7 @@ module Aspera
276
286
  end
277
287
  end
278
288
  # need generator for further checks
279
- gen_info[:generator]=Aspera::Preview::Generator.new(@gen_options,gen_info[:src],gen_info[:dst],@tmp_folder,entry['content_type'],false)
289
+ gen_info[:generator]=Aspera::Preview::Generator.new(@gen_options,gen_info[:src],gen_info[:dst],@tmp_folder,entry['content_type'])
280
290
  # get conversion_type (if known) and check if supported
281
291
  next false unless gen_info[:generator].supported?
282
292
  # shall we skip it ?
@@ -313,12 +323,14 @@ module Aspera
313
323
  end # generate_preview
314
324
 
315
325
  # scan all files in provided folder entry
326
+ # @param scan_start subpath to start folder scan inside
316
327
  def scan_folder_files(top_entry,scan_start=nil)
317
328
  if !scan_start.nil?
318
329
  # canonical path: start with / and ends with /
319
330
  scan_start='/'+scan_start.split('/').select{|i|!i.empty?}.join('/')
320
331
  scan_start="#{scan_start}/" #unless scan_start.end_with?('/')
321
332
  end
333
+ filter_block=Aspera::Node.file_matcher(options.get_option(:value,:optional))
322
334
  Log.log.debug("scan: #{top_entry} : #{scan_start}".green)
323
335
  # don't use recursive call, use list instead
324
336
  entries_to_process=[top_entry]
@@ -334,7 +346,11 @@ module Aspera
334
346
  Log.log.debug("item:#{entry}")
335
347
  case entry['type']
336
348
  when 'file'
337
- generate_preview(entry)
349
+ if filter_block.call(entry)
350
+ generate_preview(entry)
351
+ else
352
+ Log.log.debug('skip by filter')
353
+ end
338
354
  when 'link'
339
355
  Log.log.debug('Ignoring link.')
340
356
  when 'folder'
@@ -360,6 +376,8 @@ module Aspera
360
376
  end
361
377
  end
362
378
 
379
+ ACTIONS=[:scan,:events,:trevents,:check,:test]
380
+
363
381
  def execute_action
364
382
  command=self.options.get_next_command(ACTIONS)
365
383
  unless [:check,:test].include?(command)
@@ -402,6 +420,7 @@ module Aspera
402
420
  end
403
421
  end
404
422
  end
423
+ Aspera::Preview::FileTypes.instance.use_mimemagic = self.options.get_option(:mimemagic,:mandatory)
405
424
  case command
406
425
  when :scan
407
426
  scan_path=self.options.get_option(:scan_path,:optional)
@@ -417,30 +436,20 @@ module Aspera
417
436
  end
418
437
  scan_folder_files(folder_info,scan_path)
419
438
  return Main.result_status('scan finished')
420
- when :events
439
+ when :events,:trevents
421
440
  iteration_data=[]
422
441
  iteration_persistency=nil
423
442
  if self.options.get_option(:once_only,:mandatory)
424
443
  iteration_persistency=PersistencyActionOnce.new(
425
444
  manager: @agents[:persistency],
426
445
  data: iteration_data,
427
- ids: ['preview_iteration_events',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
446
+ ids: ["preview_iteration_#{command}",self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
428
447
  end
429
- iteration_data[0]=process_file_events(iteration_data[0])
430
- iteration_persistency.save unless iteration_persistency.nil?
431
- return Main.result_status('events finished')
432
- when :trevents
433
- iteration_data=[]
434
- iteration_persistency=nil
435
- if self.options.get_option(:once_only,:mandatory)
436
- iteration_persistency=PersistencyActionOnce.new(
437
- manager: @agents[:persistency],
438
- data: iteration_data,
439
- ids: ['preview_iteration_transfer',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
440
- end
441
- iteration_data[0]=process_transfer_events(iteration_data[0])
448
+
449
+ # call method specified
450
+ iteration_data[0]=send("process_#{command}",iteration_data[0])
442
451
  iteration_persistency.save unless iteration_persistency.nil?
443
- return Main.result_status('trevents finished')
452
+ return Main.result_status("#{command} finished")
444
453
  when :check
445
454
  Aspera::Preview::Utils.check_tools(@skip_types)
446
455
  return Main.result_status('tools validated')
@@ -448,8 +457,9 @@ module Aspera
448
457
  format = self.options.get_next_argument('format',Aspera::Preview::Generator::PREVIEW_FORMATS)
449
458
  source = self.options.get_next_argument('source file')
450
459
  dest=preview_filename(format,self.options.get_option(:case,:optional))
451
- g=Aspera::Preview::Generator.new(@gen_options,source,dest,@tmp_folder)
452
- raise "format not supported: #{format}" unless g.supported?
460
+ g=Aspera::Preview::Generator.new(@gen_options,source,dest,@tmp_folder,nil)
461
+ raise "cannot find file type for #{source}" if g.conversion_type.nil?
462
+ raise "out format #{format} not supported" unless g.supported?
453
463
  g.generate
454
464
  return Main.result_status("generated: #{dest}")
455
465
  else