aspera-cli 4.0.0.pre3 → 4.2.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +695 -205
  3. data/bin/dascli +13 -0
  4. data/docs/README.erb.md +615 -157
  5. data/docs/test_env.conf +23 -5
  6. data/docs/transfer_spec.html +1 -1
  7. data/examples/aoc.rb +14 -3
  8. data/examples/faspex4.rb +78 -0
  9. data/lib/aspera/aoc.rb +87 -108
  10. data/lib/aspera/cli/formater.rb +2 -0
  11. data/lib/aspera/cli/main.rb +46 -34
  12. data/lib/aspera/cli/plugin.rb +9 -4
  13. data/lib/aspera/cli/plugins/alee.rb +1 -1
  14. data/lib/aspera/cli/plugins/aoc.rb +207 -182
  15. data/lib/aspera/cli/plugins/ats.rb +2 -2
  16. data/lib/aspera/cli/plugins/config.rb +173 -117
  17. data/lib/aspera/cli/plugins/console.rb +2 -2
  18. data/lib/aspera/cli/plugins/faspex.rb +51 -36
  19. data/lib/aspera/cli/plugins/faspex5.rb +82 -41
  20. data/lib/aspera/cli/plugins/node.rb +3 -3
  21. data/lib/aspera/cli/plugins/preview.rb +35 -25
  22. data/lib/aspera/cli/plugins/server.rb +23 -8
  23. data/lib/aspera/cli/transfer_agent.rb +7 -6
  24. data/lib/aspera/cli/version.rb +1 -1
  25. data/lib/aspera/cos_node.rb +33 -28
  26. data/lib/aspera/environment.rb +2 -2
  27. data/lib/aspera/fasp/connect.rb +28 -21
  28. data/lib/aspera/fasp/http_gw.rb +140 -28
  29. data/lib/aspera/fasp/installation.rb +101 -53
  30. data/lib/aspera/fasp/local.rb +88 -45
  31. data/lib/aspera/fasp/manager.rb +15 -0
  32. data/lib/aspera/fasp/node.rb +4 -4
  33. data/lib/aspera/fasp/parameters.rb +6 -18
  34. data/lib/aspera/fasp/resume_policy.rb +13 -12
  35. data/lib/aspera/log.rb +1 -1
  36. data/lib/aspera/node.rb +61 -1
  37. data/lib/aspera/oauth.rb +49 -46
  38. data/lib/aspera/persistency_folder.rb +9 -4
  39. data/lib/aspera/preview/file_types.rb +53 -21
  40. data/lib/aspera/preview/generator.rb +3 -3
  41. data/lib/aspera/rest.rb +29 -18
  42. data/lib/aspera/secrets.rb +20 -0
  43. data/lib/aspera/temp_file_manager.rb +19 -0
  44. data/lib/aspera/web_auth.rb +105 -0
  45. metadata +42 -22
@@ -14,13 +14,13 @@ module Aspera
14
14
  self.options.parse_options!
15
15
  end
16
16
 
17
- ACTIONS=[:transfer,:nagios_check]
17
+ ACTIONS=[:transfer,:health]
18
18
 
19
19
  def execute_action
20
20
  api_console=basic_auth_api('api')
21
21
  command=self.options.get_next_command(ACTIONS)
22
22
  case command
23
- when :nagios_check
23
+ when :health
24
24
  nagios=Nagios.new
25
25
  begin
26
26
  api_console.read('ssh_keys')
@@ -26,6 +26,7 @@ module Aspera
26
26
  self.options.add_opt_simple(:delivery_info,'package delivery information (extended value)')
27
27
  self.options.add_opt_simple(:source_name,'create package from remote source (by name)')
28
28
  self.options.add_opt_simple(:storage,'Faspex local storage definition')
29
+ self.options.add_opt_simple(:recipient,'use if recipient is a dropbox (with *)')
29
30
  self.options.add_opt_list(:box,[:inbox,:sent,:archive],'package box')
30
31
  self.options.set_option(:box,:inbox)
31
32
  self.options.parse_options!
@@ -44,7 +45,7 @@ module Aspera
44
45
  result={
45
46
  :base_url => "#{publink_uri.scheme}://#{publink_uri.host}#{port_add}#{base}",
46
47
  :subpath => subpath,
47
- :query => URI::decode_www_form(publink_uri.query).inject({}){|m,v|m[v.first]=v.last;m}
48
+ :query => URI::decode_www_form(publink_uri.query).inject({}){|h,v|h[v.first]=v.last;h}
48
49
  }
49
50
  Log.dump('publink',result)
50
51
  return result
@@ -52,7 +53,7 @@ module Aspera
52
53
 
53
54
  # get faspe: URI from entry in xml, and fix problems..
54
55
  def self.get_fasp_uri_from_entry(entry)
55
- raise CliBadArgument, 'package is empty' unless entry.has_key?('link')
56
+ raise CliBadArgument, 'package has no link (deleted?)' unless entry.has_key?('link')
56
57
  result=entry['link'].select{|e| e['rel'].eql?('package')}.first['href']
57
58
  # tags in the end of URL is not well % encoded... there are "=" that should be %3D
58
59
  # TODO: enter ticket to Faspex ?
@@ -72,7 +73,7 @@ module Aspera
72
73
  def self.get_source_id(source_list,source_name)
73
74
  source_ids=source_list.select { |i| i['name'].eql?(source_name) }
74
75
  if source_ids.empty?
75
- raise CliError,"No such Faspex source #{field_sym.to_s}: #{field_value} in [#{source_list.map{|i| i[field_sym.to_s]}.join(', ')}]"
76
+ raise CliError,%Q{No such Faspex source "#{source_name}" in [#{source_list.map{|i| %Q{"#{i['name']}"}}.join(', ')}]}
76
77
  end
77
78
  return source_ids.first['id']
78
79
  end
@@ -101,19 +102,26 @@ module Aspera
101
102
  return @api_v4
102
103
  end
103
104
 
104
- ACTIONS=[ :nagios_check,:package, :source, :me, :dropbox, :v4, :address_book, :login_methods ]
105
+ ACTIONS=[ :health,:package, :source, :me, :dropbox, :v4, :address_book, :login_methods ]
105
106
 
106
107
  # we match recv command on atom feed on this field
107
108
  PACKAGE_MATCH_FIELD='package_id'
108
109
 
109
110
  def mailbox_all_entries
110
- mailbox=self.options.get_option(:box,:mandatory).to_s
111
+ recipient_name=self.options.get_option(:recipient,:optional) || self.options.get_option(:username,:mandatory)
112
+ mailbox=self.options.get_option(:box,:mandatory)
111
113
  all_inbox_xml=api_v3.call({:operation=>'GET',:subpath=>"#{mailbox}.atom",:headers=>{'Accept'=>'application/xml'}})[:http].body
112
114
  all_inbox_data=XmlSimple.xml_in(all_inbox_xml, {'ForceArray' => true})
113
115
  Log.dump(:all_inbox_data,all_inbox_data)
114
116
  result=all_inbox_data.has_key?('entry') ? all_inbox_data['entry'] : []
115
117
  result.each do |e|
116
- e[PACKAGE_MATCH_FIELD]=e['to'].first['recipient_delivery_id'].first
118
+ case mailbox
119
+ when :inbox,:archive
120
+ recipient=e['to'].select{|i|i['name'].first.eql?(recipient_name)}.first
121
+ e[PACKAGE_MATCH_FIELD]=recipient.nil? ? 'n/a' : recipient['recipient_delivery_id'].first
122
+ when :sent
123
+ e[PACKAGE_MATCH_FIELD]=e['delivery_id'].first
124
+ end
117
125
  end
118
126
  # remove dropbox packages
119
127
  result.select!{|p|p['metadata'].first['field'].select{|j|j['name'].eql?('_dropbox_name')}.empty? rescue false}
@@ -155,7 +163,7 @@ module Aspera
155
163
  def execute_action
156
164
  command=self.options.get_next_command(ACTIONS)
157
165
  case command
158
- when :nagios_check
166
+ when :health
159
167
  nagios=Nagios.new
160
168
  begin
161
169
  api_v3.read('me')
@@ -201,9 +209,18 @@ module Aspera
201
209
  #Log.dump('transfer_spec',transfer_spec)
202
210
  return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
203
211
  when :recv
204
- public_link_url=self.options.get_option(:link,:optional)
205
- if !public_link_url.nil?
206
- link_data=self.class.get_link_data(public_link_url)
212
+ link_url=self.options.get_option(:link,:optional)
213
+ # list of faspex ID/URI to download
214
+ pkg_id_uri=nil
215
+ skip_ids_data=[]
216
+ skip_ids_persistency=nil
217
+ case link_url
218
+ when nil
219
+ # usual case: no link
220
+ when /^faspe:/
221
+ pkg_id_uri=[{:id=>'package',:uri=>link_url}]
222
+ else
223
+ link_data=self.class.get_link_data(link_url)
207
224
  if !link_data[:subpath].match(%r{external_deliveries/})
208
225
  raise CliBadArgument,"pub link is #{link_data[:subpath]}, expecting external_deliveries/"
209
226
  end
@@ -211,7 +228,7 @@ module Aspera
211
228
  api_public_link=Rest.new({:base_url=>link_data[:base_url]})
212
229
  pkgdatares=api_public_link.call({:operation=>'GET',:subpath=>link_data[:subpath],:url_params=>{:passcode=>link_data[:query]['passcode']},:headers=>{'Accept'=>'application/xml'}})
213
230
  if !pkgdatares[:http].body.start_with?('<?xml ')
214
- OpenApplication.instance.uri(public_link_url)
231
+ OpenApplication.instance.uri(link_url)
215
232
  raise CliError, 'no such package'
216
233
  end
217
234
  package_entry=XmlSimple.xml_in(pkgdatares[:http].body, {'ForceArray' => false})
@@ -220,32 +237,30 @@ module Aspera
220
237
  transfer_spec['direction']='receive'
221
238
  return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
222
239
  end # public link
223
- # get command line parameters
224
- delivid=self.options.get_option(:id,:mandatory)
225
- # list of faspex ID/URI to download
226
- pkg_id_uri=nil
227
- skip_ids_data=[]
228
- skip_ids_persistency=nil
229
- if self.options.get_option(:once_only,:mandatory)
230
- skip_ids_persistency=PersistencyActionOnce.new(
231
- manager: @agents[:persistency],
232
- data: skip_ids_data,
233
- ids: ['faspex_recv',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),self.options.get_option(:box,:mandatory).to_s])
234
- end
235
- if delivid.eql?(VAL_ALL)
236
- pkg_id_uri=mailbox_all_entries.map{|i|{:id=>i[PACKAGE_MATCH_FIELD],:uri=>self.class.get_fasp_uri_from_entry(i)}}
237
- # TODO : remove ids from skip not present in inbox
238
- # skip_ids_data.select!{|id|pkg_id_uri.select{|p|p[:id].eql?(id)}}
239
- pkg_id_uri.select!{|i|!skip_ids_data.include?(i[:id])}
240
- else
241
- # TODO: delivery id is the right one if package was receive by group
242
- endpoint=case self.options.get_option(:box,:mandatory)
243
- when :inbox,:archive;'received'
244
- when :sent;'sent'
240
+ if pkg_id_uri.nil?
241
+ # get command line parameters
242
+ delivid=self.options.get_option(:id,:mandatory)
243
+ if self.options.get_option(:once_only,:mandatory)
244
+ skip_ids_persistency=PersistencyActionOnce.new(
245
+ manager: @agents[:persistency],
246
+ data: skip_ids_data,
247
+ ids: ['faspex_recv',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),self.options.get_option(:box,:mandatory).to_s])
248
+ end
249
+ if delivid.eql?(VAL_ALL)
250
+ pkg_id_uri=mailbox_all_entries.map{|i|{:id=>i[PACKAGE_MATCH_FIELD],:uri=>self.class.get_fasp_uri_from_entry(i)}}
251
+ # TODO : remove ids from skip not present in inbox
252
+ # skip_ids_data.select!{|id|pkg_id_uri.select{|p|p[:id].eql?(id)}}
253
+ pkg_id_uri.select!{|i|!skip_ids_data.include?(i[:id])}
254
+ else
255
+ # TODO: delivery id is the right one if package was receive by group
256
+ endpoint=case self.options.get_option(:box,:mandatory)
257
+ when :inbox,:archive;'received'
258
+ when :sent; 'sent'
259
+ end
260
+ entry_xml=api_v3.call({:operation=>'GET',:subpath=>"#{endpoint}/#{delivid}",:headers=>{'Accept'=>'application/xml'}})[:http].body
261
+ package_entry=XmlSimple.xml_in(entry_xml, {'ForceArray' => true})
262
+ pkg_id_uri=[{:id=>delivid,:uri=>self.class.get_fasp_uri_from_entry(package_entry)}]
245
263
  end
246
- entry_xml=api_v3.call({:operation=>'GET',:subpath=>"#{endpoint}/#{delivid}",:headers=>{'Accept'=>'application/xml'}})[:http].body
247
- package_entry=XmlSimple.xml_in(entry_xml, {'ForceArray' => true})
248
- pkg_id_uri=[{:id=>delivid,:uri=>self.class.get_fasp_uri_from_entry(package_entry)}]
249
264
  end
250
265
  Log.dump(:pkg_id_uri,pkg_id_uri)
251
266
  return Main.result_status('no package') if pkg_id_uri.empty?
@@ -1,5 +1,6 @@
1
1
  require 'aspera/cli/basic_auth_plugin'
2
2
  require 'aspera/persistency_action_once'
3
+ require 'securerandom'
3
4
 
4
5
  module Aspera
5
6
  module Cli
@@ -8,74 +9,114 @@ module Aspera
8
9
  VAL_ALL='ALL'
9
10
  def initialize(env)
10
11
  super(env)
11
- #self.options.add_opt_simple(:delivery_info,'package delivery information (extended value)')
12
- #self.options.parse_options!
12
+ options.add_opt_simple(:client_id,'API client identifier in application')
13
+ options.add_opt_simple(:client_secret,'API client secret in application')
14
+ options.add_opt_simple(:redirect_uri,'API client redirect URI')
15
+ options.add_opt_list(:auth,Oauth.auth_types.clone.push(:boot),'type of Oauth authentication')
16
+ options.add_opt_simple(:private_key,'RSA private key PEM value for JWT (prefix file path with @val:@file:)')
17
+ options.set_option(:auth,:jwt)
18
+ options.parse_options!
13
19
  end
14
- ACTIONS=[ :node, :package ]
15
20
 
16
- # http://apie-next-ui-shell-dev.mybluemix.net/explorer/catalog/aspera/product/ibm-aspera/api/faspex5-api/spec/openapi
21
+ def set_api
22
+ faxpex5_api_base_url=options.get_option(:url,:mandatory)
23
+ faxpex5_api_v5_url="#{faxpex5_api_base_url}/api/v5"
24
+ faxpex5_api_auth_url="#{faxpex5_api_base_url}/auth"
25
+ case options.get_option(:auth,:mandatory)
26
+ when :boot
27
+ # the password here is the token copied directly from browser in developer mode
28
+ @api_v5=Rest.new({
29
+ :base_url => faxpex5_api_v5_url,
30
+ :headers => {'Authorization'=>options.get_option(:password,:mandatory)},
31
+ })
32
+ when :web
33
+ # opens a browser and ask user to auth using web
34
+ @api_v5=Rest.new({
35
+ :base_url => faxpex5_api_v5_url,
36
+ :auth => {
37
+ :type => :oauth2,
38
+ :base_url => faxpex5_api_auth_url,
39
+ :grant => :web,
40
+ :state => SecureRandom.uuid,
41
+ :client_id => options.get_option(:client_id,:mandatory),
42
+ :redirect_uri => options.get_option(:redirect_uri,:mandatory),
43
+ }})
44
+ when :jwt
45
+ # currently Faspex 5 beta 3 only supports non-user based apis (e.g. jobs)
46
+ app_client_id=options.get_option(:client_id,:mandatory)
47
+ @api_v5=Rest.new({
48
+ :base_url => faxpex5_api_v5_url,
49
+ :auth => {
50
+ :type => :oauth2,
51
+ :base_url => faxpex5_api_auth_url,
52
+ :grant => :jwt,
53
+ :client_id => app_client_id,
54
+ :client_secret => options.get_option(:client_secret,:mandatory),
55
+ :jwt_subject => "client:#{app_client_id}", # TODO Mmmm
56
+ :jwt_audience => app_client_id, # TODO Mmmm
57
+ :jwt_private_key_obj => OpenSSL::PKey::RSA.new(options.get_option(:private_key,:mandatory)),
58
+ :jwt_is_f5 => true, # TODO: remove when clarified
59
+ :jwt_headers => {typ: 'JWT'}
60
+ }})
61
+ end
62
+ end
63
+
64
+ ACTIONS=[ :node, :package, :auth_client, :jobs ]
65
+
66
+ #
17
67
  def execute_action
18
- # get parameters
19
- faxpex5_api_base_url=self.options.get_option(:url,:mandatory)
20
- faxpex5_username=self.options.get_option(:username,:mandatory)
21
- faxpex5_password=self.options.get_option(:password,:mandatory)
22
- faxpex5_api_base_url+='/api/v5'
23
- # create object for REST calls to Shares2
24
- api_v5=Rest.new({
25
- :base_url => faxpex5_api_base_url,
26
- :auth => {
27
- :type => :oauth2,
28
- :base_url => faxpex5_api_base_url,
29
- :grant => :body_data,
30
- :token_field =>'auth_token',
31
- :path_token => 'authenticate',
32
- :path_authorize => :unused,
33
- :userpass_body => {name: faxpex5_username,password: faxpex5_password}
34
- }})
35
- command=self.options.get_next_command(ACTIONS)
68
+ set_api
69
+ command=options.get_next_command(ACTIONS)
36
70
  case command
71
+ when :auth_client
72
+ api_auth=Rest.new(@api_v5.params.merge({base_url: @api_v5.params[:base_url].gsub(/api\/v5$/,'auth')}))
73
+ return self.entity_action(api_auth,'oauth_clients',nil,:id,nil,true)
37
74
  when :node
38
- return self.entity_action(api_v5,'nodes',nil,:id,nil,true)
75
+ return self.entity_action(@api_v5,'nodes',nil,:id,nil,true)
76
+ when :jobs
77
+ # to test JWT
78
+ return self.entity_action(@api_v5,'jobs',nil,:id,nil,true)
39
79
  when :package
40
- command=self.options.get_next_command([:list,:show,:send,:receive])
80
+ command=options.get_next_command([:list,:show,:send,:receive])
41
81
  case command
42
82
  when :list
43
- parameters=self.options.get_option(:value,:optional)
44
- return {:type => :object_list, :data=>api_v5.read('packages',parameters)[:data]['packages']}
83
+ parameters=options.get_option(:value,:optional)
84
+ return {:type => :object_list, :data=>@api_v5.read('packages',parameters)[:data]['packages']}
45
85
  when :show
46
- id=self.options.get_option(:id,:mandatory)
47
- return {:type => :single_object, :data=>api_v5.read("packages/#{id}")[:data]}
86
+ id=options.get_option(:id,:mandatory)
87
+ return {:type => :single_object, :data=>@api_v5.read("packages/#{id}")[:data]}
48
88
  when :send
49
- parameters=self.options.get_option(:value,:mandatory)
50
- raise CliBadArgument,'package value must be hash, refer to API' unless parameters.is_a?(Hash)
51
- package=api_v5.create('packages',parameters)[:data]
52
- transfer_spec=api_v5.create("packages/#{package['id']}/transfer_spec/upload",{transfer_type: 'Connect'})[:data]
89
+ parameters=options.get_option(:value,:mandatory)
90
+ raise CliBadArgument,'value must be hash, refer to API' unless parameters.is_a?(Hash)
91
+ package=@api_v5.create('packages',parameters)[:data]
92
+ transfer_spec=@api_v5.create("packages/#{package['id']}/transfer_spec/upload",{transfer_type: 'Connect'})[:data]
53
93
  transfer_spec.delete('authentication')
54
94
  return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
55
95
  when :receive
56
96
  pkg_type='received'
57
- pack_id=self.options.get_option(:id,:mandatory)
97
+ pack_id=options.get_option(:id,:mandatory)
58
98
  package_ids=[pack_id]
59
99
  skip_ids_data=[]
60
100
  skip_ids_persistency=nil
61
- if self.options.get_option(:once_only,:mandatory)
101
+ if options.get_option(:once_only,:mandatory)
102
+ # read ids from persistency
62
103
  skip_ids_persistency=PersistencyActionOnce.new(
63
104
  manager: @agents[:persistency],
64
105
  data: skip_ids_data,
65
- ids: ['faspex_recv',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),pkg_type])
106
+ ids: ['faspex_recv',options.get_option(:url,:mandatory),options.get_option(:username,:mandatory),pkg_type])
66
107
  end
67
108
  if pack_id.eql?(VAL_ALL)
68
- # todo: if packages have same name, they will overwrite
69
- parameters=self.options.get_option(:value,:optional)
70
- parameters||={"type"=>"received","subtype"=>"mypackages","limit"=>1000}
71
- raise CliBadArgument,'value filter must be hash (API GET)' unless parameters.is_a?(Hash)
72
- package_ids=api_v5.read('packages',parameters)[:data]['packages'].map{|p|p['id']}
109
+ # TODO: if packages have same name, they will overwrite
110
+ parameters=options.get_option(:value,:optional)
111
+ parameters||={'type'=>'received','subtype'=>'mypackages','limit'=>1000}
112
+ raise CliBadArgument,'value filter must be Hash (API GET)' unless parameters.is_a?(Hash)
113
+ package_ids=@api_v5.read('packages',parameters)[:data]['packages'].map{|p|p['id']}
73
114
  package_ids.select!{|i|!skip_ids_data.include?(i)}
74
115
  end
75
116
  result_transfer=[]
76
117
  package_ids.each do |id|
77
118
  # TODO: allow from sent as well ?
78
- transfer_spec=api_v5.create("packages/#{id}/transfer_spec/download",{transfer_type: 'Connect', type: pkg_type})[:data]
119
+ transfer_spec=@api_v5.create("packages/#{id}/transfer_spec/download",{transfer_type: 'Connect', type: pkg_type})[:data]
79
120
  transfer_spec.delete('authentication')
80
121
  statuses=self.transfer.start(transfer_spec,{:src=>:node_gen3})
81
122
  result_transfer.push({'package'=>id,'status'=>statuses.map{|i|i.to_s}.join(',')})
@@ -88,7 +88,7 @@ module Aspera
88
88
  raise StandardError,"expect: nil, String or Array"
89
89
  end
90
90
 
91
- SIMPLE_ACTIONS=[:nagios_check,:events, :space, :info, :license, :mkdir, :mklink, :mkfile, :rename, :delete, :search ]
91
+ SIMPLE_ACTIONS=[:health,:events, :space, :info, :license, :mkdir, :mklink, :mkfile, :rename, :delete, :search ]
92
92
 
93
93
  COMMON_ACTIONS=[:browse, :upload, :download, :api_details ].concat(SIMPLE_ACTIONS)
94
94
 
@@ -96,7 +96,7 @@ module Aspera
96
96
  # prefix_path is used to list remote sources in Faspex
97
97
  def execute_simple_common(command,prefix_path)
98
98
  case command
99
- when :nagios_check
99
+ when :health
100
100
  nagios=Nagios.new
101
101
  begin
102
102
  info=@api_node.read('info')[:data]
@@ -341,7 +341,7 @@ module Aspera
341
341
  raise "error"
342
342
  end
343
343
  when :access_key
344
- return self.entity_action(@api_node,'access_keys',['id','root_file_id','storage'],:id,'self')
344
+ return self.entity_action(@api_node,'access_keys',nil,:id,'self')
345
345
  when :service
346
346
  command=self.options.get_next_command([ :list, :create, :delete])
347
347
  if [:delete].include?(command)
@@ -4,6 +4,7 @@ require 'aspera/preview/options'
4
4
  require 'aspera/preview/utils'
5
5
  require 'aspera/preview/file_types'
6
6
  require 'aspera/persistency_action_once'
7
+ require 'aspera/node'
7
8
  require 'aspera/hash_ext'
8
9
  require 'date'
9
10
  require 'securerandom'
@@ -56,9 +57,11 @@ module Aspera
56
57
  self.options.add_opt_simple(:case,'basename of output for for test')
57
58
  self.options.add_opt_simple(:scan_path,'subpath in folder id to start scan in (default=/)')
58
59
  self.options.add_opt_simple(:scan_id,'forder id in storage to start scan in, default is access key main folder id')
60
+ self.options.add_opt_boolean(:mimemagic,'use Mime type detection of gem mimemagic')
59
61
  self.options.add_opt_list(:overwrite,[:always,:never,:mtime],'when to overwrite result file')
60
62
  self.options.add_opt_list(:file_access,[:local,:remote],'how to read and write files in repository')
61
63
  self.options.set_option(:temp_folder,Dir.tmpdir)
64
+ self.options.set_option(:mimemagic,:false)
62
65
 
63
66
  # add other options for generator (and set default values)
64
67
  Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
@@ -100,8 +103,6 @@ module Aspera
100
103
  return @preview_formats_to_generate.map{|i|i.to_s}.join(',')
101
104
  end
102
105
 
103
- ACTIONS=[:scan,:events,:trevents,:check,:test]
104
-
105
106
  # /files/id/files is normally cached in redis, but we can discard the cache
106
107
  # but /files/id is not cached
107
108
  def get_folder_entries(file_id,request_args=nil)
@@ -112,14 +113,23 @@ module Aspera
112
113
  end
113
114
 
114
115
  # old version based on folders
115
- def process_transfer_events(iteration_token)
116
+ def process_trevents(iteration_token)
116
117
  events_filter={
117
118
  'access_key'=>@access_key_self['id'],
118
119
  'type'=>'download.ended'
119
120
  }
120
121
  # optionally by iteration token
121
122
  events_filter['iteration_token']=iteration_token unless iteration_token.nil?
122
- events=@api_node.read('events',events_filter)[:data]
123
+ begin
124
+ events=@api_node.read('events',events_filter)[:data]
125
+ rescue RestCallError => e
126
+ if e.message.include?('Invalid iteration_token')
127
+ Log.log.warn("Retrying without iteration token: #{e}")
128
+ events_filter.delete('iteration_token')
129
+ retry
130
+ end
131
+ raise e
132
+ end
123
133
  return if events.empty?
124
134
  events.each do |event|
125
135
  next unless event['data']['direction'].eql?('receive')
@@ -137,7 +147,7 @@ module Aspera
137
147
  end
138
148
 
139
149
  # requests recent events on node api and process newly modified folders
140
- def process_file_events(iteration_token)
150
+ def process_events(iteration_token)
141
151
  # get new file creation by access key (TODO: what if file already existed?)
142
152
  events_filter={
143
153
  'access_key'=>@access_key_self['id'],
@@ -276,7 +286,7 @@ module Aspera
276
286
  end
277
287
  end
278
288
  # need generator for further checks
279
- gen_info[:generator]=Aspera::Preview::Generator.new(@gen_options,gen_info[:src],gen_info[:dst],@tmp_folder,entry['content_type'],false)
289
+ gen_info[:generator]=Aspera::Preview::Generator.new(@gen_options,gen_info[:src],gen_info[:dst],@tmp_folder,entry['content_type'])
280
290
  # get conversion_type (if known) and check if supported
281
291
  next false unless gen_info[:generator].supported?
282
292
  # shall we skip it ?
@@ -313,12 +323,14 @@ module Aspera
313
323
  end # generate_preview
314
324
 
315
325
  # scan all files in provided folder entry
326
+ # @param scan_start subpath to start folder scan inside
316
327
  def scan_folder_files(top_entry,scan_start=nil)
317
328
  if !scan_start.nil?
318
329
  # canonical path: start with / and ends with /
319
330
  scan_start='/'+scan_start.split('/').select{|i|!i.empty?}.join('/')
320
331
  scan_start="#{scan_start}/" #unless scan_start.end_with?('/')
321
332
  end
333
+ filter_block=Aspera::Node.file_matcher(options.get_option(:value,:optional))
322
334
  Log.log.debug("scan: #{top_entry} : #{scan_start}".green)
323
335
  # don't use recursive call, use list instead
324
336
  entries_to_process=[top_entry]
@@ -334,7 +346,11 @@ module Aspera
334
346
  Log.log.debug("item:#{entry}")
335
347
  case entry['type']
336
348
  when 'file'
337
- generate_preview(entry)
349
+ if filter_block.call(entry)
350
+ generate_preview(entry)
351
+ else
352
+ Log.log.debug('skip by filter')
353
+ end
338
354
  when 'link'
339
355
  Log.log.debug('Ignoring link.')
340
356
  when 'folder'
@@ -360,6 +376,8 @@ module Aspera
360
376
  end
361
377
  end
362
378
 
379
+ ACTIONS=[:scan,:events,:trevents,:check,:test]
380
+
363
381
  def execute_action
364
382
  command=self.options.get_next_command(ACTIONS)
365
383
  unless [:check,:test].include?(command)
@@ -402,6 +420,7 @@ module Aspera
402
420
  end
403
421
  end
404
422
  end
423
+ Aspera::Preview::FileTypes.instance.use_mimemagic = self.options.get_option(:mimemagic,:mandatory)
405
424
  case command
406
425
  when :scan
407
426
  scan_path=self.options.get_option(:scan_path,:optional)
@@ -417,30 +436,20 @@ module Aspera
417
436
  end
418
437
  scan_folder_files(folder_info,scan_path)
419
438
  return Main.result_status('scan finished')
420
- when :events
439
+ when :events,:trevents
421
440
  iteration_data=[]
422
441
  iteration_persistency=nil
423
442
  if self.options.get_option(:once_only,:mandatory)
424
443
  iteration_persistency=PersistencyActionOnce.new(
425
444
  manager: @agents[:persistency],
426
445
  data: iteration_data,
427
- ids: ['preview_iteration_events',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
446
+ ids: ["preview_iteration_#{command}",self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
428
447
  end
429
- iteration_data[0]=process_file_events(iteration_data[0])
430
- iteration_persistency.save unless iteration_persistency.nil?
431
- return Main.result_status('events finished')
432
- when :trevents
433
- iteration_data=[]
434
- iteration_persistency=nil
435
- if self.options.get_option(:once_only,:mandatory)
436
- iteration_persistency=PersistencyActionOnce.new(
437
- manager: @agents[:persistency],
438
- data: iteration_data,
439
- ids: ['preview_iteration_transfer',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
440
- end
441
- iteration_data[0]=process_transfer_events(iteration_data[0])
448
+
449
+ # call method specified
450
+ iteration_data[0]=send("process_#{command}",iteration_data[0])
442
451
  iteration_persistency.save unless iteration_persistency.nil?
443
- return Main.result_status('trevents finished')
452
+ return Main.result_status("#{command} finished")
444
453
  when :check
445
454
  Aspera::Preview::Utils.check_tools(@skip_types)
446
455
  return Main.result_status('tools validated')
@@ -448,8 +457,9 @@ module Aspera
448
457
  format = self.options.get_next_argument('format',Aspera::Preview::Generator::PREVIEW_FORMATS)
449
458
  source = self.options.get_next_argument('source file')
450
459
  dest=preview_filename(format,self.options.get_option(:case,:optional))
451
- g=Aspera::Preview::Generator.new(@gen_options,source,dest,@tmp_folder)
452
- raise "format not supported: #{format}" unless g.supported?
460
+ g=Aspera::Preview::Generator.new(@gen_options,source,dest,@tmp_folder,nil)
461
+ raise "cannot find file type for #{source}" if g.conversion_type.nil?
462
+ raise "out format #{format} not supported" unless g.supported?
453
463
  g.generate
454
464
  return Main.result_status("generated: #{dest}")
455
465
  else