aspera-cli 4.0.0.pre3 → 4.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +695 -205
- data/bin/dascli +13 -0
- data/docs/README.erb.md +615 -157
- data/docs/test_env.conf +23 -5
- data/docs/transfer_spec.html +1 -1
- data/examples/aoc.rb +14 -3
- data/examples/faspex4.rb +78 -0
- data/lib/aspera/aoc.rb +87 -108
- data/lib/aspera/cli/formater.rb +2 -0
- data/lib/aspera/cli/main.rb +46 -34
- data/lib/aspera/cli/plugin.rb +9 -4
- data/lib/aspera/cli/plugins/alee.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +207 -182
- data/lib/aspera/cli/plugins/ats.rb +2 -2
- data/lib/aspera/cli/plugins/config.rb +173 -117
- data/lib/aspera/cli/plugins/console.rb +2 -2
- data/lib/aspera/cli/plugins/faspex.rb +51 -36
- data/lib/aspera/cli/plugins/faspex5.rb +82 -41
- data/lib/aspera/cli/plugins/node.rb +3 -3
- data/lib/aspera/cli/plugins/preview.rb +35 -25
- data/lib/aspera/cli/plugins/server.rb +23 -8
- data/lib/aspera/cli/transfer_agent.rb +7 -6
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cos_node.rb +33 -28
- data/lib/aspera/environment.rb +2 -2
- data/lib/aspera/fasp/connect.rb +28 -21
- data/lib/aspera/fasp/http_gw.rb +140 -28
- data/lib/aspera/fasp/installation.rb +101 -53
- data/lib/aspera/fasp/local.rb +88 -45
- data/lib/aspera/fasp/manager.rb +15 -0
- data/lib/aspera/fasp/node.rb +4 -4
- data/lib/aspera/fasp/parameters.rb +6 -18
- data/lib/aspera/fasp/resume_policy.rb +13 -12
- data/lib/aspera/log.rb +1 -1
- data/lib/aspera/node.rb +61 -1
- data/lib/aspera/oauth.rb +49 -46
- data/lib/aspera/persistency_folder.rb +9 -4
- data/lib/aspera/preview/file_types.rb +53 -21
- data/lib/aspera/preview/generator.rb +3 -3
- data/lib/aspera/rest.rb +29 -18
- data/lib/aspera/secrets.rb +20 -0
- data/lib/aspera/temp_file_manager.rb +19 -0
- data/lib/aspera/web_auth.rb +105 -0
- metadata +42 -22
@@ -14,13 +14,13 @@ module Aspera
|
|
14
14
|
self.options.parse_options!
|
15
15
|
end
|
16
16
|
|
17
|
-
ACTIONS=[:transfer,:
|
17
|
+
ACTIONS=[:transfer,:health]
|
18
18
|
|
19
19
|
def execute_action
|
20
20
|
api_console=basic_auth_api('api')
|
21
21
|
command=self.options.get_next_command(ACTIONS)
|
22
22
|
case command
|
23
|
-
when :
|
23
|
+
when :health
|
24
24
|
nagios=Nagios.new
|
25
25
|
begin
|
26
26
|
api_console.read('ssh_keys')
|
@@ -26,6 +26,7 @@ module Aspera
|
|
26
26
|
self.options.add_opt_simple(:delivery_info,'package delivery information (extended value)')
|
27
27
|
self.options.add_opt_simple(:source_name,'create package from remote source (by name)')
|
28
28
|
self.options.add_opt_simple(:storage,'Faspex local storage definition')
|
29
|
+
self.options.add_opt_simple(:recipient,'use if recipient is a dropbox (with *)')
|
29
30
|
self.options.add_opt_list(:box,[:inbox,:sent,:archive],'package box')
|
30
31
|
self.options.set_option(:box,:inbox)
|
31
32
|
self.options.parse_options!
|
@@ -44,7 +45,7 @@ module Aspera
|
|
44
45
|
result={
|
45
46
|
:base_url => "#{publink_uri.scheme}://#{publink_uri.host}#{port_add}#{base}",
|
46
47
|
:subpath => subpath,
|
47
|
-
:query => URI::decode_www_form(publink_uri.query).inject({}){|
|
48
|
+
:query => URI::decode_www_form(publink_uri.query).inject({}){|h,v|h[v.first]=v.last;h}
|
48
49
|
}
|
49
50
|
Log.dump('publink',result)
|
50
51
|
return result
|
@@ -52,7 +53,7 @@ module Aspera
|
|
52
53
|
|
53
54
|
# get faspe: URI from entry in xml, and fix problems..
|
54
55
|
def self.get_fasp_uri_from_entry(entry)
|
55
|
-
raise CliBadArgument, 'package
|
56
|
+
raise CliBadArgument, 'package has no link (deleted?)' unless entry.has_key?('link')
|
56
57
|
result=entry['link'].select{|e| e['rel'].eql?('package')}.first['href']
|
57
58
|
# tags in the end of URL is not well % encoded... there are "=" that should be %3D
|
58
59
|
# TODO: enter ticket to Faspex ?
|
@@ -72,7 +73,7 @@ module Aspera
|
|
72
73
|
def self.get_source_id(source_list,source_name)
|
73
74
|
source_ids=source_list.select { |i| i['name'].eql?(source_name) }
|
74
75
|
if source_ids.empty?
|
75
|
-
raise CliError
|
76
|
+
raise CliError,%Q{No such Faspex source "#{source_name}" in [#{source_list.map{|i| %Q{"#{i['name']}"}}.join(', ')}]}
|
76
77
|
end
|
77
78
|
return source_ids.first['id']
|
78
79
|
end
|
@@ -101,19 +102,26 @@ module Aspera
|
|
101
102
|
return @api_v4
|
102
103
|
end
|
103
104
|
|
104
|
-
ACTIONS=[ :
|
105
|
+
ACTIONS=[ :health,:package, :source, :me, :dropbox, :v4, :address_book, :login_methods ]
|
105
106
|
|
106
107
|
# we match recv command on atom feed on this field
|
107
108
|
PACKAGE_MATCH_FIELD='package_id'
|
108
109
|
|
109
110
|
def mailbox_all_entries
|
110
|
-
|
111
|
+
recipient_name=self.options.get_option(:recipient,:optional) || self.options.get_option(:username,:mandatory)
|
112
|
+
mailbox=self.options.get_option(:box,:mandatory)
|
111
113
|
all_inbox_xml=api_v3.call({:operation=>'GET',:subpath=>"#{mailbox}.atom",:headers=>{'Accept'=>'application/xml'}})[:http].body
|
112
114
|
all_inbox_data=XmlSimple.xml_in(all_inbox_xml, {'ForceArray' => true})
|
113
115
|
Log.dump(:all_inbox_data,all_inbox_data)
|
114
116
|
result=all_inbox_data.has_key?('entry') ? all_inbox_data['entry'] : []
|
115
117
|
result.each do |e|
|
116
|
-
|
118
|
+
case mailbox
|
119
|
+
when :inbox,:archive
|
120
|
+
recipient=e['to'].select{|i|i['name'].first.eql?(recipient_name)}.first
|
121
|
+
e[PACKAGE_MATCH_FIELD]=recipient.nil? ? 'n/a' : recipient['recipient_delivery_id'].first
|
122
|
+
when :sent
|
123
|
+
e[PACKAGE_MATCH_FIELD]=e['delivery_id'].first
|
124
|
+
end
|
117
125
|
end
|
118
126
|
# remove dropbox packages
|
119
127
|
result.select!{|p|p['metadata'].first['field'].select{|j|j['name'].eql?('_dropbox_name')}.empty? rescue false}
|
@@ -155,7 +163,7 @@ module Aspera
|
|
155
163
|
def execute_action
|
156
164
|
command=self.options.get_next_command(ACTIONS)
|
157
165
|
case command
|
158
|
-
when :
|
166
|
+
when :health
|
159
167
|
nagios=Nagios.new
|
160
168
|
begin
|
161
169
|
api_v3.read('me')
|
@@ -201,9 +209,18 @@ module Aspera
|
|
201
209
|
#Log.dump('transfer_spec',transfer_spec)
|
202
210
|
return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
|
203
211
|
when :recv
|
204
|
-
|
205
|
-
|
206
|
-
|
212
|
+
link_url=self.options.get_option(:link,:optional)
|
213
|
+
# list of faspex ID/URI to download
|
214
|
+
pkg_id_uri=nil
|
215
|
+
skip_ids_data=[]
|
216
|
+
skip_ids_persistency=nil
|
217
|
+
case link_url
|
218
|
+
when nil
|
219
|
+
# usual case: no link
|
220
|
+
when /^faspe:/
|
221
|
+
pkg_id_uri=[{:id=>'package',:uri=>link_url}]
|
222
|
+
else
|
223
|
+
link_data=self.class.get_link_data(link_url)
|
207
224
|
if !link_data[:subpath].match(%r{external_deliveries/})
|
208
225
|
raise CliBadArgument,"pub link is #{link_data[:subpath]}, expecting external_deliveries/"
|
209
226
|
end
|
@@ -211,7 +228,7 @@ module Aspera
|
|
211
228
|
api_public_link=Rest.new({:base_url=>link_data[:base_url]})
|
212
229
|
pkgdatares=api_public_link.call({:operation=>'GET',:subpath=>link_data[:subpath],:url_params=>{:passcode=>link_data[:query]['passcode']},:headers=>{'Accept'=>'application/xml'}})
|
213
230
|
if !pkgdatares[:http].body.start_with?('<?xml ')
|
214
|
-
OpenApplication.instance.uri(
|
231
|
+
OpenApplication.instance.uri(link_url)
|
215
232
|
raise CliError, 'no such package'
|
216
233
|
end
|
217
234
|
package_entry=XmlSimple.xml_in(pkgdatares[:http].body, {'ForceArray' => false})
|
@@ -220,32 +237,30 @@ module Aspera
|
|
220
237
|
transfer_spec['direction']='receive'
|
221
238
|
return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
|
222
239
|
end # public link
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
240
|
+
if pkg_id_uri.nil?
|
241
|
+
# get command line parameters
|
242
|
+
delivid=self.options.get_option(:id,:mandatory)
|
243
|
+
if self.options.get_option(:once_only,:mandatory)
|
244
|
+
skip_ids_persistency=PersistencyActionOnce.new(
|
245
|
+
manager: @agents[:persistency],
|
246
|
+
data: skip_ids_data,
|
247
|
+
ids: ['faspex_recv',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),self.options.get_option(:box,:mandatory).to_s])
|
248
|
+
end
|
249
|
+
if delivid.eql?(VAL_ALL)
|
250
|
+
pkg_id_uri=mailbox_all_entries.map{|i|{:id=>i[PACKAGE_MATCH_FIELD],:uri=>self.class.get_fasp_uri_from_entry(i)}}
|
251
|
+
# TODO : remove ids from skip not present in inbox
|
252
|
+
# skip_ids_data.select!{|id|pkg_id_uri.select{|p|p[:id].eql?(id)}}
|
253
|
+
pkg_id_uri.select!{|i|!skip_ids_data.include?(i[:id])}
|
254
|
+
else
|
255
|
+
# TODO: delivery id is the right one if package was receive by group
|
256
|
+
endpoint=case self.options.get_option(:box,:mandatory)
|
257
|
+
when :inbox,:archive;'received'
|
258
|
+
when :sent; 'sent'
|
259
|
+
end
|
260
|
+
entry_xml=api_v3.call({:operation=>'GET',:subpath=>"#{endpoint}/#{delivid}",:headers=>{'Accept'=>'application/xml'}})[:http].body
|
261
|
+
package_entry=XmlSimple.xml_in(entry_xml, {'ForceArray' => true})
|
262
|
+
pkg_id_uri=[{:id=>delivid,:uri=>self.class.get_fasp_uri_from_entry(package_entry)}]
|
245
263
|
end
|
246
|
-
entry_xml=api_v3.call({:operation=>'GET',:subpath=>"#{endpoint}/#{delivid}",:headers=>{'Accept'=>'application/xml'}})[:http].body
|
247
|
-
package_entry=XmlSimple.xml_in(entry_xml, {'ForceArray' => true})
|
248
|
-
pkg_id_uri=[{:id=>delivid,:uri=>self.class.get_fasp_uri_from_entry(package_entry)}]
|
249
264
|
end
|
250
265
|
Log.dump(:pkg_id_uri,pkg_id_uri)
|
251
266
|
return Main.result_status('no package') if pkg_id_uri.empty?
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'aspera/cli/basic_auth_plugin'
|
2
2
|
require 'aspera/persistency_action_once'
|
3
|
+
require 'securerandom'
|
3
4
|
|
4
5
|
module Aspera
|
5
6
|
module Cli
|
@@ -8,74 +9,114 @@ module Aspera
|
|
8
9
|
VAL_ALL='ALL'
|
9
10
|
def initialize(env)
|
10
11
|
super(env)
|
11
|
-
|
12
|
-
|
12
|
+
options.add_opt_simple(:client_id,'API client identifier in application')
|
13
|
+
options.add_opt_simple(:client_secret,'API client secret in application')
|
14
|
+
options.add_opt_simple(:redirect_uri,'API client redirect URI')
|
15
|
+
options.add_opt_list(:auth,Oauth.auth_types.clone.push(:boot),'type of Oauth authentication')
|
16
|
+
options.add_opt_simple(:private_key,'RSA private key PEM value for JWT (prefix file path with @val:@file:)')
|
17
|
+
options.set_option(:auth,:jwt)
|
18
|
+
options.parse_options!
|
13
19
|
end
|
14
|
-
ACTIONS=[ :node, :package ]
|
15
20
|
|
16
|
-
|
21
|
+
def set_api
|
22
|
+
faxpex5_api_base_url=options.get_option(:url,:mandatory)
|
23
|
+
faxpex5_api_v5_url="#{faxpex5_api_base_url}/api/v5"
|
24
|
+
faxpex5_api_auth_url="#{faxpex5_api_base_url}/auth"
|
25
|
+
case options.get_option(:auth,:mandatory)
|
26
|
+
when :boot
|
27
|
+
# the password here is the token copied directly from browser in developer mode
|
28
|
+
@api_v5=Rest.new({
|
29
|
+
:base_url => faxpex5_api_v5_url,
|
30
|
+
:headers => {'Authorization'=>options.get_option(:password,:mandatory)},
|
31
|
+
})
|
32
|
+
when :web
|
33
|
+
# opens a browser and ask user to auth using web
|
34
|
+
@api_v5=Rest.new({
|
35
|
+
:base_url => faxpex5_api_v5_url,
|
36
|
+
:auth => {
|
37
|
+
:type => :oauth2,
|
38
|
+
:base_url => faxpex5_api_auth_url,
|
39
|
+
:grant => :web,
|
40
|
+
:state => SecureRandom.uuid,
|
41
|
+
:client_id => options.get_option(:client_id,:mandatory),
|
42
|
+
:redirect_uri => options.get_option(:redirect_uri,:mandatory),
|
43
|
+
}})
|
44
|
+
when :jwt
|
45
|
+
# currently Faspex 5 beta 3 only supports non-user based apis (e.g. jobs)
|
46
|
+
app_client_id=options.get_option(:client_id,:mandatory)
|
47
|
+
@api_v5=Rest.new({
|
48
|
+
:base_url => faxpex5_api_v5_url,
|
49
|
+
:auth => {
|
50
|
+
:type => :oauth2,
|
51
|
+
:base_url => faxpex5_api_auth_url,
|
52
|
+
:grant => :jwt,
|
53
|
+
:client_id => app_client_id,
|
54
|
+
:client_secret => options.get_option(:client_secret,:mandatory),
|
55
|
+
:jwt_subject => "client:#{app_client_id}", # TODO Mmmm
|
56
|
+
:jwt_audience => app_client_id, # TODO Mmmm
|
57
|
+
:jwt_private_key_obj => OpenSSL::PKey::RSA.new(options.get_option(:private_key,:mandatory)),
|
58
|
+
:jwt_is_f5 => true, # TODO: remove when clarified
|
59
|
+
:jwt_headers => {typ: 'JWT'}
|
60
|
+
}})
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
ACTIONS=[ :node, :package, :auth_client, :jobs ]
|
65
|
+
|
66
|
+
#
|
17
67
|
def execute_action
|
18
|
-
|
19
|
-
|
20
|
-
faxpex5_username=self.options.get_option(:username,:mandatory)
|
21
|
-
faxpex5_password=self.options.get_option(:password,:mandatory)
|
22
|
-
faxpex5_api_base_url+='/api/v5'
|
23
|
-
# create object for REST calls to Shares2
|
24
|
-
api_v5=Rest.new({
|
25
|
-
:base_url => faxpex5_api_base_url,
|
26
|
-
:auth => {
|
27
|
-
:type => :oauth2,
|
28
|
-
:base_url => faxpex5_api_base_url,
|
29
|
-
:grant => :body_data,
|
30
|
-
:token_field =>'auth_token',
|
31
|
-
:path_token => 'authenticate',
|
32
|
-
:path_authorize => :unused,
|
33
|
-
:userpass_body => {name: faxpex5_username,password: faxpex5_password}
|
34
|
-
}})
|
35
|
-
command=self.options.get_next_command(ACTIONS)
|
68
|
+
set_api
|
69
|
+
command=options.get_next_command(ACTIONS)
|
36
70
|
case command
|
71
|
+
when :auth_client
|
72
|
+
api_auth=Rest.new(@api_v5.params.merge({base_url: @api_v5.params[:base_url].gsub(/api\/v5$/,'auth')}))
|
73
|
+
return self.entity_action(api_auth,'oauth_clients',nil,:id,nil,true)
|
37
74
|
when :node
|
38
|
-
return self.entity_action(api_v5,'nodes',nil,:id,nil,true)
|
75
|
+
return self.entity_action(@api_v5,'nodes',nil,:id,nil,true)
|
76
|
+
when :jobs
|
77
|
+
# to test JWT
|
78
|
+
return self.entity_action(@api_v5,'jobs',nil,:id,nil,true)
|
39
79
|
when :package
|
40
|
-
command=
|
80
|
+
command=options.get_next_command([:list,:show,:send,:receive])
|
41
81
|
case command
|
42
82
|
when :list
|
43
|
-
parameters=
|
44
|
-
return {:type => :object_list, :data
|
83
|
+
parameters=options.get_option(:value,:optional)
|
84
|
+
return {:type => :object_list, :data=>@api_v5.read('packages',parameters)[:data]['packages']}
|
45
85
|
when :show
|
46
|
-
id=
|
47
|
-
return {:type => :single_object, :data
|
86
|
+
id=options.get_option(:id,:mandatory)
|
87
|
+
return {:type => :single_object, :data=>@api_v5.read("packages/#{id}")[:data]}
|
48
88
|
when :send
|
49
|
-
parameters=
|
50
|
-
raise CliBadArgument,'
|
51
|
-
package
|
52
|
-
transfer_spec
|
89
|
+
parameters=options.get_option(:value,:mandatory)
|
90
|
+
raise CliBadArgument,'value must be hash, refer to API' unless parameters.is_a?(Hash)
|
91
|
+
package=@api_v5.create('packages',parameters)[:data]
|
92
|
+
transfer_spec=@api_v5.create("packages/#{package['id']}/transfer_spec/upload",{transfer_type: 'Connect'})[:data]
|
53
93
|
transfer_spec.delete('authentication')
|
54
94
|
return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
|
55
95
|
when :receive
|
56
96
|
pkg_type='received'
|
57
|
-
pack_id=
|
97
|
+
pack_id=options.get_option(:id,:mandatory)
|
58
98
|
package_ids=[pack_id]
|
59
99
|
skip_ids_data=[]
|
60
100
|
skip_ids_persistency=nil
|
61
|
-
if
|
101
|
+
if options.get_option(:once_only,:mandatory)
|
102
|
+
# read ids from persistency
|
62
103
|
skip_ids_persistency=PersistencyActionOnce.new(
|
63
104
|
manager: @agents[:persistency],
|
64
105
|
data: skip_ids_data,
|
65
|
-
ids: ['faspex_recv',
|
106
|
+
ids: ['faspex_recv',options.get_option(:url,:mandatory),options.get_option(:username,:mandatory),pkg_type])
|
66
107
|
end
|
67
108
|
if pack_id.eql?(VAL_ALL)
|
68
|
-
#
|
69
|
-
parameters=
|
70
|
-
parameters||={
|
71
|
-
raise CliBadArgument,'value filter must be
|
72
|
-
package_ids
|
109
|
+
# TODO: if packages have same name, they will overwrite
|
110
|
+
parameters=options.get_option(:value,:optional)
|
111
|
+
parameters||={'type'=>'received','subtype'=>'mypackages','limit'=>1000}
|
112
|
+
raise CliBadArgument,'value filter must be Hash (API GET)' unless parameters.is_a?(Hash)
|
113
|
+
package_ids=@api_v5.read('packages',parameters)[:data]['packages'].map{|p|p['id']}
|
73
114
|
package_ids.select!{|i|!skip_ids_data.include?(i)}
|
74
115
|
end
|
75
116
|
result_transfer=[]
|
76
117
|
package_ids.each do |id|
|
77
118
|
# TODO: allow from sent as well ?
|
78
|
-
transfer_spec
|
119
|
+
transfer_spec=@api_v5.create("packages/#{id}/transfer_spec/download",{transfer_type: 'Connect', type: pkg_type})[:data]
|
79
120
|
transfer_spec.delete('authentication')
|
80
121
|
statuses=self.transfer.start(transfer_spec,{:src=>:node_gen3})
|
81
122
|
result_transfer.push({'package'=>id,'status'=>statuses.map{|i|i.to_s}.join(',')})
|
@@ -88,7 +88,7 @@ module Aspera
|
|
88
88
|
raise StandardError,"expect: nil, String or Array"
|
89
89
|
end
|
90
90
|
|
91
|
-
SIMPLE_ACTIONS=[:
|
91
|
+
SIMPLE_ACTIONS=[:health,:events, :space, :info, :license, :mkdir, :mklink, :mkfile, :rename, :delete, :search ]
|
92
92
|
|
93
93
|
COMMON_ACTIONS=[:browse, :upload, :download, :api_details ].concat(SIMPLE_ACTIONS)
|
94
94
|
|
@@ -96,7 +96,7 @@ module Aspera
|
|
96
96
|
# prefix_path is used to list remote sources in Faspex
|
97
97
|
def execute_simple_common(command,prefix_path)
|
98
98
|
case command
|
99
|
-
when :
|
99
|
+
when :health
|
100
100
|
nagios=Nagios.new
|
101
101
|
begin
|
102
102
|
info=@api_node.read('info')[:data]
|
@@ -341,7 +341,7 @@ module Aspera
|
|
341
341
|
raise "error"
|
342
342
|
end
|
343
343
|
when :access_key
|
344
|
-
return self.entity_action(@api_node,'access_keys',
|
344
|
+
return self.entity_action(@api_node,'access_keys',nil,:id,'self')
|
345
345
|
when :service
|
346
346
|
command=self.options.get_next_command([ :list, :create, :delete])
|
347
347
|
if [:delete].include?(command)
|
@@ -4,6 +4,7 @@ require 'aspera/preview/options'
|
|
4
4
|
require 'aspera/preview/utils'
|
5
5
|
require 'aspera/preview/file_types'
|
6
6
|
require 'aspera/persistency_action_once'
|
7
|
+
require 'aspera/node'
|
7
8
|
require 'aspera/hash_ext'
|
8
9
|
require 'date'
|
9
10
|
require 'securerandom'
|
@@ -56,9 +57,11 @@ module Aspera
|
|
56
57
|
self.options.add_opt_simple(:case,'basename of output for for test')
|
57
58
|
self.options.add_opt_simple(:scan_path,'subpath in folder id to start scan in (default=/)')
|
58
59
|
self.options.add_opt_simple(:scan_id,'forder id in storage to start scan in, default is access key main folder id')
|
60
|
+
self.options.add_opt_boolean(:mimemagic,'use Mime type detection of gem mimemagic')
|
59
61
|
self.options.add_opt_list(:overwrite,[:always,:never,:mtime],'when to overwrite result file')
|
60
62
|
self.options.add_opt_list(:file_access,[:local,:remote],'how to read and write files in repository')
|
61
63
|
self.options.set_option(:temp_folder,Dir.tmpdir)
|
64
|
+
self.options.set_option(:mimemagic,:false)
|
62
65
|
|
63
66
|
# add other options for generator (and set default values)
|
64
67
|
Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
|
@@ -100,8 +103,6 @@ module Aspera
|
|
100
103
|
return @preview_formats_to_generate.map{|i|i.to_s}.join(',')
|
101
104
|
end
|
102
105
|
|
103
|
-
ACTIONS=[:scan,:events,:trevents,:check,:test]
|
104
|
-
|
105
106
|
# /files/id/files is normally cached in redis, but we can discard the cache
|
106
107
|
# but /files/id is not cached
|
107
108
|
def get_folder_entries(file_id,request_args=nil)
|
@@ -112,14 +113,23 @@ module Aspera
|
|
112
113
|
end
|
113
114
|
|
114
115
|
# old version based on folders
|
115
|
-
def
|
116
|
+
def process_trevents(iteration_token)
|
116
117
|
events_filter={
|
117
118
|
'access_key'=>@access_key_self['id'],
|
118
119
|
'type'=>'download.ended'
|
119
120
|
}
|
120
121
|
# optionally by iteration token
|
121
122
|
events_filter['iteration_token']=iteration_token unless iteration_token.nil?
|
122
|
-
|
123
|
+
begin
|
124
|
+
events=@api_node.read('events',events_filter)[:data]
|
125
|
+
rescue RestCallError => e
|
126
|
+
if e.message.include?('Invalid iteration_token')
|
127
|
+
Log.log.warn("Retrying without iteration token: #{e}")
|
128
|
+
events_filter.delete('iteration_token')
|
129
|
+
retry
|
130
|
+
end
|
131
|
+
raise e
|
132
|
+
end
|
123
133
|
return if events.empty?
|
124
134
|
events.each do |event|
|
125
135
|
next unless event['data']['direction'].eql?('receive')
|
@@ -137,7 +147,7 @@ module Aspera
|
|
137
147
|
end
|
138
148
|
|
139
149
|
# requests recent events on node api and process newly modified folders
|
140
|
-
def
|
150
|
+
def process_events(iteration_token)
|
141
151
|
# get new file creation by access key (TODO: what if file already existed?)
|
142
152
|
events_filter={
|
143
153
|
'access_key'=>@access_key_self['id'],
|
@@ -276,7 +286,7 @@ module Aspera
|
|
276
286
|
end
|
277
287
|
end
|
278
288
|
# need generator for further checks
|
279
|
-
gen_info[:generator]=Aspera::Preview::Generator.new(@gen_options,gen_info[:src],gen_info[:dst],@tmp_folder,entry['content_type']
|
289
|
+
gen_info[:generator]=Aspera::Preview::Generator.new(@gen_options,gen_info[:src],gen_info[:dst],@tmp_folder,entry['content_type'])
|
280
290
|
# get conversion_type (if known) and check if supported
|
281
291
|
next false unless gen_info[:generator].supported?
|
282
292
|
# shall we skip it ?
|
@@ -313,12 +323,14 @@ module Aspera
|
|
313
323
|
end # generate_preview
|
314
324
|
|
315
325
|
# scan all files in provided folder entry
|
326
|
+
# @param scan_start subpath to start folder scan inside
|
316
327
|
def scan_folder_files(top_entry,scan_start=nil)
|
317
328
|
if !scan_start.nil?
|
318
329
|
# canonical path: start with / and ends with /
|
319
330
|
scan_start='/'+scan_start.split('/').select{|i|!i.empty?}.join('/')
|
320
331
|
scan_start="#{scan_start}/" #unless scan_start.end_with?('/')
|
321
332
|
end
|
333
|
+
filter_block=Aspera::Node.file_matcher(options.get_option(:value,:optional))
|
322
334
|
Log.log.debug("scan: #{top_entry} : #{scan_start}".green)
|
323
335
|
# don't use recursive call, use list instead
|
324
336
|
entries_to_process=[top_entry]
|
@@ -334,7 +346,11 @@ module Aspera
|
|
334
346
|
Log.log.debug("item:#{entry}")
|
335
347
|
case entry['type']
|
336
348
|
when 'file'
|
337
|
-
|
349
|
+
if filter_block.call(entry)
|
350
|
+
generate_preview(entry)
|
351
|
+
else
|
352
|
+
Log.log.debug('skip by filter')
|
353
|
+
end
|
338
354
|
when 'link'
|
339
355
|
Log.log.debug('Ignoring link.')
|
340
356
|
when 'folder'
|
@@ -360,6 +376,8 @@ module Aspera
|
|
360
376
|
end
|
361
377
|
end
|
362
378
|
|
379
|
+
ACTIONS=[:scan,:events,:trevents,:check,:test]
|
380
|
+
|
363
381
|
def execute_action
|
364
382
|
command=self.options.get_next_command(ACTIONS)
|
365
383
|
unless [:check,:test].include?(command)
|
@@ -402,6 +420,7 @@ module Aspera
|
|
402
420
|
end
|
403
421
|
end
|
404
422
|
end
|
423
|
+
Aspera::Preview::FileTypes.instance.use_mimemagic = self.options.get_option(:mimemagic,:mandatory)
|
405
424
|
case command
|
406
425
|
when :scan
|
407
426
|
scan_path=self.options.get_option(:scan_path,:optional)
|
@@ -417,30 +436,20 @@ module Aspera
|
|
417
436
|
end
|
418
437
|
scan_folder_files(folder_info,scan_path)
|
419
438
|
return Main.result_status('scan finished')
|
420
|
-
when :events
|
439
|
+
when :events,:trevents
|
421
440
|
iteration_data=[]
|
422
441
|
iteration_persistency=nil
|
423
442
|
if self.options.get_option(:once_only,:mandatory)
|
424
443
|
iteration_persistency=PersistencyActionOnce.new(
|
425
444
|
manager: @agents[:persistency],
|
426
445
|
data: iteration_data,
|
427
|
-
ids:
|
446
|
+
ids: ["preview_iteration_#{command}",self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
|
428
447
|
end
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
when :trevents
|
433
|
-
iteration_data=[]
|
434
|
-
iteration_persistency=nil
|
435
|
-
if self.options.get_option(:once_only,:mandatory)
|
436
|
-
iteration_persistency=PersistencyActionOnce.new(
|
437
|
-
manager: @agents[:persistency],
|
438
|
-
data: iteration_data,
|
439
|
-
ids: ['preview_iteration_transfer',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
|
440
|
-
end
|
441
|
-
iteration_data[0]=process_transfer_events(iteration_data[0])
|
448
|
+
|
449
|
+
# call method specified
|
450
|
+
iteration_data[0]=send("process_#{command}",iteration_data[0])
|
442
451
|
iteration_persistency.save unless iteration_persistency.nil?
|
443
|
-
return Main.result_status(
|
452
|
+
return Main.result_status("#{command} finished")
|
444
453
|
when :check
|
445
454
|
Aspera::Preview::Utils.check_tools(@skip_types)
|
446
455
|
return Main.result_status('tools validated')
|
@@ -448,8 +457,9 @@ module Aspera
|
|
448
457
|
format = self.options.get_next_argument('format',Aspera::Preview::Generator::PREVIEW_FORMATS)
|
449
458
|
source = self.options.get_next_argument('source file')
|
450
459
|
dest=preview_filename(format,self.options.get_option(:case,:optional))
|
451
|
-
g=Aspera::Preview::Generator.new(@gen_options,source,dest,@tmp_folder)
|
452
|
-
raise "
|
460
|
+
g=Aspera::Preview::Generator.new(@gen_options,source,dest,@tmp_folder,nil)
|
461
|
+
raise "cannot find file type for #{source}" if g.conversion_type.nil?
|
462
|
+
raise "out format #{format} not supported" unless g.supported?
|
453
463
|
g.generate
|
454
464
|
return Main.result_status("generated: #{dest}")
|
455
465
|
else
|