aspera-cli 4.0.0.pre3 → 4.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +695 -205
- data/bin/dascli +13 -0
- data/docs/README.erb.md +615 -157
- data/docs/test_env.conf +23 -5
- data/docs/transfer_spec.html +1 -1
- data/examples/aoc.rb +14 -3
- data/examples/faspex4.rb +78 -0
- data/lib/aspera/aoc.rb +87 -108
- data/lib/aspera/cli/formater.rb +2 -0
- data/lib/aspera/cli/main.rb +46 -34
- data/lib/aspera/cli/plugin.rb +9 -4
- data/lib/aspera/cli/plugins/alee.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +207 -182
- data/lib/aspera/cli/plugins/ats.rb +2 -2
- data/lib/aspera/cli/plugins/config.rb +173 -117
- data/lib/aspera/cli/plugins/console.rb +2 -2
- data/lib/aspera/cli/plugins/faspex.rb +51 -36
- data/lib/aspera/cli/plugins/faspex5.rb +82 -41
- data/lib/aspera/cli/plugins/node.rb +3 -3
- data/lib/aspera/cli/plugins/preview.rb +35 -25
- data/lib/aspera/cli/plugins/server.rb +23 -8
- data/lib/aspera/cli/transfer_agent.rb +7 -6
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cos_node.rb +33 -28
- data/lib/aspera/environment.rb +2 -2
- data/lib/aspera/fasp/connect.rb +28 -21
- data/lib/aspera/fasp/http_gw.rb +140 -28
- data/lib/aspera/fasp/installation.rb +101 -53
- data/lib/aspera/fasp/local.rb +88 -45
- data/lib/aspera/fasp/manager.rb +15 -0
- data/lib/aspera/fasp/node.rb +4 -4
- data/lib/aspera/fasp/parameters.rb +6 -18
- data/lib/aspera/fasp/resume_policy.rb +13 -12
- data/lib/aspera/log.rb +1 -1
- data/lib/aspera/node.rb +61 -1
- data/lib/aspera/oauth.rb +49 -46
- data/lib/aspera/persistency_folder.rb +9 -4
- data/lib/aspera/preview/file_types.rb +53 -21
- data/lib/aspera/preview/generator.rb +3 -3
- data/lib/aspera/rest.rb +29 -18
- data/lib/aspera/secrets.rb +20 -0
- data/lib/aspera/temp_file_manager.rb +19 -0
- data/lib/aspera/web_auth.rb +105 -0
- metadata +42 -22
@@ -14,13 +14,13 @@ module Aspera
|
|
14
14
|
self.options.parse_options!
|
15
15
|
end
|
16
16
|
|
17
|
-
ACTIONS=[:transfer,:
|
17
|
+
ACTIONS=[:transfer,:health]
|
18
18
|
|
19
19
|
def execute_action
|
20
20
|
api_console=basic_auth_api('api')
|
21
21
|
command=self.options.get_next_command(ACTIONS)
|
22
22
|
case command
|
23
|
-
when :
|
23
|
+
when :health
|
24
24
|
nagios=Nagios.new
|
25
25
|
begin
|
26
26
|
api_console.read('ssh_keys')
|
@@ -26,6 +26,7 @@ module Aspera
|
|
26
26
|
self.options.add_opt_simple(:delivery_info,'package delivery information (extended value)')
|
27
27
|
self.options.add_opt_simple(:source_name,'create package from remote source (by name)')
|
28
28
|
self.options.add_opt_simple(:storage,'Faspex local storage definition')
|
29
|
+
self.options.add_opt_simple(:recipient,'use if recipient is a dropbox (with *)')
|
29
30
|
self.options.add_opt_list(:box,[:inbox,:sent,:archive],'package box')
|
30
31
|
self.options.set_option(:box,:inbox)
|
31
32
|
self.options.parse_options!
|
@@ -44,7 +45,7 @@ module Aspera
|
|
44
45
|
result={
|
45
46
|
:base_url => "#{publink_uri.scheme}://#{publink_uri.host}#{port_add}#{base}",
|
46
47
|
:subpath => subpath,
|
47
|
-
:query => URI::decode_www_form(publink_uri.query).inject({}){|
|
48
|
+
:query => URI::decode_www_form(publink_uri.query).inject({}){|h,v|h[v.first]=v.last;h}
|
48
49
|
}
|
49
50
|
Log.dump('publink',result)
|
50
51
|
return result
|
@@ -52,7 +53,7 @@ module Aspera
|
|
52
53
|
|
53
54
|
# get faspe: URI from entry in xml, and fix problems..
|
54
55
|
def self.get_fasp_uri_from_entry(entry)
|
55
|
-
raise CliBadArgument, 'package
|
56
|
+
raise CliBadArgument, 'package has no link (deleted?)' unless entry.has_key?('link')
|
56
57
|
result=entry['link'].select{|e| e['rel'].eql?('package')}.first['href']
|
57
58
|
# tags in the end of URL is not well % encoded... there are "=" that should be %3D
|
58
59
|
# TODO: enter ticket to Faspex ?
|
@@ -72,7 +73,7 @@ module Aspera
|
|
72
73
|
def self.get_source_id(source_list,source_name)
|
73
74
|
source_ids=source_list.select { |i| i['name'].eql?(source_name) }
|
74
75
|
if source_ids.empty?
|
75
|
-
raise CliError
|
76
|
+
raise CliError,%Q{No such Faspex source "#{source_name}" in [#{source_list.map{|i| %Q{"#{i['name']}"}}.join(', ')}]}
|
76
77
|
end
|
77
78
|
return source_ids.first['id']
|
78
79
|
end
|
@@ -101,19 +102,26 @@ module Aspera
|
|
101
102
|
return @api_v4
|
102
103
|
end
|
103
104
|
|
104
|
-
ACTIONS=[ :
|
105
|
+
ACTIONS=[ :health,:package, :source, :me, :dropbox, :v4, :address_book, :login_methods ]
|
105
106
|
|
106
107
|
# we match recv command on atom feed on this field
|
107
108
|
PACKAGE_MATCH_FIELD='package_id'
|
108
109
|
|
109
110
|
def mailbox_all_entries
|
110
|
-
|
111
|
+
recipient_name=self.options.get_option(:recipient,:optional) || self.options.get_option(:username,:mandatory)
|
112
|
+
mailbox=self.options.get_option(:box,:mandatory)
|
111
113
|
all_inbox_xml=api_v3.call({:operation=>'GET',:subpath=>"#{mailbox}.atom",:headers=>{'Accept'=>'application/xml'}})[:http].body
|
112
114
|
all_inbox_data=XmlSimple.xml_in(all_inbox_xml, {'ForceArray' => true})
|
113
115
|
Log.dump(:all_inbox_data,all_inbox_data)
|
114
116
|
result=all_inbox_data.has_key?('entry') ? all_inbox_data['entry'] : []
|
115
117
|
result.each do |e|
|
116
|
-
|
118
|
+
case mailbox
|
119
|
+
when :inbox,:archive
|
120
|
+
recipient=e['to'].select{|i|i['name'].first.eql?(recipient_name)}.first
|
121
|
+
e[PACKAGE_MATCH_FIELD]=recipient.nil? ? 'n/a' : recipient['recipient_delivery_id'].first
|
122
|
+
when :sent
|
123
|
+
e[PACKAGE_MATCH_FIELD]=e['delivery_id'].first
|
124
|
+
end
|
117
125
|
end
|
118
126
|
# remove dropbox packages
|
119
127
|
result.select!{|p|p['metadata'].first['field'].select{|j|j['name'].eql?('_dropbox_name')}.empty? rescue false}
|
@@ -155,7 +163,7 @@ module Aspera
|
|
155
163
|
def execute_action
|
156
164
|
command=self.options.get_next_command(ACTIONS)
|
157
165
|
case command
|
158
|
-
when :
|
166
|
+
when :health
|
159
167
|
nagios=Nagios.new
|
160
168
|
begin
|
161
169
|
api_v3.read('me')
|
@@ -201,9 +209,18 @@ module Aspera
|
|
201
209
|
#Log.dump('transfer_spec',transfer_spec)
|
202
210
|
return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
|
203
211
|
when :recv
|
204
|
-
|
205
|
-
|
206
|
-
|
212
|
+
link_url=self.options.get_option(:link,:optional)
|
213
|
+
# list of faspex ID/URI to download
|
214
|
+
pkg_id_uri=nil
|
215
|
+
skip_ids_data=[]
|
216
|
+
skip_ids_persistency=nil
|
217
|
+
case link_url
|
218
|
+
when nil
|
219
|
+
# usual case: no link
|
220
|
+
when /^faspe:/
|
221
|
+
pkg_id_uri=[{:id=>'package',:uri=>link_url}]
|
222
|
+
else
|
223
|
+
link_data=self.class.get_link_data(link_url)
|
207
224
|
if !link_data[:subpath].match(%r{external_deliveries/})
|
208
225
|
raise CliBadArgument,"pub link is #{link_data[:subpath]}, expecting external_deliveries/"
|
209
226
|
end
|
@@ -211,7 +228,7 @@ module Aspera
|
|
211
228
|
api_public_link=Rest.new({:base_url=>link_data[:base_url]})
|
212
229
|
pkgdatares=api_public_link.call({:operation=>'GET',:subpath=>link_data[:subpath],:url_params=>{:passcode=>link_data[:query]['passcode']},:headers=>{'Accept'=>'application/xml'}})
|
213
230
|
if !pkgdatares[:http].body.start_with?('<?xml ')
|
214
|
-
OpenApplication.instance.uri(
|
231
|
+
OpenApplication.instance.uri(link_url)
|
215
232
|
raise CliError, 'no such package'
|
216
233
|
end
|
217
234
|
package_entry=XmlSimple.xml_in(pkgdatares[:http].body, {'ForceArray' => false})
|
@@ -220,32 +237,30 @@ module Aspera
|
|
220
237
|
transfer_spec['direction']='receive'
|
221
238
|
return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
|
222
239
|
end # public link
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
240
|
+
if pkg_id_uri.nil?
|
241
|
+
# get command line parameters
|
242
|
+
delivid=self.options.get_option(:id,:mandatory)
|
243
|
+
if self.options.get_option(:once_only,:mandatory)
|
244
|
+
skip_ids_persistency=PersistencyActionOnce.new(
|
245
|
+
manager: @agents[:persistency],
|
246
|
+
data: skip_ids_data,
|
247
|
+
ids: ['faspex_recv',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),self.options.get_option(:box,:mandatory).to_s])
|
248
|
+
end
|
249
|
+
if delivid.eql?(VAL_ALL)
|
250
|
+
pkg_id_uri=mailbox_all_entries.map{|i|{:id=>i[PACKAGE_MATCH_FIELD],:uri=>self.class.get_fasp_uri_from_entry(i)}}
|
251
|
+
# TODO : remove ids from skip not present in inbox
|
252
|
+
# skip_ids_data.select!{|id|pkg_id_uri.select{|p|p[:id].eql?(id)}}
|
253
|
+
pkg_id_uri.select!{|i|!skip_ids_data.include?(i[:id])}
|
254
|
+
else
|
255
|
+
# TODO: delivery id is the right one if package was receive by group
|
256
|
+
endpoint=case self.options.get_option(:box,:mandatory)
|
257
|
+
when :inbox,:archive;'received'
|
258
|
+
when :sent; 'sent'
|
259
|
+
end
|
260
|
+
entry_xml=api_v3.call({:operation=>'GET',:subpath=>"#{endpoint}/#{delivid}",:headers=>{'Accept'=>'application/xml'}})[:http].body
|
261
|
+
package_entry=XmlSimple.xml_in(entry_xml, {'ForceArray' => true})
|
262
|
+
pkg_id_uri=[{:id=>delivid,:uri=>self.class.get_fasp_uri_from_entry(package_entry)}]
|
245
263
|
end
|
246
|
-
entry_xml=api_v3.call({:operation=>'GET',:subpath=>"#{endpoint}/#{delivid}",:headers=>{'Accept'=>'application/xml'}})[:http].body
|
247
|
-
package_entry=XmlSimple.xml_in(entry_xml, {'ForceArray' => true})
|
248
|
-
pkg_id_uri=[{:id=>delivid,:uri=>self.class.get_fasp_uri_from_entry(package_entry)}]
|
249
264
|
end
|
250
265
|
Log.dump(:pkg_id_uri,pkg_id_uri)
|
251
266
|
return Main.result_status('no package') if pkg_id_uri.empty?
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'aspera/cli/basic_auth_plugin'
|
2
2
|
require 'aspera/persistency_action_once'
|
3
|
+
require 'securerandom'
|
3
4
|
|
4
5
|
module Aspera
|
5
6
|
module Cli
|
@@ -8,74 +9,114 @@ module Aspera
|
|
8
9
|
VAL_ALL='ALL'
|
9
10
|
def initialize(env)
|
10
11
|
super(env)
|
11
|
-
|
12
|
-
|
12
|
+
options.add_opt_simple(:client_id,'API client identifier in application')
|
13
|
+
options.add_opt_simple(:client_secret,'API client secret in application')
|
14
|
+
options.add_opt_simple(:redirect_uri,'API client redirect URI')
|
15
|
+
options.add_opt_list(:auth,Oauth.auth_types.clone.push(:boot),'type of Oauth authentication')
|
16
|
+
options.add_opt_simple(:private_key,'RSA private key PEM value for JWT (prefix file path with @val:@file:)')
|
17
|
+
options.set_option(:auth,:jwt)
|
18
|
+
options.parse_options!
|
13
19
|
end
|
14
|
-
ACTIONS=[ :node, :package ]
|
15
20
|
|
16
|
-
|
21
|
+
def set_api
|
22
|
+
faxpex5_api_base_url=options.get_option(:url,:mandatory)
|
23
|
+
faxpex5_api_v5_url="#{faxpex5_api_base_url}/api/v5"
|
24
|
+
faxpex5_api_auth_url="#{faxpex5_api_base_url}/auth"
|
25
|
+
case options.get_option(:auth,:mandatory)
|
26
|
+
when :boot
|
27
|
+
# the password here is the token copied directly from browser in developer mode
|
28
|
+
@api_v5=Rest.new({
|
29
|
+
:base_url => faxpex5_api_v5_url,
|
30
|
+
:headers => {'Authorization'=>options.get_option(:password,:mandatory)},
|
31
|
+
})
|
32
|
+
when :web
|
33
|
+
# opens a browser and ask user to auth using web
|
34
|
+
@api_v5=Rest.new({
|
35
|
+
:base_url => faxpex5_api_v5_url,
|
36
|
+
:auth => {
|
37
|
+
:type => :oauth2,
|
38
|
+
:base_url => faxpex5_api_auth_url,
|
39
|
+
:grant => :web,
|
40
|
+
:state => SecureRandom.uuid,
|
41
|
+
:client_id => options.get_option(:client_id,:mandatory),
|
42
|
+
:redirect_uri => options.get_option(:redirect_uri,:mandatory),
|
43
|
+
}})
|
44
|
+
when :jwt
|
45
|
+
# currently Faspex 5 beta 3 only supports non-user based apis (e.g. jobs)
|
46
|
+
app_client_id=options.get_option(:client_id,:mandatory)
|
47
|
+
@api_v5=Rest.new({
|
48
|
+
:base_url => faxpex5_api_v5_url,
|
49
|
+
:auth => {
|
50
|
+
:type => :oauth2,
|
51
|
+
:base_url => faxpex5_api_auth_url,
|
52
|
+
:grant => :jwt,
|
53
|
+
:client_id => app_client_id,
|
54
|
+
:client_secret => options.get_option(:client_secret,:mandatory),
|
55
|
+
:jwt_subject => "client:#{app_client_id}", # TODO Mmmm
|
56
|
+
:jwt_audience => app_client_id, # TODO Mmmm
|
57
|
+
:jwt_private_key_obj => OpenSSL::PKey::RSA.new(options.get_option(:private_key,:mandatory)),
|
58
|
+
:jwt_is_f5 => true, # TODO: remove when clarified
|
59
|
+
:jwt_headers => {typ: 'JWT'}
|
60
|
+
}})
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
ACTIONS=[ :node, :package, :auth_client, :jobs ]
|
65
|
+
|
66
|
+
#
|
17
67
|
def execute_action
|
18
|
-
|
19
|
-
|
20
|
-
faxpex5_username=self.options.get_option(:username,:mandatory)
|
21
|
-
faxpex5_password=self.options.get_option(:password,:mandatory)
|
22
|
-
faxpex5_api_base_url+='/api/v5'
|
23
|
-
# create object for REST calls to Shares2
|
24
|
-
api_v5=Rest.new({
|
25
|
-
:base_url => faxpex5_api_base_url,
|
26
|
-
:auth => {
|
27
|
-
:type => :oauth2,
|
28
|
-
:base_url => faxpex5_api_base_url,
|
29
|
-
:grant => :body_data,
|
30
|
-
:token_field =>'auth_token',
|
31
|
-
:path_token => 'authenticate',
|
32
|
-
:path_authorize => :unused,
|
33
|
-
:userpass_body => {name: faxpex5_username,password: faxpex5_password}
|
34
|
-
}})
|
35
|
-
command=self.options.get_next_command(ACTIONS)
|
68
|
+
set_api
|
69
|
+
command=options.get_next_command(ACTIONS)
|
36
70
|
case command
|
71
|
+
when :auth_client
|
72
|
+
api_auth=Rest.new(@api_v5.params.merge({base_url: @api_v5.params[:base_url].gsub(/api\/v5$/,'auth')}))
|
73
|
+
return self.entity_action(api_auth,'oauth_clients',nil,:id,nil,true)
|
37
74
|
when :node
|
38
|
-
return self.entity_action(api_v5,'nodes',nil,:id,nil,true)
|
75
|
+
return self.entity_action(@api_v5,'nodes',nil,:id,nil,true)
|
76
|
+
when :jobs
|
77
|
+
# to test JWT
|
78
|
+
return self.entity_action(@api_v5,'jobs',nil,:id,nil,true)
|
39
79
|
when :package
|
40
|
-
command=
|
80
|
+
command=options.get_next_command([:list,:show,:send,:receive])
|
41
81
|
case command
|
42
82
|
when :list
|
43
|
-
parameters=
|
44
|
-
return {:type => :object_list, :data
|
83
|
+
parameters=options.get_option(:value,:optional)
|
84
|
+
return {:type => :object_list, :data=>@api_v5.read('packages',parameters)[:data]['packages']}
|
45
85
|
when :show
|
46
|
-
id=
|
47
|
-
return {:type => :single_object, :data
|
86
|
+
id=options.get_option(:id,:mandatory)
|
87
|
+
return {:type => :single_object, :data=>@api_v5.read("packages/#{id}")[:data]}
|
48
88
|
when :send
|
49
|
-
parameters=
|
50
|
-
raise CliBadArgument,'
|
51
|
-
package
|
52
|
-
transfer_spec
|
89
|
+
parameters=options.get_option(:value,:mandatory)
|
90
|
+
raise CliBadArgument,'value must be hash, refer to API' unless parameters.is_a?(Hash)
|
91
|
+
package=@api_v5.create('packages',parameters)[:data]
|
92
|
+
transfer_spec=@api_v5.create("packages/#{package['id']}/transfer_spec/upload",{transfer_type: 'Connect'})[:data]
|
53
93
|
transfer_spec.delete('authentication')
|
54
94
|
return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
|
55
95
|
when :receive
|
56
96
|
pkg_type='received'
|
57
|
-
pack_id=
|
97
|
+
pack_id=options.get_option(:id,:mandatory)
|
58
98
|
package_ids=[pack_id]
|
59
99
|
skip_ids_data=[]
|
60
100
|
skip_ids_persistency=nil
|
61
|
-
if
|
101
|
+
if options.get_option(:once_only,:mandatory)
|
102
|
+
# read ids from persistency
|
62
103
|
skip_ids_persistency=PersistencyActionOnce.new(
|
63
104
|
manager: @agents[:persistency],
|
64
105
|
data: skip_ids_data,
|
65
|
-
ids: ['faspex_recv',
|
106
|
+
ids: ['faspex_recv',options.get_option(:url,:mandatory),options.get_option(:username,:mandatory),pkg_type])
|
66
107
|
end
|
67
108
|
if pack_id.eql?(VAL_ALL)
|
68
|
-
#
|
69
|
-
parameters=
|
70
|
-
parameters||={
|
71
|
-
raise CliBadArgument,'value filter must be
|
72
|
-
package_ids
|
109
|
+
# TODO: if packages have same name, they will overwrite
|
110
|
+
parameters=options.get_option(:value,:optional)
|
111
|
+
parameters||={'type'=>'received','subtype'=>'mypackages','limit'=>1000}
|
112
|
+
raise CliBadArgument,'value filter must be Hash (API GET)' unless parameters.is_a?(Hash)
|
113
|
+
package_ids=@api_v5.read('packages',parameters)[:data]['packages'].map{|p|p['id']}
|
73
114
|
package_ids.select!{|i|!skip_ids_data.include?(i)}
|
74
115
|
end
|
75
116
|
result_transfer=[]
|
76
117
|
package_ids.each do |id|
|
77
118
|
# TODO: allow from sent as well ?
|
78
|
-
transfer_spec
|
119
|
+
transfer_spec=@api_v5.create("packages/#{id}/transfer_spec/download",{transfer_type: 'Connect', type: pkg_type})[:data]
|
79
120
|
transfer_spec.delete('authentication')
|
80
121
|
statuses=self.transfer.start(transfer_spec,{:src=>:node_gen3})
|
81
122
|
result_transfer.push({'package'=>id,'status'=>statuses.map{|i|i.to_s}.join(',')})
|
@@ -88,7 +88,7 @@ module Aspera
|
|
88
88
|
raise StandardError,"expect: nil, String or Array"
|
89
89
|
end
|
90
90
|
|
91
|
-
SIMPLE_ACTIONS=[:
|
91
|
+
SIMPLE_ACTIONS=[:health,:events, :space, :info, :license, :mkdir, :mklink, :mkfile, :rename, :delete, :search ]
|
92
92
|
|
93
93
|
COMMON_ACTIONS=[:browse, :upload, :download, :api_details ].concat(SIMPLE_ACTIONS)
|
94
94
|
|
@@ -96,7 +96,7 @@ module Aspera
|
|
96
96
|
# prefix_path is used to list remote sources in Faspex
|
97
97
|
def execute_simple_common(command,prefix_path)
|
98
98
|
case command
|
99
|
-
when :
|
99
|
+
when :health
|
100
100
|
nagios=Nagios.new
|
101
101
|
begin
|
102
102
|
info=@api_node.read('info')[:data]
|
@@ -341,7 +341,7 @@ module Aspera
|
|
341
341
|
raise "error"
|
342
342
|
end
|
343
343
|
when :access_key
|
344
|
-
return self.entity_action(@api_node,'access_keys',
|
344
|
+
return self.entity_action(@api_node,'access_keys',nil,:id,'self')
|
345
345
|
when :service
|
346
346
|
command=self.options.get_next_command([ :list, :create, :delete])
|
347
347
|
if [:delete].include?(command)
|
@@ -4,6 +4,7 @@ require 'aspera/preview/options'
|
|
4
4
|
require 'aspera/preview/utils'
|
5
5
|
require 'aspera/preview/file_types'
|
6
6
|
require 'aspera/persistency_action_once'
|
7
|
+
require 'aspera/node'
|
7
8
|
require 'aspera/hash_ext'
|
8
9
|
require 'date'
|
9
10
|
require 'securerandom'
|
@@ -56,9 +57,11 @@ module Aspera
|
|
56
57
|
self.options.add_opt_simple(:case,'basename of output for for test')
|
57
58
|
self.options.add_opt_simple(:scan_path,'subpath in folder id to start scan in (default=/)')
|
58
59
|
self.options.add_opt_simple(:scan_id,'forder id in storage to start scan in, default is access key main folder id')
|
60
|
+
self.options.add_opt_boolean(:mimemagic,'use Mime type detection of gem mimemagic')
|
59
61
|
self.options.add_opt_list(:overwrite,[:always,:never,:mtime],'when to overwrite result file')
|
60
62
|
self.options.add_opt_list(:file_access,[:local,:remote],'how to read and write files in repository')
|
61
63
|
self.options.set_option(:temp_folder,Dir.tmpdir)
|
64
|
+
self.options.set_option(:mimemagic,:false)
|
62
65
|
|
63
66
|
# add other options for generator (and set default values)
|
64
67
|
Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
|
@@ -100,8 +103,6 @@ module Aspera
|
|
100
103
|
return @preview_formats_to_generate.map{|i|i.to_s}.join(',')
|
101
104
|
end
|
102
105
|
|
103
|
-
ACTIONS=[:scan,:events,:trevents,:check,:test]
|
104
|
-
|
105
106
|
# /files/id/files is normally cached in redis, but we can discard the cache
|
106
107
|
# but /files/id is not cached
|
107
108
|
def get_folder_entries(file_id,request_args=nil)
|
@@ -112,14 +113,23 @@ module Aspera
|
|
112
113
|
end
|
113
114
|
|
114
115
|
# old version based on folders
|
115
|
-
def
|
116
|
+
def process_trevents(iteration_token)
|
116
117
|
events_filter={
|
117
118
|
'access_key'=>@access_key_self['id'],
|
118
119
|
'type'=>'download.ended'
|
119
120
|
}
|
120
121
|
# optionally by iteration token
|
121
122
|
events_filter['iteration_token']=iteration_token unless iteration_token.nil?
|
122
|
-
|
123
|
+
begin
|
124
|
+
events=@api_node.read('events',events_filter)[:data]
|
125
|
+
rescue RestCallError => e
|
126
|
+
if e.message.include?('Invalid iteration_token')
|
127
|
+
Log.log.warn("Retrying without iteration token: #{e}")
|
128
|
+
events_filter.delete('iteration_token')
|
129
|
+
retry
|
130
|
+
end
|
131
|
+
raise e
|
132
|
+
end
|
123
133
|
return if events.empty?
|
124
134
|
events.each do |event|
|
125
135
|
next unless event['data']['direction'].eql?('receive')
|
@@ -137,7 +147,7 @@ module Aspera
|
|
137
147
|
end
|
138
148
|
|
139
149
|
# requests recent events on node api and process newly modified folders
|
140
|
-
def
|
150
|
+
def process_events(iteration_token)
|
141
151
|
# get new file creation by access key (TODO: what if file already existed?)
|
142
152
|
events_filter={
|
143
153
|
'access_key'=>@access_key_self['id'],
|
@@ -276,7 +286,7 @@ module Aspera
|
|
276
286
|
end
|
277
287
|
end
|
278
288
|
# need generator for further checks
|
279
|
-
gen_info[:generator]=Aspera::Preview::Generator.new(@gen_options,gen_info[:src],gen_info[:dst],@tmp_folder,entry['content_type']
|
289
|
+
gen_info[:generator]=Aspera::Preview::Generator.new(@gen_options,gen_info[:src],gen_info[:dst],@tmp_folder,entry['content_type'])
|
280
290
|
# get conversion_type (if known) and check if supported
|
281
291
|
next false unless gen_info[:generator].supported?
|
282
292
|
# shall we skip it ?
|
@@ -313,12 +323,14 @@ module Aspera
|
|
313
323
|
end # generate_preview
|
314
324
|
|
315
325
|
# scan all files in provided folder entry
|
326
|
+
# @param scan_start subpath to start folder scan inside
|
316
327
|
def scan_folder_files(top_entry,scan_start=nil)
|
317
328
|
if !scan_start.nil?
|
318
329
|
# canonical path: start with / and ends with /
|
319
330
|
scan_start='/'+scan_start.split('/').select{|i|!i.empty?}.join('/')
|
320
331
|
scan_start="#{scan_start}/" #unless scan_start.end_with?('/')
|
321
332
|
end
|
333
|
+
filter_block=Aspera::Node.file_matcher(options.get_option(:value,:optional))
|
322
334
|
Log.log.debug("scan: #{top_entry} : #{scan_start}".green)
|
323
335
|
# don't use recursive call, use list instead
|
324
336
|
entries_to_process=[top_entry]
|
@@ -334,7 +346,11 @@ module Aspera
|
|
334
346
|
Log.log.debug("item:#{entry}")
|
335
347
|
case entry['type']
|
336
348
|
when 'file'
|
337
|
-
|
349
|
+
if filter_block.call(entry)
|
350
|
+
generate_preview(entry)
|
351
|
+
else
|
352
|
+
Log.log.debug('skip by filter')
|
353
|
+
end
|
338
354
|
when 'link'
|
339
355
|
Log.log.debug('Ignoring link.')
|
340
356
|
when 'folder'
|
@@ -360,6 +376,8 @@ module Aspera
|
|
360
376
|
end
|
361
377
|
end
|
362
378
|
|
379
|
+
ACTIONS=[:scan,:events,:trevents,:check,:test]
|
380
|
+
|
363
381
|
def execute_action
|
364
382
|
command=self.options.get_next_command(ACTIONS)
|
365
383
|
unless [:check,:test].include?(command)
|
@@ -402,6 +420,7 @@ module Aspera
|
|
402
420
|
end
|
403
421
|
end
|
404
422
|
end
|
423
|
+
Aspera::Preview::FileTypes.instance.use_mimemagic = self.options.get_option(:mimemagic,:mandatory)
|
405
424
|
case command
|
406
425
|
when :scan
|
407
426
|
scan_path=self.options.get_option(:scan_path,:optional)
|
@@ -417,30 +436,20 @@ module Aspera
|
|
417
436
|
end
|
418
437
|
scan_folder_files(folder_info,scan_path)
|
419
438
|
return Main.result_status('scan finished')
|
420
|
-
when :events
|
439
|
+
when :events,:trevents
|
421
440
|
iteration_data=[]
|
422
441
|
iteration_persistency=nil
|
423
442
|
if self.options.get_option(:once_only,:mandatory)
|
424
443
|
iteration_persistency=PersistencyActionOnce.new(
|
425
444
|
manager: @agents[:persistency],
|
426
445
|
data: iteration_data,
|
427
|
-
ids:
|
446
|
+
ids: ["preview_iteration_#{command}",self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
|
428
447
|
end
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
when :trevents
|
433
|
-
iteration_data=[]
|
434
|
-
iteration_persistency=nil
|
435
|
-
if self.options.get_option(:once_only,:mandatory)
|
436
|
-
iteration_persistency=PersistencyActionOnce.new(
|
437
|
-
manager: @agents[:persistency],
|
438
|
-
data: iteration_data,
|
439
|
-
ids: ['preview_iteration_transfer',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
|
440
|
-
end
|
441
|
-
iteration_data[0]=process_transfer_events(iteration_data[0])
|
448
|
+
|
449
|
+
# call method specified
|
450
|
+
iteration_data[0]=send("process_#{command}",iteration_data[0])
|
442
451
|
iteration_persistency.save unless iteration_persistency.nil?
|
443
|
-
return Main.result_status(
|
452
|
+
return Main.result_status("#{command} finished")
|
444
453
|
when :check
|
445
454
|
Aspera::Preview::Utils.check_tools(@skip_types)
|
446
455
|
return Main.result_status('tools validated')
|
@@ -448,8 +457,9 @@ module Aspera
|
|
448
457
|
format = self.options.get_next_argument('format',Aspera::Preview::Generator::PREVIEW_FORMATS)
|
449
458
|
source = self.options.get_next_argument('source file')
|
450
459
|
dest=preview_filename(format,self.options.get_option(:case,:optional))
|
451
|
-
g=Aspera::Preview::Generator.new(@gen_options,source,dest,@tmp_folder)
|
452
|
-
raise "
|
460
|
+
g=Aspera::Preview::Generator.new(@gen_options,source,dest,@tmp_folder,nil)
|
461
|
+
raise "cannot find file type for #{source}" if g.conversion_type.nil?
|
462
|
+
raise "out format #{format} not supported" unless g.supported?
|
453
463
|
g.generate
|
454
464
|
return Main.result_status("generated: #{dest}")
|
455
465
|
else
|