aspera-cli 4.6.0 → 4.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +427 -300
- data/bin/ascli +2 -1
- data/bin/asession +1 -0
- data/docs/test_env.conf +2 -0
- data/examples/aoc.rb +4 -3
- data/examples/faspex4.rb +21 -19
- data/examples/proxy.pac +1 -1
- data/examples/transfer.rb +15 -15
- data/lib/aspera/aoc.rb +135 -124
- data/lib/aspera/ascmd.rb +85 -75
- data/lib/aspera/ats_api.rb +11 -10
- data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
- data/lib/aspera/cli/extended_value.rb +42 -33
- data/lib/aspera/cli/formater.rb +138 -111
- data/lib/aspera/cli/info.rb +17 -0
- data/lib/aspera/cli/listener/line_dump.rb +3 -2
- data/lib/aspera/cli/listener/logger.rb +2 -1
- data/lib/aspera/cli/listener/progress.rb +16 -18
- data/lib/aspera/cli/listener/progress_multi.rb +13 -16
- data/lib/aspera/cli/main.rb +122 -130
- data/lib/aspera/cli/manager.rb +146 -154
- data/lib/aspera/cli/plugin.rb +38 -34
- data/lib/aspera/cli/plugins/alee.rb +6 -6
- data/lib/aspera/cli/plugins/aoc.rb +273 -276
- data/lib/aspera/cli/plugins/ats.rb +82 -76
- data/lib/aspera/cli/plugins/bss.rb +14 -16
- data/lib/aspera/cli/plugins/config.rb +350 -306
- data/lib/aspera/cli/plugins/console.rb +23 -19
- data/lib/aspera/cli/plugins/cos.rb +18 -18
- data/lib/aspera/cli/plugins/faspex.rb +180 -159
- data/lib/aspera/cli/plugins/faspex5.rb +64 -54
- data/lib/aspera/cli/plugins/node.rb +147 -140
- data/lib/aspera/cli/plugins/orchestrator.rb +68 -66
- data/lib/aspera/cli/plugins/preview.rb +92 -96
- data/lib/aspera/cli/plugins/server.rb +79 -75
- data/lib/aspera/cli/plugins/shares.rb +23 -24
- data/lib/aspera/cli/plugins/sync.rb +20 -22
- data/lib/aspera/cli/transfer_agent.rb +40 -39
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +35 -27
- data/lib/aspera/command_line_builder.rb +48 -34
- data/lib/aspera/cos_node.rb +29 -21
- data/lib/aspera/data_repository.rb +3 -2
- data/lib/aspera/environment.rb +50 -45
- data/lib/aspera/fasp/agent_base.rb +22 -20
- data/lib/aspera/fasp/agent_connect.rb +13 -11
- data/lib/aspera/fasp/agent_direct.rb +48 -59
- data/lib/aspera/fasp/agent_httpgw.rb +33 -39
- data/lib/aspera/fasp/agent_node.rb +15 -13
- data/lib/aspera/fasp/agent_trsdk.rb +12 -14
- data/lib/aspera/fasp/error.rb +2 -1
- data/lib/aspera/fasp/error_info.rb +68 -52
- data/lib/aspera/fasp/installation.rb +106 -94
- data/lib/aspera/fasp/listener.rb +1 -0
- data/lib/aspera/fasp/parameters.rb +83 -92
- data/lib/aspera/fasp/parameters.yaml +305 -249
- data/lib/aspera/fasp/resume_policy.rb +11 -14
- data/lib/aspera/fasp/transfer_spec.rb +26 -0
- data/lib/aspera/fasp/uri.rb +22 -21
- data/lib/aspera/faspex_gw.rb +55 -90
- data/lib/aspera/hash_ext.rb +4 -3
- data/lib/aspera/id_generator.rb +8 -7
- data/lib/aspera/keychain/encrypted_hash.rb +17 -16
- data/lib/aspera/keychain/macos_security.rb +6 -10
- data/lib/aspera/log.rb +25 -20
- data/lib/aspera/nagios.rb +13 -12
- data/lib/aspera/node.rb +30 -22
- data/lib/aspera/oauth.rb +175 -226
- data/lib/aspera/open_application.rb +4 -3
- data/lib/aspera/persistency_action_once.rb +6 -6
- data/lib/aspera/persistency_folder.rb +5 -9
- data/lib/aspera/preview/file_types.rb +6 -5
- data/lib/aspera/preview/generator.rb +25 -24
- data/lib/aspera/preview/options.rb +16 -14
- data/lib/aspera/preview/utils.rb +98 -98
- data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
- data/lib/aspera/proxy_auto_config.rb +111 -20
- data/lib/aspera/rest.rb +115 -113
- data/lib/aspera/rest_call_error.rb +2 -2
- data/lib/aspera/rest_error_analyzer.rb +23 -25
- data/lib/aspera/rest_errors_aspera.rb +15 -14
- data/lib/aspera/ssh.rb +12 -10
- data/lib/aspera/sync.rb +42 -41
- data/lib/aspera/temp_file_manager.rb +18 -14
- data/lib/aspera/timer_limiter.rb +2 -1
- data/lib/aspera/uri_reader.rb +7 -5
- data/lib/aspera/web_auth.rb +79 -76
- metadata +64 -21
- data/docs/Makefile +0 -65
- data/docs/README.erb.md +0 -4424
- data/docs/README.md +0 -13
- data/docs/diagrams.txt +0 -49
- data/docs/doc_tools.rb +0 -58
- data/lib/aspera/cli/plugins/shares2.rb +0 -114
- data/lib/aspera/fasp/default.rb +0 -17
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'aspera/cli/plugins/node'
|
2
3
|
require 'xmlsimple'
|
3
4
|
|
@@ -7,47 +8,47 @@ module Aspera
|
|
7
8
|
class Orchestrator < BasicAuthPlugin
|
8
9
|
def initialize(env)
|
9
10
|
super(env)
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
11
|
+
options.add_opt_simple(:params,'parameters hash table, use @json:{"param":"value"}')
|
12
|
+
options.add_opt_simple(:result,"specify result value as: 'work step:parameter'")
|
13
|
+
options.add_opt_boolean(:synchronous,'work step:parameter expected as result')
|
14
|
+
options.add_opt_list(:ret_style,[:header,:arg,:ext],'how return type is requested in api')
|
15
|
+
options.add_opt_list(:auth_style,[:arg_pass,:head_basic,:apikey],'authentication type')
|
16
|
+
options.set_option(:params,{})
|
17
|
+
options.set_option(:synchronous,:no)
|
18
|
+
options.set_option(:ret_style,:arg)
|
19
|
+
options.set_option(:auth_style,:head_basic)
|
20
|
+
options.parse_options!
|
20
21
|
end
|
21
22
|
|
22
|
-
ACTIONS=[:info, :workflow, :plugins, :processes]
|
23
|
+
ACTIONS=[:info, :workflow, :plugins, :processes].freeze
|
23
24
|
|
24
25
|
# for JSON format: add extension ".json" or add url parameter: format=json or Accept: application/json
|
25
26
|
# id can be: a parameter id=x, or at the end of url /id, for workflows: work_order[workflow_id]=wf_id
|
26
|
-
def call_API_orig(endpoint,id=nil,url_params={:
|
27
|
-
# calls are GET
|
28
|
-
call_args={:
|
29
|
-
# specify id if necessary
|
30
|
-
call_args[:subpath]=call_args[:subpath]+'/'+id unless id.nil?
|
31
|
-
unless url_params.nil?
|
32
|
-
if url_params.has_key?(:format)
|
33
|
-
call_args[:headers]={'Accept'=>'application/'+url_params[:format].to_s}
|
34
|
-
end
|
35
|
-
call_args[:headers]={'Accept'=>accept} unless accept.nil?
|
36
|
-
# add params if necessary
|
37
|
-
call_args[:url_params]=url_params
|
38
|
-
end
|
39
|
-
return @api_orch.call(call_args)
|
40
|
-
end
|
27
|
+
# def call_API_orig(endpoint,id=nil,url_params={format: :json},accept=nil)
|
28
|
+
# # calls are GET
|
29
|
+
# call_args={operation: 'GET',subpath: endpoint}
|
30
|
+
# # specify id if necessary
|
31
|
+
# call_args[:subpath]=call_args[:subpath]+'/'+id unless id.nil?
|
32
|
+
# unless url_params.nil?
|
33
|
+
# if url_params.has_key?(:format)
|
34
|
+
# call_args[:headers]={'Accept'=>'application/'+url_params[:format].to_s}
|
35
|
+
# end
|
36
|
+
# call_args[:headers]={'Accept'=>accept} unless accept.nil?
|
37
|
+
# # add params if necessary
|
38
|
+
# call_args[:url_params]=url_params
|
39
|
+
# end
|
40
|
+
# return @api_orch.call(call_args)
|
41
|
+
# end
|
41
42
|
|
42
|
-
def
|
43
|
+
def call_ao(endpoint,opt={})
|
43
44
|
opt[:prefix]='api' unless opt.has_key?(:prefix)
|
44
45
|
# calls are GET
|
45
|
-
call_args={:
|
46
|
+
call_args={operation: 'GET',subpath: endpoint}
|
46
47
|
# specify prefix if necessary
|
47
48
|
call_args[:subpath]="#{opt[:prefix]}/#{call_args[:subpath]}" unless opt[:prefix].nil?
|
48
49
|
# specify id if necessary
|
49
50
|
call_args[:subpath]="#{call_args[:subpath]}/#{opt[:id]}" if opt.has_key?(:id)
|
50
|
-
call_type=
|
51
|
+
call_type=options.get_option(:ret_style,:mandatory)
|
51
52
|
call_type=opt[:ret_style] if opt.has_key?(:ret_style)
|
52
53
|
format='json'
|
53
54
|
format=opt[:format] if opt.has_key?(:format)
|
@@ -61,82 +62,83 @@ module Aspera
|
|
61
62
|
call_args[:url_params][:format]=format
|
62
63
|
when :ext
|
63
64
|
call_args[:subpath]="#{call_args[:subpath]}.#{format}"
|
64
|
-
else raise
|
65
|
+
else raise 'unexpected'
|
65
66
|
end
|
66
67
|
end
|
67
68
|
result=@api_orch.call(call_args)
|
68
|
-
result[:data]=XmlSimple.xml_in(result[:http].body, opt[:xml_opt]||{
|
69
|
+
result[:data]=XmlSimple.xml_in(result[:http].body, opt[:xml_opt]||{'ForceArray' => true}) if format.eql?('xml')
|
69
70
|
return result
|
70
71
|
end
|
71
72
|
|
72
73
|
def execute_action
|
73
|
-
rest_params={:
|
74
|
-
case
|
74
|
+
rest_params={base_url: options.get_option(:url,:mandatory)}
|
75
|
+
case options.get_option(:auth_style,:mandatory)
|
75
76
|
when :arg_pass
|
76
77
|
rest_params[:auth]={
|
77
|
-
|
78
|
-
:
|
79
|
-
'login' =>
|
80
|
-
'password' =>
|
78
|
+
type: :url,
|
79
|
+
url_creds: {
|
80
|
+
'login' =>options.get_option(:username,:mandatory),
|
81
|
+
'password' =>options.get_option(:password,:mandatory) }}
|
81
82
|
when :head_basic
|
82
83
|
rest_params[:auth]={
|
83
|
-
|
84
|
-
:
|
85
|
-
:
|
84
|
+
type: :basic,
|
85
|
+
username: options.get_option(:username,:mandatory),
|
86
|
+
password: options.get_option(:password,:mandatory) }
|
86
87
|
when :apikey
|
87
|
-
raise
|
88
|
+
raise 'Not implemented'
|
88
89
|
end
|
89
90
|
|
90
91
|
@api_orch=Rest.new(rest_params)
|
91
92
|
|
92
|
-
command1=
|
93
|
+
command1=options.get_next_command(ACTIONS)
|
93
94
|
case command1
|
94
95
|
when :info
|
95
|
-
result=
|
96
|
-
return {:
|
97
|
-
# result=
|
96
|
+
result=call_ao('remote_node_ping',format: 'xml', xml_opt: {'ForceArray' => false})
|
97
|
+
return {type: :single_object,data: result[:data]}
|
98
|
+
# result=call_ao('workflows',prefix: nil,format: nil)
|
98
99
|
# version='unknown'
|
99
100
|
# if m=result[:http].body.match(/\(Orchestrator v([1-9]+\.[\.0-9a-f\-]+)\)/)
|
100
101
|
# version=m[1]
|
101
102
|
# end
|
102
|
-
# return {:
|
103
|
+
# return {type: :single_object,data: {'version'=>version}}
|
103
104
|
when :processes
|
104
105
|
# TODO: Jira ? API has only XML format
|
105
|
-
result=
|
106
|
-
return {:
|
106
|
+
result=call_ao('processes_status',format: 'xml')
|
107
|
+
return {type: :object_list,data: result[:data]['process']}
|
107
108
|
when :plugins
|
108
109
|
# TODO: Jira ? only json format on url
|
109
|
-
result=
|
110
|
-
return {:
|
110
|
+
result=call_ao('plugin_version')[:data]
|
111
|
+
return {type: :object_list,data: result['Plugin']}
|
111
112
|
when :workflow
|
112
|
-
command=
|
113
|
+
command=options.get_next_command([:list, :status, :inputs, :details, :start, :export])
|
113
114
|
unless [:list].include?(command)
|
114
|
-
wf_id=
|
115
|
+
wf_id=instance_identifier()
|
115
116
|
end
|
116
117
|
case command
|
117
118
|
when :status
|
118
119
|
options={}
|
119
120
|
options[:id]=wf_id unless wf_id.eql?('ALL')
|
120
|
-
result=
|
121
|
-
return {:
|
121
|
+
result=call_ao('workflows_status',options)[:data]
|
122
|
+
return {type: :object_list,data: result['workflows']['workflow']}
|
122
123
|
when :list
|
123
|
-
result=
|
124
|
-
return {:
|
124
|
+
result=call_ao('workflows_list',id: 0)[:data]
|
125
|
+
return {type: :object_list,data: result['workflows']['workflow'],
|
126
|
+
fields: ['id','portable_id','name','published_status','published_revision_id','latest_revision_id','last_modification']}
|
125
127
|
when :details
|
126
|
-
result=
|
127
|
-
return {:
|
128
|
+
result=call_ao('workflow_details',id: wf_id)[:data]
|
129
|
+
return {type: :object_list,data: result['workflows']['workflow']['statuses']}
|
128
130
|
when :inputs
|
129
|
-
result=
|
130
|
-
return {:
|
131
|
+
result=call_ao('workflow_inputs_spec',id: wf_id)[:data]
|
132
|
+
return {type: :single_object,data: result['workflow_inputs_spec']}
|
131
133
|
when :export
|
132
|
-
result=
|
133
|
-
return {:
|
134
|
+
result=call_ao('export_workflow',id: wf_id,format: nil)[:http]
|
135
|
+
return {type: :text,data: result.body}
|
134
136
|
when :start
|
135
137
|
result={
|
136
|
-
:
|
137
|
-
:
|
138
|
+
type: :single_object,
|
139
|
+
data: nil
|
138
140
|
}
|
139
|
-
call_params={:
|
141
|
+
call_params={format: :json}
|
140
142
|
override_accept=nil
|
141
143
|
# set external parameters if any
|
142
144
|
self.options.get_option(:params,:mandatory).each do |name,value|
|
@@ -159,7 +161,7 @@ module Aspera
|
|
159
161
|
result[:type]=:text
|
160
162
|
override_accept='text/plain'
|
161
163
|
end
|
162
|
-
result[:data]=
|
164
|
+
result[:data]=call_ao('initiate',id: wf_id,args: call_params,accept: override_accept)[:data]
|
163
165
|
return result
|
164
166
|
end # wf command
|
165
167
|
else raise "ERROR, unknown command: [#{command}]"
|
@@ -1,8 +1,10 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'aspera/cli/basic_auth_plugin'
|
2
3
|
require 'aspera/preview/generator'
|
3
4
|
require 'aspera/preview/options'
|
4
5
|
require 'aspera/preview/utils'
|
5
6
|
require 'aspera/preview/file_types'
|
7
|
+
require 'aspera/fasp/transfer_spec'
|
6
8
|
require 'aspera/persistency_action_once'
|
7
9
|
require 'aspera/node'
|
8
10
|
require 'aspera/hash_ext'
|
@@ -31,10 +33,7 @@ module Aspera
|
|
31
33
|
|
32
34
|
# option_skip_format has special accessors
|
33
35
|
attr_accessor :option_previews_folder
|
34
|
-
attr_accessor :option_folder_reset_cache
|
35
|
-
attr_accessor :option_skip_folders
|
36
|
-
attr_accessor :option_overwrite
|
37
|
-
attr_accessor :option_file_access
|
36
|
+
attr_accessor :option_folder_reset_cache, :option_skip_folders, :option_overwrite, :option_file_access
|
38
37
|
def initialize(env)
|
39
38
|
super(env)
|
40
39
|
@skip_types=[]
|
@@ -46,43 +45,43 @@ module Aspera
|
|
46
45
|
# used to trigger periodic processing
|
47
46
|
@periodic=TimerLimiter.new(LOG_LIMITER_SEC)
|
48
47
|
# link CLI options to gen_info attributes
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
48
|
+
options.set_obj_attr(:skip_format,self,:option_skip_format,[]) # no skip
|
49
|
+
options.set_obj_attr(:folder_reset_cache,self,:option_folder_reset_cache,:no)
|
50
|
+
options.set_obj_attr(:skip_types,self,:option_skip_types)
|
51
|
+
options.set_obj_attr(:previews_folder,self,:option_previews_folder,DEFAULT_PREVIEWS_FOLDER)
|
52
|
+
options.set_obj_attr(:skip_folders,self,:option_skip_folders,[]) # no skip
|
53
|
+
options.set_obj_attr(:overwrite,self,:option_overwrite,:mtime)
|
54
|
+
options.set_obj_attr(:file_access,self,:option_file_access,:local)
|
55
|
+
options.add_opt_list(:skip_format,Aspera::Preview::Generator::PREVIEW_FORMATS,'skip this preview format (multiple possible)')
|
56
|
+
options.add_opt_list(:folder_reset_cache,[:no,:header,:read],'force detection of generated preview by refresh cache')
|
57
|
+
options.add_opt_simple(:skip_types,'skip types in comma separated list')
|
58
|
+
options.add_opt_simple(:previews_folder,'preview folder in storage root')
|
59
|
+
options.add_opt_simple(:temp_folder,'path to temp folder')
|
60
|
+
options.add_opt_simple(:skip_folders,'list of folder to skip')
|
61
|
+
options.add_opt_simple(:case,'basename of output for for test')
|
62
|
+
options.add_opt_simple(:scan_path,'subpath in folder id to start scan in (default=/)')
|
63
|
+
options.add_opt_simple(:scan_id,'forder id in storage to start scan in, default is access key main folder id')
|
64
|
+
options.add_opt_boolean(:mimemagic,'use Mime type detection of gem mimemagic')
|
65
|
+
options.add_opt_list(:overwrite,[:always,:never,:mtime],'when to overwrite result file')
|
66
|
+
options.add_opt_list(:file_access,[:local,:remote],'how to read and write files in repository')
|
67
|
+
options.set_option(:temp_folder,Dir.tmpdir)
|
68
|
+
options.set_option(:mimemagic,false)
|
70
69
|
|
71
70
|
# add other options for generator (and set default values)
|
72
71
|
Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
|
73
|
-
|
72
|
+
options.set_obj_attr(opt[:name],@gen_options,opt[:name],opt[:default])
|
74
73
|
if opt.has_key?(:values)
|
75
|
-
|
74
|
+
options.add_opt_list(opt[:name],opt[:values],opt[:description])
|
76
75
|
elsif [:yes,:no].include?(opt[:default])
|
77
|
-
|
76
|
+
options.add_opt_boolean(opt[:name],opt[:description])
|
78
77
|
else
|
79
|
-
|
78
|
+
options.add_opt_simple(opt[:name],opt[:description])
|
80
79
|
end
|
81
80
|
end
|
82
81
|
|
83
|
-
|
82
|
+
options.parse_options!
|
84
83
|
raise 'skip_folder shall be an Array, use @json:[...]' unless @option_skip_folders.is_a?(Array)
|
85
|
-
@tmp_folder=File.join(
|
84
|
+
@tmp_folder=File.join(options.get_option(:temp_folder,:mandatory),"#{TMP_DIR_PREFIX}.#{SecureRandom.uuid}")
|
86
85
|
FileUtils.mkdir_p(@tmp_folder)
|
87
86
|
Log.log.debug("tmpdir: #{@tmp_folder}")
|
88
87
|
end
|
@@ -97,7 +96,7 @@ module Aspera
|
|
97
96
|
end
|
98
97
|
|
99
98
|
def option_skip_types
|
100
|
-
return @skip_types.map
|
99
|
+
return @skip_types.map(&:to_s).join(',')
|
101
100
|
end
|
102
101
|
|
103
102
|
def option_skip_format=(value)
|
@@ -105,7 +104,7 @@ module Aspera
|
|
105
104
|
end
|
106
105
|
|
107
106
|
def option_skip_format
|
108
|
-
return @preview_formats_to_generate.map
|
107
|
+
return @preview_formats_to_generate.map(&:to_s).join(',')
|
109
108
|
end
|
110
109
|
|
111
110
|
# /files/id/files is normally cached in redis, but we can discard the cache
|
@@ -113,7 +112,7 @@ module Aspera
|
|
113
112
|
def get_folder_entries(file_id,request_args=nil)
|
114
113
|
headers={'Accept'=>'application/json'}
|
115
114
|
headers.merge!({'X-Aspera-Cache-Control'=>'no-cache'}) if @option_folder_reset_cache.eql?(:header)
|
116
|
-
return @api_node.call({:
|
115
|
+
return @api_node.call({operation: 'GET',subpath: "files/#{file_id}/files",headers: headers,url_params: request_args})[:data]
|
117
116
|
#return @api_node.read("files/#{file_id}/files",request_args)[:data]
|
118
117
|
end
|
119
118
|
|
@@ -138,9 +137,9 @@ module Aspera
|
|
138
137
|
end
|
139
138
|
return if events.empty?
|
140
139
|
events.each do |event|
|
141
|
-
if event['data']['direction'].eql?(
|
142
|
-
event['data']['status'].eql?('completed')
|
143
|
-
event['data']['error_code'].eql?(0)
|
140
|
+
if event['data']['direction'].eql?(Fasp::TransferSpec::DIRECTION_RECEIVE) &&
|
141
|
+
event['data']['status'].eql?('completed') &&
|
142
|
+
event['data']['error_code'].eql?(0) &&
|
144
143
|
event['data'].dig('tags','aspera',PREV_GEN_TAG).nil?
|
145
144
|
folder_id=event.dig('data','tags','aspera','node','file_id')
|
146
145
|
folder_id||=event.dig('data','file_id')
|
@@ -149,13 +148,13 @@ module Aspera
|
|
149
148
|
scan_folder_files(folder_entry) unless folder_entry.nil?
|
150
149
|
end
|
151
150
|
end
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
151
|
+
# log/persist periodically or last one
|
152
|
+
next unless @periodic.trigger? || event.equal?(events.last)
|
153
|
+
Log.log.info("Processed event #{event['id']}")
|
154
|
+
# save checkpoint to avoid losing processing in case of error
|
155
|
+
if !iteration_persistency.nil?
|
156
|
+
iteration_persistency.data[0]=event['id'].to_s
|
157
|
+
iteration_persistency.save
|
159
158
|
end
|
160
159
|
end
|
161
160
|
end
|
@@ -175,7 +174,7 @@ module Aspera
|
|
175
174
|
# process only files
|
176
175
|
if event.dig('data','type').eql?('file')
|
177
176
|
file_entry=@api_node.read("files/#{event['data']['id']}")[:data] rescue nil
|
178
|
-
if !file_entry.nil?
|
177
|
+
if !file_entry.nil? &&
|
179
178
|
@option_skip_folders.select{|d|file_entry['path'].start_with?(d)}.empty?
|
180
179
|
file_entry['parent_file_id']=event['data']['parent_file_id']
|
181
180
|
if event['types'].include?('file.deleted')
|
@@ -186,30 +185,30 @@ module Aspera
|
|
186
185
|
end
|
187
186
|
end
|
188
187
|
end
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
188
|
+
# log/persist periodically or last one
|
189
|
+
next unless @periodic.trigger? || event.equal?(events.last)
|
190
|
+
Log.log.info("Processing event #{event['id']}")
|
191
|
+
# save checkpoint to avoid losing processing in case of error
|
192
|
+
if !iteration_persistency.nil?
|
193
|
+
iteration_persistency.data[0]=event['id'].to_s
|
194
|
+
iteration_persistency.save
|
196
195
|
end
|
197
196
|
end
|
198
197
|
end
|
199
198
|
|
200
199
|
def do_transfer(direction,folder_id,source_filename,destination='/')
|
201
|
-
raise
|
200
|
+
raise 'error' if destination.nil? && direction.eql?(Fasp::TransferSpec::DIRECTION_RECEIVE)
|
202
201
|
if @default_transfer_spec.nil?
|
203
202
|
# make a dummy call to get some default transfer parameters
|
204
203
|
res=@api_node.create('files/upload_setup',{'transfer_requests'=>[{'transfer_request'=>{'paths'=>[{}],'destination_root'=>'/'}}]})
|
205
204
|
template_ts=res[:data]['transfer_specs'].first['transfer_spec']
|
206
205
|
# get ports, anyway that should be 33001 for both. add remote_user ?
|
207
|
-
@default_transfer_spec=['ssh_port','fasp_port'].
|
208
|
-
if
|
209
|
-
Log.log.warn(
|
210
|
-
@default_transfer_spec['remote_user']=Aspera::Fasp::
|
206
|
+
@default_transfer_spec=['ssh_port','fasp_port'].each_with_object({}){|e,h|h[e]=template_ts[e];}
|
207
|
+
if !@default_transfer_spec['remote_user'].eql?(Aspera::Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
|
208
|
+
Log.log.warn('remote_user shall be xfer')
|
209
|
+
@default_transfer_spec['remote_user']=Aspera::Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER
|
211
210
|
end
|
212
|
-
Aspera::Node
|
211
|
+
Aspera::Node.set_ak_basic_token(@default_transfer_spec,@access_key_self['id'],options.get_option(:password,:mandatory))
|
213
212
|
# note: we use the same address for ascp than for node api instead of the one from upload_setup
|
214
213
|
# TODO: configurable ? useful ?
|
215
214
|
@default_transfer_spec['remote_host']=@transfer_server_address
|
@@ -225,33 +224,34 @@ module Aspera
|
|
225
224
|
})
|
226
225
|
# force destination
|
227
226
|
# tspec['destination_root']=destination
|
228
|
-
|
229
|
-
Main.result_transfer(
|
227
|
+
transfer.option_transfer_spec_deep_merge({'destination_root'=>destination})
|
228
|
+
Main.result_transfer(transfer.start(tspec,{src: :node_gen4}))
|
230
229
|
end
|
231
230
|
|
232
|
-
def get_infos_local(gen_infos,entry
|
231
|
+
def get_infos_local(gen_infos,entry)
|
233
232
|
local_original_filepath=File.join(@local_storage_root,entry['path'])
|
234
233
|
original_mtime=File.mtime(local_original_filepath)
|
235
234
|
# out
|
236
|
-
local_entry_preview_dir
|
235
|
+
local_entry_preview_dir=File.join(@local_preview_folder, entry_preview_folder_name(entry))
|
237
236
|
gen_infos.each do |gen_info|
|
238
237
|
gen_info[:src]=local_original_filepath
|
239
238
|
gen_info[:dst]=File.join(local_entry_preview_dir, gen_info[:base_dest])
|
240
239
|
gen_info[:preview_exist]=File.exist?(gen_info[:dst])
|
241
240
|
gen_info[:preview_newer_than_original] = (gen_info[:preview_exist] and (File.mtime(gen_info[:dst])>original_mtime))
|
242
241
|
end
|
242
|
+
return local_entry_preview_dir
|
243
243
|
end
|
244
244
|
|
245
|
-
def get_infos_remote(gen_infos,entry
|
245
|
+
def get_infos_remote(gen_infos,entry)
|
246
246
|
#Log.log.debug(">>>> get_infos_remote #{entry}".red)
|
247
247
|
# store source directly here
|
248
248
|
local_original_filepath=File.join(@tmp_folder,entry['name'])
|
249
249
|
#original_mtime=DateTime.parse(entry['modified_time'])
|
250
250
|
# out: where previews are generated
|
251
|
-
local_entry_preview_dir
|
251
|
+
local_entry_preview_dir=File.join(@tmp_folder,entry_preview_folder_name(entry))
|
252
252
|
file_info=@api_node.read("files/#{entry['id']}")[:data]
|
253
253
|
#TODO: this does not work because previews is hidden in api (gen4)
|
254
|
-
#this_preview_folder_entries=get_folder_entries(@previews_folder_entry['id'],{:
|
254
|
+
#this_preview_folder_entries=get_folder_entries(@previews_folder_entry['id'],{name: @entry_preview_folder_name})
|
255
255
|
# TODO: use gen3 api to list files and get date
|
256
256
|
gen_infos.each do |gen_info|
|
257
257
|
gen_info[:src]=local_original_filepath
|
@@ -261,6 +261,7 @@ module Aspera
|
|
261
261
|
# TODO: get change time and compare, useful ?
|
262
262
|
gen_info[:preview_newer_than_original] = gen_info[:preview_exist]
|
263
263
|
end
|
264
|
+
return local_entry_preview_dir
|
264
265
|
end
|
265
266
|
|
266
267
|
# defined by node api
|
@@ -270,29 +271,23 @@ module Aspera
|
|
270
271
|
|
271
272
|
def preview_filename(preview_format,filename=nil)
|
272
273
|
filename||=PREVIEW_BASENAME
|
273
|
-
return "#{filename}.#{preview_format
|
274
|
+
return "#{filename}.#{preview_format}"
|
274
275
|
end
|
275
276
|
|
276
277
|
# generate preview files for one folder entry (file) if necessary
|
277
278
|
# entry must contain "parent_file_id" if remote.
|
278
279
|
def generate_preview(entry)
|
279
|
-
#Log.log.debug(">>>> #{entry}".red)
|
280
|
-
# folder where previews will be generated for this particular entry
|
281
|
-
local_entry_preview_dir=String.new
|
282
280
|
# prepare generic information
|
283
281
|
gen_infos=@preview_formats_to_generate.map do |preview_format|
|
284
282
|
{
|
285
|
-
:preview_format
|
286
|
-
:
|
283
|
+
preview_format: preview_format,
|
284
|
+
base_dest: preview_filename(preview_format)
|
287
285
|
}
|
288
286
|
end
|
289
287
|
# lets gather some infos on possibly existing previews
|
290
288
|
# it depends if files access locally or remotely
|
291
|
-
|
292
|
-
|
293
|
-
else # direct local file system access
|
294
|
-
get_infos_local(gen_infos,entry,local_entry_preview_dir)
|
295
|
-
end
|
289
|
+
# folder where previews will be generated for this particular entry
|
290
|
+
local_entry_preview_dir=@access_remote ? get_infos_remote(gen_infos,entry) : get_infos_local(gen_infos,entry)
|
296
291
|
# here we have the status on preview files
|
297
292
|
# let's find if they need generation
|
298
293
|
gen_infos.select! do |gen_info|
|
@@ -324,7 +319,7 @@ module Aspera
|
|
324
319
|
if @access_remote
|
325
320
|
raise 'missing parent_file_id in entry' if entry['parent_file_id'].nil?
|
326
321
|
# download original file to temp folder
|
327
|
-
do_transfer(
|
322
|
+
do_transfer(Fasp::TransferSpec::DIRECTION_RECEIVE,entry['parent_file_id'],entry['name'],@tmp_folder)
|
328
323
|
end
|
329
324
|
Log.log.info("source: #{entry['id']}: #{entry['path']})")
|
330
325
|
gen_infos.each do |gen_info|
|
@@ -332,7 +327,7 @@ module Aspera
|
|
332
327
|
end
|
333
328
|
if @access_remote
|
334
329
|
# upload
|
335
|
-
do_transfer(
|
330
|
+
do_transfer(Fasp::TransferSpec::DIRECTION_SEND,@previews_folder_entry['id'],local_entry_preview_dir)
|
336
331
|
# cleanup after upload
|
337
332
|
FileUtils.rm_rf(local_entry_preview_dir)
|
338
333
|
File.delete(File.join(@tmp_folder,entry['name']))
|
@@ -341,8 +336,8 @@ module Aspera
|
|
341
336
|
if @option_folder_reset_cache.eql?(:read)
|
342
337
|
@api_node.read("files/#{entry['id']}")
|
343
338
|
end
|
344
|
-
rescue => e
|
345
|
-
Log.log.error("#{e.message}")
|
339
|
+
rescue StandardError => e
|
340
|
+
Log.log.error("Ignore: #{e.message}")
|
346
341
|
Log.log.debug(e.backtrace.join("\n").red)
|
347
342
|
end # generate_preview
|
348
343
|
|
@@ -351,7 +346,7 @@ module Aspera
|
|
351
346
|
def scan_folder_files(top_entry,scan_start=nil)
|
352
347
|
if !scan_start.nil?
|
353
348
|
# canonical path: start with / and ends with /
|
354
|
-
scan_start='/'+scan_start.split('/').
|
349
|
+
scan_start='/'+scan_start.split('/').reject(&:empty?).join('/')
|
355
350
|
scan_start="#{scan_start}/" #unless scan_start.end_with?('/')
|
356
351
|
end
|
357
352
|
filter_block=Aspera::Node.file_matcher(options.get_option(:value,:optional))
|
@@ -364,7 +359,7 @@ module Aspera
|
|
364
359
|
entry_path_with_slash=entry['path']
|
365
360
|
Log.log.info("processing entry #{entry_path_with_slash}") if @periodic.trigger?
|
366
361
|
entry_path_with_slash="#{entry_path_with_slash}/" unless entry_path_with_slash.end_with?('/')
|
367
|
-
if !scan_start.nil?
|
362
|
+
if !scan_start.nil? && !scan_start.start_with?(entry_path_with_slash) && !entry_path_with_slash.start_with?(scan_start)
|
368
363
|
Log.log.debug("#{entry['path']} folder (skip start)".bg_red)
|
369
364
|
next
|
370
365
|
end
|
@@ -399,7 +394,7 @@ module Aspera
|
|
399
394
|
else
|
400
395
|
Log.log.warn("unknown entry type: #{entry['type']}")
|
401
396
|
end
|
402
|
-
rescue => e
|
397
|
+
rescue StandardError => e
|
403
398
|
Log.log.warn("An error occured: #{e}, ignoring")
|
404
399
|
end
|
405
400
|
end
|
@@ -408,7 +403,7 @@ module Aspera
|
|
408
403
|
ACTIONS=[:scan,:events,:trevents,:check,:test]
|
409
404
|
|
410
405
|
def execute_action
|
411
|
-
command=
|
406
|
+
command=options.get_next_command(ACTIONS)
|
412
407
|
unless [:check,:test].include?(command)
|
413
408
|
# this will use node api
|
414
409
|
@api_node=basic_auth_api
|
@@ -426,10 +421,10 @@ module Aspera
|
|
426
421
|
@option_skip_folders.push('/'+@option_previews_folder)
|
427
422
|
if @access_remote
|
428
423
|
# note the filter "name", it's why we take the first one
|
429
|
-
@previews_folder_entry=get_folder_entries(@access_key_self['root_file_id'],{:
|
424
|
+
@previews_folder_entry=get_folder_entries(@access_key_self['root_file_id'],{name: @option_previews_folder}).first
|
430
425
|
raise CliError,"Folder #{@option_previews_folder} does not exist on node. Please create it in the storage root, or specify an alternate name." if @previews_folder_entry.nil?
|
431
426
|
else
|
432
|
-
raise
|
427
|
+
raise 'only local storage allowed in this mode' unless @access_key_self['storage']['type'].eql?('local')
|
433
428
|
@local_storage_root=@access_key_self['storage']['path']
|
434
429
|
#TODO: option to override @local_storage_root='xxx'
|
435
430
|
@local_storage_root=@local_storage_root[LOCAL_STORAGE_PCVL.length..-1] if @local_storage_root.start_with?(LOCAL_STORAGE_PCVL)
|
@@ -449,13 +444,14 @@ module Aspera
|
|
449
444
|
end
|
450
445
|
end
|
451
446
|
end
|
452
|
-
Aspera::Preview::FileTypes.instance.use_mimemagic =
|
447
|
+
Aspera::Preview::FileTypes.instance.use_mimemagic = options.get_option(:mimemagic,:mandatory)
|
453
448
|
case command
|
454
449
|
when :scan
|
455
|
-
scan_path=
|
456
|
-
scan_id=
|
450
|
+
scan_path=options.get_option(:scan_path,:optional)
|
451
|
+
scan_id=options.get_option(:scan_id,:optional)
|
457
452
|
# by default start at root
|
458
|
-
folder_info=
|
453
|
+
folder_info=
|
454
|
+
if scan_id.nil?
|
459
455
|
{ 'id' => @access_key_self['root_file_id'],
|
460
456
|
'name' => '/',
|
461
457
|
'type' => 'folder',
|
@@ -467,11 +463,11 @@ module Aspera
|
|
467
463
|
return Main.result_status('scan finished')
|
468
464
|
when :events,:trevents
|
469
465
|
iteration_persistency=nil
|
470
|
-
if
|
466
|
+
if options.get_option(:once_only,:mandatory)
|
471
467
|
iteration_persistency=PersistencyActionOnce.new(
|
472
468
|
manager: @agents[:persistency],
|
473
469
|
data: [],
|
474
|
-
id: IdGenerator.from_list(['preview_iteration',command.to_s,
|
470
|
+
id: IdGenerator.from_list(['preview_iteration',command.to_s,options.get_option(:url,:mandatory),options.get_option(:username,:mandatory)]))
|
475
471
|
end
|
476
472
|
# call processing method specified by command line command
|
477
473
|
send("process_#{command}",iteration_persistency)
|
@@ -480,16 +476,16 @@ module Aspera
|
|
480
476
|
Aspera::Preview::Utils.check_tools(@skip_types)
|
481
477
|
return Main.result_status('tools validated')
|
482
478
|
when :test
|
483
|
-
format =
|
484
|
-
source =
|
485
|
-
dest=preview_filename(format,
|
479
|
+
format = options.get_next_argument('format',Aspera::Preview::Generator::PREVIEW_FORMATS)
|
480
|
+
source = options.get_next_argument('source file')
|
481
|
+
dest=preview_filename(format,options.get_option(:case,:optional))
|
486
482
|
g=Aspera::Preview::Generator.new(@gen_options,source,dest,@tmp_folder,nil)
|
487
483
|
raise "cannot find file type for #{source}" if g.conversion_type.nil?
|
488
484
|
raise "out format #{format} not supported" unless g.supported?
|
489
485
|
g.generate
|
490
486
|
return Main.result_status("generated: #{dest}")
|
491
487
|
else
|
492
|
-
raise
|
488
|
+
raise 'error'
|
493
489
|
end
|
494
490
|
ensure
|
495
491
|
FileUtils.rm_rf(@tmp_folder)
|