aspera-cli 4.4.0 → 4.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2095 -1503
- data/bin/ascli +2 -1
- data/bin/asession +4 -5
- data/docs/test_env.conf +3 -0
- data/examples/aoc.rb +4 -3
- data/examples/faspex4.rb +25 -25
- data/examples/proxy.pac +1 -1
- data/examples/transfer.rb +17 -17
- data/lib/aspera/aoc.rb +238 -185
- data/lib/aspera/ascmd.rb +93 -83
- data/lib/aspera/ats_api.rb +11 -10
- data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
- data/lib/aspera/cli/extended_value.rb +42 -33
- data/lib/aspera/cli/formater.rb +142 -108
- data/lib/aspera/cli/info.rb +17 -0
- data/lib/aspera/cli/listener/line_dump.rb +3 -2
- data/lib/aspera/cli/listener/logger.rb +2 -1
- data/lib/aspera/cli/listener/progress.rb +16 -18
- data/lib/aspera/cli/listener/progress_multi.rb +18 -21
- data/lib/aspera/cli/main.rb +173 -149
- data/lib/aspera/cli/manager.rb +163 -168
- data/lib/aspera/cli/plugin.rb +43 -31
- data/lib/aspera/cli/plugins/alee.rb +6 -6
- data/lib/aspera/cli/plugins/aoc.rb +405 -370
- data/lib/aspera/cli/plugins/ats.rb +86 -79
- data/lib/aspera/cli/plugins/bss.rb +14 -16
- data/lib/aspera/cli/plugins/config.rb +580 -362
- data/lib/aspera/cli/plugins/console.rb +23 -19
- data/lib/aspera/cli/plugins/cos.rb +18 -18
- data/lib/aspera/cli/plugins/faspex.rb +201 -158
- data/lib/aspera/cli/plugins/faspex5.rb +80 -57
- data/lib/aspera/cli/plugins/node.rb +183 -166
- data/lib/aspera/cli/plugins/orchestrator.rb +71 -67
- data/lib/aspera/cli/plugins/preview.rb +92 -96
- data/lib/aspera/cli/plugins/server.rb +79 -75
- data/lib/aspera/cli/plugins/shares.rb +35 -19
- data/lib/aspera/cli/plugins/sync.rb +20 -22
- data/lib/aspera/cli/transfer_agent.rb +76 -113
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +35 -27
- data/lib/aspera/command_line_builder.rb +48 -34
- data/lib/aspera/cos_node.rb +29 -21
- data/lib/aspera/data_repository.rb +3 -2
- data/lib/aspera/environment.rb +50 -45
- data/lib/aspera/fasp/{manager.rb → agent_base.rb} +28 -25
- data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +52 -43
- data/lib/aspera/fasp/{local.rb → agent_direct.rb} +58 -72
- data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +37 -43
- data/lib/aspera/fasp/{node.rb → agent_node.rb} +35 -16
- data/lib/aspera/fasp/agent_trsdk.rb +104 -0
- data/lib/aspera/fasp/error.rb +2 -1
- data/lib/aspera/fasp/error_info.rb +68 -52
- data/lib/aspera/fasp/installation.rb +152 -124
- data/lib/aspera/fasp/listener.rb +1 -0
- data/lib/aspera/fasp/parameters.rb +87 -92
- data/lib/aspera/fasp/parameters.yaml +305 -249
- data/lib/aspera/fasp/resume_policy.rb +11 -14
- data/lib/aspera/fasp/transfer_spec.rb +26 -0
- data/lib/aspera/fasp/uri.rb +22 -21
- data/lib/aspera/faspex_gw.rb +55 -89
- data/lib/aspera/hash_ext.rb +4 -3
- data/lib/aspera/id_generator.rb +8 -7
- data/lib/aspera/keychain/encrypted_hash.rb +121 -0
- data/lib/aspera/keychain/macos_security.rb +90 -0
- data/lib/aspera/log.rb +55 -37
- data/lib/aspera/nagios.rb +13 -12
- data/lib/aspera/node.rb +30 -25
- data/lib/aspera/oauth.rb +175 -226
- data/lib/aspera/open_application.rb +4 -3
- data/lib/aspera/persistency_action_once.rb +6 -6
- data/lib/aspera/persistency_folder.rb +5 -9
- data/lib/aspera/preview/file_types.rb +6 -5
- data/lib/aspera/preview/generator.rb +25 -24
- data/lib/aspera/preview/options.rb +16 -14
- data/lib/aspera/preview/utils.rb +98 -98
- data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
- data/lib/aspera/proxy_auto_config.rb +111 -20
- data/lib/aspera/rest.rb +154 -135
- data/lib/aspera/rest_call_error.rb +2 -2
- data/lib/aspera/rest_error_analyzer.rb +23 -25
- data/lib/aspera/rest_errors_aspera.rb +15 -14
- data/lib/aspera/ssh.rb +12 -10
- data/lib/aspera/sync.rb +42 -41
- data/lib/aspera/temp_file_manager.rb +18 -14
- data/lib/aspera/timer_limiter.rb +2 -1
- data/lib/aspera/uri_reader.rb +7 -5
- data/lib/aspera/web_auth.rb +79 -76
- metadata +116 -29
- data/docs/Makefile +0 -66
- data/docs/README.erb.md +0 -3973
- data/docs/README.md +0 -13
- data/docs/diagrams.txt +0 -49
- data/docs/doc_tools.rb +0 -58
- data/lib/aspera/api_detector.rb +0 -60
- data/lib/aspera/cli/plugins/shares2.rb +0 -114
- data/lib/aspera/secrets.rb +0 -20
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
require 'aspera/cli/plugins/node'
|
|
2
3
|
require 'xmlsimple'
|
|
3
4
|
|
|
@@ -7,47 +8,47 @@ module Aspera
|
|
|
7
8
|
class Orchestrator < BasicAuthPlugin
|
|
8
9
|
def initialize(env)
|
|
9
10
|
super(env)
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
11
|
+
options.add_opt_simple(:params,'parameters hash table, use @json:{"param":"value"}')
|
|
12
|
+
options.add_opt_simple(:result,"specify result value as: 'work step:parameter'")
|
|
13
|
+
options.add_opt_boolean(:synchronous,'work step:parameter expected as result')
|
|
14
|
+
options.add_opt_list(:ret_style,[:header,:arg,:ext],'how return type is requested in api')
|
|
15
|
+
options.add_opt_list(:auth_style,[:arg_pass,:head_basic,:apikey],'authentication type')
|
|
16
|
+
options.set_option(:params,{})
|
|
17
|
+
options.set_option(:synchronous,:no)
|
|
18
|
+
options.set_option(:ret_style,:arg)
|
|
19
|
+
options.set_option(:auth_style,:head_basic)
|
|
20
|
+
options.parse_options!
|
|
20
21
|
end
|
|
21
22
|
|
|
22
|
-
ACTIONS=[:info, :workflow, :plugins, :processes]
|
|
23
|
+
ACTIONS=[:info, :workflow, :plugins, :processes].freeze
|
|
23
24
|
|
|
24
25
|
# for JSON format: add extension ".json" or add url parameter: format=json or Accept: application/json
|
|
25
26
|
# id can be: a parameter id=x, or at the end of url /id, for workflows: work_order[workflow_id]=wf_id
|
|
26
|
-
def call_API_orig(endpoint,id=nil,url_params={:
|
|
27
|
-
# calls are GET
|
|
28
|
-
call_args={:
|
|
29
|
-
# specify id if necessary
|
|
30
|
-
call_args[:subpath]=call_args[:subpath]+'/'+id unless id.nil?
|
|
31
|
-
unless url_params.nil?
|
|
32
|
-
if url_params.has_key?(:format)
|
|
33
|
-
call_args[:headers]={'Accept'=>'application/'+url_params[:format].to_s}
|
|
34
|
-
end
|
|
35
|
-
call_args[:headers]={'Accept'=>accept} unless accept.nil?
|
|
36
|
-
# add params if necessary
|
|
37
|
-
call_args[:url_params]=url_params
|
|
38
|
-
end
|
|
39
|
-
return @api_orch.call(call_args)
|
|
40
|
-
end
|
|
27
|
+
# def call_API_orig(endpoint,id=nil,url_params={format: :json},accept=nil)
|
|
28
|
+
# # calls are GET
|
|
29
|
+
# call_args={operation: 'GET',subpath: endpoint}
|
|
30
|
+
# # specify id if necessary
|
|
31
|
+
# call_args[:subpath]=call_args[:subpath]+'/'+id unless id.nil?
|
|
32
|
+
# unless url_params.nil?
|
|
33
|
+
# if url_params.has_key?(:format)
|
|
34
|
+
# call_args[:headers]={'Accept'=>'application/'+url_params[:format].to_s}
|
|
35
|
+
# end
|
|
36
|
+
# call_args[:headers]={'Accept'=>accept} unless accept.nil?
|
|
37
|
+
# # add params if necessary
|
|
38
|
+
# call_args[:url_params]=url_params
|
|
39
|
+
# end
|
|
40
|
+
# return @api_orch.call(call_args)
|
|
41
|
+
# end
|
|
41
42
|
|
|
42
|
-
def
|
|
43
|
+
def call_ao(endpoint,opt={})
|
|
43
44
|
opt[:prefix]='api' unless opt.has_key?(:prefix)
|
|
44
45
|
# calls are GET
|
|
45
|
-
call_args={:
|
|
46
|
+
call_args={operation: 'GET',subpath: endpoint}
|
|
46
47
|
# specify prefix if necessary
|
|
47
48
|
call_args[:subpath]="#{opt[:prefix]}/#{call_args[:subpath]}" unless opt[:prefix].nil?
|
|
48
49
|
# specify id if necessary
|
|
49
50
|
call_args[:subpath]="#{call_args[:subpath]}/#{opt[:id]}" if opt.has_key?(:id)
|
|
50
|
-
call_type=
|
|
51
|
+
call_type=options.get_option(:ret_style,:mandatory)
|
|
51
52
|
call_type=opt[:ret_style] if opt.has_key?(:ret_style)
|
|
52
53
|
format='json'
|
|
53
54
|
format=opt[:format] if opt.has_key?(:format)
|
|
@@ -61,80 +62,83 @@ module Aspera
|
|
|
61
62
|
call_args[:url_params][:format]=format
|
|
62
63
|
when :ext
|
|
63
64
|
call_args[:subpath]="#{call_args[:subpath]}.#{format}"
|
|
64
|
-
else raise
|
|
65
|
+
else raise 'unexpected'
|
|
65
66
|
end
|
|
66
67
|
end
|
|
67
68
|
result=@api_orch.call(call_args)
|
|
68
|
-
result[:data]=XmlSimple.xml_in(result[:http].body, opt[:xml_opt]||{
|
|
69
|
+
result[:data]=XmlSimple.xml_in(result[:http].body, opt[:xml_opt]||{'ForceArray' => true}) if format.eql?('xml')
|
|
69
70
|
return result
|
|
70
71
|
end
|
|
71
72
|
|
|
72
73
|
def execute_action
|
|
73
|
-
rest_params={:
|
|
74
|
-
case
|
|
74
|
+
rest_params={base_url: options.get_option(:url,:mandatory)}
|
|
75
|
+
case options.get_option(:auth_style,:mandatory)
|
|
75
76
|
when :arg_pass
|
|
76
77
|
rest_params[:auth]={
|
|
77
|
-
|
|
78
|
-
:
|
|
79
|
-
'login' =>
|
|
80
|
-
'password' =>
|
|
78
|
+
type: :url,
|
|
79
|
+
url_creds: {
|
|
80
|
+
'login' =>options.get_option(:username,:mandatory),
|
|
81
|
+
'password' =>options.get_option(:password,:mandatory) }}
|
|
81
82
|
when :head_basic
|
|
82
83
|
rest_params[:auth]={
|
|
83
|
-
|
|
84
|
-
:
|
|
85
|
-
:
|
|
84
|
+
type: :basic,
|
|
85
|
+
username: options.get_option(:username,:mandatory),
|
|
86
|
+
password: options.get_option(:password,:mandatory) }
|
|
86
87
|
when :apikey
|
|
87
|
-
raise
|
|
88
|
+
raise 'Not implemented'
|
|
88
89
|
end
|
|
89
90
|
|
|
90
91
|
@api_orch=Rest.new(rest_params)
|
|
91
92
|
|
|
92
|
-
command1=
|
|
93
|
+
command1=options.get_next_command(ACTIONS)
|
|
93
94
|
case command1
|
|
94
95
|
when :info
|
|
95
|
-
result=
|
|
96
|
-
return {:
|
|
97
|
-
# result=
|
|
96
|
+
result=call_ao('remote_node_ping',format: 'xml', xml_opt: {'ForceArray' => false})
|
|
97
|
+
return {type: :single_object,data: result[:data]}
|
|
98
|
+
# result=call_ao('workflows',prefix: nil,format: nil)
|
|
98
99
|
# version='unknown'
|
|
99
100
|
# if m=result[:http].body.match(/\(Orchestrator v([1-9]+\.[\.0-9a-f\-]+)\)/)
|
|
100
101
|
# version=m[1]
|
|
101
102
|
# end
|
|
102
|
-
# return {:
|
|
103
|
+
# return {type: :single_object,data: {'version'=>version}}
|
|
103
104
|
when :processes
|
|
104
105
|
# TODO: Jira ? API has only XML format
|
|
105
|
-
result=
|
|
106
|
-
return {:
|
|
106
|
+
result=call_ao('processes_status',format: 'xml')
|
|
107
|
+
return {type: :object_list,data: result[:data]['process']}
|
|
107
108
|
when :plugins
|
|
108
109
|
# TODO: Jira ? only json format on url
|
|
109
|
-
result=
|
|
110
|
-
return {:
|
|
110
|
+
result=call_ao('plugin_version')[:data]
|
|
111
|
+
return {type: :object_list,data: result['Plugin']}
|
|
111
112
|
when :workflow
|
|
112
|
-
command=
|
|
113
|
-
unless [:list
|
|
114
|
-
wf_id=
|
|
113
|
+
command=options.get_next_command([:list, :status, :inputs, :details, :start, :export])
|
|
114
|
+
unless [:list].include?(command)
|
|
115
|
+
wf_id=instance_identifier()
|
|
115
116
|
end
|
|
116
117
|
case command
|
|
117
118
|
when :status
|
|
118
|
-
|
|
119
|
-
|
|
119
|
+
options={}
|
|
120
|
+
options[:id]=wf_id unless wf_id.eql?('ALL')
|
|
121
|
+
result=call_ao('workflows_status',options)[:data]
|
|
122
|
+
return {type: :object_list,data: result['workflows']['workflow']}
|
|
120
123
|
when :list
|
|
121
|
-
result=
|
|
122
|
-
return {:
|
|
124
|
+
result=call_ao('workflows_list',id: 0)[:data]
|
|
125
|
+
return {type: :object_list,data: result['workflows']['workflow'],
|
|
126
|
+
fields: ['id','portable_id','name','published_status','published_revision_id','latest_revision_id','last_modification']}
|
|
123
127
|
when :details
|
|
124
|
-
result=
|
|
125
|
-
return {:
|
|
128
|
+
result=call_ao('workflow_details',id: wf_id)[:data]
|
|
129
|
+
return {type: :object_list,data: result['workflows']['workflow']['statuses']}
|
|
126
130
|
when :inputs
|
|
127
|
-
result=
|
|
128
|
-
return {:
|
|
131
|
+
result=call_ao('workflow_inputs_spec',id: wf_id)[:data]
|
|
132
|
+
return {type: :single_object,data: result['workflow_inputs_spec']}
|
|
129
133
|
when :export
|
|
130
|
-
result=
|
|
131
|
-
return {:
|
|
134
|
+
result=call_ao('export_workflow',id: wf_id,format: nil)[:http]
|
|
135
|
+
return {type: :text,data: result.body}
|
|
132
136
|
when :start
|
|
133
137
|
result={
|
|
134
|
-
:
|
|
135
|
-
:
|
|
138
|
+
type: :single_object,
|
|
139
|
+
data: nil
|
|
136
140
|
}
|
|
137
|
-
call_params={:
|
|
141
|
+
call_params={format: :json}
|
|
138
142
|
override_accept=nil
|
|
139
143
|
# set external parameters if any
|
|
140
144
|
self.options.get_option(:params,:mandatory).each do |name,value|
|
|
@@ -157,7 +161,7 @@ module Aspera
|
|
|
157
161
|
result[:type]=:text
|
|
158
162
|
override_accept='text/plain'
|
|
159
163
|
end
|
|
160
|
-
result[:data]=
|
|
164
|
+
result[:data]=call_ao('initiate',id: wf_id,args: call_params,accept: override_accept)[:data]
|
|
161
165
|
return result
|
|
162
166
|
end # wf command
|
|
163
167
|
else raise "ERROR, unknown command: [#{command}]"
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
require 'aspera/cli/basic_auth_plugin'
|
|
2
3
|
require 'aspera/preview/generator'
|
|
3
4
|
require 'aspera/preview/options'
|
|
4
5
|
require 'aspera/preview/utils'
|
|
5
6
|
require 'aspera/preview/file_types'
|
|
7
|
+
require 'aspera/fasp/transfer_spec'
|
|
6
8
|
require 'aspera/persistency_action_once'
|
|
7
9
|
require 'aspera/node'
|
|
8
10
|
require 'aspera/hash_ext'
|
|
@@ -31,10 +33,7 @@ module Aspera
|
|
|
31
33
|
|
|
32
34
|
# option_skip_format has special accessors
|
|
33
35
|
attr_accessor :option_previews_folder
|
|
34
|
-
attr_accessor :option_folder_reset_cache
|
|
35
|
-
attr_accessor :option_skip_folders
|
|
36
|
-
attr_accessor :option_overwrite
|
|
37
|
-
attr_accessor :option_file_access
|
|
36
|
+
attr_accessor :option_folder_reset_cache, :option_skip_folders, :option_overwrite, :option_file_access
|
|
38
37
|
def initialize(env)
|
|
39
38
|
super(env)
|
|
40
39
|
@skip_types=[]
|
|
@@ -46,43 +45,43 @@ module Aspera
|
|
|
46
45
|
# used to trigger periodic processing
|
|
47
46
|
@periodic=TimerLimiter.new(LOG_LIMITER_SEC)
|
|
48
47
|
# link CLI options to gen_info attributes
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
48
|
+
options.set_obj_attr(:skip_format,self,:option_skip_format,[]) # no skip
|
|
49
|
+
options.set_obj_attr(:folder_reset_cache,self,:option_folder_reset_cache,:no)
|
|
50
|
+
options.set_obj_attr(:skip_types,self,:option_skip_types)
|
|
51
|
+
options.set_obj_attr(:previews_folder,self,:option_previews_folder,DEFAULT_PREVIEWS_FOLDER)
|
|
52
|
+
options.set_obj_attr(:skip_folders,self,:option_skip_folders,[]) # no skip
|
|
53
|
+
options.set_obj_attr(:overwrite,self,:option_overwrite,:mtime)
|
|
54
|
+
options.set_obj_attr(:file_access,self,:option_file_access,:local)
|
|
55
|
+
options.add_opt_list(:skip_format,Aspera::Preview::Generator::PREVIEW_FORMATS,'skip this preview format (multiple possible)')
|
|
56
|
+
options.add_opt_list(:folder_reset_cache,[:no,:header,:read],'force detection of generated preview by refresh cache')
|
|
57
|
+
options.add_opt_simple(:skip_types,'skip types in comma separated list')
|
|
58
|
+
options.add_opt_simple(:previews_folder,'preview folder in storage root')
|
|
59
|
+
options.add_opt_simple(:temp_folder,'path to temp folder')
|
|
60
|
+
options.add_opt_simple(:skip_folders,'list of folder to skip')
|
|
61
|
+
options.add_opt_simple(:case,'basename of output for for test')
|
|
62
|
+
options.add_opt_simple(:scan_path,'subpath in folder id to start scan in (default=/)')
|
|
63
|
+
options.add_opt_simple(:scan_id,'forder id in storage to start scan in, default is access key main folder id')
|
|
64
|
+
options.add_opt_boolean(:mimemagic,'use Mime type detection of gem mimemagic')
|
|
65
|
+
options.add_opt_list(:overwrite,[:always,:never,:mtime],'when to overwrite result file')
|
|
66
|
+
options.add_opt_list(:file_access,[:local,:remote],'how to read and write files in repository')
|
|
67
|
+
options.set_option(:temp_folder,Dir.tmpdir)
|
|
68
|
+
options.set_option(:mimemagic,false)
|
|
70
69
|
|
|
71
70
|
# add other options for generator (and set default values)
|
|
72
71
|
Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
|
|
73
|
-
|
|
72
|
+
options.set_obj_attr(opt[:name],@gen_options,opt[:name],opt[:default])
|
|
74
73
|
if opt.has_key?(:values)
|
|
75
|
-
|
|
74
|
+
options.add_opt_list(opt[:name],opt[:values],opt[:description])
|
|
76
75
|
elsif [:yes,:no].include?(opt[:default])
|
|
77
|
-
|
|
76
|
+
options.add_opt_boolean(opt[:name],opt[:description])
|
|
78
77
|
else
|
|
79
|
-
|
|
78
|
+
options.add_opt_simple(opt[:name],opt[:description])
|
|
80
79
|
end
|
|
81
80
|
end
|
|
82
81
|
|
|
83
|
-
|
|
82
|
+
options.parse_options!
|
|
84
83
|
raise 'skip_folder shall be an Array, use @json:[...]' unless @option_skip_folders.is_a?(Array)
|
|
85
|
-
@tmp_folder=File.join(
|
|
84
|
+
@tmp_folder=File.join(options.get_option(:temp_folder,:mandatory),"#{TMP_DIR_PREFIX}.#{SecureRandom.uuid}")
|
|
86
85
|
FileUtils.mkdir_p(@tmp_folder)
|
|
87
86
|
Log.log.debug("tmpdir: #{@tmp_folder}")
|
|
88
87
|
end
|
|
@@ -97,7 +96,7 @@ module Aspera
|
|
|
97
96
|
end
|
|
98
97
|
|
|
99
98
|
def option_skip_types
|
|
100
|
-
return @skip_types.map
|
|
99
|
+
return @skip_types.map(&:to_s).join(',')
|
|
101
100
|
end
|
|
102
101
|
|
|
103
102
|
def option_skip_format=(value)
|
|
@@ -105,7 +104,7 @@ module Aspera
|
|
|
105
104
|
end
|
|
106
105
|
|
|
107
106
|
def option_skip_format
|
|
108
|
-
return @preview_formats_to_generate.map
|
|
107
|
+
return @preview_formats_to_generate.map(&:to_s).join(',')
|
|
109
108
|
end
|
|
110
109
|
|
|
111
110
|
# /files/id/files is normally cached in redis, but we can discard the cache
|
|
@@ -113,7 +112,7 @@ module Aspera
|
|
|
113
112
|
def get_folder_entries(file_id,request_args=nil)
|
|
114
113
|
headers={'Accept'=>'application/json'}
|
|
115
114
|
headers.merge!({'X-Aspera-Cache-Control'=>'no-cache'}) if @option_folder_reset_cache.eql?(:header)
|
|
116
|
-
return @api_node.call({:
|
|
115
|
+
return @api_node.call({operation: 'GET',subpath: "files/#{file_id}/files",headers: headers,url_params: request_args})[:data]
|
|
117
116
|
#return @api_node.read("files/#{file_id}/files",request_args)[:data]
|
|
118
117
|
end
|
|
119
118
|
|
|
@@ -138,9 +137,9 @@ module Aspera
|
|
|
138
137
|
end
|
|
139
138
|
return if events.empty?
|
|
140
139
|
events.each do |event|
|
|
141
|
-
if event['data']['direction'].eql?(
|
|
142
|
-
event['data']['status'].eql?('completed')
|
|
143
|
-
event['data']['error_code'].eql?(0)
|
|
140
|
+
if event['data']['direction'].eql?(Fasp::TransferSpec::DIRECTION_RECEIVE) &&
|
|
141
|
+
event['data']['status'].eql?('completed') &&
|
|
142
|
+
event['data']['error_code'].eql?(0) &&
|
|
144
143
|
event['data'].dig('tags','aspera',PREV_GEN_TAG).nil?
|
|
145
144
|
folder_id=event.dig('data','tags','aspera','node','file_id')
|
|
146
145
|
folder_id||=event.dig('data','file_id')
|
|
@@ -149,13 +148,13 @@ module Aspera
|
|
|
149
148
|
scan_folder_files(folder_entry) unless folder_entry.nil?
|
|
150
149
|
end
|
|
151
150
|
end
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
151
|
+
# log/persist periodically or last one
|
|
152
|
+
next unless @periodic.trigger? || event.equal?(events.last)
|
|
153
|
+
Log.log.info("Processed event #{event['id']}")
|
|
154
|
+
# save checkpoint to avoid losing processing in case of error
|
|
155
|
+
if !iteration_persistency.nil?
|
|
156
|
+
iteration_persistency.data[0]=event['id'].to_s
|
|
157
|
+
iteration_persistency.save
|
|
159
158
|
end
|
|
160
159
|
end
|
|
161
160
|
end
|
|
@@ -175,7 +174,7 @@ module Aspera
|
|
|
175
174
|
# process only files
|
|
176
175
|
if event.dig('data','type').eql?('file')
|
|
177
176
|
file_entry=@api_node.read("files/#{event['data']['id']}")[:data] rescue nil
|
|
178
|
-
if !file_entry.nil?
|
|
177
|
+
if !file_entry.nil? &&
|
|
179
178
|
@option_skip_folders.select{|d|file_entry['path'].start_with?(d)}.empty?
|
|
180
179
|
file_entry['parent_file_id']=event['data']['parent_file_id']
|
|
181
180
|
if event['types'].include?('file.deleted')
|
|
@@ -186,30 +185,30 @@ module Aspera
|
|
|
186
185
|
end
|
|
187
186
|
end
|
|
188
187
|
end
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
188
|
+
# log/persist periodically or last one
|
|
189
|
+
next unless @periodic.trigger? || event.equal?(events.last)
|
|
190
|
+
Log.log.info("Processing event #{event['id']}")
|
|
191
|
+
# save checkpoint to avoid losing processing in case of error
|
|
192
|
+
if !iteration_persistency.nil?
|
|
193
|
+
iteration_persistency.data[0]=event['id'].to_s
|
|
194
|
+
iteration_persistency.save
|
|
196
195
|
end
|
|
197
196
|
end
|
|
198
197
|
end
|
|
199
198
|
|
|
200
199
|
def do_transfer(direction,folder_id,source_filename,destination='/')
|
|
201
|
-
raise
|
|
200
|
+
raise 'error' if destination.nil? && direction.eql?(Fasp::TransferSpec::DIRECTION_RECEIVE)
|
|
202
201
|
if @default_transfer_spec.nil?
|
|
203
202
|
# make a dummy call to get some default transfer parameters
|
|
204
203
|
res=@api_node.create('files/upload_setup',{'transfer_requests'=>[{'transfer_request'=>{'paths'=>[{}],'destination_root'=>'/'}}]})
|
|
205
204
|
template_ts=res[:data]['transfer_specs'].first['transfer_spec']
|
|
206
205
|
# get ports, anyway that should be 33001 for both. add remote_user ?
|
|
207
|
-
@default_transfer_spec=['ssh_port','fasp_port'].
|
|
208
|
-
if
|
|
209
|
-
Log.log.warn(
|
|
210
|
-
@default_transfer_spec['remote_user']=Aspera::
|
|
206
|
+
@default_transfer_spec=['ssh_port','fasp_port'].each_with_object({}){|e,h|h[e]=template_ts[e];}
|
|
207
|
+
if !@default_transfer_spec['remote_user'].eql?(Aspera::Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
|
|
208
|
+
Log.log.warn('remote_user shall be xfer')
|
|
209
|
+
@default_transfer_spec['remote_user']=Aspera::Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER
|
|
211
210
|
end
|
|
212
|
-
Aspera::Node
|
|
211
|
+
Aspera::Node.set_ak_basic_token(@default_transfer_spec,@access_key_self['id'],options.get_option(:password,:mandatory))
|
|
213
212
|
# note: we use the same address for ascp than for node api instead of the one from upload_setup
|
|
214
213
|
# TODO: configurable ? useful ?
|
|
215
214
|
@default_transfer_spec['remote_host']=@transfer_server_address
|
|
@@ -225,33 +224,34 @@ module Aspera
|
|
|
225
224
|
})
|
|
226
225
|
# force destination
|
|
227
226
|
# tspec['destination_root']=destination
|
|
228
|
-
|
|
229
|
-
Main.result_transfer(
|
|
227
|
+
transfer.option_transfer_spec_deep_merge({'destination_root'=>destination})
|
|
228
|
+
Main.result_transfer(transfer.start(tspec,{src: :node_gen4}))
|
|
230
229
|
end
|
|
231
230
|
|
|
232
|
-
def get_infos_local(gen_infos,entry
|
|
231
|
+
def get_infos_local(gen_infos,entry)
|
|
233
232
|
local_original_filepath=File.join(@local_storage_root,entry['path'])
|
|
234
233
|
original_mtime=File.mtime(local_original_filepath)
|
|
235
234
|
# out
|
|
236
|
-
local_entry_preview_dir
|
|
235
|
+
local_entry_preview_dir=File.join(@local_preview_folder, entry_preview_folder_name(entry))
|
|
237
236
|
gen_infos.each do |gen_info|
|
|
238
237
|
gen_info[:src]=local_original_filepath
|
|
239
238
|
gen_info[:dst]=File.join(local_entry_preview_dir, gen_info[:base_dest])
|
|
240
239
|
gen_info[:preview_exist]=File.exist?(gen_info[:dst])
|
|
241
240
|
gen_info[:preview_newer_than_original] = (gen_info[:preview_exist] and (File.mtime(gen_info[:dst])>original_mtime))
|
|
242
241
|
end
|
|
242
|
+
return local_entry_preview_dir
|
|
243
243
|
end
|
|
244
244
|
|
|
245
|
-
def get_infos_remote(gen_infos,entry
|
|
245
|
+
def get_infos_remote(gen_infos,entry)
|
|
246
246
|
#Log.log.debug(">>>> get_infos_remote #{entry}".red)
|
|
247
247
|
# store source directly here
|
|
248
248
|
local_original_filepath=File.join(@tmp_folder,entry['name'])
|
|
249
249
|
#original_mtime=DateTime.parse(entry['modified_time'])
|
|
250
250
|
# out: where previews are generated
|
|
251
|
-
local_entry_preview_dir
|
|
251
|
+
local_entry_preview_dir=File.join(@tmp_folder,entry_preview_folder_name(entry))
|
|
252
252
|
file_info=@api_node.read("files/#{entry['id']}")[:data]
|
|
253
253
|
#TODO: this does not work because previews is hidden in api (gen4)
|
|
254
|
-
#this_preview_folder_entries=get_folder_entries(@previews_folder_entry['id'],{:
|
|
254
|
+
#this_preview_folder_entries=get_folder_entries(@previews_folder_entry['id'],{name: @entry_preview_folder_name})
|
|
255
255
|
# TODO: use gen3 api to list files and get date
|
|
256
256
|
gen_infos.each do |gen_info|
|
|
257
257
|
gen_info[:src]=local_original_filepath
|
|
@@ -261,6 +261,7 @@ module Aspera
|
|
|
261
261
|
# TODO: get change time and compare, useful ?
|
|
262
262
|
gen_info[:preview_newer_than_original] = gen_info[:preview_exist]
|
|
263
263
|
end
|
|
264
|
+
return local_entry_preview_dir
|
|
264
265
|
end
|
|
265
266
|
|
|
266
267
|
# defined by node api
|
|
@@ -270,29 +271,23 @@ module Aspera
|
|
|
270
271
|
|
|
271
272
|
def preview_filename(preview_format,filename=nil)
|
|
272
273
|
filename||=PREVIEW_BASENAME
|
|
273
|
-
return "#{filename}.#{preview_format
|
|
274
|
+
return "#{filename}.#{preview_format}"
|
|
274
275
|
end
|
|
275
276
|
|
|
276
277
|
# generate preview files for one folder entry (file) if necessary
|
|
277
278
|
# entry must contain "parent_file_id" if remote.
|
|
278
279
|
def generate_preview(entry)
|
|
279
|
-
#Log.log.debug(">>>> #{entry}".red)
|
|
280
|
-
# folder where previews will be generated for this particular entry
|
|
281
|
-
local_entry_preview_dir=String.new
|
|
282
280
|
# prepare generic information
|
|
283
281
|
gen_infos=@preview_formats_to_generate.map do |preview_format|
|
|
284
282
|
{
|
|
285
|
-
:preview_format
|
|
286
|
-
:
|
|
283
|
+
preview_format: preview_format,
|
|
284
|
+
base_dest: preview_filename(preview_format)
|
|
287
285
|
}
|
|
288
286
|
end
|
|
289
287
|
# lets gather some infos on possibly existing previews
|
|
290
288
|
# it depends if files access locally or remotely
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
else # direct local file system access
|
|
294
|
-
get_infos_local(gen_infos,entry,local_entry_preview_dir)
|
|
295
|
-
end
|
|
289
|
+
# folder where previews will be generated for this particular entry
|
|
290
|
+
local_entry_preview_dir=@access_remote ? get_infos_remote(gen_infos,entry) : get_infos_local(gen_infos,entry)
|
|
296
291
|
# here we have the status on preview files
|
|
297
292
|
# let's find if they need generation
|
|
298
293
|
gen_infos.select! do |gen_info|
|
|
@@ -324,7 +319,7 @@ module Aspera
|
|
|
324
319
|
if @access_remote
|
|
325
320
|
raise 'missing parent_file_id in entry' if entry['parent_file_id'].nil?
|
|
326
321
|
# download original file to temp folder
|
|
327
|
-
do_transfer(
|
|
322
|
+
do_transfer(Fasp::TransferSpec::DIRECTION_RECEIVE,entry['parent_file_id'],entry['name'],@tmp_folder)
|
|
328
323
|
end
|
|
329
324
|
Log.log.info("source: #{entry['id']}: #{entry['path']})")
|
|
330
325
|
gen_infos.each do |gen_info|
|
|
@@ -332,7 +327,7 @@ module Aspera
|
|
|
332
327
|
end
|
|
333
328
|
if @access_remote
|
|
334
329
|
# upload
|
|
335
|
-
do_transfer(
|
|
330
|
+
do_transfer(Fasp::TransferSpec::DIRECTION_SEND,@previews_folder_entry['id'],local_entry_preview_dir)
|
|
336
331
|
# cleanup after upload
|
|
337
332
|
FileUtils.rm_rf(local_entry_preview_dir)
|
|
338
333
|
File.delete(File.join(@tmp_folder,entry['name']))
|
|
@@ -341,8 +336,8 @@ module Aspera
|
|
|
341
336
|
if @option_folder_reset_cache.eql?(:read)
|
|
342
337
|
@api_node.read("files/#{entry['id']}")
|
|
343
338
|
end
|
|
344
|
-
rescue => e
|
|
345
|
-
Log.log.error("#{e.message}")
|
|
339
|
+
rescue StandardError => e
|
|
340
|
+
Log.log.error("Ignore: #{e.message}")
|
|
346
341
|
Log.log.debug(e.backtrace.join("\n").red)
|
|
347
342
|
end # generate_preview
|
|
348
343
|
|
|
@@ -351,7 +346,7 @@ module Aspera
|
|
|
351
346
|
def scan_folder_files(top_entry,scan_start=nil)
|
|
352
347
|
if !scan_start.nil?
|
|
353
348
|
# canonical path: start with / and ends with /
|
|
354
|
-
scan_start='/'+scan_start.split('/').
|
|
349
|
+
scan_start='/'+scan_start.split('/').reject(&:empty?).join('/')
|
|
355
350
|
scan_start="#{scan_start}/" #unless scan_start.end_with?('/')
|
|
356
351
|
end
|
|
357
352
|
filter_block=Aspera::Node.file_matcher(options.get_option(:value,:optional))
|
|
@@ -364,7 +359,7 @@ module Aspera
|
|
|
364
359
|
entry_path_with_slash=entry['path']
|
|
365
360
|
Log.log.info("processing entry #{entry_path_with_slash}") if @periodic.trigger?
|
|
366
361
|
entry_path_with_slash="#{entry_path_with_slash}/" unless entry_path_with_slash.end_with?('/')
|
|
367
|
-
if !scan_start.nil?
|
|
362
|
+
if !scan_start.nil? && !scan_start.start_with?(entry_path_with_slash) && !entry_path_with_slash.start_with?(scan_start)
|
|
368
363
|
Log.log.debug("#{entry['path']} folder (skip start)".bg_red)
|
|
369
364
|
next
|
|
370
365
|
end
|
|
@@ -399,7 +394,7 @@ module Aspera
|
|
|
399
394
|
else
|
|
400
395
|
Log.log.warn("unknown entry type: #{entry['type']}")
|
|
401
396
|
end
|
|
402
|
-
rescue => e
|
|
397
|
+
rescue StandardError => e
|
|
403
398
|
Log.log.warn("An error occured: #{e}, ignoring")
|
|
404
399
|
end
|
|
405
400
|
end
|
|
@@ -408,7 +403,7 @@ module Aspera
|
|
|
408
403
|
ACTIONS=[:scan,:events,:trevents,:check,:test]
|
|
409
404
|
|
|
410
405
|
def execute_action
|
|
411
|
-
command=
|
|
406
|
+
command=options.get_next_command(ACTIONS)
|
|
412
407
|
unless [:check,:test].include?(command)
|
|
413
408
|
# this will use node api
|
|
414
409
|
@api_node=basic_auth_api
|
|
@@ -426,10 +421,10 @@ module Aspera
|
|
|
426
421
|
@option_skip_folders.push('/'+@option_previews_folder)
|
|
427
422
|
if @access_remote
|
|
428
423
|
# note the filter "name", it's why we take the first one
|
|
429
|
-
@previews_folder_entry=get_folder_entries(@access_key_self['root_file_id'],{:
|
|
424
|
+
@previews_folder_entry=get_folder_entries(@access_key_self['root_file_id'],{name: @option_previews_folder}).first
|
|
430
425
|
raise CliError,"Folder #{@option_previews_folder} does not exist on node. Please create it in the storage root, or specify an alternate name." if @previews_folder_entry.nil?
|
|
431
426
|
else
|
|
432
|
-
raise
|
|
427
|
+
raise 'only local storage allowed in this mode' unless @access_key_self['storage']['type'].eql?('local')
|
|
433
428
|
@local_storage_root=@access_key_self['storage']['path']
|
|
434
429
|
#TODO: option to override @local_storage_root='xxx'
|
|
435
430
|
@local_storage_root=@local_storage_root[LOCAL_STORAGE_PCVL.length..-1] if @local_storage_root.start_with?(LOCAL_STORAGE_PCVL)
|
|
@@ -449,13 +444,14 @@ module Aspera
|
|
|
449
444
|
end
|
|
450
445
|
end
|
|
451
446
|
end
|
|
452
|
-
Aspera::Preview::FileTypes.instance.use_mimemagic =
|
|
447
|
+
Aspera::Preview::FileTypes.instance.use_mimemagic = options.get_option(:mimemagic,:mandatory)
|
|
453
448
|
case command
|
|
454
449
|
when :scan
|
|
455
|
-
scan_path=
|
|
456
|
-
scan_id=
|
|
450
|
+
scan_path=options.get_option(:scan_path,:optional)
|
|
451
|
+
scan_id=options.get_option(:scan_id,:optional)
|
|
457
452
|
# by default start at root
|
|
458
|
-
folder_info=
|
|
453
|
+
folder_info=
|
|
454
|
+
if scan_id.nil?
|
|
459
455
|
{ 'id' => @access_key_self['root_file_id'],
|
|
460
456
|
'name' => '/',
|
|
461
457
|
'type' => 'folder',
|
|
@@ -467,11 +463,11 @@ module Aspera
|
|
|
467
463
|
return Main.result_status('scan finished')
|
|
468
464
|
when :events,:trevents
|
|
469
465
|
iteration_persistency=nil
|
|
470
|
-
if
|
|
466
|
+
if options.get_option(:once_only,:mandatory)
|
|
471
467
|
iteration_persistency=PersistencyActionOnce.new(
|
|
472
468
|
manager: @agents[:persistency],
|
|
473
469
|
data: [],
|
|
474
|
-
id: IdGenerator.from_list(['preview_iteration',command.to_s,
|
|
470
|
+
id: IdGenerator.from_list(['preview_iteration',command.to_s,options.get_option(:url,:mandatory),options.get_option(:username,:mandatory)]))
|
|
475
471
|
end
|
|
476
472
|
# call processing method specified by command line command
|
|
477
473
|
send("process_#{command}",iteration_persistency)
|
|
@@ -480,16 +476,16 @@ module Aspera
|
|
|
480
476
|
Aspera::Preview::Utils.check_tools(@skip_types)
|
|
481
477
|
return Main.result_status('tools validated')
|
|
482
478
|
when :test
|
|
483
|
-
format =
|
|
484
|
-
source =
|
|
485
|
-
dest=preview_filename(format,
|
|
479
|
+
format = options.get_next_argument('format',Aspera::Preview::Generator::PREVIEW_FORMATS)
|
|
480
|
+
source = options.get_next_argument('source file')
|
|
481
|
+
dest=preview_filename(format,options.get_option(:case,:optional))
|
|
486
482
|
g=Aspera::Preview::Generator.new(@gen_options,source,dest,@tmp_folder,nil)
|
|
487
483
|
raise "cannot find file type for #{source}" if g.conversion_type.nil?
|
|
488
484
|
raise "out format #{format} not supported" unless g.supported?
|
|
489
485
|
g.generate
|
|
490
486
|
return Main.result_status("generated: #{dest}")
|
|
491
487
|
else
|
|
492
|
-
raise
|
|
488
|
+
raise 'error'
|
|
493
489
|
end
|
|
494
490
|
ensure
|
|
495
491
|
FileUtils.rm_rf(@tmp_folder)
|