aspera-cli 4.4.0 → 4.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2095 -1503
  3. data/bin/ascli +2 -1
  4. data/bin/asession +4 -5
  5. data/docs/test_env.conf +3 -0
  6. data/examples/aoc.rb +4 -3
  7. data/examples/faspex4.rb +25 -25
  8. data/examples/proxy.pac +1 -1
  9. data/examples/transfer.rb +17 -17
  10. data/lib/aspera/aoc.rb +238 -185
  11. data/lib/aspera/ascmd.rb +93 -83
  12. data/lib/aspera/ats_api.rb +11 -10
  13. data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
  14. data/lib/aspera/cli/extended_value.rb +42 -33
  15. data/lib/aspera/cli/formater.rb +142 -108
  16. data/lib/aspera/cli/info.rb +17 -0
  17. data/lib/aspera/cli/listener/line_dump.rb +3 -2
  18. data/lib/aspera/cli/listener/logger.rb +2 -1
  19. data/lib/aspera/cli/listener/progress.rb +16 -18
  20. data/lib/aspera/cli/listener/progress_multi.rb +18 -21
  21. data/lib/aspera/cli/main.rb +173 -149
  22. data/lib/aspera/cli/manager.rb +163 -168
  23. data/lib/aspera/cli/plugin.rb +43 -31
  24. data/lib/aspera/cli/plugins/alee.rb +6 -6
  25. data/lib/aspera/cli/plugins/aoc.rb +405 -370
  26. data/lib/aspera/cli/plugins/ats.rb +86 -79
  27. data/lib/aspera/cli/plugins/bss.rb +14 -16
  28. data/lib/aspera/cli/plugins/config.rb +580 -362
  29. data/lib/aspera/cli/plugins/console.rb +23 -19
  30. data/lib/aspera/cli/plugins/cos.rb +18 -18
  31. data/lib/aspera/cli/plugins/faspex.rb +201 -158
  32. data/lib/aspera/cli/plugins/faspex5.rb +80 -57
  33. data/lib/aspera/cli/plugins/node.rb +183 -166
  34. data/lib/aspera/cli/plugins/orchestrator.rb +71 -67
  35. data/lib/aspera/cli/plugins/preview.rb +92 -96
  36. data/lib/aspera/cli/plugins/server.rb +79 -75
  37. data/lib/aspera/cli/plugins/shares.rb +35 -19
  38. data/lib/aspera/cli/plugins/sync.rb +20 -22
  39. data/lib/aspera/cli/transfer_agent.rb +76 -113
  40. data/lib/aspera/cli/version.rb +2 -1
  41. data/lib/aspera/colors.rb +35 -27
  42. data/lib/aspera/command_line_builder.rb +48 -34
  43. data/lib/aspera/cos_node.rb +29 -21
  44. data/lib/aspera/data_repository.rb +3 -2
  45. data/lib/aspera/environment.rb +50 -45
  46. data/lib/aspera/fasp/{manager.rb → agent_base.rb} +28 -25
  47. data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +52 -43
  48. data/lib/aspera/fasp/{local.rb → agent_direct.rb} +58 -72
  49. data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +37 -43
  50. data/lib/aspera/fasp/{node.rb → agent_node.rb} +35 -16
  51. data/lib/aspera/fasp/agent_trsdk.rb +104 -0
  52. data/lib/aspera/fasp/error.rb +2 -1
  53. data/lib/aspera/fasp/error_info.rb +68 -52
  54. data/lib/aspera/fasp/installation.rb +152 -124
  55. data/lib/aspera/fasp/listener.rb +1 -0
  56. data/lib/aspera/fasp/parameters.rb +87 -92
  57. data/lib/aspera/fasp/parameters.yaml +305 -249
  58. data/lib/aspera/fasp/resume_policy.rb +11 -14
  59. data/lib/aspera/fasp/transfer_spec.rb +26 -0
  60. data/lib/aspera/fasp/uri.rb +22 -21
  61. data/lib/aspera/faspex_gw.rb +55 -89
  62. data/lib/aspera/hash_ext.rb +4 -3
  63. data/lib/aspera/id_generator.rb +8 -7
  64. data/lib/aspera/keychain/encrypted_hash.rb +121 -0
  65. data/lib/aspera/keychain/macos_security.rb +90 -0
  66. data/lib/aspera/log.rb +55 -37
  67. data/lib/aspera/nagios.rb +13 -12
  68. data/lib/aspera/node.rb +30 -25
  69. data/lib/aspera/oauth.rb +175 -226
  70. data/lib/aspera/open_application.rb +4 -3
  71. data/lib/aspera/persistency_action_once.rb +6 -6
  72. data/lib/aspera/persistency_folder.rb +5 -9
  73. data/lib/aspera/preview/file_types.rb +6 -5
  74. data/lib/aspera/preview/generator.rb +25 -24
  75. data/lib/aspera/preview/options.rb +16 -14
  76. data/lib/aspera/preview/utils.rb +98 -98
  77. data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
  78. data/lib/aspera/proxy_auto_config.rb +111 -20
  79. data/lib/aspera/rest.rb +154 -135
  80. data/lib/aspera/rest_call_error.rb +2 -2
  81. data/lib/aspera/rest_error_analyzer.rb +23 -25
  82. data/lib/aspera/rest_errors_aspera.rb +15 -14
  83. data/lib/aspera/ssh.rb +12 -10
  84. data/lib/aspera/sync.rb +42 -41
  85. data/lib/aspera/temp_file_manager.rb +18 -14
  86. data/lib/aspera/timer_limiter.rb +2 -1
  87. data/lib/aspera/uri_reader.rb +7 -5
  88. data/lib/aspera/web_auth.rb +79 -76
  89. metadata +116 -29
  90. data/docs/Makefile +0 -66
  91. data/docs/README.erb.md +0 -3973
  92. data/docs/README.md +0 -13
  93. data/docs/diagrams.txt +0 -49
  94. data/docs/doc_tools.rb +0 -58
  95. data/lib/aspera/api_detector.rb +0 -60
  96. data/lib/aspera/cli/plugins/shares2.rb +0 -114
  97. data/lib/aspera/secrets.rb +0 -20
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  require 'aspera/cli/plugins/node'
2
3
  require 'xmlsimple'
3
4
 
@@ -7,47 +8,47 @@ module Aspera
7
8
  class Orchestrator < BasicAuthPlugin
8
9
  def initialize(env)
9
10
  super(env)
10
- self.options.add_opt_simple(:params,"parameters hash table, use @json:{\"param\":\"value\"}")
11
- self.options.add_opt_simple(:result,"specify result value as: 'work step:parameter'")
12
- self.options.add_opt_boolean(:synchronous,"work step:parameter expected as result")
13
- self.options.add_opt_list(:ret_style,[:header,:arg,:ext],'how return type is requested in api')
14
- self.options.add_opt_list(:auth_style,[:arg_pass,:head_basic,:apikey],'authentication type')
15
- self.options.set_option(:params,{})
16
- self.options.set_option(:synchronous,:no)
17
- self.options.set_option(:ret_style,:arg)
18
- self.options.set_option(:auth_style,:head_basic)
19
- self.options.parse_options!
11
+ options.add_opt_simple(:params,'parameters hash table, use @json:{"param":"value"}')
12
+ options.add_opt_simple(:result,"specify result value as: 'work step:parameter'")
13
+ options.add_opt_boolean(:synchronous,'work step:parameter expected as result')
14
+ options.add_opt_list(:ret_style,[:header,:arg,:ext],'how return type is requested in api')
15
+ options.add_opt_list(:auth_style,[:arg_pass,:head_basic,:apikey],'authentication type')
16
+ options.set_option(:params,{})
17
+ options.set_option(:synchronous,:no)
18
+ options.set_option(:ret_style,:arg)
19
+ options.set_option(:auth_style,:head_basic)
20
+ options.parse_options!
20
21
  end
21
22
 
22
- ACTIONS=[:info, :workflow, :plugins, :processes]
23
+ ACTIONS=[:info, :workflow, :plugins, :processes].freeze
23
24
 
24
25
  # for JSON format: add extension ".json" or add url parameter: format=json or Accept: application/json
25
26
  # id can be: a parameter id=x, or at the end of url /id, for workflows: work_order[workflow_id]=wf_id
26
- def call_API_orig(endpoint,id=nil,url_params={:format=>:json},accept=nil)
27
- # calls are GET
28
- call_args={:operation=>'GET',:subpath=>endpoint}
29
- # specify id if necessary
30
- call_args[:subpath]=call_args[:subpath]+'/'+id unless id.nil?
31
- unless url_params.nil?
32
- if url_params.has_key?(:format)
33
- call_args[:headers]={'Accept'=>'application/'+url_params[:format].to_s}
34
- end
35
- call_args[:headers]={'Accept'=>accept} unless accept.nil?
36
- # add params if necessary
37
- call_args[:url_params]=url_params
38
- end
39
- return @api_orch.call(call_args)
40
- end
27
+ # def call_API_orig(endpoint,id=nil,url_params={format: :json},accept=nil)
28
+ # # calls are GET
29
+ # call_args={operation: 'GET',subpath: endpoint}
30
+ # # specify id if necessary
31
+ # call_args[:subpath]=call_args[:subpath]+'/'+id unless id.nil?
32
+ # unless url_params.nil?
33
+ # if url_params.has_key?(:format)
34
+ # call_args[:headers]={'Accept'=>'application/'+url_params[:format].to_s}
35
+ # end
36
+ # call_args[:headers]={'Accept'=>accept} unless accept.nil?
37
+ # # add params if necessary
38
+ # call_args[:url_params]=url_params
39
+ # end
40
+ # return @api_orch.call(call_args)
41
+ # end
41
42
 
42
- def call_API(endpoint,opt={})
43
+ def call_ao(endpoint,opt={})
43
44
  opt[:prefix]='api' unless opt.has_key?(:prefix)
44
45
  # calls are GET
45
- call_args={:operation=>'GET',:subpath=>endpoint}
46
+ call_args={operation: 'GET',subpath: endpoint}
46
47
  # specify prefix if necessary
47
48
  call_args[:subpath]="#{opt[:prefix]}/#{call_args[:subpath]}" unless opt[:prefix].nil?
48
49
  # specify id if necessary
49
50
  call_args[:subpath]="#{call_args[:subpath]}/#{opt[:id]}" if opt.has_key?(:id)
50
- call_type=self.options.get_option(:ret_style,:mandatory)
51
+ call_type=options.get_option(:ret_style,:mandatory)
51
52
  call_type=opt[:ret_style] if opt.has_key?(:ret_style)
52
53
  format='json'
53
54
  format=opt[:format] if opt.has_key?(:format)
@@ -61,80 +62,83 @@ module Aspera
61
62
  call_args[:url_params][:format]=format
62
63
  when :ext
63
64
  call_args[:subpath]="#{call_args[:subpath]}.#{format}"
64
- else raise "unexpected"
65
+ else raise 'unexpected'
65
66
  end
66
67
  end
67
68
  result=@api_orch.call(call_args)
68
- result[:data]=XmlSimple.xml_in(result[:http].body, opt[:xml_opt]||{"ForceArray" => true}) if format.eql?('xml')
69
+ result[:data]=XmlSimple.xml_in(result[:http].body, opt[:xml_opt]||{'ForceArray' => true}) if format.eql?('xml')
69
70
  return result
70
71
  end
71
72
 
72
73
  def execute_action
73
- rest_params={:base_url => self.options.get_option(:url,:mandatory)}
74
- case self.options.get_option(:auth_style,:mandatory)
74
+ rest_params={base_url: options.get_option(:url,:mandatory)}
75
+ case options.get_option(:auth_style,:mandatory)
75
76
  when :arg_pass
76
77
  rest_params[:auth]={
77
- :type => :url,
78
- :url_creds => {
79
- 'login' =>self.options.get_option(:username,:mandatory),
80
- 'password' =>self.options.get_option(:password,:mandatory) }}
78
+ type: :url,
79
+ url_creds: {
80
+ 'login' =>options.get_option(:username,:mandatory),
81
+ 'password' =>options.get_option(:password,:mandatory) }}
81
82
  when :head_basic
82
83
  rest_params[:auth]={
83
- :type => :basic,
84
- :username =>self.options.get_option(:username,:mandatory),
85
- :password =>self.options.get_option(:password,:mandatory) }
84
+ type: :basic,
85
+ username: options.get_option(:username,:mandatory),
86
+ password: options.get_option(:password,:mandatory) }
86
87
  when :apikey
87
- raise "Not implemented"
88
+ raise 'Not implemented'
88
89
  end
89
90
 
90
91
  @api_orch=Rest.new(rest_params)
91
92
 
92
- command1=self.options.get_next_command(ACTIONS)
93
+ command1=options.get_next_command(ACTIONS)
93
94
  case command1
94
95
  when :info
95
- result=call_API('remote_node_ping',format: 'xml', xml_opt: {"ForceArray" => false})
96
- return {:type=>:single_object,:data=>result[:data]}
97
- # result=call_API('workflows',prefix: nil,format: nil)
96
+ result=call_ao('remote_node_ping',format: 'xml', xml_opt: {'ForceArray' => false})
97
+ return {type: :single_object,data: result[:data]}
98
+ # result=call_ao('workflows',prefix: nil,format: nil)
98
99
  # version='unknown'
99
100
  # if m=result[:http].body.match(/\(Orchestrator v([1-9]+\.[\.0-9a-f\-]+)\)/)
100
101
  # version=m[1]
101
102
  # end
102
- # return {:type=>:single_object,:data=>{'version'=>version}}
103
+ # return {type: :single_object,data: {'version'=>version}}
103
104
  when :processes
104
105
  # TODO: Jira ? API has only XML format
105
- result=call_API('processes_status',format: 'xml')
106
- return {:type=>:object_list,:data=>result[:data]['process']}
106
+ result=call_ao('processes_status',format: 'xml')
107
+ return {type: :object_list,data: result[:data]['process']}
107
108
  when :plugins
108
109
  # TODO: Jira ? only json format on url
109
- result=call_API('plugin_version')[:data]
110
- return {:type=>:object_list,:data=>result['Plugin']}
110
+ result=call_ao('plugin_version')[:data]
111
+ return {type: :object_list,data: result['Plugin']}
111
112
  when :workflow
112
- command=self.options.get_next_command([:list, :status, :inputs, :details, :start, :export])
113
- unless [:list, :status].include?(command)
114
- wf_id=self.options.get_option(:id,:mandatory)
113
+ command=options.get_next_command([:list, :status, :inputs, :details, :start, :export])
114
+ unless [:list].include?(command)
115
+ wf_id=instance_identifier()
115
116
  end
116
117
  case command
117
118
  when :status
118
- result=call_API('workflows_status')[:data]
119
- return {:type=>:object_list,:data=>result['workflows']['workflow']}
119
+ options={}
120
+ options[:id]=wf_id unless wf_id.eql?('ALL')
121
+ result=call_ao('workflows_status',options)[:data]
122
+ return {type: :object_list,data: result['workflows']['workflow']}
120
123
  when :list
121
- result=call_API('workflows_list',id: 0)[:data]
122
- return {:type=>:object_list,:data=>result['workflows']['workflow'],:fields=>["id","portable_id","name","published_status","published_revision_id","latest_revision_id","last_modification"]}
124
+ result=call_ao('workflows_list',id: 0)[:data]
125
+ return {type: :object_list,data: result['workflows']['workflow'],
126
+ fields: ['id','portable_id','name','published_status','published_revision_id','latest_revision_id','last_modification']}
123
127
  when :details
124
- result=call_API('workflow_details',id: wf_id)[:data]
125
- return {:type=>:object_list,:data=>result['workflows']['workflow']['statuses']}
128
+ result=call_ao('workflow_details',id: wf_id)[:data]
129
+ return {type: :object_list,data: result['workflows']['workflow']['statuses']}
126
130
  when :inputs
127
- result=call_API('workflow_inputs_spec',id: wf_id)[:data]
128
- return {:type=>:single_object,:data=>result['workflow_inputs_spec']}
131
+ result=call_ao('workflow_inputs_spec',id: wf_id)[:data]
132
+ return {type: :single_object,data: result['workflow_inputs_spec']}
129
133
  when :export
130
- result=call_API('export_workflow',id: wf_id,format: nil)[:http]
131
- return {:type=>:text,:data=>result.body}
134
+ result=call_ao('export_workflow',id: wf_id,format: nil)[:http]
135
+ return {type: :text,data: result.body}
132
136
  when :start
133
137
  result={
134
- :type=>:single_object,
135
- :data=>nil
138
+ type: :single_object,
139
+ data: nil
136
140
  }
137
- call_params={:format=>:json}
141
+ call_params={format: :json}
138
142
  override_accept=nil
139
143
  # set external parameters if any
140
144
  self.options.get_option(:params,:mandatory).each do |name,value|
@@ -157,7 +161,7 @@ module Aspera
157
161
  result[:type]=:text
158
162
  override_accept='text/plain'
159
163
  end
160
- result[:data]=call_API('initiate',id: wf_id,args: call_params,accept: override_accept)[:data]
164
+ result[:data]=call_ao('initiate',id: wf_id,args: call_params,accept: override_accept)[:data]
161
165
  return result
162
166
  end # wf command
163
167
  else raise "ERROR, unknown command: [#{command}]"
@@ -1,8 +1,10 @@
1
+ # frozen_string_literal: true
1
2
  require 'aspera/cli/basic_auth_plugin'
2
3
  require 'aspera/preview/generator'
3
4
  require 'aspera/preview/options'
4
5
  require 'aspera/preview/utils'
5
6
  require 'aspera/preview/file_types'
7
+ require 'aspera/fasp/transfer_spec'
6
8
  require 'aspera/persistency_action_once'
7
9
  require 'aspera/node'
8
10
  require 'aspera/hash_ext'
@@ -31,10 +33,7 @@ module Aspera
31
33
 
32
34
  # option_skip_format has special accessors
33
35
  attr_accessor :option_previews_folder
34
- attr_accessor :option_folder_reset_cache
35
- attr_accessor :option_skip_folders
36
- attr_accessor :option_overwrite
37
- attr_accessor :option_file_access
36
+ attr_accessor :option_folder_reset_cache, :option_skip_folders, :option_overwrite, :option_file_access
38
37
  def initialize(env)
39
38
  super(env)
40
39
  @skip_types=[]
@@ -46,43 +45,43 @@ module Aspera
46
45
  # used to trigger periodic processing
47
46
  @periodic=TimerLimiter.new(LOG_LIMITER_SEC)
48
47
  # link CLI options to gen_info attributes
49
- self.options.set_obj_attr(:skip_format,self,:option_skip_format,[]) # no skip
50
- self.options.set_obj_attr(:folder_reset_cache,self,:option_folder_reset_cache,:no)
51
- self.options.set_obj_attr(:skip_types,self,:option_skip_types)
52
- self.options.set_obj_attr(:previews_folder,self,:option_previews_folder,DEFAULT_PREVIEWS_FOLDER)
53
- self.options.set_obj_attr(:skip_folders,self,:option_skip_folders,[]) # no skip
54
- self.options.set_obj_attr(:overwrite,self,:option_overwrite,:mtime)
55
- self.options.set_obj_attr(:file_access,self,:option_file_access,:local)
56
- self.options.add_opt_list(:skip_format,Aspera::Preview::Generator::PREVIEW_FORMATS,'skip this preview format (multiple possible)')
57
- self.options.add_opt_list(:folder_reset_cache,[:no,:header,:read],'force detection of generated preview by refresh cache')
58
- self.options.add_opt_simple(:skip_types,'skip types in comma separated list')
59
- self.options.add_opt_simple(:previews_folder,'preview folder in storage root')
60
- self.options.add_opt_simple(:temp_folder,'path to temp folder')
61
- self.options.add_opt_simple(:skip_folders,'list of folder to skip')
62
- self.options.add_opt_simple(:case,'basename of output for for test')
63
- self.options.add_opt_simple(:scan_path,'subpath in folder id to start scan in (default=/)')
64
- self.options.add_opt_simple(:scan_id,'forder id in storage to start scan in, default is access key main folder id')
65
- self.options.add_opt_boolean(:mimemagic,'use Mime type detection of gem mimemagic')
66
- self.options.add_opt_list(:overwrite,[:always,:never,:mtime],'when to overwrite result file')
67
- self.options.add_opt_list(:file_access,[:local,:remote],'how to read and write files in repository')
68
- self.options.set_option(:temp_folder,Dir.tmpdir)
69
- self.options.set_option(:mimemagic,:false)
48
+ options.set_obj_attr(:skip_format,self,:option_skip_format,[]) # no skip
49
+ options.set_obj_attr(:folder_reset_cache,self,:option_folder_reset_cache,:no)
50
+ options.set_obj_attr(:skip_types,self,:option_skip_types)
51
+ options.set_obj_attr(:previews_folder,self,:option_previews_folder,DEFAULT_PREVIEWS_FOLDER)
52
+ options.set_obj_attr(:skip_folders,self,:option_skip_folders,[]) # no skip
53
+ options.set_obj_attr(:overwrite,self,:option_overwrite,:mtime)
54
+ options.set_obj_attr(:file_access,self,:option_file_access,:local)
55
+ options.add_opt_list(:skip_format,Aspera::Preview::Generator::PREVIEW_FORMATS,'skip this preview format (multiple possible)')
56
+ options.add_opt_list(:folder_reset_cache,[:no,:header,:read],'force detection of generated preview by refresh cache')
57
+ options.add_opt_simple(:skip_types,'skip types in comma separated list')
58
+ options.add_opt_simple(:previews_folder,'preview folder in storage root')
59
+ options.add_opt_simple(:temp_folder,'path to temp folder')
60
+ options.add_opt_simple(:skip_folders,'list of folder to skip')
61
+ options.add_opt_simple(:case,'basename of output for for test')
62
+ options.add_opt_simple(:scan_path,'subpath in folder id to start scan in (default=/)')
63
+ options.add_opt_simple(:scan_id,'forder id in storage to start scan in, default is access key main folder id')
64
+ options.add_opt_boolean(:mimemagic,'use Mime type detection of gem mimemagic')
65
+ options.add_opt_list(:overwrite,[:always,:never,:mtime],'when to overwrite result file')
66
+ options.add_opt_list(:file_access,[:local,:remote],'how to read and write files in repository')
67
+ options.set_option(:temp_folder,Dir.tmpdir)
68
+ options.set_option(:mimemagic,false)
70
69
 
71
70
  # add other options for generator (and set default values)
72
71
  Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
73
- self.options.set_obj_attr(opt[:name],@gen_options,opt[:name],opt[:default])
72
+ options.set_obj_attr(opt[:name],@gen_options,opt[:name],opt[:default])
74
73
  if opt.has_key?(:values)
75
- self.options.add_opt_list(opt[:name],opt[:values],opt[:description])
74
+ options.add_opt_list(opt[:name],opt[:values],opt[:description])
76
75
  elsif [:yes,:no].include?(opt[:default])
77
- self.options.add_opt_boolean(opt[:name],opt[:description])
76
+ options.add_opt_boolean(opt[:name],opt[:description])
78
77
  else
79
- self.options.add_opt_simple(opt[:name],opt[:description])
78
+ options.add_opt_simple(opt[:name],opt[:description])
80
79
  end
81
80
  end
82
81
 
83
- self.options.parse_options!
82
+ options.parse_options!
84
83
  raise 'skip_folder shall be an Array, use @json:[...]' unless @option_skip_folders.is_a?(Array)
85
- @tmp_folder=File.join(self.options.get_option(:temp_folder,:mandatory),"#{TMP_DIR_PREFIX}.#{SecureRandom.uuid}")
84
+ @tmp_folder=File.join(options.get_option(:temp_folder,:mandatory),"#{TMP_DIR_PREFIX}.#{SecureRandom.uuid}")
86
85
  FileUtils.mkdir_p(@tmp_folder)
87
86
  Log.log.debug("tmpdir: #{@tmp_folder}")
88
87
  end
@@ -97,7 +96,7 @@ module Aspera
97
96
  end
98
97
 
99
98
  def option_skip_types
100
- return @skip_types.map{|i|i.to_s}.join(',')
99
+ return @skip_types.map(&:to_s).join(',')
101
100
  end
102
101
 
103
102
  def option_skip_format=(value)
@@ -105,7 +104,7 @@ module Aspera
105
104
  end
106
105
 
107
106
  def option_skip_format
108
- return @preview_formats_to_generate.map{|i|i.to_s}.join(',')
107
+ return @preview_formats_to_generate.map(&:to_s).join(',')
109
108
  end
110
109
 
111
110
  # /files/id/files is normally cached in redis, but we can discard the cache
@@ -113,7 +112,7 @@ module Aspera
113
112
  def get_folder_entries(file_id,request_args=nil)
114
113
  headers={'Accept'=>'application/json'}
115
114
  headers.merge!({'X-Aspera-Cache-Control'=>'no-cache'}) if @option_folder_reset_cache.eql?(:header)
116
- return @api_node.call({:operation=>'GET',:subpath=>"files/#{file_id}/files",:headers=>headers,:url_params=>request_args})[:data]
115
+ return @api_node.call({operation: 'GET',subpath: "files/#{file_id}/files",headers: headers,url_params: request_args})[:data]
117
116
  #return @api_node.read("files/#{file_id}/files",request_args)[:data]
118
117
  end
119
118
 
@@ -138,9 +137,9 @@ module Aspera
138
137
  end
139
138
  return if events.empty?
140
139
  events.each do |event|
141
- if event['data']['direction'].eql?('receive') and
142
- event['data']['status'].eql?('completed') and
143
- event['data']['error_code'].eql?(0) and
140
+ if event['data']['direction'].eql?(Fasp::TransferSpec::DIRECTION_RECEIVE) &&
141
+ event['data']['status'].eql?('completed') &&
142
+ event['data']['error_code'].eql?(0) &&
144
143
  event['data'].dig('tags','aspera',PREV_GEN_TAG).nil?
145
144
  folder_id=event.dig('data','tags','aspera','node','file_id')
146
145
  folder_id||=event.dig('data','file_id')
@@ -149,13 +148,13 @@ module Aspera
149
148
  scan_folder_files(folder_entry) unless folder_entry.nil?
150
149
  end
151
150
  end
152
- if @periodic.trigger? or event.equal?(events.last)
153
- Log.log.info("Processed event #{event['id']}")
154
- # save checkpoint to avoid losing processing in case of error
155
- if !iteration_persistency.nil?
156
- iteration_persistency.data[0]=event['id'].to_s
157
- iteration_persistency.save
158
- end
151
+ # log/persist periodically or last one
152
+ next unless @periodic.trigger? || event.equal?(events.last)
153
+ Log.log.info("Processed event #{event['id']}")
154
+ # save checkpoint to avoid losing processing in case of error
155
+ if !iteration_persistency.nil?
156
+ iteration_persistency.data[0]=event['id'].to_s
157
+ iteration_persistency.save
159
158
  end
160
159
  end
161
160
  end
@@ -175,7 +174,7 @@ module Aspera
175
174
  # process only files
176
175
  if event.dig('data','type').eql?('file')
177
176
  file_entry=@api_node.read("files/#{event['data']['id']}")[:data] rescue nil
178
- if !file_entry.nil? and
177
+ if !file_entry.nil? &&
179
178
  @option_skip_folders.select{|d|file_entry['path'].start_with?(d)}.empty?
180
179
  file_entry['parent_file_id']=event['data']['parent_file_id']
181
180
  if event['types'].include?('file.deleted')
@@ -186,30 +185,30 @@ module Aspera
186
185
  end
187
186
  end
188
187
  end
189
- if @periodic.trigger? or event.equal?(events.last)
190
- Log.log.info("Processing event #{event['id']}")
191
- # save checkpoint to avoid losing processing in case of error
192
- if !iteration_persistency.nil?
193
- iteration_persistency.data[0]=event['id'].to_s
194
- iteration_persistency.save
195
- end
188
+ # log/persist periodically or last one
189
+ next unless @periodic.trigger? || event.equal?(events.last)
190
+ Log.log.info("Processing event #{event['id']}")
191
+ # save checkpoint to avoid losing processing in case of error
192
+ if !iteration_persistency.nil?
193
+ iteration_persistency.data[0]=event['id'].to_s
194
+ iteration_persistency.save
196
195
  end
197
196
  end
198
197
  end
199
198
 
200
199
  def do_transfer(direction,folder_id,source_filename,destination='/')
201
- raise "error" if destination.nil? and direction.eql?('receive')
200
+ raise 'error' if destination.nil? && direction.eql?(Fasp::TransferSpec::DIRECTION_RECEIVE)
202
201
  if @default_transfer_spec.nil?
203
202
  # make a dummy call to get some default transfer parameters
204
203
  res=@api_node.create('files/upload_setup',{'transfer_requests'=>[{'transfer_request'=>{'paths'=>[{}],'destination_root'=>'/'}}]})
205
204
  template_ts=res[:data]['transfer_specs'].first['transfer_spec']
206
205
  # get ports, anyway that should be 33001 for both. add remote_user ?
207
- @default_transfer_spec=['ssh_port','fasp_port'].inject({}){|h,e|h[e]=template_ts[e];h}
208
- if ! @default_transfer_spec['remote_user'].eql?(Aspera::Node::ACCESS_KEY_TRANSFER_USER)
209
- Log.log.warn("remote_user shall be xfer")
210
- @default_transfer_spec['remote_user']=Aspera::Node::ACCESS_KEY_TRANSFER_USER
206
+ @default_transfer_spec=['ssh_port','fasp_port'].each_with_object({}){|e,h|h[e]=template_ts[e];}
207
+ if !@default_transfer_spec['remote_user'].eql?(Aspera::Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
208
+ Log.log.warn('remote_user shall be xfer')
209
+ @default_transfer_spec['remote_user']=Aspera::Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER
211
210
  end
212
- Aspera::Node::set_ak_basic_token(@default_transfer_spec,@access_key_self['id'],self.options.get_option(:password,:mandatory))
211
+ Aspera::Node.set_ak_basic_token(@default_transfer_spec,@access_key_self['id'],options.get_option(:password,:mandatory))
213
212
  # note: we use the same address for ascp than for node api instead of the one from upload_setup
214
213
  # TODO: configurable ? useful ?
215
214
  @default_transfer_spec['remote_host']=@transfer_server_address
@@ -225,33 +224,34 @@ module Aspera
225
224
  })
226
225
  # force destination
227
226
  # tspec['destination_root']=destination
228
- self.transfer.option_transfer_spec_deep_merge({'destination_root'=>destination})
229
- Main.result_transfer(self.transfer.start(tspec,{:src=>:node_gen4}))
227
+ transfer.option_transfer_spec_deep_merge({'destination_root'=>destination})
228
+ Main.result_transfer(transfer.start(tspec,{src: :node_gen4}))
230
229
  end
231
230
 
232
- def get_infos_local(gen_infos,entry,local_entry_preview_dir)
231
+ def get_infos_local(gen_infos,entry)
233
232
  local_original_filepath=File.join(@local_storage_root,entry['path'])
234
233
  original_mtime=File.mtime(local_original_filepath)
235
234
  # out
236
- local_entry_preview_dir.replace(File.join(@local_preview_folder, entry_preview_folder_name(entry)))
235
+ local_entry_preview_dir=File.join(@local_preview_folder, entry_preview_folder_name(entry))
237
236
  gen_infos.each do |gen_info|
238
237
  gen_info[:src]=local_original_filepath
239
238
  gen_info[:dst]=File.join(local_entry_preview_dir, gen_info[:base_dest])
240
239
  gen_info[:preview_exist]=File.exist?(gen_info[:dst])
241
240
  gen_info[:preview_newer_than_original] = (gen_info[:preview_exist] and (File.mtime(gen_info[:dst])>original_mtime))
242
241
  end
242
+ return local_entry_preview_dir
243
243
  end
244
244
 
245
- def get_infos_remote(gen_infos,entry,local_entry_preview_dir)
245
+ def get_infos_remote(gen_infos,entry)
246
246
  #Log.log.debug(">>>> get_infos_remote #{entry}".red)
247
247
  # store source directly here
248
248
  local_original_filepath=File.join(@tmp_folder,entry['name'])
249
249
  #original_mtime=DateTime.parse(entry['modified_time'])
250
250
  # out: where previews are generated
251
- local_entry_preview_dir.replace(File.join(@tmp_folder,entry_preview_folder_name(entry)))
251
+ local_entry_preview_dir=File.join(@tmp_folder,entry_preview_folder_name(entry))
252
252
  file_info=@api_node.read("files/#{entry['id']}")[:data]
253
253
  #TODO: this does not work because previews is hidden in api (gen4)
254
- #this_preview_folder_entries=get_folder_entries(@previews_folder_entry['id'],{:name=>@entry_preview_folder_name})
254
+ #this_preview_folder_entries=get_folder_entries(@previews_folder_entry['id'],{name: @entry_preview_folder_name})
255
255
  # TODO: use gen3 api to list files and get date
256
256
  gen_infos.each do |gen_info|
257
257
  gen_info[:src]=local_original_filepath
@@ -261,6 +261,7 @@ module Aspera
261
261
  # TODO: get change time and compare, useful ?
262
262
  gen_info[:preview_newer_than_original] = gen_info[:preview_exist]
263
263
  end
264
+ return local_entry_preview_dir
264
265
  end
265
266
 
266
267
  # defined by node api
@@ -270,29 +271,23 @@ module Aspera
270
271
 
271
272
  def preview_filename(preview_format,filename=nil)
272
273
  filename||=PREVIEW_BASENAME
273
- return "#{filename}.#{preview_format.to_s}"
274
+ return "#{filename}.#{preview_format}"
274
275
  end
275
276
 
276
277
  # generate preview files for one folder entry (file) if necessary
277
278
  # entry must contain "parent_file_id" if remote.
278
279
  def generate_preview(entry)
279
- #Log.log.debug(">>>> #{entry}".red)
280
- # folder where previews will be generated for this particular entry
281
- local_entry_preview_dir=String.new
282
280
  # prepare generic information
283
281
  gen_infos=@preview_formats_to_generate.map do |preview_format|
284
282
  {
285
- :preview_format => preview_format,
286
- :base_dest => preview_filename(preview_format)
283
+ preview_format: preview_format,
284
+ base_dest: preview_filename(preview_format)
287
285
  }
288
286
  end
289
287
  # lets gather some infos on possibly existing previews
290
288
  # it depends if files access locally or remotely
291
- if @access_remote
292
- get_infos_remote(gen_infos,entry,local_entry_preview_dir)
293
- else # direct local file system access
294
- get_infos_local(gen_infos,entry,local_entry_preview_dir)
295
- end
289
+ # folder where previews will be generated for this particular entry
290
+ local_entry_preview_dir=@access_remote ? get_infos_remote(gen_infos,entry) : get_infos_local(gen_infos,entry)
296
291
  # here we have the status on preview files
297
292
  # let's find if they need generation
298
293
  gen_infos.select! do |gen_info|
@@ -324,7 +319,7 @@ module Aspera
324
319
  if @access_remote
325
320
  raise 'missing parent_file_id in entry' if entry['parent_file_id'].nil?
326
321
  # download original file to temp folder
327
- do_transfer('receive',entry['parent_file_id'],entry['name'],@tmp_folder)
322
+ do_transfer(Fasp::TransferSpec::DIRECTION_RECEIVE,entry['parent_file_id'],entry['name'],@tmp_folder)
328
323
  end
329
324
  Log.log.info("source: #{entry['id']}: #{entry['path']})")
330
325
  gen_infos.each do |gen_info|
@@ -332,7 +327,7 @@ module Aspera
332
327
  end
333
328
  if @access_remote
334
329
  # upload
335
- do_transfer('send',@previews_folder_entry['id'],local_entry_preview_dir)
330
+ do_transfer(Fasp::TransferSpec::DIRECTION_SEND,@previews_folder_entry['id'],local_entry_preview_dir)
336
331
  # cleanup after upload
337
332
  FileUtils.rm_rf(local_entry_preview_dir)
338
333
  File.delete(File.join(@tmp_folder,entry['name']))
@@ -341,8 +336,8 @@ module Aspera
341
336
  if @option_folder_reset_cache.eql?(:read)
342
337
  @api_node.read("files/#{entry['id']}")
343
338
  end
344
- rescue => e
345
- Log.log.error("#{e.message}")
339
+ rescue StandardError => e
340
+ Log.log.error("Ignore: #{e.message}")
346
341
  Log.log.debug(e.backtrace.join("\n").red)
347
342
  end # generate_preview
348
343
 
@@ -351,7 +346,7 @@ module Aspera
351
346
  def scan_folder_files(top_entry,scan_start=nil)
352
347
  if !scan_start.nil?
353
348
  # canonical path: start with / and ends with /
354
- scan_start='/'+scan_start.split('/').select{|i|!i.empty?}.join('/')
349
+ scan_start='/'+scan_start.split('/').reject(&:empty?).join('/')
355
350
  scan_start="#{scan_start}/" #unless scan_start.end_with?('/')
356
351
  end
357
352
  filter_block=Aspera::Node.file_matcher(options.get_option(:value,:optional))
@@ -364,7 +359,7 @@ module Aspera
364
359
  entry_path_with_slash=entry['path']
365
360
  Log.log.info("processing entry #{entry_path_with_slash}") if @periodic.trigger?
366
361
  entry_path_with_slash="#{entry_path_with_slash}/" unless entry_path_with_slash.end_with?('/')
367
- if !scan_start.nil? and !scan_start.start_with?(entry_path_with_slash) and !entry_path_with_slash.start_with?(scan_start)
362
+ if !scan_start.nil? && !scan_start.start_with?(entry_path_with_slash) && !entry_path_with_slash.start_with?(scan_start)
368
363
  Log.log.debug("#{entry['path']} folder (skip start)".bg_red)
369
364
  next
370
365
  end
@@ -399,7 +394,7 @@ module Aspera
399
394
  else
400
395
  Log.log.warn("unknown entry type: #{entry['type']}")
401
396
  end
402
- rescue => e
397
+ rescue StandardError => e
403
398
  Log.log.warn("An error occured: #{e}, ignoring")
404
399
  end
405
400
  end
@@ -408,7 +403,7 @@ module Aspera
408
403
  ACTIONS=[:scan,:events,:trevents,:check,:test]
409
404
 
410
405
  def execute_action
411
- command=self.options.get_next_command(ACTIONS)
406
+ command=options.get_next_command(ACTIONS)
412
407
  unless [:check,:test].include?(command)
413
408
  # this will use node api
414
409
  @api_node=basic_auth_api
@@ -426,10 +421,10 @@ module Aspera
426
421
  @option_skip_folders.push('/'+@option_previews_folder)
427
422
  if @access_remote
428
423
  # note the filter "name", it's why we take the first one
429
- @previews_folder_entry=get_folder_entries(@access_key_self['root_file_id'],{:name=>@option_previews_folder}).first
424
+ @previews_folder_entry=get_folder_entries(@access_key_self['root_file_id'],{name: @option_previews_folder}).first
430
425
  raise CliError,"Folder #{@option_previews_folder} does not exist on node. Please create it in the storage root, or specify an alternate name." if @previews_folder_entry.nil?
431
426
  else
432
- raise "only local storage allowed in this mode" unless @access_key_self['storage']['type'].eql?('local')
427
+ raise 'only local storage allowed in this mode' unless @access_key_self['storage']['type'].eql?('local')
433
428
  @local_storage_root=@access_key_self['storage']['path']
434
429
  #TODO: option to override @local_storage_root='xxx'
435
430
  @local_storage_root=@local_storage_root[LOCAL_STORAGE_PCVL.length..-1] if @local_storage_root.start_with?(LOCAL_STORAGE_PCVL)
@@ -449,13 +444,14 @@ module Aspera
449
444
  end
450
445
  end
451
446
  end
452
- Aspera::Preview::FileTypes.instance.use_mimemagic = self.options.get_option(:mimemagic,:mandatory)
447
+ Aspera::Preview::FileTypes.instance.use_mimemagic = options.get_option(:mimemagic,:mandatory)
453
448
  case command
454
449
  when :scan
455
- scan_path=self.options.get_option(:scan_path,:optional)
456
- scan_id=self.options.get_option(:scan_id,:optional)
450
+ scan_path=options.get_option(:scan_path,:optional)
451
+ scan_id=options.get_option(:scan_id,:optional)
457
452
  # by default start at root
458
- folder_info=if scan_id.nil?
453
+ folder_info=
454
+ if scan_id.nil?
459
455
  { 'id' => @access_key_self['root_file_id'],
460
456
  'name' => '/',
461
457
  'type' => 'folder',
@@ -467,11 +463,11 @@ module Aspera
467
463
  return Main.result_status('scan finished')
468
464
  when :events,:trevents
469
465
  iteration_persistency=nil
470
- if self.options.get_option(:once_only,:mandatory)
466
+ if options.get_option(:once_only,:mandatory)
471
467
  iteration_persistency=PersistencyActionOnce.new(
472
468
  manager: @agents[:persistency],
473
469
  data: [],
474
- id: IdGenerator.from_list(['preview_iteration',command.to_s,self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)]))
470
+ id: IdGenerator.from_list(['preview_iteration',command.to_s,options.get_option(:url,:mandatory),options.get_option(:username,:mandatory)]))
475
471
  end
476
472
  # call processing method specified by command line command
477
473
  send("process_#{command}",iteration_persistency)
@@ -480,16 +476,16 @@ module Aspera
480
476
  Aspera::Preview::Utils.check_tools(@skip_types)
481
477
  return Main.result_status('tools validated')
482
478
  when :test
483
- format = self.options.get_next_argument('format',Aspera::Preview::Generator::PREVIEW_FORMATS)
484
- source = self.options.get_next_argument('source file')
485
- dest=preview_filename(format,self.options.get_option(:case,:optional))
479
+ format = options.get_next_argument('format',Aspera::Preview::Generator::PREVIEW_FORMATS)
480
+ source = options.get_next_argument('source file')
481
+ dest=preview_filename(format,options.get_option(:case,:optional))
486
482
  g=Aspera::Preview::Generator.new(@gen_options,source,dest,@tmp_folder,nil)
487
483
  raise "cannot find file type for #{source}" if g.conversion_type.nil?
488
484
  raise "out format #{format} not supported" unless g.supported?
489
485
  g.generate
490
486
  return Main.result_status("generated: #{dest}")
491
487
  else
492
- raise "error"
488
+ raise 'error'
493
489
  end
494
490
  ensure
495
491
  FileUtils.rm_rf(@tmp_folder)