etna 0.1.35 → 0.1.39

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: bf1f86e15c74113a4591afd3d8a43064b6ed3b448bdfad3739baa27061df7913
4
- data.tar.gz: ed543c8775c89679b4755ca3119899639fc125df2d7e65c3515998f06ae45de8
3
+ metadata.gz: c21212f38bbad3cf757eb84a9e453428c9d0b6f195fd34eb81f98aeff2746490
4
+ data.tar.gz: 3581821244c087cbbfbac162f9a5c61911e06ab64923c603a144d36ca3bedff0
5
5
  SHA512:
6
- metadata.gz: 8523211434696c74c03ca5319a089c74dba9bc3c3cc086f723dcadd1789c91f2e6c7fe43701cd3ac467460bc5a8ee1f4c82c032a5d0ac16e666a565212fba209
7
- data.tar.gz: 60e3338c3021ea2e1a54dcfb7f2a15c9ffce71aed5a12db7739992649cfd9518f19edc49262a0095ba09c8783e0ef499bcb3015aebca0d2b7751e7096cbf88eb
6
+ metadata.gz: 3289530a2e874d2b79b0da66ea787e30b335b7a60626e4b67f1be9919067d4b79752db4e878ca8851dbce6c68a1e59a573bcbc60314bb7a35268cd37cc2eeaae
7
+ data.tar.gz: 2758348f253992e08fbf589d2b5444844becf69e179872c630804a7e8885568189168edd056c56beabe4ed5dc3e0905f46c51595896bd7bdd1255fbe4ece71ec
data/lib/etna.rb CHANGED
@@ -21,6 +21,7 @@ require_relative './etna/environment_scoped'
21
21
  require_relative './etna/filesystem'
22
22
  require_relative './etna/formatting'
23
23
  require_relative './etna/cwl'
24
+ require_relative './etna/metrics'
24
25
 
25
26
  class EtnaApp
26
27
  include Etna::Application
@@ -8,6 +8,7 @@ require_relative './command'
8
8
  require_relative './generate_autocompletion_script'
9
9
  require 'singleton'
10
10
  require 'rollbar'
11
+ require 'fileutils'
11
12
 
12
13
  module Etna::Application
13
14
  def self.included(other)
@@ -31,6 +32,12 @@ module Etna::Application
31
32
  raise "Could not find application instance from #{namespace}, and not subclass of Application found."
32
33
  end
33
34
 
35
+ # Used to find the application in development recorded vcr tests.
36
+ # see spec/vcr.rb
37
+ def dev_route
38
+ "#{self.class.name.split('::').first.downcase}.development.local"
39
+ end
40
+
34
41
  def self.register(app)
35
42
  @instance = app
36
43
  end
@@ -53,6 +60,72 @@ module Etna::Application
53
60
  end
54
61
  end
55
62
 
63
+ # This will cause metrics to persist to a file.
64
+ # NOTE -- /tmp/metrics.bin should be a persistent mount when using this.
65
+ # You will still need to export metrics in the text format for the node_exporter on the host machine to
66
+ # export them to prometheus. Ensure that the /tmp/metrics.bin is on a named volume or a bind mount, either is fine.
67
+ def enable_job_metrics!
68
+ require 'prometheus'
69
+ Prometheus::Client.config.data_store = Prometheus::Client::DataStores::DirectFileStore.new({
70
+ dir: "/tmp/metrics.bin"
71
+ })
72
+ end
73
+
74
+ def setup_yabeda
75
+ application = self.id
76
+ Yabeda.configure do
77
+ default_tag :application, application
78
+
79
+ group :etna do
80
+ histogram :response_time do
81
+ comment "Time spent by a controller returning a response"
82
+ unit :seconds
83
+ tags [:controller, :action, :user_hash, :project_name]
84
+ buckets [0.01, 0.1, 0.3, 0.5, 1, 5]
85
+ end
86
+
87
+ counter :visits do
88
+ comment "Counts visits to the controller"
89
+ tags [:controller, :action, :user_hash, :project_name]
90
+ end
91
+
92
+ counter :rollbar_errors do
93
+ comment "Counts errors detected by and sent to rollbar"
94
+ end
95
+
96
+ gauge :last_command_completion do
97
+ comment "Unix time of last time command was completed"
98
+ tags [:command, :status, :application]
99
+ end
100
+
101
+ histogram :command_runtime do
102
+ comment "Time spent processing a given command"
103
+ tags [:command, :status, :application]
104
+ unit :seconds
105
+ buckets [0.1, 1, 5, 60, 300, 1500]
106
+ end
107
+ end
108
+ end
109
+
110
+ Yabeda.configure!
111
+ end
112
+
113
+ # Writes all metrics currently gathered to a text format prometheus file. If /tmp/metrics.prom is bind mounted
114
+ # to the host directed bound to the node_exporter's file exporter directory, these will be exported to
115
+ # prometheus. Combine this enable_job_metrics! for maximum effect.
116
+ def write_job_metrics(name)
117
+ node_metrics_dir = "/tmp/metrics.prom"
118
+ ::FileUtils.mkdir_p(node_metrics_dir)
119
+
120
+ tmp_file = ::File.join(node_metrics_dir, "#{name}.prom.$$")
121
+ ::File.open(tmp_file, "w") do |f|
122
+ f.write(Prometheus::Client::Formats::Text.marshal(Prometheus::Client.registry))
123
+ end
124
+
125
+ require 'fileutils'
126
+ ::FileUtils.mv(tmp_file, ::File.join(node_metrics_dir, "#{name}.prom"))
127
+ end
128
+
56
129
  def setup_logger
57
130
  @logger = Etna::Logger.new(
58
131
  # The name of the log_file, required.
@@ -102,17 +175,32 @@ module Etna::Application
102
175
  end
103
176
 
104
177
  def run_command(config, *args, &block)
178
+ application = self.id
179
+ status = 'success'
180
+ start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
105
181
  cmd, cmd_args, cmd_kwds = find_command(*args)
106
- cmd.setup(config)
107
182
 
108
- if block_given?
109
- return unless yield [cmd, cmd_args]
110
- end
183
+ begin
184
+ cmd.setup(config)
185
+ if block_given?
186
+ return unless yield [cmd, cmd_args]
187
+ end
111
188
 
112
- cmd.execute(*cmd.fill_in_missing_params(cmd_args), **cmd_kwds)
113
- rescue => e
114
- Rollbar.error(e)
115
- raise
189
+ cmd.execute(*cmd.fill_in_missing_params(cmd_args), **cmd_kwds)
190
+ rescue => e
191
+ Rollbar.error(e)
192
+ status = 'failed'
193
+ raise
194
+ ensure
195
+ if Yabeda.configured?
196
+ tags = { command: cmd.class.name, status: status, application: application }
197
+ dur = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
198
+
199
+ Yabeda.etna.last_command_completion.set(tags, Time.now.to_i)
200
+ Yabeda.etna.command_runtime.measure(tags, dur)
201
+ write_job_metrics("run_command")
202
+ end
203
+ end
116
204
  end
117
205
  end
118
206
 
data/lib/etna/auth.rb CHANGED
@@ -86,7 +86,7 @@ module Etna
86
86
  host: application.config(:janus)[:host]
87
87
  )
88
88
 
89
- response = janus_client.validate_task_token()
89
+ response = janus_client.validate_task_token
90
90
 
91
91
  return false unless response.code == '200'
92
92
 
data/lib/etna/client.rb CHANGED
@@ -173,7 +173,7 @@ module Etna
173
173
  verify_mode = @ignore_ssl ?
174
174
  OpenSSL::SSL::VERIFY_NONE :
175
175
  OpenSSL::SSL::VERIFY_PEER
176
- Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode) do |http|
176
+ Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode, read_timeout: 300) do |http|
177
177
  http.request(data) do |response|
178
178
  status_check!(response)
179
179
  yield response
@@ -183,7 +183,7 @@ module Etna
183
183
  verify_mode = @ignore_ssl ?
184
184
  OpenSSL::SSL::VERIFY_NONE :
185
185
  OpenSSL::SSL::VERIFY_PEER
186
- Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode) do |http|
186
+ Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode, read_timeout: 300) do |http|
187
187
  response = http.request(data)
188
188
  status_check!(response)
189
189
  return response
@@ -29,7 +29,8 @@ module Etna
29
29
  def token_will_expire?(offset=3000)
30
30
  # offset in seconds
31
31
  # Will the user's token expire in the given amount of time?
32
- epoch_seconds = JSON.parse(Base64.urlsafe_decode64(token.split('.')[1]))["exp"]
32
+ payload = JSON.parse(Base64.urlsafe_decode64(token.split('.')[1]))
33
+ epoch_seconds = payload["exp"]
33
34
  expiration = DateTime.strptime(epoch_seconds.to_s, "%s")
34
35
  expiration <= DateTime.now.new_offset + offset
35
36
  end
@@ -58,20 +58,19 @@ module Etna
58
58
  TokenResponse.new(token)
59
59
  end
60
60
 
61
- def validate_task_token(validate_task_token_request = ValidateTaskTokenRequest.new)
62
- token = nil
63
- @etna_client.post('/api/tokens/task/validate', validate_task_token_request)
61
+ def validate_task_token
62
+ @etna_client.post('/api/tokens/validate_task')
64
63
  end
65
64
 
66
65
  def get_nonce
67
66
  @etna_client.get('/api/tokens/nonce').body
68
67
  end
69
68
 
70
- def generate_token(token_type, signed_nonce: nil, project_name: nil)
69
+ def generate_token(token_type, signed_nonce: nil, project_name: nil, read_only: false)
71
70
  response = @etna_client.with_headers(
72
71
  'Authorization' => signed_nonce ? "Signed-Nonce #{signed_nonce}" : nil
73
72
  ) do
74
- post('/api/tokens/generate', token_type: token_type, project_name: project_name)
73
+ post('/api/tokens/generate', token_type: token_type, project_name: project_name, read_only: read_only)
75
74
  end
76
75
 
77
76
  response.body
@@ -228,14 +228,44 @@ module Etna
228
228
  end
229
229
 
230
230
  folder_contents.files.all.each do |file|
231
- rename_file(Etna::Clients::Metis::RenameFileRequest.new(
232
- bucket_name: source_bucket,
231
+ # If file exists in destination, delete the older file.
232
+ list_dest_folder_request = Etna::Clients::Metis::ListFolderRequest.new(
233
+ bucket_name: dest_bucket,
233
234
  project_name: project_name,
234
- file_path: file.file_path,
235
- new_bucket_name: dest_bucket,
236
- new_file_path: file.file_path,
237
- create_parent: true)
235
+ folder_path: ::File.dirname(file.file_path)
238
236
  )
237
+
238
+ dest_file = list_folder(list_dest_folder_request).files.all.find { |f| f.file_name == file.file_name }
239
+
240
+ should_rename = true
241
+ if (dest_file && file.updated_at <= dest_file.updated_at)
242
+ # Delete source file if it's out of date
243
+ delete_file(Etna::Clients::Metis::DeleteFileRequest.new(
244
+ bucket_name: source_bucket,
245
+ project_name: project_name,
246
+ file_path: file.file_path,
247
+ ))
248
+
249
+ should_rename = false
250
+ elsif (dest_file && file.updated_at > dest_file.updated_at)
251
+ # Delete dest file if it's out of date
252
+ delete_file(Etna::Clients::Metis::DeleteFileRequest.new(
253
+ bucket_name: dest_bucket,
254
+ project_name: project_name,
255
+ file_path: dest_file.file_path,
256
+ ))
257
+ end
258
+
259
+ if should_rename
260
+ rename_file(Etna::Clients::Metis::RenameFileRequest.new(
261
+ bucket_name: source_bucket,
262
+ project_name: project_name,
263
+ file_path: file.file_path,
264
+ new_bucket_name: dest_bucket,
265
+ new_file_path: file.file_path,
266
+ create_parent: true)
267
+ )
268
+ end
239
269
  end
240
270
 
241
271
  # Now delete the source folder
@@ -1,3 +1,4 @@
1
- require_relative './workflows/metis_download_workflow'
2
- require_relative './workflows/metis_upload_workflow'
3
- require_relative './workflows/sync_metis_data_workflow'
1
+ require_relative "./workflows/metis_download_workflow"
2
+ require_relative "./workflows/metis_upload_workflow"
3
+ require_relative "./workflows/sync_metis_data_workflow"
4
+ require_relative "./workflows/ingest_metis_data_workflow"
@@ -0,0 +1,31 @@
1
+ require "ostruct"
2
+ require "fileutils"
3
+ require "tempfile"
4
+
5
+ module Etna
6
+ module Clients
7
+ class Metis
8
+ class IngestMetisDataWorkflow < Struct.new(:metis_filesystem, :ingest_filesystem, :logger, keyword_init: true)
9
+ # Since we are doing manual triage of files,
10
+ # do not automatically copy directory trees.
11
+ # srcs must be a list of full paths to files.
12
+ def copy_files(srcs)
13
+ srcs.each do |src|
14
+ next unless ingest_filesystem.exist?(src)
15
+
16
+ logger&.info("Copying file #{src} (#{Etna::Formatting.as_size(ingest_filesystem.stat(src).size)})")
17
+
18
+ # For ingestion triage, just copy over the exact path + filename.
19
+ copy_file(dest: src, src: src)
20
+ end
21
+ end
22
+
23
+ def copy_file(dest:, src:)
24
+ ingest_filesystem.with_readable(src, "r") do |file|
25
+ metis_filesystem.do_streaming_upload(file, dest, file.size)
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
@@ -30,7 +30,7 @@ module Etna
30
30
  project_name: project_name,
31
31
  bucket_name: bucket_name,
32
32
  folder_path: dir,
33
- ))
33
+ )) unless dir == "."
34
34
 
35
35
  authorize_response = metis_client.authorize_upload(AuthorizeUploadRequest.new(
36
36
  project_name: project_name,
@@ -19,25 +19,30 @@ module Etna
19
19
  @logger.warn(request_msg(line))
20
20
  end
21
21
 
22
+ def handle_error(e)
23
+ case e
24
+ when Etna::Error
25
+ Rollbar.error(e)
26
+ @logger.error(request_msg("Exiting with #{e.status}, #{e.message}"))
27
+ return failure(e.status, error: e.message)
28
+ else
29
+ Rollbar.error(e)
30
+ @logger.error(request_msg('Caught unspecified error'))
31
+ @logger.error(request_msg(e.message))
32
+ e.backtrace.each do |trace|
33
+ @logger.error(request_msg(trace))
34
+ end
35
+ return failure(500, error: 'Server error.')
36
+ end
37
+ end
38
+
22
39
  def response(&block)
23
40
  return instance_eval(&block) if block_given?
24
-
25
41
  return send(@action) if @action
26
42
 
27
-
28
43
  [501, {}, ['This controller is not implemented.']]
29
- rescue Etna::Error => e
30
- Rollbar.error(e)
31
- @logger.error(request_msg("Exiting with #{e.status}, #{e.message}"))
32
- return failure(e.status, error: e.message)
33
44
  rescue Exception => e
34
- Rollbar.error(e)
35
- @logger.error(request_msg('Caught unspecified error'))
36
- @logger.error(request_msg(e.message))
37
- e.backtrace.each do |trace|
38
- @logger.error(request_msg(trace))
39
- end
40
- return failure(500, error: 'Server error.')
45
+ handle_error(e)
41
46
  end
42
47
 
43
48
  def require_params(*params)
@@ -52,7 +52,12 @@ class DirectedGraph
52
52
  result[grandparent] << child_node if result.include?(grandparent)
53
53
  end
54
54
 
55
- result[child_node] = []
55
+ # Depending on the graph shape, diamonds could lead to
56
+ # resetting of previously calculated dependencies.
57
+ # Here we avoid resetting existing entries in `result`
58
+ # and instead concatenate them if they already exist.
59
+ result[child_node] = [] unless result.include?(child_node)
60
+ result[n].concat(result[child_node]) if result.include?(child_node) && result.include?(n)
56
61
  end
57
62
  end
58
63
 
@@ -3,6 +3,8 @@ require 'fileutils'
3
3
  require 'open3'
4
4
  require 'securerandom'
5
5
  require 'concurrent-ruby'
6
+ require 'net/sftp'
7
+ require 'net/ssh'
6
8
 
7
9
  module Etna
8
10
  # A class that encapsulates opening / reading file system entries that abstracts normal file access in order
@@ -50,6 +52,11 @@ module Etna
50
52
  ::FileUtils.mv(src, dest)
51
53
  end
52
54
 
55
+ def stat(src)
56
+ raise "stat not supported by #{self.class.name}" unless self.class == Filesystem
57
+ ::File.stat(src)
58
+ end
59
+
53
60
  class EmptyIO < StringIO
54
61
  def write(*args)
55
62
  # Do nothing -- always leave empty
@@ -369,7 +376,59 @@ module Etna
369
376
  end
370
377
  end
371
378
 
379
+ class SftpFilesystem < Filesystem
380
+ def initialize(host:, username:, password: nil, port: 22, **args)
381
+ @username = username
382
+ @password = password
383
+ @host = host
384
+ @port = port
385
+ end
386
+
387
+ def ssh
388
+ @ssh ||= Net::SSH.start(@host, @username, password: @password)
389
+ end
390
+
391
+ def sftp
392
+ @sftp ||= begin
393
+ conn = Net::SFTP::Session.new(ssh)
394
+ conn.loop { conn.opening? }
395
+
396
+ conn
397
+ end
398
+ end
399
+
400
+ def with_readable(src, opts = 'r', &block)
401
+ sftp.file.open(src, opts, &block)
402
+ end
403
+
404
+ def ls(dir)
405
+ sftp.dir.entries(dir)
406
+ end
407
+
408
+ def exist?(src)
409
+ begin
410
+ sftp.file.open(src)
411
+ rescue Net::SFTP::StatusException
412
+ return false
413
+ end
414
+ return true
415
+ end
416
+
417
+ def stat(src)
418
+ sftp.file.open(src).stat
419
+ end
420
+ end
421
+
372
422
  class Mock < Filesystem
423
+ class MockStat
424
+ def initialize
425
+ end
426
+
427
+ def size
428
+ 0
429
+ end
430
+ end
431
+
373
432
  def initialize(&new_io)
374
433
  @files = {}
375
434
  @dirs = {}
@@ -438,6 +497,10 @@ module Etna
438
497
  def exist?(src)
439
498
  @files.include?(src) || @dirs.include?(src)
440
499
  end
500
+
501
+ def stat(src)
502
+ @files[src].respond_to?(:stat) ? @files[src].stat : MockStat.new
503
+ end
441
504
  end
442
505
  end
443
506
  end
data/lib/etna/logger.rb CHANGED
@@ -40,6 +40,10 @@ module Etna
40
40
  error(trace)
41
41
  end
42
42
 
43
+ if defined? Yabeda
44
+ Yabeda.etna.rollbar_errors.increment({}, 1) rescue nil
45
+ end
46
+
43
47
  Rollbar.error(e)
44
48
  end
45
49
 
@@ -0,0 +1,26 @@
1
+
2
+ module Etna
3
+ class MetricsExporter
4
+ def initialize(app, path: '/metrics')
5
+ @app = app
6
+ @path = path
7
+ end
8
+
9
+ def exporter
10
+ @exporter ||= begin
11
+ exporter = Yabeda::Prometheus::Exporter.new(@app, path: @path)
12
+ Rack::Auth::Basic.new(exporter) do |user, pw|
13
+ user == 'prometheus' && pw == ENV['METRICS_PW']
14
+ end
15
+ end
16
+ end
17
+
18
+ def call(env)
19
+ if env['PATH_INFO'] == @path
20
+ exporter.call(env)
21
+ else
22
+ @app.call(env)
23
+ end
24
+ end
25
+ end
26
+ end
data/lib/etna/route.rb CHANGED
@@ -1,3 +1,6 @@
1
+ require 'digest'
2
+ require 'date'
3
+
1
4
  module Etna
2
5
  class Route
3
6
  attr_reader :name
@@ -58,6 +61,63 @@ module Etna
58
61
  end
59
62
 
60
63
  def call(app, request)
64
+ start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
65
+
66
+ try_yabeda(request) do |tags|
67
+ Yabeda.etna.visits.increment(tags)
68
+ end
69
+
70
+ begin
71
+ process_call(app, request)
72
+ ensure
73
+ try_yabeda(request) do |tags|
74
+ dur = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
75
+ Yabeda.etna.response_time.measure(tags, dur)
76
+ end
77
+ end
78
+ end
79
+
80
+ def hash_user_email(email)
81
+ secret = Etna::Application.instance.config(:user_hash_secret) || 'notsosecret'
82
+ digest = email + secret + Date.today.to_s
83
+
84
+ if @name
85
+ digest += @name.to_s
86
+ else
87
+ digest += @route.to_s
88
+ end
89
+
90
+ Digest::MD5.hexdigest(digest)
91
+ end
92
+
93
+ def try_yabeda(request, &block)
94
+ if @action
95
+ controller, action = @action.split('#')
96
+ elsif @name
97
+ controller = "none"
98
+ action = @name
99
+ else
100
+ controller = "none"
101
+ action = @route
102
+ end
103
+
104
+ params = request.env['rack.request.params']
105
+ user = request.env['etna.user']
106
+ user_hash = user ? hash_user_email(user.email) : 'unknown'
107
+ project_name = "unknown"
108
+
109
+ if params && (params.include?(:project_name) || params.include?('project_name'))
110
+ project_name = params[:project_name] || params['project_name']
111
+ end
112
+
113
+ begin
114
+ block.call({ controller: controller, action: action, user_hash: user_hash, project_name: project_name })
115
+ rescue => e
116
+ raise e unless Etna::Application.instance.environment == :production
117
+ end
118
+ end
119
+
120
+ def process_call(app, request)
61
121
  update_params(request)
62
122
 
63
123
  unless authorized?(request)
@@ -148,7 +208,10 @@ module Etna
148
208
  params = request.env['rack.request.params']
149
209
 
150
210
  @auth[:user].all? do |constraint, param_name|
151
- user.respond_to?(constraint) && user.send(constraint, params[param_name])
211
+ user.respond_to?(constraint) && (
212
+ param_name.is_a?(Symbol) ?
213
+ user.send(constraint, params[param_name]) :
214
+ user.send(constraint, param_name))
152
215
  end
153
216
  end
154
217
 
data/lib/etna/server.rb CHANGED
@@ -74,6 +74,11 @@ module Etna
74
74
  def initialize
75
75
  # Setup logging.
76
76
  application.setup_logger
77
+
78
+ # This needs to be required before yabeda invocation, but cannot belong at the top of any module since clients
79
+ # do not install yabeda.
80
+ require 'yabeda'
81
+ application.setup_yabeda
77
82
  end
78
83
 
79
84
  private
data/lib/etna/spec/vcr.rb CHANGED
@@ -4,13 +4,32 @@ require 'openssl'
4
4
  require 'digest/sha2'
5
5
  require 'base64'
6
6
 
7
- def setup_base_vcr(spec_helper_dir)
7
+ def setup_base_vcr(spec_helper_dir, server: nil, application: nil)
8
8
  VCR.configure do |c|
9
9
  c.hook_into :webmock
10
10
  c.cassette_serializers
11
11
  c.cassette_library_dir = ::File.join(spec_helper_dir, 'fixtures', 'cassettes')
12
12
  c.allow_http_connections_when_no_cassette = true
13
13
 
14
+ c.register_request_matcher :verify_uri_route do |request_1, request_2|
15
+ next true if server.nil? || application.nil?
16
+
17
+ route_match = request_1.uri =~ /https:\/\/#{application.dev_route}(.*)/
18
+ if route_match && route_match[1]
19
+ def request_1.request_method
20
+ method.to_s.upcase
21
+ end
22
+
23
+ def request_1.path
24
+ URI(uri).path
25
+ end
26
+
27
+ !!server.find_route(request_1)
28
+ else
29
+ true
30
+ end
31
+ end
32
+
14
33
  c.register_request_matcher :try_body do |request_1, request_2|
15
34
  if request_1.headers['Content-Type'].first =~ /application\/json/
16
35
  if request_2.headers['Content-Type'].first =~ /application\/json/
@@ -39,7 +58,7 @@ def setup_base_vcr(spec_helper_dir)
39
58
  else
40
59
  ENV['RERECORD'] ? :all : :once
41
60
  end,
42
- match_requests_on: [:method, :uri, :try_body]
61
+ match_requests_on: [:method, :uri, :try_body, :verify_uri_route]
43
62
  }
44
63
 
45
64
  # Filter the authorization headers of any request by replacing any occurrence of that request's
@@ -42,7 +42,6 @@ module Etna
42
42
  # We do this to support Metis client tests, we pass in tokens with multiple "."-separated parts, so
43
43
  # have to account for that.
44
44
  payload = JSON.parse(Base64.decode64(token.split('.')[1]))
45
-
46
45
  request.env['etna.user'] = Etna::User.new(payload.map{|k,v| [k.to_sym, v]}.to_h, token)
47
46
  end
48
47
 
data/lib/etna/user.rb CHANGED
@@ -7,7 +7,7 @@ module Etna
7
7
  }
8
8
 
9
9
  def initialize params, token=nil
10
- @name, @email, @encoded_permissions, encoded_flags = params.values_at(:name, :email, :perm, :flags)
10
+ @name, @email, @encoded_permissions, encoded_flags, @task = params.values_at(:name, :email, :perm, :flags, :task)
11
11
 
12
12
  @flags = encoded_flags&.split(/;/) || []
13
13
  @token = token unless !token
@@ -16,6 +16,10 @@ module Etna
16
16
 
17
17
  attr_reader :name, :email, :token
18
18
 
19
+ def task?
20
+ !!@task
21
+ end
22
+
19
23
  def permissions
20
24
  @permissions ||= @encoded_permissions.split(/\;/).map do |roles|
21
25
  role, projects = roles.split(/:/)
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: etna
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.35
4
+ version: 0.1.39
5
5
  platform: ruby
6
6
  authors:
7
7
  - Saurabh Asthana
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-04-26 00:00:00.000000000 Z
11
+ date: 2021-07-22 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rack
@@ -94,6 +94,20 @@ dependencies:
94
94
  - - ">="
95
95
  - !ruby/object:Gem::Version
96
96
  version: '0'
97
+ - !ruby/object:Gem::Dependency
98
+ name: net-sftp
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: 3.0.0
104
+ type: :runtime
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: 3.0.0
97
111
  description: See summary
98
112
  email: Saurabh.Asthana@ucsf.edu
99
113
  executables:
@@ -144,6 +158,7 @@ files:
144
158
  - lib/etna/clients/metis/client.rb
145
159
  - lib/etna/clients/metis/models.rb
146
160
  - lib/etna/clients/metis/workflows.rb
161
+ - lib/etna/clients/metis/workflows/ingest_metis_data_workflow.rb
147
162
  - lib/etna/clients/metis/workflows/metis_download_workflow.rb
148
163
  - lib/etna/clients/metis/workflows/metis_upload_workflow.rb
149
164
  - lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb
@@ -168,6 +183,7 @@ files:
168
183
  - lib/etna/hmac.rb
169
184
  - lib/etna/json_serializable_struct.rb
170
185
  - lib/etna/logger.rb
186
+ - lib/etna/metrics.rb
171
187
  - lib/etna/multipart_serializable_nested_hash.rb
172
188
  - lib/etna/parse_body.rb
173
189
  - lib/etna/route.rb