etna 0.1.34 → 0.1.38
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/commands.rb +1 -1
- data/lib/etna.rb +1 -0
- data/lib/etna/application.rb +96 -8
- data/lib/etna/auth.rb +1 -1
- data/lib/etna/client.rb +2 -2
- data/lib/etna/clients/base_client.rb +2 -1
- data/lib/etna/clients/janus/client.rb +4 -5
- data/lib/etna/clients/metis/workflows.rb +4 -3
- data/lib/etna/clients/metis/workflows/ingest_metis_data_workflow.rb +31 -0
- data/lib/etna/clients/metis/workflows/metis_upload_workflow.rb +1 -1
- data/lib/etna/controller.rb +18 -13
- data/lib/etna/directed_graph.rb +6 -1
- data/lib/etna/filesystem.rb +63 -0
- data/lib/etna/logger.rb +4 -0
- data/lib/etna/metrics.rb +26 -0
- data/lib/etna/route.rb +64 -1
- data/lib/etna/server.rb +5 -0
- data/lib/etna/spec/vcr.rb +21 -2
- data/lib/etna/test_auth.rb +0 -1
- data/lib/etna/user.rb +5 -1
- metadata +18 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e7c665ee736c06bf1b566467985638cd41a5b3536f848fe2971174aab26ba3a4
|
4
|
+
data.tar.gz: 22bd58b6dd1a4902ee91f6588f53cf56a4bec9db5b42d9236ed9cd7c4310a2f0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 04ad878b6f9593a06b65b7652ba3b6da9b0a2aa4de57981ef91247b19eb606e5018cc01e45d033ccdaf5b755cb92149e92a1d26218c47fce6c1a77085dce300e
|
7
|
+
data.tar.gz: 26134c1553f33e59e13f5ec737bd11b0bd8e62e1f54c3b1c5963560d1078cb00d5ce0255c01bddae86dc19f9bb38ae5a0dfc6b9d2845e955e55df0299ad1f456
|
data/lib/commands.rb
CHANGED
@@ -339,7 +339,7 @@ class EtnaApp
|
|
339
339
|
request = Etna::Clients::Magma::RetrievalRequest.new(project_name: project_name)
|
340
340
|
request.model_name = model_name
|
341
341
|
request.attribute_names = 'all'
|
342
|
-
request.record_names =
|
342
|
+
request.record_names = []
|
343
343
|
model = magma_client.retrieve(request).models.model(model_name)
|
344
344
|
model_parent_name = model.template.attributes.all.select do |attribute|
|
345
345
|
attribute.attribute_type == Etna::Clients::Magma::AttributeType::PARENT
|
data/lib/etna.rb
CHANGED
data/lib/etna/application.rb
CHANGED
@@ -8,6 +8,7 @@ require_relative './command'
|
|
8
8
|
require_relative './generate_autocompletion_script'
|
9
9
|
require 'singleton'
|
10
10
|
require 'rollbar'
|
11
|
+
require 'fileutils'
|
11
12
|
|
12
13
|
module Etna::Application
|
13
14
|
def self.included(other)
|
@@ -31,6 +32,12 @@ module Etna::Application
|
|
31
32
|
raise "Could not find application instance from #{namespace}, and not subclass of Application found."
|
32
33
|
end
|
33
34
|
|
35
|
+
# Used to find the application in development recorded vcr tests.
|
36
|
+
# see spec/vcr.rb
|
37
|
+
def dev_route
|
38
|
+
"#{self.class.name.split('::').first.downcase}.development.local"
|
39
|
+
end
|
40
|
+
|
34
41
|
def self.register(app)
|
35
42
|
@instance = app
|
36
43
|
end
|
@@ -53,6 +60,72 @@ module Etna::Application
|
|
53
60
|
end
|
54
61
|
end
|
55
62
|
|
63
|
+
# This will cause metrics to persist to a file.
|
64
|
+
# NOTE -- /tmp/metrics.bin should be a persistent mount when using this.
|
65
|
+
# You will still need to export metrics in the text format for the node_exporter on the host machine to
|
66
|
+
# export them to prometheus. Ensure that the /tmp/metrics.bin is on a named volume or a bind mount, either is fine.
|
67
|
+
def enable_job_metrics!
|
68
|
+
require 'prometheus'
|
69
|
+
Prometheus::Client.config.data_store = Prometheus::Client::DataStores::DirectFileStore.new({
|
70
|
+
dir: "/tmp/metrics.bin"
|
71
|
+
})
|
72
|
+
end
|
73
|
+
|
74
|
+
def setup_yabeda
|
75
|
+
application = self.id
|
76
|
+
Yabeda.configure do
|
77
|
+
default_tag :application, application
|
78
|
+
|
79
|
+
group :etna do
|
80
|
+
histogram :response_time do
|
81
|
+
comment "Time spent by a controller returning a response"
|
82
|
+
unit :seconds
|
83
|
+
tags [:controller, :action, :user_hash, :project_name]
|
84
|
+
buckets [0.01, 0.1, 0.3, 0.5, 1, 5]
|
85
|
+
end
|
86
|
+
|
87
|
+
counter :visits do
|
88
|
+
comment "Counts visits to the controller"
|
89
|
+
tags [:controller, :action, :user_hash, :project_name]
|
90
|
+
end
|
91
|
+
|
92
|
+
counter :rollbar_errors do
|
93
|
+
comment "Counts errors detected by and sent to rollbar"
|
94
|
+
end
|
95
|
+
|
96
|
+
gauge :last_command_completion do
|
97
|
+
comment "Unix time of last time command was completed"
|
98
|
+
tags [:command, :status, :application]
|
99
|
+
end
|
100
|
+
|
101
|
+
histogram :command_runtime do
|
102
|
+
comment "Time spent processing a given command"
|
103
|
+
tags [:command, :status, :application]
|
104
|
+
unit :seconds
|
105
|
+
buckets [0.1, 1, 5, 60, 300, 1500]
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
Yabeda.configure!
|
111
|
+
end
|
112
|
+
|
113
|
+
# Writes all metrics currently gathered to a text format prometheus file. If /tmp/metrics.prom is bind mounted
|
114
|
+
# to the host directed bound to the node_exporter's file exporter directory, these will be exported to
|
115
|
+
# prometheus. Combine this enable_job_metrics! for maximum effect.
|
116
|
+
def write_job_metrics(name)
|
117
|
+
node_metrics_dir = "/tmp/metrics.prom"
|
118
|
+
::FileUtils.mkdir_p(node_metrics_dir)
|
119
|
+
|
120
|
+
tmp_file = ::File.join(node_metrics_dir, "#{name}.prom.$$")
|
121
|
+
::File.open(tmp_file, "w") do |f|
|
122
|
+
f.write(Prometheus::Client::Formats::Text.marshal(Prometheus::Client.registry))
|
123
|
+
end
|
124
|
+
|
125
|
+
require 'fileutils'
|
126
|
+
::FileUtils.mv(tmp_file, ::File.join(node_metrics_dir, "#{name}.prom"))
|
127
|
+
end
|
128
|
+
|
56
129
|
def setup_logger
|
57
130
|
@logger = Etna::Logger.new(
|
58
131
|
# The name of the log_file, required.
|
@@ -102,17 +175,32 @@ module Etna::Application
|
|
102
175
|
end
|
103
176
|
|
104
177
|
def run_command(config, *args, &block)
|
178
|
+
application = self.id
|
179
|
+
status = 'success'
|
180
|
+
start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
105
181
|
cmd, cmd_args, cmd_kwds = find_command(*args)
|
106
|
-
cmd.setup(config)
|
107
182
|
|
108
|
-
|
109
|
-
|
110
|
-
|
183
|
+
begin
|
184
|
+
cmd.setup(config)
|
185
|
+
if block_given?
|
186
|
+
return unless yield [cmd, cmd_args]
|
187
|
+
end
|
111
188
|
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
189
|
+
cmd.execute(*cmd.fill_in_missing_params(cmd_args), **cmd_kwds)
|
190
|
+
rescue => e
|
191
|
+
Rollbar.error(e)
|
192
|
+
status = 'failed'
|
193
|
+
raise
|
194
|
+
ensure
|
195
|
+
if Yabeda.configured?
|
196
|
+
tags = { command: cmd.class.name, status: status, application: application }
|
197
|
+
dur = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
|
198
|
+
|
199
|
+
Yabeda.etna.last_command_completion.set(tags, Time.now.to_i)
|
200
|
+
Yabeda.etna.command_runtime.measure(tags, dur)
|
201
|
+
write_job_metrics("run_command")
|
202
|
+
end
|
203
|
+
end
|
116
204
|
end
|
117
205
|
end
|
118
206
|
|
data/lib/etna/auth.rb
CHANGED
data/lib/etna/client.rb
CHANGED
@@ -173,7 +173,7 @@ module Etna
|
|
173
173
|
verify_mode = @ignore_ssl ?
|
174
174
|
OpenSSL::SSL::VERIFY_NONE :
|
175
175
|
OpenSSL::SSL::VERIFY_PEER
|
176
|
-
Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode) do |http|
|
176
|
+
Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode, read_timeout: 300) do |http|
|
177
177
|
http.request(data) do |response|
|
178
178
|
status_check!(response)
|
179
179
|
yield response
|
@@ -183,7 +183,7 @@ module Etna
|
|
183
183
|
verify_mode = @ignore_ssl ?
|
184
184
|
OpenSSL::SSL::VERIFY_NONE :
|
185
185
|
OpenSSL::SSL::VERIFY_PEER
|
186
|
-
Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode) do |http|
|
186
|
+
Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode, read_timeout: 300) do |http|
|
187
187
|
response = http.request(data)
|
188
188
|
status_check!(response)
|
189
189
|
return response
|
@@ -29,7 +29,8 @@ module Etna
|
|
29
29
|
def token_will_expire?(offset=3000)
|
30
30
|
# offset in seconds
|
31
31
|
# Will the user's token expire in the given amount of time?
|
32
|
-
|
32
|
+
payload = JSON.parse(Base64.urlsafe_decode64(token.split('.')[1]))
|
33
|
+
epoch_seconds = payload["exp"]
|
33
34
|
expiration = DateTime.strptime(epoch_seconds.to_s, "%s")
|
34
35
|
expiration <= DateTime.now.new_offset + offset
|
35
36
|
end
|
@@ -58,20 +58,19 @@ module Etna
|
|
58
58
|
TokenResponse.new(token)
|
59
59
|
end
|
60
60
|
|
61
|
-
def validate_task_token
|
62
|
-
|
63
|
-
@etna_client.post('/api/tokens/task/validate', validate_task_token_request)
|
61
|
+
def validate_task_token
|
62
|
+
@etna_client.post('/api/tokens/validate_task')
|
64
63
|
end
|
65
64
|
|
66
65
|
def get_nonce
|
67
66
|
@etna_client.get('/api/tokens/nonce').body
|
68
67
|
end
|
69
68
|
|
70
|
-
def generate_token(token_type, signed_nonce: nil, project_name: nil)
|
69
|
+
def generate_token(token_type, signed_nonce: nil, project_name: nil, read_only: false)
|
71
70
|
response = @etna_client.with_headers(
|
72
71
|
'Authorization' => signed_nonce ? "Signed-Nonce #{signed_nonce}" : nil
|
73
72
|
) do
|
74
|
-
post('/api/tokens/generate', token_type: token_type, project_name: project_name)
|
73
|
+
post('/api/tokens/generate', token_type: token_type, project_name: project_name, read_only: read_only)
|
75
74
|
end
|
76
75
|
|
77
76
|
response.body
|
@@ -1,3 +1,4 @@
|
|
1
|
-
require_relative
|
2
|
-
require_relative
|
3
|
-
require_relative
|
1
|
+
require_relative "./workflows/metis_download_workflow"
|
2
|
+
require_relative "./workflows/metis_upload_workflow"
|
3
|
+
require_relative "./workflows/sync_metis_data_workflow"
|
4
|
+
require_relative "./workflows/ingest_metis_data_workflow"
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require "ostruct"
|
2
|
+
require "fileutils"
|
3
|
+
require "tempfile"
|
4
|
+
|
5
|
+
module Etna
|
6
|
+
module Clients
|
7
|
+
class Metis
|
8
|
+
class IngestMetisDataWorkflow < Struct.new(:metis_filesystem, :ingest_filesystem, :logger, keyword_init: true)
|
9
|
+
# Since we are doing manual triage of files,
|
10
|
+
# do not automatically copy directory trees.
|
11
|
+
# srcs must be a list of full paths to files.
|
12
|
+
def copy_files(srcs)
|
13
|
+
srcs.each do |src|
|
14
|
+
next unless ingest_filesystem.exist?(src)
|
15
|
+
|
16
|
+
logger&.info("Copying file #{src} (#{Etna::Formatting.as_size(ingest_filesystem.stat(src).size)})")
|
17
|
+
|
18
|
+
# For ingestion triage, just copy over the exact path + filename.
|
19
|
+
copy_file(dest: src, src: src)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def copy_file(dest:, src:)
|
24
|
+
ingest_filesystem.with_readable(src, "r") do |file|
|
25
|
+
metis_filesystem.do_streaming_upload(file, dest, file.size)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
data/lib/etna/controller.rb
CHANGED
@@ -19,25 +19,30 @@ module Etna
|
|
19
19
|
@logger.warn(request_msg(line))
|
20
20
|
end
|
21
21
|
|
22
|
+
def handle_error(e)
|
23
|
+
case e
|
24
|
+
when Etna::Error
|
25
|
+
Rollbar.error(e)
|
26
|
+
@logger.error(request_msg("Exiting with #{e.status}, #{e.message}"))
|
27
|
+
return failure(e.status, error: e.message)
|
28
|
+
else
|
29
|
+
Rollbar.error(e)
|
30
|
+
@logger.error(request_msg('Caught unspecified error'))
|
31
|
+
@logger.error(request_msg(e.message))
|
32
|
+
e.backtrace.each do |trace|
|
33
|
+
@logger.error(request_msg(trace))
|
34
|
+
end
|
35
|
+
return failure(500, error: 'Server error.')
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
22
39
|
def response(&block)
|
23
40
|
return instance_eval(&block) if block_given?
|
24
|
-
|
25
41
|
return send(@action) if @action
|
26
42
|
|
27
|
-
|
28
43
|
[501, {}, ['This controller is not implemented.']]
|
29
|
-
rescue Etna::Error => e
|
30
|
-
Rollbar.error(e)
|
31
|
-
@logger.error(request_msg("Exiting with #{e.status}, #{e.message}"))
|
32
|
-
return failure(e.status, error: e.message)
|
33
44
|
rescue Exception => e
|
34
|
-
|
35
|
-
@logger.error(request_msg('Caught unspecified error'))
|
36
|
-
@logger.error(request_msg(e.message))
|
37
|
-
e.backtrace.each do |trace|
|
38
|
-
@logger.error(request_msg(trace))
|
39
|
-
end
|
40
|
-
return failure(500, error: 'Server error.')
|
45
|
+
handle_error(e)
|
41
46
|
end
|
42
47
|
|
43
48
|
def require_params(*params)
|
data/lib/etna/directed_graph.rb
CHANGED
@@ -52,7 +52,12 @@ class DirectedGraph
|
|
52
52
|
result[grandparent] << child_node if result.include?(grandparent)
|
53
53
|
end
|
54
54
|
|
55
|
-
|
55
|
+
# Depending on the graph shape, diamonds could lead to
|
56
|
+
# resetting of previously calculated dependencies.
|
57
|
+
# Here we avoid resetting existing entries in `result`
|
58
|
+
# and instead concatenate them if they already exist.
|
59
|
+
result[child_node] = [] unless result.include?(child_node)
|
60
|
+
result[n].concat(result[child_node]) if result.include?(child_node) && result.include?(n)
|
56
61
|
end
|
57
62
|
end
|
58
63
|
|
data/lib/etna/filesystem.rb
CHANGED
@@ -3,6 +3,8 @@ require 'fileutils'
|
|
3
3
|
require 'open3'
|
4
4
|
require 'securerandom'
|
5
5
|
require 'concurrent-ruby'
|
6
|
+
require 'net/sftp'
|
7
|
+
require 'net/ssh'
|
6
8
|
|
7
9
|
module Etna
|
8
10
|
# A class that encapsulates opening / reading file system entries that abstracts normal file access in order
|
@@ -50,6 +52,11 @@ module Etna
|
|
50
52
|
::FileUtils.mv(src, dest)
|
51
53
|
end
|
52
54
|
|
55
|
+
def stat(src)
|
56
|
+
raise "stat not supported by #{self.class.name}" unless self.class == Filesystem
|
57
|
+
::File.stat(src)
|
58
|
+
end
|
59
|
+
|
53
60
|
class EmptyIO < StringIO
|
54
61
|
def write(*args)
|
55
62
|
# Do nothing -- always leave empty
|
@@ -369,7 +376,59 @@ module Etna
|
|
369
376
|
end
|
370
377
|
end
|
371
378
|
|
379
|
+
class SftpFilesystem < Filesystem
|
380
|
+
def initialize(host:, username:, password: nil, port: 22, **args)
|
381
|
+
@username = username
|
382
|
+
@password = password
|
383
|
+
@host = host
|
384
|
+
@port = port
|
385
|
+
end
|
386
|
+
|
387
|
+
def ssh
|
388
|
+
@ssh ||= Net::SSH.start(@host, @username, password: @password)
|
389
|
+
end
|
390
|
+
|
391
|
+
def sftp
|
392
|
+
@sftp ||= begin
|
393
|
+
conn = Net::SFTP::Session.new(ssh)
|
394
|
+
conn.loop { conn.opening? }
|
395
|
+
|
396
|
+
conn
|
397
|
+
end
|
398
|
+
end
|
399
|
+
|
400
|
+
def with_readable(src, opts = 'r', &block)
|
401
|
+
sftp.file.open(src, opts, &block)
|
402
|
+
end
|
403
|
+
|
404
|
+
def ls(dir)
|
405
|
+
sftp.dir.entries(dir)
|
406
|
+
end
|
407
|
+
|
408
|
+
def exist?(src)
|
409
|
+
begin
|
410
|
+
sftp.file.open(src)
|
411
|
+
rescue Net::SFTP::StatusException
|
412
|
+
return false
|
413
|
+
end
|
414
|
+
return true
|
415
|
+
end
|
416
|
+
|
417
|
+
def stat(src)
|
418
|
+
sftp.file.open(src).stat
|
419
|
+
end
|
420
|
+
end
|
421
|
+
|
372
422
|
class Mock < Filesystem
|
423
|
+
class MockStat
|
424
|
+
def initialize
|
425
|
+
end
|
426
|
+
|
427
|
+
def size
|
428
|
+
0
|
429
|
+
end
|
430
|
+
end
|
431
|
+
|
373
432
|
def initialize(&new_io)
|
374
433
|
@files = {}
|
375
434
|
@dirs = {}
|
@@ -438,6 +497,10 @@ module Etna
|
|
438
497
|
def exist?(src)
|
439
498
|
@files.include?(src) || @dirs.include?(src)
|
440
499
|
end
|
500
|
+
|
501
|
+
def stat(src)
|
502
|
+
@files[src].respond_to?(:stat) ? @files[src].stat : MockStat.new
|
503
|
+
end
|
441
504
|
end
|
442
505
|
end
|
443
506
|
end
|
data/lib/etna/logger.rb
CHANGED
data/lib/etna/metrics.rb
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
|
2
|
+
module Etna
|
3
|
+
class MetricsExporter
|
4
|
+
def initialize(app, path: '/metrics')
|
5
|
+
@app = app
|
6
|
+
@path = path
|
7
|
+
end
|
8
|
+
|
9
|
+
def exporter
|
10
|
+
@exporter ||= begin
|
11
|
+
exporter = Yabeda::Prometheus::Exporter.new(@app, path: @path)
|
12
|
+
Rack::Auth::Basic.new(exporter) do |user, pw|
|
13
|
+
user == 'prometheus' && pw == ENV['METRICS_PW']
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def call(env)
|
19
|
+
if env['PATH_INFO'] == @path
|
20
|
+
exporter.call(env)
|
21
|
+
else
|
22
|
+
@app.call(env)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
data/lib/etna/route.rb
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
require 'digest'
|
2
|
+
require 'date'
|
3
|
+
|
1
4
|
module Etna
|
2
5
|
class Route
|
3
6
|
attr_reader :name
|
@@ -58,6 +61,63 @@ module Etna
|
|
58
61
|
end
|
59
62
|
|
60
63
|
def call(app, request)
|
64
|
+
start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
65
|
+
|
66
|
+
try_yabeda(request) do |tags|
|
67
|
+
Yabeda.etna.visits.increment(tags)
|
68
|
+
end
|
69
|
+
|
70
|
+
begin
|
71
|
+
process_call(app, request)
|
72
|
+
ensure
|
73
|
+
try_yabeda(request) do |tags|
|
74
|
+
dur = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
|
75
|
+
Yabeda.etna.response_time.measure(tags, dur)
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def hash_user_email(email)
|
81
|
+
secret = Etna::Application.instance.config(:user_hash_secret) || 'notsosecret'
|
82
|
+
digest = email + secret + Date.today.to_s
|
83
|
+
|
84
|
+
if @name
|
85
|
+
digest += @name.to_s
|
86
|
+
else
|
87
|
+
digest += @route.to_s
|
88
|
+
end
|
89
|
+
|
90
|
+
Digest::MD5.hexdigest(digest)
|
91
|
+
end
|
92
|
+
|
93
|
+
def try_yabeda(request, &block)
|
94
|
+
if @action
|
95
|
+
controller, action = @action.split('#')
|
96
|
+
elsif @name
|
97
|
+
controller = "none"
|
98
|
+
action = @name
|
99
|
+
else
|
100
|
+
controller = "none"
|
101
|
+
action = @route
|
102
|
+
end
|
103
|
+
|
104
|
+
params = request.env['rack.request.params']
|
105
|
+
user = request.env['etna.user']
|
106
|
+
user_hash = user ? hash_user_email(user.email) : 'unknown'
|
107
|
+
project_name = "unknown"
|
108
|
+
|
109
|
+
if params && (params.include?(:project_name) || params.include?('project_name'))
|
110
|
+
project_name = params[:project_name] || params['project_name']
|
111
|
+
end
|
112
|
+
|
113
|
+
begin
|
114
|
+
block.call({ controller: controller, action: action, user_hash: user_hash, project_name: project_name })
|
115
|
+
rescue => e
|
116
|
+
raise e unless Etna::Application.instance.environment == :production
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
def process_call(app, request)
|
61
121
|
update_params(request)
|
62
122
|
|
63
123
|
unless authorized?(request)
|
@@ -148,7 +208,10 @@ module Etna
|
|
148
208
|
params = request.env['rack.request.params']
|
149
209
|
|
150
210
|
@auth[:user].all? do |constraint, param_name|
|
151
|
-
user.respond_to?(constraint) &&
|
211
|
+
user.respond_to?(constraint) && (
|
212
|
+
param_name.is_a?(Symbol) ?
|
213
|
+
user.send(constraint, params[param_name]) :
|
214
|
+
user.send(constraint, param_name))
|
152
215
|
end
|
153
216
|
end
|
154
217
|
|
data/lib/etna/server.rb
CHANGED
@@ -74,6 +74,11 @@ module Etna
|
|
74
74
|
def initialize
|
75
75
|
# Setup logging.
|
76
76
|
application.setup_logger
|
77
|
+
|
78
|
+
# This needs to be required before yabeda invocation, but cannot belong at the top of any module since clients
|
79
|
+
# do not install yabeda.
|
80
|
+
require 'yabeda'
|
81
|
+
application.setup_yabeda
|
77
82
|
end
|
78
83
|
|
79
84
|
private
|
data/lib/etna/spec/vcr.rb
CHANGED
@@ -4,13 +4,32 @@ require 'openssl'
|
|
4
4
|
require 'digest/sha2'
|
5
5
|
require 'base64'
|
6
6
|
|
7
|
-
def setup_base_vcr(spec_helper_dir)
|
7
|
+
def setup_base_vcr(spec_helper_dir, server: nil, application: nil)
|
8
8
|
VCR.configure do |c|
|
9
9
|
c.hook_into :webmock
|
10
10
|
c.cassette_serializers
|
11
11
|
c.cassette_library_dir = ::File.join(spec_helper_dir, 'fixtures', 'cassettes')
|
12
12
|
c.allow_http_connections_when_no_cassette = true
|
13
13
|
|
14
|
+
c.register_request_matcher :verify_uri_route do |request_1, request_2|
|
15
|
+
next true if server.nil? || application.nil?
|
16
|
+
|
17
|
+
route_match = request_1.uri =~ /https:\/\/#{application.dev_route}(.*)/
|
18
|
+
if route_match && route_match[1]
|
19
|
+
def request_1.request_method
|
20
|
+
method.to_s.upcase
|
21
|
+
end
|
22
|
+
|
23
|
+
def request_1.path
|
24
|
+
URI(uri).path
|
25
|
+
end
|
26
|
+
|
27
|
+
!!server.find_route(request_1)
|
28
|
+
else
|
29
|
+
true
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
14
33
|
c.register_request_matcher :try_body do |request_1, request_2|
|
15
34
|
if request_1.headers['Content-Type'].first =~ /application\/json/
|
16
35
|
if request_2.headers['Content-Type'].first =~ /application\/json/
|
@@ -39,7 +58,7 @@ def setup_base_vcr(spec_helper_dir)
|
|
39
58
|
else
|
40
59
|
ENV['RERECORD'] ? :all : :once
|
41
60
|
end,
|
42
|
-
match_requests_on: [:method, :uri, :try_body]
|
61
|
+
match_requests_on: [:method, :uri, :try_body, :verify_uri_route]
|
43
62
|
}
|
44
63
|
|
45
64
|
# Filter the authorization headers of any request by replacing any occurrence of that request's
|
data/lib/etna/test_auth.rb
CHANGED
@@ -42,7 +42,6 @@ module Etna
|
|
42
42
|
# We do this to support Metis client tests, we pass in tokens with multiple "."-separated parts, so
|
43
43
|
# have to account for that.
|
44
44
|
payload = JSON.parse(Base64.decode64(token.split('.')[1]))
|
45
|
-
|
46
45
|
request.env['etna.user'] = Etna::User.new(payload.map{|k,v| [k.to_sym, v]}.to_h, token)
|
47
46
|
end
|
48
47
|
|
data/lib/etna/user.rb
CHANGED
@@ -7,7 +7,7 @@ module Etna
|
|
7
7
|
}
|
8
8
|
|
9
9
|
def initialize params, token=nil
|
10
|
-
@name, @email, @encoded_permissions, encoded_flags = params.values_at(:name, :email, :perm, :flags)
|
10
|
+
@name, @email, @encoded_permissions, encoded_flags, @task = params.values_at(:name, :email, :perm, :flags, :task)
|
11
11
|
|
12
12
|
@flags = encoded_flags&.split(/;/) || []
|
13
13
|
@token = token unless !token
|
@@ -16,6 +16,10 @@ module Etna
|
|
16
16
|
|
17
17
|
attr_reader :name, :email, :token
|
18
18
|
|
19
|
+
def task?
|
20
|
+
!!@task
|
21
|
+
end
|
22
|
+
|
19
23
|
def permissions
|
20
24
|
@permissions ||= @encoded_permissions.split(/\;/).map do |roles|
|
21
25
|
role, projects = roles.split(/:/)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: etna
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.38
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Saurabh Asthana
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-07-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rack
|
@@ -94,6 +94,20 @@ dependencies:
|
|
94
94
|
- - ">="
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: net-sftp
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: 3.0.0
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: 3.0.0
|
97
111
|
description: See summary
|
98
112
|
email: Saurabh.Asthana@ucsf.edu
|
99
113
|
executables:
|
@@ -144,6 +158,7 @@ files:
|
|
144
158
|
- lib/etna/clients/metis/client.rb
|
145
159
|
- lib/etna/clients/metis/models.rb
|
146
160
|
- lib/etna/clients/metis/workflows.rb
|
161
|
+
- lib/etna/clients/metis/workflows/ingest_metis_data_workflow.rb
|
147
162
|
- lib/etna/clients/metis/workflows/metis_download_workflow.rb
|
148
163
|
- lib/etna/clients/metis/workflows/metis_upload_workflow.rb
|
149
164
|
- lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb
|
@@ -168,6 +183,7 @@ files:
|
|
168
183
|
- lib/etna/hmac.rb
|
169
184
|
- lib/etna/json_serializable_struct.rb
|
170
185
|
- lib/etna/logger.rb
|
186
|
+
- lib/etna/metrics.rb
|
171
187
|
- lib/etna/multipart_serializable_nested_hash.rb
|
172
188
|
- lib/etna/parse_body.rb
|
173
189
|
- lib/etna/route.rb
|