etna 0.1.35 → 0.1.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/etna.rb +1 -0
- data/lib/etna/application.rb +96 -8
- data/lib/etna/auth.rb +1 -1
- data/lib/etna/client.rb +2 -2
- data/lib/etna/clients/base_client.rb +2 -1
- data/lib/etna/clients/janus/client.rb +4 -5
- data/lib/etna/clients/metis/client.rb +36 -6
- data/lib/etna/clients/metis/workflows.rb +4 -3
- data/lib/etna/clients/metis/workflows/ingest_metis_data_workflow.rb +31 -0
- data/lib/etna/clients/metis/workflows/metis_upload_workflow.rb +1 -1
- data/lib/etna/controller.rb +18 -13
- data/lib/etna/directed_graph.rb +6 -1
- data/lib/etna/filesystem.rb +63 -0
- data/lib/etna/logger.rb +4 -0
- data/lib/etna/metrics.rb +26 -0
- data/lib/etna/route.rb +64 -1
- data/lib/etna/server.rb +5 -0
- data/lib/etna/spec/vcr.rb +21 -2
- data/lib/etna/test_auth.rb +0 -1
- data/lib/etna/user.rb +5 -1
- metadata +18 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c21212f38bbad3cf757eb84a9e453428c9d0b6f195fd34eb81f98aeff2746490
|
|
4
|
+
data.tar.gz: 3581821244c087cbbfbac162f9a5c61911e06ab64923c603a144d36ca3bedff0
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 3289530a2e874d2b79b0da66ea787e30b335b7a60626e4b67f1be9919067d4b79752db4e878ca8851dbce6c68a1e59a573bcbc60314bb7a35268cd37cc2eeaae
|
|
7
|
+
data.tar.gz: 2758348f253992e08fbf589d2b5444844becf69e179872c630804a7e8885568189168edd056c56beabe4ed5dc3e0905f46c51595896bd7bdd1255fbe4ece71ec
|
data/lib/etna.rb
CHANGED
data/lib/etna/application.rb
CHANGED
|
@@ -8,6 +8,7 @@ require_relative './command'
|
|
|
8
8
|
require_relative './generate_autocompletion_script'
|
|
9
9
|
require 'singleton'
|
|
10
10
|
require 'rollbar'
|
|
11
|
+
require 'fileutils'
|
|
11
12
|
|
|
12
13
|
module Etna::Application
|
|
13
14
|
def self.included(other)
|
|
@@ -31,6 +32,12 @@ module Etna::Application
|
|
|
31
32
|
raise "Could not find application instance from #{namespace}, and not subclass of Application found."
|
|
32
33
|
end
|
|
33
34
|
|
|
35
|
+
# Used to find the application in development recorded vcr tests.
|
|
36
|
+
# see spec/vcr.rb
|
|
37
|
+
def dev_route
|
|
38
|
+
"#{self.class.name.split('::').first.downcase}.development.local"
|
|
39
|
+
end
|
|
40
|
+
|
|
34
41
|
def self.register(app)
|
|
35
42
|
@instance = app
|
|
36
43
|
end
|
|
@@ -53,6 +60,72 @@ module Etna::Application
|
|
|
53
60
|
end
|
|
54
61
|
end
|
|
55
62
|
|
|
63
|
+
# This will cause metrics to persist to a file.
|
|
64
|
+
# NOTE -- /tmp/metrics.bin should be a persistent mount when using this.
|
|
65
|
+
# You will still need to export metrics in the text format for the node_exporter on the host machine to
|
|
66
|
+
# export them to prometheus. Ensure that the /tmp/metrics.bin is on a named volume or a bind mount, either is fine.
|
|
67
|
+
def enable_job_metrics!
|
|
68
|
+
require 'prometheus'
|
|
69
|
+
Prometheus::Client.config.data_store = Prometheus::Client::DataStores::DirectFileStore.new({
|
|
70
|
+
dir: "/tmp/metrics.bin"
|
|
71
|
+
})
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def setup_yabeda
|
|
75
|
+
application = self.id
|
|
76
|
+
Yabeda.configure do
|
|
77
|
+
default_tag :application, application
|
|
78
|
+
|
|
79
|
+
group :etna do
|
|
80
|
+
histogram :response_time do
|
|
81
|
+
comment "Time spent by a controller returning a response"
|
|
82
|
+
unit :seconds
|
|
83
|
+
tags [:controller, :action, :user_hash, :project_name]
|
|
84
|
+
buckets [0.01, 0.1, 0.3, 0.5, 1, 5]
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
counter :visits do
|
|
88
|
+
comment "Counts visits to the controller"
|
|
89
|
+
tags [:controller, :action, :user_hash, :project_name]
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
counter :rollbar_errors do
|
|
93
|
+
comment "Counts errors detected by and sent to rollbar"
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
gauge :last_command_completion do
|
|
97
|
+
comment "Unix time of last time command was completed"
|
|
98
|
+
tags [:command, :status, :application]
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
histogram :command_runtime do
|
|
102
|
+
comment "Time spent processing a given command"
|
|
103
|
+
tags [:command, :status, :application]
|
|
104
|
+
unit :seconds
|
|
105
|
+
buckets [0.1, 1, 5, 60, 300, 1500]
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
Yabeda.configure!
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# Writes all metrics currently gathered to a text format prometheus file. If /tmp/metrics.prom is bind mounted
|
|
114
|
+
# to the host directed bound to the node_exporter's file exporter directory, these will be exported to
|
|
115
|
+
# prometheus. Combine this enable_job_metrics! for maximum effect.
|
|
116
|
+
def write_job_metrics(name)
|
|
117
|
+
node_metrics_dir = "/tmp/metrics.prom"
|
|
118
|
+
::FileUtils.mkdir_p(node_metrics_dir)
|
|
119
|
+
|
|
120
|
+
tmp_file = ::File.join(node_metrics_dir, "#{name}.prom.$$")
|
|
121
|
+
::File.open(tmp_file, "w") do |f|
|
|
122
|
+
f.write(Prometheus::Client::Formats::Text.marshal(Prometheus::Client.registry))
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
require 'fileutils'
|
|
126
|
+
::FileUtils.mv(tmp_file, ::File.join(node_metrics_dir, "#{name}.prom"))
|
|
127
|
+
end
|
|
128
|
+
|
|
56
129
|
def setup_logger
|
|
57
130
|
@logger = Etna::Logger.new(
|
|
58
131
|
# The name of the log_file, required.
|
|
@@ -102,17 +175,32 @@ module Etna::Application
|
|
|
102
175
|
end
|
|
103
176
|
|
|
104
177
|
def run_command(config, *args, &block)
|
|
178
|
+
application = self.id
|
|
179
|
+
status = 'success'
|
|
180
|
+
start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
105
181
|
cmd, cmd_args, cmd_kwds = find_command(*args)
|
|
106
|
-
cmd.setup(config)
|
|
107
182
|
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
183
|
+
begin
|
|
184
|
+
cmd.setup(config)
|
|
185
|
+
if block_given?
|
|
186
|
+
return unless yield [cmd, cmd_args]
|
|
187
|
+
end
|
|
111
188
|
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
189
|
+
cmd.execute(*cmd.fill_in_missing_params(cmd_args), **cmd_kwds)
|
|
190
|
+
rescue => e
|
|
191
|
+
Rollbar.error(e)
|
|
192
|
+
status = 'failed'
|
|
193
|
+
raise
|
|
194
|
+
ensure
|
|
195
|
+
if Yabeda.configured?
|
|
196
|
+
tags = { command: cmd.class.name, status: status, application: application }
|
|
197
|
+
dur = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
|
|
198
|
+
|
|
199
|
+
Yabeda.etna.last_command_completion.set(tags, Time.now.to_i)
|
|
200
|
+
Yabeda.etna.command_runtime.measure(tags, dur)
|
|
201
|
+
write_job_metrics("run_command")
|
|
202
|
+
end
|
|
203
|
+
end
|
|
116
204
|
end
|
|
117
205
|
end
|
|
118
206
|
|
data/lib/etna/auth.rb
CHANGED
data/lib/etna/client.rb
CHANGED
|
@@ -173,7 +173,7 @@ module Etna
|
|
|
173
173
|
verify_mode = @ignore_ssl ?
|
|
174
174
|
OpenSSL::SSL::VERIFY_NONE :
|
|
175
175
|
OpenSSL::SSL::VERIFY_PEER
|
|
176
|
-
Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode) do |http|
|
|
176
|
+
Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode, read_timeout: 300) do |http|
|
|
177
177
|
http.request(data) do |response|
|
|
178
178
|
status_check!(response)
|
|
179
179
|
yield response
|
|
@@ -183,7 +183,7 @@ module Etna
|
|
|
183
183
|
verify_mode = @ignore_ssl ?
|
|
184
184
|
OpenSSL::SSL::VERIFY_NONE :
|
|
185
185
|
OpenSSL::SSL::VERIFY_PEER
|
|
186
|
-
Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode) do |http|
|
|
186
|
+
Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode, read_timeout: 300) do |http|
|
|
187
187
|
response = http.request(data)
|
|
188
188
|
status_check!(response)
|
|
189
189
|
return response
|
|
@@ -29,7 +29,8 @@ module Etna
|
|
|
29
29
|
def token_will_expire?(offset=3000)
|
|
30
30
|
# offset in seconds
|
|
31
31
|
# Will the user's token expire in the given amount of time?
|
|
32
|
-
|
|
32
|
+
payload = JSON.parse(Base64.urlsafe_decode64(token.split('.')[1]))
|
|
33
|
+
epoch_seconds = payload["exp"]
|
|
33
34
|
expiration = DateTime.strptime(epoch_seconds.to_s, "%s")
|
|
34
35
|
expiration <= DateTime.now.new_offset + offset
|
|
35
36
|
end
|
|
@@ -58,20 +58,19 @@ module Etna
|
|
|
58
58
|
TokenResponse.new(token)
|
|
59
59
|
end
|
|
60
60
|
|
|
61
|
-
def validate_task_token
|
|
62
|
-
|
|
63
|
-
@etna_client.post('/api/tokens/task/validate', validate_task_token_request)
|
|
61
|
+
def validate_task_token
|
|
62
|
+
@etna_client.post('/api/tokens/validate_task')
|
|
64
63
|
end
|
|
65
64
|
|
|
66
65
|
def get_nonce
|
|
67
66
|
@etna_client.get('/api/tokens/nonce').body
|
|
68
67
|
end
|
|
69
68
|
|
|
70
|
-
def generate_token(token_type, signed_nonce: nil, project_name: nil)
|
|
69
|
+
def generate_token(token_type, signed_nonce: nil, project_name: nil, read_only: false)
|
|
71
70
|
response = @etna_client.with_headers(
|
|
72
71
|
'Authorization' => signed_nonce ? "Signed-Nonce #{signed_nonce}" : nil
|
|
73
72
|
) do
|
|
74
|
-
post('/api/tokens/generate', token_type: token_type, project_name: project_name)
|
|
73
|
+
post('/api/tokens/generate', token_type: token_type, project_name: project_name, read_only: read_only)
|
|
75
74
|
end
|
|
76
75
|
|
|
77
76
|
response.body
|
|
@@ -228,14 +228,44 @@ module Etna
|
|
|
228
228
|
end
|
|
229
229
|
|
|
230
230
|
folder_contents.files.all.each do |file|
|
|
231
|
-
|
|
232
|
-
|
|
231
|
+
# If file exists in destination, delete the older file.
|
|
232
|
+
list_dest_folder_request = Etna::Clients::Metis::ListFolderRequest.new(
|
|
233
|
+
bucket_name: dest_bucket,
|
|
233
234
|
project_name: project_name,
|
|
234
|
-
|
|
235
|
-
new_bucket_name: dest_bucket,
|
|
236
|
-
new_file_path: file.file_path,
|
|
237
|
-
create_parent: true)
|
|
235
|
+
folder_path: ::File.dirname(file.file_path)
|
|
238
236
|
)
|
|
237
|
+
|
|
238
|
+
dest_file = list_folder(list_dest_folder_request).files.all.find { |f| f.file_name == file.file_name }
|
|
239
|
+
|
|
240
|
+
should_rename = true
|
|
241
|
+
if (dest_file && file.updated_at <= dest_file.updated_at)
|
|
242
|
+
# Delete source file if it's out of date
|
|
243
|
+
delete_file(Etna::Clients::Metis::DeleteFileRequest.new(
|
|
244
|
+
bucket_name: source_bucket,
|
|
245
|
+
project_name: project_name,
|
|
246
|
+
file_path: file.file_path,
|
|
247
|
+
))
|
|
248
|
+
|
|
249
|
+
should_rename = false
|
|
250
|
+
elsif (dest_file && file.updated_at > dest_file.updated_at)
|
|
251
|
+
# Delete dest file if it's out of date
|
|
252
|
+
delete_file(Etna::Clients::Metis::DeleteFileRequest.new(
|
|
253
|
+
bucket_name: dest_bucket,
|
|
254
|
+
project_name: project_name,
|
|
255
|
+
file_path: dest_file.file_path,
|
|
256
|
+
))
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
if should_rename
|
|
260
|
+
rename_file(Etna::Clients::Metis::RenameFileRequest.new(
|
|
261
|
+
bucket_name: source_bucket,
|
|
262
|
+
project_name: project_name,
|
|
263
|
+
file_path: file.file_path,
|
|
264
|
+
new_bucket_name: dest_bucket,
|
|
265
|
+
new_file_path: file.file_path,
|
|
266
|
+
create_parent: true)
|
|
267
|
+
)
|
|
268
|
+
end
|
|
239
269
|
end
|
|
240
270
|
|
|
241
271
|
# Now delete the source folder
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
-
require_relative
|
|
2
|
-
require_relative
|
|
3
|
-
require_relative
|
|
1
|
+
require_relative "./workflows/metis_download_workflow"
|
|
2
|
+
require_relative "./workflows/metis_upload_workflow"
|
|
3
|
+
require_relative "./workflows/sync_metis_data_workflow"
|
|
4
|
+
require_relative "./workflows/ingest_metis_data_workflow"
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
require "ostruct"
|
|
2
|
+
require "fileutils"
|
|
3
|
+
require "tempfile"
|
|
4
|
+
|
|
5
|
+
module Etna
|
|
6
|
+
module Clients
|
|
7
|
+
class Metis
|
|
8
|
+
class IngestMetisDataWorkflow < Struct.new(:metis_filesystem, :ingest_filesystem, :logger, keyword_init: true)
|
|
9
|
+
# Since we are doing manual triage of files,
|
|
10
|
+
# do not automatically copy directory trees.
|
|
11
|
+
# srcs must be a list of full paths to files.
|
|
12
|
+
def copy_files(srcs)
|
|
13
|
+
srcs.each do |src|
|
|
14
|
+
next unless ingest_filesystem.exist?(src)
|
|
15
|
+
|
|
16
|
+
logger&.info("Copying file #{src} (#{Etna::Formatting.as_size(ingest_filesystem.stat(src).size)})")
|
|
17
|
+
|
|
18
|
+
# For ingestion triage, just copy over the exact path + filename.
|
|
19
|
+
copy_file(dest: src, src: src)
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def copy_file(dest:, src:)
|
|
24
|
+
ingest_filesystem.with_readable(src, "r") do |file|
|
|
25
|
+
metis_filesystem.do_streaming_upload(file, dest, file.size)
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
data/lib/etna/controller.rb
CHANGED
|
@@ -19,25 +19,30 @@ module Etna
|
|
|
19
19
|
@logger.warn(request_msg(line))
|
|
20
20
|
end
|
|
21
21
|
|
|
22
|
+
def handle_error(e)
|
|
23
|
+
case e
|
|
24
|
+
when Etna::Error
|
|
25
|
+
Rollbar.error(e)
|
|
26
|
+
@logger.error(request_msg("Exiting with #{e.status}, #{e.message}"))
|
|
27
|
+
return failure(e.status, error: e.message)
|
|
28
|
+
else
|
|
29
|
+
Rollbar.error(e)
|
|
30
|
+
@logger.error(request_msg('Caught unspecified error'))
|
|
31
|
+
@logger.error(request_msg(e.message))
|
|
32
|
+
e.backtrace.each do |trace|
|
|
33
|
+
@logger.error(request_msg(trace))
|
|
34
|
+
end
|
|
35
|
+
return failure(500, error: 'Server error.')
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
22
39
|
def response(&block)
|
|
23
40
|
return instance_eval(&block) if block_given?
|
|
24
|
-
|
|
25
41
|
return send(@action) if @action
|
|
26
42
|
|
|
27
|
-
|
|
28
43
|
[501, {}, ['This controller is not implemented.']]
|
|
29
|
-
rescue Etna::Error => e
|
|
30
|
-
Rollbar.error(e)
|
|
31
|
-
@logger.error(request_msg("Exiting with #{e.status}, #{e.message}"))
|
|
32
|
-
return failure(e.status, error: e.message)
|
|
33
44
|
rescue Exception => e
|
|
34
|
-
|
|
35
|
-
@logger.error(request_msg('Caught unspecified error'))
|
|
36
|
-
@logger.error(request_msg(e.message))
|
|
37
|
-
e.backtrace.each do |trace|
|
|
38
|
-
@logger.error(request_msg(trace))
|
|
39
|
-
end
|
|
40
|
-
return failure(500, error: 'Server error.')
|
|
45
|
+
handle_error(e)
|
|
41
46
|
end
|
|
42
47
|
|
|
43
48
|
def require_params(*params)
|
data/lib/etna/directed_graph.rb
CHANGED
|
@@ -52,7 +52,12 @@ class DirectedGraph
|
|
|
52
52
|
result[grandparent] << child_node if result.include?(grandparent)
|
|
53
53
|
end
|
|
54
54
|
|
|
55
|
-
|
|
55
|
+
# Depending on the graph shape, diamonds could lead to
|
|
56
|
+
# resetting of previously calculated dependencies.
|
|
57
|
+
# Here we avoid resetting existing entries in `result`
|
|
58
|
+
# and instead concatenate them if they already exist.
|
|
59
|
+
result[child_node] = [] unless result.include?(child_node)
|
|
60
|
+
result[n].concat(result[child_node]) if result.include?(child_node) && result.include?(n)
|
|
56
61
|
end
|
|
57
62
|
end
|
|
58
63
|
|
data/lib/etna/filesystem.rb
CHANGED
|
@@ -3,6 +3,8 @@ require 'fileutils'
|
|
|
3
3
|
require 'open3'
|
|
4
4
|
require 'securerandom'
|
|
5
5
|
require 'concurrent-ruby'
|
|
6
|
+
require 'net/sftp'
|
|
7
|
+
require 'net/ssh'
|
|
6
8
|
|
|
7
9
|
module Etna
|
|
8
10
|
# A class that encapsulates opening / reading file system entries that abstracts normal file access in order
|
|
@@ -50,6 +52,11 @@ module Etna
|
|
|
50
52
|
::FileUtils.mv(src, dest)
|
|
51
53
|
end
|
|
52
54
|
|
|
55
|
+
def stat(src)
|
|
56
|
+
raise "stat not supported by #{self.class.name}" unless self.class == Filesystem
|
|
57
|
+
::File.stat(src)
|
|
58
|
+
end
|
|
59
|
+
|
|
53
60
|
class EmptyIO < StringIO
|
|
54
61
|
def write(*args)
|
|
55
62
|
# Do nothing -- always leave empty
|
|
@@ -369,7 +376,59 @@ module Etna
|
|
|
369
376
|
end
|
|
370
377
|
end
|
|
371
378
|
|
|
379
|
+
class SftpFilesystem < Filesystem
|
|
380
|
+
def initialize(host:, username:, password: nil, port: 22, **args)
|
|
381
|
+
@username = username
|
|
382
|
+
@password = password
|
|
383
|
+
@host = host
|
|
384
|
+
@port = port
|
|
385
|
+
end
|
|
386
|
+
|
|
387
|
+
def ssh
|
|
388
|
+
@ssh ||= Net::SSH.start(@host, @username, password: @password)
|
|
389
|
+
end
|
|
390
|
+
|
|
391
|
+
def sftp
|
|
392
|
+
@sftp ||= begin
|
|
393
|
+
conn = Net::SFTP::Session.new(ssh)
|
|
394
|
+
conn.loop { conn.opening? }
|
|
395
|
+
|
|
396
|
+
conn
|
|
397
|
+
end
|
|
398
|
+
end
|
|
399
|
+
|
|
400
|
+
def with_readable(src, opts = 'r', &block)
|
|
401
|
+
sftp.file.open(src, opts, &block)
|
|
402
|
+
end
|
|
403
|
+
|
|
404
|
+
def ls(dir)
|
|
405
|
+
sftp.dir.entries(dir)
|
|
406
|
+
end
|
|
407
|
+
|
|
408
|
+
def exist?(src)
|
|
409
|
+
begin
|
|
410
|
+
sftp.file.open(src)
|
|
411
|
+
rescue Net::SFTP::StatusException
|
|
412
|
+
return false
|
|
413
|
+
end
|
|
414
|
+
return true
|
|
415
|
+
end
|
|
416
|
+
|
|
417
|
+
def stat(src)
|
|
418
|
+
sftp.file.open(src).stat
|
|
419
|
+
end
|
|
420
|
+
end
|
|
421
|
+
|
|
372
422
|
class Mock < Filesystem
|
|
423
|
+
class MockStat
|
|
424
|
+
def initialize
|
|
425
|
+
end
|
|
426
|
+
|
|
427
|
+
def size
|
|
428
|
+
0
|
|
429
|
+
end
|
|
430
|
+
end
|
|
431
|
+
|
|
373
432
|
def initialize(&new_io)
|
|
374
433
|
@files = {}
|
|
375
434
|
@dirs = {}
|
|
@@ -438,6 +497,10 @@ module Etna
|
|
|
438
497
|
def exist?(src)
|
|
439
498
|
@files.include?(src) || @dirs.include?(src)
|
|
440
499
|
end
|
|
500
|
+
|
|
501
|
+
def stat(src)
|
|
502
|
+
@files[src].respond_to?(:stat) ? @files[src].stat : MockStat.new
|
|
503
|
+
end
|
|
441
504
|
end
|
|
442
505
|
end
|
|
443
506
|
end
|
data/lib/etna/logger.rb
CHANGED
data/lib/etna/metrics.rb
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
|
|
2
|
+
module Etna
|
|
3
|
+
class MetricsExporter
|
|
4
|
+
def initialize(app, path: '/metrics')
|
|
5
|
+
@app = app
|
|
6
|
+
@path = path
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
def exporter
|
|
10
|
+
@exporter ||= begin
|
|
11
|
+
exporter = Yabeda::Prometheus::Exporter.new(@app, path: @path)
|
|
12
|
+
Rack::Auth::Basic.new(exporter) do |user, pw|
|
|
13
|
+
user == 'prometheus' && pw == ENV['METRICS_PW']
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def call(env)
|
|
19
|
+
if env['PATH_INFO'] == @path
|
|
20
|
+
exporter.call(env)
|
|
21
|
+
else
|
|
22
|
+
@app.call(env)
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
data/lib/etna/route.rb
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
require 'digest'
|
|
2
|
+
require 'date'
|
|
3
|
+
|
|
1
4
|
module Etna
|
|
2
5
|
class Route
|
|
3
6
|
attr_reader :name
|
|
@@ -58,6 +61,63 @@ module Etna
|
|
|
58
61
|
end
|
|
59
62
|
|
|
60
63
|
def call(app, request)
|
|
64
|
+
start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
65
|
+
|
|
66
|
+
try_yabeda(request) do |tags|
|
|
67
|
+
Yabeda.etna.visits.increment(tags)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
begin
|
|
71
|
+
process_call(app, request)
|
|
72
|
+
ensure
|
|
73
|
+
try_yabeda(request) do |tags|
|
|
74
|
+
dur = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
|
|
75
|
+
Yabeda.etna.response_time.measure(tags, dur)
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def hash_user_email(email)
|
|
81
|
+
secret = Etna::Application.instance.config(:user_hash_secret) || 'notsosecret'
|
|
82
|
+
digest = email + secret + Date.today.to_s
|
|
83
|
+
|
|
84
|
+
if @name
|
|
85
|
+
digest += @name.to_s
|
|
86
|
+
else
|
|
87
|
+
digest += @route.to_s
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
Digest::MD5.hexdigest(digest)
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def try_yabeda(request, &block)
|
|
94
|
+
if @action
|
|
95
|
+
controller, action = @action.split('#')
|
|
96
|
+
elsif @name
|
|
97
|
+
controller = "none"
|
|
98
|
+
action = @name
|
|
99
|
+
else
|
|
100
|
+
controller = "none"
|
|
101
|
+
action = @route
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
params = request.env['rack.request.params']
|
|
105
|
+
user = request.env['etna.user']
|
|
106
|
+
user_hash = user ? hash_user_email(user.email) : 'unknown'
|
|
107
|
+
project_name = "unknown"
|
|
108
|
+
|
|
109
|
+
if params && (params.include?(:project_name) || params.include?('project_name'))
|
|
110
|
+
project_name = params[:project_name] || params['project_name']
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
begin
|
|
114
|
+
block.call({ controller: controller, action: action, user_hash: user_hash, project_name: project_name })
|
|
115
|
+
rescue => e
|
|
116
|
+
raise e unless Etna::Application.instance.environment == :production
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def process_call(app, request)
|
|
61
121
|
update_params(request)
|
|
62
122
|
|
|
63
123
|
unless authorized?(request)
|
|
@@ -148,7 +208,10 @@ module Etna
|
|
|
148
208
|
params = request.env['rack.request.params']
|
|
149
209
|
|
|
150
210
|
@auth[:user].all? do |constraint, param_name|
|
|
151
|
-
user.respond_to?(constraint) &&
|
|
211
|
+
user.respond_to?(constraint) && (
|
|
212
|
+
param_name.is_a?(Symbol) ?
|
|
213
|
+
user.send(constraint, params[param_name]) :
|
|
214
|
+
user.send(constraint, param_name))
|
|
152
215
|
end
|
|
153
216
|
end
|
|
154
217
|
|
data/lib/etna/server.rb
CHANGED
|
@@ -74,6 +74,11 @@ module Etna
|
|
|
74
74
|
def initialize
|
|
75
75
|
# Setup logging.
|
|
76
76
|
application.setup_logger
|
|
77
|
+
|
|
78
|
+
# This needs to be required before yabeda invocation, but cannot belong at the top of any module since clients
|
|
79
|
+
# do not install yabeda.
|
|
80
|
+
require 'yabeda'
|
|
81
|
+
application.setup_yabeda
|
|
77
82
|
end
|
|
78
83
|
|
|
79
84
|
private
|
data/lib/etna/spec/vcr.rb
CHANGED
|
@@ -4,13 +4,32 @@ require 'openssl'
|
|
|
4
4
|
require 'digest/sha2'
|
|
5
5
|
require 'base64'
|
|
6
6
|
|
|
7
|
-
def setup_base_vcr(spec_helper_dir)
|
|
7
|
+
def setup_base_vcr(spec_helper_dir, server: nil, application: nil)
|
|
8
8
|
VCR.configure do |c|
|
|
9
9
|
c.hook_into :webmock
|
|
10
10
|
c.cassette_serializers
|
|
11
11
|
c.cassette_library_dir = ::File.join(spec_helper_dir, 'fixtures', 'cassettes')
|
|
12
12
|
c.allow_http_connections_when_no_cassette = true
|
|
13
13
|
|
|
14
|
+
c.register_request_matcher :verify_uri_route do |request_1, request_2|
|
|
15
|
+
next true if server.nil? || application.nil?
|
|
16
|
+
|
|
17
|
+
route_match = request_1.uri =~ /https:\/\/#{application.dev_route}(.*)/
|
|
18
|
+
if route_match && route_match[1]
|
|
19
|
+
def request_1.request_method
|
|
20
|
+
method.to_s.upcase
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def request_1.path
|
|
24
|
+
URI(uri).path
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
!!server.find_route(request_1)
|
|
28
|
+
else
|
|
29
|
+
true
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
14
33
|
c.register_request_matcher :try_body do |request_1, request_2|
|
|
15
34
|
if request_1.headers['Content-Type'].first =~ /application\/json/
|
|
16
35
|
if request_2.headers['Content-Type'].first =~ /application\/json/
|
|
@@ -39,7 +58,7 @@ def setup_base_vcr(spec_helper_dir)
|
|
|
39
58
|
else
|
|
40
59
|
ENV['RERECORD'] ? :all : :once
|
|
41
60
|
end,
|
|
42
|
-
match_requests_on: [:method, :uri, :try_body]
|
|
61
|
+
match_requests_on: [:method, :uri, :try_body, :verify_uri_route]
|
|
43
62
|
}
|
|
44
63
|
|
|
45
64
|
# Filter the authorization headers of any request by replacing any occurrence of that request's
|
data/lib/etna/test_auth.rb
CHANGED
|
@@ -42,7 +42,6 @@ module Etna
|
|
|
42
42
|
# We do this to support Metis client tests, we pass in tokens with multiple "."-separated parts, so
|
|
43
43
|
# have to account for that.
|
|
44
44
|
payload = JSON.parse(Base64.decode64(token.split('.')[1]))
|
|
45
|
-
|
|
46
45
|
request.env['etna.user'] = Etna::User.new(payload.map{|k,v| [k.to_sym, v]}.to_h, token)
|
|
47
46
|
end
|
|
48
47
|
|
data/lib/etna/user.rb
CHANGED
|
@@ -7,7 +7,7 @@ module Etna
|
|
|
7
7
|
}
|
|
8
8
|
|
|
9
9
|
def initialize params, token=nil
|
|
10
|
-
@name, @email, @encoded_permissions, encoded_flags = params.values_at(:name, :email, :perm, :flags)
|
|
10
|
+
@name, @email, @encoded_permissions, encoded_flags, @task = params.values_at(:name, :email, :perm, :flags, :task)
|
|
11
11
|
|
|
12
12
|
@flags = encoded_flags&.split(/;/) || []
|
|
13
13
|
@token = token unless !token
|
|
@@ -16,6 +16,10 @@ module Etna
|
|
|
16
16
|
|
|
17
17
|
attr_reader :name, :email, :token
|
|
18
18
|
|
|
19
|
+
def task?
|
|
20
|
+
!!@task
|
|
21
|
+
end
|
|
22
|
+
|
|
19
23
|
def permissions
|
|
20
24
|
@permissions ||= @encoded_permissions.split(/\;/).map do |roles|
|
|
21
25
|
role, projects = roles.split(/:/)
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: etna
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.39
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Saurabh Asthana
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2021-
|
|
11
|
+
date: 2021-07-22 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rack
|
|
@@ -94,6 +94,20 @@ dependencies:
|
|
|
94
94
|
- - ">="
|
|
95
95
|
- !ruby/object:Gem::Version
|
|
96
96
|
version: '0'
|
|
97
|
+
- !ruby/object:Gem::Dependency
|
|
98
|
+
name: net-sftp
|
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
|
100
|
+
requirements:
|
|
101
|
+
- - ">="
|
|
102
|
+
- !ruby/object:Gem::Version
|
|
103
|
+
version: 3.0.0
|
|
104
|
+
type: :runtime
|
|
105
|
+
prerelease: false
|
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
107
|
+
requirements:
|
|
108
|
+
- - ">="
|
|
109
|
+
- !ruby/object:Gem::Version
|
|
110
|
+
version: 3.0.0
|
|
97
111
|
description: See summary
|
|
98
112
|
email: Saurabh.Asthana@ucsf.edu
|
|
99
113
|
executables:
|
|
@@ -144,6 +158,7 @@ files:
|
|
|
144
158
|
- lib/etna/clients/metis/client.rb
|
|
145
159
|
- lib/etna/clients/metis/models.rb
|
|
146
160
|
- lib/etna/clients/metis/workflows.rb
|
|
161
|
+
- lib/etna/clients/metis/workflows/ingest_metis_data_workflow.rb
|
|
147
162
|
- lib/etna/clients/metis/workflows/metis_download_workflow.rb
|
|
148
163
|
- lib/etna/clients/metis/workflows/metis_upload_workflow.rb
|
|
149
164
|
- lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb
|
|
@@ -168,6 +183,7 @@ files:
|
|
|
168
183
|
- lib/etna/hmac.rb
|
|
169
184
|
- lib/etna/json_serializable_struct.rb
|
|
170
185
|
- lib/etna/logger.rb
|
|
186
|
+
- lib/etna/metrics.rb
|
|
171
187
|
- lib/etna/multipart_serializable_nested_hash.rb
|
|
172
188
|
- lib/etna/parse_body.rb
|
|
173
189
|
- lib/etna/route.rb
|