etna 0.1.40 → 0.1.41
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/etna.rb +1 -0
- data/lib/etna/application.rb +2 -12
- data/lib/etna/clients/metis/workflows/ingest_metis_data_workflow.rb +10 -6
- data/lib/etna/filesystem.rb +83 -23
- data/lib/etna/remote.rb +38 -0
- metadata +20 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 44b6c9fde1d01fab95038052180531fafd4a120e77ffbd12a8ac14e28122cd05
|
4
|
+
data.tar.gz: e1cb315eaeb4d35efc9e7f181e0cad767833b98f7ce3f26386e040c758c8483c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3129a01c4181a772ab6752ed319396485e06f774b8091bff1ed9cbf6543a34820e0cc0706fd313ce5b3fb848a25f347de17bc376dc66b36458975d6298f48608
|
7
|
+
data.tar.gz: 82c57cff18c826a0a2664a63e0a531df57c75af30ceb3408d1d8ec1b9cb942fd25bd11253e6d880db6973df4cd60ade0764fac798a543af1e7601bfcc21abc45
|
data/lib/etna.rb
CHANGED
data/lib/etna/application.rb
CHANGED
@@ -60,17 +60,6 @@ module Etna::Application
|
|
60
60
|
end
|
61
61
|
end
|
62
62
|
|
63
|
-
# This will cause metrics to persist to a file.
|
64
|
-
# NOTE -- /tmp/metrics.bin should be a persistent mount when using this.
|
65
|
-
# You will still need to export metrics in the text format for the node_exporter on the host machine to
|
66
|
-
# export them to prometheus. Ensure that the /tmp/metrics.bin is on a named volume or a bind mount, either is fine.
|
67
|
-
def enable_job_metrics!
|
68
|
-
require 'prometheus'
|
69
|
-
Prometheus::Client.config.data_store = Prometheus::Client::DataStores::DirectFileStore.new({
|
70
|
-
dir: "/tmp/metrics.bin"
|
71
|
-
})
|
72
|
-
end
|
73
|
-
|
74
63
|
def setup_yabeda
|
75
64
|
application = self.id
|
76
65
|
Yabeda.configure do
|
@@ -198,7 +187,8 @@ module Etna::Application
|
|
198
187
|
|
199
188
|
Yabeda.etna.last_command_completion.set(tags, Time.now.to_i)
|
200
189
|
Yabeda.etna.command_runtime.measure(tags, dur)
|
201
|
-
|
190
|
+
|
191
|
+
write_job_metrics("run_command.#{cmd.class.name}")
|
202
192
|
end
|
203
193
|
end
|
204
194
|
end
|
@@ -9,20 +9,24 @@ module Etna
|
|
9
9
|
# Since we are doing manual triage of files,
|
10
10
|
# do not automatically copy directory trees.
|
11
11
|
# srcs must be a list of full paths to files.
|
12
|
-
def copy_files(srcs)
|
12
|
+
def copy_files(srcs, &block)
|
13
13
|
srcs.each do |src|
|
14
|
-
|
14
|
+
if !ingest_filesystem.exist?(src)
|
15
|
+
logger&.warn("#{src} does not exist on source filesystem. Skipping.")
|
16
|
+
next
|
17
|
+
end
|
15
18
|
|
16
19
|
logger&.info("Copying file #{src} (#{Etna::Formatting.as_size(ingest_filesystem.stat(src).size)})")
|
17
20
|
|
18
21
|
# For ingestion triage, just copy over the exact path + filename.
|
19
|
-
copy_file(dest: src, src: src)
|
22
|
+
copy_file(dest: src, src: src, &block)
|
20
23
|
end
|
21
24
|
end
|
22
25
|
|
23
|
-
def copy_file(dest:, src
|
24
|
-
ingest_filesystem.with_readable(src, "r") do |
|
25
|
-
metis_filesystem.do_streaming_upload(
|
26
|
+
def copy_file(dest:, src:, &block)
|
27
|
+
ingest_filesystem.with_readable(src, "r") do |io|
|
28
|
+
metis_filesystem.do_streaming_upload(io, dest, ingest_filesystem.stat(src).size)
|
29
|
+
yield src if block_given?
|
26
30
|
end
|
27
31
|
end
|
28
32
|
end
|
data/lib/etna/filesystem.rb
CHANGED
@@ -3,8 +3,7 @@ require 'fileutils'
|
|
3
3
|
require 'open3'
|
4
4
|
require 'securerandom'
|
5
5
|
require 'concurrent-ruby'
|
6
|
-
require '
|
7
|
-
require 'net/ssh'
|
6
|
+
require 'curb'
|
8
7
|
|
9
8
|
module Etna
|
10
9
|
# A class that encapsulates opening / reading file system entries that abstracts normal file access in order
|
@@ -377,55 +376,116 @@ module Etna
|
|
377
376
|
end
|
378
377
|
|
379
378
|
class SftpFilesystem < Filesystem
|
379
|
+
include WithPipeConsumer
|
380
|
+
|
381
|
+
class SftpFile
|
382
|
+
attr_reader :size, :name
|
383
|
+
|
384
|
+
def initialize(metadata)
|
385
|
+
@metadata_parts = metadata.split(" ")
|
386
|
+
@size = @metadata_parts[4].to_i
|
387
|
+
@perms = @metadata_parts.first
|
388
|
+
@name = @metadata_parts[8]
|
389
|
+
end
|
390
|
+
end
|
391
|
+
|
380
392
|
def initialize(host:, username:, password: nil, port: 22, **args)
|
381
393
|
@username = username
|
382
394
|
@password = password
|
383
395
|
@host = host
|
384
396
|
@port = port
|
397
|
+
|
398
|
+
@dir_listings = {}
|
399
|
+
end
|
400
|
+
|
401
|
+
def url(src)
|
402
|
+
"sftp://#{@host}/#{src}"
|
385
403
|
end
|
386
404
|
|
387
|
-
def
|
388
|
-
@
|
405
|
+
def authn
|
406
|
+
"#{@username}:#{@password}"
|
389
407
|
end
|
390
408
|
|
391
|
-
def
|
392
|
-
|
393
|
-
|
394
|
-
|
409
|
+
def curl_cmd(path, opts=[])
|
410
|
+
connection = Curl::Easy.new(url(path))
|
411
|
+
connection.http_auth_types = :basic
|
412
|
+
connection.username = @username
|
413
|
+
connection.password = @password
|
395
414
|
|
396
|
-
|
415
|
+
connection
|
416
|
+
end
|
417
|
+
|
418
|
+
def sftp_file_from_path(src)
|
419
|
+
file = ls(::File.dirname(src)).split("\n").map do |listing|
|
420
|
+
SftpFile.new(listing)
|
421
|
+
end.select do |file|
|
422
|
+
file.name == ::File.basename(src)
|
397
423
|
end
|
424
|
+
|
425
|
+
raise "#{src} not found" if file.empty?
|
426
|
+
|
427
|
+
file.first
|
398
428
|
end
|
399
429
|
|
400
|
-
def
|
401
|
-
|
430
|
+
def mkcommand(rd, wd, file, opts, size_hint: nil)
|
431
|
+
env = {}
|
432
|
+
cmd = [env, "curl"]
|
433
|
+
|
434
|
+
cmd << "-u"
|
435
|
+
cmd << authn
|
436
|
+
cmd << "-o"
|
437
|
+
cmd << "-"
|
438
|
+
cmd << "-N"
|
439
|
+
cmd << url(file)
|
440
|
+
|
441
|
+
if opts.include?('r')
|
442
|
+
cmd << {out: wd}
|
443
|
+
end
|
444
|
+
|
445
|
+
cmd
|
402
446
|
end
|
403
447
|
|
404
|
-
def
|
405
|
-
|
448
|
+
def with_readable(src, opts = 'r', &block)
|
449
|
+
raise "#{src} does not exist" unless exist?(src)
|
450
|
+
|
451
|
+
sftp_file = sftp_file_from_path(src)
|
452
|
+
|
453
|
+
mkio(src, opts, size_hint: sftp_file.size, &block)
|
406
454
|
end
|
407
455
|
|
408
456
|
def exist?(src)
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
414
|
-
|
457
|
+
files = ls(::File.dirname(src))
|
458
|
+
files.include?(::File.basename(src))
|
459
|
+
end
|
460
|
+
|
461
|
+
def ls(dir)
|
462
|
+
dir = dir + "/" unless "/" == dir[-1] # Trailing / makes curl list directory
|
463
|
+
|
464
|
+
return @dir_listings[dir] if @dir_listings.has_key?(dir)
|
465
|
+
|
466
|
+
listing = ''
|
467
|
+
connection = curl_cmd(dir)
|
468
|
+
connection.on_body { |data| listing << data; data.size }
|
469
|
+
connection.perform
|
470
|
+
|
471
|
+
@dir_listings[dir] = listing
|
472
|
+
|
473
|
+
listing
|
415
474
|
end
|
416
475
|
|
417
476
|
def stat(src)
|
418
|
-
|
477
|
+
sftp_file_from_path(src)
|
419
478
|
end
|
420
479
|
end
|
421
480
|
|
422
481
|
class Mock < Filesystem
|
423
482
|
class MockStat
|
424
|
-
def initialize
|
483
|
+
def initialize(io)
|
484
|
+
@io = io
|
425
485
|
end
|
426
486
|
|
427
487
|
def size
|
428
|
-
0
|
488
|
+
@io.respond_to?(:length) ? @io.length : 0
|
429
489
|
end
|
430
490
|
end
|
431
491
|
|
@@ -499,7 +559,7 @@ module Etna
|
|
499
559
|
end
|
500
560
|
|
501
561
|
def stat(src)
|
502
|
-
@files[src].respond_to?(:stat) ? @files[src].stat : MockStat.new
|
562
|
+
@files[src].respond_to?(:stat) ? @files[src].stat : MockStat.new(@files[src])
|
503
563
|
end
|
504
564
|
end
|
505
565
|
end
|
data/lib/etna/remote.rb
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
require "net/ssh"
|
2
|
+
|
3
|
+
module Etna
|
4
|
+
class RemoteSSH
|
5
|
+
class RemoteSSHError < Exception
|
6
|
+
end
|
7
|
+
|
8
|
+
def initialize(host:, username:, password: nil, port: 22, root:, **args)
|
9
|
+
@username = username
|
10
|
+
@password = password
|
11
|
+
@host = host
|
12
|
+
@port = port
|
13
|
+
@root = root
|
14
|
+
end
|
15
|
+
|
16
|
+
def ssh
|
17
|
+
@ssh ||= Net::SSH.start(@host, @username, password: @password)
|
18
|
+
end
|
19
|
+
|
20
|
+
def mkdir_p(dir)
|
21
|
+
output = ssh.exec!("mkdir -p #{dir}")
|
22
|
+
|
23
|
+
raise RemoteSSHError.new("Unable to mkdir -p, #{output}") unless 0 == output.exitstatus
|
24
|
+
end
|
25
|
+
|
26
|
+
def lftp_get(username:, password:, host:, remote_filename:, &block)
|
27
|
+
full_local_path = ::File.join(@root, host, remote_filename)
|
28
|
+
full_local_dir = ::File.dirname(full_local_path)
|
29
|
+
mkdir_p(full_local_dir)
|
30
|
+
|
31
|
+
cmd = "lftp sftp://#{username}:#{password}@#{host} -e \"get #{remote_filename} -o #{full_local_path}; bye\""
|
32
|
+
|
33
|
+
output = ssh.exec!(cmd)
|
34
|
+
raise RemoteSSHError.new("LFTP get failure: #{output}") unless 0 == output.exitstatus
|
35
|
+
yield remote_filename if block_given?
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: etna
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.41
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Saurabh Asthana
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-08-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rack
|
@@ -95,19 +95,33 @@ dependencies:
|
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: '0'
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
|
-
name:
|
98
|
+
name: curb
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
101
|
- - ">="
|
102
102
|
- !ruby/object:Gem::Version
|
103
|
-
version:
|
103
|
+
version: '0'
|
104
104
|
type: :runtime
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
108
|
- - ">="
|
109
109
|
- !ruby/object:Gem::Version
|
110
|
-
version:
|
110
|
+
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: net-ssh
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :runtime
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
111
125
|
description: See summary
|
112
126
|
email: Saurabh.Asthana@ucsf.edu
|
113
127
|
executables:
|
@@ -186,6 +200,7 @@ files:
|
|
186
200
|
- lib/etna/metrics.rb
|
187
201
|
- lib/etna/multipart_serializable_nested_hash.rb
|
188
202
|
- lib/etna/parse_body.rb
|
203
|
+
- lib/etna/remote.rb
|
189
204
|
- lib/etna/route.rb
|
190
205
|
- lib/etna/server.rb
|
191
206
|
- lib/etna/sign_service.rb
|