etna 0.1.40 → 0.1.41

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e83d3c30c23693011d59b103f8b5a9372950eba1b3123376a288826204d47c62
-  data.tar.gz: c3b50952e71d91f4361a87e872ba5532849644abe42fbc9e480549726a6bbf93
+  metadata.gz: 44b6c9fde1d01fab95038052180531fafd4a120e77ffbd12a8ac14e28122cd05
+  data.tar.gz: e1cb315eaeb4d35efc9e7f181e0cad767833b98f7ce3f26386e040c758c8483c
 SHA512:
-  metadata.gz: c3c561cdd7db4631c267f2edba8421b94baba5de790ca1ff9edd06cd21d039d7224d868eaa43ba7090b4114b595830ac777c1c4688ec15c9320cde90530be4e5
-  data.tar.gz: 7148582a82f7e90357ed6658c7337bce7b34dd532b4a899ac27a2d74f40f34224cb6c225b2aa4db4164c467aa8a5c189d7acf28a1c462097e619e9875fbdc4d1
+  metadata.gz: 3129a01c4181a772ab6752ed319396485e06f774b8091bff1ed9cbf6543a34820e0cc0706fd313ce5b3fb848a25f347de17bc376dc66b36458975d6298f48608
+  data.tar.gz: 82c57cff18c826a0a2664a63e0a531df57c75af30ceb3408d1d8ec1b9cb942fd25bd11253e6d880db6973df4cd60ade0764fac798a543af1e7601bfcc21abc45

data/lib/etna.rb

@@ -22,6 +22,7 @@ require_relative './etna/filesystem'
 require_relative './etna/formatting'
 require_relative './etna/cwl'
 require_relative './etna/metrics'
+require_relative './etna/remote'
 
 class EtnaApp
   include Etna::Application

data/lib/etna/application.rb

@@ -60,17 +60,6 @@ module Etna::Application
     end
   end
 
-  # This will cause metrics to persist to a file.
-  # NOTE -- /tmp/metrics.bin should be a persistent mount when using this.
-  # You will still need to export metrics in the text format for the node_exporter on the host machine to
-  # export them to prometheus.  Ensure that the /tmp/metrics.bin is on a named volume or a bind mount, either is fine.
-  def enable_job_metrics!
-    require 'prometheus'
-    Prometheus::Client.config.data_store = Prometheus::Client::DataStores::DirectFileStore.new({
-      dir: "/tmp/metrics.bin"
-    })
-  end
-
   def setup_yabeda
     application = self.id
     Yabeda.configure do
@@ -198,7 +187,8 @@ module Etna::Application
 
         Yabeda.etna.last_command_completion.set(tags, Time.now.to_i)
         Yabeda.etna.command_runtime.measure(tags, dur)
-        write_job_metrics("run_command")
+
+        write_job_metrics("run_command.#{cmd.class.name}")
       end
     end
   end

data/lib/etna/clients/metis/workflows/ingest_metis_data_workflow.rb

@@ -9,20 +9,24 @@ module Etna
         # Since we are doing manual triage of files,
         #   do not automatically copy directory trees.
         # srcs must be a list of full paths to files.
-        def copy_files(srcs)
+        def copy_files(srcs, &block)
           srcs.each do |src|
-            next unless ingest_filesystem.exist?(src)
+            if !ingest_filesystem.exist?(src)
+              logger&.warn("#{src} does not exist on source filesystem. Skipping.")
+              next
+            end
 
             logger&.info("Copying file #{src} (#{Etna::Formatting.as_size(ingest_filesystem.stat(src).size)})")
 
             # For ingestion triage, just copy over the exact path + filename.
-            copy_file(dest: src, src: src)
+            copy_file(dest: src, src: src, &block)
           end
         end
 
-        def copy_file(dest:, src:)
-          ingest_filesystem.with_readable(src, "r") do |file|
-            metis_filesystem.do_streaming_upload(file, dest, file.size)
+        def copy_file(dest:, src:, &block)
+          ingest_filesystem.with_readable(src, "r") do |io|
+            metis_filesystem.do_streaming_upload(io, dest, ingest_filesystem.stat(src).size)
+            yield src if block_given?
           end
         end
       end

data/lib/etna/filesystem.rb

@@ -3,8 +3,7 @@ require 'fileutils'
 require 'open3'
 require 'securerandom'
 require 'concurrent-ruby'
-require 'net/sftp'
-require 'net/ssh'
+require 'curb'
 
 module Etna
   # A class that encapsulates opening / reading file system entries that abstracts normal file access in order
@@ -377,55 +376,116 @@ module Etna
     end
 
     class SftpFilesystem < Filesystem
+      include WithPipeConsumer
+
+      class SftpFile
+        attr_reader :size, :name
+
+        def initialize(metadata)
+          @metadata_parts = metadata.split(" ")
+          @size = @metadata_parts[4].to_i
+          @perms = @metadata_parts.first
+          @name = @metadata_parts[8]
+        end
+      end
+
       def initialize(host:, username:, password: nil, port: 22, **args)
         @username = username
         @password = password
         @host = host
         @port = port
+
+        @dir_listings = {}
+      end
+
+      def url(src)
+        "sftp://#{@host}/#{src}"
       end
 
-      def ssh
-        @ssh ||= Net::SSH.start(@host, @username, password: @password)
+      def authn
+        "#{@username}:#{@password}"
       end
 
-      def sftp
-        @sftp ||= begin
-          conn = Net::SFTP::Session.new(ssh)
-          conn.loop { conn.opening? }
+      def curl_cmd(path, opts=[])
+        connection = Curl::Easy.new(url(path))
+        connection.http_auth_types = :basic
+        connection.username = @username
+        connection.password = @password
 
-          conn
+        connection
+      end
+
+      def sftp_file_from_path(src)
+        file = ls(::File.dirname(src)).split("\n").map do |listing|
+          SftpFile.new(listing)
+        end.select do |file|
+          file.name == ::File.basename(src)
         end
+
+        raise "#{src} not found" if file.empty?
+
+        file.first
       end
 
-      def with_readable(src, opts = 'r', &block)
-        sftp.file.open(src, opts, &block)
+      def mkcommand(rd, wd, file, opts, size_hint: nil)
+        env = {}
+        cmd = [env, "curl"]
+
+        cmd << "-u"
+        cmd << authn
+        cmd << "-o"
+        cmd << "-"
+        cmd << "-N"
+        cmd << url(file)
+
+        if opts.include?('r')
+          cmd << {out: wd}
+        end
+
+        cmd
       end
 
-      def ls(dir)
-        sftp.dir.entries(dir)
+      def with_readable(src, opts = 'r', &block)
+        raise "#{src} does not exist" unless exist?(src)
+
+        sftp_file = sftp_file_from_path(src)
+
+        mkio(src, opts, size_hint: sftp_file.size, &block)
       end
 
       def exist?(src)
-        begin
-          sftp.file.open(src)
-        rescue Net::SFTP::StatusException
-          return false
-        end
-        return true
+        files = ls(::File.dirname(src))
+        files.include?(::File.basename(src))
+      end
+
+      def ls(dir)
+        dir = dir + "/" unless "/" == dir[-1]  # Trailing / makes curl list directory
+
+        return @dir_listings[dir] if @dir_listings.has_key?(dir)
+
+        listing = ''
+        connection = curl_cmd(dir)
+        connection.on_body { |data| listing << data; data.size }
+        connection.perform
+
+        @dir_listings[dir] = listing
+
+        listing
       end
 
       def stat(src)
-        sftp.file.open(src).stat
+        sftp_file_from_path(src)
       end
     end
 
     class Mock < Filesystem
       class MockStat
-        def initialize
+        def initialize(io)
+          @io = io
         end
 
         def size
-          0
+          @io.respond_to?(:length) ? @io.length : 0
         end
       end
 
@@ -499,7 +559,7 @@ module Etna
       end
 
       def stat(src)
-        @files[src].respond_to?(:stat) ? @files[src].stat : MockStat.new
+        @files[src].respond_to?(:stat) ? @files[src].stat : MockStat.new(@files[src])
       end
     end
   end

data/lib/etna/remote.rb

@@ -0,0 +1,38 @@
+require "net/ssh"
+
+module Etna
+  class RemoteSSH
+    class RemoteSSHError < Exception
+    end
+
+    def initialize(host:, username:, password: nil, port: 22, root:, **args)
+      @username = username
+      @password = password
+      @host = host
+      @port = port
+      @root = root
+    end
+
+    def ssh
+      @ssh ||= Net::SSH.start(@host, @username, password: @password)
+    end
+
+    def mkdir_p(dir)
+      output = ssh.exec!("mkdir -p #{dir}")
+
+      raise RemoteSSHError.new("Unable to mkdir -p, #{output}") unless 0 == output.exitstatus
+    end
+
+    def lftp_get(username:, password:, host:, remote_filename:, &block)
+      full_local_path = ::File.join(@root, host, remote_filename)
+      full_local_dir = ::File.dirname(full_local_path)
+      mkdir_p(full_local_dir)
+
+      cmd = "lftp sftp://#{username}:#{password}@#{host}  -e \"get #{remote_filename} -o #{full_local_path}; bye\""
+
+      output = ssh.exec!(cmd)
+      raise RemoteSSHError.new("LFTP get failure: #{output}") unless 0 == output.exitstatus
+      yield remote_filename if block_given?
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: etna
 version: !ruby/object:Gem::Version
-  version: 0.1.40
+  version: 0.1.41
 platform: ruby
 authors:
 - Saurabh Asthana
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-07-29 00:00:00.000000000 Z
+date: 2021-08-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -95,19 +95,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: net-sftp
+  name: curb
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: net-ssh
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: See summary
 email: Saurabh.Asthana@ucsf.edu
 executables:
@@ -186,6 +200,7 @@ files:
 - lib/etna/metrics.rb
 - lib/etna/multipart_serializable_nested_hash.rb
 - lib/etna/parse_body.rb
+- lib/etna/remote.rb
 - lib/etna/route.rb
 - lib/etna/server.rb
 - lib/etna/sign_service.rb