etna 0.1.37 → 0.1.41

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 484c950fffbafcf5132df923bc0ef88a79faa6c285ccde653805aac526e4a97e
-  data.tar.gz: f513b75b048e816acc494c09cb0eccbc2c38cda02c2723266fe7bd98c8595f3a
+  metadata.gz: 44b6c9fde1d01fab95038052180531fafd4a120e77ffbd12a8ac14e28122cd05
+  data.tar.gz: e1cb315eaeb4d35efc9e7f181e0cad767833b98f7ce3f26386e040c758c8483c
 SHA512:
-  metadata.gz: 6ad0e99925dc6110c8f80f4a1e1417d610b28838497f09afe8be1c69a65fec95a7d549af987aaccde2b42480347bf336838be5398ef580d3024476206b4d611d
-  data.tar.gz: 44eb05596c635e7ab96df7971fed6b24bfee7bd545238e173e3d1b57bdcfe7b30db5deea355d2711b192d026d8fb72afc32cbcbcf4a83ef92848184433b7a813
+  metadata.gz: 3129a01c4181a772ab6752ed319396485e06f774b8091bff1ed9cbf6543a34820e0cc0706fd313ce5b3fb848a25f347de17bc376dc66b36458975d6298f48608
+  data.tar.gz: 82c57cff18c826a0a2664a63e0a531df57c75af30ceb3408d1d8ec1b9cb942fd25bd11253e6d880db6973df4cd60ade0764fac798a543af1e7601bfcc21abc45

data/lib/etna.rb

@@ -22,6 +22,7 @@ require_relative './etna/filesystem'
 require_relative './etna/formatting'
 require_relative './etna/cwl'
 require_relative './etna/metrics'
+require_relative './etna/remote'
 
 class EtnaApp
   include Etna::Application

data/lib/etna/application.rb

@@ -61,11 +61,49 @@ module Etna::Application
   end
 
   def setup_yabeda
+    application = self.id
+    Yabeda.configure do
+      default_tag :application, application
+
+      group :etna do
+        histogram :response_time do
+          comment "Time spent by a controller returning a response"
+          unit :seconds
+          tags [:controller, :action, :user_hash, :project_name]
+          buckets [0.01, 0.1, 0.3, 0.5, 1, 5]
+        end
+
+        counter :visits do
+          comment "Counts visits to the controller"
+          tags [:controller, :action, :user_hash, :project_name]
+        end
+
+        counter :rollbar_errors do
+          comment "Counts errors detected by and sent to rollbar"
+        end
+
+        gauge :last_command_completion do
+          comment "Unix time of last time command was completed"
+          tags [:command, :status, :application]
+        end
+
+        histogram :command_runtime do
+          comment "Time spent processing a given command"
+          tags [:command, :status, :application]
+          unit :seconds
+          buckets [0.1, 1, 5, 60, 300, 1500]
+        end
+      end
+    end
+
     Yabeda.configure!
   end
 
+  # Writes all metrics currently gathered to a text format prometheus file.  If /tmp/metrics.prom is bind mounted
+  # to the host directed bound to the node_exporter's file exporter directory, these will be exported to
+  # prometheus.  Combine this enable_job_metrics! for maximum effect.
   def write_job_metrics(name)
-    node_metrics_dir = config(:node_metrics_dir) || "/tmp/metrics.prom"
+    node_metrics_dir = "/tmp/metrics.prom"
     ::FileUtils.mkdir_p(node_metrics_dir)
 
     tmp_file = ::File.join(node_metrics_dir, "#{name}.prom.$$")
@@ -126,17 +164,33 @@ module Etna::Application
   end
 
   def run_command(config, *args, &block)
+    application = self.id
+    status = 'success'
+    start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
     cmd, cmd_args, cmd_kwds = find_command(*args)
-    cmd.setup(config)
 
-    if block_given?
-      return unless yield [cmd, cmd_args]
-    end
+    begin
+      cmd.setup(config)
+      if block_given?
+        return unless yield [cmd, cmd_args]
+      end
 
-    cmd.execute(*cmd.fill_in_missing_params(cmd_args), **cmd_kwds)
-  rescue => e
-    Rollbar.error(e)
-    raise
+      cmd.execute(*cmd.fill_in_missing_params(cmd_args), **cmd_kwds)
+    rescue => e
+      Rollbar.error(e)
+      status = 'failed'
+      raise
+    ensure
+      if defined?(Yabeda) && Yabeda.configured?
+        tags = { command: cmd.class.name, status: status, application: application }
+        dur = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
+
+        Yabeda.etna.last_command_completion.set(tags, Time.now.to_i)
+        Yabeda.etna.command_runtime.measure(tags, dur)
+
+        write_job_metrics("run_command.#{cmd.class.name}")
+      end
+    end
   end
 end
 

data/lib/etna/clients/metis/client.rb

@@ -210,6 +210,52 @@ module Etna
         }
       end
 
+      def resolve_conflicts_and_verify_rename(project_name:, source_bucket:, dest_bucket:, folder:, file:)
+        parent_folder = ::File.dirname(file.file_path)
+
+        should_rename_original = true
+
+        # If the destination folder already exists, check to see if
+        #   the file also exists, otherwise we risk a
+        #   rename conflict.
+        create_folder_request = CreateFolderRequest.new(
+          project_name: project_name,
+          bucket_name: dest_bucket,
+          folder_path: parent_folder
+        )
+
+        if folder_exists?(create_folder_request)
+          # If file exists in destination, delete the older file.
+          list_dest_folder_request = Etna::Clients::Metis::ListFolderRequest.new(
+            bucket_name: dest_bucket,
+            project_name: project_name,
+            folder_path: parent_folder
+          )
+
+          dest_file = list_folder(list_dest_folder_request).files.all.find { |f| f.file_name == file.file_name }
+
+          if (dest_file && file.updated_at <= dest_file.updated_at)
+            # Delete source file if it's out of date
+            delete_file(Etna::Clients::Metis::DeleteFileRequest.new(
+              bucket_name: source_bucket,
+              project_name: project_name,
+              file_path: file.file_path,
+            ))
+
+            should_rename_original = false
+          elsif (dest_file && file.updated_at > dest_file.updated_at)
+            # Delete dest file if it's out of date
+            delete_file(Etna::Clients::Metis::DeleteFileRequest.new(
+              bucket_name: dest_bucket,
+              project_name: project_name,
+              file_path: dest_file.file_path,
+            ))
+          end
+        end
+
+        should_rename_original
+      end
+
       def recursively_rename_folder(project_name:, source_bucket:, dest_bucket:, folder:)
         folder_contents = list_folder(
           Etna::Clients::Metis::ListFolderRequest.new(
@@ -228,6 +274,13 @@ module Etna
         end
 
         folder_contents.files.all.each do |file|
+          should_rename = resolve_conflicts_and_verify_rename(
+            project_name: project_name,
+            source_bucket: source_bucket,
+            dest_bucket: dest_bucket,
+            folder: folder,
+            file: file)
+
           rename_file(Etna::Clients::Metis::RenameFileRequest.new(
             bucket_name: source_bucket,
             project_name: project_name,
@@ -235,7 +288,7 @@ module Etna
             new_bucket_name: dest_bucket,
             new_file_path: file.file_path,
             create_parent: true)
-          )
+          ) if should_rename
         end
 
         # Now delete the source folder

data/lib/etna/clients/metis/workflows.rb

@@ -1,3 +1,4 @@
-require_relative './workflows/metis_download_workflow'
-require_relative './workflows/metis_upload_workflow'
-require_relative './workflows/sync_metis_data_workflow'
+require_relative "./workflows/metis_download_workflow"
+require_relative "./workflows/metis_upload_workflow"
+require_relative "./workflows/sync_metis_data_workflow"
+require_relative "./workflows/ingest_metis_data_workflow"

data/lib/etna/clients/metis/workflows/ingest_metis_data_workflow.rb

@@ -0,0 +1,35 @@
+require "ostruct"
+require "fileutils"
+require "tempfile"
+
+module Etna
+  module Clients
+    class Metis
+      class IngestMetisDataWorkflow < Struct.new(:metis_filesystem, :ingest_filesystem, :logger, keyword_init: true)
+        # Since we are doing manual triage of files,
+        #   do not automatically copy directory trees.
+        # srcs must be a list of full paths to files.
+        def copy_files(srcs, &block)
+          srcs.each do |src|
+            if !ingest_filesystem.exist?(src)
+              logger&.warn("#{src} does not exist on source filesystem. Skipping.")
+              next
+            end
+
+            logger&.info("Copying file #{src} (#{Etna::Formatting.as_size(ingest_filesystem.stat(src).size)})")
+
+            # For ingestion triage, just copy over the exact path + filename.
+            copy_file(dest: src, src: src, &block)
+          end
+        end
+
+        def copy_file(dest:, src:, &block)
+          ingest_filesystem.with_readable(src, "r") do |io|
+            metis_filesystem.do_streaming_upload(io, dest, ingest_filesystem.stat(src).size)
+            yield src if block_given?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/etna/clients/metis/workflows/metis_upload_workflow.rb

@@ -30,7 +30,7 @@ module Etna
               project_name: project_name,
               bucket_name: bucket_name,
               folder_path: dir,
-          ))
+          )) unless dir == "."
 
           authorize_response = metis_client.authorize_upload(AuthorizeUploadRequest.new(
               project_name: project_name,

data/lib/etna/controller.rb

@@ -19,25 +19,30 @@ module Etna
       @logger.warn(request_msg(line))
     end
 
+    def handle_error(e)
+      case e
+      when Etna::Error
+        Rollbar.error(e)
+        @logger.error(request_msg("Exiting with #{e.status}, #{e.message}"))
+        return failure(e.status, error: e.message)
+      else
+        Rollbar.error(e)
+        @logger.error(request_msg('Caught unspecified error'))
+        @logger.error(request_msg(e.message))
+        e.backtrace.each do |trace|
+          @logger.error(request_msg(trace))
+        end
+        return failure(500, error: 'Server error.')
+      end
+    end
+
     def response(&block)
       return instance_eval(&block) if block_given?
-
       return send(@action) if @action
 
-
       [501, {}, ['This controller is not implemented.']]
-    rescue Etna::Error => e
-      Rollbar.error(e)
-      @logger.error(request_msg("Exiting with #{e.status}, #{e.message}"))
-      return failure(e.status, error: e.message)
     rescue Exception => e
-      Rollbar.error(e)
-      @logger.error(request_msg('Caught unspecified error'))
-      @logger.error(request_msg(e.message))
-      e.backtrace.each do |trace|
-        @logger.error(request_msg(trace))
-      end
-      return failure(500, error: 'Server error.')
+      handle_error(e)
     end
 
     def require_params(*params)

data/lib/etna/filesystem.rb

@@ -3,6 +3,7 @@ require 'fileutils'
 require 'open3'
 require 'securerandom'
 require 'concurrent-ruby'
+require 'curb'
 
 module Etna
   # A class that encapsulates opening / reading file system entries that abstracts normal file access in order
@@ -50,6 +51,11 @@ module Etna
       ::FileUtils.mv(src, dest)
     end
 
+    def stat(src)
+      raise "stat not supported by #{self.class.name}" unless self.class == Filesystem
+      ::File.stat(src)
+    end
+
     class EmptyIO < StringIO
       def write(*args)
         # Do nothing -- always leave empty
@@ -369,7 +375,120 @@ module Etna
       end
     end
 
+    class SftpFilesystem < Filesystem
+      include WithPipeConsumer
+
+      class SftpFile
+        attr_reader :size, :name
+
+        def initialize(metadata)
+          @metadata_parts = metadata.split(" ")
+          @size = @metadata_parts[4].to_i
+          @perms = @metadata_parts.first
+          @name = @metadata_parts[8]
+        end
+      end
+
+      def initialize(host:, username:, password: nil, port: 22, **args)
+        @username = username
+        @password = password
+        @host = host
+        @port = port
+
+        @dir_listings = {}
+      end
+
+      def url(src)
+        "sftp://#{@host}/#{src}"
+      end
+
+      def authn
+        "#{@username}:#{@password}"
+      end
+
+      def curl_cmd(path, opts=[])
+        connection = Curl::Easy.new(url(path))
+        connection.http_auth_types = :basic
+        connection.username = @username
+        connection.password = @password
+
+        connection
+      end
+
+      def sftp_file_from_path(src)
+        file = ls(::File.dirname(src)).split("\n").map do |listing|
+          SftpFile.new(listing)
+        end.select do |file|
+          file.name == ::File.basename(src)
+        end
+
+        raise "#{src} not found" if file.empty?
+
+        file.first
+      end
+
+      def mkcommand(rd, wd, file, opts, size_hint: nil)
+        env = {}
+        cmd = [env, "curl"]
+
+        cmd << "-u"
+        cmd << authn
+        cmd << "-o"
+        cmd << "-"
+        cmd << "-N"
+        cmd << url(file)
+
+        if opts.include?('r')
+          cmd << {out: wd}
+        end
+
+        cmd
+      end
+
+      def with_readable(src, opts = 'r', &block)
+        raise "#{src} does not exist" unless exist?(src)
+
+        sftp_file = sftp_file_from_path(src)
+
+        mkio(src, opts, size_hint: sftp_file.size, &block)
+      end
+
+      def exist?(src)
+        files = ls(::File.dirname(src))
+        files.include?(::File.basename(src))
+      end
+
+      def ls(dir)
+        dir = dir + "/" unless "/" == dir[-1]  # Trailing / makes curl list directory
+
+        return @dir_listings[dir] if @dir_listings.has_key?(dir)
+
+        listing = ''
+        connection = curl_cmd(dir)
+        connection.on_body { |data| listing << data; data.size }
+        connection.perform
+
+        @dir_listings[dir] = listing
+
+        listing
+      end
+
+      def stat(src)
+        sftp_file_from_path(src)
+      end
+    end
+
     class Mock < Filesystem
+      class MockStat
+        def initialize(io)
+          @io = io
+        end
+
+        def size
+          @io.respond_to?(:length) ? @io.length : 0
+        end
+      end
+
       def initialize(&new_io)
         @files = {}
         @dirs = {}
@@ -438,6 +557,10 @@ module Etna
       def exist?(src)
         @files.include?(src) || @dirs.include?(src)
       end
+
+      def stat(src)
+        @files[src].respond_to?(:stat) ? @files[src].stat : MockStat.new(@files[src])
+      end
     end
   end
 end

data/lib/etna/logger.rb

@@ -40,6 +40,10 @@ module Etna
         error(trace)
       end
 
+      if defined? Yabeda
+        Yabeda.etna.rollbar_errors.increment({}, 1) rescue nil
+      end
+
       Rollbar.error(e)
     end
 

data/lib/etna/remote.rb

@@ -0,0 +1,38 @@
+require "net/ssh"
+
+module Etna
+  class RemoteSSH
+    class RemoteSSHError < Exception
+    end
+
+    def initialize(host:, username:, password: nil, port: 22, root:, **args)
+      @username = username
+      @password = password
+      @host = host
+      @port = port
+      @root = root
+    end
+
+    def ssh
+      @ssh ||= Net::SSH.start(@host, @username, password: @password)
+    end
+
+    def mkdir_p(dir)
+      output = ssh.exec!("mkdir -p #{dir}")
+
+      raise RemoteSSHError.new("Unable to mkdir -p, #{output}") unless 0 == output.exitstatus
+    end
+
+    def lftp_get(username:, password:, host:, remote_filename:, &block)
+      full_local_path = ::File.join(@root, host, remote_filename)
+      full_local_dir = ::File.dirname(full_local_path)
+      mkdir_p(full_local_dir)
+
+      cmd = "lftp sftp://#{username}:#{password}@#{host}  -e \"get #{remote_filename} -o #{full_local_path}; bye\""
+
+      output = ssh.exec!(cmd)
+      raise RemoteSSHError.new("LFTP get failure: #{output}") unless 0 == output.exitstatus
+      yield remote_filename if block_given?
+    end
+  end
+end

data/lib/etna/route.rb

@@ -1,3 +1,6 @@
+require 'digest'
+require 'date'
+
 module Etna
   class Route
     attr_reader :name
@@ -58,6 +61,63 @@ module Etna
     end
 
     def call(app, request)
+      start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+      try_yabeda(request)  do |tags|
+        Yabeda.etna.visits.increment(tags)
+      end
+
+      begin
+        process_call(app, request)
+      ensure
+        try_yabeda(request) do |tags|
+          dur = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
+          Yabeda.etna.response_time.measure(tags, dur)
+        end
+      end
+    end
+
+    def hash_user_email(email)
+      secret = Etna::Application.instance.config(:user_hash_secret) || 'notsosecret'
+      digest = email + secret + Date.today.to_s
+
+      if @name
+        digest += @name.to_s
+      else
+        digest += @route.to_s
+      end
+
+      Digest::MD5.hexdigest(digest)
+    end
+
+    def try_yabeda(request, &block)
+      if @action
+        controller, action = @action.split('#')
+      elsif @name
+        controller = "none"
+        action = @name
+      else
+        controller = "none"
+        action = @route
+      end
+
+      params = request.env['rack.request.params']
+      user = request.env['etna.user']
+      user_hash = user ? hash_user_email(user.email) : 'unknown'
+      project_name = "unknown"
+
+      if params && (params.include?(:project_name) || params.include?('project_name'))
+        project_name = params[:project_name] || params['project_name']
+      end
+
+      begin
+        block.call({ controller: controller, action: action, user_hash: user_hash, project_name: project_name })
+      rescue => e
+        raise e unless Etna::Application.instance.environment == :production
+      end
+    end
+
+    def process_call(app, request)
       update_params(request)
 
       unless authorized?(request)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: etna
 version: !ruby/object:Gem::Version
-  version: 0.1.37
+  version: 0.1.41
 platform: ruby
 authors:
 - Saurabh Asthana
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-05-27 00:00:00.000000000 Z
+date: 2021-08-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -94,6 +94,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: curb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: net-ssh
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: See summary
 email: Saurabh.Asthana@ucsf.edu
 executables:
@@ -144,6 +172,7 @@ files:
 - lib/etna/clients/metis/client.rb
 - lib/etna/clients/metis/models.rb
 - lib/etna/clients/metis/workflows.rb
+- lib/etna/clients/metis/workflows/ingest_metis_data_workflow.rb
 - lib/etna/clients/metis/workflows/metis_download_workflow.rb
 - lib/etna/clients/metis/workflows/metis_upload_workflow.rb
 - lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb
@@ -171,6 +200,7 @@ files:
 - lib/etna/metrics.rb
 - lib/etna/multipart_serializable_nested_hash.rb
 - lib/etna/parse_body.rb
+- lib/etna/remote.rb
 - lib/etna/route.rb
 - lib/etna/server.rb
 - lib/etna/sign_service.rb