etna 0.1.37 → 0.1.38

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 484c950fffbafcf5132df923bc0ef88a79faa6c285ccde653805aac526e4a97e
-  data.tar.gz: f513b75b048e816acc494c09cb0eccbc2c38cda02c2723266fe7bd98c8595f3a
+  metadata.gz: e7c665ee736c06bf1b566467985638cd41a5b3536f848fe2971174aab26ba3a4
+  data.tar.gz: 22bd58b6dd1a4902ee91f6588f53cf56a4bec9db5b42d9236ed9cd7c4310a2f0
 SHA512:
-  metadata.gz: 6ad0e99925dc6110c8f80f4a1e1417d610b28838497f09afe8be1c69a65fec95a7d549af987aaccde2b42480347bf336838be5398ef580d3024476206b4d611d
-  data.tar.gz: 44eb05596c635e7ab96df7971fed6b24bfee7bd545238e173e3d1b57bdcfe7b30db5deea355d2711b192d026d8fb72afc32cbcbcf4a83ef92848184433b7a813
+  metadata.gz: 04ad878b6f9593a06b65b7652ba3b6da9b0a2aa4de57981ef91247b19eb606e5018cc01e45d033ccdaf5b755cb92149e92a1d26218c47fce6c1a77085dce300e
+  data.tar.gz: 26134c1553f33e59e13f5ec737bd11b0bd8e62e1f54c3b1c5963560d1078cb00d5ce0255c01bddae86dc19f9bb38ae5a0dfc6b9d2845e955e55df0299ad1f456

data/lib/etna/application.rb

@@ -60,12 +60,61 @@ module Etna::Application
     end
   end
 
+  # This will cause metrics to persist to a file.
+  # NOTE -- /tmp/metrics.bin should be a persistent mount when using this.
+  # You will still need to export metrics in the text format for the node_exporter on the host machine to
+  # export them to prometheus.  Ensure that the /tmp/metrics.bin is on a named volume or a bind mount, either is fine.
+  def enable_job_metrics!
+    require 'prometheus'
+    Prometheus::Client.config.data_store = Prometheus::Client::DataStores::DirectFileStore.new({
+      dir: "/tmp/metrics.bin"
+    })
+  end
+
   def setup_yabeda
+    application = self.id
+    Yabeda.configure do
+      default_tag :application, application
+
+      group :etna do
+        histogram :response_time do
+          comment "Time spent by a controller returning a response"
+          unit :seconds
+          tags [:controller, :action, :user_hash, :project_name]
+          buckets [0.01, 0.1, 0.3, 0.5, 1, 5]
+        end
+
+        counter :visits do
+          comment "Counts visits to the controller"
+          tags [:controller, :action, :user_hash, :project_name]
+        end
+
+        counter :rollbar_errors do
+          comment "Counts errors detected by and sent to rollbar"
+        end
+
+        gauge :last_command_completion do
+          comment "Unix time of last time command was completed"
+          tags [:command, :status, :application]
+        end
+
+        histogram :command_runtime do
+          comment "Time spent processing a given command"
+          tags [:command, :status, :application]
+          unit :seconds
+          buckets [0.1, 1, 5, 60, 300, 1500]
+        end
+      end
+    end
+
     Yabeda.configure!
   end
 
+  # Writes all metrics currently gathered to a text format prometheus file.  If /tmp/metrics.prom is bind mounted
+  # to the host directed bound to the node_exporter's file exporter directory, these will be exported to
+  # prometheus.  Combine this enable_job_metrics! for maximum effect.
   def write_job_metrics(name)
-    node_metrics_dir = config(:node_metrics_dir) || "/tmp/metrics.prom"
+    node_metrics_dir = "/tmp/metrics.prom"
     ::FileUtils.mkdir_p(node_metrics_dir)
 
     tmp_file = ::File.join(node_metrics_dir, "#{name}.prom.$$")
@@ -126,17 +175,32 @@ module Etna::Application
   end
 
   def run_command(config, *args, &block)
+    application = self.id
+    status = 'success'
+    start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
     cmd, cmd_args, cmd_kwds = find_command(*args)
-    cmd.setup(config)
 
-    if block_given?
-      return unless yield [cmd, cmd_args]
-    end
+    begin
+      cmd.setup(config)
+      if block_given?
+        return unless yield [cmd, cmd_args]
+      end
 
-    cmd.execute(*cmd.fill_in_missing_params(cmd_args), **cmd_kwds)
-  rescue => e
-    Rollbar.error(e)
-    raise
+      cmd.execute(*cmd.fill_in_missing_params(cmd_args), **cmd_kwds)
+    rescue => e
+      Rollbar.error(e)
+      status = 'failed'
+      raise
+    ensure
+      if Yabeda.configured?
+        tags = { command: cmd.class.name, status: status, application: application }
+        dur = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
+
+        Yabeda.etna.last_command_completion.set(tags, Time.now.to_i)
+        Yabeda.etna.command_runtime.measure(tags, dur)
+        write_job_metrics("run_command")
+      end
+    end
   end
 end
 

data/lib/etna/clients/metis/workflows.rb

@@ -1,3 +1,4 @@
-require_relative './workflows/metis_download_workflow'
-require_relative './workflows/metis_upload_workflow'
-require_relative './workflows/sync_metis_data_workflow'
+require_relative "./workflows/metis_download_workflow"
+require_relative "./workflows/metis_upload_workflow"
+require_relative "./workflows/sync_metis_data_workflow"
+require_relative "./workflows/ingest_metis_data_workflow"

data/lib/etna/clients/metis/workflows/ingest_metis_data_workflow.rb

@@ -0,0 +1,31 @@
+require "ostruct"
+require "fileutils"
+require "tempfile"
+
+module Etna
+  module Clients
+    class Metis
+      class IngestMetisDataWorkflow < Struct.new(:metis_filesystem, :ingest_filesystem, :logger, keyword_init: true)
+        # Since we are doing manual triage of files,
+        #   do not automatically copy directory trees.
+        # srcs must be a list of full paths to files.
+        def copy_files(srcs)
+          srcs.each do |src|
+            next unless ingest_filesystem.exist?(src)
+
+            logger&.info("Copying file #{src} (#{Etna::Formatting.as_size(ingest_filesystem.stat(src).size)})")
+
+            # For ingestion triage, just copy over the exact path + filename.
+            copy_file(dest: src, src: src)
+          end
+        end
+
+        def copy_file(dest:, src:)
+          ingest_filesystem.with_readable(src, "r") do |file|
+            metis_filesystem.do_streaming_upload(file, dest, file.size)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/etna/clients/metis/workflows/metis_upload_workflow.rb

@@ -30,7 +30,7 @@ module Etna
               project_name: project_name,
               bucket_name: bucket_name,
               folder_path: dir,
-          ))
+          )) unless dir == "."
 
           authorize_response = metis_client.authorize_upload(AuthorizeUploadRequest.new(
               project_name: project_name,

data/lib/etna/controller.rb

@@ -19,25 +19,30 @@ module Etna
       @logger.warn(request_msg(line))
     end
 
+    def handle_error(e)
+      case e
+      when Etna::Error
+        Rollbar.error(e)
+        @logger.error(request_msg("Exiting with #{e.status}, #{e.message}"))
+        return failure(e.status, error: e.message)
+      else
+        Rollbar.error(e)
+        @logger.error(request_msg('Caught unspecified error'))
+        @logger.error(request_msg(e.message))
+        e.backtrace.each do |trace|
+          @logger.error(request_msg(trace))
+        end
+        return failure(500, error: 'Server error.')
+      end
+    end
+
     def response(&block)
       return instance_eval(&block) if block_given?
-
       return send(@action) if @action
 
-
       [501, {}, ['This controller is not implemented.']]
-    rescue Etna::Error => e
-      Rollbar.error(e)
-      @logger.error(request_msg("Exiting with #{e.status}, #{e.message}"))
-      return failure(e.status, error: e.message)
     rescue Exception => e
-      Rollbar.error(e)
-      @logger.error(request_msg('Caught unspecified error'))
-      @logger.error(request_msg(e.message))
-      e.backtrace.each do |trace|
-        @logger.error(request_msg(trace))
-      end
-      return failure(500, error: 'Server error.')
+      handle_error(e)
     end
 
     def require_params(*params)

data/lib/etna/filesystem.rb

@@ -3,6 +3,8 @@ require 'fileutils'
 require 'open3'
 require 'securerandom'
 require 'concurrent-ruby'
+require 'net/sftp'
+require 'net/ssh'
 
 module Etna
   # A class that encapsulates opening / reading file system entries that abstracts normal file access in order
@@ -50,6 +52,11 @@ module Etna
       ::FileUtils.mv(src, dest)
     end
 
+    def stat(src)
+      raise "stat not supported by #{self.class.name}" unless self.class == Filesystem
+      ::File.stat(src)
+    end
+
     class EmptyIO < StringIO
       def write(*args)
         # Do nothing -- always leave empty
@@ -369,7 +376,59 @@ module Etna
       end
     end
 
+    class SftpFilesystem < Filesystem
+      def initialize(host:, username:, password: nil, port: 22, **args)
+        @username = username
+        @password = password
+        @host = host
+        @port = port
+      end
+
+      def ssh
+        @ssh ||= Net::SSH.start(@host, @username, password: @password)
+      end
+
+      def sftp
+        @sftp ||= begin
+          conn = Net::SFTP::Session.new(ssh)
+          conn.loop { conn.opening? }
+
+          conn
+        end
+      end
+
+      def with_readable(src, opts = 'r', &block)
+        sftp.file.open(src, opts, &block)
+      end
+
+      def ls(dir)
+        sftp.dir.entries(dir)
+      end
+
+      def exist?(src)
+        begin
+          sftp.file.open(src)
+        rescue Net::SFTP::StatusException
+          return false
+        end
+        return true
+      end
+
+      def stat(src)
+        sftp.file.open(src).stat
+      end
+    end
+
     class Mock < Filesystem
+      class MockStat
+        def initialize
+        end
+
+        def size
+          0
+        end
+      end
+
       def initialize(&new_io)
         @files = {}
         @dirs = {}
@@ -438,6 +497,10 @@ module Etna
       def exist?(src)
         @files.include?(src) || @dirs.include?(src)
       end
+
+      def stat(src)
+        @files[src].respond_to?(:stat) ? @files[src].stat : MockStat.new
+      end
     end
   end
 end

data/lib/etna/logger.rb

@@ -40,6 +40,10 @@ module Etna
         error(trace)
       end
 
+      if defined? Yabeda
+        Yabeda.etna.rollbar_errors.increment({}, 1) rescue nil
+      end
+
       Rollbar.error(e)
     end
 

data/lib/etna/route.rb

@@ -1,3 +1,6 @@
+require 'digest'
+require 'date'
+
 module Etna
   class Route
     attr_reader :name
@@ -58,6 +61,63 @@ module Etna
     end
 
     def call(app, request)
+      start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+      try_yabeda(request)  do |tags|
+        Yabeda.etna.visits.increment(tags)
+      end
+
+      begin
+        process_call(app, request)
+      ensure
+        try_yabeda(request) do |tags|
+          dur = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
+          Yabeda.etna.response_time.measure(tags, dur)
+        end
+      end
+    end
+
+    def hash_user_email(email)
+      secret = Etna::Application.instance.config(:user_hash_secret) || 'notsosecret'
+      digest = email + secret + Date.today.to_s
+
+      if @name
+        digest += @name.to_s
+      else
+        digest += @route.to_s
+      end
+
+      Digest::MD5.hexdigest(digest)
+    end
+
+    def try_yabeda(request, &block)
+      if @action
+        controller, action = @action.split('#')
+      elsif @name
+        controller = "none"
+        action = @name
+      else
+        controller = "none"
+        action = @route
+      end
+
+      params = request.env['rack.request.params']
+      user = request.env['etna.user']
+      user_hash = user ? hash_user_email(user.email) : 'unknown'
+      project_name = "unknown"
+
+      if params && (params.include?(:project_name) || params.include?('project_name'))
+        project_name = params[:project_name] || params['project_name']
+      end
+
+      begin
+        block.call({ controller: controller, action: action, user_hash: user_hash, project_name: project_name })
+      rescue => e
+        raise e unless Etna::Application.instance.environment == :production
+      end
+    end
+
+    def process_call(app, request)
       update_params(request)
 
       unless authorized?(request)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: etna
 version: !ruby/object:Gem::Version
-  version: 0.1.37
+  version: 0.1.38
 platform: ruby
 authors:
 - Saurabh Asthana
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-05-27 00:00:00.000000000 Z
+date: 2021-07-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: net-sftp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.0
 description: See summary
 email: Saurabh.Asthana@ucsf.edu
 executables:
@@ -144,6 +158,7 @@ files:
 - lib/etna/clients/metis/client.rb
 - lib/etna/clients/metis/models.rb
 - lib/etna/clients/metis/workflows.rb
+- lib/etna/clients/metis/workflows/ingest_metis_data_workflow.rb
 - lib/etna/clients/metis/workflows/metis_download_workflow.rb
 - lib/etna/clients/metis/workflows/metis_upload_workflow.rb
 - lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb