etna 0.1.36 → 0.1.40

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a6d9abd6b64884e701b1055215018b85939ee0bf6cc19275eccae0ebffe92856
-  data.tar.gz: a382140c854224855b317923551d467baa3b29202ec6ebccc2fb8d7782115d06
+  metadata.gz: e83d3c30c23693011d59b103f8b5a9372950eba1b3123376a288826204d47c62
+  data.tar.gz: c3b50952e71d91f4361a87e872ba5532849644abe42fbc9e480549726a6bbf93
 SHA512:
-  metadata.gz: e150a0e7dac99e3c637151033d4105617910188f9c61c8e2812b2fa75c3d3967e95bb2fa43babb98802b14640658e02167c81b825fa0e1be41cef6340a74d4fa
-  data.tar.gz: 6f7365324ceed077161b4507397380837cc0d355f6cbd4dfff22b2abb527eaeafc2455e3cb3138d7b9a8085d3b4a2b3cf05b86417f2458a9bc80da859c77ed24
+  metadata.gz: c3c561cdd7db4631c267f2edba8421b94baba5de790ca1ff9edd06cd21d039d7224d868eaa43ba7090b4114b595830ac777c1c4688ec15c9320cde90530be4e5
+  data.tar.gz: 7148582a82f7e90357ed6658c7337bce7b34dd532b4a899ac27a2d74f40f34224cb6c225b2aa4db4164c467aa8a5c189d7acf28a1c462097e619e9875fbdc4d1

data/lib/etna.rb

@@ -21,6 +21,7 @@ require_relative './etna/environment_scoped'
 require_relative './etna/filesystem'
 require_relative './etna/formatting'
 require_relative './etna/cwl'
+require_relative './etna/metrics'
 
 class EtnaApp
   include Etna::Application

data/lib/etna/application.rb

@@ -8,6 +8,7 @@ require_relative './command'
 require_relative './generate_autocompletion_script'
 require 'singleton'
 require 'rollbar'
+require 'fileutils'
 
 module Etna::Application
   def self.included(other)
@@ -59,6 +60,72 @@ module Etna::Application
     end
   end
 
+  # This will cause metrics to persist to a file.
+  # NOTE -- /tmp/metrics.bin should be a persistent mount when using this.
+  # You will still need to export metrics in the text format for the node_exporter on the host machine to
+  # export them to prometheus.  Ensure that the /tmp/metrics.bin is on a named volume or a bind mount, either is fine.
+  def enable_job_metrics!
+    require 'prometheus'
+    Prometheus::Client.config.data_store = Prometheus::Client::DataStores::DirectFileStore.new({
+      dir: "/tmp/metrics.bin"
+    })
+  end
+
+  def setup_yabeda
+    application = self.id
+    Yabeda.configure do
+      default_tag :application, application
+
+      group :etna do
+        histogram :response_time do
+          comment "Time spent by a controller returning a response"
+          unit :seconds
+          tags [:controller, :action, :user_hash, :project_name]
+          buckets [0.01, 0.1, 0.3, 0.5, 1, 5]
+        end
+
+        counter :visits do
+          comment "Counts visits to the controller"
+          tags [:controller, :action, :user_hash, :project_name]
+        end
+
+        counter :rollbar_errors do
+          comment "Counts errors detected by and sent to rollbar"
+        end
+
+        gauge :last_command_completion do
+          comment "Unix time of last time command was completed"
+          tags [:command, :status, :application]
+        end
+
+        histogram :command_runtime do
+          comment "Time spent processing a given command"
+          tags [:command, :status, :application]
+          unit :seconds
+          buckets [0.1, 1, 5, 60, 300, 1500]
+        end
+      end
+    end
+
+    Yabeda.configure!
+  end
+
+  # Writes all metrics currently gathered to a text format prometheus file.  If /tmp/metrics.prom is bind mounted
+  # to the host directed bound to the node_exporter's file exporter directory, these will be exported to
+  # prometheus.  Combine this enable_job_metrics! for maximum effect.
+  def write_job_metrics(name)
+    node_metrics_dir = "/tmp/metrics.prom"
+    ::FileUtils.mkdir_p(node_metrics_dir)
+
+    tmp_file = ::File.join(node_metrics_dir, "#{name}.prom.$$")
+    ::File.open(tmp_file, "w") do |f|
+      f.write(Prometheus::Client::Formats::Text.marshal(Prometheus::Client.registry))
+    end
+
+    require 'fileutils'
+    ::FileUtils.mv(tmp_file, ::File.join(node_metrics_dir, "#{name}.prom"))
+  end
+
   def setup_logger
     @logger = Etna::Logger.new(
       # The name of the log_file, required.
@@ -108,17 +175,32 @@ module Etna::Application
   end
 
   def run_command(config, *args, &block)
+    application = self.id
+    status = 'success'
+    start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
     cmd, cmd_args, cmd_kwds = find_command(*args)
-    cmd.setup(config)
 
-    if block_given?
-      return unless yield [cmd, cmd_args]
-    end
+    begin
+      cmd.setup(config)
+      if block_given?
+        return unless yield [cmd, cmd_args]
+      end
 
-    cmd.execute(*cmd.fill_in_missing_params(cmd_args), **cmd_kwds)
-  rescue => e
-    Rollbar.error(e)
-    raise
+      cmd.execute(*cmd.fill_in_missing_params(cmd_args), **cmd_kwds)
+    rescue => e
+      Rollbar.error(e)
+      status = 'failed'
+      raise
+    ensure
+      if defined?(Yabeda) && Yabeda.configured?
+        tags = { command: cmd.class.name, status: status, application: application }
+        dur = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
+
+        Yabeda.etna.last_command_completion.set(tags, Time.now.to_i)
+        Yabeda.etna.command_runtime.measure(tags, dur)
+        write_job_metrics("run_command")
+      end
+    end
   end
 end
 

data/lib/etna/clients/metis/client.rb

@@ -210,6 +210,52 @@ module Etna
         }
       end
 
+      def resolve_conflicts_and_verify_rename(project_name:, source_bucket:, dest_bucket:, folder:, file:)
+        parent_folder = ::File.dirname(file.file_path)
+
+        should_rename_original = true
+
+        # If the destination folder already exists, check to see if
+        #   the file also exists, otherwise we risk a
+        #   rename conflict.
+        create_folder_request = CreateFolderRequest.new(
+          project_name: project_name,
+          bucket_name: dest_bucket,
+          folder_path: parent_folder
+        )
+
+        if folder_exists?(create_folder_request)
+          # If file exists in destination, delete the older file.
+          list_dest_folder_request = Etna::Clients::Metis::ListFolderRequest.new(
+            bucket_name: dest_bucket,
+            project_name: project_name,
+            folder_path: parent_folder
+          )
+
+          dest_file = list_folder(list_dest_folder_request).files.all.find { |f| f.file_name == file.file_name }
+
+          if (dest_file && file.updated_at <= dest_file.updated_at)
+            # Delete source file if it's out of date
+            delete_file(Etna::Clients::Metis::DeleteFileRequest.new(
+              bucket_name: source_bucket,
+              project_name: project_name,
+              file_path: file.file_path,
+            ))
+
+            should_rename_original = false
+          elsif (dest_file && file.updated_at > dest_file.updated_at)
+            # Delete dest file if it's out of date
+            delete_file(Etna::Clients::Metis::DeleteFileRequest.new(
+              bucket_name: dest_bucket,
+              project_name: project_name,
+              file_path: dest_file.file_path,
+            ))
+          end
+        end
+
+        should_rename_original
+      end
+
       def recursively_rename_folder(project_name:, source_bucket:, dest_bucket:, folder:)
         folder_contents = list_folder(
           Etna::Clients::Metis::ListFolderRequest.new(
@@ -228,6 +274,13 @@ module Etna
         end
 
         folder_contents.files.all.each do |file|
+          should_rename = resolve_conflicts_and_verify_rename(
+            project_name: project_name,
+            source_bucket: source_bucket,
+            dest_bucket: dest_bucket,
+            folder: folder,
+            file: file)
+
           rename_file(Etna::Clients::Metis::RenameFileRequest.new(
             bucket_name: source_bucket,
             project_name: project_name,
@@ -235,7 +288,7 @@ module Etna
             new_bucket_name: dest_bucket,
             new_file_path: file.file_path,
             create_parent: true)
-          )
+          ) if should_rename
         end
 
         # Now delete the source folder

data/lib/etna/clients/metis/workflows.rb

@@ -1,3 +1,4 @@
-require_relative './workflows/metis_download_workflow'
-require_relative './workflows/metis_upload_workflow'
-require_relative './workflows/sync_metis_data_workflow'
+require_relative "./workflows/metis_download_workflow"
+require_relative "./workflows/metis_upload_workflow"
+require_relative "./workflows/sync_metis_data_workflow"
+require_relative "./workflows/ingest_metis_data_workflow"

data/lib/etna/clients/metis/workflows/ingest_metis_data_workflow.rb

@@ -0,0 +1,31 @@
+require "ostruct"
+require "fileutils"
+require "tempfile"
+
+module Etna
+  module Clients
+    class Metis
+      class IngestMetisDataWorkflow < Struct.new(:metis_filesystem, :ingest_filesystem, :logger, keyword_init: true)
+        # Since we are doing manual triage of files,
+        #   do not automatically copy directory trees.
+        # srcs must be a list of full paths to files.
+        def copy_files(srcs)
+          srcs.each do |src|
+            next unless ingest_filesystem.exist?(src)
+
+            logger&.info("Copying file #{src} (#{Etna::Formatting.as_size(ingest_filesystem.stat(src).size)})")
+
+            # For ingestion triage, just copy over the exact path + filename.
+            copy_file(dest: src, src: src)
+          end
+        end
+
+        def copy_file(dest:, src:)
+          ingest_filesystem.with_readable(src, "r") do |file|
+            metis_filesystem.do_streaming_upload(file, dest, file.size)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/etna/clients/metis/workflows/metis_upload_workflow.rb

@@ -30,7 +30,7 @@ module Etna
               project_name: project_name,
               bucket_name: bucket_name,
               folder_path: dir,
-          ))
+          )) unless dir == "."
 
           authorize_response = metis_client.authorize_upload(AuthorizeUploadRequest.new(
               project_name: project_name,

data/lib/etna/controller.rb

@@ -19,25 +19,30 @@ module Etna
       @logger.warn(request_msg(line))
     end
 
+    def handle_error(e)
+      case e
+      when Etna::Error
+        Rollbar.error(e)
+        @logger.error(request_msg("Exiting with #{e.status}, #{e.message}"))
+        return failure(e.status, error: e.message)
+      else
+        Rollbar.error(e)
+        @logger.error(request_msg('Caught unspecified error'))
+        @logger.error(request_msg(e.message))
+        e.backtrace.each do |trace|
+          @logger.error(request_msg(trace))
+        end
+        return failure(500, error: 'Server error.')
+      end
+    end
+
     def response(&block)
       return instance_eval(&block) if block_given?
-
       return send(@action) if @action
 
-
       [501, {}, ['This controller is not implemented.']]
-    rescue Etna::Error => e
-      Rollbar.error(e)
-      @logger.error(request_msg("Exiting with #{e.status}, #{e.message}"))
-      return failure(e.status, error: e.message)
     rescue Exception => e
-      Rollbar.error(e)
-      @logger.error(request_msg('Caught unspecified error'))
-      @logger.error(request_msg(e.message))
-      e.backtrace.each do |trace|
-        @logger.error(request_msg(trace))
-      end
-      return failure(500, error: 'Server error.')
+      handle_error(e)
     end
 
     def require_params(*params)

data/lib/etna/directed_graph.rb

@@ -52,7 +52,12 @@ class DirectedGraph
             result[grandparent] << child_node if result.include?(grandparent)
           end
 
-          result[child_node] = []
+          # Depending on the graph shape, diamonds could lead to
+          #   resetting of previously calculated dependencies.
+          # Here we avoid resetting existing entries in `result`
+          #   and instead concatenate them if they already exist.
+          result[child_node] = [] unless result.include?(child_node)
+          result[n].concat(result[child_node]) if result.include?(child_node) && result.include?(n)
         end
       end
 

data/lib/etna/filesystem.rb

@@ -3,6 +3,8 @@ require 'fileutils'
 require 'open3'
 require 'securerandom'
 require 'concurrent-ruby'
+require 'net/sftp'
+require 'net/ssh'
 
 module Etna
   # A class that encapsulates opening / reading file system entries that abstracts normal file access in order
@@ -50,6 +52,11 @@ module Etna
       ::FileUtils.mv(src, dest)
     end
 
+    def stat(src)
+      raise "stat not supported by #{self.class.name}" unless self.class == Filesystem
+      ::File.stat(src)
+    end
+
     class EmptyIO < StringIO
       def write(*args)
         # Do nothing -- always leave empty
@@ -369,7 +376,59 @@ module Etna
       end
     end
 
+    class SftpFilesystem < Filesystem
+      def initialize(host:, username:, password: nil, port: 22, **args)
+        @username = username
+        @password = password
+        @host = host
+        @port = port
+      end
+
+      def ssh
+        @ssh ||= Net::SSH.start(@host, @username, password: @password)
+      end
+
+      def sftp
+        @sftp ||= begin
+          conn = Net::SFTP::Session.new(ssh)
+          conn.loop { conn.opening? }
+
+          conn
+        end
+      end
+
+      def with_readable(src, opts = 'r', &block)
+        sftp.file.open(src, opts, &block)
+      end
+
+      def ls(dir)
+        sftp.dir.entries(dir)
+      end
+
+      def exist?(src)
+        begin
+          sftp.file.open(src)
+        rescue Net::SFTP::StatusException
+          return false
+        end
+        return true
+      end
+
+      def stat(src)
+        sftp.file.open(src).stat
+      end
+    end
+
     class Mock < Filesystem
+      class MockStat
+        def initialize
+        end
+
+        def size
+          0
+        end
+      end
+
       def initialize(&new_io)
         @files = {}
         @dirs = {}
@@ -438,6 +497,10 @@ module Etna
       def exist?(src)
         @files.include?(src) || @dirs.include?(src)
       end
+
+      def stat(src)
+        @files[src].respond_to?(:stat) ? @files[src].stat : MockStat.new
+      end
     end
   end
 end

data/lib/etna/logger.rb

@@ -40,6 +40,10 @@ module Etna
         error(trace)
       end
 
+      if defined? Yabeda
+        Yabeda.etna.rollbar_errors.increment({}, 1) rescue nil
+      end
+
       Rollbar.error(e)
     end
 

data/lib/etna/metrics.rb

@@ -0,0 +1,26 @@
+
+module Etna
+  class MetricsExporter
+    def initialize(app, path: '/metrics')
+      @app = app
+      @path = path
+    end
+
+    def exporter
+      @exporter ||= begin
+        exporter = Yabeda::Prometheus::Exporter.new(@app, path: @path)
+        Rack::Auth::Basic.new(exporter) do |user, pw|
+          user == 'prometheus' && pw == ENV['METRICS_PW']
+        end
+      end
+    end
+
+    def call(env)
+      if env['PATH_INFO'] == @path
+        exporter.call(env)
+      else
+        @app.call(env)
+      end
+    end
+  end
+end

data/lib/etna/route.rb

@@ -1,3 +1,6 @@
+require 'digest'
+require 'date'
+
 module Etna
   class Route
     attr_reader :name
@@ -58,6 +61,63 @@ module Etna
     end
 
     def call(app, request)
+      start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+      try_yabeda(request)  do |tags|
+        Yabeda.etna.visits.increment(tags)
+      end
+
+      begin
+        process_call(app, request)
+      ensure
+        try_yabeda(request) do |tags|
+          dur = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
+          Yabeda.etna.response_time.measure(tags, dur)
+        end
+      end
+    end
+
+    def hash_user_email(email)
+      secret = Etna::Application.instance.config(:user_hash_secret) || 'notsosecret'
+      digest = email + secret + Date.today.to_s
+
+      if @name
+        digest += @name.to_s
+      else
+        digest += @route.to_s
+      end
+
+      Digest::MD5.hexdigest(digest)
+    end
+
+    def try_yabeda(request, &block)
+      if @action
+        controller, action = @action.split('#')
+      elsif @name
+        controller = "none"
+        action = @name
+      else
+        controller = "none"
+        action = @route
+      end
+
+      params = request.env['rack.request.params']
+      user = request.env['etna.user']
+      user_hash = user ? hash_user_email(user.email) : 'unknown'
+      project_name = "unknown"
+
+      if params && (params.include?(:project_name) || params.include?('project_name'))
+        project_name = params[:project_name] || params['project_name']
+      end
+
+      begin
+        block.call({ controller: controller, action: action, user_hash: user_hash, project_name: project_name })
+      rescue => e
+        raise e unless Etna::Application.instance.environment == :production
+      end
+    end
+
+    def process_call(app, request)
       update_params(request)
 
       unless authorized?(request)
@@ -148,7 +208,10 @@ module Etna
       params = request.env['rack.request.params']
 
       @auth[:user].all? do |constraint, param_name|
-        user.respond_to?(constraint) && user.send(constraint, params[param_name])
+        user.respond_to?(constraint) && (
+          param_name.is_a?(Symbol) ?
+            user.send(constraint, params[param_name]) :
+            user.send(constraint, param_name))
       end
     end
 

data/lib/etna/server.rb

@@ -74,6 +74,11 @@ module Etna
     def initialize
       # Setup logging.
       application.setup_logger
+
+      # This needs to be required before yabeda invocation, but cannot belong at the top of any module since clients
+      # do not install yabeda.
+      require 'yabeda'
+      application.setup_yabeda
     end
 
     private

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: etna
 version: !ruby/object:Gem::Version
-  version: 0.1.36
+  version: 0.1.40
 platform: ruby
 authors:
 - Saurabh Asthana
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-05-10 00:00:00.000000000 Z
+date: 2021-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: net-sftp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.0
 description: See summary
 email: Saurabh.Asthana@ucsf.edu
 executables:
@@ -144,6 +158,7 @@ files:
 - lib/etna/clients/metis/client.rb
 - lib/etna/clients/metis/models.rb
 - lib/etna/clients/metis/workflows.rb
+- lib/etna/clients/metis/workflows/ingest_metis_data_workflow.rb
 - lib/etna/clients/metis/workflows/metis_download_workflow.rb
 - lib/etna/clients/metis/workflows/metis_upload_workflow.rb
 - lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb
@@ -168,6 +183,7 @@ files:
 - lib/etna/hmac.rb
 - lib/etna/json_serializable_struct.rb
 - lib/etna/logger.rb
+- lib/etna/metrics.rb
 - lib/etna/multipart_serializable_nested_hash.rb
 - lib/etna/parse_body.rb
 - lib/etna/route.rb