etna 0.1.44 → 0.1.45

data/lib/commands.rb

@@ -2,6 +2,7 @@ require 'date'
 require 'logger'
 require 'rollbar'
 require 'tempfile'
+require 'securerandom'
 require_relative 'helpers'
 require 'yaml'
 
@@ -88,6 +89,123 @@ class EtnaApp
     end
   end
 
+  class Metis
+    include Etna::CommandExecutor
+    string_flags << "--project-name"
+    string_flags << "--bucket-name"
+    string_flags << "--path"
+    ROLLING_COUNT = 14
+
+    class PutArchive < Etna::Command
+      include WithEtnaClients
+
+      def execute(file, project_name:, bucket_name:, path:)
+        @project_name = project_name
+        @bucket_name = bucket_name
+        basename = ::File.basename(path)
+        dir = ::File.dirname(path)
+
+        puts "Creating archive folder"
+        metis_client.create_folder(Etna::Clients::Metis::CreateFolderRequest.new(
+          project_name: project_name,
+          bucket_name: bucket_name,
+          folder_path: dir
+        ))
+
+        puts "Listing archive folder"
+        files = metis_client.list_folder(Etna::Clients::Metis::ListFolderRequest.new(
+          project_name: project_name,
+          bucket_name: bucket_name,
+          folder_path: dir
+        )).files.all
+
+        if files.select { |f| f.file_name == basename }.length > 0
+          timestr = DateTime.now.strftime("%Y%m%d_%H%M")
+          puts "Backing up #{dir}/#{basename} to #{dir}/#{basename}.#{timestr}"
+          metis_client.rename_file(Etna::Clients::Metis::RenameFileRequest.new(
+            project_name: project_name,
+            bucket_name: bucket_name,
+            file_path: ::File.join(dir, basename),
+            new_bucket_name: bucket_name,
+            new_file_path: ::File.join(dir, "#{basename}.#{timestr}"),
+          ))
+        end
+
+        create_upload_workflow.do_upload(
+          ::File.open(file, "r"),
+          path
+        ) do |progress|
+          case progress[0]
+          when :error
+            puts("Error while uploading: #{progress[1].to_s}")
+          else
+          end
+        end
+        puts "Completed upload"
+
+        backups = files.select { |f| f.file_name != basename }.sort_by(&:file_name).reverse
+        if backups.length > ROLLING_COUNT
+          backups.slice(ROLLING_COUNT..-1).reverse.each do |f|
+            puts "Removing rolling back up #{f.file_name}"
+            metis_client.delete_file(Etna::Clients::Metis::DeleteFileRequest.new(
+              project_name: project_name,
+              bucket_name: bucket_name,
+              file_path: ::File.join(dir, f.file_name)
+            ))
+          end
+        end
+      end
+
+      def create_upload_workflow
+        Etna::Clients::Metis::MetisUploadWorkflow.new(
+          metis_client: @metis_client,
+          metis_uid: SecureRandom.hex,
+          project_name: @project_name,
+          bucket_name: @bucket_name,
+          max_attempts: 3
+        )
+      end
+    end
+
+    class PullArchive < Etna::Command
+      include WithEtnaClients
+
+      def execute(project_name:, bucket_name:, path:)
+        @project_name = project_name
+        @bucket_name = bucket_name
+        basename = ::File.basename(path)
+        dir = ::File.dirname(path)
+
+        files = metis_client.list_folder(Etna::Clients::Metis::ListFolderRequest.new(
+          project_name: project_name,
+          bucket_name: bucket_name,
+          folder_path: dir
+        )).files.all
+
+        files.sort_by(&:file_name).reverse
+
+        archived = files.select { |f| f.file_name == basename }.first
+        if archived.nil?
+          archived = files.first
+        end
+
+        tmp = Tempfile.new('download', Dir.pwd)
+        begin
+          puts "Downloading to #{basename} from #{archived.file_name}"
+          metis_client.download_file(archived) do |chunk|
+            tmp.write(chunk)
+          end
+
+          # Atomic operation -- ensure we only place the file in position when it successfully loads
+          ::File.rename(tmp.path, ::File.join(::Dir.pwd, basename))
+        rescue
+          tmp.close!
+          raise
+        end
+      end
+    end
+  end
+
   class Administrate
     include Etna::CommandExecutor
 

data/lib/etna/application.rb

@@ -50,41 +50,8 @@ module Etna::Application
     Etna::Application.register(self)
   end
 
-  # a <- b
-  def deep_merge(a, b)
-    if a.is_a?(Hash)
-      if b.is_a?(Hash)
-        b.keys.each do |b_key|
-          a[b_key] = deep_merge(a[b_key], b[b_key])
-        end
-
-        return a
-      end
-    end
-
-    a.nil? ? b : a
-  end
-
   def configure(opts)
-    @config = opts
-
-    # Apply environmental variables of the form "ETNA__x__y__z_FILE"
-    # by reading the given file.
-    prefix = "ETNA__"
-    ENV.keys.select { |k| k.start_with?(prefix) && k.end_with?("_FILE") }.each do |key|
-      path = key.sub(/_FILE/, '').split("__", -1)
-      path.shift # drop the first, just app name
-
-      target = @config
-      while path.length > 1
-        n = path.shift
-        target = (target[n.downcase.to_sym] ||= {})
-      end
-
-      v = YAML.load(File.read(ENV[key]))
-      target[path.last.downcase.to_sym] =
-        deep_merge(target[path.last.downcase.to_sym], v)
-    end
+    @config = Etna::EnvironmentVariables.load_from_env('ETNA', root: opts) { |path, value| load_config_from_env_path(path, value) }
 
     if (rollbar_config = config(:rollbar)) && rollbar_config[:access_token]
       Rollbar.configure do |config|
@@ -93,6 +60,14 @@ module Etna::Application
     end
   end
 
+  def load_config_from_env_path(path, value)
+    return nil if path.empty?
+    return nil unless path.last.end_with?('_file')
+    path.last.sub!(/_file$/, '')
+
+    [path.map(&:to_sym), YAML.load(File.read(value))]
+  end
+
   def setup_yabeda
     application = self.id
     Yabeda.configure do
@@ -186,6 +161,10 @@ module Etna::Application
     (ENV["#{self.class.name.upcase}_ENV"] || :development).to_sym
   end
 
+  def test?
+    environment == "test"
+  end
+
   def id
     ENV["APP_NAME"] || self.class.name.snake_case.split(/::/).last
   end

data/lib/etna/auth.rb

@@ -18,12 +18,16 @@ module Etna
       if [ approve_noauth(request), approve_hmac(request), approve_user(request) ].all?{|approved| !approved}
         return fail_or_redirect(request)
       end
-
+  
       @app.call(request.env)
     end
 
     private
 
+    def janus
+      @janus ||= Etna::JanusUtils.new
+    end
+
     def application
       @application ||= Etna::Application.instance
     end
@@ -61,7 +65,7 @@ module Etna
         application.config(:auth_redirect).chomp('/') + '/login'
       )
       uri.query = URI.encode_www_form(refer: request.url)
-      return [ 302, { 'Location' => uri.to_s }, [] ]
+      Etna::Redirect(request).to(uri.to_s)
     end
 
     def approve_noauth(request)
@@ -77,52 +81,26 @@ module Etna
       return true if route && route.ignore_janus?
 
       # process task tokens
-      payload['task'] ? valid_task_token?(token) : true
-    end
-
-    def resource_projects(token)
-      return [] unless has_janus_config?
-
-      janus_client(token).get_projects.projects.select do |project|
-        project.resource
-      end
-    rescue
-      # If encounter any issue with Janus, we'll return no resource projects
-      []
-    end
-
-    def janus_client(token)
-      Etna::Clients::Janus.new(
-        token: token,
-        host: application.config(:janus)[:host]
-      )
+      payload[:task] ? janus.valid_task_token?(token) : true
     end
 
-    def valid_task_token?(token)
-      return false unless has_janus_config?
-
-      response = janus_client(token).validate_task_token
-
-      return false unless response.code == '200'
-
-      return true
+    def symbolize_payload_keys(payload)
+      payload.map{|k,v| [k.to_sym, v]}.to_h
     end
 
-    def has_janus_config?
-      application.config(:janus) && application.config(:janus)[:host]
+    def permissions(payload)
+      Etna::Permissions.from_encoded_permissions(payload[:perm])
     end
 
     def update_payload(payload, token, request)
       route = server.find_route(request)
 
-      payload = payload.map{|k,v| [k.to_sym, v]}.to_h
-
       return payload unless route
 
       begin      
-        permissions = Etna::Permissions.from_encoded_permissions(payload[:perm])
+        permissions = permissions(payload)
 
-        resource_projects(token).each do |resource_project|
+        janus.resource_projects(token).each do |resource_project|
           permissions.add_permission(
             Etna::Permission.new('v', resource_project.project_name)
           )
@@ -141,6 +119,8 @@ module Etna
       begin
         payload, header = application.sign.jwt_decode(token)
 
+        payload = symbolize_payload_keys(payload)
+
         return false unless janus_approved?(payload, token, request)
         return request.env['etna.user'] = Etna::User.new(
           update_payload(payload, token, request),

data/lib/etna/clients/janus/models.rb

@@ -115,6 +115,10 @@ module Etna
         def resource
           !!@raw[:resource]
         end
+
+        def requires_agreement
+          !!@raw[:requires_agreement]
+        end
       end
 
       class TokenResponse

data/lib/etna/clients/magma/workflows/create_project_workflow.rb

@@ -59,7 +59,7 @@ module Etna
         end
 
         def create_magma_project_record!
-          magma_client.update(Etna::Clients::Magma::UpdateRequest.new(
+          magma_client.update_json(Etna::Clients::Magma::UpdateRequest.new(
             project_name: project_name,
             revisions: {
                 'project' => { project_name => { name: project_name } },

data/lib/etna/controller.rb

@@ -1,4 +1,5 @@
 require 'erb'
+require 'net/smtp'
 
 module Etna
   class Controller
@@ -44,7 +45,7 @@ module Etna
     rescue Exception => e
       error = e
     ensure
-      log_request
+      log_request if !@request.env['etna.dont_log'] || error
       return handle_error(error) if error
     end
 
@@ -76,6 +77,24 @@ module Etna
       end
     end
 
+    def send_email(to_name, to_email, subject, content)
+      message = <<MESSAGE_END
+From: Data Library <noreply@janus>
+To: #{to_name} <#{to_email}>
+Subject: #{subject}
+
+#{content}
+MESSAGE_END
+
+      unless @server.send(:application).test?
+        Net::SMTP.start('smtp.ucsf.edu') do |smtp|
+          smtp.send_message message, 'noreply@janus', to_email
+        end
+      end
+    rescue => e
+      @logger.log_error(e)
+    end
+
     def require_params(*params)
       missing_params = params.reject{|p| @params.key?(p) }
       raise Etna::BadRequest, "Missing param #{missing_params.join(', ')}" unless missing_params.empty?

data/lib/etna/cwl.rb

@@ -559,6 +559,14 @@ module Etna
       def id
         @attributes['id']
       end
+
+      def type
+        @attributes['type']
+      end
+
+      def format
+        @attributes['format']
+      end
     end
 
     class WorkflowInputParameter < Cwl

data/lib/etna/environment_variables.rb

@@ -0,0 +1,50 @@
+module Etna
+  module EnvironmentVariables
+    # a <- b
+    def self.deep_merge(a, b)
+      if a.is_a?(Hash)
+        if b.is_a?(Hash)
+          b.keys.each do |b_key|
+            a[b_key] = deep_merge(a[b_key], b[b_key])
+          end
+
+          return a
+        end
+      end
+
+      a.nil? ? b : a
+    end
+
+    def self.load_from_env(prefix, root: {}, env: ENV, downcase: true, sep: '__', &path_to_value_mapper)
+      env.keys.each do |key|
+        next unless key.start_with?(prefix + sep)
+
+        path = key.split(sep, -1)
+        path.shift
+        if downcase
+          path.each(&:downcase!)
+        end
+
+        result = path_to_value_mapper.call(path, env[key])
+        next unless result
+
+        path, value = result
+
+        if path.empty?
+          root = EnvironmentVariables.deep_merge(root, value)
+          next
+        end
+
+        target = root
+        while path.length > 1
+          n = path.shift
+          target = (target[n] ||= {})
+        end
+
+        target[path.last] = EnvironmentVariables.deep_merge(target[path.last], value)
+      end
+
+      root
+    end
+  end
+end

data/lib/etna/injection.rb

@@ -0,0 +1,57 @@
+module Etna
+  class Injection
+    # Extend into class
+    module FromHash
+      def from_hash(hash, hash_has_string_keys, rest: nil, key_rest: nil)
+        ::Etna::Injection.inject_new(self, hash ,hash_has_string_keys, rest: rest, key_rest: key_rest) do |missing_p|
+          raise "required argument '#{missing_p}' of #{self.name} is missing!"
+        end
+      end
+    end
+
+    def self.inject_new(cls, hash, hash_has_string_keys, rest: nil, key_rest: nil, &missing_req_param_cb)
+      args, kwds = prep_args(
+        cls.instance_method(:initialize), hash, hash_has_string_keys,
+        rest: rest, key_rest: key_rest, &missing_req_param_cb
+      )
+
+      cls.new(*args, **kwds)
+    end
+
+    def self.prep_args(method, hash, hash_has_string_keys, rest: nil, key_rest: nil, &missing_req_param_cb)
+      new_k_params = {}
+      new_p_params = []
+
+      method.parameters.each do |type, p_key|
+        h_key = hash_has_string_keys ? p_key.to_s : p_key
+        if type == :rest && rest
+          new_p_params.append(*rest)
+        elsif type == :keyrest && key_rest
+          new_k_params.update(key_rest)
+        elsif type == :req || type == :opt
+          if hash.include?(h_key)
+            new_p_params << hash[h_key]
+          elsif type == :req
+            if block_given?
+              new_p_params << missing_req_param_cb.call(h_key)
+            else
+              new_p_params << nil
+            end
+          end
+        elsif type == :keyreq || type == :key
+          if hash.include?(h_key)
+            new_k_params[p_key] = hash[h_key]
+          elsif type == :keyreq
+            if block_given?
+              new_k_params[p_key] = missing_req_param_cb.call(h_key)
+            else
+              new_k_params[p_key] = nil
+            end
+          end
+        end
+      end
+
+      [new_p_params, new_k_params]
+    end
+  end
+end

data/lib/etna/janus_utils.rb

@@ -0,0 +1,55 @@
+# Utility class to work with Janus for authz and authn
+module Etna
+  class JanusUtils
+    def initialize
+    end
+
+    def projects(token)
+      return [] unless has_janus_config?
+
+      janus_client(token).get_projects.projects
+    rescue
+      # If encounter any issue with Janus, we'll return no projects
+      []
+    end
+
+    def resource_projects(token)
+      projects(token).select do |project|
+        !!project.resource && !project.requires_agreement
+      end
+    end
+
+    def community_projects(token)
+      projects(token).select do |project|
+        !!project.resource && !!project.requires_agreement
+      end
+    end
+
+    def janus_client(token)
+      Etna::Clients::Janus.new(
+        token: token,
+        host: application.config(:janus)[:host],
+      )
+    end
+
+    def valid_task_token?(token)
+      return false unless has_janus_config?
+
+      response = janus_client(token).validate_task_token
+
+      return false unless response.code == "200"
+
+      return true
+    end
+
+    private
+
+    def application
+      @application ||= Etna::Application.instance
+    end
+
+    def has_janus_config?
+      application.config(:janus) && application.config(:janus)[:host]
+    end
+  end
+end

data/lib/etna/permissions.rb

@@ -50,6 +50,10 @@ module Etna
       @permissions << permission
     end
 
+    def any?(&block)
+      @permissions.any?(&block)
+    end
+
     private
 
     def current_project_names
@@ -64,6 +68,7 @@ module Etna
       "A" => :admin,
       "E" => :editor,
       "V" => :viewer,
+      "G" => :guest
     }
 
     def initialize(role_key, project_name)

data/lib/etna/redirect.rb

@@ -0,0 +1,36 @@
+module Etna
+  def self.Redirect(req)
+    Redirect.new(req)
+  end
+
+  class Redirect
+    def initialize(request)
+      @request = request
+    end
+
+    def to(path, &block)
+      return Rack::Response.new(
+        [ { errors: [ 'Cannot redirect out of domain' ] }.to_json ], 422,
+        { 'Content-Type' => 'application/json' }
+      ).finish unless matches_domain?(path)
+
+      response = Rack::Response.new
+
+      response.redirect(path.gsub("http://", "https://"), 302)
+
+      yield response if block_given?
+
+      response.finish
+    end
+
+    private
+
+    def matches_domain?(path)
+      top_domain(@request.hostname) == top_domain(URI(path).host)
+    end
+
+    def top_domain(host_name)
+      host_name.split('.')[-2..-1].join('.')
+    end
+  end
+end

data/lib/etna/route.rb

@@ -15,6 +15,7 @@ module Etna
       @block = block
       @match_ext = options[:match_ext]
       @log_redact_keys = options[:log_redact_keys]
+      @dont_log = options[:dont_log]
     end
 
     def to_hash
@@ -41,7 +42,7 @@ module Etna
       if params
         PARAM_TYPES.reduce(route) do |path,pat|
           path.gsub(pat) do
-           URI.encode( params[$1.to_sym], UNSAFE)
+            params[$1.to_sym].split('/').map { |c| URI.encode_www_form_component(c) }.join('/')
           end
         end
       else
@@ -123,10 +124,19 @@ module Etna
       update_params(request)
 
       unless authorized?(request)
+        if cc_available?(request)
+          if request.content_type == 'application/json'
+            return [ 403, { 'Content-Type' => 'application/json' }, [ { error: 'You are forbidden from performing this action, but you can visit the project home page and request access.' }.to_json ] ]
+          else
+            return cc_redirect(request)
+          end
+        end
+
         return [ 403, { 'Content-Type' => 'application/json' }, [ { error: 'You are forbidden from performing this action.' }.to_json ] ]
       end
 
       request.env['etna.redact_keys'] = @log_redact_keys
+      request.env['etna.dont_log'] = @dont_log
 
       if @action
         controller, action = @action.split('#')
@@ -160,6 +170,23 @@ module Etna
 
     private
 
+    def janus
+      @janus ||= Etna::JanusUtils.new
+    end
+
+    # If the application asks for a code of conduct redirect
+    def cc_redirect(request, msg = 'You are unauthorized')
+      return [ 401, { 'Content-Type' => 'text/html' }, [msg] ] unless application.config(:auth_redirect)
+
+      params = request.env['rack.request.params']
+
+      uri = URI(
+        application.config(:auth_redirect).chomp('/') + "/#{params[:project_name]}/cc"
+      )
+      uri.query = URI.encode_www_form(refer: request.url)
+      Etna::Redirect(request).to(uri.to_s)
+    end
+
     def application
       @application ||= Etna::Application.instance
     end
@@ -189,6 +216,24 @@ module Etna
       end
     end
 
+    def cc_available?(request)
+      user = request.env['etna.user']
+
+      return false unless user
+
+      params = request.env['rack.request.params']
+
+      return false unless params[:project_name]
+
+      # Only check for a CC if the user does not currently have permissions
+      #   for the project
+      return false if user.permissions.keys.include?(params[:project_name])
+
+      !janus.community_projects(user.token).select do |project|
+        project.project_name == params[:project_name]
+      end.first.nil?
+    end
+
     def hmac_authorized?(request)
       # either there is no hmac requirement, or we have a valid hmac
       !@auth[:hmac] || request.env['etna.hmac']&.valid?
@@ -211,7 +256,7 @@ module Etna
         Hash[
           match.names.map(&:to_sym).zip(
             match.captures.map do |capture|
-              URI.decode(capture)
+              capture.split('/').map {|c| URI.decode_www_form_component(c) }.join('/')
             end
           )
         ]