etna 0.1.42 → 0.1.45

data/lib/etna/clients/metis/workflows/walk_metis_workflow.rb

@@ -0,0 +1,31 @@
+module Etna
+  module Clients
+    class Metis
+      class WalkMetisWorkflow < Struct.new(:metis_client, :project_name,
+          :bucket_name, :logger, :root_dir, keyword_init: true)
+        def each(&block)
+          q = [self.root_dir]
+
+          while (n = q.pop)
+            req = Etna::Clients::Metis::ListFolderRequest.new(
+              project_name: project_name,
+              bucket_name: bucket_name,
+              folder_path: n
+            )
+            next unless metis_client.folder_exists?(req)
+            resp = metis_client.list_folder(req)
+
+            resp.files.all.sort_by { |f| f.file_path[self.root_dir.length..-1] }.each do |file|
+              yield [file, file.file_path[self.root_dir.length..-1]]
+            end
+
+            resp.folders.all.sort_by { |f| f.folder_path[self.root_dir.length..-1] }.each do |f|
+              yield [f, f.folder_path[self.root_dir.length..-1]]
+              q << f.folder_path
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/etna/clients/metis/workflows.rb

@@ -2,3 +2,5 @@ require_relative "./workflows/metis_download_workflow"
 require_relative "./workflows/metis_upload_workflow"
 require_relative "./workflows/sync_metis_data_workflow"
 require_relative "./workflows/ingest_metis_data_workflow"
+require_relative "./workflows/walk_metis_workflow"
+require_relative "./workflows/walk_metis_diff_workflow"

data/lib/etna/controller.rb

@@ -1,4 +1,5 @@
 require 'erb'
+require 'net/smtp'
 
 module Etna
   class Controller
@@ -44,10 +45,56 @@ module Etna
     rescue Exception => e
       error = e
     ensure
-      log_request
+      log_request if !@request.env['etna.dont_log'] || error
       return handle_error(error) if error
     end
 
+    def try_stream(content_type, &block)
+      if @request.env['rack.hijack?']
+        @request.env['rack.hijack'].call
+        stream = @request.env['rack.hijack_io']
+
+        headers = [
+          "HTTP/1.1 200 OK",
+          "Content-Type: #{content_type}"
+        ]
+        stream.write(headers.map { |header| header + "\r\n" }.join)
+        stream.write("\r\n")
+        stream.flush
+
+        Thread.new do
+          block.call(stream)
+        ensure
+          stream.close
+        end
+
+        # IO is now streaming and will be processed by above thread.
+        @response.close
+      else
+        @response['Content-Type'] = content_type
+        block.call(@response)
+        @response.finish
+      end
+    end
+
+    def send_email(to_name, to_email, subject, content)
+      message = <<MESSAGE_END
+From: Data Library <noreply@janus>
+To: #{to_name} <#{to_email}>
+Subject: #{subject}
+
+#{content}
+MESSAGE_END
+
+      unless @server.send(:application).test?
+        Net::SMTP.start('smtp.ucsf.edu') do |smtp|
+          smtp.send_message message, 'noreply@janus', to_email
+        end
+      end
+    rescue => e
+      @logger.log_error(e)
+    end
+
     def require_params(*params)
       missing_params = params.reject{|p| @params.key?(p) }
       raise Etna::BadRequest, "Missing param #{missing_params.join(', ')}" unless missing_params.empty?

data/lib/etna/cwl.rb

@@ -559,6 +559,14 @@ module Etna
       def id
         @attributes['id']
       end
+
+      def type
+        @attributes['type']
+      end
+
+      def format
+        @attributes['format']
+      end
     end
 
     class WorkflowInputParameter < Cwl

data/lib/etna/environment_variables.rb

@@ -0,0 +1,50 @@
+module Etna
+  module EnvironmentVariables
+    # a <- b
+    def self.deep_merge(a, b)
+      if a.is_a?(Hash)
+        if b.is_a?(Hash)
+          b.keys.each do |b_key|
+            a[b_key] = deep_merge(a[b_key], b[b_key])
+          end
+
+          return a
+        end
+      end
+
+      a.nil? ? b : a
+    end
+
+    def self.load_from_env(prefix, root: {}, env: ENV, downcase: true, sep: '__', &path_to_value_mapper)
+      env.keys.each do |key|
+        next unless key.start_with?(prefix + sep)
+
+        path = key.split(sep, -1)
+        path.shift
+        if downcase
+          path.each(&:downcase!)
+        end
+
+        result = path_to_value_mapper.call(path, env[key])
+        next unless result
+
+        path, value = result
+
+        if path.empty?
+          root = EnvironmentVariables.deep_merge(root, value)
+          next
+        end
+
+        target = root
+        while path.length > 1
+          n = path.shift
+          target = (target[n] ||= {})
+        end
+
+        target[path.last] = EnvironmentVariables.deep_merge(target[path.last], value)
+      end
+
+      root
+    end
+  end
+end

data/lib/etna/injection.rb

@@ -0,0 +1,57 @@
+module Etna
+  class Injection
+    # Extend into class
+    module FromHash
+      def from_hash(hash, hash_has_string_keys, rest: nil, key_rest: nil)
+        ::Etna::Injection.inject_new(self, hash ,hash_has_string_keys, rest: rest, key_rest: key_rest) do |missing_p|
+          raise "required argument '#{missing_p}' of #{self.name} is missing!"
+        end
+      end
+    end
+
+    def self.inject_new(cls, hash, hash_has_string_keys, rest: nil, key_rest: nil, &missing_req_param_cb)
+      args, kwds = prep_args(
+        cls.instance_method(:initialize), hash, hash_has_string_keys,
+        rest: rest, key_rest: key_rest, &missing_req_param_cb
+      )
+
+      cls.new(*args, **kwds)
+    end
+
+    def self.prep_args(method, hash, hash_has_string_keys, rest: nil, key_rest: nil, &missing_req_param_cb)
+      new_k_params = {}
+      new_p_params = []
+
+      method.parameters.each do |type, p_key|
+        h_key = hash_has_string_keys ? p_key.to_s : p_key
+        if type == :rest && rest
+          new_p_params.append(*rest)
+        elsif type == :keyrest && key_rest
+          new_k_params.update(key_rest)
+        elsif type == :req || type == :opt
+          if hash.include?(h_key)
+            new_p_params << hash[h_key]
+          elsif type == :req
+            if block_given?
+              new_p_params << missing_req_param_cb.call(h_key)
+            else
+              new_p_params << nil
+            end
+          end
+        elsif type == :keyreq || type == :key
+          if hash.include?(h_key)
+            new_k_params[p_key] = hash[h_key]
+          elsif type == :keyreq
+            if block_given?
+              new_k_params[p_key] = missing_req_param_cb.call(h_key)
+            else
+              new_k_params[p_key] = nil
+            end
+          end
+        end
+      end
+
+      [new_p_params, new_k_params]
+    end
+  end
+end

data/lib/etna/janus_utils.rb

@@ -0,0 +1,55 @@
+# Utility class to work with Janus for authz and authn
+module Etna
+  class JanusUtils
+    def initialize
+    end
+
+    def projects(token)
+      return [] unless has_janus_config?
+
+      janus_client(token).get_projects.projects
+    rescue
+      # If encounter any issue with Janus, we'll return no projects
+      []
+    end
+
+    def resource_projects(token)
+      projects(token).select do |project|
+        !!project.resource && !project.requires_agreement
+      end
+    end
+
+    def community_projects(token)
+      projects(token).select do |project|
+        !!project.resource && !!project.requires_agreement
+      end
+    end
+
+    def janus_client(token)
+      Etna::Clients::Janus.new(
+        token: token,
+        host: application.config(:janus)[:host],
+      )
+    end
+
+    def valid_task_token?(token)
+      return false unless has_janus_config?
+
+      response = janus_client(token).validate_task_token
+
+      return false unless response.code == "200"
+
+      return true
+    end
+
+    private
+
+    def application
+      @application ||= Etna::Application.instance
+    end
+
+    def has_janus_config?
+      application.config(:janus) && application.config(:janus)[:host]
+    end
+  end
+end

data/lib/etna/permissions.rb

@@ -0,0 +1,104 @@
+module Etna
+  class Permissions
+    def initialize(permissions)
+      @permissions = permissions
+    end
+
+    def self.from_encoded_permissions(encoded_permissions)
+      perms = encoded_permissions.split(/\;/).map do |roles|
+        role, projects = roles.split(/:/)
+
+        projects.split(/\,/).reduce([]) do |perms, project_name|
+          perms << Etna::Permission.new(role, project_name)
+        end
+      end.flatten
+
+      Etna::Permissions.new(perms)
+    end
+
+    def self.from_hash(permissions_hash)
+      perms = permissions_hash.map do |project_name, role_hash|
+        Etna::Permission.new(
+          Etna::Role.new(role_hash[:role], role_hash[:restricted]).key,
+          project_name
+        )
+      end
+
+      Etna::Permissions.new(perms)
+    end
+
+    def to_string
+      @permissions_string ||= @permissions.group_by(&:role_key)
+        .sort_by(&:first)
+        .map do |role_key, permissions|
+        [
+          role_key,
+          permissions.map(&:project_name).sort.join(","),
+        ].join(":")
+      end.join(";")
+    end
+
+    def to_hash
+      @permissions_hash ||= @permissions.map do |permission|
+        [permission.project_name, permission.to_hash]
+      end.to_h
+    end
+
+    def add_permission(permission)
+      return if current_project_names.include?(permission.project_name)
+
+      @permissions << permission
+    end
+
+    def any?(&block)
+      @permissions.any?(&block)
+    end
+
+    private
+
+    def current_project_names
+      @permissions.map(&:project_name)
+    end
+  end
+
+  class Permission
+    attr_reader :role, :project_name, :role_key
+
+    ROLE_NAMES = {
+      "A" => :admin,
+      "E" => :editor,
+      "V" => :viewer,
+      "G" => :guest
+    }
+
+    def initialize(role_key, project_name)
+      @role_key = role_key
+      @role = Etna::Role.new(ROLE_NAMES[role_key.upcase], role_key == role_key.upcase)
+      @project_name = project_name
+    end
+
+    def to_hash
+      role.to_hash
+    end
+  end
+
+  class Role
+    attr_reader :role, :restricted
+    def initialize(role, restricted)
+      @role = role
+      @restricted = restricted
+    end
+
+    def key
+      role_key = role.to_s[0]
+      restricted ? role_key.upcase : role_key
+    end
+
+    def to_hash
+      {
+        role: role,
+        restricted: restricted,
+      }
+    end
+  end
+end

data/lib/etna/redirect.rb

@@ -0,0 +1,36 @@
+module Etna
+  def self.Redirect(req)
+    Redirect.new(req)
+  end
+
+  class Redirect
+    def initialize(request)
+      @request = request
+    end
+
+    def to(path, &block)
+      return Rack::Response.new(
+        [ { errors: [ 'Cannot redirect out of domain' ] }.to_json ], 422,
+        { 'Content-Type' => 'application/json' }
+      ).finish unless matches_domain?(path)
+
+      response = Rack::Response.new
+
+      response.redirect(path.gsub("http://", "https://"), 302)
+
+      yield response if block_given?
+
+      response.finish
+    end
+
+    private
+
+    def matches_domain?(path)
+      top_domain(@request.hostname) == top_domain(URI(path).host)
+    end
+
+    def top_domain(host_name)
+      host_name.split('.')[-2..-1].join('.')
+    end
+  end
+end

data/lib/etna/route.rb

@@ -15,6 +15,7 @@ module Etna
       @block = block
       @match_ext = options[:match_ext]
       @log_redact_keys = options[:log_redact_keys]
+      @dont_log = options[:dont_log]
     end
 
     def to_hash
@@ -41,7 +42,7 @@ module Etna
       if params
         PARAM_TYPES.reduce(route) do |path,pat|
           path.gsub(pat) do
-           URI.encode( params[$1.to_sym], UNSAFE)
+            params[$1.to_sym].split('/').map { |c| URI.encode_www_form_component(c) }.join('/')
           end
         end
       else
@@ -123,10 +124,19 @@ module Etna
       update_params(request)
 
       unless authorized?(request)
+        if cc_available?(request)
+          if request.content_type == 'application/json'
+            return [ 403, { 'Content-Type' => 'application/json' }, [ { error: 'You are forbidden from performing this action, but you can visit the project home page and request access.' }.to_json ] ]
+          else
+            return cc_redirect(request)
+          end
+        end
+
         return [ 403, { 'Content-Type' => 'application/json' }, [ { error: 'You are forbidden from performing this action.' }.to_json ] ]
       end
 
       request.env['etna.redact_keys'] = @log_redact_keys
+      request.env['etna.dont_log'] = @dont_log
 
       if @action
         controller, action = @action.split('#')
@@ -154,8 +164,29 @@ module Etna
       @auth && @auth[:ignore_janus]
     end
 
+    def has_user_constraint?(constraint)
+      @auth && @auth[:user] && @auth[:user][constraint]
+    end
+
     private
 
+    def janus
+      @janus ||= Etna::JanusUtils.new
+    end
+
+    # If the application asks for a code of conduct redirect
+    def cc_redirect(request, msg = 'You are unauthorized')
+      return [ 401, { 'Content-Type' => 'text/html' }, [msg] ] unless application.config(:auth_redirect)
+
+      params = request.env['rack.request.params']
+
+      uri = URI(
+        application.config(:auth_redirect).chomp('/') + "/#{params[:project_name]}/cc"
+      )
+      uri.query = URI.encode_www_form(refer: request.url)
+      Etna::Redirect(request).to(uri.to_s)
+    end
+
     def application
       @application ||= Etna::Application.instance
     end
@@ -185,6 +216,24 @@ module Etna
       end
     end
 
+    def cc_available?(request)
+      user = request.env['etna.user']
+
+      return false unless user
+
+      params = request.env['rack.request.params']
+
+      return false unless params[:project_name]
+
+      # Only check for a CC if the user does not currently have permissions
+      #   for the project
+      return false if user.permissions.keys.include?(params[:project_name])
+
+      !janus.community_projects(user.token).select do |project|
+        project.project_name == params[:project_name]
+      end.first.nil?
+    end
+
     def hmac_authorized?(request)
       # either there is no hmac requirement, or we have a valid hmac
       !@auth[:hmac] || request.env['etna.hmac']&.valid?
@@ -207,7 +256,7 @@ module Etna
         Hash[
           match.names.map(&:to_sym).zip(
             match.captures.map do |capture|
-              URI.decode(capture)
+              capture.split('/').map {|c| URI.decode_www_form_component(c) }.join('/')
             end
           )
         ]

data/lib/etna/spec/auth.rb

@@ -18,11 +18,22 @@ module Etna::Spec
       },
       non_user: {
         email: 'nessus@centaurs.org', name: 'Nessus', perm: ''
+      },
+      guest: {
+        email: 'sinon@troy.org', name: 'Sinon', perm: 'g:labors'
       }
     }
 
     def auth_header(user_type)
       header(*Etna::TestAuth.token_header(AUTH_USERS[user_type]))
     end
+
+    def below_admin_roles
+      [:editor, :viewer, :guest]
+    end
+    
+    def below_editor_roles
+      [:viewer, :guest]
+    end
   end
 end

data/lib/etna/spec/vcr.rb

@@ -1,9 +1,27 @@
+require 'cgi'
 require 'webmock/rspec'
 require 'vcr'
 require 'openssl'
 require 'digest/sha2'
 require 'base64'
 
+def clean_query(json_or_string)
+  if json_or_string.is_a?(Hash) && json_or_string.include?('upload_path')
+    json_or_string['upload_path'] = clean_query(json_or_string['upload_path'])
+    json_or_string
+  elsif json_or_string.is_a?(String)
+    uri = URI(json_or_string)
+
+    if uri.query&.include?('X-Etna-Signature')
+      uri.query = 'etna-signature'
+    end
+
+    uri.to_s
+  else
+    json_or_string
+  end
+end
+
 def setup_base_vcr(spec_helper_dir, server: nil, application: nil)
   VCR.configure do |c|
     c.hook_into :webmock
@@ -30,6 +48,10 @@ def setup_base_vcr(spec_helper_dir, server: nil, application: nil)
       end
     end
 
+    c.register_request_matcher :try_uri do |request_1, request_2|
+      clean_query(request_1.uri) == clean_query(request_2.uri)
+    end
+
     c.register_request_matcher :try_body do |request_1, request_2|
       if request_1.headers['Content-Type'].first =~ /application\/json/
         if request_2.headers['Content-Type'].first =~ /application\/json/
@@ -40,7 +62,7 @@ def setup_base_vcr(spec_helper_dir, server: nil, application: nil)
             JSON.parse(request_2.body) rescue 'not-json'
           end
 
-          request_1_json == request_2_json
+          clean_query(request_1_json) == clean_query(request_2_json)
         else
           false
         end
@@ -49,7 +71,9 @@ def setup_base_vcr(spec_helper_dir, server: nil, application: nil)
       end
     end
 
-    # c.debug_logger = File.open('log/vcr_debug.log', 'w')
+    if File.exists?('log')
+      c.debug_logger = File.open('log/vcr_debug.log', 'w')
+    end
 
     c.default_cassette_options = {
         serialize_with: :compressed,
@@ -58,7 +82,7 @@ def setup_base_vcr(spec_helper_dir, server: nil, application: nil)
         else
           ENV['RERECORD'] ? :all : :once
         end,
-        match_requests_on: [:method, :uri, :try_body, :verify_uri_route]
+        match_requests_on: [:method, :try_uri, :try_body, :verify_uri_route]
     }
 
     # Filter the authorization headers of any request by replacing any occurrence of that request's
@@ -111,6 +135,7 @@ def setup_base_vcr(spec_helper_dir, server: nil, application: nil)
 end
 
 def prepare_vcr_secret
+  p ENV
   secret = ENV["CI_SECRET"]
 
   if (secret.nil? || secret.empty?) && ENV['IS_CI'] != '1'

data/lib/etna/test_auth.rb

@@ -31,6 +31,24 @@ module Etna
       end.to_h
     end
 
+    def update_payload(payload, token, request)
+      route = server.find_route(request)
+
+      payload = payload.map{|k,v| [k.to_sym, v]}.to_h
+
+      return payload unless route
+
+      begin      
+        permissions = Etna::Permissions.from_encoded_permissions(payload[:perm])
+
+        # Skip making an actual call to Janus. This behavior is tested in auth_spec
+
+        payload[:perm] = permissions.to_string
+      end if (!route.ignore_janus? && route.has_user_constraint?(:can_view?))
+
+      payload
+    end
+
     def approve_user(request)
       token = auth(request,:etna)
 
@@ -42,7 +60,10 @@ module Etna
       # We do this to support Metis client tests, we pass in tokens with multiple "."-separated parts, so
       #   have to account for that.
       payload = JSON.parse(Base64.decode64(token.split('.')[1]))
-      request.env['etna.user'] = Etna::User.new(payload.map{|k,v| [k.to_sym, v]}.to_h, token)
+      request.env['etna.user'] = Etna::User.new(
+        update_payload(payload, token, request),
+        token
+      )
     end
 
     def approve_hmac(request)