etna 0.1.43 → 0.1.46

data/lib/etna/clients/metis/client.rb

@@ -24,8 +24,13 @@ module Etna
       end
 
       def list_folder(list_folder_request = ListFolderRequest.new)
-        FoldersAndFilesResponse.new(
-          @etna_client.folder_list(list_folder_request.to_h))
+        if list_folder_request.folder_path != ''
+          FoldersAndFilesResponse.new(
+            @etna_client.folder_list(list_folder_request.to_h))
+        else
+          FoldersAndFilesResponse.new(
+            @etna_client.bucket_list(list_folder_request.to_h))
+        end
       end
 
       def list_folder_by_id(list_folder_by_id_request = ListFolderByIdRequest.new)

data/lib/etna/clients/metis/models.rb

@@ -50,7 +50,10 @@ module Etna
         end
       end
 
-      class ListFolderRequest < Struct.new(:project_name, :bucket_name, :folder_path, keyword_init: true)
+      class FolderRequest < Struct.new(:project_name, :bucket_name, :folder_path, keyword_init: true)
+      end
+
+      class ListFolderRequest < FolderRequest
         include JsonSerializableStruct
 
         def initialize(**params)
@@ -110,7 +113,7 @@ module Etna
         end
       end
 
-      class CreateFolderRequest < Struct.new(:project_name, :bucket_name, :folder_path, keyword_init: true)
+      class CreateFolderRequest < FolderRequest
         include JsonSerializableStruct
 
         def initialize(**params)
@@ -125,7 +128,7 @@ module Etna
         end
       end
 
-      class DeleteFolderRequest < Struct.new(:project_name, :bucket_name, :folder_path, keyword_init: true)
+      class DeleteFolderRequest < FolderRequest
         include JsonSerializableStruct
 
         def initialize(**params)

data/lib/etna/clients/metis/workflows/metis_download_workflow.rb

@@ -18,7 +18,7 @@ module Etna
           completed = 0.0
           start = Time.now
 
-          unless dest_file_or_io.is_a?(IO)
+          unless dest_file_or_io.is_a?(IO) || dest_file_or_io.is_a?(StringIO)
             ::File.open(dest_file_or_io, 'w') do |io|
               return do_download(dest_file_or_io, metis_file, &block)
             end

data/lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb

@@ -6,6 +6,8 @@ module Etna
   module Clients
     class Metis
       class SyncMetisDataWorkflow < Struct.new(:metis_client, :filesystem, :project_name, :bucket_name, :logger, keyword_init: true)
+        DOWNLOAD_REGEX = /^https:\/\/[^\/]*\/(?<project_name>.*)\/download\/(?<bucket_name>.*)\/(?<file_path>[^\?]*).*$/
+
         def copy_directory(src, dest, root = dest)
           response = metis_client.list_folder(ListFolderRequest.new(project_name: project_name, bucket_name: bucket_name, folder_path: src))
 
@@ -20,6 +22,27 @@ module Etna
         end
 
         def copy_file(dest:, url:, stub: false)
+          # This does not work due to the magma bucket's restrictions, but if it did work, it'd be super sweet.
+          # url_match = DOWNLOAD_REGEX.match(url)
+          #
+          # if filesystem.instance_of?(Etna::Filesystem::Metis) && !url_match.nil?
+          #   bucket_name = url_match[:bucket_name]
+          #   project_name = url_match[:project_name]
+          #   file_path = url_match[:file_path]
+          #
+          #   metis_client.copy_files(
+          #     Etna::Clients::Metis::CopyFilesRequest.new(
+          #       project_name: project_name,
+          #       revisions: [
+          #         Etna::Clients::Metis::CopyRevision.new(
+          #           source: "metis://#{project_name}/#{bucket_name}/#{file_path}",
+          #           dest: "metis://#{filesystem.project_name}/#{filesystem.bucket_name}#{dest}",
+          #         )
+          #       ]
+          #     )
+          #   )
+          # end
+
           metadata = metis_client.file_metadata(url)
           size = metadata[:size]
 

data/lib/etna/clients/metis/workflows/walk_metis_diff_workflow.rb

@@ -0,0 +1,95 @@
+module Etna
+  module Clients
+    class Metis
+      class WalkMetisDiffWorkflow < Struct.new(:left_walker, :right_walker, keyword_init: true)
+        # Iterates entries of the form [kind, left | nil, right | nil]
+        # where kind is one of
+        #   :left_unique | :right_unique | :left_is_folder | :right_is_folder
+        #   :unknown | :equal | :right_older | :left_older
+        # and left / right is one of
+        #   nil | Etna::Clients::Metis::File | Etna::Clients::Metis::Folder
+        def each(&block)
+          left_enum = self.left_walker.to_enum
+          right_enum = self.right_walker.to_enum
+
+          l, l_path = next_or_nil(left_enum)
+          r, r_path = next_or_nil(right_enum)
+
+          while l && r
+            if l_path == r_path
+              yield [compare_file_or_folders(l, r), l, r]
+
+              l, l_path = next_or_nil(left_enum)
+              r, r_path = next_or_nil(right_enum)
+            elsif l_path < r_path
+              yield [:left_unique, l, nil]
+              l, l_path = next_or_nil(left_enum)
+            else
+              yield [:right_unique, nil, r]
+              r, r_path = next_or_nil(right_enum)
+            end
+          end
+
+          while l
+            yield [:left_unique, l, nil]
+            l, l_path = next_or_nil(left_enum)
+          end
+
+          while r
+            yield [:right_unique, nil, r]
+            r, r_path = next_or_nil(right_enum)
+          end
+        end
+
+        def next_or_nil(enum)
+          enum.next
+        rescue StopIteration
+          [nil, nil]
+        end
+
+        def compare_file_or_folders(l, r)
+          if l.is_a?(Etna::Clients::Metis::Folder)
+            if r.is_a?(Etna::Clients::Metis::Folder)
+              return compare_file_or_folder_age(l, r)
+            end
+
+            return :left_is_folder
+          end
+
+          if r.is_a?(Etna::Clients::Metis::Folder)
+            return :right_is_folder
+          end
+
+
+          if l.file_hash.nil? || r.file_hash.nil?
+            return :unknown
+          end
+
+          if l.file_hash == r.file_hash
+            return :equal
+          end
+
+          return compare_file_or_folder_age(l, r)
+        end
+
+        def compare_file_or_folder_age(l, r)
+          if l.updated_at.nil?
+            return :unknown
+          end
+
+          if r.updated_at.nil?
+            return :unknown
+          end
+
+          if l.updated_at < r.updated_at
+            return :left_older
+          elsif l.updated_at > r.updated_at
+            return :right_older
+          else
+            return :equal
+          end
+        end
+      end
+    end
+  end
+end

data/lib/etna/clients/metis/workflows/walk_metis_workflow.rb

@@ -0,0 +1,31 @@
+module Etna
+  module Clients
+    class Metis
+      class WalkMetisWorkflow < Struct.new(:metis_client, :project_name,
+          :bucket_name, :logger, :root_dir, keyword_init: true)
+        def each(&block)
+          q = [self.root_dir]
+
+          while (n = q.pop)
+            req = Etna::Clients::Metis::ListFolderRequest.new(
+              project_name: project_name,
+              bucket_name: bucket_name,
+              folder_path: n
+            )
+            next unless metis_client.folder_exists?(req)
+            resp = metis_client.list_folder(req)
+
+            resp.files.all.sort_by { |f| f.file_path[self.root_dir.length..-1] }.each do |file|
+              yield [file, file.file_path[self.root_dir.length..-1]]
+            end
+
+            resp.folders.all.sort_by { |f| f.folder_path[self.root_dir.length..-1] }.each do |f|
+              yield [f, f.folder_path[self.root_dir.length..-1]]
+              q << f.folder_path
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/etna/clients/metis/workflows.rb

@@ -2,3 +2,5 @@ require_relative "./workflows/metis_download_workflow"
 require_relative "./workflows/metis_upload_workflow"
 require_relative "./workflows/sync_metis_data_workflow"
 require_relative "./workflows/ingest_metis_data_workflow"
+require_relative "./workflows/walk_metis_workflow"
+require_relative "./workflows/walk_metis_diff_workflow"

data/lib/etna/controller.rb

@@ -1,4 +1,5 @@
 require 'erb'
+require 'net/smtp'
 
 module Etna
   class Controller
@@ -44,10 +45,56 @@ module Etna
     rescue Exception => e
       error = e
     ensure
-      log_request
+      log_request if !@request.env['etna.dont_log'] || error
       return handle_error(error) if error
     end
 
+    def try_stream(content_type, &block)
+      if @request.env['rack.hijack?']
+        @request.env['rack.hijack'].call
+        stream = @request.env['rack.hijack_io']
+
+        headers = [
+          "HTTP/1.1 200 OK",
+          "Content-Type: #{content_type}"
+        ]
+        stream.write(headers.map { |header| header + "\r\n" }.join)
+        stream.write("\r\n")
+        stream.flush
+
+        Thread.new do
+          block.call(stream)
+        ensure
+          stream.close
+        end
+
+        # IO is now streaming and will be processed by above thread.
+        @response.close
+      else
+        @response['Content-Type'] = content_type
+        block.call(@response)
+        @response.finish
+      end
+    end
+
+    def send_email(to_name, to_email, subject, content)
+      message = <<MESSAGE_END
+From: Data Library <noreply@janus>
+To: #{to_name} <#{to_email}>
+Subject: #{subject}
+
+#{content}
+MESSAGE_END
+
+      unless @server.send(:application).test?
+        Net::SMTP.start('smtp.ucsf.edu') do |smtp|
+          smtp.send_message message, 'noreply@janus', to_email
+        end
+      end
+    rescue => e
+      @logger.log_error(e)
+    end
+
     def require_params(*params)
       missing_params = params.reject{|p| @params.key?(p) }
       raise Etna::BadRequest, "Missing param #{missing_params.join(', ')}" unless missing_params.empty?

data/lib/etna/cwl.rb

@@ -559,6 +559,14 @@ module Etna
       def id
         @attributes['id']
       end
+
+      def type
+        @attributes['type']
+      end
+
+      def format
+        @attributes['format']
+      end
     end
 
     class WorkflowInputParameter < Cwl

data/lib/etna/environment_variables.rb

@@ -0,0 +1,50 @@
+module Etna
+  module EnvironmentVariables
+    # a <- b
+    def self.deep_merge(a, b)
+      if a.is_a?(Hash)
+        if b.is_a?(Hash)
+          b.keys.each do |b_key|
+            a[b_key] = deep_merge(a[b_key], b[b_key])
+          end
+
+          return a
+        end
+      end
+
+      a.nil? ? b : a
+    end
+
+    def self.load_from_env(prefix, root: {}, env: ENV, downcase: true, sep: '__', &path_to_value_mapper)
+      env.keys.each do |key|
+        next unless key.start_with?(prefix + sep)
+
+        path = key.split(sep, -1)
+        path.shift
+        if downcase
+          path.each(&:downcase!)
+        end
+
+        result = path_to_value_mapper.call(path, env[key])
+        next unless result
+
+        path, value = result
+
+        if path.empty?
+          root = EnvironmentVariables.deep_merge(root, value)
+          next
+        end
+
+        target = root
+        while path.length > 1
+          n = path.shift
+          target = (target[n] ||= {})
+        end
+
+        target[path.last] = EnvironmentVariables.deep_merge(target[path.last], value)
+      end
+
+      root
+    end
+  end
+end

data/lib/etna/filesystem.rb

@@ -251,6 +251,8 @@ module Etna
     end
 
     class Metis < Filesystem
+      attr_reader :project_name, :bucket_name
+
       def initialize(metis_client:, project_name:, bucket_name:, root: '/', uuid: SecureRandom.uuid)
         @metis_client = metis_client
         @project_name = project_name

data/lib/etna/injection.rb

@@ -0,0 +1,57 @@
+module Etna
+  class Injection
+    # Extend into class
+    module FromHash
+      def from_hash(hash, hash_has_string_keys, rest: nil, key_rest: nil)
+        ::Etna::Injection.inject_new(self, hash ,hash_has_string_keys, rest: rest, key_rest: key_rest) do |missing_p|
+          raise "required argument '#{missing_p}' of #{self.name} is missing!"
+        end
+      end
+    end
+
+    def self.inject_new(cls, hash, hash_has_string_keys, rest: nil, key_rest: nil, &missing_req_param_cb)
+      args, kwds = prep_args(
+        cls.instance_method(:initialize), hash, hash_has_string_keys,
+        rest: rest, key_rest: key_rest, &missing_req_param_cb
+      )
+
+      cls.new(*args, **kwds)
+    end
+
+    def self.prep_args(method, hash, hash_has_string_keys, rest: nil, key_rest: nil, &missing_req_param_cb)
+      new_k_params = {}
+      new_p_params = []
+
+      method.parameters.each do |type, p_key|
+        h_key = hash_has_string_keys ? p_key.to_s : p_key
+        if type == :rest && rest
+          new_p_params.append(*rest)
+        elsif type == :keyrest && key_rest
+          new_k_params.update(key_rest)
+        elsif type == :req || type == :opt
+          if hash.include?(h_key)
+            new_p_params << hash[h_key]
+          elsif type == :req
+            if block_given?
+              new_p_params << missing_req_param_cb.call(h_key)
+            else
+              new_p_params << nil
+            end
+          end
+        elsif type == :keyreq || type == :key
+          if hash.include?(h_key)
+            new_k_params[p_key] = hash[h_key]
+          elsif type == :keyreq
+            if block_given?
+              new_k_params[p_key] = missing_req_param_cb.call(h_key)
+            else
+              new_k_params[p_key] = nil
+            end
+          end
+        end
+      end
+
+      [new_p_params, new_k_params]
+    end
+  end
+end

data/lib/etna/janus_utils.rb

@@ -0,0 +1,55 @@
+# Utility class to work with Janus for authz and authn
+module Etna
+  class JanusUtils
+    def initialize
+    end
+
+    def projects(token)
+      return [] unless has_janus_config?
+
+      janus_client(token).get_projects.projects
+    rescue
+      # If encounter any issue with Janus, we'll return no projects
+      []
+    end
+
+    def resource_projects(token)
+      projects(token).select do |project|
+        !!project.resource && !project.requires_agreement
+      end
+    end
+
+    def community_projects(token)
+      projects(token).select do |project|
+        !!project.resource && !!project.requires_agreement
+      end
+    end
+
+    def janus_client(token)
+      Etna::Clients::Janus.new(
+        token: token,
+        host: application.config(:janus)[:host],
+      )
+    end
+
+    def valid_task_token?(token)
+      return false unless has_janus_config?
+
+      response = janus_client(token).validate_task_token
+
+      return false unless response.code == "200"
+
+      return true
+    end
+
+    private
+
+    def application
+      @application ||= Etna::Application.instance
+    end
+
+    def has_janus_config?
+      application.config(:janus) && application.config(:janus)[:host]
+    end
+  end
+end

data/lib/etna/permissions.rb

@@ -0,0 +1,104 @@
+module Etna
+  class Permissions
+    def initialize(permissions)
+      @permissions = permissions
+    end
+
+    def self.from_encoded_permissions(encoded_permissions)
+      perms = encoded_permissions.split(/\;/).map do |roles|
+        role, projects = roles.split(/:/)
+
+        projects.split(/\,/).reduce([]) do |perms, project_name|
+          perms << Etna::Permission.new(role, project_name)
+        end
+      end.flatten
+
+      Etna::Permissions.new(perms)
+    end
+
+    def self.from_hash(permissions_hash)
+      perms = permissions_hash.map do |project_name, role_hash|
+        Etna::Permission.new(
+          Etna::Role.new(role_hash[:role], role_hash[:restricted]).key,
+          project_name
+        )
+      end
+
+      Etna::Permissions.new(perms)
+    end
+
+    def to_string
+      @permissions_string ||= @permissions.group_by(&:role_key)
+        .sort_by(&:first)
+        .map do |role_key, permissions|
+        [
+          role_key,
+          permissions.map(&:project_name).sort.join(","),
+        ].join(":")
+      end.join(";")
+    end
+
+    def to_hash
+      @permissions_hash ||= @permissions.map do |permission|
+        [permission.project_name, permission.to_hash]
+      end.to_h
+    end
+
+    def add_permission(permission)
+      return if current_project_names.include?(permission.project_name)
+
+      @permissions << permission
+    end
+
+    def any?(&block)
+      @permissions.any?(&block)
+    end
+
+    private
+
+    def current_project_names
+      @permissions.map(&:project_name)
+    end
+  end
+
+  class Permission
+    attr_reader :role, :project_name, :role_key
+
+    ROLE_NAMES = {
+      "A" => :admin,
+      "E" => :editor,
+      "V" => :viewer,
+      "G" => :guest
+    }
+
+    def initialize(role_key, project_name)
+      @role_key = role_key
+      @role = Etna::Role.new(ROLE_NAMES[role_key.upcase], role_key == role_key.upcase)
+      @project_name = project_name
+    end
+
+    def to_hash
+      role.to_hash
+    end
+  end
+
+  class Role
+    attr_reader :role, :restricted
+    def initialize(role, restricted)
+      @role = role
+      @restricted = restricted
+    end
+
+    def key
+      role_key = role.to_s[0]
+      restricted ? role_key.upcase : role_key
+    end
+
+    def to_hash
+      {
+        role: role,
+        restricted: restricted,
+      }
+    end
+  end
+end

data/lib/etna/redirect.rb

@@ -0,0 +1,36 @@
+module Etna
+  def self.Redirect(req)
+    Redirect.new(req)
+  end
+
+  class Redirect
+    def initialize(request)
+      @request = request
+    end
+
+    def to(path, &block)
+      return Rack::Response.new(
+        [ { errors: [ 'Cannot redirect out of domain' ] }.to_json ], 422,
+        { 'Content-Type' => 'application/json' }
+      ).finish unless matches_domain?(path)
+
+      response = Rack::Response.new
+
+      response.redirect(path.gsub("http://", "https://"), 302)
+
+      yield response if block_given?
+
+      response.finish
+    end
+
+    private
+
+    def matches_domain?(path)
+      top_domain(@request.hostname) == top_domain(URI(path).host)
+    end
+
+    def top_domain(host_name)
+      host_name.split('.')[-2..-1].join('.')
+    end
+  end
+end