etna 0.1.27 → 0.1.32

data/lib/etna/directed_graph.rb

@@ -18,6 +18,56 @@ class DirectedGraph
     parents[parent] = parent_parents
   end
 
+  def serialized_path_from(root)
+    seen = Set.new
+    [].tap do |result|
+      result << root
+      seen.add(root)
+      path_q = paths_from(root)
+
+      until path_q.empty?
+        next_path = path_q.shift
+        next if next_path.nil?
+
+        until next_path.empty?
+          next_n = next_path.shift
+          next if next_n.nil?
+          next if seen.include?(next_n)
+
+          if @parents[next_n].keys.any? { |p| !seen.include?(p) }
+            next_path.unshift(next_n)
+            path_q.push(next_path)
+            break
+          else
+            result << next_n
+            seen.add(next_n)
+          end
+        end
+      end
+    end
+  end
+
+  def paths_from(root)
+    [].tap do |result|
+      parents_of_map = descendants(root)
+      seen = Set.new
+
+      parents_of_map.to_a.sort_by { |k, parents| [-parents.length, k.inspect] }.each do |k, parents|
+        unless seen.include?(k)
+          if @children[k].keys.empty?
+            result << parents + [k]
+          else
+            @children[k].keys.dup.sort.each do |c|
+              result << parents + [k, c]
+            end
+          end
+        end
+
+        parents.each { |p| seen.add(p) }
+      end
+    end
+  end
+
   def descendants(parent)
     seen = Set.new
 

data/lib/etna/filesystem.rb

@@ -1,36 +1,52 @@
 require 'yaml'
 require 'fileutils'
 require 'open3'
+require 'securerandom'
+require 'concurrent-ruby'
 
 module Etna
   # A class that encapsulates opening / reading file system entries that abstracts normal file access in order
   # to make stubbing, substituting, and testing easier.
   class Filesystem
     def with_writeable(dest, opts = 'w', size_hint: nil, &block)
+      raise "with_writeable not supported by #{self.class.name}" unless self.class == Filesystem
       ::File.open(dest, opts, &block)
     end
 
+    def ls(dir)
+      raise "ls not supported by #{self.class.name}" unless self.class == Filesystem
+      ::Dir.entries(dir).select { |p| !p.start_with?('.') }.map do |path|
+        ::File.file?(::File.join(dir, path)) ? [:file, path] : [:dir, path]
+      end
+    end
+
     def with_readable(src, opts = 'r', &block)
+      raise "with_readable not supported by #{self.class.name}" unless self.class == Filesystem
       ::File.open(src, opts, &block)
     end
 
     def mkdir_p(dir)
+      raise "mkdir_p not supported by #{self.class.name}" unless self.class == Filesystem
       ::FileUtils.mkdir_p(dir)
     end
 
     def rm_rf(dir)
+      raise "rm_rf not supported by #{self.class.name}" unless self.class == Filesystem
       ::FileUtils.rm_rf(dir)
     end
 
     def tmpdir
+      raise "tmpdir not supported by #{self.class.name}" unless self.class == Filesystem
       ::Dir.mktmpdir
     end
 
     def exist?(src)
+      raise "exist? not supported by #{self.class.name}" unless self.class == Filesystem
       ::File.exist?(src)
     end
 
     def mv(src, dest)
+      raise "mv not supported by #{self.class.name}" unless self.class == Filesystem
       ::FileUtils.mv(src, dest)
     end
 
@@ -179,7 +195,7 @@ module Etna
           cmd << remote_path
           cmd << local_path
 
-          cmd << { out: wd }
+          cmd << {out: wd}
         elsif opts.include?('w')
           cmd << '--mode=send'
           cmd << "--host=#{@host}"
@@ -187,7 +203,7 @@ module Etna
           cmd << local_path
           cmd << remote_path
 
-          cmd << { in: rd }
+          cmd << {in: rd}
         end
 
         cmd
@@ -196,23 +212,10 @@ module Etna
 
     # Genentech's aspera deployment doesn't support modern commands, unfortunately...
     class GeneAsperaCliFilesystem < AsperaCliFilesystem
-      def tmpdir
-        raise "tmpdir is not supported"
-      end
-
-      def rm_rf
-        raise "rm_rf is not supported"
-      end
-
       def mkdir_p(dest)
         # Pass through -- this file system creates containing directories by default, womp womp.
       end
 
-      def mv
-        raise "mv is not supported"
-      end
-
-
       def mkcommand(rd, wd, file, opts, size_hint: nil)
         if opts.include?('w')
           super.map do |e|
@@ -230,6 +233,8 @@ module Etna
       end
 
       def with_writeable(dest, opts = 'w', size_hint: nil, &block)
+        raise "#{self.class.name} requires size_hint in with_writeable" if size_hint.nil?
+
         super do |io|
           io.write("File: #{::File.basename(dest)}\n")
           io.write("Size: #{size_hint}\n")
@@ -239,6 +244,131 @@ module Etna
       end
     end
 
+    class Metis < Filesystem
+      def initialize(metis_client:, project_name:, bucket_name:, root: '/', uuid: SecureRandom.uuid)
+        @metis_client = metis_client
+        @project_name = project_name
+        @bucket_name = bucket_name
+        @root = root
+        @metis_uid = uuid
+      end
+
+      def metis_path_of(path)
+        joined = ::File.join(@root, path)
+        joined[0] == "/" ? joined.slice(1..-1) : joined
+      end
+
+      def create_upload_workflow
+        Etna::Clients::Metis::MetisUploadWorkflow.new(metis_client: @metis_client, metis_uid: @metis_uid, project_name: @project_name, bucket_name: @bucket_name, max_attempts: 3)
+      end
+
+      def with_hot_pipe(opts, receiver, *args, &block)
+        rp, wp = IO.pipe
+        begin
+          executor = Concurrent::SingleThreadExecutor.new(fallback_policy: :abort)
+          begin
+            if opts.include?('w')
+              future = Concurrent::Promises.future_on(executor) do
+                self.send(receiver, rp, *args)
+              rescue => e
+                Etna::Application.instance.logger.log_error(e)
+                raise e
+              ensure
+                rp.close
+              end
+
+              yield wp
+            else
+              future = Concurrent::Promises.future_on(executor) do
+                self.send(receiver, wp, *args)
+              rescue => e
+                Etna::Application.instance.logger.log_error(e)
+                raise e
+              ensure
+                wp.close
+              end
+
+              yield rp
+            end
+
+            future.wait!
+          ensure
+            executor.shutdown
+            executor.kill unless executor.wait_for_termination(5)
+          end
+        ensure
+          rp.close
+          wp.close
+        end
+      end
+
+      def do_streaming_upload(rp, dest, size_hint)
+        streaming_upload = Etna::Clients::Metis::MetisUploadWorkflow::StreamingIOUpload.new(readable_io: rp, size_hint: size_hint)
+        create_upload_workflow.do_upload(
+            streaming_upload,
+            metis_path_of(dest)
+        )
+      end
+
+      def with_writeable(dest, opts = 'w', size_hint: nil, &block)
+        self.with_hot_pipe(opts, :do_streaming_upload, dest, size_hint) do |wp|
+          yield wp
+        end
+      end
+
+      def create_download_workflow
+        Etna::Clients::Metis::MetisDownloadWorkflow.new(metis_client: @metis_client, project_name: @project_name, bucket_name: @bucket_name, max_attempts: 3)
+      end
+
+      def do_streaming_download(wp, metis_file)
+        create_download_workflow.do_download(wp, metis_file)
+      end
+
+      def with_readable(src, opts = 'r', &block)
+        metis_file = list_metis_directory(::File.dirname(src)).files.all.find { |f| f.file_name == ::File.basename(src) }
+        raise "Metis file at #{@project_name}/#{@bucket_name}/#{@root}/#{src} not found.  No such file" if metis_file.nil?
+
+        self.with_hot_pipe(opts, :do_streaming_download, metis_file) do |rp|
+          yield rp
+        end
+      end
+
+      def list_metis_directory(path)
+        @metis_client.list_folder(Etna::Clients::Metis::ListFolderRequest.new(project_name: @project_name, bucket_name: @bucket_name, folder_path: metis_path_of(path)))
+      end
+
+      def mkdir_p(dir)
+        create_folder_request = Etna::Clients::Metis::CreateFolderRequest.new(
+            project_name: @project_name,
+            bucket_name: @bucket_name,
+            folder_path: metis_path_of(dir),
+        )
+        @metis_client.create_folder(create_folder_request)
+      end
+
+      def ls(dir)
+        response = list_metis_directory(::File.dirname(dir))
+        response.files.map { |f| [:file, f.file_name] } + response.folders.map { |f| [:folder, f.folder_name] }
+      end
+
+      def exist?(src)
+        begin
+          response = list_metis_directory(::File.dirname(src))
+        rescue Etna::Error => e
+          if e.status == 404
+            return false
+          elsif e.message =~ /Invalid folder/
+            return false
+          end
+
+          raise e
+        end
+
+        response.files.all.any? { |f| f.file_name == ::File.basename(src) } ||
+            response.folders.all.any? { |f| f.folder_name == ::File.basename(src) }
+      end
+    end
+
     class Mock < Filesystem
       def initialize(&new_io)
         @files = {}

data/lib/etna/generate_autocompletion_script.rb

@@ -29,10 +29,12 @@ module Etna
       write %Q(elif [[ -z "$(echo $all_flag_completion_names | xargs)" ]]; then)
       write "return"
       write %Q(elif [[ "$all_flag_completion_names" =~ $1\\  ]]; then)
+      write %Q(if ! [[ "$multi_flags" =~ $1\\  ]]; then)
       write %Q(all_flag_completion_names="${all_flag_completion_names//$1\\ /}")
+      write 'fi'
       write 'a=$1'
       write 'shift'
-      write %Q(if [[ "$string_flag_completion_names" =~ $a\\  ]]; then)
+      write %Q(if [[ "$arg_flag_completion_names" =~ $a\\  ]]; then)
       write 'if [[ "$#" == "1" ]];  then'
       write %Q(a="${a//--/}")
       write %Q(a="${a//-/_}")
@@ -58,12 +60,14 @@ module Etna
 
     def enable_flags(flags_container)
       boolean_flags = flags_container.boolean_flags
-      string_flags = flags_container.string_flags
-      flags = boolean_flags + string_flags
+      multi_flags = flags_container.multi_flags
+      arg_flags = flags_container.string_flags + multi_flags
+      flags = boolean_flags + arg_flags
       write %Q(all_flag_completion_names="$all_flag_completion_names #{flags.join(' ')} ")
-      write %Q(string_flag_completion_names="$string_flag_completion_names #{string_flags.join(' ')} ")
+      write %Q(arg_flag_completion_names="$arg_flag_completion_names #{arg_flags.join(' ')} ")
+      write %Q(multi_flags="$multi_flags #{multi_flags.join(' ')} ")
 
-      string_flags.each do |flag|
+      arg_flags.each do |flag|
         write %Q(declare _completions_for_#{flag_as_parameter(flag)}="#{completions_for(flag_as_parameter(flag)).join(' ')}")
       end
     end
@@ -107,7 +111,8 @@ function _#{name}_completions() {
 
 function _#{name}_inner_completions() {
   local all_flag_completion_names=''
-  local string_flag_completion_names=''
+  local arg_flag_completion_names=''
+  local multi_flags=''
   local all_completion_names=''
   local i=''
   local a=''

data/lib/etna/hmac.rb

@@ -20,14 +20,14 @@ module Etna
     end
 
     # this returns arguments for URI::HTTP.build
-    def url_params
+    def url_params(with_headers=true)
       params = {
         signature: signature,
         expiration: @expiration,
         nonce: @nonce,
         id: @id.to_s,
         headers: @headers.keys.join(','),
-      }.merge(@headers).map do |name, value|
+      }.merge(with_headers ? @headers : {}).map do |name, value|
         [
           "X-Etna-#{ name.to_s.split(/_/).map(&:capitalize).join('-') }",
           value

data/lib/etna/route.rb

@@ -9,6 +9,7 @@ module Etna
       @name = route_name(options)
       @route = route.gsub(/\A(?=[^\/])/, '/')
       @block = block
+      @match_ext = options[:match_ext]
     end
 
     def to_hash
@@ -72,9 +73,7 @@ module Etna
         user = request.env['etna.user']
 
         params = request.env['rack.request.params'].map do |key,value|
-          value = value.to_s
-          value = value[0..500] + "..." + value[-100..-1] if value.length > 600
-          [ key, value ]
+          [ key, redact(key, value) ]
         end.to_h
 
         logger.warn("User #{user ? user.email : :unknown} calling #{controller}##{action} with params #{params}")
@@ -94,6 +93,39 @@ module Etna
 
     private
 
+    def application
+      @application ||= Etna::Application.instance
+    end
+
+    def compact(value)
+      value = value.to_s
+      value = value[0..500] + "..." + value[-100..-1] if value.length > 600
+      value
+    end
+
+    def redact_keys
+      @redact_keys ||= application.config(:log_redact_keys).split(",").map do |key|
+        key.to_sym
+      end
+    end
+
+    def redact(key, value)
+      # From configuration, redact any values for the supplied key values, so they
+      #   don't appear in the logs.
+      return compact(value) unless application.config(:log_redact_keys)
+
+      if value.is_a?(Hash)
+        redacted_value = value.map do |value_key, value_value|
+          [ value_key, redact(value_key, value_value) ]
+        end.to_h
+        return redacted_value
+      elsif redact_keys.include?(key)
+        return "*"
+      end
+
+      return compact(value)
+    end
+
     def authorized?(request)
       # If there is no @auth requirement, they are ok - this doesn't preclude
       # them being rejected in the controller response
@@ -145,13 +177,21 @@ module Etna
       )
     end
 
+    def separator_free_match
+      if @match_ext
+        '(?<\1>[^\/\?]+)'
+      else
+        '(?<\1>[^\.\/\?]+)'
+      end
+    end
+
     def route_regexp
       @route_regexp ||=
         Regexp.new(
           '\A' +
           @route.
             # any :params match separator-free strings
-            gsub(NAMED_PARAM, '(?<\1>[^\.\/\?]+)').
+            gsub(NAMED_PARAM, separator_free_match).
             # any *params match arbitrary strings
             gsub(GLOB_PARAM, '(?<\1>.+)').
             # ignore any trailing slashes in the route

data/lib/etna/spec/auth.rb

@@ -2,22 +2,22 @@ module Etna::Spec
   module Auth
     AUTH_USERS = {
       superuser: {
-        email: 'zeus@olympus.org', first: 'Zeus', perm: 'A:administration'
+        email: 'zeus@olympus.org', name: 'Zeus', perm: 'A:administration'
       },
       admin: {
-        email: 'hera@olympus.org', first: 'Hera', perm: 'a:labors'
+        email: 'hera@olympus.org', name: 'Hera', perm: 'a:labors'
       },
       editor: {
-        email: 'eurystheus@twelve-labors.org', first: 'Eurystheus', perm: 'E:labors'
+        email: 'eurystheus@twelve-labors.org', name: 'Eurystheus', perm: 'E:labors'
       },
       restricted_editor: {
-        email: 'copreus@twelve-labors.org', first: 'Copreus', perm: 'e:labors'
+        email: 'copreus@twelve-labors.org', name: 'Copreus', perm: 'e:labors'
       },
       viewer: {
-        email: 'hercules@twelve-labors.org', first: 'Hercules', perm: 'v:labors'
+        email: 'hercules@twelve-labors.org', name: 'Hercules', perm: 'v:labors'
       },
       non_user: {
-        email: 'nessus@centaurs.org', first: 'Nessus', perm: ''
+        email: 'nessus@centaurs.org', name: 'Nessus', perm: ''
       }
     }