etna 0.1.28 → 0.1.29

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f72e476ca58afe56ac2d2334ae35dfe159409892f54bda2624b73d83da574309
-  data.tar.gz: 89e4ccce963ca79bddcd668da5817c94d95ce609612d2d34a635707588dfe88a
+  metadata.gz: 8d4c9465edd6fd280a9cd91b9c3abbc3ef08114a46e3ffb41e14a20ce612fe7f
+  data.tar.gz: bfd29114b8db331c92f79be1fe765fa6b64c7b5ee4a8543cd75f6c9fe7c38a84
 SHA512:
-  metadata.gz: 76f01865ad042215703f305f1ceaf7eeee4febacb114490481e91c9cbbfbf83bedb004d725d18f90d4ffa34efb8ed91741c91e0244fb60f704fba7e3f2e65057
-  data.tar.gz: e8c352582c61d655881d8290a3be01a03cd27c4b3167d60bab077256a5222cfba94a7401130cff14582230fdcef2068498abaf66fda4176b0c3fc486b3cf202e
+  metadata.gz: 8ea7b21a563d743cdb6f8a06f92ca15fdeb56ecb291b27ff32a555a8f863cd6ce4400fc6954ae5620a9766bbafbed861609f52309f4de3d407f117d4f9b3f1b1
+  data.tar.gz: 6a9530ee4beb8e6a6d7940c30a2a2bc7412dc4420492e650cb731eac4e5e958a4ca630a966aede4591706ae79212e3290b1a080b5fafe5549df24cc302cbc82b

data/lib/etna/clients/magma/workflows/create_project_workflow.rb

@@ -70,7 +70,7 @@ module Etna
           puts "Creating Janus project."
           create_janus_project!
           puts "Done! Adding you as an administrator on the project."
-          add_janus_user(user['email'], "#{user['first']} #{user['last']}", 'editor')
+          add_janus_user(user['email'], "#{user['name']}", 'editor')
           promote_to_administrator(user['email'])
           update_magma_client_token!
 

data/lib/etna/clients/magma/workflows/crud_workflow.rb

@@ -36,8 +36,17 @@ module Etna
           documents = Documents.new({})
           last_page = nil
           while last_page.nil? || last_page.models.model_keys.map { |k| last_page.models.model(k).documents.raw.length }.sum > 0
+            attempts = 0
             begin
+              attempts += 1
               last_page = magma_client.retrieve(request)
+            # Unfortunately, paging in magma is not great and times out from time to time.
+            rescue Net::ReadTimeout => e
+              if attempts > 5
+                raise e
+              end
+
+              retry
             rescue Etna::Error => e
               raise e unless e.message.include?('not found')
               break

data/lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb

@@ -1,5 +1,4 @@
 require 'ostruct'
-require 'digest'
 require 'fileutils'
 require 'tempfile'
 
@@ -9,11 +8,11 @@ module Etna
       class MaterializeDataWorkflow < Struct.new(
           :metis_client, :magma_client, :project_name,
           :model_name, :model_filters, :model_attributes_mask,
-          :filesystem, :logger, :stub_files,
-          :skip_tmpdir, keyword_init: true)
+          :filesystem, :logger, :stub_files, :concurrency,
+          :record_names, keyword_init: true)
 
         def initialize(**kwds)
-          super(**({filesystem: Etna::Filesystem.new}.update(kwds)))
+          super(**({filesystem: Etna::Filesystem.new, concurrency: 10, record_names: "all"}.update(kwds)))
         end
 
         def magma_crud
@@ -25,31 +24,47 @@ module Etna
         end
 
         def materialize_all(dest)
-          tmpdir = skip_tmpdir ? nil : filesystem.tmpdir
+          templates = {}
+
+          semaphore = Concurrent::Semaphore.new(concurrency)
+          errors = Queue.new
+
+          model_walker.walk_from(
+              model_name,
+              record_names,
+              model_attributes_mask: model_attributes_mask,
+              model_filters: model_filters,
+              page_size: 500,
+          ) do |template, document|
+            logger&.info("Materializing #{template.name}##{document[template.identifier]}")
+            templates[template.name] = template
+
+            begin
+              if (error = errors.pop(true))
+                raise error
+              end
+            rescue ThreadError
+            end
 
-          begin
-            model_walker.walk_from(
-                model_name,
-                model_attributes_mask: model_attributes_mask,
-                model_filters: model_filters,
-            ) do |template, document|
-              logger&.info("Materializing #{template.name}##{document[template.identifier]}")
-              materialize_record(dest, tmpdir, template, document)
+            semaphore.acquire
+            Thread.new do
+              begin
+                materialize_record(dest, template, document)
+              rescue => e
+                errors << e
+              ensure
+                semaphore.release
+              end
             end
-          ensure
-            filesystem.rm_rf(tmpdir) unless skip_tmpdir
           end
-        end
 
-        def each_root_record
-          request = RetrievalRequest.new(project_name: project_name, model_name: model_name, record_names: "all",
-              filter: filter, page_size: 100, page: 1)
-          magma_crud.page_records(model_name, request) do |response|
-            model = response.models.model(model_name)
-            template = model.template
-            model.documents.document_keys.each do |key|
-              yield template, model.documents.document(key)
+          semaphore.acquire(concurrency)
+
+          begin
+            if (error = errors.pop(true))
+              raise error
             end
+          rescue ThreadError
           end
         end
 
@@ -78,11 +93,10 @@ module Etna
           @sync_metis_data_workflow ||= Etna::Clients::Metis::SyncMetisDataWorkflow.new(
               metis_client: metis_client,
               logger: logger,
-              skip_tmpdir: skip_tmpdir,
               filesystem: filesystem)
         end
 
-        def materialize_record(dest_dir, tmpdir, template, record)
+        def materialize_record(dest_dir, template, record)
           record_to_serialize = record.dup
 
           each_file(template, record) do |attr_name, url, filename, idx|
@@ -91,13 +105,14 @@ module Etna
             end
 
             dest_file = File.join(dest_dir, metadata_file_name(record_name: record[template.identifier], record_model_name: template.name, ext: "_#{attr_name}_#{idx}#{File.extname(filename)}"))
-            sync_metis_data_workflow.copy_file(bin_root_dir: dest_dir, tmpdir: tmpdir, dest: dest_file, url: url, stub: stub_files)
-            record_to_serialize[attr_name] << { file: dest_file, original_filename: filename }
+            sync_metis_data_workflow.copy_file(dest: dest_file, url: url, stub: stub_files)
+            record_to_serialize[attr_name] << {file: dest_file, original_filename: filename}
           end
 
           dest_file = File.join(dest_dir, metadata_file_name(record_name: record[template.identifier], record_model_name: template.name, ext: '.json'))
           filesystem.mkdir_p(File.dirname(dest_file))
           json = record_to_serialize.to_json
+
           filesystem.with_writeable(dest_file, "w", size_hint: json.bytes.length) do |io|
             io.write(json)
           end

data/lib/etna/clients/magma/workflows/walk_model_tree_workflow.rb

@@ -7,6 +7,10 @@ module Etna
   module Clients
     class Magma
       class WalkModelTreeWorkflow < Struct.new(:magma_crud, :logger, keyword_init: true)
+        def initialize(**args)
+          super(**({}.update(args)))
+        end
+
         def walk_from(
             model_name,
             record_names = 'all',
@@ -43,7 +47,8 @@ module Etna
 
               template.attributes.attribute_keys.each do |attr_name|
                 attributes_mask = model_attributes_mask[model_name]
-                next if !attributes_mask.nil? && !attributes_mask.include?(attr_name) && attr_name != template.identifier
+                black_listed = !attributes_mask.nil? && !attributes_mask.include?(attr_name)
+                next if black_listed && attr_name != template.identifier && attr_name != 'parent'
                 attributes << attr_name
 
                 attr = template.attributes.attribute(attr_name)
@@ -58,7 +63,7 @@ module Etna
                 elsif attr.attribute_type == AttributeType::CHILD
                   related_models[attr.link_model_name] ||= Set.new
                   links << attr_name
-                elsif attr.attribute_type == AttributeType::PARENT
+                elsif attr.attribute_type == AttributeType::PARENT && !black_listed
                   related_models[attr.link_model_name] ||= Set.new
                   links << attr_name
                 end

data/lib/etna/clients/metis/client.rb

@@ -98,7 +98,7 @@ module Etna
         @etna_client.get(download_path) do |response|
           return {
               etag: response['ETag'].gsub(/"/, ''),
-              size: response['Content-Length'],
+              size: response['Content-Length'].to_i,
           }
         end
       end

data/lib/etna/clients/metis/workflows/metis_download_workflow.rb

@@ -13,23 +13,27 @@ module Etna
         end
 
         # TODO: Might be possible to use range headers to select and resume downloads on failure in the future.
-        def do_download(dest_file, metis_file, &block)
+        def do_download(dest_file_or_io, metis_file, &block)
           size = metis_file.size
           completed = 0.0
           start = Time.now
 
-          ::File.open(dest_file, "w") do |io|
-            metis_client.download_file(metis_file) do |chunk|
-              io.write chunk
-              completed += chunk.size
-
-              block.call([
-                  :progress,
-                  size == 0 ? 1 : completed / size,
-                  (completed / (Time.now - start)).round(2),
-              ]) unless block.nil?
+          unless dest_file_or_io.is_a?(IO)
+            ::File.open(dest_file_or_io, 'w') do |io|
+              return do_download(dest_file_or_io, metis_file, &block)
             end
           end
+
+          metis_client.download_file(metis_file) do |chunk|
+            dest_file_or_io.write chunk
+            completed += chunk.size
+
+            block.call([
+                :progress,
+                size == 0 ? 1 : completed / size,
+                (completed / (Time.now - start)).round(2),
+            ]) unless block.nil?
+          end
         end
       end
     end

data/lib/etna/clients/metis/workflows/metis_upload_workflow.rb

@@ -4,17 +4,26 @@ require 'fileutils'
 require 'tempfile'
 require 'securerandom'
 
+$digest_mutx = Mutex.new
+
 module Etna
   module Clients
     class Metis
       class MetisUploadWorkflow < Struct.new(:metis_client, :metis_uid, :project_name, :bucket_name, :max_attempts, keyword_init: true)
+        class StreamingUploadError < StandardError
+        end
+
 
         def initialize(args)
           super({max_attempts: 3, metis_uid: SecureRandom.hex}.update(args))
         end
 
-        def do_upload(source_file, dest_path, &block)
-          upload = Upload.new(source_file: source_file)
+        def do_upload(source_file_or_upload, dest_path, &block)
+          unless source_file_or_upload.is_a?(Upload)
+            upload = Upload.new(source_file: source_file_or_upload)
+          else
+            upload = source_file_or_upload
+          end
 
           dir = ::File.dirname(dest_path)
           metis_client.create_folder(CreateFolderRequest.new(
@@ -68,9 +77,11 @@ module Etna
 
               unsent_zero_byte_file = false
             rescue Etna::Error => e
-              m = yield [:error, e] unless block.nil?
-              if m == false
-                raise e
+              unless block.nil?
+                m = yield [:error, e]
+                if m == false
+                  raise e
+                end
               end
 
               if e.status == 422
@@ -90,13 +101,17 @@ module Etna
           end
         end
 
-        class Upload < Struct.new(:source_file, :next_blob_size, :current_byte_position, keyword_init: true)
+        class Upload
           INITIAL_BLOB_SIZE = 2 ** 10
           MAX_BLOB_SIZE = 2 ** 22
           ZERO_HASH = 'd41d8cd98f00b204e9800998ecf8427e'
 
-          def initialize(**args)
-            super
+          attr_accessor :source_file, :next_blob_size, :current_byte_position
+
+          def initialize(source_file: nil, next_blob_size: nil, current_byte_position: nil)
+            self.source_file = source_file
+            self.next_blob_size = next_blob_size
+            self.current_byte_position = current_byte_position
             self.next_blob_size = [file_size, INITIAL_BLOB_SIZE].min
             self.current_byte_position = 0
           end
@@ -108,10 +123,10 @@ module Etna
           def advance_position!
             self.current_byte_position = self.current_byte_position + self.next_blob_size
             self.next_blob_size = [
-                MAX_BLOB_SIZE,
-                # in fact we should stop when we hit the end of the file
-                file_size - current_byte_position
-            ].min
+                                      MAX_BLOB_SIZE,
+                                      # in fact we should stop when we hit the end of the file
+                                      file_size - current_byte_position
+                                  ].min
           end
 
           def complete?
@@ -119,7 +134,14 @@ module Etna
           end
 
           def next_blob_hash
-            Digest::MD5.hexdigest(next_blob_bytes)
+            bytes = next_blob_bytes
+            if bytes.empty?
+              return ZERO_HASH
+            end
+
+            $digest_mutx.synchronize do
+              return Digest::MD5.hexdigest(bytes)
+            end
           end
 
           def next_blob_bytes
@@ -131,6 +153,54 @@ module Etna
             self.next_blob_size = upload_response.next_blob_size
           end
         end
+
+        class StreamingIOUpload < Upload
+          def initialize(readable_io:, size_hint: 0, **args)
+            @readable_io = readable_io
+            @size_hint = size_hint
+            @read_position = 0
+            @last_bytes = ""
+            super(**args)
+          end
+
+          def file_size
+            @size_hint
+          end
+
+          def next_blob_bytes
+            next_left = current_byte_position
+            next_right = current_byte_position + next_blob_size
+
+            if next_right < @read_position
+              raise StreamingUploadError.new("Upload needs restart, but source is streaming and ephemeral. #{next_right} #{@read_position} You need to restart the source stream and create a new upload.")
+            elsif @read_position < next_left
+              # read from the stream and discard until we are positioned for the next read.
+              data = @readable_io.read(next_left - @read_position)
+              raise StreamingUploadError.new("Unexpected EOF in read stream") if data.nil?
+
+              @read_position += data.bytes.length
+            end
+
+            # If we have consumed all requested data, return what we have consumed.
+            # If we have requested no data, make sure to provide "" as the result.
+            if next_right == @read_position
+              return @last_bytes
+            end
+
+            if @read_position != next_left
+              raise StreamingUploadError.new("Alignment error, source data does not match expected upload resume. #{@read_position} #{next_left} Restart the upload to address.")
+            end
+
+            @last_bytes = "".tap do |bytes|
+              while @read_position < next_right
+                bytes << @readable_io.read(next_right - @read_position).tap do |data|
+                  raise StreamingUploadError.new("Unexpected EOF in read stream") if data.nil?
+                  @read_position += data.bytes.length
+                end
+              end
+            end
+          end
+        end
       end
     end
   end

data/lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb

@@ -1,107 +1,71 @@
 require 'ostruct'
-require 'digest'
 require 'fileutils'
 require 'tempfile'
 
 module Etna
   module Clients
     class Metis
-      class SyncMetisDataWorkflow < Struct.new(:metis_client, :filesystem, :project_name, :bucket_name,
-          :logger, :skip_tmpdir, keyword_init: true)
-        def copy_directory(src, dest, root = dest, tmpdir = nil)
-          own_tmpdir = tmpdir.nil? && !skip_tmpdir
-          if own_tmpdir
-            tmpdir = filesystem.tmpdir
-          end
-
-          begin
-            response = metis_client.list_folder(ListFolderRequest.new(project_name: project_name, bucket_name: bucket_name, folder_path: src))
+      class SyncMetisDataWorkflow < Struct.new(:metis_client, :filesystem, :project_name, :bucket_name, :logger, keyword_init: true)
+        def copy_directory(src, dest, root = dest)
+          response = metis_client.list_folder(ListFolderRequest.new(project_name: project_name, bucket_name: bucket_name, folder_path: src))
 
-            response.files.all.each do |file|
-              logger&.info("Copying file #{file.file_path} (#{Etna::Formatting.as_size(file.size)})")
-              copy_file(bin_root_dir: root, tmpdir: tmpdir, dest: ::File.join(dest, file.file_name), url: file.download_url)
-            end
-
-            response.folders.all.each do |folder|
-              copy_directory(::File.join(src, folder.folder_name), ::File.join(dest, folder.folder_name), root, tmpdir)
-            end
-          ensure
-            filesystem.rm_rf(tmpdir) if own_tmpdir
+          response.files.all.each do |file|
+            logger&.info("Copying file #{file.file_path} (#{Etna::Formatting.as_size(file.size)})")
+            copy_file(dest: ::File.join(dest, file.file_name), url: file.download_url)
           end
-        end
-
-        def bin_file_name(etag:)
-          "bin/#{etag}"
-        end
 
-        def with_maybe_intermediate_tmp_dest(bin_file_name:, tmpdir:, dest_file_name:, &block)
-          filesystem.mkdir_p(::File.dirname(dest_file_name))
-          if tmpdir.nil?
-            yield dest_file_name
-          else
-            tmp_file = ::File.join(tmpdir, ::File.basename(bin_file_name))
-            yield tmp_file
-            filesystem.mv(tmp_file, dest_file_name)
+          response.folders.all.each do |folder|
+            copy_directory(::File.join(src, folder.folder_name), ::File.join(dest, folder.folder_name), root)
           end
         end
 
-        def copy_file(bin_root_dir:, tmpdir:, dest:, url:, stub: false)
+        def copy_file(dest:, url:, stub: false)
           metadata = metis_client.file_metadata(url)
-          etag = metadata[:etag]
           size = metadata[:size]
 
-          dest_bin_file = ::File.join(bin_root_dir, bin_file_name(etag: etag))
-          # Already materialized, continue
-          if filesystem.exist?(dest_bin_file)
-            return
-          end
-
-          with_maybe_intermediate_tmp_dest(bin_file_name: dest_bin_file, tmpdir: tmpdir, dest_file_name: dest) do |tmp_file|
-            upload_timings = []
-            upload_amount = 0
-            last_rate = 0.00001
-
-            filesystem.with_writeable(tmp_file, "w", size_hint: size) do |io|
-              if stub
-                io.write("(stub) #{size} bytes")
-              else
-                metis_client.download_file(url) do |chunk|
-                  io.write(chunk)
-
-                  upload_timings << [chunk.length, Time.now.to_f]
-                  upload_amount += chunk.length
-
-                  if upload_timings.length > 150
-                    s, _ = upload_timings.shift
-                    upload_amount -= s
-                  end
+          tmp_file = dest
+          upload_timings = []
+          upload_amount = 0
+          last_rate = 0.00001
+          remaining = size
+
+          filesystem.with_writeable(tmp_file, "w", size_hint: size) do |io|
+            if stub
+              io.write("(stub) #{size} bytes")
+            else
+              metis_client.download_file(url) do |chunk|
+                io.write(chunk)
+
+                upload_timings << [chunk.length, Time.now.to_f]
+                upload_amount += chunk.length
+                remaining -= chunk.length
+
+                if upload_timings.length > 150
+                  s, _ = upload_timings.shift
+                  upload_amount -= s
+                end
 
-                  _, start_time = upload_timings.first
-                  _, end_time = upload_timings.last
+                _, start_time = upload_timings.first
+                _, end_time = upload_timings.last
 
-                  if start_time == end_time
-                    next
-                  end
+                if start_time == end_time
+                  next
+                end
 
-                  rate = upload_amount / (end_time - start_time)
+                rate = upload_amount / (end_time - start_time)
 
-                  if rate / last_rate > 1.3 || rate / last_rate < 0.7
-                    logger&.info("Uploading #{Etna::Formatting.as_size(rate)} per second")
+                if rate / last_rate > 1.3 || rate / last_rate < 0.7
+                  logger&.info("Uploading #{Etna::Formatting.as_size(rate)} per second, #{Etna::Formatting.as_size(remaining)} remaining")
 
-                    if rate == 0
-                      last_rate = 0.0001
-                    else
-                      last_rate = rate
-                    end
+                  if rate == 0
+                    last_rate = 0.0001
+                  else
+                    last_rate = rate
                   end
                 end
               end
-            end
-          end
 
-          filesystem.mkdir_p(::File.dirname(dest_bin_file))
-          filesystem.with_writeable(dest_bin_file, 'w', size_hint: 0) do |io|
-            # empty file marking that this etag has been moved, to save a future write.
+            end
           end
         end
       end

data/lib/etna/cwl.rb

@@ -584,6 +584,10 @@ module Etna
       def default
         @attributes['default']
       end
+
+      def type
+        @attributes['type']
+      end
     end
 
     class StepOutput < Cwl

data/lib/etna/directed_graph.rb

@@ -23,7 +23,7 @@ class DirectedGraph
       parents_of_map = descendants(root)
       seen = Set.new
 
-      parents_of_map.to_a.sort_by { |k, parents| [-parents.length, k] }.each do |k, parents|
+      parents_of_map.to_a.sort_by { |k, parents| [-parents.length, k.inspect] }.each do |k, parents|
         unless seen.include?(k)
           if @children[k].keys.empty?
             result << parents + [k]

data/lib/etna/filesystem.rb

@@ -1,36 +1,52 @@
 require 'yaml'
 require 'fileutils'
 require 'open3'
+require 'securerandom'
+require 'concurrent-ruby'
 
 module Etna
   # A class that encapsulates opening / reading file system entries that abstracts normal file access in order
   # to make stubbing, substituting, and testing easier.
   class Filesystem
     def with_writeable(dest, opts = 'w', size_hint: nil, &block)
+      raise "with_writeable not supported by #{self.class.name}" unless self.class == Filesystem
       ::File.open(dest, opts, &block)
     end
 
+    def ls(dir)
+      raise "ls not supported by #{self.class.name}" unless self.class == Filesystem
+      ::Dir.entries(dir).select { |p| !p.start_with?('.') }.map do |path|
+        ::File.file?(::File.join(dir, path)) ? [:file, path] : [:dir, path]
+      end
+    end
+
     def with_readable(src, opts = 'r', &block)
+      raise "with_readable not supported by #{self.class.name}" unless self.class == Filesystem
       ::File.open(src, opts, &block)
     end
 
     def mkdir_p(dir)
+      raise "mkdir_p not supported by #{self.class.name}" unless self.class == Filesystem
       ::FileUtils.mkdir_p(dir)
     end
 
     def rm_rf(dir)
+      raise "rm_rf not supported by #{self.class.name}" unless self.class == Filesystem
       ::FileUtils.rm_rf(dir)
     end
 
     def tmpdir
+      raise "tmpdir not supported by #{self.class.name}" unless self.class == Filesystem
       ::Dir.mktmpdir
     end
 
     def exist?(src)
+      raise "exist? not supported by #{self.class.name}" unless self.class == Filesystem
       ::File.exist?(src)
     end
 
     def mv(src, dest)
+      raise "mv not supported by #{self.class.name}" unless self.class == Filesystem
       ::FileUtils.mv(src, dest)
     end
 
@@ -179,7 +195,7 @@ module Etna
           cmd << remote_path
           cmd << local_path
 
-          cmd << { out: wd }
+          cmd << {out: wd}
         elsif opts.include?('w')
           cmd << '--mode=send'
           cmd << "--host=#{@host}"
@@ -187,7 +203,7 @@ module Etna
           cmd << local_path
           cmd << remote_path
 
-          cmd << { in: rd }
+          cmd << {in: rd}
         end
 
         cmd
@@ -196,23 +212,10 @@ module Etna
 
     # Genentech's aspera deployment doesn't support modern commands, unfortunately...
     class GeneAsperaCliFilesystem < AsperaCliFilesystem
-      def tmpdir
-        raise "tmpdir is not supported"
-      end
-
-      def rm_rf
-        raise "rm_rf is not supported"
-      end
-
       def mkdir_p(dest)
         # Pass through -- this file system creates containing directories by default, womp womp.
       end
 
-      def mv
-        raise "mv is not supported"
-      end
-
-
       def mkcommand(rd, wd, file, opts, size_hint: nil)
         if opts.include?('w')
           super.map do |e|
@@ -241,6 +244,131 @@ module Etna
       end
     end
 
+    class Metis < Filesystem
+      def initialize(metis_client:, project_name:, bucket_name:, root: '/', uuid: SecureRandom.uuid)
+        @metis_client = metis_client
+        @project_name = project_name
+        @bucket_name = bucket_name
+        @root = root
+        @metis_uid = uuid
+      end
+
+      def metis_path_of(path)
+        joined = ::File.join(@root, path)
+        joined[0] == "/" ? joined.slice(1..-1) : joined
+      end
+
+      def create_upload_workflow
+        Etna::Clients::Metis::MetisUploadWorkflow.new(metis_client: @metis_client, metis_uid: @metis_uid, project_name: @project_name, bucket_name: @bucket_name, max_attempts: 3)
+      end
+
+      def with_hot_pipe(opts, receiver, *args, &block)
+        rp, wp = IO.pipe
+        begin
+          executor = Concurrent::SingleThreadExecutor.new(fallback_policy: :abort)
+          begin
+            if opts.include?('w')
+              future = Concurrent::Promises.future_on(executor) do
+                self.send(receiver, rp, *args)
+              rescue => e
+                Etna::Application.instance.logger.log_error(e)
+                raise e
+              ensure
+                rp.close
+              end
+
+              yield wp
+            else
+              future = Concurrent::Promises.future_on(executor) do
+                self.send(receiver, wp, *args)
+              rescue => e
+                Etna::Application.instance.logger.log_error(e)
+                raise e
+              ensure
+                wp.close
+              end
+
+              yield rp
+            end
+
+            future.wait!
+          ensure
+            executor.shutdown
+            executor.kill unless executor.wait_for_termination(5)
+          end
+        ensure
+          rp.close
+          wp.close
+        end
+      end
+
+      def do_streaming_upload(rp, dest, size_hint)
+        streaming_upload = Etna::Clients::Metis::MetisUploadWorkflow::StreamingIOUpload.new(readable_io: rp, size_hint: size_hint)
+        create_upload_workflow.do_upload(
+            streaming_upload,
+            metis_path_of(dest)
+        )
+      end
+
+      def with_writeable(dest, opts = 'w', size_hint: nil, &block)
+        self.with_hot_pipe(opts, :do_streaming_upload, dest, size_hint) do |wp|
+          yield wp
+        end
+      end
+
+      def create_download_workflow
+        Etna::Clients::Metis::MetisDownloadWorkflow.new(metis_client: @metis_client, project_name: @project_name, bucket_name: @bucket_name, max_attempts: 3)
+      end
+
+      def do_streaming_download(wp, metis_file)
+        create_download_workflow.do_download(wp, metis_file)
+      end
+
+      def with_readable(src, opts = 'r', &block)
+        metis_file = list_metis_directory(::File.dirname(src)).files.all.find { |f| f.file_name == ::File.basename(src) }
+        raise "Metis file at #{@project_name}/#{@bucket_name}/#{@root}/#{src} not found.  No such file" if metis_file.nil?
+
+        self.with_hot_pipe(opts, :do_streaming_download, metis_file) do |rp|
+          yield rp
+        end
+      end
+
+      def list_metis_directory(path)
+        @metis_client.list_folder(Etna::Clients::Metis::ListFolderRequest.new(project_name: @project_name, bucket_name: @bucket_name, folder_path: metis_path_of(path)))
+      end
+
+      def mkdir_p(dir)
+        create_folder_request = Etna::Clients::Metis::CreateFolderRequest.new(
+            project_name: @project_name,
+            bucket_name: @bucket_name,
+            folder_path: metis_path_of(dir),
+        )
+        @metis_client.create_folder(create_folder_request)
+      end
+
+      def ls(dir)
+        response = list_metis_directory(::File.dirname(dir))
+        response.files.map { |f| [:file, f.file_name] } + response.folders.map { |f| [:folder, f.folder_name] }
+      end
+
+      def exist?(src)
+        begin
+          response = list_metis_directory(::File.dirname(src))
+        rescue Etna::Error => e
+          if e.status == 404
+            return false
+          elsif e.message =~ /Invalid folder/
+            return false
+          end
+
+          raise e
+        end
+
+        response.files.all.any? { |f| f.file_name == ::File.basename(src) } ||
+            response.folders.all.any? { |f| f.folder_name == ::File.basename(src) }
+      end
+    end
+
     class Mock < Filesystem
       def initialize(&new_io)
         @files = {}

data/lib/etna/spec/auth.rb

@@ -2,22 +2,22 @@ module Etna::Spec
   module Auth
     AUTH_USERS = {
       superuser: {
-        email: 'zeus@olympus.org', first: 'Zeus', perm: 'A:administration'
+        email: 'zeus@olympus.org', name: 'Zeus', perm: 'A:administration'
       },
       admin: {
-        email: 'hera@olympus.org', first: 'Hera', perm: 'a:labors'
+        email: 'hera@olympus.org', name: 'Hera', perm: 'a:labors'
       },
       editor: {
-        email: 'eurystheus@twelve-labors.org', first: 'Eurystheus', perm: 'E:labors'
+        email: 'eurystheus@twelve-labors.org', name: 'Eurystheus', perm: 'E:labors'
       },
       restricted_editor: {
-        email: 'copreus@twelve-labors.org', first: 'Copreus', perm: 'e:labors'
+        email: 'copreus@twelve-labors.org', name: 'Copreus', perm: 'e:labors'
       },
       viewer: {
-        email: 'hercules@twelve-labors.org', first: 'Hercules', perm: 'v:labors'
+        email: 'hercules@twelve-labors.org', name: 'Hercules', perm: 'v:labors'
       },
       non_user: {
-        email: 'nessus@centaurs.org', first: 'Nessus', perm: ''
+        email: 'nessus@centaurs.org', name: 'Nessus', perm: ''
       }
     }
 

data/lib/etna/user.rb

@@ -7,14 +7,14 @@ module Etna
     }
 
     def initialize params, token=nil
-      @first, @last, @email, @encoded_permissions, encoded_flags = params.values_at(:first, :last, :email, :perm, :flags)
+      @name, @email, @encoded_permissions, encoded_flags = params.values_at(:name, :email, :perm, :flags)
 
       @flags = encoded_flags&.split(/;/) || []
       @token = token unless !token
       raise ArgumentError, "No email given!" unless @email
     end
 
-    attr_reader :first, :last, :email, :token
+    attr_reader :name, :email, :token
 
     def permissions
       @permissions ||= @encoded_permissions.split(/\;/).map do |roles|
@@ -36,10 +36,6 @@ module Etna
       @flags.include?(flag)
     end
 
-    def name
-      "#{first} #{last}"
-    end
-
     def projects
       permissions.keys
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: etna
 version: !ruby/object:Gem::Version
-  version: 0.1.28
+  version: 0.1.29
 platform: ruby
 authors:
 - Saurabh Asthana
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-17 00:00:00.000000000 Z
+date: 2021-02-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -80,6 +80,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby-ext
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: See summary
 email: Saurabh.Asthana@ucsf.edu
 executables: