etna 0.1.28 → 0.1.33

data/lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb

@@ -1,107 +1,71 @@
 require 'ostruct'
-require 'digest'
 require 'fileutils'
 require 'tempfile'
 
 module Etna
   module Clients
     class Metis
-      class SyncMetisDataWorkflow < Struct.new(:metis_client, :filesystem, :project_name, :bucket_name,
-          :logger, :skip_tmpdir, keyword_init: true)
-        def copy_directory(src, dest, root = dest, tmpdir = nil)
-          own_tmpdir = tmpdir.nil? && !skip_tmpdir
-          if own_tmpdir
-            tmpdir = filesystem.tmpdir
-          end
-
-          begin
-            response = metis_client.list_folder(ListFolderRequest.new(project_name: project_name, bucket_name: bucket_name, folder_path: src))
+      class SyncMetisDataWorkflow < Struct.new(:metis_client, :filesystem, :project_name, :bucket_name, :logger, keyword_init: true)
+        def copy_directory(src, dest, root = dest)
+          response = metis_client.list_folder(ListFolderRequest.new(project_name: project_name, bucket_name: bucket_name, folder_path: src))
 
-            response.files.all.each do |file|
-              logger&.info("Copying file #{file.file_path} (#{Etna::Formatting.as_size(file.size)})")
-              copy_file(bin_root_dir: root, tmpdir: tmpdir, dest: ::File.join(dest, file.file_name), url: file.download_url)
-            end
-
-            response.folders.all.each do |folder|
-              copy_directory(::File.join(src, folder.folder_name), ::File.join(dest, folder.folder_name), root, tmpdir)
-            end
-          ensure
-            filesystem.rm_rf(tmpdir) if own_tmpdir
+          response.files.all.each do |file|
+            logger&.info("Copying file #{file.file_path} (#{Etna::Formatting.as_size(file.size)})")
+            copy_file(dest: ::File.join(dest, file.file_name), url: file.download_url)
           end
-        end
-
-        def bin_file_name(etag:)
-          "bin/#{etag}"
-        end
 
-        def with_maybe_intermediate_tmp_dest(bin_file_name:, tmpdir:, dest_file_name:, &block)
-          filesystem.mkdir_p(::File.dirname(dest_file_name))
-          if tmpdir.nil?
-            yield dest_file_name
-          else
-            tmp_file = ::File.join(tmpdir, ::File.basename(bin_file_name))
-            yield tmp_file
-            filesystem.mv(tmp_file, dest_file_name)
+          response.folders.all.each do |folder|
+            copy_directory(::File.join(src, folder.folder_name), ::File.join(dest, folder.folder_name), root)
           end
         end
 
-        def copy_file(bin_root_dir:, tmpdir:, dest:, url:, stub: false)
+        def copy_file(dest:, url:, stub: false)
           metadata = metis_client.file_metadata(url)
-          etag = metadata[:etag]
           size = metadata[:size]
 
-          dest_bin_file = ::File.join(bin_root_dir, bin_file_name(etag: etag))
-          # Already materialized, continue
-          if filesystem.exist?(dest_bin_file)
-            return
-          end
-
-          with_maybe_intermediate_tmp_dest(bin_file_name: dest_bin_file, tmpdir: tmpdir, dest_file_name: dest) do |tmp_file|
-            upload_timings = []
-            upload_amount = 0
-            last_rate = 0.00001
-
-            filesystem.with_writeable(tmp_file, "w", size_hint: size) do |io|
-              if stub
-                io.write("(stub) #{size} bytes")
-              else
-                metis_client.download_file(url) do |chunk|
-                  io.write(chunk)
-
-                  upload_timings << [chunk.length, Time.now.to_f]
-                  upload_amount += chunk.length
-
-                  if upload_timings.length > 150
-                    s, _ = upload_timings.shift
-                    upload_amount -= s
-                  end
+          tmp_file = dest
+          upload_timings = []
+          upload_amount = 0
+          last_rate = 0.00001
+          remaining = size
+
+          filesystem.with_writeable(tmp_file, "w", size_hint: size) do |io|
+            if stub
+              io.write("(stub) #{size} bytes")
+            else
+              metis_client.download_file(url) do |chunk|
+                io.write(chunk)
+
+                upload_timings << [chunk.length, Time.now.to_f]
+                upload_amount += chunk.length
+                remaining -= chunk.length
+
+                if upload_timings.length > 150
+                  s, _ = upload_timings.shift
+                  upload_amount -= s
+                end
 
-                  _, start_time = upload_timings.first
-                  _, end_time = upload_timings.last
+                _, start_time = upload_timings.first
+                _, end_time = upload_timings.last
 
-                  if start_time == end_time
-                    next
-                  end
+                if start_time == end_time
+                  next
+                end
 
-                  rate = upload_amount / (end_time - start_time)
+                rate = upload_amount / (end_time - start_time)
 
-                  if rate / last_rate > 1.3 || rate / last_rate < 0.7
-                    logger&.info("Uploading #{Etna::Formatting.as_size(rate)} per second")
+                if rate / last_rate > 1.3 || rate / last_rate < 0.7
+                  logger&.info("Uploading #{Etna::Formatting.as_size(rate)} per second, #{Etna::Formatting.as_size(remaining)} remaining")
 
-                    if rate == 0
-                      last_rate = 0.0001
-                    else
-                      last_rate = rate
-                    end
+                  if rate == 0
+                    last_rate = 0.0001
+                  else
+                    last_rate = rate
                   end
                 end
               end
-            end
-          end
 
-          filesystem.mkdir_p(::File.dirname(dest_bin_file))
-          filesystem.with_writeable(dest_bin_file, 'w', size_hint: 0) do |io|
-            # empty file marking that this etag has been moved, to save a future write.
+            end
           end
         end
       end

data/lib/etna/command.rb

@@ -203,6 +203,7 @@ module Etna
       @subcommands ||= self.class.constants.sort.reduce({}) do |acc, n|
         acc.tap do
           c = self.class.const_get(n)
+          next unless c.respond_to?(:instance_methods)
           next unless c.instance_methods.include?(:find_command)
           v = c.new(self)
           acc[v.command_name] = v

data/lib/etna/cwl.rb

@@ -584,6 +584,10 @@ module Etna
       def default
         @attributes['default']
       end
+
+      def type
+        @attributes['type']
+      end
     end
 
     class StepOutput < Cwl

data/lib/etna/directed_graph.rb

@@ -7,6 +7,59 @@ class DirectedGraph
   attr_reader :children
   attr_reader :parents
 
+  def full_parentage(n)
+    [].tap do |result|
+      q = @parents[n].keys.dup
+      seen = Set.new
+
+      until q.empty?
+        n = q.shift
+        next if seen.include?(n)
+        seen.add(n)
+
+        result << n
+        q.push(*@parents[n].keys)
+      end
+
+      result.uniq!
+    end
+  end
+
+  def as_normalized_hash(root, include_root = true)
+    q = [root]
+    {}.tap do |result|
+      if include_root
+        result[root] = []
+      end
+
+      seen = Set.new
+
+      until q.empty?
+        n = q.shift
+        next if seen.include?(n)
+        seen.add(n)
+
+        parentage = full_parentage(n)
+
+        @children[n].keys.each do |child_node|
+          q << child_node
+
+          if result.include?(n)
+            result[n] << child_node
+          end
+
+          parentage.each do |grandparent|
+            result[grandparent] << child_node if result.include?(grandparent)
+          end
+
+          result[child_node] = []
+        end
+      end
+
+      result.values.each(&:uniq!)
+    end
+  end
+
   def add_connection(parent, child)
     children = @children[parent] ||= {}
     child_children = @children[child] ||= {}
@@ -18,12 +71,42 @@ class DirectedGraph
     parents[parent] = parent_parents
   end
 
-  def paths_from(root)
+  def serialized_path_from(root, include_root = true)
+    seen = Set.new
+    [].tap do |result|
+      result << root if include_root
+      seen.add(root)
+      path_q = paths_from(root, include_root)
+      traversables = path_q.flatten
+
+      until path_q.empty?
+        next_path = path_q.shift
+        next if next_path.nil?
+
+        until next_path.empty?
+          next_n = next_path.shift
+          next if next_n.nil?
+          next if seen.include?(next_n)
+
+          if @parents[next_n].keys.any? { |p| !seen.include?(p) && traversables.include?(p) }
+            next_path.unshift(next_n)
+            path_q.push(next_path)
+            break
+          else
+            result << next_n
+            seen.add(next_n)
+          end
+        end
+      end
+    end
+  end
+
+  def paths_from(root, include_root = true)
     [].tap do |result|
-      parents_of_map = descendants(root)
+      parents_of_map = descendants(root, include_root)
       seen = Set.new
 
-      parents_of_map.to_a.sort_by { |k, parents| [-parents.length, k] }.each do |k, parents|
+      parents_of_map.to_a.sort_by { |k, parents| [-parents.length, k.inspect] }.each do |k, parents|
         unless seen.include?(k)
           if @children[k].keys.empty?
             result << parents + [k]
@@ -39,7 +122,7 @@ class DirectedGraph
     end
   end
 
-  def descendants(parent)
+  def descendants(parent, include_root = true)
     seen = Set.new
 
     seen.add(parent)
@@ -61,7 +144,7 @@ class DirectedGraph
     while child = queue.pop
       next if seen.include? child
       seen.add(child)
-      path = (paths[child] ||= [parent])
+      path = (paths[child] ||= (include_root ? [parent] : []))
 
       @children[child].keys.each do |child_child|
         queue.push child_child

data/lib/etna/filesystem.rb

@@ -1,36 +1,52 @@
 require 'yaml'
 require 'fileutils'
 require 'open3'
+require 'securerandom'
+require 'concurrent-ruby'
 
 module Etna
   # A class that encapsulates opening / reading file system entries that abstracts normal file access in order
   # to make stubbing, substituting, and testing easier.
   class Filesystem
     def with_writeable(dest, opts = 'w', size_hint: nil, &block)
+      raise "with_writeable not supported by #{self.class.name}" unless self.class == Filesystem
       ::File.open(dest, opts, &block)
     end
 
+    def ls(dir)
+      raise "ls not supported by #{self.class.name}" unless self.class == Filesystem
+      ::Dir.entries(dir).select { |p| !p.start_with?('.') }.map do |path|
+        ::File.file?(::File.join(dir, path)) ? [:file, path] : [:dir, path]
+      end
+    end
+
     def with_readable(src, opts = 'r', &block)
+      raise "with_readable not supported by #{self.class.name}" unless self.class == Filesystem
       ::File.open(src, opts, &block)
     end
 
     def mkdir_p(dir)
+      raise "mkdir_p not supported by #{self.class.name}" unless self.class == Filesystem
       ::FileUtils.mkdir_p(dir)
     end
 
     def rm_rf(dir)
+      raise "rm_rf not supported by #{self.class.name}" unless self.class == Filesystem
       ::FileUtils.rm_rf(dir)
     end
 
     def tmpdir
+      raise "tmpdir not supported by #{self.class.name}" unless self.class == Filesystem
       ::Dir.mktmpdir
     end
 
     def exist?(src)
+      raise "exist? not supported by #{self.class.name}" unless self.class == Filesystem
       ::File.exist?(src)
     end
 
     def mv(src, dest)
+      raise "mv not supported by #{self.class.name}" unless self.class == Filesystem
       ::FileUtils.mv(src, dest)
     end
 
@@ -179,7 +195,7 @@ module Etna
           cmd << remote_path
           cmd << local_path
 
-          cmd << { out: wd }
+          cmd << {out: wd}
         elsif opts.include?('w')
           cmd << '--mode=send'
           cmd << "--host=#{@host}"
@@ -187,7 +203,7 @@ module Etna
           cmd << local_path
           cmd << remote_path
 
-          cmd << { in: rd }
+          cmd << {in: rd}
         end
 
         cmd
@@ -196,23 +212,10 @@ module Etna
 
     # Genentech's aspera deployment doesn't support modern commands, unfortunately...
     class GeneAsperaCliFilesystem < AsperaCliFilesystem
-      def tmpdir
-        raise "tmpdir is not supported"
-      end
-
-      def rm_rf
-        raise "rm_rf is not supported"
-      end
-
       def mkdir_p(dest)
         # Pass through -- this file system creates containing directories by default, womp womp.
       end
 
-      def mv
-        raise "mv is not supported"
-      end
-
-
       def mkcommand(rd, wd, file, opts, size_hint: nil)
         if opts.include?('w')
           super.map do |e|
@@ -241,6 +244,131 @@ module Etna
       end
     end
 
+    class Metis < Filesystem
+      def initialize(metis_client:, project_name:, bucket_name:, root: '/', uuid: SecureRandom.uuid)
+        @metis_client = metis_client
+        @project_name = project_name
+        @bucket_name = bucket_name
+        @root = root
+        @metis_uid = uuid
+      end
+
+      def metis_path_of(path)
+        joined = ::File.join(@root, path)
+        joined[0] == "/" ? joined.slice(1..-1) : joined
+      end
+
+      def create_upload_workflow
+        Etna::Clients::Metis::MetisUploadWorkflow.new(metis_client: @metis_client, metis_uid: @metis_uid, project_name: @project_name, bucket_name: @bucket_name, max_attempts: 3)
+      end
+
+      def with_hot_pipe(opts, receiver, *args, &block)
+        rp, wp = IO.pipe
+        begin
+          executor = Concurrent::SingleThreadExecutor.new(fallback_policy: :abort)
+          begin
+            if opts.include?('w')
+              future = Concurrent::Promises.future_on(executor) do
+                self.send(receiver, rp, *args)
+              rescue => e
+                Etna::Application.instance.logger.log_error(e)
+                raise e
+              ensure
+                rp.close
+              end
+
+              yield wp
+            else
+              future = Concurrent::Promises.future_on(executor) do
+                self.send(receiver, wp, *args)
+              rescue => e
+                Etna::Application.instance.logger.log_error(e)
+                raise e
+              ensure
+                wp.close
+              end
+
+              yield rp
+            end
+
+            future.wait!
+          ensure
+            executor.shutdown
+            executor.kill unless executor.wait_for_termination(5)
+          end
+        ensure
+          rp.close
+          wp.close
+        end
+      end
+
+      def do_streaming_upload(rp, dest, size_hint)
+        streaming_upload = Etna::Clients::Metis::MetisUploadWorkflow::StreamingIOUpload.new(readable_io: rp, size_hint: size_hint)
+        create_upload_workflow.do_upload(
+            streaming_upload,
+            metis_path_of(dest)
+        )
+      end
+
+      def with_writeable(dest, opts = 'w', size_hint: nil, &block)
+        self.with_hot_pipe(opts, :do_streaming_upload, dest, size_hint) do |wp|
+          yield wp
+        end
+      end
+
+      def create_download_workflow
+        Etna::Clients::Metis::MetisDownloadWorkflow.new(metis_client: @metis_client, project_name: @project_name, bucket_name: @bucket_name, max_attempts: 3)
+      end
+
+      def do_streaming_download(wp, metis_file)
+        create_download_workflow.do_download(wp, metis_file)
+      end
+
+      def with_readable(src, opts = 'r', &block)
+        metis_file = list_metis_directory(::File.dirname(src)).files.all.find { |f| f.file_name == ::File.basename(src) }
+        raise "Metis file at #{@project_name}/#{@bucket_name}/#{@root}/#{src} not found.  No such file" if metis_file.nil?
+
+        self.with_hot_pipe(opts, :do_streaming_download, metis_file) do |rp|
+          yield rp
+        end
+      end
+
+      def list_metis_directory(path)
+        @metis_client.list_folder(Etna::Clients::Metis::ListFolderRequest.new(project_name: @project_name, bucket_name: @bucket_name, folder_path: metis_path_of(path)))
+      end
+
+      def mkdir_p(dir)
+        create_folder_request = Etna::Clients::Metis::CreateFolderRequest.new(
+            project_name: @project_name,
+            bucket_name: @bucket_name,
+            folder_path: metis_path_of(dir),
+        )
+        @metis_client.create_folder(create_folder_request)
+      end
+
+      def ls(dir)
+        response = list_metis_directory(::File.dirname(dir))
+        response.files.map { |f| [:file, f.file_name] } + response.folders.map { |f| [:folder, f.folder_name] }
+      end
+
+      def exist?(src)
+        begin
+          response = list_metis_directory(::File.dirname(src))
+        rescue Etna::Error => e
+          if e.status == 404
+            return false
+          elsif e.message =~ /Invalid folder/
+            return false
+          end
+
+          raise e
+        end
+
+        response.files.all.any? { |f| f.file_name == ::File.basename(src) } ||
+            response.folders.all.any? { |f| f.folder_name == ::File.basename(src) }
+      end
+    end
+
     class Mock < Filesystem
       def initialize(&new_io)
         @files = {}