etna 0.1.25 → 0.1.26

data/lib/commands.rb

@@ -351,14 +351,16 @@ class EtnaApp
 
         boolean_flags << '--commit'
         string_flags << '--models'
+        string_flags << '--record_names'
 
-        def execute(project_name, redcap_tokens, models: "all", commit: false)
+        def execute(project_name, redcap_tokens, models: "all", record_names: nil, commit: false)
           raise "Must provide at least one REDCap token (comma-separated)." unless redcap_tokens.split(',').length > 0
 
           puts "NOTE: This is a **preview** of what the data loading will look like. Use the --commit flag to load records into Magma." unless commit
 
           polyphemus_client.job(Etna::Clients::Polyphemus::RedcapJobRequest.new(
             model_names: "all" == models ? "all" : models.split(','),
+            record_names: nil == record_names || "existing" == record_names ? record_names : record_names.split(','),
             redcap_tokens: redcap_tokens.split(','),
             project_name: project_name,
             commit: commit

data/lib/etna.rb

@@ -19,6 +19,7 @@ require_relative './etna/clients'
 require_relative './etna/csvs'
 require_relative './etna/environment_scoped'
 require_relative './etna/filesystem'
+require_relative './etna/formatting'
 
 class EtnaApp
   include Etna::Application

data/lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb

@@ -24,8 +24,8 @@ module Etna
           @model_walker ||= WalkModelTreeWorkflow.new(magma_crud: magma_crud, logger: logger)
         end
 
-        def with_materialized_dir(&block)
-          tmp_dir = filesystem.tmpdir
+        def materialize_all(dest = filesystem.tmpdir)
+          tmpdir = filesystem.tmpdir
 
           begin
             model_walker.walk_from(
@@ -34,12 +34,10 @@ module Etna
                 model_filters: model_filters,
             ) do |template, document|
               logger&.info("Materializing #{template.name}##{document[template.identifier]}")
-              materialize_record(tmp_dir, template, document)
+              materialize_record(dest, tmpdir, template, document)
             end
-
-            yield tmp_dir
           ensure
-            filesystem.rm_rf(tmp_dir)
+            filesystem.rm_rf(tmpdir)
           end
         end
 
@@ -76,58 +74,35 @@ module Etna
           end
         end
 
-        def materialize_record(dest_dir, template, record)
+        def sync_metis_data_workflow
+          @sync_metis_data_workflow ||= Etna::Clients::Metis::SyncMetisDataWorkflow.new(
+              metis_client: metis_client,
+              logger: logger,
+              filesystem: filesystem)
+        end
+
+        def materialize_record(dest_dir, tmpdir, template, record)
           record_to_serialize = record.dup
-          metadata_path = metadata_file_name(record_name: record[template.identifier], record_model_name: template.name)
 
           each_file(template, record) do |attr_name, url, filename, idx|
-            metadata = metis_client.file_metadata(url)
-            etag = metadata[:etag]
-            size = metadata[:size]
-
             if idx == 0
               record_to_serialize[attr_name] = []
             end
 
-            dest_file = bin_file_name(etag: etag)
+            dest_file = File.join(dest_dir, metadata_file_name(record_name: record[template.identifier], record_model_name: template.name, ext: "_#{attr_name}_#{idx}#{File.extname(filename)}"))
+            sync_metis_data_workflow.copy_file(bin_root_dir: dest_dir, tmpdir: tmpdir, dest: dest_file, url: url, stub: stub_files)
             record_to_serialize[attr_name] << { file: dest_file, original_filename: filename }
-
-            # Already materialized, continue
-            if filesystem.exist?(dest_file)
-              next
-            end
-
-            logger&.info("materializing file #{filename} (#{size} bytes)")
-            filesystem.mkdir_p(File.dirname(File.join(dest_dir, dest_file)))
-
-            filesystem.with_writeable(File.join(dest_dir, dest_file), "w") do |io|
-              if stub_files
-                io.write("(stub) #{filename}: #{size} bytes")
-              else
-                metis_client.download_file(url) do |chunk|
-                  if Random.rand < 0.1
-                    logger&.info("Writing #{chunk.length} bytes into #{dest_file}")
-                  end
-
-                  io.write(chunk)
-                end
-              end
-            end
           end
 
-          dest_file = File.join(dest_dir, metadata_path)
+          dest_file = File.join(dest_dir, metadata_file_name(record_name: record[template.identifier], record_model_name: template.name, ext: '.json'))
           filesystem.mkdir_p(File.dirname(dest_file))
           filesystem.with_writeable(dest_file, "w") do |io|
             io.write(record_to_serialize.to_json)
           end
         end
 
-        def metadata_file_name(record_name:, record_model_name:)
-          "#{record_model_name}/#{record_name.gsub(/\s/, '_')}.json"
-        end
-
-        def bin_file_name(etag:)
-          "bin/#{etag}"
+        def metadata_file_name(record_name:, record_model_name:, ext:)
+          "#{record_model_name}/#{record_name.gsub(/\s/, '_')}#{ext}"
         end
       end
     end

data/lib/etna/clients/metis/models.rb

@@ -171,7 +171,11 @@ module Etna
 
       class FoldersAndFilesResponse < FoldersResponse
         def files
-          Files.new(raw[:files])
+          Files.new(raw[:files] || [])
+        end
+
+        def folders
+          Folders.new(raw[:folders] || [])
         end
       end
 
@@ -253,6 +257,10 @@ module Etna
           raw[:folder_path]
         end
 
+        def folder_name
+          raw[:folder_name]
+        end
+
         def bucket_name
           raw[:bucket_name]
         end

data/lib/etna/clients/metis/workflows.rb

@@ -1,2 +1,3 @@
 require_relative './workflows/metis_download_workflow'
 require_relative './workflows/metis_upload_workflow'
+require_relative './workflows/sync_metis_data_workflow'

data/lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb

@@ -0,0 +1,102 @@
+require 'ostruct'
+require 'digest'
+require 'fileutils'
+require 'tempfile'
+
+module Etna
+  module Clients
+    class Metis
+      class SyncMetisDataWorkflow < Struct.new(:metis_client, :filesystem, :project_name, :bucket_name, :logger, keyword_init: true)
+        def copy_directory(src, dest, root = dest, tmpdir = nil)
+          own_tmpdir = tmpdir.nil?
+          if own_tmpdir
+            tmpdir = filesystem.tmpdir
+          end
+
+          begin
+            response = metis_client.list_folder(ListFolderRequest.new(project_name: project_name, bucket_name: bucket_name, folder_path: src))
+
+            response.files.all.each do |file|
+              logger&.info("Copying file #{file.file_path} (#{Etna::Formatting.as_size(file.size)})")
+              copy_file(bin_root_dir: root, tmpdir: tmpdir, dest: ::File.join(dest, file.file_name), url: file.download_url)
+            end
+
+            response.folders.all.each do |folder|
+              copy_directory(::File.join(src, folder.folder_name), ::File.join(dest, folder.folder_name), root, tmpdir)
+            end
+          ensure
+            filesystem.rm_rf(tmpdir) if own_tmpdir
+          end
+        end
+
+        def bin_file_name(etag:)
+          "bin/#{etag}"
+        end
+
+        def copy_file(bin_root_dir:, tmpdir:, dest:, url:, stub: false)
+          metadata = metis_client.file_metadata(url)
+          etag = metadata[:etag]
+          size = metadata[:size]
+
+          dest_bin_file = ::File.join(bin_root_dir, bin_file_name(etag: etag))
+          # Already materialized, continue
+          if filesystem.exist?(dest_bin_file)
+            return
+          end
+
+          tmp_file = ::File.join(tmpdir, etag)
+
+
+          upload_timings = []
+          upload_amount = 0
+          last_rate = 0.00001
+
+          filesystem.with_writeable(tmp_file, "w", size_hint: size) do |io|
+            if stub
+              io.write("(stub) #{size} bytes")
+            else
+              metis_client.download_file(url) do |chunk|
+                io.write(chunk)
+
+                upload_timings << [chunk.length, Time.now.to_f]
+                upload_amount += chunk.length
+
+                if upload_timings.length > 150
+                  s, _ = upload_timings.shift
+                  upload_amount -= s
+                end
+
+                _, start_time = upload_timings.first
+                _, end_time = upload_timings.last
+
+                if start_time == end_time
+                  next
+                end
+
+                rate = upload_amount / (end_time - start_time)
+
+                if rate / last_rate > 1.3 || rate / last_rate < 0.7
+                  logger&.info("Uploading #{Etna::Formatting.as_size(rate)} per second")
+
+                  if rate == 0
+                    last_rate = 0.0001
+                  else
+                    last_rate = rate
+                  end
+                end
+              end
+            end
+          end
+
+          filesystem.mkdir_p(::File.dirname(dest))
+          filesystem.mv(tmp_file, dest)
+
+          filesystem.mkdir_p(::File.dirname(dest_bin_file))
+          filesystem.with_writeable(dest_bin_file, 'w', size_hint: 0) do |io|
+            # empty file
+          end
+        end
+      end
+    end
+  end
+end

data/lib/etna/clients/polyphemus/models.rb

@@ -13,11 +13,11 @@ module Etna
         end
       end
 
-      class RedcapJobRequest < Struct.new(:model_names, :redcap_tokens, :commit, :project_name, keyword_init: true)
+      class RedcapJobRequest < Struct.new(:model_names, :redcap_tokens, :commit, :project_name, :record_names, keyword_init: true)
         include JsonSerializableStruct
 
         def initialize(**params)
-          super({model_names: 'all', commit: false}.update(params))
+          super({model_names: 'all', record_names: nil, commit: false}.update(params))
         end
 
         def to_json
@@ -26,7 +26,8 @@ module Etna
             job_params: {
               commit: commit,
               model_names: model_names,
-              redcap_tokens: redcap_tokens
+              redcap_tokens: redcap_tokens,
+              record_names: record_names
             }
           }.to_json
         end

data/lib/etna/filesystem.rb

@@ -1,8 +1,12 @@
+require 'yaml'
+require 'fileutils'
+require 'open3'
+
 module Etna
   # A class that encapsulates opening / reading file system entries that abstracts normal file access in order
   # to make stubbing, substituting, and testing easier.
   class Filesystem
-    def with_writeable(dest, opts = 'w', &block)
+    def with_writeable(dest, opts = 'w', size_hint: nil, &block)
       ::File.open(dest, opts, &block)
     end
 
@@ -11,29 +15,185 @@ module Etna
     end
 
     def mkdir_p(dir)
-      require 'fileutils'
       ::FileUtils.mkdir_p(dir)
     end
 
     def rm_rf(dir)
-      require 'fileutils'
-      FileUtils.rm_rf(dir)
+      ::FileUtils.rm_rf(dir)
     end
 
     def tmpdir
-      ::Dir.tmpdir
+      ::Dir.mktmpdir
     end
 
     def exist?(src)
       ::File.exist?(src)
     end
 
+    def mv(src, dest)
+      ::FileUtils.mv(src, dest)
+    end
+
     class EmptyIO < StringIO
       def write(*args)
         # Do nothing -- always leave empty
       end
     end
 
+    module WithPipeConsumer
+      def mkio(file, opts, size_hint: nil, &block)
+        rd, wd = IO.pipe
+
+        pid = spawn(*mkcommand(rd, wd, file, opts, size_hint: size_hint))
+        q = Queue.new
+
+        closer = Thread.new do
+          _, status = Process.wait2 pid
+          q << status
+        end
+
+        begin
+          if opts.include?('w')
+            rd.close
+            yield wd
+            wd.close
+          else
+            wd.close
+            yield rd
+            rd.close
+          end
+
+          closer.join
+        rescue => e
+          wd.close
+          rd.close
+          Process.kill("HUP", pid)
+          raise e
+        end
+
+        status = q.pop
+        raise IOError.new("Failed to run external process, got status code #{status}") unless status.success?
+      end
+    end
+
+    class AsperaCliFilesystem < Filesystem
+      include WithPipeConsumer
+
+      def initialize(ascli_bin:, ascp_bin:, host:, username:, password: nil, key_file: nil, port: 33001)
+        @ascli_bin = ascli_bin
+        @ascp_bin = ascp_bin
+        @username = username
+        @password = password
+        @key_file = key_file
+        @host = host
+        @port = port
+
+        @config_file = File.join(Dir.mktmpdir, "config.yml")
+        config = {}
+        config["config"] = {"version" => `#{ascli_bin} --version`.chomp}
+        config["default"] = {"server" => "clifilesystem"}
+        server_config = config["clifilesystem"] = {
+            "url" => "ssh://#{host}:#{port}",
+            "username" => username,
+            "ssh_options" => {append_all_supported_algorithms: true},
+        }
+
+        if password
+          server_config["password"] = password
+        elsif key_file
+          server_config["ssh_keys"] = key_file
+        else
+          raise "One of password or key_file must be provided"
+        end
+
+        ::File.open(@config_file, "w") do |file|
+          file.write(config.to_yaml)
+        end
+      end
+
+      def run_ascli_cmd(cmd, *opts)
+        output, status = Open3.capture2(@ascli_bin, "server", cmd, *opts, "--format=json", "--config=#{@config_file}")
+
+        if status.success?
+          return JSON.parse(output)
+        end
+
+        nil
+      end
+
+      def with_writeable(dest, opts = 'w', size_hint: nil, &block)
+        mkio(dest, opts, size_hint: size_hint, &block)
+      end
+
+      def with_readable(src, opts = 'r', &block)
+        mkio(src, opts, &block)
+      end
+
+      def mkdir_p(dir)
+        raise "Failed to mkdir #{dir}" unless run_ascli_cmd("mkdir", dir)
+      end
+
+      def rm_rf(dir)
+        raise "Failed to rm_rf #{dir}" unless run_ascli_cmd("rm", dir)
+      end
+
+      def tmpdir
+        tmpdir = "/Upload/Temp/#{SecureRandom.hex}"
+        mkdir_p(tmpdir)
+        tmpdir
+      end
+
+      def exist?(src)
+        !run_ascli_cmd("ls", src).nil?
+      end
+
+      def mv(src, dest)
+        raise "Failed to mv #{src} to #{dest}" unless run_ascli_cmd("mv", src, dest)
+      end
+
+      def mkcommand(rd, wd, file, opts, size_hint: nil)
+        env = {}
+        cmd = [env, @ascp_bin]
+
+        if @password
+          env['ASPERA_SCP_PASS'] = @password
+        else
+          cmd << "-i"
+          cmd << @key_file
+        end
+
+        cmd << "-P"
+        cmd << @port.to_s
+
+        remote_path = file
+        # https://download.asperasoft.com/download/docs/entsrv/3.9.1/es_admin_linux/webhelp/index.html#dita/stdio_2.html
+        local_path = "stdio://"
+        if size_hint
+          local_path += "/?#{size_hint}"
+        end
+
+        if opts.include?('r')
+          cmd << '--mode=recv'
+          cmd << "--host=#{@host}"
+          cmd << "--user=#{@username}"
+          cmd << remote_path
+          cmd << local_path
+
+          cmd << { out: wd }
+        elsif opts.include?('w')
+          cmd << '--mode=send'
+          cmd << "--host=#{@host}"
+          cmd << "--user=#{@username}"
+          cmd << local_path
+          cmd << remote_path
+
+          cmd << { in: rd }
+        end
+
+        cmd
+      end
+    end
+
     class Mock < Filesystem
       def initialize(&new_io)
         @files = {}
@@ -49,7 +209,7 @@ module Etna
         end
       end
 
-      def with_writeable(dest, opts = 'w', &block)
+      def with_writeable(dest, opts = 'w', size_hint: nil, &block)
         if @dirs.include?(dest)
           raise IOError.new("Path #{dest} is a directory")
         end
@@ -68,21 +228,36 @@ module Etna
         end
       end
 
+      def mv(src, dest)
+        if exist?(dest)
+          raise "#{dest} already exists, cannot move"
+        end
+
+        if @dirs.include?(src)
+          @dirs[dest] = @dirs.delete(src)
+        elsif @files.include?(src)
+          @files[dest] = @files.delete(src)
+        else
+          raise "#{src} does not exist, cannot move"
+        end
+      end
+
       def tmpdir
         require 'securerandom'
         "/tmp-#{SecureRandom::uuid}"
       end
 
       def with_readable(src, opts = 'r', &block)
-        if @dirs.include?(dest)
-          raise IOError.new("Path #{dest} is a directory")
+        if @dirs.include?(src)
+          raise IOError.new("Path #{src} is a directory")
         end
 
-        if !@files.include?(dest)
-          raise IOError.new("Path #{dest} does not exist")
+        if !@files.include?(src)
+          raise IOError.new("Path #{src} does not exist")
         end
 
-        yield (@files[dest] ||= mkio(src, opts))
+        @files[src].rewind
+        yield @files[src]
       end
 
       def exist?(src)