etna 0.1.26 → 0.1.27

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7075c5bf9637903a5aae31e83dbec99fc903945b362b5b454a3d31f95c03c6eb
-  data.tar.gz: 48e957e8286a613f7a1cafab2512976e625211782ae09a402053161af0f69b87
+  metadata.gz: 99facdcccd4282d230f93f3ccd779af9f5f58878c5a33e2be0209e95b715a88e
+  data.tar.gz: b56796ac6f84ec988a07275bdaf78d14d12409cf7c3b0a4d2b8b30c0009d8130
 SHA512:
-  metadata.gz: 850effed7485f90957cb3ee53ebb97abaa3fe9c32d9de9ff16c5d30585b61f34ccf00bb88c702ee9f1271c8a883526cebc36218904c311c276d6de51aca0d1ba
-  data.tar.gz: 1c90072a96341cf75b869d08f9e1cd1e36c2a616873048d83c6c1cd2ffe57850c618745c80ee1e16e63c2c8c4c090cbb8c99ab19ed1541ec06d8b43d29824d0a
+  metadata.gz: c4a505a48d40c6ed15a97810002809260d43f9420c9339d5a00c6c78957430751cfa609312e99d6c50346cb0ebad300cc7bb6fbba12fde602e43005981556eab
+  data.tar.gz: 3edfd8838451970ed00a470c6b843a0e4b166c31ed7bb67eb2a8a1ce42c217414e6c92776aa4d4db2e0fa0cc2c27f50e2b05b339a28f4893c9137ec71ff8e2ff

data/etna.completion

@@ -499,10 +499,10 @@ COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
 return
 fi
 shift
-all_flag_completion_names="$all_flag_completion_names --commit --models --record_names "
-string_flag_completion_names="$string_flag_completion_names --models --record_names "
+all_flag_completion_names="$all_flag_completion_names --commit --models --mode "
+string_flag_completion_names="$string_flag_completion_names --models --mode "
 declare _completions_for_models="__models__"
-declare _completions_for_record_names="__record_names__"
+declare _completions_for_mode="__mode__"
 while [[ "$#" != "0" ]]; do
 if [[ "$#" == "1" ]];  then
 all_completion_names=""

data/lib/commands.rb

@@ -351,16 +351,16 @@ class EtnaApp
 
         boolean_flags << '--commit'
         string_flags << '--models'
-        string_flags << '--record_names'
+        string_flags << '--mode'
 
-        def execute(project_name, redcap_tokens, models: "all", record_names: nil, commit: false)
+        def execute(project_name, redcap_tokens, models: "all", mode: nil, commit: false)
           raise "Must provide at least one REDCap token (comma-separated)." unless redcap_tokens.split(',').length > 0
 
           puts "NOTE: This is a **preview** of what the data loading will look like. Use the --commit flag to load records into Magma." unless commit
 
           polyphemus_client.job(Etna::Clients::Polyphemus::RedcapJobRequest.new(
             model_names: "all" == models ? "all" : models.split(','),
-            record_names: nil == record_names || "existing" == record_names ? record_names : record_names.split(','),
+            mode: mode,
             redcap_tokens: redcap_tokens.split(','),
             project_name: project_name,
             commit: commit

data/lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb

@@ -10,7 +10,7 @@ module Etna
           :metis_client, :magma_client, :project_name,
           :model_name, :model_filters, :model_attributes_mask,
           :filesystem, :logger, :stub_files,
-          keyword_init: true)
+          :skip_tmpdir, keyword_init: true)
 
         def initialize(**kwds)
           super(**({filesystem: Etna::Filesystem.new}.update(kwds)))
@@ -24,8 +24,8 @@ module Etna
           @model_walker ||= WalkModelTreeWorkflow.new(magma_crud: magma_crud, logger: logger)
         end
 
-        def materialize_all(dest = filesystem.tmpdir)
-          tmpdir = filesystem.tmpdir
+        def materialize_all(dest)
+          tmpdir = skip_tmpdir ? nil : filesystem.tmpdir
 
           begin
             model_walker.walk_from(
@@ -37,7 +37,7 @@ module Etna
               materialize_record(dest, tmpdir, template, document)
             end
           ensure
-            filesystem.rm_rf(tmpdir)
+            filesystem.rm_rf(tmpdir) unless skip_tmpdir
           end
         end
 
@@ -78,6 +78,7 @@ module Etna
           @sync_metis_data_workflow ||= Etna::Clients::Metis::SyncMetisDataWorkflow.new(
               metis_client: metis_client,
               logger: logger,
+              skip_tmpdir: skip_tmpdir,
               filesystem: filesystem)
         end
 

data/lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb

@@ -6,9 +6,10 @@ require 'tempfile'
 module Etna
   module Clients
     class Metis
-      class SyncMetisDataWorkflow < Struct.new(:metis_client, :filesystem, :project_name, :bucket_name, :logger, keyword_init: true)
+      class SyncMetisDataWorkflow < Struct.new(:metis_client, :filesystem, :project_name, :bucket_name,
+          :logger, :skip_tmpdir, keyword_init: true)
         def copy_directory(src, dest, root = dest, tmpdir = nil)
-          own_tmpdir = tmpdir.nil?
+          own_tmpdir = tmpdir.nil? && !skip_tmpdir
           if own_tmpdir
             tmpdir = filesystem.tmpdir
           end
@@ -33,6 +34,17 @@ module Etna
           "bin/#{etag}"
         end
 
+        def with_maybe_intermediate_tmp_dest(bin_file_name:, tmpdir:, dest_file_name:, &block)
+          filesystem.mkdir_p(::File.dirname(dest_file_name))
+          if tmpdir.nil?
+            yield dest_file_name
+          else
+            tmp_file = ::File.join(tmpdir, ::File.basename(bin_file_name))
+            yield tmp_file
+            filesystem.mv(tmp_file, dest_file_name)
+          end
+        end
+
         def copy_file(bin_root_dir:, tmpdir:, dest:, url:, stub: false)
           metadata = metis_client.file_metadata(url)
           etag = metadata[:etag]
@@ -44,56 +56,52 @@ module Etna
             return
           end
 
-          tmp_file = ::File.join(tmpdir, etag)
-
-
-          upload_timings = []
-          upload_amount = 0
-          last_rate = 0.00001
+          with_maybe_intermediate_tmp_dest(bin_file_name: dest_bin_file, tmpdir: tmpdir, dest_file_name: dest) do |tmp_file|
+            upload_timings = []
+            upload_amount = 0
+            last_rate = 0.00001
 
-          filesystem.with_writeable(tmp_file, "w", size_hint: size) do |io|
-            if stub
-              io.write("(stub) #{size} bytes")
-            else
-              metis_client.download_file(url) do |chunk|
-                io.write(chunk)
+            filesystem.with_writeable(tmp_file, "w", size_hint: size) do |io|
+              if stub
+                io.write("(stub) #{size} bytes")
+              else
+                metis_client.download_file(url) do |chunk|
+                  io.write(chunk)
 
-                upload_timings << [chunk.length, Time.now.to_f]
-                upload_amount += chunk.length
+                  upload_timings << [chunk.length, Time.now.to_f]
+                  upload_amount += chunk.length
 
-                if upload_timings.length > 150
-                  s, _ = upload_timings.shift
-                  upload_amount -= s
-                end
+                  if upload_timings.length > 150
+                    s, _ = upload_timings.shift
+                    upload_amount -= s
+                  end
 
-                _, start_time = upload_timings.first
-                _, end_time = upload_timings.last
+                  _, start_time = upload_timings.first
+                  _, end_time = upload_timings.last
 
-                if start_time == end_time
-                  next
-                end
+                  if start_time == end_time
+                    next
+                  end
 
-                rate = upload_amount / (end_time - start_time)
+                  rate = upload_amount / (end_time - start_time)
 
-                if rate / last_rate > 1.3 || rate / last_rate < 0.7
-                  logger&.info("Uploading #{Etna::Formatting.as_size(rate)} per second")
+                  if rate / last_rate > 1.3 || rate / last_rate < 0.7
+                    logger&.info("Uploading #{Etna::Formatting.as_size(rate)} per second")
 
-                  if rate == 0
-                    last_rate = 0.0001
-                  else
-                    last_rate = rate
+                    if rate == 0
+                      last_rate = 0.0001
+                    else
+                      last_rate = rate
+                    end
                   end
                 end
               end
             end
           end
 
-          filesystem.mkdir_p(::File.dirname(dest))
-          filesystem.mv(tmp_file, dest)
-
           filesystem.mkdir_p(::File.dirname(dest_bin_file))
           filesystem.with_writeable(dest_bin_file, 'w', size_hint: 0) do |io|
-            # empty file
+            # empty file marking that this etag has been moved, to save a future write.
           end
         end
       end

data/lib/etna/clients/polyphemus/models.rb

@@ -13,11 +13,11 @@ module Etna
         end
       end
 
-      class RedcapJobRequest < Struct.new(:model_names, :redcap_tokens, :commit, :project_name, :record_names, keyword_init: true)
+      class RedcapJobRequest < Struct.new(:model_names, :redcap_tokens, :commit, :project_name, :mode, keyword_init: true)
         include JsonSerializableStruct
 
         def initialize(**params)
-          super({model_names: 'all', record_names: nil, commit: false}.update(params))
+          super({model_names: 'all', mode: nil, commit: false}.update(params))
         end
 
         def to_json
@@ -27,7 +27,7 @@ module Etna
               commit: commit,
               model_names: model_names,
               redcap_tokens: redcap_tokens,
-              record_names: record_names
+              mode: mode
             }
           }.to_json
         end

data/lib/etna/filesystem.rb

@@ -194,6 +194,51 @@ module Etna
       end
     end
 
+    # Genentech's aspera deployment doesn't support modern commands, unfortunately...
+    class GeneAsperaCliFilesystem < AsperaCliFilesystem
+      def tmpdir
+        raise "tmpdir is not supported"
+      end
+
+      def rm_rf
+        raise "rm_rf is not supported"
+      end
+
+      def mkdir_p(dest)
+        # Pass through -- this file system creates containing directories by default, womp womp.
+      end
+
+      def mv
+        raise "mv is not supported"
+      end
+
+
+      def mkcommand(rd, wd, file, opts, size_hint: nil)
+        if opts.include?('w')
+          super.map do |e|
+            if e.instance_of?(String) && e.start_with?("stdio://")
+              "stdio-tar://"
+            elsif e == file
+              ::File.dirname(file)
+            else
+              e
+            end
+          end.insert(2, "-d")
+        else
+          super
+        end
+      end
+
+      def with_writeable(dest, opts = 'w', size_hint: nil, &block)
+        super do |io|
+          io.write("File: #{::File.basename(dest)}\n")
+          io.write("Size: #{size_hint}\n")
+          io.write("\n")
+          yield io
+        end
+      end
+    end
+
     class Mock < Filesystem
       def initialize(&new_io)
         @files = {}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: etna
 version: !ruby/object:Gem::Version
-  version: 0.1.26
+  version: 0.1.27
 platform: ruby
 authors:
 - Saurabh Asthana
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-12 00:00:00.000000000 Z
+date: 2021-01-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack