etna 0.1.39 → 0.1.43

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c21212f38bbad3cf757eb84a9e453428c9d0b6f195fd34eb81f98aeff2746490
-  data.tar.gz: 3581821244c087cbbfbac162f9a5c61911e06ab64923c603a144d36ca3bedff0
+  metadata.gz: 2e30f0d83fdf8f5781f723080ddd042d371f3131a415c85ec441b56178cef27c
+  data.tar.gz: a543a3464ca34e0d6daedddbf702cf067f7dfdcbdc972e4e9a9a626118d5fbbd
 SHA512:
-  metadata.gz: 3289530a2e874d2b79b0da66ea787e30b335b7a60626e4b67f1be9919067d4b79752db4e878ca8851dbce6c68a1e59a573bcbc60314bb7a35268cd37cc2eeaae
-  data.tar.gz: 2758348f253992e08fbf589d2b5444844becf69e179872c630804a7e8885568189168edd056c56beabe4ed5dc3e0905f46c51595896bd7bdd1255fbe4ece71ec
+  metadata.gz: d11b9f2882e7499decb4854c5c399bce24e31b14ff52c83006708bd4f83706e4a754d2c96838241b1885ae2ad4f2eb360a0f815b38f5009f1a75fb4fe7246275
+  data.tar.gz: 3447fa61b285fdbe78e5e0646b36d489f3fd2677cce2415ebe001f8b691ddc281b9514a12bd9424ed9023f0b58609482edba77156066377ab1840e06bf359a5a

data/etna.completion

@@ -84,7 +84,7 @@ arg_flag_completion_names="$arg_flag_completion_names  "
 multi_flags="$multi_flags  "
 while [[ "$#" != "0" ]]; do
 if [[ "$#" == "1" ]];  then
-all_completion_names="apply_template attributes copy_template help load_from_redcap"
+all_completion_names="apply_template attributes copy_template help load_from_redcap set_date_shift_root"
 all_completion_names="$all_completion_names $all_flag_completion_names"
 if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
 return
@@ -568,6 +568,54 @@ return
 fi
 done
 return
+elif [[ "$1" == "set_date_shift_root" ]]; then
+shift
+if [[ "$#" == "1" ]];  then
+all_completion_names="__project_name__"
+if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
+return
+fi
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+fi
+shift
+all_flag_completion_names="$all_flag_completion_names --date-shift-root --target-model "
+arg_flag_completion_names="$arg_flag_completion_names --target-model "
+multi_flags="$multi_flags  "
+declare _completions_for_target_model="__target_model__"
+while [[ "$#" != "0" ]]; do
+if [[ "$#" == "1" ]];  then
+all_completion_names=""
+all_completion_names="$all_completion_names $all_flag_completion_names"
+if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
+return
+fi
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+elif [[ -z "$(echo $all_flag_completion_names | xargs)" ]]; then
+return
+elif [[ "$all_flag_completion_names" =~ $1\  ]]; then
+if ! [[ "$multi_flags" =~ $1\  ]]; then
+all_flag_completion_names="${all_flag_completion_names//$1\ /}"
+fi
+a=$1
+shift
+if [[ "$arg_flag_completion_names" =~ $a\  ]]; then
+if [[ "$#" == "1" ]];  then
+a="${a//--/}"
+a="${a//-/_}"
+i="_completions_for_$a"
+all_completion_names="${!i}"
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+fi
+shift
+fi
+else
+return
+fi
+done
+return
 elif [[ -z "$(echo $all_flag_completion_names | xargs)" ]]; then
 return
 elif [[ "$all_flag_completion_names" =~ $1\  ]]; then
@@ -738,10 +786,11 @@ COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
 return
 elif [[ "$1" == "generate" ]]; then
 shift
-all_flag_completion_names="$all_flag_completion_names --task --project-name "
-arg_flag_completion_names="$arg_flag_completion_names --project-name "
+all_flag_completion_names="$all_flag_completion_names --task --project-name --email "
+arg_flag_completion_names="$arg_flag_completion_names --project-name --email "
 multi_flags="$multi_flags  "
 declare _completions_for_project_name="__project_name__"
+declare _completions_for_email="__email__"
 while [[ "$#" != "0" ]]; do
 if [[ "$#" == "1" ]];  then
 all_completion_names=""

data/lib/commands.rb

@@ -411,6 +411,25 @@ class EtnaApp
           end
         end
       end
+
+      class SetDateShiftRoot < Etna::Command
+        include WithEtnaClients
+        include WithLogger
+        include StrongConfirmation
+
+        boolean_flags << '--date-shift-root'
+        string_flags << '--target-model'
+
+        def execute(project_name, target_model: 'subject', date_shift_root: false)
+          magma_client.update_model(Etna::Clients::Magma::UpdateModelRequest.new(
+            actions:[Etna::Clients::Magma::SetDateShiftRootAction.new(
+              model_name: target_model,
+              date_shift_root: date_shift_root
+            )],
+            project_name: project_name,
+          ))
+        end
+      end
     end
   end
 

data/lib/etna/application.rb

@@ -53,6 +53,21 @@ module Etna::Application
   def configure(opts)
     @config = opts
 
+    # Apply environmental variables of the form "APP__x__y__z"
+    prefix = "#{self.class.name.upcase}__"
+    ENV.keys.select { |k| k.start_with?(prefix) }.each do |key|
+      path = key.split("__", -1)
+      path.shift # drop the first, just app name
+
+      target = @config
+      while path.length > 1
+        n = path.shift
+        target = (target[n.downcase.to_sym] ||= {})
+      end
+
+      target[path.last.downcase.to_sym] ||= ENV[key]
+    end
+
     if (rollbar_config = config(:rollbar)) && rollbar_config[:access_token]
       Rollbar.configure do |config|
         config.access_token = rollbar_config[:access_token]
@@ -60,17 +75,6 @@ module Etna::Application
     end
   end
 
-  # This will cause metrics to persist to a file.
-  # NOTE -- /tmp/metrics.bin should be a persistent mount when using this.
-  # You will still need to export metrics in the text format for the node_exporter on the host machine to
-  # export them to prometheus.  Ensure that the /tmp/metrics.bin is on a named volume or a bind mount, either is fine.
-  def enable_job_metrics!
-    require 'prometheus'
-    Prometheus::Client.config.data_store = Prometheus::Client::DataStores::DirectFileStore.new({
-      dir: "/tmp/metrics.bin"
-    })
-  end
-
   def setup_yabeda
     application = self.id
     Yabeda.configure do
@@ -192,13 +196,14 @@ module Etna::Application
       status = 'failed'
       raise
     ensure
-      if Yabeda.configured?
+      if defined?(Yabeda) && Yabeda.configured?
         tags = { command: cmd.class.name, status: status, application: application }
         dur = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
 
         Yabeda.etna.last_command_completion.set(tags, Time.now.to_i)
         Yabeda.etna.command_runtime.measure(tags, dur)
-        write_job_metrics("run_command")
+
+        write_job_metrics("run_command.#{cmd.class.name}")
       end
     end
   end

data/lib/etna/censor.rb

@@ -0,0 +1,36 @@
+module Etna
+  class Censor
+    def initialize(log_redact_keys)
+      @log_redact_keys = log_redact_keys
+    end
+
+    def redact_keys
+      @log_redact_keys
+    end
+
+    def redact(key, value)
+      # Redact any values for the supplied key values, so they
+      #   don't appear in the logs.
+      return compact(value) unless redact_keys
+
+      if redact_keys.include?(key)
+        return "*"
+      elsif value.is_a?(Hash)
+        redacted_value = value.map do |value_key, value_value|
+          [value_key, redact(value_key, value_value)]
+        end.to_h
+        return redacted_value
+      end
+
+      return compact(value)
+    end
+
+    private
+
+    def compact(value)
+      value = value.to_s
+      value = value[0..500] + "..." + value[-100..-1] if value.length > 600
+      value
+    end
+  end
+end

data/lib/etna/clients/magma/models.rb

@@ -58,11 +58,19 @@ module Etna
         end
       end
 
-      class AddModelAction < Struct.new(:action_name, :model_name, :parent_model_name, :parent_link_type, :identifier, keyword_init: true)
+      class AddModelAction < Struct.new(:action_name, :model_name, :parent_model_name, :parent_link_type, :identifier, :date_shift_root, keyword_init: true)
         include JsonSerializableStruct
 
         def initialize(**args)
-          super({action_name: 'add_model'}.update(args))
+          super({action_name: 'add_model', date_shift_root: false}.update(args))
+        end
+      end
+
+      class SetDateShiftRootAction < Struct.new(:action_name, :model_name, :date_shift_root, keyword_init: true)
+        include JsonSerializableStruct
+
+        def initialize(**args)
+          super({action_name: 'set_date_shift_root'}.update(args))
         end
       end
 
@@ -620,6 +628,7 @@ module Etna
         MATRIX = AttributeType.new("matrix")
         PARENT = AttributeType.new("parent")
         TABLE = AttributeType.new("table")
+        SHIFTED_DATE_TIME = AttributeType.new("shifted_date_time")
       end
 
       class ParentLinkType < String

data/lib/etna/clients/metis/client.rb

@@ -28,6 +28,21 @@ module Etna
           @etna_client.folder_list(list_folder_request.to_h))
       end
 
+      def list_folder_by_id(list_folder_by_id_request = ListFolderByIdRequest.new)
+        FoldersAndFilesResponse.new(
+          @etna_client.folder_list_by_id(list_folder_by_id_request.to_h))
+      end
+
+      def touch_folder(touch_folder_request = TouchFolderRequest.new)
+        FoldersResponse.new(
+          @etna_client.folder_touch(touch_folder_request.to_h))
+      end
+
+      def touch_file(touch_file_request = TouchFileRequest.new)
+        FilesResponse.new(
+          @etna_client.file_touch(touch_file_request.to_h))
+      end
+
       def ensure_parent_folder_exists(project_name:, bucket_name:, path:)
         create_folder_request = CreateFolderRequest.new(
           project_name: project_name,
@@ -181,7 +196,16 @@ module Etna
 
         return if found_folders.length == 0
 
-        found_folders.each { |folder|
+        rename_folders(
+          project_name: project_name,
+          source_bucket: source_bucket,
+          source_folders: found_folders,
+          dest_bucket: dest_bucket
+        )
+      end
+
+      def rename_folders(project_name:, source_bucket:, source_folders:, dest_bucket:)
+        source_folders.each { |folder|
           # If the destination folder already exists, we need to copy the files
           #   over to it and delete the source folder.
           create_folder_request = CreateFolderRequest.new(
@@ -210,34 +234,30 @@ module Etna
         }
       end
 
-      def recursively_rename_folder(project_name:, source_bucket:, dest_bucket:, folder:)
-        folder_contents = list_folder(
-          Etna::Clients::Metis::ListFolderRequest.new(
-            project_name: project_name,
-            bucket_name: source_bucket,
-            folder_path: folder.folder_path
-        ))
+      def resolve_conflicts_and_verify_rename(project_name:, source_bucket:, dest_bucket:, folder:, file:)
+        parent_folder = ::File.dirname(file.file_path)
 
-        folder_contents.folders.all.each do |sub_folder|
-          recursively_rename_folder(
-            project_name: project_name,
-            source_bucket: source_bucket,
-            dest_bucket: dest_bucket,
-            folder: sub_folder
-          )
-        end
+        should_rename_original = true
 
-        folder_contents.files.all.each do |file|
+        # If the destination folder already exists, check to see if
+        #   the file also exists, otherwise we risk a
+        #   rename conflict.
+        create_folder_request = CreateFolderRequest.new(
+          project_name: project_name,
+          bucket_name: dest_bucket,
+          folder_path: parent_folder
+        )
+
+        if folder_exists?(create_folder_request)
           # If file exists in destination, delete the older file.
           list_dest_folder_request = Etna::Clients::Metis::ListFolderRequest.new(
             bucket_name: dest_bucket,
             project_name: project_name,
-            folder_path: ::File.dirname(file.file_path)
+            folder_path: parent_folder
           )
 
           dest_file = list_folder(list_dest_folder_request).files.all.find { |f| f.file_name == file.file_name }
 
-          should_rename = true
           if (dest_file && file.updated_at <= dest_file.updated_at)
             # Delete source file if it's out of date
             delete_file(Etna::Clients::Metis::DeleteFileRequest.new(
@@ -246,7 +266,7 @@ module Etna
               file_path: file.file_path,
             ))
 
-            should_rename = false
+            should_rename_original = false
           elsif (dest_file && file.updated_at > dest_file.updated_at)
             # Delete dest file if it's out of date
             delete_file(Etna::Clients::Metis::DeleteFileRequest.new(
@@ -255,17 +275,44 @@ module Etna
               file_path: dest_file.file_path,
             ))
           end
+        end
 
-          if should_rename
-            rename_file(Etna::Clients::Metis::RenameFileRequest.new(
-              bucket_name: source_bucket,
-              project_name: project_name,
-              file_path: file.file_path,
-              new_bucket_name: dest_bucket,
-              new_file_path: file.file_path,
-              create_parent: true)
-            )
-          end
+        should_rename_original
+      end
+
+      def recursively_rename_folder(project_name:, source_bucket:, dest_bucket:, folder:)
+        folder_contents = list_folder(
+          Etna::Clients::Metis::ListFolderRequest.new(
+            project_name: project_name,
+            bucket_name: source_bucket,
+            folder_path: folder.folder_path
+        ))
+
+        folder_contents.folders.all.each do |sub_folder|
+          recursively_rename_folder(
+            project_name: project_name,
+            source_bucket: source_bucket,
+            dest_bucket: dest_bucket,
+            folder: sub_folder
+          )
+        end
+
+        folder_contents.files.all.each do |file|
+          should_rename = resolve_conflicts_and_verify_rename(
+            project_name: project_name,
+            source_bucket: source_bucket,
+            dest_bucket: dest_bucket,
+            folder: folder,
+            file: file)
+
+          rename_file(Etna::Clients::Metis::RenameFileRequest.new(
+            bucket_name: source_bucket,
+            project_name: project_name,
+            file_path: file.file_path,
+            new_bucket_name: dest_bucket,
+            new_file_path: file.file_path,
+            create_parent: true)
+          ) if should_rename
         end
 
         # Now delete the source folder

data/lib/etna/clients/metis/models.rb

@@ -65,6 +65,51 @@ module Etna
         end
       end
 
+      class ListFolderByIdRequest < Struct.new(:project_name, :bucket_name, :folder_id, keyword_init: true)
+        include JsonSerializableStruct
+
+        def initialize(**params)
+          super({}.update(params))
+        end
+
+        def to_h
+          # The :project_name comes in from Polyphemus as a symbol value,
+          #   we need to make sure it's a string because it's going
+          #   in the URL.
+          super().compact.transform_values(&:to_s)
+        end
+      end
+
+      class TouchFolderRequest < Struct.new(:project_name, :bucket_name, :folder_path, keyword_init: true)
+        include JsonSerializableStruct
+
+        def initialize(**params)
+          super({}.update(params))
+        end
+
+        def to_h
+          # The :project_name comes in from Polyphemus as a symbol value,
+          #   we need to make sure it's a string because it's going
+          #   in the URL.
+          super().compact.transform_values(&:to_s)
+        end
+      end
+
+      class TouchFileRequest < Struct.new(:project_name, :bucket_name, :file_path, keyword_init: true)
+        include JsonSerializableStruct
+
+        def initialize(**params)
+          super({}.update(params))
+        end
+
+        def to_h
+          # The :project_name comes in from Polyphemus as a symbol value,
+          #   we need to make sure it's a string because it's going
+          #   in the URL.
+          super().compact.transform_values(&:to_s)
+        end
+      end
+
       class CreateFolderRequest < Struct.new(:project_name, :bucket_name, :folder_path, keyword_init: true)
         include JsonSerializableStruct
 
@@ -110,11 +155,11 @@ module Etna
         end
       end
 
-      class FindRequest < Struct.new(:project_name, :bucket_name, :limit, :offset, :params, keyword_init: true)
+      class FindRequest < Struct.new(:project_name, :bucket_name, :limit, :offset, :params, :hide_paths, keyword_init: true)
         include JsonSerializableStruct
 
         def initialize(**args)
-          super({params: []}.update(args))
+          super({params: [], hide_paths: false}.update(args))
         end
 
         def add_param(param)
@@ -126,6 +171,17 @@ module Etna
           #   easier to do from a JSON string
           JSON.parse(to_json, :symbolize_names => true)
         end
+
+        def clone
+          FindRequest.new(
+            project_name: self.project_name,
+            bucket_name: self.bucket_name,
+            limit: self.limit,
+            offset: self.offset,
+            params: self.params.dup,
+            hide_paths: self.hide_paths
+          )
+        end
       end
 
       class FindParam < Struct.new(:attribute, :predicate, :value, :type, keyword_init: true)
@@ -225,6 +281,13 @@ module Etna
           @raw = raw
         end
 
+        def with_containing_folder(folder)
+          folder_path = folder.is_a?(Folder) ? folder.folder_path : folder
+          File.new({}.update(self.raw).update({
+            file_path: ::File.join(folder_path, self.file_name)
+          }))
+        end
+
         def file_path
           raw[:file_path]
         end
@@ -259,6 +322,14 @@ module Etna
         def size
           raw[:size]
         end
+
+        def file_hash
+          raw[:file_hash]
+        end
+
+        def folder_id
+          raw[:folder_id]
+        end
       end
 
       class Folder
@@ -279,6 +350,19 @@ module Etna
         def bucket_name
           raw[:bucket_name]
         end
+
+        def project_name
+          raw[:project_name]
+        end
+
+        def updated_at
+          time = raw[:updated_at]
+          time.nil? ? nil : Time.parse(time)
+        end
+
+        def id
+          raw[:id]
+        end
       end
 
       class AuthorizeUploadRequest < Struct.new(:project_name, :bucket_name, :file_path, keyword_init: true)

data/lib/etna/clients/metis/workflows/ingest_metis_data_workflow.rb

@@ -9,20 +9,24 @@ module Etna
         # Since we are doing manual triage of files,
         #   do not automatically copy directory trees.
         # srcs must be a list of full paths to files.
-        def copy_files(srcs)
+        def copy_files(srcs, &block)
           srcs.each do |src|
-            next unless ingest_filesystem.exist?(src)
+            if !ingest_filesystem.exist?(src)
+              logger&.warn("#{src} does not exist on source filesystem. Skipping.")
+              next
+            end
 
             logger&.info("Copying file #{src} (#{Etna::Formatting.as_size(ingest_filesystem.stat(src).size)})")
 
             # For ingestion triage, just copy over the exact path + filename.
-            copy_file(dest: src, src: src)
+            copy_file(dest: src, src: src, &block)
           end
         end
 
-        def copy_file(dest:, src:)
-          ingest_filesystem.with_readable(src, "r") do |file|
-            metis_filesystem.do_streaming_upload(file, dest, file.size)
+        def copy_file(dest:, src:, &block)
+          ingest_filesystem.with_readable(src, "r") do |io|
+            metis_filesystem.do_streaming_upload(io, dest, ingest_filesystem.stat(src).size)
+            yield src if block_given?
           end
         end
       end

data/lib/etna/clients/metis/workflows/metis_upload_workflow.rb

@@ -110,8 +110,6 @@ module Etna
 
           def initialize(source_file: nil, next_blob_size: nil, current_byte_position: nil)
             self.source_file = source_file
-            self.next_blob_size = next_blob_size
-            self.current_byte_position = current_byte_position
             self.next_blob_size = [file_size, INITIAL_BLOB_SIZE].min
             self.current_byte_position = 0
           end

data/lib/etna/controller.rb

@@ -42,7 +42,10 @@ module Etna
 
       [501, {}, ['This controller is not implemented.']]
     rescue Exception => e
-      handle_error(e)
+      error = e
+    ensure
+      log_request
+      return handle_error(error) if error
     end
 
     def require_params(*params)
@@ -82,8 +85,36 @@ module Etna
       @response.finish
     end
 
+    def config_hosts
+      [:janus, :magma, :timur, :metis, :vulcan, :polyphemus].map do |host|
+        [ :"#{host}_host", @server.send(:application).config(host)&.dig(:host) ]
+      end.to_h.compact
+    end
+
     private
 
+    def redact_keys
+      @request.env['etna.redact_keys']
+    end
+
+    def add_redact_keys(new_redact_keys=[])
+      @request.env['etna.redact_keys'] = (@request.env['etna.redact_keys'] || []).concat(new_redact_keys)
+    end
+
+    def log_request
+      censor = Etna::Censor.new(redact_keys)
+
+      redacted_params = @params.map do |key,value|
+        [ key, censor.redact(key, value) ]
+      end.to_h
+
+      log("User #{@user ? @user.email : :unknown} calling #{controller_name}##{@action} with params #{redacted_params}")
+    end
+
+    def controller_name
+      self.class.name.sub("Kernel::", "").sub("Controller", "").downcase
+    end
+
     def success(msg, content_type='text/plain')
       @response['Content-Type'] = content_type
       @response.write(msg)

data/lib/etna/filesystem.rb

@@ -3,8 +3,7 @@ require 'fileutils'
 require 'open3'
 require 'securerandom'
 require 'concurrent-ruby'
-require 'net/sftp'
-require 'net/ssh'
+require 'curb'
 
 module Etna
   # A class that encapsulates opening / reading file system entries that abstracts normal file access in order
@@ -377,55 +376,116 @@ module Etna
     end
 
     class SftpFilesystem < Filesystem
+      include WithPipeConsumer
+
+      class SftpFile
+        attr_reader :size, :name
+
+        def initialize(metadata)
+          @metadata_parts = metadata.split(" ")
+          @size = @metadata_parts[4].to_i
+          @perms = @metadata_parts.first
+          @name = @metadata_parts[8]
+        end
+      end
+
       def initialize(host:, username:, password: nil, port: 22, **args)
         @username = username
         @password = password
         @host = host
         @port = port
+
+        @dir_listings = {}
+      end
+
+      def url(src)
+        "sftp://#{@host}/#{src}"
       end
 
-      def ssh
-        @ssh ||= Net::SSH.start(@host, @username, password: @password)
+      def authn
+        "#{@username}:#{@password}"
       end
 
-      def sftp
-        @sftp ||= begin
-          conn = Net::SFTP::Session.new(ssh)
-          conn.loop { conn.opening? }
+      def curl_cmd(path, opts=[])
+        connection = Curl::Easy.new(url(path))
+        connection.http_auth_types = :basic
+        connection.username = @username
+        connection.password = @password
 
-          conn
+        connection
+      end
+
+      def sftp_file_from_path(src)
+        file = ls(::File.dirname(src)).split("\n").map do |listing|
+          SftpFile.new(listing)
+        end.select do |file|
+          file.name == ::File.basename(src)
         end
+
+        raise "#{src} not found" if file.empty?
+
+        file.first
       end
 
-      def with_readable(src, opts = 'r', &block)
-        sftp.file.open(src, opts, &block)
+      def mkcommand(rd, wd, file, opts, size_hint: nil)
+        env = {}
+        cmd = [env, "curl"]
+
+        cmd << "-u"
+        cmd << authn
+        cmd << "-o"
+        cmd << "-"
+        cmd << "-N"
+        cmd << url(file)
+
+        if opts.include?('r')
+          cmd << {out: wd}
+        end
+
+        cmd
       end
 
-      def ls(dir)
-        sftp.dir.entries(dir)
+      def with_readable(src, opts = 'r', &block)
+        raise "#{src} does not exist" unless exist?(src)
+
+        sftp_file = sftp_file_from_path(src)
+
+        mkio(src, opts, size_hint: sftp_file.size, &block)
       end
 
       def exist?(src)
-        begin
-          sftp.file.open(src)
-        rescue Net::SFTP::StatusException
-          return false
-        end
-        return true
+        files = ls(::File.dirname(src))
+        files.include?(::File.basename(src))
+      end
+
+      def ls(dir)
+        dir = dir + "/" unless "/" == dir[-1]  # Trailing / makes curl list directory
+
+        return @dir_listings[dir] if @dir_listings.has_key?(dir)
+
+        listing = ''
+        connection = curl_cmd(dir)
+        connection.on_body { |data| listing << data; data.size }
+        connection.perform
+
+        @dir_listings[dir] = listing
+
+        listing
       end
 
       def stat(src)
-        sftp.file.open(src).stat
+        sftp_file_from_path(src)
       end
     end
 
     class Mock < Filesystem
       class MockStat
-        def initialize
+        def initialize(io)
+          @io = io
         end
 
         def size
-          0
+          @io.respond_to?(:length) ? @io.length : 0
         end
       end
 
@@ -499,7 +559,7 @@ module Etna
       end
 
       def stat(src)
-        @files[src].respond_to?(:stat) ? @files[src].stat : MockStat.new
+        @files[src].respond_to?(:stat) ? @files[src].stat : MockStat.new(@files[src])
       end
     end
   end

data/lib/etna/remote.rb

@@ -0,0 +1,38 @@
+require "net/ssh"
+
+module Etna
+  class RemoteSSH
+    class RemoteSSHError < Exception
+    end
+
+    def initialize(host:, username:, password: nil, port: 22, root:, **args)
+      @username = username
+      @password = password
+      @host = host
+      @port = port
+      @root = root
+    end
+
+    def ssh
+      @ssh ||= Net::SSH.start(@host, @username, password: @password)
+    end
+
+    def mkdir_p(dir)
+      output = ssh.exec!("mkdir -p #{dir}")
+
+      raise RemoteSSHError.new("Unable to mkdir -p, #{output}") unless 0 == output.exitstatus
+    end
+
+    def lftp_get(username:, password:, host:, remote_filename:, &block)
+      full_local_path = ::File.join(@root, host, remote_filename)
+      full_local_dir = ::File.dirname(full_local_path)
+      mkdir_p(full_local_dir)
+
+      cmd = "lftp sftp://#{username}:#{password}@#{host}  -e \"get #{remote_filename} -o #{full_local_path}; bye\""
+
+      output = ssh.exec!(cmd)
+      raise RemoteSSHError.new("LFTP get failure: #{output}") unless 0 == output.exitstatus
+      yield remote_filename if block_given?
+    end
+  end
+end

data/lib/etna/route.rb

@@ -1,5 +1,6 @@
 require 'digest'
 require 'date'
+require_relative "./censor"
 
 module Etna
   class Route
@@ -13,6 +14,7 @@ module Etna
       @route = route.gsub(/\A(?=[^\/])/, '/')
       @block = block
       @match_ext = options[:match_ext]
+      @log_redact_keys = options[:log_redact_keys]
     end
 
     def to_hash
@@ -124,6 +126,8 @@ module Etna
         return [ 403, { 'Content-Type' => 'application/json' }, [ { error: 'You are forbidden from performing this action.' }.to_json ] ]
       end
 
+      request.env['etna.redact_keys'] = @log_redact_keys
+
       if @action
         controller, action = @action.split('#')
         controller_class = Kernel.const_get(
@@ -132,11 +136,6 @@ module Etna
         logger = request.env['etna.logger']
         user = request.env['etna.user']
 
-        params = request.env['rack.request.params'].map do |key,value|
-          [ key, redact(key, value) ]
-        end.to_h
-
-        logger.warn("User #{user ? user.email : :unknown} calling #{controller}##{action} with params #{params}")
         return controller_class.new(request, action).response
       elsif @block
         application = Etna::Application.find(app.class).class
@@ -161,35 +160,6 @@ module Etna
       @application ||= Etna::Application.instance
     end
 
-    def compact(value)
-      value = value.to_s
-      value = value[0..500] + "..." + value[-100..-1] if value.length > 600
-      value
-    end
-
-    def redact_keys
-      @redact_keys ||= application.config(:log_redact_keys).split(",").map do |key|
-        key.to_sym
-      end
-    end
-
-    def redact(key, value)
-      # From configuration, redact any values for the supplied key values, so they
-      #   don't appear in the logs.
-      return compact(value) unless application.config(:log_redact_keys)
-
-      if value.is_a?(Hash)
-        redacted_value = value.map do |value_key, value_value|
-          [ value_key, redact(value_key, value_value) ]
-        end.to_h
-        return redacted_value
-      elsif redact_keys.include?(key)
-        return "*"
-      end
-
-      return compact(value)
-    end
-
     def authorized?(request)
       # If there is no @auth requirement, they are ok - this doesn't preclude
       # them being rejected in the controller response

data/lib/etna/synchronize_db.rb

@@ -0,0 +1,14 @@
+module Etna
+  class SynchronizeDb
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      # Do a coarse checkout of the connection
+      Etna::Application.instance.db.synchronize do
+        @app.call(env)
+      end
+    end
+  end
+end

data/lib/etna.rb

@@ -22,6 +22,8 @@ require_relative './etna/filesystem'
 require_relative './etna/formatting'
 require_relative './etna/cwl'
 require_relative './etna/metrics'
+require_relative './etna/remote'
+require_relative './etna/synchronize_db'
 
 class EtnaApp
   include Etna::Application

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: etna
 version: !ruby/object:Gem::Version
-  version: 0.1.39
+  version: 0.1.43
 platform: ruby
 authors:
 - Saurabh Asthana
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-07-22 00:00:00.000000000 Z
+date: 2021-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -95,19 +95,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: net-sftp
+  name: curb
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: net-ssh
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: See summary
 email: Saurabh.Asthana@ucsf.edu
 executables:
@@ -124,6 +138,7 @@ files:
 - lib/etna.rb
 - lib/etna/application.rb
 - lib/etna/auth.rb
+- lib/etna/censor.rb
 - lib/etna/client.rb
 - lib/etna/clients.rb
 - lib/etna/clients/base_client.rb
@@ -186,6 +201,7 @@ files:
 - lib/etna/metrics.rb
 - lib/etna/multipart_serializable_nested_hash.rb
 - lib/etna/parse_body.rb
+- lib/etna/remote.rb
 - lib/etna/route.rb
 - lib/etna/server.rb
 - lib/etna/sign_service.rb
@@ -193,6 +209,7 @@ files:
 - lib/etna/spec/auth.rb
 - lib/etna/spec/vcr.rb
 - lib/etna/symbolize_params.rb
+- lib/etna/synchronize_db.rb
 - lib/etna/templates/attribute_actions_template.json
 - lib/etna/test_auth.rb
 - lib/etna/user.rb