etna 0.1.41 → 0.1.42

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 44b6c9fde1d01fab95038052180531fafd4a120e77ffbd12a8ac14e28122cd05
-  data.tar.gz: e1cb315eaeb4d35efc9e7f181e0cad767833b98f7ce3f26386e040c758c8483c
+  metadata.gz: 0d599cb16d9abc7427f3fbd3760fe293a039f355f4ebd94243757d9d434dbe27
+  data.tar.gz: '0834c3692bdd1db0008272b478f854b5da5b31aea19e0f42435e12f1dcee333d'
 SHA512:
-  metadata.gz: 3129a01c4181a772ab6752ed319396485e06f774b8091bff1ed9cbf6543a34820e0cc0706fd313ce5b3fb848a25f347de17bc376dc66b36458975d6298f48608
-  data.tar.gz: 82c57cff18c826a0a2664a63e0a531df57c75af30ceb3408d1d8ec1b9cb942fd25bd11253e6d880db6973df4cd60ade0764fac798a543af1e7601bfcc21abc45
+  metadata.gz: 8a3c1e354ccc5af8de5fee06bbc629d43b170d51bb55eee76d1d3fc3157a4b4e6d1bfab52830d9003d26944cd54cc4a0405c472752b5cd1ea36b8f3216f80e13
+  data.tar.gz: 5004d00104765e18193116ee76574a27db03fc40ad5d2ec6fea7731ab5081dd1c7589081fcb98b3cc8be1c8645d2659548df2983e9b6506e6914739351ee26a8

data/etna.completion

@@ -84,7 +84,7 @@ arg_flag_completion_names="$arg_flag_completion_names  "
 multi_flags="$multi_flags  "
 while [[ "$#" != "0" ]]; do
 if [[ "$#" == "1" ]];  then
-all_completion_names="apply_template attributes copy_template help load_from_redcap"
+all_completion_names="apply_template attributes copy_template help load_from_redcap set_date_shift_root"
 all_completion_names="$all_completion_names $all_flag_completion_names"
 if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
 return
@@ -568,6 +568,54 @@ return
 fi
 done
 return
+elif [[ "$1" == "set_date_shift_root" ]]; then
+shift
+if [[ "$#" == "1" ]];  then
+all_completion_names="__project_name__"
+if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
+return
+fi
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+fi
+shift
+all_flag_completion_names="$all_flag_completion_names --date-shift-root --target-model "
+arg_flag_completion_names="$arg_flag_completion_names --target-model "
+multi_flags="$multi_flags  "
+declare _completions_for_target_model="__target_model__"
+while [[ "$#" != "0" ]]; do
+if [[ "$#" == "1" ]];  then
+all_completion_names=""
+all_completion_names="$all_completion_names $all_flag_completion_names"
+if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
+return
+fi
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+elif [[ -z "$(echo $all_flag_completion_names | xargs)" ]]; then
+return
+elif [[ "$all_flag_completion_names" =~ $1\  ]]; then
+if ! [[ "$multi_flags" =~ $1\  ]]; then
+all_flag_completion_names="${all_flag_completion_names//$1\ /}"
+fi
+a=$1
+shift
+if [[ "$arg_flag_completion_names" =~ $a\  ]]; then
+if [[ "$#" == "1" ]];  then
+a="${a//--/}"
+a="${a//-/_}"
+i="_completions_for_$a"
+all_completion_names="${!i}"
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+fi
+shift
+fi
+else
+return
+fi
+done
+return
 elif [[ -z "$(echo $all_flag_completion_names | xargs)" ]]; then
 return
 elif [[ "$all_flag_completion_names" =~ $1\  ]]; then
@@ -738,10 +786,11 @@ COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
 return
 elif [[ "$1" == "generate" ]]; then
 shift
-all_flag_completion_names="$all_flag_completion_names --task --project-name "
-arg_flag_completion_names="$arg_flag_completion_names --project-name "
+all_flag_completion_names="$all_flag_completion_names --task --project-name --email "
+arg_flag_completion_names="$arg_flag_completion_names --project-name --email "
 multi_flags="$multi_flags  "
 declare _completions_for_project_name="__project_name__"
+declare _completions_for_email="__email__"
 while [[ "$#" != "0" ]]; do
 if [[ "$#" == "1" ]];  then
 all_completion_names=""

data/lib/commands.rb

@@ -411,6 +411,25 @@ class EtnaApp
           end
         end
       end
+
+      class SetDateShiftRoot < Etna::Command
+        include WithEtnaClients
+        include WithLogger
+        include StrongConfirmation
+
+        boolean_flags << '--date-shift-root'
+        string_flags << '--target-model'
+
+        def execute(project_name, target_model: 'subject', date_shift_root: false)
+          magma_client.update_model(Etna::Clients::Magma::UpdateModelRequest.new(
+            actions:[Etna::Clients::Magma::SetDateShiftRootAction.new(
+              model_name: target_model,
+              date_shift_root: date_shift_root
+            )],
+            project_name: project_name,
+          ))
+        end
+      end
     end
   end
 

data/lib/etna/application.rb

@@ -53,6 +53,21 @@ module Etna::Application
   def configure(opts)
     @config = opts
 
+    # Apply environmental variables of the form "APP__x__y__z"
+    prefix = "#{self.class.name.upcase}__"
+    ENV.keys.select { |k| k.start_with?(prefix) }.each do |key|
+      path = key.split("__", -1)
+      path.shift # drop the first, just app name
+
+      target = @config
+      while path.length > 1
+        n = path.shift
+        target = (target[n.downcase.to_sym] ||= {})
+      end
+
+      target[path.last.downcase.to_sym] ||= ENV[key]
+    end
+
     if (rollbar_config = config(:rollbar)) && rollbar_config[:access_token]
       Rollbar.configure do |config|
         config.access_token = rollbar_config[:access_token]

data/lib/etna/censor.rb

@@ -0,0 +1,36 @@
+module Etna
+  class Censor
+    def initialize(log_redact_keys)
+      @log_redact_keys = log_redact_keys
+    end
+
+    def redact_keys
+      @log_redact_keys
+    end
+
+    def redact(key, value)
+      # Redact any values for the supplied key values, so they
+      #   don't appear in the logs.
+      return compact(value) unless redact_keys
+
+      if redact_keys.include?(key)
+        return "*"
+      elsif value.is_a?(Hash)
+        redacted_value = value.map do |value_key, value_value|
+          [value_key, redact(value_key, value_value)]
+        end.to_h
+        return redacted_value
+      end
+
+      return compact(value)
+    end
+
+    private
+
+    def compact(value)
+      value = value.to_s
+      value = value[0..500] + "..." + value[-100..-1] if value.length > 600
+      value
+    end
+  end
+end

data/lib/etna/clients/magma/models.rb

@@ -58,11 +58,19 @@ module Etna
         end
       end
 
-      class AddModelAction < Struct.new(:action_name, :model_name, :parent_model_name, :parent_link_type, :identifier, keyword_init: true)
+      class AddModelAction < Struct.new(:action_name, :model_name, :parent_model_name, :parent_link_type, :identifier, :date_shift_root, keyword_init: true)
         include JsonSerializableStruct
 
         def initialize(**args)
-          super({action_name: 'add_model'}.update(args))
+          super({action_name: 'add_model', date_shift_root: false}.update(args))
+        end
+      end
+
+      class SetDateShiftRootAction < Struct.new(:action_name, :model_name, :date_shift_root, keyword_init: true)
+        include JsonSerializableStruct
+
+        def initialize(**args)
+          super({action_name: 'set_date_shift_root'}.update(args))
         end
       end
 

data/lib/etna/clients/metis/client.rb

@@ -28,6 +28,21 @@ module Etna
           @etna_client.folder_list(list_folder_request.to_h))
       end
 
+      def list_folder_by_id(list_folder_by_id_request = ListFolderByIdRequest.new)
+        FoldersAndFilesResponse.new(
+          @etna_client.folder_list_by_id(list_folder_by_id_request.to_h))
+      end
+
+      def touch_folder(touch_folder_request = TouchFolderRequest.new)
+        FoldersResponse.new(
+          @etna_client.folder_touch(touch_folder_request.to_h))
+      end
+
+      def touch_file(touch_file_request = TouchFileRequest.new)
+        FilesResponse.new(
+          @etna_client.file_touch(touch_file_request.to_h))
+      end
+
       def ensure_parent_folder_exists(project_name:, bucket_name:, path:)
         create_folder_request = CreateFolderRequest.new(
           project_name: project_name,
@@ -181,7 +196,16 @@ module Etna
 
         return if found_folders.length == 0
 
-        found_folders.each { |folder|
+        rename_folders(
+          project_name: project_name,
+          source_bucket: source_bucket,
+          source_folders: found_folders,
+          dest_bucket: dest_bucket
+        )
+      end
+
+      def rename_folders(project_name:, source_bucket:, source_folders:, dest_bucket:)
+        source_folders.each { |folder|
           # If the destination folder already exists, we need to copy the files
           #   over to it and delete the source folder.
           create_folder_request = CreateFolderRequest.new(

data/lib/etna/clients/metis/models.rb

@@ -65,6 +65,51 @@ module Etna
         end
       end
 
+      class ListFolderByIdRequest < Struct.new(:project_name, :bucket_name, :folder_id, keyword_init: true)
+        include JsonSerializableStruct
+
+        def initialize(**params)
+          super({}.update(params))
+        end
+
+        def to_h
+          # The :project_name comes in from Polyphemus as a symbol value,
+          #   we need to make sure it's a string because it's going
+          #   in the URL.
+          super().compact.transform_values(&:to_s)
+        end
+      end
+
+      class TouchFolderRequest < Struct.new(:project_name, :bucket_name, :folder_path, keyword_init: true)
+        include JsonSerializableStruct
+
+        def initialize(**params)
+          super({}.update(params))
+        end
+
+        def to_h
+          # The :project_name comes in from Polyphemus as a symbol value,
+          #   we need to make sure it's a string because it's going
+          #   in the URL.
+          super().compact.transform_values(&:to_s)
+        end
+      end
+
+      class TouchFileRequest < Struct.new(:project_name, :bucket_name, :file_path, keyword_init: true)
+        include JsonSerializableStruct
+
+        def initialize(**params)
+          super({}.update(params))
+        end
+
+        def to_h
+          # The :project_name comes in from Polyphemus as a symbol value,
+          #   we need to make sure it's a string because it's going
+          #   in the URL.
+          super().compact.transform_values(&:to_s)
+        end
+      end
+
       class CreateFolderRequest < Struct.new(:project_name, :bucket_name, :folder_path, keyword_init: true)
         include JsonSerializableStruct
 
@@ -110,11 +155,11 @@ module Etna
         end
       end
 
-      class FindRequest < Struct.new(:project_name, :bucket_name, :limit, :offset, :params, keyword_init: true)
+      class FindRequest < Struct.new(:project_name, :bucket_name, :limit, :offset, :params, :hide_paths, keyword_init: true)
         include JsonSerializableStruct
 
         def initialize(**args)
-          super({params: []}.update(args))
+          super({params: [], hide_paths: false}.update(args))
         end
 
         def add_param(param)
@@ -126,6 +171,17 @@ module Etna
           #   easier to do from a JSON string
           JSON.parse(to_json, :symbolize_names => true)
         end
+
+        def clone
+          FindRequest.new(
+            project_name: self.project_name,
+            bucket_name: self.bucket_name,
+            limit: self.limit,
+            offset: self.offset,
+            params: self.params.dup,
+            hide_paths: self.hide_paths
+          )
+        end
       end
 
       class FindParam < Struct.new(:attribute, :predicate, :value, :type, keyword_init: true)
@@ -225,6 +281,13 @@ module Etna
           @raw = raw
         end
 
+        def with_containing_folder(folder)
+          folder_path = folder.is_a?(Folder) ? folder.folder_path : folder
+          File.new({}.update(self.raw).update({
+            file_path: ::File.join(folder_path, self.file_name)
+          }))
+        end
+
         def file_path
           raw[:file_path]
         end
@@ -259,6 +322,14 @@ module Etna
         def size
           raw[:size]
         end
+
+        def file_hash
+          raw[:file_hash]
+        end
+
+        def folder_id
+          raw[:folder_id]
+        end
       end
 
       class Folder
@@ -279,6 +350,19 @@ module Etna
         def bucket_name
           raw[:bucket_name]
         end
+
+        def project_name
+          raw[:project_name]
+        end
+
+        def updated_at
+          time = raw[:updated_at]
+          time.nil? ? nil : Time.parse(time)
+        end
+
+        def id
+          raw[:id]
+        end
       end
 
       class AuthorizeUploadRequest < Struct.new(:project_name, :bucket_name, :file_path, keyword_init: true)

data/lib/etna/clients/metis/workflows/metis_upload_workflow.rb

@@ -110,8 +110,6 @@ module Etna
 
           def initialize(source_file: nil, next_blob_size: nil, current_byte_position: nil)
             self.source_file = source_file
-            self.next_blob_size = next_blob_size
-            self.current_byte_position = current_byte_position
             self.next_blob_size = [file_size, INITIAL_BLOB_SIZE].min
             self.current_byte_position = 0
           end

data/lib/etna/controller.rb

@@ -42,7 +42,10 @@ module Etna
 
       [501, {}, ['This controller is not implemented.']]
     rescue Exception => e
-      handle_error(e)
+      error = e
+    ensure
+      log_request
+      return handle_error(error) if error
     end
 
     def require_params(*params)
@@ -82,8 +85,36 @@ module Etna
       @response.finish
     end
 
+    def config_hosts
+      [:janus, :magma, :timur, :metis, :vulcan, :polyphemus].map do |host|
+        [ :"#{host}_host", @server.send(:application).config(host)&.dig(:host) ]
+      end.to_h.compact
+    end
+
     private
 
+    def redact_keys
+      @request.env['etna.redact_keys']
+    end
+
+    def add_redact_keys(new_redact_keys=[])
+      @request.env['etna.redact_keys'] = (@request.env['etna.redact_keys'] || []).concat(new_redact_keys)
+    end
+
+    def log_request
+      censor = Etna::Censor.new(redact_keys)
+
+      redacted_params = @params.map do |key,value|
+        [ key, censor.redact(key, value) ]
+      end.to_h
+
+      log("User #{@user ? @user.email : :unknown} calling #{controller_name}##{@action} with params #{redacted_params}")
+    end
+
+    def controller_name
+      self.class.name.sub("Kernel::", "").sub("Controller", "").downcase
+    end
+
     def success(msg, content_type='text/plain')
       @response['Content-Type'] = content_type
       @response.write(msg)

data/lib/etna/route.rb

@@ -1,5 +1,6 @@
 require 'digest'
 require 'date'
+require_relative "./censor"
 
 module Etna
   class Route
@@ -13,6 +14,7 @@ module Etna
       @route = route.gsub(/\A(?=[^\/])/, '/')
       @block = block
       @match_ext = options[:match_ext]
+      @log_redact_keys = options[:log_redact_keys]
     end
 
     def to_hash
@@ -124,6 +126,8 @@ module Etna
         return [ 403, { 'Content-Type' => 'application/json' }, [ { error: 'You are forbidden from performing this action.' }.to_json ] ]
       end
 
+      request.env['etna.redact_keys'] = @log_redact_keys
+
       if @action
         controller, action = @action.split('#')
         controller_class = Kernel.const_get(
@@ -132,11 +136,6 @@ module Etna
         logger = request.env['etna.logger']
         user = request.env['etna.user']
 
-        params = request.env['rack.request.params'].map do |key,value|
-          [ key, redact(key, value) ]
-        end.to_h
-
-        logger.warn("User #{user ? user.email : :unknown} calling #{controller}##{action} with params #{params}")
         return controller_class.new(request, action).response
       elsif @block
         application = Etna::Application.find(app.class).class
@@ -161,35 +160,6 @@ module Etna
       @application ||= Etna::Application.instance
     end
 
-    def compact(value)
-      value = value.to_s
-      value = value[0..500] + "..." + value[-100..-1] if value.length > 600
-      value
-    end
-
-    def redact_keys
-      @redact_keys ||= application.config(:log_redact_keys).split(",").map do |key|
-        key.to_sym
-      end
-    end
-
-    def redact(key, value)
-      # From configuration, redact any values for the supplied key values, so they
-      #   don't appear in the logs.
-      return compact(value) unless application.config(:log_redact_keys)
-
-      if value.is_a?(Hash)
-        redacted_value = value.map do |value_key, value_value|
-          [ value_key, redact(value_key, value_value) ]
-        end.to_h
-        return redacted_value
-      elsif redact_keys.include?(key)
-        return "*"
-      end
-
-      return compact(value)
-    end
-
     def authorized?(request)
       # If there is no @auth requirement, they are ok - this doesn't preclude
       # them being rejected in the controller response

data/lib/etna/synchronize_db.rb

@@ -0,0 +1,14 @@
+module Etna
+  class SynchronizeDb
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      # Do a coarse checkout of the connection
+      Etna::Application.instance.db.synchronize do
+        @app.call(env)
+      end
+    end
+  end
+end

data/lib/etna.rb

@@ -23,6 +23,7 @@ require_relative './etna/formatting'
 require_relative './etna/cwl'
 require_relative './etna/metrics'
 require_relative './etna/remote'
+require_relative './etna/synchronize_db'
 
 class EtnaApp
   include Etna::Application

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: etna
 version: !ruby/object:Gem::Version
-  version: 0.1.41
+  version: 0.1.42
 platform: ruby
 authors:
 - Saurabh Asthana
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-11 00:00:00.000000000 Z
+date: 2021-10-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -138,6 +138,7 @@ files:
 - lib/etna.rb
 - lib/etna/application.rb
 - lib/etna/auth.rb
+- lib/etna/censor.rb
 - lib/etna/client.rb
 - lib/etna/clients.rb
 - lib/etna/clients/base_client.rb
@@ -208,6 +209,7 @@ files:
 - lib/etna/spec/auth.rb
 - lib/etna/spec/vcr.rb
 - lib/etna/symbolize_params.rb
+- lib/etna/synchronize_db.rb
 - lib/etna/templates/attribute_actions_template.json
 - lib/etna/test_auth.rb
 - lib/etna/user.rb