etna 0.1.41 → 0.1.44

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 44b6c9fde1d01fab95038052180531fafd4a120e77ffbd12a8ac14e28122cd05
-  data.tar.gz: e1cb315eaeb4d35efc9e7f181e0cad767833b98f7ce3f26386e040c758c8483c
+  metadata.gz: bea4cbc1d94fca11f52c02b75ef487d11e7b0a6f2aaa7d64c03a09803891d627
+  data.tar.gz: cb764eb044b4b711ddcc4ea46efa598763a34fedf2dc1984a9a2e88234094e2c
 SHA512:
-  metadata.gz: 3129a01c4181a772ab6752ed319396485e06f774b8091bff1ed9cbf6543a34820e0cc0706fd313ce5b3fb848a25f347de17bc376dc66b36458975d6298f48608
-  data.tar.gz: 82c57cff18c826a0a2664a63e0a531df57c75af30ceb3408d1d8ec1b9cb942fd25bd11253e6d880db6973df4cd60ade0764fac798a543af1e7601bfcc21abc45
+  metadata.gz: 30100186f9af0e44654cc0a72d23eff04872f9479e119d308ac749bac4c7125358ea6e62f0a5cafd751512068715ac7831cbb5770ec554394c35c8c3781c30ce
+  data.tar.gz: dcee2b46965993e00d0dcae0a516b501a2afbbe968f1c9df35e45fcff4cd05535af7de52ddfec7e345943f3a1f4536aca196dd8150632d1a628559af482c6d7a

data/etna.completion

@@ -84,7 +84,7 @@ arg_flag_completion_names="$arg_flag_completion_names  "
 multi_flags="$multi_flags  "
 while [[ "$#" != "0" ]]; do
 if [[ "$#" == "1" ]];  then
-all_completion_names="apply_template attributes copy_template help load_from_redcap"
+all_completion_names="apply_template attributes copy_template help load_from_redcap set_date_shift_root"
 all_completion_names="$all_completion_names $all_flag_completion_names"
 if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
 return
@@ -568,6 +568,54 @@ return
 fi
 done
 return
+elif [[ "$1" == "set_date_shift_root" ]]; then
+shift
+if [[ "$#" == "1" ]];  then
+all_completion_names="__project_name__"
+if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
+return
+fi
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+fi
+shift
+all_flag_completion_names="$all_flag_completion_names --date-shift-root --target-model "
+arg_flag_completion_names="$arg_flag_completion_names --target-model "
+multi_flags="$multi_flags  "
+declare _completions_for_target_model="__target_model__"
+while [[ "$#" != "0" ]]; do
+if [[ "$#" == "1" ]];  then
+all_completion_names=""
+all_completion_names="$all_completion_names $all_flag_completion_names"
+if [[ -z "$(echo $all_completion_names | xargs)" ]]; then
+return
+fi
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+elif [[ -z "$(echo $all_flag_completion_names | xargs)" ]]; then
+return
+elif [[ "$all_flag_completion_names" =~ $1\  ]]; then
+if ! [[ "$multi_flags" =~ $1\  ]]; then
+all_flag_completion_names="${all_flag_completion_names//$1\ /}"
+fi
+a=$1
+shift
+if [[ "$arg_flag_completion_names" =~ $a\  ]]; then
+if [[ "$#" == "1" ]];  then
+a="${a//--/}"
+a="${a//-/_}"
+i="_completions_for_$a"
+all_completion_names="${!i}"
+COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
+return
+fi
+shift
+fi
+else
+return
+fi
+done
+return
 elif [[ -z "$(echo $all_flag_completion_names | xargs)" ]]; then
 return
 elif [[ "$all_flag_completion_names" =~ $1\  ]]; then
@@ -738,10 +786,11 @@ COMPREPLY=($(compgen -W "$all_completion_names" -- "$1"))
 return
 elif [[ "$1" == "generate" ]]; then
 shift
-all_flag_completion_names="$all_flag_completion_names --task --project-name "
-arg_flag_completion_names="$arg_flag_completion_names --project-name "
+all_flag_completion_names="$all_flag_completion_names --task --project-name --email "
+arg_flag_completion_names="$arg_flag_completion_names --project-name --email "
 multi_flags="$multi_flags  "
 declare _completions_for_project_name="__project_name__"
+declare _completions_for_email="__email__"
 while [[ "$#" != "0" ]]; do
 if [[ "$#" == "1" ]];  then
 all_completion_names=""

data/lib/commands.rb

@@ -411,6 +411,25 @@ class EtnaApp
           end
         end
       end
+
+      class SetDateShiftRoot < Etna::Command
+        include WithEtnaClients
+        include WithLogger
+        include StrongConfirmation
+
+        boolean_flags << '--date-shift-root'
+        string_flags << '--target-model'
+
+        def execute(project_name, target_model: 'subject', date_shift_root: false)
+          magma_client.update_model(Etna::Clients::Magma::UpdateModelRequest.new(
+            actions:[Etna::Clients::Magma::SetDateShiftRootAction.new(
+              model_name: target_model,
+              date_shift_root: date_shift_root
+            )],
+            project_name: project_name,
+          ))
+        end
+      end
     end
   end
 

data/lib/etna/application.rb

@@ -50,9 +50,42 @@ module Etna::Application
     Etna::Application.register(self)
   end
 
+  # a <- b
+  def deep_merge(a, b)
+    if a.is_a?(Hash)
+      if b.is_a?(Hash)
+        b.keys.each do |b_key|
+          a[b_key] = deep_merge(a[b_key], b[b_key])
+        end
+
+        return a
+      end
+    end
+
+    a.nil? ? b : a
+  end
+
   def configure(opts)
     @config = opts
 
+    # Apply environmental variables of the form "ETNA__x__y__z_FILE"
+    # by reading the given file.
+    prefix = "ETNA__"
+    ENV.keys.select { |k| k.start_with?(prefix) && k.end_with?("_FILE") }.each do |key|
+      path = key.sub(/_FILE/, '').split("__", -1)
+      path.shift # drop the first, just app name
+
+      target = @config
+      while path.length > 1
+        n = path.shift
+        target = (target[n.downcase.to_sym] ||= {})
+      end
+
+      v = YAML.load(File.read(ENV[key]))
+      target[path.last.downcase.to_sym] =
+        deep_merge(target[path.last.downcase.to_sym], v)
+    end
+
     if (rollbar_config = config(:rollbar)) && rollbar_config[:access_token]
       Rollbar.configure do |config|
         config.access_token = rollbar_config[:access_token]

data/lib/etna/auth.rb

@@ -76,23 +76,63 @@ module Etna
       # some routes don't need janus approval
       return true if route && route.ignore_janus?
 
-      # only process task tokens right now
-      return true unless payload['task']
+      # process task tokens
+      payload['task'] ? valid_task_token?(token) : true
+    end
+
+    def resource_projects(token)
+      return [] unless has_janus_config?
 
-      return false unless application.config(:janus) && application.config(:janus)[:host]
+      janus_client(token).get_projects.projects.select do |project|
+        project.resource
+      end
+    rescue
+      # If encounter any issue with Janus, we'll return no resource projects
+      []
+    end
 
-      janus_client = Etna::Clients::Janus.new(
+    def janus_client(token)
+      Etna::Clients::Janus.new(
         token: token,
         host: application.config(:janus)[:host]
       )
+    end
+
+    def valid_task_token?(token)
+      return false unless has_janus_config?
 
-      response = janus_client.validate_task_token
+      response = janus_client(token).validate_task_token
 
       return false unless response.code == '200'
 
       return true
     end
 
+    def has_janus_config?
+      application.config(:janus) && application.config(:janus)[:host]
+    end
+
+    def update_payload(payload, token, request)
+      route = server.find_route(request)
+
+      payload = payload.map{|k,v| [k.to_sym, v]}.to_h
+
+      return payload unless route
+
+      begin      
+        permissions = Etna::Permissions.from_encoded_permissions(payload[:perm])
+
+        resource_projects(token).each do |resource_project|
+          permissions.add_permission(
+            Etna::Permission.new('v', resource_project.project_name)
+          )
+        end
+        payload[:perm] = permissions.to_string
+      end if (!route.ignore_janus? && route.has_user_constraint?(:can_view?))
+
+      payload
+    end
+
     def approve_user(request)
       token = request.cookies[application.config(:token_name)] || auth(request, :etna)
 
@@ -102,8 +142,11 @@ module Etna
         payload, header = application.sign.jwt_decode(token)
 
         return false unless janus_approved?(payload, token, request)
-        return request.env['etna.user'] = Etna::User.new(payload.map{|k,v| [k.to_sym, v]}.to_h, token)
-      rescue
+        return request.env['etna.user'] = Etna::User.new(
+          update_payload(payload, token, request),
+          token)
+      rescue => e
+        application.logger.log_error(e)
         # bail out if anything goes wrong
         return false
       end

data/lib/etna/censor.rb

@@ -0,0 +1,36 @@
+module Etna
+  class Censor
+    def initialize(log_redact_keys)
+      @log_redact_keys = log_redact_keys
+    end
+
+    def redact_keys
+      @log_redact_keys
+    end
+
+    def redact(key, value)
+      # Redact any values for the supplied key values, so they
+      #   don't appear in the logs.
+      return compact(value) unless redact_keys
+
+      if redact_keys.include?(key)
+        return "*"
+      elsif value.is_a?(Hash)
+        redacted_value = value.map do |value_key, value_value|
+          [value_key, redact(value_key, value_value)]
+        end.to_h
+        return redacted_value
+      end
+
+      return compact(value)
+    end
+
+    private
+
+    def compact(value)
+      value = value.to_s
+      value = value[0..500] + "..." + value[-100..-1] if value.length > 600
+      value
+    end
+  end
+end

data/lib/etna/clients/janus/client.rb

@@ -8,25 +8,33 @@ module Etna
   module Clients
     class Janus < Etna::Clients::BaseClient
       def get_project(get_project_request = GetProjectRequest.new)
-        html = nil
+        json = nil
         @etna_client.get(
-          "/project/#{get_project_request.project_name}",
-          get_project_request) do |res|
-          html = res.body
+          "/api/admin/#{get_project_request.project_name}/info") do |res|
+            json = JSON.parse(res.body, symbolize_names: true)
         end
 
-        HtmlResponse.new(html)
+        GetProjectResponse.new(json)
+      end
+
+      def get_projects()
+        json = nil
+        @etna_client.get('/api/user/projects') do |res|
+          json = JSON.parse(res.body, symbolize_names: true)
+        end
+
+        GetProjectsResponse.new(json)
       end
 
       def add_project(add_project_request = AddProjectRequest.new)
-        @etna_client.post('/add_project', add_project_request) do |res|
+        @etna_client.post('/api/admin/add_project', add_project_request) do |res|
           # Redirect, no response data
         end
       end
 
       def add_user(add_user_request = AddUserRequest.new)
         @etna_client.post(
-          "/add_user/#{add_user_request.project_name}",
+          "/api/admin/#{add_user_request.project_name}/add_user",
           add_user_request) do |res|
           # Redirect, no response data
         end
@@ -34,7 +42,7 @@ module Etna
 
       def update_permission(update_permission_request = UpdatePermissionRequest.new)
         @etna_client.post(
-          "/update_permission/#{update_permission_request.project_name}",
+          "/api/admin/#{update_permission_request.project_name}/update_permission",
           update_permission_request) do |res|
           # Redirect, no response data
         end
@@ -42,16 +50,7 @@ module Etna
 
       def refresh_token(refresh_token_request = RefreshTokenRequest.new)
         token = nil
-        @etna_client.get('/refresh_token', refresh_token_request) do |res|
-          token = res.body
-        end
-
-        TokenResponse.new(token)
-      end
-
-      def viewer_token(viewer_token_request = ViewerTokenRequest.new)
-        token = nil
-        @etna_client.get('/viewer_token', viewer_token_request) do |res|
+        @etna_client.post('/api/tokens/generate') do |res|
           token = res.body
         end
 

data/lib/etna/clients/janus/models.rb

@@ -69,6 +69,54 @@ module Etna
         end
       end
 
+      class GetProjectResponse
+        attr_reader :raw
+
+        def initialize(raw = '')
+          @raw = raw
+        end
+
+        def project
+          Project.new(@raw[:project])
+        end
+      end
+
+      class GetProjectsResponse
+        attr_reader :raw
+
+        def initialize(raw = '')
+          @raw = raw
+        end
+
+        def projects
+          @raw[:projects].map do |project|
+            Project.new(project)
+          end
+        end
+      end
+
+      class Project
+        def initialize(raw = '')
+          @raw = raw
+        end
+
+        def project_name
+          @raw[:project_name]
+        end
+
+        def project_name_full
+          @raw[:project_name_full]
+        end
+
+        def permissions
+          @raw[:permissions] || []
+        end
+
+        def resource
+          !!@raw[:resource]
+        end
+      end
+
       class TokenResponse
         attr_reader :raw
 

data/lib/etna/clients/magma/client.rb

@@ -6,6 +6,47 @@ require_relative './models'
 
 module Etna
   module Clients
+    class LocalMagmaClient # May only be used from within magma app.
+      def logger
+        ::Magma.instance.logger
+      end
+
+      def user
+        permissions = Etna::Permissions.new([])
+        permissions.add_permission(Etna::Permission.new('A', 'administration'))
+        @user ||= Etna::User.new({name: 'Admin', email: 'etnaagent@ucsf.edu', perm: permissions.to_string})
+      end
+
+      def fake_request(request)
+        Rack::Request.new({
+          'rack.request.params' => request.as_json,
+          'etna.logger' => self.logger,
+          'etna.user' => user,
+          'etna.request_id' => 'local',
+        })
+      end
+
+      def parse_json_response(response)
+        status, headers, body = response
+        body = body.join('')
+        if status < 200 || status >= 300
+          raise Etna::Error.new(body, status)
+        end
+
+        JSON.parse(body)
+      end
+
+      def retrieve(retrieval_request = RetrievalRequest.new)
+        controller = ::RetrieveController.new(fake_request(retrieval_request), nil)
+        Magma::RetrievalResponse.new(parse_json_response(controller.action))
+      end
+
+      def update_model(update_model_request = UpdateModelRequest.new)
+        controller = ::UpdateModelController.new(fake_request(update_model_request), nil)
+        Magma::UpdateModelResponse.new(parse_json_response(controller.action))
+      end
+    end
+
     class Magma < Etna::Clients::BaseClient
       # This endpoint returns models and records by name:
       # e.g. params:

data/lib/etna/clients/magma/models.rb

@@ -22,12 +22,12 @@ module Etna
         include JsonSerializableStruct
       end
 
-      class UpdateRequest < Struct.new(:revisions, :project_name, keyword_init: true)
+      class UpdateRequest < Struct.new(:revisions, :project_name, :dry_run, keyword_init: true)
         include JsonSerializableStruct
         include MultipartSerializableNestedHash
 
         def initialize(**params)
-          super({revisions: {}}.update(params))
+          super({revisions: {}, dry_run: false}.update(params))
         end
 
         def update_revision(model_name, record_name, attrs)
@@ -58,11 +58,19 @@ module Etna
         end
       end
 
-      class AddModelAction < Struct.new(:action_name, :model_name, :parent_model_name, :parent_link_type, :identifier, keyword_init: true)
+      class AddModelAction < Struct.new(:action_name, :model_name, :parent_model_name, :parent_link_type, :identifier, :date_shift_root, keyword_init: true)
         include JsonSerializableStruct
 
         def initialize(**args)
-          super({action_name: 'add_model'}.update(args))
+          super({action_name: 'add_model', date_shift_root: false}.update(args))
+        end
+      end
+
+      class SetDateShiftRootAction < Struct.new(:action_name, :model_name, :date_shift_root, keyword_init: true)
+        include JsonSerializableStruct
+
+        def initialize(**args)
+          super({action_name: 'set_date_shift_root'}.update(args))
         end
       end
 
@@ -102,7 +110,7 @@ module Etna
         include JsonSerializableStruct
       end
 
-      class AddProjectAction < Struct.new(:action_name, keyword_init: true)
+      class AddProjectAction < Struct.new(:action_name, :no_metis_bucket, keyword_init: true)
         include JsonSerializableStruct
 
         def initialize(**args)
@@ -207,8 +215,8 @@ module Etna
         end
 
         def model(model_key)
-          return nil unless raw.include?(model_key)
-          Model.new(raw[model_key])
+          return nil unless raw.include?(model_key.to_s)
+          Model.new(raw[model_key.to_s])
         end
 
         def all
@@ -426,8 +434,8 @@ module Etna
         end
 
         def attribute(attribute_key)
-          return nil unless raw.include?(attribute_key)
-          Attribute.new(raw[attribute_key])
+          return nil unless raw.include?(attribute_key.to_s)
+          Attribute.new(raw[attribute_key.to_s])
         end
 
         def build_attribute(key)
@@ -620,6 +628,7 @@ module Etna
         MATRIX = AttributeType.new("matrix")
         PARENT = AttributeType.new("parent")
         TABLE = AttributeType.new("table")
+        SHIFTED_DATE_TIME = AttributeType.new("shifted_date_time")
       end
 
       class ParentLinkType < String

data/lib/etna/clients/magma/workflows/json_validators.rb

@@ -172,10 +172,6 @@ module Etna
           link_attributes.each do |attribute|
             check_key("attribute #{attribute.attribute_name}", attribute.raw, 'link_model_name')
 
-            if attribute.attribute_name != attribute.link_model_name
-              @errors << "Linked model, \"#{attribute.link_model_name}\", does not match attribute #{attribute.attribute_name}, link attribute names must match the model name."
-            end
-
             unless @models.model_keys.include?(attribute.link_model_name)
               @errors << "Linked model, \"#{attribute.link_model_name}\", on attribute #{attribute.attribute_name} does not exist!"
             end

data/lib/etna/clients/magma/workflows/model_synchronization_workflow.rb

@@ -29,7 +29,7 @@ module Etna
         end
 
         def execute_planned!
-          @planned_actions.each do |action|
+          planned_actions.each do |action|
             execute_update(action)
           end
         end
@@ -161,7 +161,7 @@ module Etna
           return if target_attributes.attribute_keys.include?(target_link_model_name)
 
           add_link = AddLinkAction.new
-          add_link.links << AddLinkDefinition.new(model_name: target_model_name, attribute_name: target_link_model_name, type: source_attribute.attribute_type)
+          add_link.links << AddLinkDefinition.new(model_name: target_model_name, attribute_name: attribute_name, type: source_attribute.attribute_type)
           add_link.links << AddLinkDefinition.new(model_name: target_link_model_name, attribute_name: reciprocal.attribute_name, type: reciprocal.attribute_type)
 
           queue_update(add_link)

data/lib/etna/clients/metis/client.rb

@@ -24,8 +24,28 @@ module Etna
       end
 
       def list_folder(list_folder_request = ListFolderRequest.new)
+        if list_folder_request.folder_path != ''
+          FoldersAndFilesResponse.new(
+            @etna_client.folder_list(list_folder_request.to_h))
+        else
+          FoldersAndFilesResponse.new(
+            @etna_client.bucket_list(list_folder_request.to_h))
+        end
+      end
+
+      def list_folder_by_id(list_folder_by_id_request = ListFolderByIdRequest.new)
         FoldersAndFilesResponse.new(
-          @etna_client.folder_list(list_folder_request.to_h))
+          @etna_client.folder_list_by_id(list_folder_by_id_request.to_h))
+      end
+
+      def touch_folder(touch_folder_request = TouchFolderRequest.new)
+        FoldersResponse.new(
+          @etna_client.folder_touch(touch_folder_request.to_h))
+      end
+
+      def touch_file(touch_file_request = TouchFileRequest.new)
+        FilesResponse.new(
+          @etna_client.file_touch(touch_file_request.to_h))
       end
 
       def ensure_parent_folder_exists(project_name:, bucket_name:, path:)
@@ -181,7 +201,16 @@ module Etna
 
         return if found_folders.length == 0
 
-        found_folders.each { |folder|
+        rename_folders(
+          project_name: project_name,
+          source_bucket: source_bucket,
+          source_folders: found_folders,
+          dest_bucket: dest_bucket
+        )
+      end
+
+      def rename_folders(project_name:, source_bucket:, source_folders:, dest_bucket:)
+        source_folders.each { |folder|
           # If the destination folder already exists, we need to copy the files
           #   over to it and delete the source folder.
           create_folder_request = CreateFolderRequest.new(