etna 0.1.43 → 0.1.46

data/lib/commands.rb

@@ -2,6 +2,7 @@ require 'date'
 require 'logger'
 require 'rollbar'
 require 'tempfile'
+require 'securerandom'
 require_relative 'helpers'
 require 'yaml'
 
@@ -88,6 +89,123 @@ class EtnaApp
     end
   end
 
+  class Metis
+    include Etna::CommandExecutor
+    string_flags << "--project-name"
+    string_flags << "--bucket-name"
+    string_flags << "--path"
+    ROLLING_COUNT = 14
+
+    class PutArchive < Etna::Command
+      include WithEtnaClients
+
+      def execute(file, project_name:, bucket_name:, path:)
+        @project_name = project_name
+        @bucket_name = bucket_name
+        basename = ::File.basename(path)
+        dir = ::File.dirname(path)
+
+        puts "Creating archive folder"
+        metis_client.create_folder(Etna::Clients::Metis::CreateFolderRequest.new(
+          project_name: project_name,
+          bucket_name: bucket_name,
+          folder_path: dir
+        ))
+
+        puts "Listing archive folder"
+        files = metis_client.list_folder(Etna::Clients::Metis::ListFolderRequest.new(
+          project_name: project_name,
+          bucket_name: bucket_name,
+          folder_path: dir
+        )).files.all
+
+        if files.select { |f| f.file_name == basename }.length > 0
+          timestr = DateTime.now.strftime("%Y%m%d_%H%M")
+          puts "Backing up #{dir}/#{basename} to #{dir}/#{basename}.#{timestr}"
+          metis_client.rename_file(Etna::Clients::Metis::RenameFileRequest.new(
+            project_name: project_name,
+            bucket_name: bucket_name,
+            file_path: ::File.join(dir, basename),
+            new_bucket_name: bucket_name,
+            new_file_path: ::File.join(dir, "#{basename}.#{timestr}"),
+          ))
+        end
+
+        create_upload_workflow.do_upload(
+          ::File.open(file, "r"),
+          path
+        ) do |progress|
+          case progress[0]
+          when :error
+            puts("Error while uploading: #{progress[1].to_s}")
+          else
+          end
+        end
+        puts "Completed upload"
+
+        backups = files.select { |f| f.file_name != basename }.sort_by(&:file_name).reverse
+        if backups.length > ROLLING_COUNT
+          backups.slice(ROLLING_COUNT..-1).reverse.each do |f|
+            puts "Removing rolling back up #{f.file_name}"
+            metis_client.delete_file(Etna::Clients::Metis::DeleteFileRequest.new(
+              project_name: project_name,
+              bucket_name: bucket_name,
+              file_path: ::File.join(dir, f.file_name)
+            ))
+          end
+        end
+      end
+
+      def create_upload_workflow
+        Etna::Clients::Metis::MetisUploadWorkflow.new(
+          metis_client: @metis_client,
+          metis_uid: SecureRandom.hex,
+          project_name: @project_name,
+          bucket_name: @bucket_name,
+          max_attempts: 3
+        )
+      end
+    end
+
+    class PullArchive < Etna::Command
+      include WithEtnaClients
+
+      def execute(project_name:, bucket_name:, path:)
+        @project_name = project_name
+        @bucket_name = bucket_name
+        basename = ::File.basename(path)
+        dir = ::File.dirname(path)
+
+        files = metis_client.list_folder(Etna::Clients::Metis::ListFolderRequest.new(
+          project_name: project_name,
+          bucket_name: bucket_name,
+          folder_path: dir
+        )).files.all
+
+        files.sort_by(&:file_name).reverse
+
+        archived = files.select { |f| f.file_name == basename }.first
+        if archived.nil?
+          archived = files.first
+        end
+
+        tmp = Tempfile.new('download', Dir.pwd)
+        begin
+          puts "Downloading to #{basename} from #{archived.file_name}"
+          metis_client.download_file(archived) do |chunk|
+            tmp.write(chunk)
+          end
+
+          # Atomic operation -- ensure we only place the file in position when it successfully loads
+          ::File.rename(tmp.path, ::File.join(::Dir.pwd, basename))
+        rescue
+          tmp.close!
+          raise
+        end
+      end
+    end
+  end
+
   class Administrate
     include Etna::CommandExecutor
 

data/lib/etna/application.rb

@@ -51,22 +51,7 @@ module Etna::Application
   end
 
   def configure(opts)
-    @config = opts
-
-    # Apply environmental variables of the form "APP__x__y__z"
-    prefix = "#{self.class.name.upcase}__"
-    ENV.keys.select { |k| k.start_with?(prefix) }.each do |key|
-      path = key.split("__", -1)
-      path.shift # drop the first, just app name
-
-      target = @config
-      while path.length > 1
-        n = path.shift
-        target = (target[n.downcase.to_sym] ||= {})
-      end
-
-      target[path.last.downcase.to_sym] ||= ENV[key]
-    end
+    @config = Etna::EnvironmentVariables.load_from_env('ETNA', root: opts) { |path, value| load_config_from_env_path(path, value) }
 
     if (rollbar_config = config(:rollbar)) && rollbar_config[:access_token]
       Rollbar.configure do |config|
@@ -75,6 +60,14 @@ module Etna::Application
     end
   end
 
+  def load_config_from_env_path(path, value)
+    return nil if path.empty?
+    return nil unless path.last.end_with?('_file')
+    path.last.sub!(/_file$/, '')
+
+    [path.map(&:to_sym), YAML.load(File.read(value))]
+  end
+
   def setup_yabeda
     application = self.id
     Yabeda.configure do
@@ -168,6 +161,10 @@ module Etna::Application
     (ENV["#{self.class.name.upcase}_ENV"] || :development).to_sym
   end
 
+  def test?
+    environment == "test"
+  end
+
   def id
     ENV["APP_NAME"] || self.class.name.snake_case.split(/::/).last
   end

data/lib/etna/auth.rb

@@ -18,12 +18,16 @@ module Etna
       if [ approve_noauth(request), approve_hmac(request), approve_user(request) ].all?{|approved| !approved}
         return fail_or_redirect(request)
       end
-
+  
       @app.call(request.env)
     end
 
     private
 
+    def janus
+      @janus ||= Etna::JanusUtils.new
+    end
+
     def application
       @application ||= Etna::Application.instance
     end
@@ -61,7 +65,7 @@ module Etna
         application.config(:auth_redirect).chomp('/') + '/login'
       )
       uri.query = URI.encode_www_form(refer: request.url)
-      return [ 302, { 'Location' => uri.to_s }, [] ]
+      Etna::Redirect(request).to(uri.to_s)
     end
 
     def approve_noauth(request)
@@ -76,21 +80,35 @@ module Etna
       # some routes don't need janus approval
       return true if route && route.ignore_janus?
 
-      # only process task tokens right now
-      return true unless payload['task']
+      # process task tokens
+      payload[:task] ? janus.valid_task_token?(token) : true
+    end
 
-      return false unless application.config(:janus) && application.config(:janus)[:host]
+    def symbolize_payload_keys(payload)
+      payload.map{|k,v| [k.to_sym, v]}.to_h
+    end
 
-      janus_client = Etna::Clients::Janus.new(
-        token: token,
-        host: application.config(:janus)[:host]
-      )
+    def permissions(payload)
+      Etna::Permissions.from_encoded_permissions(payload[:perm])
+    end
 
-      response = janus_client.validate_task_token
+    def update_payload(payload, token, request)
+      route = server.find_route(request)
 
-      return false unless response.code == '200'
+      return payload unless route
 
-      return true
+      begin      
+        permissions = permissions(payload)
+
+        janus.resource_projects(token).each do |resource_project|
+          permissions.add_permission(
+            Etna::Permission.new('v', resource_project.project_name)
+          )
+        end
+        payload[:perm] = permissions.to_string
+      end if (!route.ignore_janus? && route.has_user_constraint?(:can_view?))
+
+      payload
     end
 
     def approve_user(request)
@@ -101,9 +119,14 @@ module Etna
       begin
         payload, header = application.sign.jwt_decode(token)
 
+        payload = symbolize_payload_keys(payload)
+
         return false unless janus_approved?(payload, token, request)
-        return request.env['etna.user'] = Etna::User.new(payload.map{|k,v| [k.to_sym, v]}.to_h, token)
-      rescue
+        return request.env['etna.user'] = Etna::User.new(
+          update_payload(payload, token, request),
+          token)
+      rescue => e
+        application.logger.log_error(e)
         # bail out if anything goes wrong
         return false
       end

data/lib/etna/clients/janus/client.rb

@@ -8,25 +8,33 @@ module Etna
   module Clients
     class Janus < Etna::Clients::BaseClient
       def get_project(get_project_request = GetProjectRequest.new)
-        html = nil
+        json = nil
         @etna_client.get(
-          "/project/#{get_project_request.project_name}",
-          get_project_request) do |res|
-          html = res.body
+          "/api/admin/#{get_project_request.project_name}/info") do |res|
+            json = JSON.parse(res.body, symbolize_names: true)
         end
 
-        HtmlResponse.new(html)
+        GetProjectResponse.new(json)
+      end
+
+      def get_projects()
+        json = nil
+        @etna_client.get('/api/user/projects') do |res|
+          json = JSON.parse(res.body, symbolize_names: true)
+        end
+
+        GetProjectsResponse.new(json)
       end
 
       def add_project(add_project_request = AddProjectRequest.new)
-        @etna_client.post('/add_project', add_project_request) do |res|
+        @etna_client.post('/api/admin/add_project', add_project_request) do |res|
           # Redirect, no response data
         end
       end
 
       def add_user(add_user_request = AddUserRequest.new)
         @etna_client.post(
-          "/add_user/#{add_user_request.project_name}",
+          "/api/admin/#{add_user_request.project_name}/add_user",
           add_user_request) do |res|
           # Redirect, no response data
         end
@@ -34,7 +42,7 @@ module Etna
 
       def update_permission(update_permission_request = UpdatePermissionRequest.new)
         @etna_client.post(
-          "/update_permission/#{update_permission_request.project_name}",
+          "/api/admin/#{update_permission_request.project_name}/update_permission",
           update_permission_request) do |res|
           # Redirect, no response data
         end
@@ -42,16 +50,7 @@ module Etna
 
       def refresh_token(refresh_token_request = RefreshTokenRequest.new)
         token = nil
-        @etna_client.get('/refresh_token', refresh_token_request) do |res|
-          token = res.body
-        end
-
-        TokenResponse.new(token)
-      end
-
-      def viewer_token(viewer_token_request = ViewerTokenRequest.new)
-        token = nil
-        @etna_client.get('/viewer_token', viewer_token_request) do |res|
+        @etna_client.post('/api/tokens/generate') do |res|
           token = res.body
         end
 

data/lib/etna/clients/janus/models.rb

@@ -69,6 +69,58 @@ module Etna
         end
       end
 
+      class GetProjectResponse
+        attr_reader :raw
+
+        def initialize(raw = '')
+          @raw = raw
+        end
+
+        def project
+          Project.new(@raw[:project])
+        end
+      end
+
+      class GetProjectsResponse
+        attr_reader :raw
+
+        def initialize(raw = '')
+          @raw = raw
+        end
+
+        def projects
+          @raw[:projects].map do |project|
+            Project.new(project)
+          end
+        end
+      end
+
+      class Project
+        def initialize(raw = '')
+          @raw = raw
+        end
+
+        def project_name
+          @raw[:project_name]
+        end
+
+        def project_name_full
+          @raw[:project_name_full]
+        end
+
+        def permissions
+          @raw[:permissions] || []
+        end
+
+        def resource
+          !!@raw[:resource]
+        end
+
+        def requires_agreement
+          !!@raw[:requires_agreement]
+        end
+      end
+
       class TokenResponse
         attr_reader :raw
 

data/lib/etna/clients/magma/client.rb

@@ -6,6 +6,47 @@ require_relative './models'
 
 module Etna
   module Clients
+    class LocalMagmaClient # May only be used from within magma app.
+      def logger
+        ::Magma.instance.logger
+      end
+
+      def user
+        permissions = Etna::Permissions.new([])
+        permissions.add_permission(Etna::Permission.new('A', 'administration'))
+        @user ||= Etna::User.new({name: 'Admin', email: 'etnaagent@ucsf.edu', perm: permissions.to_string})
+      end
+
+      def fake_request(request)
+        Rack::Request.new({
+          'rack.request.params' => request.as_json,
+          'etna.logger' => self.logger,
+          'etna.user' => user,
+          'etna.request_id' => 'local',
+        })
+      end
+
+      def parse_json_response(response)
+        status, headers, body = response
+        body = body.join('')
+        if status < 200 || status >= 300
+          raise Etna::Error.new(body, status)
+        end
+
+        JSON.parse(body)
+      end
+
+      def retrieve(retrieval_request = RetrievalRequest.new)
+        controller = ::RetrieveController.new(fake_request(retrieval_request), nil)
+        Magma::RetrievalResponse.new(parse_json_response(controller.action))
+      end
+
+      def update_model(update_model_request = UpdateModelRequest.new)
+        controller = ::UpdateModelController.new(fake_request(update_model_request), nil)
+        Magma::UpdateModelResponse.new(parse_json_response(controller.action))
+      end
+    end
+
     class Magma < Etna::Clients::BaseClient
       # This endpoint returns models and records by name:
       # e.g. params:

data/lib/etna/clients/magma/formatting/models_csv.rb

@@ -22,6 +22,7 @@ module Etna
                 :new_attribute_name,
                 :attribute_type,
                 :link_model_name,
+                :link_attribute_name,
                 :description,
                 :display_name,
                 :format_hint,
@@ -114,7 +115,10 @@ module Etna
             if attribute.attribute_type == Etna::Clients::Magma::AttributeType::IDENTIFIER
               # Identifiers for models whose parent link type ends up being a table are non configurable, so we don't
               # want to include them in the CSV.
-              if models.find_reciprocal(model: model, link_attribute_name: model.template.parent)&.attribute_type == Etna::Clients::Magma::AttributeType::TABLE
+              if models.find_reciprocal(
+                model: model,
+                link_attribute_name: model.name,
+                link_model: models.model(model.template.parent))&.attribute_type == Etna::Clients::Magma::AttributeType::TABLE
                 return
               end
             else
@@ -137,6 +141,7 @@ module Etna
                 attribute_name: attribute.name,
                 attribute_type: attribute.attribute_type,
                 link_model_name: attribute.link_model_name,
+                link_attribute_name: attribute.link_attribute_name,
                 reciprocal_link_type: models.find_reciprocal(model: model, attribute: attribute)&.attribute_type,
                 description: attribute.description,
                 display_name: attribute.display_name,
@@ -175,7 +180,7 @@ module Etna
           # This should line up with the attribute names _on the model itself_.
           ATTRIBUTE_ROW_ENTRIES = [
               :attribute_type,
-              :link_model_name, :description,
+              :link_model_name, :link_attribute_name, :description,
               :display_name, :format_hint,
               :restricted, :read_only,
               :validation, :attribute_group,
@@ -223,6 +228,7 @@ module Etna
             end
           rescue ImportError => e
             validation_err_block.call(e.message)
+            raise e
           end
 
 
@@ -255,7 +261,11 @@ module Etna
             end
 
             row_processor.process(:parent_link_type, :model_name, :parent_model_name) do |parent_link_type, template|
-              reciprocal = models.find_reciprocal(model_name: template.name, link_attribute_name: template.parent)
+              reciprocal = models.find_reciprocal(
+                model_name: template.name,
+                link_attribute_name: template.name,
+                link_model: models.model(template.parent)
+              )
               if reciprocal && reciprocal.attribute_type.to_s != parent_link_type
                 raise ImportError.new("Model #{template.name} was provided multiple parent_link_types: #{reciprocal.attribute_type} and #{parent_link_type}")
               end
@@ -333,7 +343,7 @@ module Etna
                 att.send(:"#{prop_name}=", value)
 
                 if att.attribute_type && att.link_model_name
-                  if att.attribute_type == Etna::Clients::Magma::AttributeType::LINK && models.find_reciprocal(model_name: template.name, attribute: att).nil?
+                  if att.attribute_type == Etna::Clients::Magma::AttributeType::LINK && models.find_reciprocal(model_name: template.name, attribute: att, link_attribute_name: att.link_attribute_name).nil?
                     models.build_model(att.link_model_name).build_template.build_attributes.build_attribute(template.name).tap do |rec_att|
                       rec_att.attribute_name = rec_att.name = template.name
                       rec_att.display_name = prettify(template.name)

data/lib/etna/clients/magma/models.rb

@@ -22,12 +22,12 @@ module Etna
         include JsonSerializableStruct
       end
 
-      class UpdateRequest < Struct.new(:revisions, :project_name, keyword_init: true)
+      class UpdateRequest < Struct.new(:revisions, :project_name, :dry_run, keyword_init: true)
         include JsonSerializableStruct
         include MultipartSerializableNestedHash
 
         def initialize(**params)
-          super({revisions: {}}.update(params))
+          super({revisions: {}, dry_run: false}.update(params))
         end
 
         def update_revision(model_name, record_name, attrs)
@@ -110,7 +110,7 @@ module Etna
         include JsonSerializableStruct
       end
 
-      class AddProjectAction < Struct.new(:action_name, keyword_init: true)
+      class AddProjectAction < Struct.new(:action_name, :no_metis_bucket, keyword_init: true)
         include JsonSerializableStruct
 
         def initialize(**args)
@@ -215,8 +215,8 @@ module Etna
         end
 
         def model(model_key)
-          return nil unless raw.include?(model_key)
-          Model.new(raw[model_key])
+          return nil unless raw.include?(model_key.to_s)
+          Model.new(raw[model_key.to_s])
         end
 
         def all
@@ -239,6 +239,9 @@ module Etna
             link_model: self.model(attribute&.link_model_name)
         )
           return nil if model.nil? || model.name.nil?
+
+          return link_model&.template&.attributes&.all&.find { |a| a.name == link_attribute_name } if link_attribute_name
+
           link_model&.template&.attributes&.all&.find { |a| a.link_model_name == model.name }
         end
 
@@ -434,8 +437,8 @@ module Etna
         end
 
         def attribute(attribute_key)
-          return nil unless raw.include?(attribute_key)
-          Attribute.new(raw[attribute_key])
+          return nil unless raw.include?(attribute_key.to_s)
+          Attribute.new(raw[attribute_key.to_s])
         end
 
         def build_attribute(key)
@@ -457,7 +460,7 @@ module Etna
         def is_edited?(other)
           # Don't just override == in case need to do a full comparison.
           editable_attribute_names = Attribute::EDITABLE_ATTRIBUTE_ATTRIBUTES.map(&:to_s)
-          
+
           self_editable = raw.slice(*editable_attribute_names)
           other_editable = other.raw.slice(*editable_attribute_names)
 
@@ -505,6 +508,14 @@ module Etna
           @raw['attribute_type'] = val
         end
 
+        def link_attribute_name
+          @raw['link_attribute_name']
+        end
+
+        def link_attribute_name=(val)
+          @raw['link_attribute_name'] = val
+        end
+
         def link_model_name
           @raw['link_model_name']
         end

data/lib/etna/clients/magma/workflows/create_project_workflow.rb

@@ -59,7 +59,7 @@ module Etna
         end
 
         def create_magma_project_record!
-          magma_client.update(Etna::Clients::Magma::UpdateRequest.new(
+          magma_client.update_json(Etna::Clients::Magma::UpdateRequest.new(
             project_name: project_name,
             revisions: {
                 'project' => { project_name => { name: project_name } },

data/lib/etna/clients/magma/workflows/json_validators.rb

@@ -113,7 +113,11 @@ module Etna
         end
 
         def parent_reciprocal_attribute
-          @models.find_reciprocal(model: @model, link_attribute_name: @model.template.parent)
+          @models.find_reciprocal(
+            model: @model,
+            link_attribute_name: @model.name,
+            link_model: @models.model(@model.template.parent)
+          )
         end
 
         def name
@@ -172,10 +176,6 @@ module Etna
           link_attributes.each do |attribute|
             check_key("attribute #{attribute.attribute_name}", attribute.raw, 'link_model_name')
 
-            if attribute.attribute_name != attribute.link_model_name
-              @errors << "Linked model, \"#{attribute.link_model_name}\", does not match attribute #{attribute.attribute_name}, link attribute names must match the model name."
-            end
-
             unless @models.model_keys.include?(attribute.link_model_name)
               @errors << "Linked model, \"#{attribute.link_model_name}\", on attribute #{attribute.attribute_name} does not exist!"
             end

data/lib/etna/clients/magma/workflows/model_synchronization_workflow.rb

@@ -29,7 +29,7 @@ module Etna
         end
 
         def execute_planned!
-          @planned_actions.each do |action|
+          planned_actions.each do |action|
             execute_update(action)
           end
         end
@@ -131,7 +131,7 @@ module Etna
             unless attribute.attribute_type == AttributeType::PARENT
               if attribute.link_model_name
                 ensure_model(attribute.link_model_name)
-                ensure_model_link(model_name, attribute.link_model_name, attribute.attribute_name)
+                ensure_model_link(model_name, attribute.link_model_name, attribute.attribute_name, attribute.link_attribute_name)
               else
                 ensure_model_attribute(model_name, attribute.attribute_name)
               end
@@ -144,26 +144,26 @@ module Etna
           end
         end
 
-        def ensure_model_link(model_name, link_model_name, attribute_name)
+        def ensure_model_link(model_name, link_model_name, attribute_name, link_attribute_name)
           return unless (model = source_models.model(model_name))
           return unless (source_attribute = model.template.attributes.attribute(attribute_name))
 
           return unless (link_model = source_models.model(link_model_name))
-          link_model_attributes = link_model.template.attributes
-          reciprocal = link_model_attributes.all.find do |attr|
-            attr.link_model_name == model_name
-          end
+          return unless (reciprocal = link_model.template.attributes.attribute(link_attribute_name))
 
           target_model_name = target_of_source(model_name)
           target_link_model_name = target_of_source(link_model_name)
 
           target_attributes = target_models.model(target_model_name).template.attributes
           return if target_attributes.attribute_keys.include?(target_link_model_name)
+          return if target_attributes.attribute_keys.include?(reciprocal.attribute_name)
+
+          # skip non-links for circular references so they don't get added twice
+          return if link_model_name == model_name && reciprocal.attribute_type != 'link'
 
           add_link = AddLinkAction.new
-          add_link.links << AddLinkDefinition.new(model_name: target_model_name, attribute_name: target_link_model_name, type: source_attribute.attribute_type)
+          add_link.links << AddLinkDefinition.new(model_name: target_model_name, attribute_name: attribute_name, type: source_attribute.attribute_type)
           add_link.links << AddLinkDefinition.new(model_name: target_link_model_name, attribute_name: reciprocal.attribute_name, type: reciprocal.attribute_type)
-
           queue_update(add_link)
         end