etna 0.1.42 → 0.1.45

data/lib/commands.rb

@@ -2,6 +2,7 @@ require 'date'
 require 'logger'
 require 'rollbar'
 require 'tempfile'
+require 'securerandom'
 require_relative 'helpers'
 require 'yaml'
 
@@ -88,6 +89,123 @@ class EtnaApp
     end
   end
 
+  class Metis
+    include Etna::CommandExecutor
+    string_flags << "--project-name"
+    string_flags << "--bucket-name"
+    string_flags << "--path"
+    ROLLING_COUNT = 14
+
+    class PutArchive < Etna::Command
+      include WithEtnaClients
+
+      def execute(file, project_name:, bucket_name:, path:)
+        @project_name = project_name
+        @bucket_name = bucket_name
+        basename = ::File.basename(path)
+        dir = ::File.dirname(path)
+
+        puts "Creating archive folder"
+        metis_client.create_folder(Etna::Clients::Metis::CreateFolderRequest.new(
+          project_name: project_name,
+          bucket_name: bucket_name,
+          folder_path: dir
+        ))
+
+        puts "Listing archive folder"
+        files = metis_client.list_folder(Etna::Clients::Metis::ListFolderRequest.new(
+          project_name: project_name,
+          bucket_name: bucket_name,
+          folder_path: dir
+        )).files.all
+
+        if files.select { |f| f.file_name == basename }.length > 0
+          timestr = DateTime.now.strftime("%Y%m%d_%H%M")
+          puts "Backing up #{dir}/#{basename} to #{dir}/#{basename}.#{timestr}"
+          metis_client.rename_file(Etna::Clients::Metis::RenameFileRequest.new(
+            project_name: project_name,
+            bucket_name: bucket_name,
+            file_path: ::File.join(dir, basename),
+            new_bucket_name: bucket_name,
+            new_file_path: ::File.join(dir, "#{basename}.#{timestr}"),
+          ))
+        end
+
+        create_upload_workflow.do_upload(
+          ::File.open(file, "r"),
+          path
+        ) do |progress|
+          case progress[0]
+          when :error
+            puts("Error while uploading: #{progress[1].to_s}")
+          else
+          end
+        end
+        puts "Completed upload"
+
+        backups = files.select { |f| f.file_name != basename }.sort_by(&:file_name).reverse
+        if backups.length > ROLLING_COUNT
+          backups.slice(ROLLING_COUNT..-1).reverse.each do |f|
+            puts "Removing rolling back up #{f.file_name}"
+            metis_client.delete_file(Etna::Clients::Metis::DeleteFileRequest.new(
+              project_name: project_name,
+              bucket_name: bucket_name,
+              file_path: ::File.join(dir, f.file_name)
+            ))
+          end
+        end
+      end
+
+      def create_upload_workflow
+        Etna::Clients::Metis::MetisUploadWorkflow.new(
+          metis_client: @metis_client,
+          metis_uid: SecureRandom.hex,
+          project_name: @project_name,
+          bucket_name: @bucket_name,
+          max_attempts: 3
+        )
+      end
+    end
+
+    class PullArchive < Etna::Command
+      include WithEtnaClients
+
+      def execute(project_name:, bucket_name:, path:)
+        @project_name = project_name
+        @bucket_name = bucket_name
+        basename = ::File.basename(path)
+        dir = ::File.dirname(path)
+
+        files = metis_client.list_folder(Etna::Clients::Metis::ListFolderRequest.new(
+          project_name: project_name,
+          bucket_name: bucket_name,
+          folder_path: dir
+        )).files.all
+
+        files.sort_by(&:file_name).reverse
+
+        archived = files.select { |f| f.file_name == basename }.first
+        if archived.nil?
+          archived = files.first
+        end
+
+        tmp = Tempfile.new('download', Dir.pwd)
+        begin
+          puts "Downloading to #{basename} from #{archived.file_name}"
+          metis_client.download_file(archived) do |chunk|
+            tmp.write(chunk)
+          end
+
+          # Atomic operation -- ensure we only place the file in position when it successfully loads
+          ::File.rename(tmp.path, ::File.join(::Dir.pwd, basename))
+        rescue
+          tmp.close!
+          raise
+        end
+      end
+    end
+  end
+
   class Administrate
     include Etna::CommandExecutor
 

data/lib/etna/application.rb

@@ -51,22 +51,7 @@ module Etna::Application
   end
 
   def configure(opts)
-    @config = opts
-
-    # Apply environmental variables of the form "APP__x__y__z"
-    prefix = "#{self.class.name.upcase}__"
-    ENV.keys.select { |k| k.start_with?(prefix) }.each do |key|
-      path = key.split("__", -1)
-      path.shift # drop the first, just app name
-
-      target = @config
-      while path.length > 1
-        n = path.shift
-        target = (target[n.downcase.to_sym] ||= {})
-      end
-
-      target[path.last.downcase.to_sym] ||= ENV[key]
-    end
+    @config = Etna::EnvironmentVariables.load_from_env('ETNA', root: opts) { |path, value| load_config_from_env_path(path, value) }
 
     if (rollbar_config = config(:rollbar)) && rollbar_config[:access_token]
       Rollbar.configure do |config|
@@ -75,6 +60,14 @@ module Etna::Application
     end
   end
 
+  def load_config_from_env_path(path, value)
+    return nil if path.empty?
+    return nil unless path.last.end_with?('_file')
+    path.last.sub!(/_file$/, '')
+
+    [path.map(&:to_sym), YAML.load(File.read(value))]
+  end
+
   def setup_yabeda
     application = self.id
     Yabeda.configure do
@@ -168,6 +161,10 @@ module Etna::Application
     (ENV["#{self.class.name.upcase}_ENV"] || :development).to_sym
   end
 
+  def test?
+    environment == "test"
+  end
+
   def id
     ENV["APP_NAME"] || self.class.name.snake_case.split(/::/).last
   end

data/lib/etna/auth.rb

@@ -18,12 +18,16 @@ module Etna
       if [ approve_noauth(request), approve_hmac(request), approve_user(request) ].all?{|approved| !approved}
         return fail_or_redirect(request)
       end
-
+  
       @app.call(request.env)
     end
 
     private
 
+    def janus
+      @janus ||= Etna::JanusUtils.new
+    end
+
     def application
       @application ||= Etna::Application.instance
     end
@@ -61,7 +65,7 @@ module Etna
         application.config(:auth_redirect).chomp('/') + '/login'
       )
       uri.query = URI.encode_www_form(refer: request.url)
-      return [ 302, { 'Location' => uri.to_s }, [] ]
+      Etna::Redirect(request).to(uri.to_s)
     end
 
     def approve_noauth(request)
@@ -76,21 +80,35 @@ module Etna
       # some routes don't need janus approval
       return true if route && route.ignore_janus?
 
-      # only process task tokens right now
-      return true unless payload['task']
+      # process task tokens
+      payload[:task] ? janus.valid_task_token?(token) : true
+    end
 
-      return false unless application.config(:janus) && application.config(:janus)[:host]
+    def symbolize_payload_keys(payload)
+      payload.map{|k,v| [k.to_sym, v]}.to_h
+    end
 
-      janus_client = Etna::Clients::Janus.new(
-        token: token,
-        host: application.config(:janus)[:host]
-      )
+    def permissions(payload)
+      Etna::Permissions.from_encoded_permissions(payload[:perm])
+    end
 
-      response = janus_client.validate_task_token
+    def update_payload(payload, token, request)
+      route = server.find_route(request)
 
-      return false unless response.code == '200'
+      return payload unless route
 
-      return true
+      begin      
+        permissions = permissions(payload)
+
+        janus.resource_projects(token).each do |resource_project|
+          permissions.add_permission(
+            Etna::Permission.new('v', resource_project.project_name)
+          )
+        end
+        payload[:perm] = permissions.to_string
+      end if (!route.ignore_janus? && route.has_user_constraint?(:can_view?))
+
+      payload
     end
 
     def approve_user(request)
@@ -101,9 +119,14 @@ module Etna
       begin
         payload, header = application.sign.jwt_decode(token)
 
+        payload = symbolize_payload_keys(payload)
+
         return false unless janus_approved?(payload, token, request)
-        return request.env['etna.user'] = Etna::User.new(payload.map{|k,v| [k.to_sym, v]}.to_h, token)
-      rescue
+        return request.env['etna.user'] = Etna::User.new(
+          update_payload(payload, token, request),
+          token)
+      rescue => e
+        application.logger.log_error(e)
         # bail out if anything goes wrong
         return false
       end

data/lib/etna/clients/janus/client.rb

@@ -8,25 +8,33 @@ module Etna
   module Clients
     class Janus < Etna::Clients::BaseClient
       def get_project(get_project_request = GetProjectRequest.new)
-        html = nil
+        json = nil
         @etna_client.get(
-          "/project/#{get_project_request.project_name}",
-          get_project_request) do |res|
-          html = res.body
+          "/api/admin/#{get_project_request.project_name}/info") do |res|
+            json = JSON.parse(res.body, symbolize_names: true)
         end
 
-        HtmlResponse.new(html)
+        GetProjectResponse.new(json)
+      end
+
+      def get_projects()
+        json = nil
+        @etna_client.get('/api/user/projects') do |res|
+          json = JSON.parse(res.body, symbolize_names: true)
+        end
+
+        GetProjectsResponse.new(json)
       end
 
       def add_project(add_project_request = AddProjectRequest.new)
-        @etna_client.post('/add_project', add_project_request) do |res|
+        @etna_client.post('/api/admin/add_project', add_project_request) do |res|
           # Redirect, no response data
         end
       end
 
       def add_user(add_user_request = AddUserRequest.new)
         @etna_client.post(
-          "/add_user/#{add_user_request.project_name}",
+          "/api/admin/#{add_user_request.project_name}/add_user",
           add_user_request) do |res|
           # Redirect, no response data
         end
@@ -34,7 +42,7 @@ module Etna
 
       def update_permission(update_permission_request = UpdatePermissionRequest.new)
         @etna_client.post(
-          "/update_permission/#{update_permission_request.project_name}",
+          "/api/admin/#{update_permission_request.project_name}/update_permission",
           update_permission_request) do |res|
           # Redirect, no response data
         end
@@ -42,16 +50,7 @@ module Etna
 
       def refresh_token(refresh_token_request = RefreshTokenRequest.new)
         token = nil
-        @etna_client.get('/refresh_token', refresh_token_request) do |res|
-          token = res.body
-        end
-
-        TokenResponse.new(token)
-      end
-
-      def viewer_token(viewer_token_request = ViewerTokenRequest.new)
-        token = nil
-        @etna_client.get('/viewer_token', viewer_token_request) do |res|
+        @etna_client.post('/api/tokens/generate') do |res|
           token = res.body
         end
 

data/lib/etna/clients/janus/models.rb

@@ -69,6 +69,58 @@ module Etna
         end
       end
 
+      class GetProjectResponse
+        attr_reader :raw
+
+        def initialize(raw = '')
+          @raw = raw
+        end
+
+        def project
+          Project.new(@raw[:project])
+        end
+      end
+
+      class GetProjectsResponse
+        attr_reader :raw
+
+        def initialize(raw = '')
+          @raw = raw
+        end
+
+        def projects
+          @raw[:projects].map do |project|
+            Project.new(project)
+          end
+        end
+      end
+
+      class Project
+        def initialize(raw = '')
+          @raw = raw
+        end
+
+        def project_name
+          @raw[:project_name]
+        end
+
+        def project_name_full
+          @raw[:project_name_full]
+        end
+
+        def permissions
+          @raw[:permissions] || []
+        end
+
+        def resource
+          !!@raw[:resource]
+        end
+
+        def requires_agreement
+          !!@raw[:requires_agreement]
+        end
+      end
+
       class TokenResponse
         attr_reader :raw
 

data/lib/etna/clients/magma/client.rb

@@ -6,6 +6,47 @@ require_relative './models'
 
 module Etna
   module Clients
+    class LocalMagmaClient # May only be used from within magma app.
+      def logger
+        ::Magma.instance.logger
+      end
+
+      def user
+        permissions = Etna::Permissions.new([])
+        permissions.add_permission(Etna::Permission.new('A', 'administration'))
+        @user ||= Etna::User.new({name: 'Admin', email: 'etnaagent@ucsf.edu', perm: permissions.to_string})
+      end
+
+      def fake_request(request)
+        Rack::Request.new({
+          'rack.request.params' => request.as_json,
+          'etna.logger' => self.logger,
+          'etna.user' => user,
+          'etna.request_id' => 'local',
+        })
+      end
+
+      def parse_json_response(response)
+        status, headers, body = response
+        body = body.join('')
+        if status < 200 || status >= 300
+          raise Etna::Error.new(body, status)
+        end
+
+        JSON.parse(body)
+      end
+
+      def retrieve(retrieval_request = RetrievalRequest.new)
+        controller = ::RetrieveController.new(fake_request(retrieval_request), nil)
+        Magma::RetrievalResponse.new(parse_json_response(controller.action))
+      end
+
+      def update_model(update_model_request = UpdateModelRequest.new)
+        controller = ::UpdateModelController.new(fake_request(update_model_request), nil)
+        Magma::UpdateModelResponse.new(parse_json_response(controller.action))
+      end
+    end
+
     class Magma < Etna::Clients::BaseClient
       # This endpoint returns models and records by name:
       # e.g. params:

data/lib/etna/clients/magma/models.rb

@@ -22,12 +22,12 @@ module Etna
         include JsonSerializableStruct
       end
 
-      class UpdateRequest < Struct.new(:revisions, :project_name, keyword_init: true)
+      class UpdateRequest < Struct.new(:revisions, :project_name, :dry_run, keyword_init: true)
         include JsonSerializableStruct
         include MultipartSerializableNestedHash
 
         def initialize(**params)
-          super({revisions: {}}.update(params))
+          super({revisions: {}, dry_run: false}.update(params))
         end
 
         def update_revision(model_name, record_name, attrs)
@@ -110,7 +110,7 @@ module Etna
         include JsonSerializableStruct
       end
 
-      class AddProjectAction < Struct.new(:action_name, keyword_init: true)
+      class AddProjectAction < Struct.new(:action_name, :no_metis_bucket, keyword_init: true)
         include JsonSerializableStruct
 
         def initialize(**args)
@@ -215,8 +215,8 @@ module Etna
         end
 
         def model(model_key)
-          return nil unless raw.include?(model_key)
-          Model.new(raw[model_key])
+          return nil unless raw.include?(model_key.to_s)
+          Model.new(raw[model_key.to_s])
         end
 
         def all
@@ -434,8 +434,8 @@ module Etna
         end
 
         def attribute(attribute_key)
-          return nil unless raw.include?(attribute_key)
-          Attribute.new(raw[attribute_key])
+          return nil unless raw.include?(attribute_key.to_s)
+          Attribute.new(raw[attribute_key.to_s])
         end
 
         def build_attribute(key)
@@ -628,6 +628,7 @@ module Etna
         MATRIX = AttributeType.new("matrix")
         PARENT = AttributeType.new("parent")
         TABLE = AttributeType.new("table")
+        SHIFTED_DATE_TIME = AttributeType.new("shifted_date_time")
       end
 
       class ParentLinkType < String

data/lib/etna/clients/magma/workflows/create_project_workflow.rb

@@ -59,7 +59,7 @@ module Etna
         end
 
         def create_magma_project_record!
-          magma_client.update(Etna::Clients::Magma::UpdateRequest.new(
+          magma_client.update_json(Etna::Clients::Magma::UpdateRequest.new(
             project_name: project_name,
             revisions: {
                 'project' => { project_name => { name: project_name } },

data/lib/etna/clients/magma/workflows/json_validators.rb

@@ -172,10 +172,6 @@ module Etna
           link_attributes.each do |attribute|
             check_key("attribute #{attribute.attribute_name}", attribute.raw, 'link_model_name')
 
-            if attribute.attribute_name != attribute.link_model_name
-              @errors << "Linked model, \"#{attribute.link_model_name}\", does not match attribute #{attribute.attribute_name}, link attribute names must match the model name."
-            end
-
             unless @models.model_keys.include?(attribute.link_model_name)
               @errors << "Linked model, \"#{attribute.link_model_name}\", on attribute #{attribute.attribute_name} does not exist!"
             end

data/lib/etna/clients/magma/workflows/model_synchronization_workflow.rb

@@ -29,7 +29,7 @@ module Etna
         end
 
         def execute_planned!
-          @planned_actions.each do |action|
+          planned_actions.each do |action|
             execute_update(action)
           end
         end
@@ -161,7 +161,7 @@ module Etna
           return if target_attributes.attribute_keys.include?(target_link_model_name)
 
           add_link = AddLinkAction.new
-          add_link.links << AddLinkDefinition.new(model_name: target_model_name, attribute_name: target_link_model_name, type: source_attribute.attribute_type)
+          add_link.links << AddLinkDefinition.new(model_name: target_model_name, attribute_name: attribute_name, type: source_attribute.attribute_type)
           add_link.links << AddLinkDefinition.new(model_name: target_link_model_name, attribute_name: reciprocal.attribute_name, type: reciprocal.attribute_type)
 
           queue_update(add_link)

data/lib/etna/clients/metis/client.rb

@@ -24,8 +24,13 @@ module Etna
       end
 
       def list_folder(list_folder_request = ListFolderRequest.new)
-        FoldersAndFilesResponse.new(
-          @etna_client.folder_list(list_folder_request.to_h))
+        if list_folder_request.folder_path != ''
+          FoldersAndFilesResponse.new(
+            @etna_client.folder_list(list_folder_request.to_h))
+        else
+          FoldersAndFilesResponse.new(
+            @etna_client.bucket_list(list_folder_request.to_h))
+        end
       end
 
       def list_folder_by_id(list_folder_by_id_request = ListFolderByIdRequest.new)

data/lib/etna/clients/metis/models.rb

@@ -50,7 +50,10 @@ module Etna
         end
       end
 
-      class ListFolderRequest < Struct.new(:project_name, :bucket_name, :folder_path, keyword_init: true)
+      class FolderRequest < Struct.new(:project_name, :bucket_name, :folder_path, keyword_init: true)
+      end
+
+      class ListFolderRequest < FolderRequest
         include JsonSerializableStruct
 
         def initialize(**params)
@@ -110,7 +113,7 @@ module Etna
         end
       end
 
-      class CreateFolderRequest < Struct.new(:project_name, :bucket_name, :folder_path, keyword_init: true)
+      class CreateFolderRequest < FolderRequest
         include JsonSerializableStruct
 
         def initialize(**params)
@@ -125,7 +128,7 @@ module Etna
         end
       end
 
-      class DeleteFolderRequest < Struct.new(:project_name, :bucket_name, :folder_path, keyword_init: true)
+      class DeleteFolderRequest < FolderRequest
         include JsonSerializableStruct
 
         def initialize(**params)

data/lib/etna/clients/metis/workflows/metis_download_workflow.rb

@@ -18,7 +18,7 @@ module Etna
           completed = 0.0
           start = Time.now
 
-          unless dest_file_or_io.is_a?(IO)
+          unless dest_file_or_io.is_a?(IO) || dest_file_or_io.is_a?(StringIO)
             ::File.open(dest_file_or_io, 'w') do |io|
               return do_download(dest_file_or_io, metis_file, &block)
             end

data/lib/etna/clients/metis/workflows/walk_metis_diff_workflow.rb

@@ -0,0 +1,95 @@
+module Etna
+  module Clients
+    class Metis
+      class WalkMetisDiffWorkflow < Struct.new(:left_walker, :right_walker, keyword_init: true)
+        # Iterates entries of the form [kind, left | nil, right | nil]
+        # where kind is one of
+        #   :left_unique | :right_unique | :left_is_folder | :right_is_folder
+        #   :unknown | :equal | :right_older | :left_older
+        # and left / right is one of
+        #   nil | Etna::Clients::Metis::File | Etna::Clients::Metis::Folder
+        def each(&block)
+          left_enum = self.left_walker.to_enum
+          right_enum = self.right_walker.to_enum
+
+          l, l_path = next_or_nil(left_enum)
+          r, r_path = next_or_nil(right_enum)
+
+          while l && r
+            if l_path == r_path
+              yield [compare_file_or_folders(l, r), l, r]
+
+              l, l_path = next_or_nil(left_enum)
+              r, r_path = next_or_nil(right_enum)
+            elsif l_path < r_path
+              yield [:left_unique, l, nil]
+              l, l_path = next_or_nil(left_enum)
+            else
+              yield [:right_unique, nil, r]
+              r, r_path = next_or_nil(right_enum)
+            end
+          end
+
+          while l
+            yield [:left_unique, l, nil]
+            l, l_path = next_or_nil(left_enum)
+          end
+
+          while r
+            yield [:right_unique, nil, r]
+            r, r_path = next_or_nil(right_enum)
+          end
+        end
+
+        def next_or_nil(enum)
+          enum.next
+        rescue StopIteration
+          [nil, nil]
+        end
+
+        def compare_file_or_folders(l, r)
+          if l.is_a?(Etna::Clients::Metis::Folder)
+            if r.is_a?(Etna::Clients::Metis::Folder)
+              return compare_file_or_folder_age(l, r)
+            end
+
+            return :left_is_folder
+          end
+
+          if r.is_a?(Etna::Clients::Metis::Folder)
+            return :right_is_folder
+          end
+
+
+          if l.file_hash.nil? || r.file_hash.nil?
+            return :unknown
+          end
+
+          if l.file_hash == r.file_hash
+            return :equal
+          end
+
+          return compare_file_or_folder_age(l, r)
+        end
+
+        def compare_file_or_folder_age(l, r)
+          if l.updated_at.nil?
+            return :unknown
+          end
+
+          if r.updated_at.nil?
+            return :unknown
+          end
+
+          if l.updated_at < r.updated_at
+            return :left_older
+          elsif l.updated_at > r.updated_at
+            return :right_older
+          else
+            return :equal
+          end
+        end
+      end
+    end
+  end
+end