etna 0.1.41 → 0.1.44

data/lib/etna/clients/metis/models.rb

@@ -50,7 +50,25 @@ module Etna
         end
       end
 
-      class ListFolderRequest < Struct.new(:project_name, :bucket_name, :folder_path, keyword_init: true)
+      class FolderRequest < Struct.new(:project_name, :bucket_name, :folder_path, keyword_init: true)
+      end
+
+      class ListFolderRequest < FolderRequest
+        include JsonSerializableStruct
+
+        def initialize(**params)
+          super({}.update(params))
+        end
+
+        def to_h
+          # The :project_name comes in from Polyphemus as a symbol value,
+          #   we need to make sure it's a string because it's going
+          #   in the URL.
+          super().compact.transform_values(&:to_s)
+        end
+      end
+
+      class ListFolderByIdRequest < Struct.new(:project_name, :bucket_name, :folder_id, keyword_init: true)
         include JsonSerializableStruct
 
         def initialize(**params)
@@ -65,7 +83,7 @@ module Etna
         end
       end
 
-      class CreateFolderRequest < Struct.new(:project_name, :bucket_name, :folder_path, keyword_init: true)
+      class TouchFolderRequest < Struct.new(:project_name, :bucket_name, :folder_path, keyword_init: true)
         include JsonSerializableStruct
 
         def initialize(**params)
@@ -80,7 +98,37 @@ module Etna
         end
       end
 
-      class DeleteFolderRequest < Struct.new(:project_name, :bucket_name, :folder_path, keyword_init: true)
+      class TouchFileRequest < Struct.new(:project_name, :bucket_name, :file_path, keyword_init: true)
+        include JsonSerializableStruct
+
+        def initialize(**params)
+          super({}.update(params))
+        end
+
+        def to_h
+          # The :project_name comes in from Polyphemus as a symbol value,
+          #   we need to make sure it's a string because it's going
+          #   in the URL.
+          super().compact.transform_values(&:to_s)
+        end
+      end
+
+      class CreateFolderRequest < FolderRequest
+        include JsonSerializableStruct
+
+        def initialize(**params)
+          super({}.update(params))
+        end
+
+        def to_h
+          # The :project_name comes in from Polyphemus as a symbol value,
+          #   we need to make sure it's a string because it's going
+          #   in the URL.
+          super().compact.transform_values(&:to_s)
+        end
+      end
+
+      class DeleteFolderRequest < FolderRequest
         include JsonSerializableStruct
 
         def initialize(**params)
@@ -110,11 +158,11 @@ module Etna
         end
       end
 
-      class FindRequest < Struct.new(:project_name, :bucket_name, :limit, :offset, :params, keyword_init: true)
+      class FindRequest < Struct.new(:project_name, :bucket_name, :limit, :offset, :params, :hide_paths, keyword_init: true)
         include JsonSerializableStruct
 
         def initialize(**args)
-          super({params: []}.update(args))
+          super({params: [], hide_paths: false}.update(args))
         end
 
         def add_param(param)
@@ -126,6 +174,17 @@ module Etna
           #   easier to do from a JSON string
           JSON.parse(to_json, :symbolize_names => true)
         end
+
+        def clone
+          FindRequest.new(
+            project_name: self.project_name,
+            bucket_name: self.bucket_name,
+            limit: self.limit,
+            offset: self.offset,
+            params: self.params.dup,
+            hide_paths: self.hide_paths
+          )
+        end
       end
 
       class FindParam < Struct.new(:attribute, :predicate, :value, :type, keyword_init: true)
@@ -225,6 +284,13 @@ module Etna
           @raw = raw
         end
 
+        def with_containing_folder(folder)
+          folder_path = folder.is_a?(Folder) ? folder.folder_path : folder
+          File.new({}.update(self.raw).update({
+            file_path: ::File.join(folder_path, self.file_name)
+          }))
+        end
+
         def file_path
           raw[:file_path]
         end
@@ -259,6 +325,14 @@ module Etna
         def size
           raw[:size]
         end
+
+        def file_hash
+          raw[:file_hash]
+        end
+
+        def folder_id
+          raw[:folder_id]
+        end
       end
 
       class Folder
@@ -279,6 +353,19 @@ module Etna
         def bucket_name
           raw[:bucket_name]
         end
+
+        def project_name
+          raw[:project_name]
+        end
+
+        def updated_at
+          time = raw[:updated_at]
+          time.nil? ? nil : Time.parse(time)
+        end
+
+        def id
+          raw[:id]
+        end
       end
 
       class AuthorizeUploadRequest < Struct.new(:project_name, :bucket_name, :file_path, keyword_init: true)

data/lib/etna/clients/metis/workflows/metis_download_workflow.rb

@@ -18,7 +18,7 @@ module Etna
           completed = 0.0
           start = Time.now
 
-          unless dest_file_or_io.is_a?(IO)
+          unless dest_file_or_io.is_a?(IO) || dest_file_or_io.is_a?(StringIO)
             ::File.open(dest_file_or_io, 'w') do |io|
               return do_download(dest_file_or_io, metis_file, &block)
             end

data/lib/etna/clients/metis/workflows/metis_upload_workflow.rb

@@ -110,8 +110,6 @@ module Etna
 
           def initialize(source_file: nil, next_blob_size: nil, current_byte_position: nil)
             self.source_file = source_file
-            self.next_blob_size = next_blob_size
-            self.current_byte_position = current_byte_position
             self.next_blob_size = [file_size, INITIAL_BLOB_SIZE].min
             self.current_byte_position = 0
           end

data/lib/etna/clients/metis/workflows/walk_metis_diff_workflow.rb

@@ -0,0 +1,95 @@
+module Etna
+  module Clients
+    class Metis
+      class WalkMetisDiffWorkflow < Struct.new(:left_walker, :right_walker, keyword_init: true)
+        # Iterates entries of the form [kind, left | nil, right | nil]
+        # where kind is one of
+        #   :left_unique | :right_unique | :left_is_folder | :right_is_folder
+        #   :unknown | :equal | :right_older | :left_older
+        # and left / right is one of
+        #   nil | Etna::Clients::Metis::File | Etna::Clients::Metis::Folder
+        def each(&block)
+          left_enum = self.left_walker.to_enum
+          right_enum = self.right_walker.to_enum
+
+          l, l_path = next_or_nil(left_enum)
+          r, r_path = next_or_nil(right_enum)
+
+          while l && r
+            if l_path == r_path
+              yield [compare_file_or_folders(l, r), l, r]
+
+              l, l_path = next_or_nil(left_enum)
+              r, r_path = next_or_nil(right_enum)
+            elsif l_path < r_path
+              yield [:left_unique, l, nil]
+              l, l_path = next_or_nil(left_enum)
+            else
+              yield [:right_unique, nil, r]
+              r, r_path = next_or_nil(right_enum)
+            end
+          end
+
+          while l
+            yield [:left_unique, l, nil]
+            l, l_path = next_or_nil(left_enum)
+          end
+
+          while r
+            yield [:right_unique, nil, r]
+            r, r_path = next_or_nil(right_enum)
+          end
+        end
+
+        def next_or_nil(enum)
+          enum.next
+        rescue StopIteration
+          [nil, nil]
+        end
+
+        def compare_file_or_folders(l, r)
+          if l.is_a?(Etna::Clients::Metis::Folder)
+            if r.is_a?(Etna::Clients::Metis::Folder)
+              return compare_file_or_folder_age(l, r)
+            end
+
+            return :left_is_folder
+          end
+
+          if r.is_a?(Etna::Clients::Metis::Folder)
+            return :right_is_folder
+          end
+
+
+          if l.file_hash.nil? || r.file_hash.nil?
+            return :unknown
+          end
+
+          if l.file_hash == r.file_hash
+            return :equal
+          end
+
+          return compare_file_or_folder_age(l, r)
+        end
+
+        def compare_file_or_folder_age(l, r)
+          if l.updated_at.nil?
+            return :unknown
+          end
+
+          if r.updated_at.nil?
+            return :unknown
+          end
+
+          if l.updated_at < r.updated_at
+            return :left_older
+          elsif l.updated_at > r.updated_at
+            return :right_older
+          else
+            return :equal
+          end
+        end
+      end
+    end
+  end
+end

data/lib/etna/clients/metis/workflows/walk_metis_workflow.rb

@@ -0,0 +1,31 @@
+module Etna
+  module Clients
+    class Metis
+      class WalkMetisWorkflow < Struct.new(:metis_client, :project_name,
+          :bucket_name, :logger, :root_dir, keyword_init: true)
+        def each(&block)
+          q = [self.root_dir]
+
+          while (n = q.pop)
+            req = Etna::Clients::Metis::ListFolderRequest.new(
+              project_name: project_name,
+              bucket_name: bucket_name,
+              folder_path: n
+            )
+            next unless metis_client.folder_exists?(req)
+            resp = metis_client.list_folder(req)
+
+            resp.files.all.sort_by { |f| f.file_path[self.root_dir.length..-1] }.each do |file|
+              yield [file, file.file_path[self.root_dir.length..-1]]
+            end
+
+            resp.folders.all.sort_by { |f| f.folder_path[self.root_dir.length..-1] }.each do |f|
+              yield [f, f.folder_path[self.root_dir.length..-1]]
+              q << f.folder_path
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/etna/clients/metis/workflows.rb

@@ -2,3 +2,5 @@ require_relative "./workflows/metis_download_workflow"
 require_relative "./workflows/metis_upload_workflow"
 require_relative "./workflows/sync_metis_data_workflow"
 require_relative "./workflows/ingest_metis_data_workflow"
+require_relative "./workflows/walk_metis_workflow"
+require_relative "./workflows/walk_metis_diff_workflow"

data/lib/etna/controller.rb

@@ -42,7 +42,38 @@ module Etna
 
       [501, {}, ['This controller is not implemented.']]
     rescue Exception => e
-      handle_error(e)
+      error = e
+    ensure
+      log_request
+      return handle_error(error) if error
+    end
+
+    def try_stream(content_type, &block)
+      if @request.env['rack.hijack?']
+        @request.env['rack.hijack'].call
+        stream = @request.env['rack.hijack_io']
+
+        headers = [
+          "HTTP/1.1 200 OK",
+          "Content-Type: #{content_type}"
+        ]
+        stream.write(headers.map { |header| header + "\r\n" }.join)
+        stream.write("\r\n")
+        stream.flush
+
+        Thread.new do
+          block.call(stream)
+        ensure
+          stream.close
+        end
+
+        # IO is now streaming and will be processed by above thread.
+        @response.close
+      else
+        @response['Content-Type'] = content_type
+        block.call(@response)
+        @response.finish
+      end
     end
 
     def require_params(*params)
@@ -82,8 +113,36 @@ module Etna
       @response.finish
     end
 
+    def config_hosts
+      [:janus, :magma, :timur, :metis, :vulcan, :polyphemus].map do |host|
+        [ :"#{host}_host", @server.send(:application).config(host)&.dig(:host) ]
+      end.to_h.compact
+    end
+
     private
 
+    def redact_keys
+      @request.env['etna.redact_keys']
+    end
+
+    def add_redact_keys(new_redact_keys=[])
+      @request.env['etna.redact_keys'] = (@request.env['etna.redact_keys'] || []).concat(new_redact_keys)
+    end
+
+    def log_request
+      censor = Etna::Censor.new(redact_keys)
+
+      redacted_params = @params.map do |key,value|
+        [ key, censor.redact(key, value) ]
+      end.to_h
+
+      log("User #{@user ? @user.email : :unknown} calling #{controller_name}##{@action} with params #{redacted_params}")
+    end
+
+    def controller_name
+      self.class.name.sub("Kernel::", "").sub("Controller", "").downcase
+    end
+
     def success(msg, content_type='text/plain')
       @response['Content-Type'] = content_type
       @response.write(msg)

data/lib/etna/permissions.rb

@@ -0,0 +1,99 @@
+module Etna
+  class Permissions
+    def initialize(permissions)
+      @permissions = permissions
+    end
+
+    def self.from_encoded_permissions(encoded_permissions)
+      perms = encoded_permissions.split(/\;/).map do |roles|
+        role, projects = roles.split(/:/)
+
+        projects.split(/\,/).reduce([]) do |perms, project_name|
+          perms << Etna::Permission.new(role, project_name)
+        end
+      end.flatten
+
+      Etna::Permissions.new(perms)
+    end
+
+    def self.from_hash(permissions_hash)
+      perms = permissions_hash.map do |project_name, role_hash|
+        Etna::Permission.new(
+          Etna::Role.new(role_hash[:role], role_hash[:restricted]).key,
+          project_name
+        )
+      end
+
+      Etna::Permissions.new(perms)
+    end
+
+    def to_string
+      @permissions_string ||= @permissions.group_by(&:role_key)
+        .sort_by(&:first)
+        .map do |role_key, permissions|
+        [
+          role_key,
+          permissions.map(&:project_name).sort.join(","),
+        ].join(":")
+      end.join(";")
+    end
+
+    def to_hash
+      @permissions_hash ||= @permissions.map do |permission|
+        [permission.project_name, permission.to_hash]
+      end.to_h
+    end
+
+    def add_permission(permission)
+      return if current_project_names.include?(permission.project_name)
+
+      @permissions << permission
+    end
+
+    private
+
+    def current_project_names
+      @permissions.map(&:project_name)
+    end
+  end
+
+  class Permission
+    attr_reader :role, :project_name, :role_key
+
+    ROLE_NAMES = {
+      "A" => :admin,
+      "E" => :editor,
+      "V" => :viewer,
+    }
+
+    def initialize(role_key, project_name)
+      @role_key = role_key
+      @role = Etna::Role.new(ROLE_NAMES[role_key.upcase], role_key == role_key.upcase)
+      @project_name = project_name
+    end
+
+    def to_hash
+      role.to_hash
+    end
+  end
+
+  class Role
+    attr_reader :role, :restricted
+    def initialize(role, restricted)
+      @role = role
+      @restricted = restricted
+    end
+
+    def key
+      role_key = role.to_s[0]
+      restricted ? role_key.upcase : role_key
+    end
+
+    def to_hash
+      {
+        role: role,
+        restricted: restricted,
+      }
+    end
+  end
+end

data/lib/etna/route.rb

@@ -1,5 +1,6 @@
 require 'digest'
 require 'date'
+require_relative "./censor"
 
 module Etna
   class Route
@@ -13,6 +14,7 @@ module Etna
       @route = route.gsub(/\A(?=[^\/])/, '/')
       @block = block
       @match_ext = options[:match_ext]
+      @log_redact_keys = options[:log_redact_keys]
     end
 
     def to_hash
@@ -124,6 +126,8 @@ module Etna
         return [ 403, { 'Content-Type' => 'application/json' }, [ { error: 'You are forbidden from performing this action.' }.to_json ] ]
       end
 
+      request.env['etna.redact_keys'] = @log_redact_keys
+
       if @action
         controller, action = @action.split('#')
         controller_class = Kernel.const_get(
@@ -132,11 +136,6 @@ module Etna
         logger = request.env['etna.logger']
         user = request.env['etna.user']
 
-        params = request.env['rack.request.params'].map do |key,value|
-          [ key, redact(key, value) ]
-        end.to_h
-
-        logger.warn("User #{user ? user.email : :unknown} calling #{controller}##{action} with params #{params}")
         return controller_class.new(request, action).response
       elsif @block
         application = Etna::Application.find(app.class).class
@@ -155,41 +154,16 @@ module Etna
       @auth && @auth[:ignore_janus]
     end
 
+    def has_user_constraint?(constraint)
+      @auth && @auth[:user] && @auth[:user][constraint]
+    end
+
     private
 
     def application
       @application ||= Etna::Application.instance
     end
 
-    def compact(value)
-      value = value.to_s
-      value = value[0..500] + "..." + value[-100..-1] if value.length > 600
-      value
-    end
-
-    def redact_keys
-      @redact_keys ||= application.config(:log_redact_keys).split(",").map do |key|
-        key.to_sym
-      end
-    end
-
-    def redact(key, value)
-      # From configuration, redact any values for the supplied key values, so they
-      #   don't appear in the logs.
-      return compact(value) unless application.config(:log_redact_keys)
-
-      if value.is_a?(Hash)
-        redacted_value = value.map do |value_key, value_value|
-          [ value_key, redact(value_key, value_value) ]
-        end.to_h
-        return redacted_value
-      elsif redact_keys.include?(key)
-        return "*"
-      end
-
-      return compact(value)
-    end
-
     def authorized?(request)
       # If there is no @auth requirement, they are ok - this doesn't preclude
       # them being rejected in the controller response

data/lib/etna/spec/vcr.rb

@@ -1,9 +1,27 @@
+require 'cgi'
 require 'webmock/rspec'
 require 'vcr'
 require 'openssl'
 require 'digest/sha2'
 require 'base64'
 
+def clean_query(json_or_string)
+  if json_or_string.is_a?(Hash) && json_or_string.include?('upload_path')
+    json_or_string['upload_path'] = clean_query(json_or_string['upload_path'])
+    json_or_string
+  elsif json_or_string.is_a?(String)
+    uri = URI(json_or_string)
+
+    if uri.query&.include?('X-Etna-Signature')
+      uri.query = 'etna-signature'
+    end
+
+    uri.to_s
+  else
+    json_or_string
+  end
+end
+
 def setup_base_vcr(spec_helper_dir, server: nil, application: nil)
   VCR.configure do |c|
     c.hook_into :webmock
@@ -30,6 +48,10 @@ def setup_base_vcr(spec_helper_dir, server: nil, application: nil)
       end
     end
 
+    c.register_request_matcher :try_uri do |request_1, request_2|
+      clean_query(request_1.uri) == clean_query(request_2.uri)
+    end
+
     c.register_request_matcher :try_body do |request_1, request_2|
       if request_1.headers['Content-Type'].first =~ /application\/json/
         if request_2.headers['Content-Type'].first =~ /application\/json/
@@ -40,7 +62,7 @@ def setup_base_vcr(spec_helper_dir, server: nil, application: nil)
             JSON.parse(request_2.body) rescue 'not-json'
           end
 
-          request_1_json == request_2_json
+          clean_query(request_1_json) == clean_query(request_2_json)
         else
           false
         end
@@ -49,7 +71,9 @@ def setup_base_vcr(spec_helper_dir, server: nil, application: nil)
       end
     end
 
-    # c.debug_logger = File.open('log/vcr_debug.log', 'w')
+    if File.exists?('log')
+      c.debug_logger = File.open('log/vcr_debug.log', 'w')
+    end
 
     c.default_cassette_options = {
         serialize_with: :compressed,
@@ -58,7 +82,7 @@ def setup_base_vcr(spec_helper_dir, server: nil, application: nil)
         else
           ENV['RERECORD'] ? :all : :once
         end,
-        match_requests_on: [:method, :uri, :try_body, :verify_uri_route]
+        match_requests_on: [:method, :try_uri, :try_body, :verify_uri_route]
     }
 
     # Filter the authorization headers of any request by replacing any occurrence of that request's

data/lib/etna/synchronize_db.rb

@@ -0,0 +1,14 @@
+module Etna
+  class SynchronizeDb
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      # Do a coarse checkout of the connection
+      Etna::Application.instance.db.synchronize do
+        @app.call(env)
+      end
+    end
+  end
+end

data/lib/etna/test_auth.rb

@@ -31,6 +31,24 @@ module Etna
       end.to_h
     end
 
+    def update_payload(payload, token, request)
+      route = server.find_route(request)
+
+      payload = payload.map{|k,v| [k.to_sym, v]}.to_h
+
+      return payload unless route
+
+      begin      
+        permissions = Etna::Permissions.from_encoded_permissions(payload[:perm])
+
+        # Skip making an actual call to Janus. This behavior is tested in auth_spec
+
+        payload[:perm] = permissions.to_string
+      end if (!route.ignore_janus? && route.has_user_constraint?(:can_view?))
+
+      payload
+    end
+
     def approve_user(request)
       token = auth(request,:etna)
 
@@ -42,7 +60,10 @@ module Etna
       # We do this to support Metis client tests, we pass in tokens with multiple "."-separated parts, so
       #   have to account for that.
       payload = JSON.parse(Base64.decode64(token.split('.')[1]))
-      request.env['etna.user'] = Etna::User.new(payload.map{|k,v| [k.to_sym, v]}.to_h, token)
+      request.env['etna.user'] = Etna::User.new(
+        update_payload(payload, token, request),
+        token
+      )
     end
 
     def approve_hmac(request)