etna 0.1.46 → 0.1.48

data/lib/commands.rb

@@ -206,7 +206,7 @@ class EtnaApp
     end
   end
 
-  class Administrate
+  class Janus
     include Etna::CommandExecutor
 
     class Token
@@ -238,6 +238,59 @@ class EtnaApp
         end
       end
     end
+  end
+
+  class Magma
+    include Etna::CommandExecutor
+
+    class Materialize < Etna::Command
+      include WithEtnaClients
+
+      string_flags << "--project-name"
+      string_flags << "--log-file"
+      string_flags << "--log-level"
+      string_flags << "--concurrency"
+
+
+      def execute(project_name:, log_file:'/dev/stdout', log_level: ::Logger::INFO, concurrency: 1)
+        logger = Etna::Logger.new(log_file, 0, 1048576)
+
+        logger.level = log_level
+
+        workflow = Etna::Clients::Magma::MaterializeDataWorkflow.new(
+            model_attributes_mask: model_attribute_pairs(project_name),
+            record_names: 'all',
+            model_filters: {},
+            metis_client: metis_client(logger: logger),
+            magma_client: magma_client(logger: logger),
+            logger: logger,
+            project_name: project_name,
+            model_name: 'project', filesystem: filesystem,
+            concurrency: concurrency.to_i)
+
+        workflow.materialize_all(project_name)
+        logger.info("Done")
+      end
+
+      def model_attribute_pairs(project_name)
+        models = magma_client.retrieve(Etna::Clients::Magma::RetrievalRequest.new(
+            project_name: project_name,
+            model_name: 'all',
+            attribute_names: 'all',
+            record_names: []
+        )).models
+
+        result = models.model_keys.map do |model_name|
+          [ model_name, models.model(model_name).template.attributes.attribute_keys ]
+        end.to_h
+
+        result
+      end
+
+      def filesystem
+        @filesystem ||= Etna::Filesystem.new
+      end
+    end
 
     class Project
       include Etna::CommandExecutor

data/lib/etna/application.rb

@@ -68,38 +68,43 @@ module Etna::Application
     [path.map(&:to_sym), YAML.load(File.read(value))]
   end
 
+  def controller_tags
+    [:controller, :action, :project_name, :user_hash]
+  end
+
   def setup_yabeda
     application = self.id
+    ctags = self.controller_tags
     Yabeda.configure do
       default_tag :application, application
+      default_tag :project_name, 'unknown'
+      default_tag :controller, 'none'
+      default_tag :action, 'none'
+      default_tag :user_hash, 'unknown'
 
       group :etna do
         histogram :response_time do
           comment "Time spent by a controller returning a response"
           unit :seconds
-          tags [:controller, :action, :user_hash, :project_name]
+          tags ctags
+          buckets [0.01, 0.1, 0.3, 0.5, 1, 5]
+        end
+
+        histogram :perf do
+          comment "Time spent inside code path"
+          unit :seconds
+          tags ctags + [:class_name, :method_name]
           buckets [0.01, 0.1, 0.3, 0.5, 1, 5]
         end
 
         counter :visits do
           comment "Counts visits to the controller"
-          tags [:controller, :action, :user_hash, :project_name]
+          tags ctags
         end
 
         counter :rollbar_errors do
           comment "Counts errors detected by and sent to rollbar"
-        end
-
-        gauge :last_command_completion do
-          comment "Unix time of last time command was completed"
-          tags [:command, :status, :application]
-        end
-
-        histogram :command_runtime do
-          comment "Time spent processing a given command"
-          tags [:command, :status, :application]
-          unit :seconds
-          buckets [0.1, 1, 5, 60, 300, 1500]
+          tags ctags
         end
       end
     end
@@ -176,9 +181,6 @@ module Etna::Application
   end
 
   def run_command(config, *args, &block)
-    application = self.id
-    status = 'success'
-    start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
     cmd, cmd_args, cmd_kwds = find_command(*args)
 
     begin
@@ -190,18 +192,7 @@ module Etna::Application
       cmd.execute(*cmd.fill_in_missing_params(cmd_args), **cmd_kwds)
     rescue => e
       Rollbar.error(e)
-      status = 'failed'
       raise
-    ensure
-      if defined?(Yabeda) && Yabeda.configured?
-        tags = { command: cmd.class.name, status: status, application: application }
-        dur = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
-
-        Yabeda.etna.last_command_completion.set(tags, Time.now.to_i)
-        Yabeda.etna.command_runtime.measure(tags, dur)
-
-        write_job_metrics("run_command.#{cmd.class.name}")
-      end
     end
   end
 end

data/lib/etna/auth.rb

@@ -18,7 +18,7 @@ module Etna
       if [ approve_noauth(request), approve_hmac(request), approve_user(request) ].all?{|approved| !approved}
         return fail_or_redirect(request)
       end
-  
+
       @app.call(request.env)
     end
 
@@ -97,7 +97,7 @@ module Etna
 
       return payload unless route
 
-      begin      
+      begin
         permissions = permissions(payload)
 
         janus.resource_projects(token).each do |resource_project|

data/lib/etna/client.rb

@@ -4,10 +4,17 @@ require 'rack/utils'
 
 module Etna
   class Client
-    def initialize(host, token, routes_available: true, ignore_ssl: false)
+
+    def initialize(host, token, routes_available: true, ignore_ssl: false, max_retries: 10, backoff_time: 15, logger: nil)
       @host = host.sub(%r!/$!, '')
       @token = token
       @ignore_ssl = ignore_ssl
+      @max_retries = max_retries
+      @backoff_time = backoff_time
+
+      default_logger = ::Etna::Logger.new('/dev/stdout', 0, 1048576)
+      default_logger.level = ::Logger::WARN
+      @logger = logger || default_logger
 
       if routes_available
         set_routes
@@ -55,7 +62,7 @@ module Etna
       uri = request_uri(endpoint)
       multipart = Net::HTTP::Post::Multipart.new uri.request_uri, content
       multipart.add_field('Authorization', "Etna #{@token}")
-      request(uri, multipart, &block)
+      retrier.retry_request(uri, multipart, &block)
     end
 
     def post(endpoint, params = {}, &block)
@@ -80,9 +87,12 @@ module Etna
 
     private
 
+    def retrier
+      @retrier ||= Retrier.new(ignore_ssl: @ignore_ssl, max_retries: @max_retries, backoff_time: @backoff_time, logger: @logger)
+    end
+
     def set_routes
       response = options('/')
-      status_check!(response)
       @routes = JSON.parse(response.body, symbolize_names: true)
     end
 
@@ -120,7 +130,7 @@ module Etna
       uri = request_uri(endpoint)
       req = type.new(uri.request_uri, request_headers)
       req.body = params.to_json
-      request(uri, req, &block)
+      retrier.retry_request(uri, req, &block)
     end
 
     def query_request(type, endpoint, params = {}, &block)
@@ -132,7 +142,7 @@ module Etna
         uri.query = URI.encode_www_form(params)
       end
       req = type.new(uri.request_uri, request_headers)
-      request(uri, req, &block)
+      retrier.retry_request(uri, req, &block)
     end
 
     def request_uri(endpoint)
@@ -140,6 +150,8 @@ module Etna
     end
 
     def request_headers
+      refresh_token
+
       {
           'Content-Type' => 'application/json',
           'Accept' => 'application/json, text/*',
@@ -149,46 +161,234 @@ module Etna
       )
     end
 
-    def status_check!(response)
-      status = response.code.to_i
-      if status >= 400
-        msg = response.content_type == 'application/json' ?
-            json_error(response.body) :
-            response.body
-        raise Etna::Error.new(msg, status)
-      end
+    def refresh_token
+      @token = TokenRefresher.new(@host, @token, @logger).active_token
     end
 
-    def json_error(body)
-      msg = JSON.parse(body, symbolize_names: true)
-      if (msg.has_key?(:errors) && msg[:errors].is_a?(Array))
-        return JSON.generate(msg[:errors])
-      elsif msg.has_key?(:error)
-        return JSON.generate(msg[:error])
+    class TokenRefresher
+      def initialize(host, token, logger)
+        @token = token
+        @host = host
+        @logger = logger
+      end
+
+      def active_token
+        token_will_expire? ?
+          refresh_token :
+          @token
+      end
+
+      private
+
+      def token_expired?
+        # Has the token already expired?
+        token_will_expire?(0)
+      end
+
+      def token_will_expire?(offset=3000)
+        return false if @token.nil?
+
+        # Will the user's token expire in the given amount of time?
+        payload = @token.split('.')[1]
+        return false if payload.nil?
+
+        epoch_seconds = JSON.parse(Base64.urlsafe_decode64(payload))["exp"]
+
+        return false if epoch_seconds.nil?
+
+        expiration = DateTime.strptime(epoch_seconds.to_s, "%s").to_time
+        expiration <= DateTime.now.new_offset.to_time + offset
+      end
+
+      def refresh_token
+        @logger.debug("Requesting a refreshed token.")
+        uri = refresh_uri
+        req = Net::HTTP::Post.new(uri.request_uri, request_headers)
+        retrier.retry_request(uri, req).body
+      end
+
+      def refresh_uri
+        URI("#{janus_host}#{refresh_endpoint}")
+      end
+
+      def janus_host
+        @host.gsub(/(metis|magma|timur|polyphemus|janus|gnomon)/, "janus")
+      end
+
+      def refresh_endpoint
+        "/api/tokens/generate"
+      end
+
+      def request_headers
+        {
+          'Content-Type' => 'application/json',
+          'Accept' => 'application/json, text/*',
+          'Authorization' => "Etna #{@token}"
+        }
+      end
+
+      def retrier
+        @retrier ||= Retrier.new(max_retries: 5, backoff_time: 10, logger: @logger)
       end
     end
 
-    def request(uri, data)
-      if block_given?
-        verify_mode = @ignore_ssl ?
-          OpenSSL::SSL::VERIFY_NONE :
-          OpenSSL::SSL::VERIFY_PEER
-        Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode, read_timeout: 300) do |http|
-          http.request(data) do |response|
+    class Retrier
+      # Ideally the retry code would be centralized with metis_client ...
+      #   unsure what would be the best approach to do that, at this moment.
+      def initialize(ignore_ssl: false, max_retries: 10, backoff_time: 15, logger:)
+        @max_retries = max_retries
+        @ignore_ssl = ignore_ssl
+        @backoff_time = backoff_time
+        @logger = logger
+      end
+
+      def retry_request(uri, req, retries: 0, &block)
+        retries += 1
+
+        begin
+          @logger.debug("\rWaiting for #{uri.host} restart"+"."*retries+"\x1b[0K")
+
+          sleep @backoff_time * retries
+        end if retries > 1
+
+        if retries < @max_retries
+          begin
+            if block_given?
+              request(uri, req) do |block_response|
+                if net_exceptions.include?(block_response.class)
+                  @logger.debug("Received #{block_response.class.name}, retrying")
+                  retry_request(uri, req, retries: retries, &block)
+                elsif block_response.is_a?(OpenSSL::SSL::SSLError)
+                  @logger.debug("SSL error, retrying")
+                  retry_request(uri, req, retries: retries, &block)
+                else
+                  status_check!(block_response)
+                  yield block_response
+                end
+              end
+            else
+              response = request(uri, req)
+            end
+          rescue OpenSSL::SSL::SSLError => e
+            if e.message =~ /write client hello/
+              @logger.debug("SSL error, retrying")
+              return retry_request(uri, req, retries: retries)
+            end
+            raise e
+          rescue *net_exceptions => e
+            @logger.debug("Received #{e.class.name}, retrying")
+            return retry_request(uri, req, retries: retries)
+          end
+
+          begin
+            retry_codes = ['503', '502', '504', '408']
+            if retry_codes.include?(response.code)
+              @logger.debug("Received response with code #{response.code}, retrying")
+              return retry_request(uri, req, retries: retries)
+            elsif response.code == '500' && response.body.start_with?("Puma caught")
+              @logger.debug("Received 500 Puma error #{response.body.split("\n").first}, retrying")
+              return retry_request(uri, req, retries: retries)
+            end
+
             status_check!(response)
-            yield response
+            return response
+          end unless block_given?
+        end
+
+        raise ::Etna::Error, "Could not contact server, giving up" unless block_given?
+      end
+
+      private
+
+      def request(uri, data)
+        if block_given?
+          verify_mode = @ignore_ssl ?
+            OpenSSL::SSL::VERIFY_NONE :
+            OpenSSL::SSL::VERIFY_PEER
+          Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode, read_timeout: 300) do |http|
+            http.request(data) do |response|
+              api_error_check!(response)
+              yield response
+            end
+          end
+        else
+          verify_mode = @ignore_ssl ?
+            OpenSSL::SSL::VERIFY_NONE :
+            OpenSSL::SSL::VERIFY_PEER
+          Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode, read_timeout: 300) do |http|
+            response = http.request(data)
+            api_error_check!(response)
+            return response
           end
         end
-      else
-        verify_mode = @ignore_ssl ?
-          OpenSSL::SSL::VERIFY_NONE :
-          OpenSSL::SSL::VERIFY_PEER
-        Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode, read_timeout: 300) do |http|
-          response = http.request(data)
-          status_check!(response)
-          return response
+      end
+
+      def api_error_check!(response)
+        status = response.code.to_i
+        raise_status_error(response) if 400 <= status && status < 500
+      end
+
+      def status_check!(response)
+        status = response.code.to_i
+        raise_status_error(response) if status >= 400
+      end
+
+      def raise_status_error(response)
+        msg = response.content_type == 'application/json' ?
+          json_error(response.body) :
+          response.body
+        raise Etna::Error.new(msg, response.code.to_i)
+      end
+
+      def json_error(body)
+        msg = JSON.parse(body, symbolize_names: true)
+        if (msg.has_key?(:errors) && msg[:errors].is_a?(Array))
+          return JSON.generate(msg[:errors])
+        elsif msg.has_key?(:error)
+          return JSON.generate(msg[:error])
         end
       end
+
+      def net_exceptions
+        retry_exceptions = [
+          Errno::ECONNREFUSED,
+          Errno::ECONNRESET,
+          Errno::ENETRESET,
+          Errno::EPIPE,
+          Errno::ECONNABORTED,
+          Errno::EHOSTDOWN,
+          Errno::EHOSTUNREACH,
+          Errno::EINVAL,
+          Errno::ETIMEDOUT,
+          Net::ReadTimeout,
+          Net::HTTPFatalError,
+          Net::HTTPBadResponse,
+          Net::HTTPHeaderSyntaxError,
+          Net::ProtocolError,
+          Net::HTTPRequestTimeOut,
+          Net::HTTPGatewayTimeOut,
+          Net::HTTPBadRequest,
+          Net::HTTPBadGateway,
+          Net::HTTPError,
+          Net::HTTPInternalServerError,
+          Net::HTTPRetriableError,
+          Net::HTTPServerError,
+          Net::HTTPServiceUnavailable,
+          Net::HTTPUnprocessableEntity,
+          Net::OpenTimeout,
+          IOError,
+          EOFError,
+          Timeout::Error
+        ]
+
+        begin
+          retry_exceptions << Net::HTTPRequestTimeout
+          retry_exceptions << Net::HTTPGatewayTimeout
+          retry_exceptions << Net::WriteTimeout
+        end if RUBY_VERSION > "2.5.8"
+
+        retry_exceptions
+      end
     end
   end
 end

data/lib/etna/clients/base_client.rb

@@ -6,7 +6,7 @@ module Etna
   module Clients
     class BaseClient
       attr_reader :host, :token, :ignore_ssl
-      def initialize(host:, token:, ignore_ssl: false)
+      def initialize(host:, token:, ignore_ssl: false, logger: nil)
         raise "#{self.class.name} client configuration is missing host." unless host
 
         @token = token
@@ -16,7 +16,8 @@ module Etna
           host,
           token,
           routes_available: false,
-          ignore_ssl: ignore_ssl)
+          ignore_ssl: ignore_ssl,
+          logger: logger)
         @host = host
         @ignore_ssl = ignore_ssl
       end

data/lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb

@@ -41,6 +41,7 @@ module Etna
 
             begin
               if (error = errors.pop(true))
+                logger&.error(error)
                 raise error
               end
             rescue ThreadError
@@ -51,6 +52,7 @@ module Etna
               begin
                 materialize_record(dest, template, document)
               rescue => e
+                logger&.error(e)
                 errors << e
               ensure
                 semaphore.release
@@ -62,6 +64,7 @@ module Etna
 
           begin
             if (error = errors.pop(true))
+              logger&.error(error)
               raise error
             end
           rescue ThreadError
@@ -105,6 +108,7 @@ module Etna
             end
 
             dest_file = File.join(dest_dir, metadata_file_name(record_name: record[template.identifier], record_model_name: template.name, ext: "_#{attr_name}_#{idx}#{File.extname(filename)}"))
+            filesystem.mkdir_p(File.dirname(dest_file))
             sync_metis_data_workflow.copy_file(dest: dest_file, url: url, stub: stub_files)
             record_to_serialize[attr_name] << {file: dest_file, original_filename: filename}
           end

data/lib/etna/clients/magma/workflows/walk_model_tree_workflow.rb

@@ -94,7 +94,6 @@ module Etna
               collections = []
               links = []
               attributes = []
-
               model = response.models.model(model_name)
 
               template.attributes.attribute_keys.each do |attr_name|

data/lib/etna/clients/metis/client.rb

@@ -10,10 +10,10 @@ module Etna
   module Clients
     class Metis < Etna::Clients::BaseClient
 
-      def initialize(host:, token:, ignore_ssl: false)
+      def initialize(host:, token:, ignore_ssl: false, logger: nil)
         raise 'Metis client configuration is missing host.' unless host
         raise 'Metis client configuration is missing token.' unless token
-        @etna_client = ::Etna::Client.new(host, token, ignore_ssl: ignore_ssl)
+        @etna_client = ::Etna::Client.new(host, token, ignore_ssl: ignore_ssl, logger: logger)
 
         @token = token
       end

data/lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb

@@ -22,36 +22,56 @@ module Etna
         end
 
         def copy_file(dest:, url:, stub: false)
-          # This does not work due to the magma bucket's restrictions, but if it did work, it'd be super sweet.
-          # url_match = DOWNLOAD_REGEX.match(url)
-          #
-          # if filesystem.instance_of?(Etna::Filesystem::Metis) && !url_match.nil?
-          #   bucket_name = url_match[:bucket_name]
-          #   project_name = url_match[:project_name]
-          #   file_path = url_match[:file_path]
-          #
-          #   metis_client.copy_files(
-          #     Etna::Clients::Metis::CopyFilesRequest.new(
-          #       project_name: project_name,
-          #       revisions: [
-          #         Etna::Clients::Metis::CopyRevision.new(
-          #           source: "metis://#{project_name}/#{bucket_name}/#{file_path}",
-          #           dest: "metis://#{filesystem.project_name}/#{filesystem.bucket_name}#{dest}",
-          #         )
-          #       ]
-          #     )
-          #   )
-          # end
+          url_match = DOWNLOAD_REGEX.match(url)
+
+          if filesystem.instance_of?(Etna::Filesystem::Metis) && !url_match.nil?
+            bucket_name = url_match[:bucket_name]
+            project_name = url_match[:project_name]
+            file_path = url_match[:file_path]
+
+            # ensure target parent directory exists
+            metis_client.ensure_parent_folder_exists(
+              project_name: filesystem.project_name,
+              bucket_name: filesystem.bucket_name,
+              path: dest
+            )
+
+            metis_client.copy_files(
+              Etna::Clients::Metis::CopyFilesRequest.new(
+                project_name: project_name,
+                revisions: [
+                  Etna::Clients::Metis::CopyRevision.new(
+                    source: "metis://#{project_name}/#{bucket_name}/#{file_path}",
+                    dest: "metis://#{filesystem.project_name}/#{filesystem.bucket_name}/#{dest}",
+                  )
+                ]
+              )
+            )
+
+            return
+          end
 
           metadata = metis_client.file_metadata(url)
           size = metadata[:size]
 
+          begin
+            if filesystem.exist?(dest) && filesystem.stat(dest).size == size
+              logger&.info "Already downloaded #{dest}"
+              return
+            end
+          rescue Etna::Filesystem::Error => e
+            unless e.message =~ /stat not supported/
+              raise e
+            end
+          end
+
           tmp_file = dest
           upload_timings = []
           upload_amount = 0
           last_rate = 0.00001
           remaining = size
 
+          logger&.info "Downloading #{dest} - #{Etna::Formatting.as_size(size)}"
           filesystem.with_writeable(tmp_file, "w", size_hint: size) do |io|
             if stub
               io.write("(stub) #{size} bytes")
@@ -78,7 +98,7 @@ module Etna
                 rate = upload_amount / (end_time - start_time)
 
                 if rate / last_rate > 1.3 || rate / last_rate < 0.7
-                  logger&.info("Uploading #{Etna::Formatting.as_size(rate)} per second, #{Etna::Formatting.as_size(remaining)} remaining")
+                  logger&.debug("Uploading #{Etna::Formatting.as_size(rate)} per second, #{Etna::Formatting.as_size(remaining)} remaining")
 
                   if rate == 0
                     last_rate = 0.0001
@@ -87,7 +107,6 @@ module Etna
                   end
                 end
               end
-
             end
           end
         end

data/lib/etna/command.rb

@@ -136,7 +136,7 @@ module Etna
             "<#{name}>..."
           when :keyrest
             "[flags...]"
-          when :key
+          when :key, :keyreq
             flag = "--#{name.to_s.gsub('_', '-')}"
             if self.class.boolean_flags.include?(flag)
               "[#{flag}]"