etna 0.1.14 → 0.1.20

data/lib/etna/clients/metis/workflows.rb

@@ -0,0 +1,2 @@
+require_relative './workflows/metis_download_workflow'
+require_relative './workflows/metis_upload_workflow'

data/lib/etna/clients/metis/workflows/metis_download_workflow.rb

@@ -0,0 +1,37 @@
+require 'ostruct'
+require 'digest'
+require 'fileutils'
+require 'tempfile'
+
+module Etna
+  module Clients
+    class Metis
+      class MetisDownloadWorkflow < Struct.new(:metis_client, :project_name, :bucket_name, :max_attempts, keyword_init: true)
+
+        def initialize(args)
+          super({max_attempts: 3}.update(args))
+        end
+
+        # TODO: Might be possible to use range headers to select and resume downloads on failure in the future.
+        def do_download(dest_file, metis_file, &block)
+          size = metis_file.size
+          completed = 0.0
+          start = Time.now
+
+          ::File.open(dest_file, "w") do |io|
+            metis_client.download_file(metis_file) do |chunk|
+              io.write chunk
+              completed += chunk.size
+
+              block.call([
+                  :progress,
+                  size == 0 ? 1 : completed / size,
+                  (completed / (Time.now - start)).round(2),
+              ]) unless block.nil?
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/etna/clients/metis/workflows/metis_upload_workflow.rb

@@ -0,0 +1,137 @@
+require 'ostruct'
+require 'digest'
+require 'fileutils'
+require 'tempfile'
+require 'securerandom'
+
+module Etna
+  module Clients
+    class Metis
+      class MetisUploadWorkflow < Struct.new(:metis_client, :metis_uid, :project_name, :bucket_name, :max_attempts, keyword_init: true)
+
+        def initialize(args)
+          super({max_attempts: 3, metis_uid: SecureRandom.hex}.update(args))
+        end
+
+        def do_upload(source_file, dest_path, &block)
+          upload = Upload.new(source_file: source_file)
+
+          dir = ::File.dirname(dest_path)
+          metis_client.create_folder(CreateFolderRequest.new(
+              project_name: project_name,
+              bucket_name: bucket_name,
+              folder_path: dir,
+          ))
+
+          authorize_response = metis_client.authorize_upload(AuthorizeUploadRequest.new(
+              project_name: project_name,
+              bucket_name: bucket_name,
+              file_path: dest_path,
+          ))
+
+          upload_parts(upload, authorize_response.upload_path, &block)
+        end
+
+        private
+
+        def upload_parts(upload, upload_path, attempt_number = 1, reset = false, &block)
+          if attempt_number > max_attempts
+            raise "Upload failed after (#{attempt_number}) attempts."
+          end
+
+          start = Time.now
+          unsent_zero_byte_file = upload.current_byte_position == 0
+
+          upload.resume_from!(metis_client.upload_start(UploadStartRequest.new(
+              upload_path: upload_path,
+              file_size: upload.file_size,
+              next_blob_size: upload.next_blob_size,
+              next_blob_hash: upload.next_blob_hash,
+              metis_uid: metis_uid,
+              reset: reset,
+          )))
+
+          until upload.complete? && !unsent_zero_byte_file
+            begin
+              blob_bytes = upload.next_blob_bytes
+              byte_position = upload.current_byte_position
+
+              upload.advance_position!
+              metis_client.upload_blob(UploadBlobRequest.new(
+                  upload_path: upload_path,
+                  next_blob_size: upload.next_blob_size,
+                  next_blob_hash: upload.next_blob_hash,
+                  blob_data: StringIO.new(blob_bytes),
+                  metis_uid: metis_uid,
+                  current_byte_position: byte_position,
+              ))
+
+              unsent_zero_byte_file = false
+            rescue Etna::Error => e
+              m = yield [:error, e] unless block.nil?
+              if m == false
+                raise e
+              end
+
+              if e.status == 422
+                return upload_parts(upload, upload_path, attempt_number + 1, true, &block)
+              elsif e.status >= 500
+                return upload_parts(upload, upload_path, attempt_number + 1, &block)
+              end
+
+              raise e
+            end
+
+            yield [
+                :progress,
+                upload.file_size == 0 ? 1.0 : upload.current_byte_position.to_f / upload.file_size,
+                (upload.current_byte_position / (Time.now - start).to_f).round(2),
+            ] unless block.nil?
+          end
+        end
+
+        class Upload < Struct.new(:source_file, :next_blob_size, :current_byte_position, keyword_init: true)
+          INITIAL_BLOB_SIZE = 2 ** 10
+          MAX_BLOB_SIZE = 2 ** 22
+          ZERO_HASH = 'd41d8cd98f00b204e9800998ecf8427e'
+
+          def initialize(**args)
+            super
+            self.next_blob_size = [file_size, INITIAL_BLOB_SIZE].min
+            self.current_byte_position = 0
+          end
+
+          def file_size
+            ::File.size(source_file)
+          end
+
+          def advance_position!
+            self.current_byte_position = self.current_byte_position + self.next_blob_size
+            self.next_blob_size = [
+                MAX_BLOB_SIZE,
+                # in fact we should stop when we hit the end of the file
+                file_size - current_byte_position
+            ].min
+          end
+
+          def complete?
+            current_byte_position >= file_size
+          end
+
+          def next_blob_hash
+            Digest::MD5.hexdigest(next_blob_bytes)
+          end
+
+          def next_blob_bytes
+            IO.binread(source_file, next_blob_size, current_byte_position)
+          end
+
+          def resume_from!(upload_response)
+            self.current_byte_position = upload_response.current_byte_position
+            self.next_blob_size = upload_response.next_blob_size
+          end
+        end
+      end
+    end
+  end
+end

data/lib/etna/clients/polyphemus.rb

@@ -0,0 +1,3 @@
+require_relative './polyphemus/client'
+require_relative './polyphemus/models'
+require_relative './polyphemus/workflows'

data/lib/etna/clients/polyphemus/client.rb

@@ -0,0 +1,33 @@
+require 'net/http/persistent'
+require 'net/http/post/multipart'
+require 'singleton'
+require_relative '../../client'
+require_relative './models'
+
+module Etna
+  module Clients
+    class Polyphemus
+      def initialize(host:, token:, ignore_ssl: false, persistent: true)
+        raise 'Polyphemus client configuration is missing host.' unless host
+        raise 'Polyphemus client configuration is missing token.' unless token
+        @etna_client = ::Etna::Client.new(
+          host,
+          token,
+          routes_available: false,
+          persistent: persistent,
+          ignore_ssl: ignore_ssl)
+      end
+
+      def configuration(configuration_request = ConfigurationRequest.new)
+        json = nil
+        @etna_client.get(
+          "/configuration",
+          configuration_request) do |res|
+          json = JSON.parse(res.body, symbolize_names: true)
+        end
+
+        ConfigurationResponse.new(json)
+      end
+    end
+  end
+end

data/lib/etna/clients/polyphemus/models.rb

@@ -0,0 +1,68 @@
+require 'ostruct'
+require_relative '../../json_serializable_struct'
+
+# TODO:  In the near future, I'd like to transition to specifying apis via SWAGGER and generating model stubs from the
+# common definitions.  For nowe I've written them out by hand here.
+module Etna
+  module Clients
+    class Polyphemus
+      class ConfigurationRequest
+        def map
+          []
+        end
+      end
+
+      class ConfigurationResponse
+        attr_reader :raw
+
+        def initialize(raw = '')
+          @raw = raw
+        end
+
+        def environment
+          @raw.keys.first
+        end
+
+        def environments
+          @raw.keys
+        end
+
+        def environment_configuration(env = environment)
+          EnvironmentConfiguration.new(@raw[env])
+        end
+      end
+
+      class EnvironmentConfiguration
+        attr_reader :raw
+
+        def initialize(raw = {})
+          @raw = raw
+        end
+
+        def magma
+          @raw['magma']
+        end
+
+        def metis
+          @raw['magma']
+        end
+
+        def janus
+          @raw['magma']
+        end
+
+        def timur
+          @raw['magma']
+        end
+
+        def polyphemus
+          @raw['magma']
+        end
+
+        def auth_redirect
+          @raw['auth_redirect']
+        end
+      end
+    end
+  end
+end

data/lib/etna/clients/polyphemus/workflows.rb

@@ -0,0 +1 @@
+require_relative './workflows/set_configuration_workflow'

data/lib/etna/clients/polyphemus/workflows/set_configuration_workflow.rb

@@ -0,0 +1,47 @@
+# Given an environment (i.e. test, development, staging),
+#   and a polyphemus host, fetches the configuration
+#   hash and sets it up in a local config file,
+#   ~/.etna.json by default.
+
+# Adds a model to a project, from a JSON file.
+# This workflow:
+# 1) Validates the model JSON file.
+# 2) Validates that the model parent and any link attributes exist in Magma.
+# 3) Adds the model to the Magma project.
+# 4) Adds any attributes to Magma.
+
+require 'json'
+require 'yaml'
+require 'ostruct'
+
+module Etna
+  module Clients
+    class Polyphemus
+      class SetConfigurationWorkflow < Struct.new(:polyphemus_client, :config_file, keyword_init: true)
+        def fetch_configuration
+          polyphemus_client.configuration
+        end
+
+        def update_configuration_file(**additional_config)
+          if File.exist?(config_file)
+            etna_config = YAML.load_file(config_file) || {}
+          else
+            etna_config = {}
+          end
+
+          config = polyphemus_client.configuration
+          env = config.environment.to_sym
+
+          # Ensure that env is the last key in the result, which becomes the new 'default' config.
+          etna_config.delete(env) if etna_config.include?(env)
+          env_config = config.environment_configuration.raw.dup
+          env_config.update(additional_config)
+          etna_config.update({ env => env_config })
+
+          File.open(config_file, 'w') { |f| YAML.dump(etna_config, f) }
+          config
+        end
+      end
+    end
+  end
+end

data/lib/etna/command.rb

@@ -1,26 +1,264 @@
+require 'rollbar'
+
+# Commands resolution works by starting from a class that includes CommandExecutor, and searches for
+# INNER (belong to their scope) classes that include :find_command (generally subclasses of Etna::Command or
+# classes that include CommandExecutor).
+# A command completes the find_command chain by returning itself, while CommandExecutors consume one of the arguments
+# to dispatch to a subcommand.
+# eg of subcommands
+#
+# ./bin/my_app my_command help
+# ./bin/my_app my_command subcommand
+# class MyApp < Etna::Application
+#   class MyCommand
+#     include Etna::CommandExecutor
+#
+#     class Subcommand < Etna::Command
+#       def execute
+#       end
+#     end
+#   end
+# end
+#
 module Etna
-  class Command
-    class << self
+  # Provides the usage DSL and method that includes information about a CommandExecutor or a Command.
+  # Commands or CommandExecutors can call 'usage' in their definition to provide a specific description
+  # to be given about that command.  By default, the command name + desc method will be shown.
+  module CommandOrExecutor
+    module Dsl
       def usage(desc)
         define_method :usage do
-          "  #{"%-30s" % name}#{desc}"
+          "  #{"%-45s" % command_name}#{desc}"
         end
       end
+
+      def boolean_flags
+        @boolean_flags ||= []
+      end
+
+      def string_flags
+        @string_flags ||= []
+      end
+    end
+
+    def flag_as_parameter(flag)
+      flag.gsub('--', '').gsub('-', '_')
     end
 
-    def name
-      self.class.name.snake_case.split(/::/).last.to_sym
+    def parse_flags(*args)
+      new_args = []
+      flags = {}
+      found_non_flag = false
+
+      until args.empty?
+        next_arg = args.shift
+
+        unless next_arg.start_with? '--'
+          new_args << next_arg
+          found_non_flag = true
+          next
+        end
+
+        arg_name = flag_as_parameter(next_arg).to_sym
+
+        if self.class.boolean_flags.include?(next_arg)
+          flags[arg_name] = true
+        elsif self.class.string_flags.include?(next_arg)
+          if args.empty?
+            raise "flag #{next_arg} requires an argument"
+          else
+            flags[arg_name] = args.shift
+          end
+        elsif !found_non_flag
+          raise "#{program_name} does not recognize flag #{next_arg}"
+        else
+          new_args << next_arg
+        end
+      end
+
+      [flags, new_args]
+    end
+
+    def completions_for(parameter)
+      if parameter == 'env' || parameter == 'environment' || parameter =~ /_env/
+        ['production', 'staging', 'development']
+      else
+        ["__#{parameter}__"]
+      end
+    end
+
+    def command_name
+      self.class.name.snake_case.split(/::/).last
+    end
+
+    def usage
+      "  #{"%-45s" % command_name}#{desc}"
+    end
+
+    def flag_argspec
+      @argspec ||= []
+    end
+
+    # Program name is used to compose the usage display.  The top level executor will just assume the running
+    # process's bin name, while other executors will use the 'command name' of the Command / Exectuor
+    # (class name derived) and whatever program_name exists for its parent scope.
+    def program_name
+      if parent.nil? || !parent.respond_to?(:program_name)
+        $PROGRAM_NAME
+      else
+        parent.program_name + " " + command_name
+      end
+    end
+
+    # By default, the description of a command maps the execute parameters into CLI descriptions,
+    # where as CommandExecutors will display their subcommands.
+    def desc
+      if respond_to?(:execute)
+        method(:execute).parameters.map do |type, name|
+          name = "..." if name.nil?
+
+          case type
+          when :req
+            "<#{name}>"
+          when :opt
+            "[<#{name}>]"
+          when :rest
+            "<#{name}>..."
+          when :keyrest
+            "[flags...]"
+          when :key
+            flag = "--#{name.to_s.gsub('_', '-')}"
+            if self.class.boolean_flags.include?(flag)
+              "[#{flag}]"
+            else
+              "[#{flag} <#{name}>]"
+            end
+          else
+            raise "Invalid command execute argument specification #{type}, unsure how to format description."
+          end
+        end.join(' ')
+      elsif respond_to?(:subcommands)
+        '<command> <args>...'
+      end
+    end
+
+    def self.included(cls)
+      cls.extend(Dsl)
+    end
+  end
+
+  # Include this module into class that dispatches to child CommandExecutors | Commands that must exist as
+  # inner classes.  Note that non-root CommandExecutors must accept and set their @parent just like commands.
+  module CommandExecutor
+    attr_reader :parent
+
+    def initialize(parent = nil)
+      super()
+      @parent = parent
+    end
+
+    def self.included(cls)
+      cls.include(CommandOrExecutor)
+      cls.const_set(:Help, Class.new(Etna::Command) do
+        usage 'List this help'
+
+        def execute
+          self.parent.help
+        end
+      end) unless cls.const_defined?(:Help)
+    end
+
+    def help
+      puts "usage: #{program_name} #{desc}"
+      subcommands.each do |name, cmd|
+        puts cmd.usage
+      end
+    end
+
+    def find_command(*args, **kwds)
+      flags, args = parse_flags(*args)
+      dispatch_to_subcommand(*args, **(kwds.update(flags)))
+    end
+
+    def dispatch_to_subcommand(cmd = 'help', *args, **kwds)
+      unless subcommands.include?(cmd)
+        cmd = 'help'
+        args = []
+      end
+
+      subcommands[cmd].find_command(*args, **kwds)
+    end
+
+    def subcommands
+      @subcommands ||= self.class.constants.sort.reduce({}) do |acc, n|
+        acc.tap do
+          c = self.class.const_get(n)
+          next unless c.instance_methods.include?(:find_command)
+          v = c.new(self)
+          acc[v.command_name] = v
+        end
+      end
+    end
+
+    def all_subcommands
+      subcommands.values + (subcommands.values.map { |s| s.respond_to?(:all_subcommands) ? s.all_subcommands : [] }.flatten)
+    end
+  end
+
+  class Command
+    include CommandOrExecutor
+
+    attr_reader :parent
+
+    def initialize(parent = nil)
+      @parent = parent
+    end
+
+    def self.parent_scope
+      parts = self.name.split('::')
+      parts.pop
+      Kernel.const_get(parts.join('::'))
+    end
+
+    def find_command(*args, **kwds)
+      flags, args = parse_flags(*args)
+      [self, args, kwds.update(flags)]
+    end
+
+    def fill_in_missing_params(args)
+      req_params = method(:execute).parameters.select { |type, name| type == :req }
+      args + (req_params[(args.length)..(req_params.length)] || []).map do |type, name|
+        puts "#{name}?"
+        STDIN.gets.chomp
+      end
+    end
+
+    def completions
+      method(:execute).parameters.map do |type, name|
+        name = "..." if name.nil?
+        if type == :req || type == :opt
+          [completions_for(name)]
+        else
+          []
+        end
+      end.inject([], &:+)
     end
 
     # To be overridden during inheritance.
     def execute
       raise 'Command is not implemented'
+    rescue => e
+      Rollbar.error(e)
+      raise
     end
 
     # To be overridden during inheritance, to e.g. connect to a database.
     # Should be called with super by inheriting method.
     def setup(config)
       Etna::Application.find(self.class).configure(config)
+    rescue => e
+      Rollbar.error(e)
+      raise
     end
   end
 end