sfn 0.0.1 → 0.3.0

data/lib/sfn/provider.rb

@@ -0,0 +1,224 @@
+require 'logger'
+require 'sfn'
+
+module Sfn
+  # Remote provider interface
+  class Provider
+
+    include Bogo::AnimalStrings
+
+    # Minimum number of seconds to wait before re-expanding in
+    # progress stack
+    STACK_EXPAND_INTERVAL = 45
+
+    # Default interval for refreshing stack list in cache
+    STACK_LIST_INTERVAL = 120
+
+    # @return [Miasma::Models::Orchestration]
+    attr_reader :connection
+    # @return [Cache]
+    attr_reader :cache
+    # @return [Thread, NilClass] stack list updater
+    attr_accessor :updater
+    # @return [TrueClass, FalseClass] async updates
+    attr_reader :async
+    # @return [Logger, NilClass] logger in use
+    attr_reader :logger
+    # @return [Numeric] interval between stack expansions
+    attr_reader :stack_expansion_interval
+    # @return [Numeric] interval between stack list updates
+    attr_reader :stack_list_interval
+
+    # Create new instance
+    #
+    # @param args [Hash]
+    # @option args [Hash] :miasma miasma connection hash
+    # @option args [Cache] :cache
+    # @option args [TrueClass, FalseClass] :async fetch stacks async (defaults true)
+    # @option args [Logger] :logger use custom logger
+    # @option args [Numeric] :stack_expansion_interval interval to wait between stack data expands
+    # @option args [Numeric] :stack_list_interval interval to wait between stack list refresh
+    def initialize(args={})
+      args = args.to_smash
+      unless(args[:miasma][:provider])
+        best_guess = args[:miasma].keys.group_by do |key|
+          key.to_s.split('_').first
+        end.sort do |x, y|
+          y.size <=> x.size
+        end.first
+        if(best_guess)
+          provider = best_guess.first.to_sym
+        else
+          raise ArgumentError.new 'Cannot auto determine :provider value for credentials'
+        end
+      else
+        provider = args[:miasma].delete(:provider).to_sym
+      end
+      if(provider == :aws)
+        if(args[:miasma][:region])
+          args[:miasma][:aws_region] = args[:miasma].delete(:region)
+        end
+      end
+      if(ENV['DEBUG'].to_s.downcase == 'true')
+        log_to = STDOUT
+      else
+        if(Gem.win_platform?)
+          log_to = 'NUL'
+        else
+          log_to = '/dev/null'
+        end
+      end
+      @logger = args.fetch(:logger, Logger.new(log_to))
+      @stack_expansion_interval = args.fetch(:stack_expansion_interval, STACK_EXPAND_INTERVAL)
+      @stack_list_interval = args.fetch(:stack_list_interval, STACK_LIST_INTERVAL)
+      @connection = Miasma.api(
+        :provider => provider,
+        :type => :orchestration,
+        :credentials => args[:miasma]
+      )
+      @cache = args.fetch(:cache, Cache.new(:local))
+      @async = args.fetch(:async, true)
+      @miamsa_args = args[:miasma].dup
+      cache.init(:stacks_lock, :lock, :timeout => 0.1)
+      cache.init(:stacks, :stamped)
+      cache.init(:stack_expansion_lock, :lock, :timeout => 0.1)
+      if(args.fetch(:fetch, false))
+        async ? update_stack_list! : fetch_stacks
+      end
+    end
+
+    # @return [Miasma::Orchestration::Stacks]
+    def stacks
+      connection.stacks.from_json(cached_stacks)
+    end
+
+    # @return [String] json representation of cached stacks
+    def cached_stacks
+      fetch_stacks unless @initial_fetch_complete
+      value = cache[:stacks].value
+      value ? MultiJson.dump(MultiJson.load(value).values) : '[]'
+    end
+
+    # @return [Miasma::Orchestration::Stack, NilClass]
+    def stack(stack_id)
+      stacks.get(stack_id)
+    end
+
+    # Store stack attribute changes
+    #
+    # @param stack_id [String]
+    # @param stack_attributes [Hash]
+    # @return [TrueClass]
+    def save_expanded_stack(stack_id, stack_attributes)
+      current_stacks = MultiJson.load(cached_stacks)
+      cache.locked_action(:stacks_lock) do
+        logger.info "Saving expanded stack attributes in cache (#{stack_id})"
+        current_stacks[stack_id] = stack_attributes.merge('Cached' => Time.now.to_i)
+        cache[:stacks].value = MultiJson.dump(current_stacks)
+      end
+      true
+    end
+
+    # Remove stack from the cache
+    #
+    # @param stack_id [String]
+    # @return [TrueClass, FalseClass]
+    def remove_stack(stack_id)
+      current_stacks = MultiJson.load(cached_stacks)
+      logger.info "Attempting to remove stack from internal cache (#{stack_id})"
+      cache.locked_action(:stacks_lock) do
+        val = current_stacks.delete(stack_id)
+        logger.info "Successfully removed stack from internal cache (#{stack_id})"
+        cache[:stacks].value = MultiJson.dump(current_stacks)
+        !!val
+      end
+    end
+
+    # Expand all lazy loaded attributes within stack
+    #
+    # @param stack [Miasma::Models::Orchestration::Stack]
+    def expand_stack(stack)
+      logger.info "Stack expansion requested (#{stack.id})"
+      if((stack.in_progress? && Time.now.to_i - stack.attributes['Cached'].to_i > stack_expansion_interval) ||
+          !stack.attributes['Cached'])
+        begin
+          expanded = false
+          cache.locked_action(:stack_expansion_lock) do
+            expanded = true
+            stack.reload
+            stack.data['Cached'] = Time.now.to_i
+          end
+          if(expanded)
+            save_expanded_stack(stack.id, stack.to_json)
+          end
+        rescue => e
+          logger.error "Stack expansion failed (#{stack.id}) - #{e.class}: #{e}"
+        end
+      else
+        logger.info "Stack has been cached within expand interval. Expansion prevented. (#{stack.id})"
+      end
+    end
+
+    # Request stack information and store in cache
+    #
+    # @return [TrueClass]
+    def fetch_stacks
+      cache.locked_action(:stacks_lock) do
+        logger.info "Lock aquired for stack update. Requesting stacks from upstream. (#{Thread.current})"
+        stacks = Hash[
+          connection.stacks.reload.all.map do |stack|
+            [stack.id, stack.attributes]
+          end
+        ]
+        if(cache[:stacks].value)
+          existing_stacks = MultiJson.load(cache[:stacks].value)
+          # Force common types
+          stacks = MultiJson.load(MultiJson.dump(stacks))
+          # Remove stacks that have been deleted
+          stale_ids = existing_stacks.keys - stacks.keys
+          stacks = existing_stacks.to_smash.deep_merge(stacks)
+          stale_ids.each do |stale_id|
+            stacks.delete(stale_id)
+          end
+        end
+        cache[:stacks].value = stacks.to_json
+        logger.info 'Stack list has been updated from upstream and cached locally'
+      end
+      @initial_fetch_complete = true
+    end
+
+    # Start async stack list update. Creates thread that loops every
+    # `self.stack_list_interval` seconds and refreshes stack list in cache
+    #
+    # @return [TrueClass, FalseClass]
+    def update_stack_list!
+      if(updater.nil? || !updater.alive?)
+        self.updater = Thread.new{
+          loop do
+            begin
+              fetch_stacks
+              sleep(stack_list_interval)
+            rescue => e
+              logger.error "Failure encountered on stack fetch: #{e.class} - #{e}"
+            end
+          end
+        }
+        true
+      else
+        false
+      end
+    end
+
+    # Build API connection for service type
+    #
+    # @param service [String, Symbol]
+    # @return [Miasma::Model]
+    def service_for(service)
+      connection.api_for(service)
+    end
+
+  end
+end
+
+# Release the monkeys!
+Sfn::MonkeyPatch::Stack

data/lib/sfn/utils.rb

@@ -0,0 +1,23 @@
+require 'sfn'
+
+module Sfn
+  # Utility classes and modules
+  module Utils
+
+    autoload :Output, 'sfn/utils/output'
+    autoload :StackParameterValidator, 'sfn/utils/stack_parameter_validator'
+    autoload :StackParameterScrubber, 'sfn/utils/stack_parameter_scrubber'
+    autoload :StackExporter, 'sfn/utils/stack_exporter'
+    autoload :Debug, 'sfn/utils/debug'
+    autoload :JSON, 'sfn/utils/json'
+    autoload :Ssher, 'sfn/utils/ssher'
+    autoload :ObjectStorage, 'sfn/utils/object_storage'
+    autoload :PathSelector, 'sfn/utils/path_selector'
+
+    # Provide methods directly from module for previous version compatibility
+    extend JSON
+    extend ObjectStorage
+    extend Bogo::AnimalStrings
+
+  end
+end

data/lib/sfn/utils/debug.rb

@@ -0,0 +1,31 @@
+require 'sfn'
+
+module Sfn
+  module Utils
+    # Debug helpers
+    module Debug
+      # Output helpers
+      module Output
+        # Write debug message
+        #
+        # @param msg [String]
+        def debug(msg)
+          puts "<SparkleFormation>: #{msg}" if ENV['DEBUG']
+        end
+      end
+
+      class << self
+        # Load module into class
+        #
+        # @param klass [Class]
+        def included(klass)
+          klass.class_eval do
+            include Output
+            extend Output
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/sfn/utils/json.rb

@@ -0,0 +1,37 @@
+require 'sfn'
+
+module Sfn
+  module Utils
+
+    # JSON helper methods
+    module JSON
+
+      # Convert to JSON
+      #
+      # @param thing [Object]
+      # @return [String]
+      def _to_json(thing)
+        MultiJson.dump(thing)
+      end
+
+      # Load JSON data
+      #
+      # @param thing [String]
+      # @return [Object]
+      def _from_json(thing)
+        MultiJson.load(thing)
+      end
+
+      # Format object into pretty JSON
+      #
+      # @param thing [Object]
+      # @return [String]
+      def _format_json(thing)
+        thing = _from_json(thing) if thing.is_a?(String)
+        MultiJson.dump(thing, :pretty => true)
+      end
+
+    end
+
+  end
+end

data/lib/sfn/utils/object_storage.rb

@@ -0,0 +1,28 @@
+require 'sfn'
+
+module Sfn
+  module Utils
+
+    # Storage helpers
+    module ObjectStorage
+
+      # Write to file
+      #
+      # @param object [Object]
+      # @param path [String] path to write object
+      # @param directory [Miasma::Models::Storage::Directory]
+      # @return [String] file path
+      def file_store(object, path, directory)
+        raise NotImplementedError.new 'Internal updated required! :('
+        content = object.is_a?(String) ? object : Utils._format_json(object)
+        directory.files.create(
+          :identity => path,
+          :body => content
+        )
+        loc = directory.service.service.name.split('::').last.downcase
+        "#{loc}://#{directory.identity}/#{path}"
+      end
+
+    end
+  end
+end

data/lib/sfn/utils/output.rb

@@ -0,0 +1,79 @@
+require 'sfn'
+
+module Sfn
+  module Utils
+    # Output Helpers
+    module Output
+
+      # Process things and return items
+      #
+      # @param things [Array] items to process
+      # @param args [Hash] options
+      # @option args [TrueClass, FalseClass] :flat flatten result array
+      # @option args [Array] :attributes attributes to extract
+      # @todo this was extracted from events and needs to be cleaned up
+      def process(things, args={})
+        @event_ids ||= []
+        processed = things.reverse.map do |thing|
+          next if @event_ids.include?(thing['id'])
+          @event_ids.push(thing['id']).compact!
+          if(args[:attributes])
+            args[:attributes].map do |key|
+              thing[key].to_s
+            end
+          else
+            thing.values
+          end
+        end
+        args[:flat] ? processed.flatten : processed
+      end
+
+      # Generate formatted titles
+      #
+      # @param thing [Object] thing being processed
+      # @param args [Hash]
+      # @option args [Array] :attributes
+      # @return [Array<String>] formatted titles
+      def get_titles(thing, args={})
+        attrs = args[:attributes] || []
+        if(attrs.empty?)
+          hash = thing.is_a?(Array) ? thing.first : thing
+          hash ||= {}
+          attrs = hash.keys
+        end
+        titles = attrs.map do |key|
+          camel(key).gsub(/([a-z])([A-Z])/, '\1 \2')
+        end.compact
+        if(args[:format])
+          titles.map{|s| @ui.color(s, :bold)}
+        else
+          titles
+        end
+      end
+
+      # Output stack related things in nice format
+      #
+      # @param stack [String] name of stack
+      # @param things [Array] things to display
+      # @param what [String] description of things for output
+      # @param args [Symbol] options (:ignore_empty_output)
+      def things_output(stack, things, what, *args)
+        unless(args.include?(:no_title))
+          output = get_titles(things, :format => true, :attributes => allowed_attributes)
+        else
+          output = []
+        end
+        columns = allowed_attributes.size
+        output += process(things, :flat => true, :attributes => allowed_attributes)
+        output.compact!
+        if(output.empty?)
+          ui.warn 'No information found' unless args.include?(:ignore_empty_output)
+        else
+          ui.info "#{what.to_s.capitalize} for stack: #{ui.color(stack, :bold)}" if stack
+          ui.info "#{ui.list(output, :uneven_columns_across, columns)}"
+        end
+      end
+
+    end
+  end
+end

data/lib/sfn/utils/path_selector.rb

@@ -0,0 +1,99 @@
+require 'sfn'
+
+module Sfn
+  module Utils
+
+    # Helper methods for path selection
+    module PathSelector
+
+      # Humanize the base name of path
+      #
+      # @param path [String]
+      # @return [String]
+      def humanize_path_basename(path)
+        File.basename(path).sub(
+          File.extname(path), ''
+        ).split(/[-_]/).map(&:capitalize).join(' ')
+      end
+
+      # Prompt user for file selection
+      #
+      # @param directory [String] path to directory
+      # @param opts [Hash] options
+      # @option opts [Array<String>] :ignore_directories directory names
+      # @option opts [String] :directories_name title for directories
+      # @option opts [String] :files_name title for files
+      # @option opts [String] :filter_prefix only return results matching filter
+      # @return [String] file path
+      def prompt_for_file(directory, opts={})
+        file_list = Dir.glob(File.join(directory, '**', '**', '*')).find_all do |file|
+          File.file?(file)
+        end
+        if(opts[:filter_prefix])
+          file_list = file_list.find_all do |file|
+            file.start_with?(options[:filter_prefix])
+          end
+        end
+        directories = file_list.map do |file|
+          File.dirname(file)
+        end.uniq
+        files = file_list.find_all do |path|
+          path.sub(directory, '').split('/').size == 2
+        end
+        if(opts[:ignore_directories])
+          directories.delete_if do |dir|
+            opts[:ignore_directories].include?(File.basename(dir))
+          end
+        end
+        if(directories.empty? && files.empty?)
+          ui.fatal 'No formation paths discoverable!'
+        else
+          output = ['Please select an entry']
+          output << '(or directory to list):' unless directories.empty?
+          ui.info output.join(' ')
+          output.clear
+          idx = 1
+          valid = {}
+          unless(directories.empty?)
+            output << ui.color("#{opts.fetch(:directories_name, 'Directories')}:", :bold)
+            directories.each do |dir|
+              valid[idx] = {:path => dir, :type => :directory}
+              output << [idx, humanize_path_basename(dir)]
+              idx += 1
+            end
+          end
+          unless(files.empty?)
+            output << ui.color("#{opts.fetch(:files_name, 'Files')}:", :bold)
+            files.each do |file|
+              valid[idx] = {:path => file, :type => :file}
+              output << [idx, humanize_path_basename(file)]
+              idx += 1
+            end
+          end
+          max = idx.to_s.length
+          output.map! do |o|
+            if(o.is_a?(Array))
+              "  #{o.first}.#{' ' * (max - o.first.to_s.length)} #{o.last}"
+            else
+              o
+            end
+          end
+          ui.info "#{output.join("\n")}\n"
+          response = ask_question('Enter selection: ').to_i
+          unless(valid[response])
+            ui.fatal 'How about using a real value'
+            exit 1
+          else
+            entry = valid[response.to_i]
+            if(entry[:type] == :directory)
+              prompt_for_file(entry[:path], opts)
+            else
+              "/#{entry[:path]}"
+            end
+          end
+        end
+      end
+
+    end
+  end
+end