yle_tf 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e7b1c7b056ec918749df48648c4648f2c48f1ea1
-  data.tar.gz: a914bf80887e137d875a462ff6fa41e0d90719e7
+  metadata.gz: dc3f6bcfcb30e2863c067ca306b13a834ec66444
+  data.tar.gz: 3ba84a88565e830b0686f49532259fab3c466d82
 SHA512:
-  metadata.gz: '097b3721ffacb30739b37cd9eacfa583a582f8e210778869a47d5f47c18269f98df41b68387e996199abd879855d9bddcf9100d2dd278793cb837ac9abdcbc83'
-  data.tar.gz: 353f0124461ff8cfd6c742f0a4eeae635793c9dabc39cce25b3aac64b76325aedfb1d62118b831d79a73b98cd15696dc4be7b659812b48fc06e0ab8999f05725
+  metadata.gz: d438544dfde60a06feb23f9c12ba19230c6892fdaeaa43bcf7aa3360f1d34469b65ec13fd8184b920ba6ec76602644e2f1583678f469bbac66f5116b13302f68
+  data.tar.gz: 7a136fb090de4474ffd55e664d371374cd70dcab53dd6851f2c85ed6ce89364b5eb105e42c6131c5863d0894c7c990bc974566a227da31d92d6b5ff6159a66c5

data/lib/yle_tf/action/terraform_init.rb

@@ -15,8 +15,8 @@ class YleTf
         config = env[:config]
         backend = backend_config(config)
 
-        Logger.debug('Initializing Terraform with backend configuration:')
-        Logger.debug(backend.to_s)
+        Logger.info('Initializing Terraform')
+        Logger.debug("Backend configuration: #{backend}")
 
         if VersionRequirement.pre_0_9?(env[:terraform_version])
           init_pre_0_9(backend)
@@ -29,17 +29,21 @@ class YleTf
 
       def init_pre_0_9(backend)
         cli_args = backend.cli_args
-        YleTf::System.cmd('terraform', 'remote', 'config', *cli_args) if cli_args
+        if cli_args
+          YleTf::System.cmd('terraform', 'remote', 'config',
+                            '-no-color', *cli_args,
+                            stdout: :debug)
+        end
 
         Logger.debug('Fetching Terraform modules')
-        YleTf::System.cmd('terraform', 'get')
+        YleTf::System.cmd('terraform', 'get', '-no-color', stdout: :debug)
       end
 
       def init(backend)
         Logger.debug('Generating the backend configuration')
         backend.generate_config do
           Logger.debug('Initializing Terraform')
-          YleTf::System.cmd('terraform', 'init', '-no-color')
+          YleTf::System.cmd('terraform', 'init', '-no-color', stdout: :debug)
         end
       end
 

data/lib/yle_tf/action/tmpdir.rb

@@ -20,7 +20,7 @@ class YleTf
           @app.call(env)
         end
       ensure
-        FileUtils.rm_r(tmpdir) if tmpdir
+        FileUtils.rm_r(tmpdir) if tmpdir && Dir.exist?(tmpdir)
       end
 
       def tmpdir_prefix(config)

data/lib/yle_tf/action/verify_terraform_version.rb

@@ -1,5 +1,6 @@
 require 'yle_tf/error'
 require 'yle_tf/logger'
+require 'yle_tf/system'
 require 'yle_tf/version_requirement'
 
 class YleTf
@@ -22,10 +23,8 @@ class YleTf
       end
 
       def terraform_version
-        # TODO: move `command` to YleTf::System
-        Regexp.last_match(1) if `terraform version` =~ /^Terraform v([^\s]+)/
-      rescue Errno::ENOENT
-        nil
+        v = YleTf::System.read_cmd('terraform', 'version', error_handler: proc {})
+        Regexp.last_match(1) if v =~ /^Terraform v([^\s]+)/
       end
 
       def verify_version(env)

data/lib/yle_tf/action/verify_tf_env.rb

@@ -12,9 +12,13 @@ class YleTf
         config = env[:config]
         all_envs = VarsFile.list_all_envs(config)
 
+        if all_envs.empty?
+          raise Error, "No Terraform vars files found in '#{VarsFile::ENV_DIR}/'"
+        end
+
         if !all_envs.include?(config.tf_env)
           raise Error, "Terraform vars file not found for the '#{config.tf_env}' " \
-            " environment. Existing envs: #{all_envs.join(', ')}"
+            "environment. Existing envs: #{all_envs.join(', ')}"
         end
 
         @app.call(env)

data/lib/yle_tf/cli.rb

@@ -5,7 +5,7 @@ class YleTf
     attr_reader :tf_options, :tf_command, :tf_command_args, :tf_env
 
     # YleTf option arguments
-    TF_OPTIONS = %w[--debug --no-hooks --only-hooks].freeze
+    TF_OPTIONS = %w[--debug --no-color --no-hooks --only-hooks].freeze
 
     HELP_ARGS = %w[-h --help help].freeze
     VERSION_ARGS = %w[-v --version version].freeze
@@ -65,7 +65,8 @@ class YleTf
         @tf_env = 'error'
       end
 
-      self.debug = true if @tf_options.include?(:debug)
+      self.debug = true if @tf_options[:debug]
+      YleTf::Logger.color = !@tf_options[:no_color]
     end
 
     # Returns `Symbol` for the arg, e.g. `"--foo-bar"` -> `:foo_bar`

data/lib/yle_tf/logger/colorize.rb

@@ -0,0 +1,22 @@
+class YleTf
+  module Logger
+    module Colorize
+      COLORS = {
+        black:   30,
+        red:     31,
+        brown:   33,
+        blue:    34,
+        magenta: 35,
+        cyan:    36,
+        gray:    37
+      }.freeze
+
+      # Wraps the message with the specified ANSI color code
+      # if the color is specified and supported
+      def self.colorize(msg, color)
+        code = COLORS[color]
+        code ? "\033[#{code}m#{msg}\033[0m" : msg
+      end
+    end
+  end
+end

data/lib/yle_tf/logger.rb

@@ -1,14 +1,17 @@
 require 'forwardable'
 require 'logger'
 require 'rubygems'
+require 'yle_tf/logger/colorize'
 
 class YleTf
   # Logger for debug, error, etc. outputs.
   # Prints to STDERR, so it does not mess with e.g. `terraform output`.
   module Logger
+    LEVELS = %i[debug info warn error fatal].freeze
+
     class << self
       extend Forwardable
-      def_delegators :logger, :debug, :info, :warn, :error, :fatal
+      def_delegators :logger, *LEVELS
       def_delegators :logger, :debug?
     end
 
@@ -28,14 +31,35 @@ class YleTf
 
     def self.log_formatter
       proc do |severity, _datetime, progname, msg|
+        msg = Colorize.colorize(msg, color(severity))
+
         if progname
-          "[#{progname}] #{severity}: #{msg}\n"
+          "#{severity}: [#{progname}] #{msg}\n"
         else
           "#{severity}: #{msg}\n"
         end
       end
     end
 
+    def self.color?
+      @color
+    end
+
+    def self.color=(value)
+      @color = value
+    end
+
+    def self.color(severity)
+      return if !color?
+
+      case severity.to_s
+      when 'FATAL', 'ERROR'
+        :red
+      when 'WARN'
+        :brown
+      end
+    end
+
     # Patches the `::Logger` in older Ruby versions to
     # accept log level as a `String`
     def self.patch_for_old_ruby(logger)

data/lib/yle_tf/system/io_handlers.rb

@@ -0,0 +1,105 @@
+require 'yle_tf/system/output_logger'
+
+class YleTf
+  class System
+    class IOHandlers
+      BLOCK_SIZE = 1024
+
+      # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength
+      def self.input_handler(handler)
+        case handler
+        when :close
+          close
+        when :console
+          console_input
+        when :dev_null
+          dev_null_input
+        when IO, StringIO
+          io_input(handler)
+        else
+          if !handler.respond_to?(:call)
+            raise YleTf::Error, "Unknown input handler #{handler.inspect}"
+          end
+          handler
+        end
+      end
+
+      def self.output_handler(handler)
+        case handler
+        when :close
+          close
+        when :console
+          console_output
+        when :dev_null
+          dev_null_output
+        when IO, StringIO
+          io_output(handler)
+        when Symbol
+          OutputLogger.new(handler)
+        else
+          if !handler.respond_to?(:call)
+            raise YleTf::Error, "Unknown output handler #{handler.inspect}"
+          end
+          handler
+        end
+      end
+      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/MethodLength
+
+      # Returns a lambda that just closes the IO
+      def self.close
+        ->(io, *) { io.close }
+      end
+
+      # Returns a lambda that pipes STDIN to the IO
+      def self.console_input
+        io_input(STDIN)
+      end
+
+      # Returns a lambda that pipes IO's output to STDOUT
+      def self.console_output
+        io_output(STDOUT)
+      end
+
+      # Returns a lambda that does nothing
+      def self.dev_null_input
+        ->(*) {}
+      end
+
+      # Returns a lambda that just consumes the IO's output
+      def self.dev_null_output
+        lambda do |io, *|
+          Thread.new do
+            while io.read; end
+          end
+        end
+      end
+
+      # Returns a lambda that pipes the source IO to the IO's input
+      def self.io_input(source)
+        lambda do |target, *|
+          Thread.new do
+            begin
+              while (data = source.readpartial(BLOCK_SIZE))
+                target.write(data)
+              end
+            ensure
+              target.close_write
+            end
+          end
+        end
+      end
+
+      # Returns a lambda that pipes IO's output to the target IO
+      # Does not close the target stream
+      def self.io_output(target)
+        lambda do |source, *|
+          Thread.new do
+            while (data = source.readpartial(BLOCK_SIZE))
+              target.write(data)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/yle_tf/system/output_logger.rb

@@ -0,0 +1,29 @@
+require 'yle_tf/logger'
+
+class YleTf
+  class System
+    class OutputLogger
+      attr_reader :level
+
+      def initialize(level)
+        @level = level.to_sym
+
+        raise YleTf::Error, "Unknown log level '#{@level}'" if !level?(@level)
+      end
+
+      def call(io, progname)
+        Thread.new do
+          io.each { |line| log(progname, line.chomp) }
+        end
+      end
+
+      def level?(level)
+        YleTf::Logger::LEVELS.include?(level)
+      end
+
+      def log(progname, line)
+        YleTf::Logger.public_send(level, progname) { line }
+      end
+    end
+  end
+end

data/lib/yle_tf/system/tf_hook_output_logger.rb

@@ -0,0 +1,33 @@
+require 'yle_tf/logger'
+require 'yle_tf/system/output_logger'
+
+class YleTf
+  class System
+    # An IO handler class for YleTf Hook output.
+    #
+    # Allows hooks to emit log messages with specific levels
+    # by prefixing a line with `<LEVEL>: `.
+    class TfHookOutputLogger < OutputLogger
+      def log(progname, line)
+        # Remove `[<progname>] ` prefix from the output line.
+        # This is mostly for backwards compatibility in Yle.
+        line.sub!(/^\[#{progname}\] /, '')
+
+        level, line = line_level(line)
+
+        YleTf::Logger.public_send(level, progname) { line }
+      end
+
+      # Extracts the log level from the line if found,
+      # otherwise returns the default level and the line as is
+      def line_level(line)
+        if (m = /^(?<level>[A-Z]+): (?<line>.*)$/.match(line))
+          line_level = m[:level].downcase.to_sym
+          return [line_level, m[:line]] if level?(line_level)
+        end
+
+        [level, line]
+      end
+    end
+  end
+end

data/lib/yle_tf/system.rb

@@ -1,22 +1,98 @@
+require 'English'
+require 'open3'
 require 'shellwords'
-
+require 'thwait'
 require 'yle_tf/error'
 require 'yle_tf/logger'
+require 'yle_tf/system/io_handlers'
 
 class YleTf
   # Helpers to execute system commands with error handling
-  #
-  # TODO: Add way to wrap stdout of the commands and direct it to `Logger`
   class System
     ExecuteError = Class.new(YleTf::Error)
 
+    DEFAULT_ERROR_HANDLER = ->(_exit_code, error) { raise error }.freeze
+
+    DEFAULT_IO_HANDLERS = {
+      stdin: :dev_null,
+      stdout: :info,
+      stderr: :error
+    }.freeze
+
+    def self.console_cmd(*args, **opts)
+      env = opts[:env] || {}
+
+      YleTf::Logger.debug { "Calling #{cmd_string(args, env)}" }
+
+      system(env, *args)
+      verify_exit_status($CHILD_STATUS, opts[:error_handler], cmd_string(args))
+    end
+
+    # Executes the command and attaches IO streams
     def self.cmd(*args, **opts)
-      env = opts[:env]
-      YleTf::Logger.debug { "Calling `#{args.shelljoin}`#{" with env '#{env}'" if env}" }
+      opts = DEFAULT_IO_HANDLERS.merge(opts)
+      env = opts[:env] || {}
+      progname = opts.fetch(:progname) { args.first }
+
+      YleTf::Logger.debug { "Calling #{cmd_string(args, env)}" }
+
+      status = Open3.popen3(env, *args, &handle_io(progname, opts))
+      verify_exit_status(status, opts[:error_handler], cmd_string(args))
+    rescue Interrupt, Errno::ENOENT => e
+      error(opts[:error_handler], "Failed to execute #{cmd_string(args)}: #{e}")
+    end
+
+    def self.read_cmd(*args, **opts)
+      buffer = StringIO.new
+      cmd(*args, opts.merge(stdout: buffer))
+      buffer.string
+    end
+
+    def self.cmd_string(args, env = nil)
+      "`#{args.shelljoin}`#{" with env '#{env}'" if env && !env.empty?}"
+    end
+
+    def self.handle_io(progname, handlers)
+      lambda do |stdin, stdout, stderr, wait_thr|
+        in_thr = attach_input_handler(handlers[:stdin], stdin, progname)
+        out_thr = [
+          attach_output_handler(handlers[:stdout], stdout, progname),
+          attach_output_handler(handlers[:stderr], stderr, progname)
+        ]
+
+        # Wait for the process to exit
+        wait_thr.value.tap do
+          YleTf::Logger.debug("`#{progname}` exited, killing input handler thread")
+          in_thr.kill if in_thr.is_a?(Thread)
+
+          YleTf::Logger.debug('Waiting for output handler threads to stop')
+          ThreadsWait.all_waits(out_thr)
+        end
+      end
+    end
+
+    def self.attach_input_handler(handler, io, progname)
+      io_proc = IOHandlers.input_handler(handler)
+      io_proc.call(io, progname)
+    end
+
+    def self.attach_output_handler(handler, io, progname)
+      io_proc = IOHandlers.output_handler(handler)
+      io_proc.call(io, progname)
+    end
+
+    def self.verify_exit_status(status, handler, cmd)
+      status.success? ||
+        error(handler,
+              "Failed to execute #{cmd} (#{status.exitstatus})",
+              status.exitstatus)
+    end
+
+    def self.error(handler, error_msg, exit_code = nil)
+      YleTf::Logger.debug(error_msg)
 
-      system(env || {}, *args) ||
-        raise(ExecuteError,
-              "Failed to execute `#{args.shelljoin}`#{" with env '#{env}'" if env}")
+      handler ||= DEFAULT_ERROR_HANDLER
+      handler.call(exit_code, ExecuteError.new(error_msg))
     end
   end
 end

data/lib/yle_tf/tf_hook.rb

@@ -4,6 +4,7 @@ require 'tmpdir'
 require 'yle_tf/error'
 require 'yle_tf/logger'
 require 'yle_tf/system'
+require 'yle_tf/system/tf_hook_output_logger'
 
 class YleTf
   class TfHook
@@ -39,8 +40,14 @@ class YleTf
     def run(tf_vars)
       fetch if !path
 
-      Logger.info("Running hook '#{description}'...")
-      YleTf::System.cmd(path, env: vars.merge(tf_vars))
+      Logger.info("Running hook '#{description}'")
+      YleTf::System.cmd(
+        path,
+        env: vars.merge(tf_vars),
+        progname: File.basename(path),
+        stdout: System::TfHookOutputLogger.new(:info),
+        stderr: System::TfHookOutputLogger.new(:error)
+      )
     ensure
       delete_tmpdir
     end
@@ -64,9 +71,9 @@ class YleTf
     end
 
     def clone_git_repo(config)
-      Logger.info("Cloning hook '#{description}' from #{config[:uri]} (#{config[:ref]})")
+      Logger.debug("Cloning hook '#{description}' from #{config[:uri]} (#{config[:ref]})")
       YleTf::System.cmd(
-        'git', 'clone', '--no-progress', '--depth=1', '--branch', config[:ref],
+        'git', 'clone', '--quiet', '--depth=1', '--branch', config[:ref],
         '--', config[:uri], config[:dir]
       )
     end
@@ -76,7 +83,7 @@ class YleTf
     end
 
     def delete_tmpdir
-      FileUtils.rm_r(@tmpdir) if @tmpdir
+      FileUtils.rm_r(@tmpdir) if @tmpdir && Dir.exist?(@tmpdir)
       @tmpdir = nil
     end
 

data/lib/yle_tf/vars_file.rb

@@ -1,14 +1,16 @@
 class YleTf
   class VarsFile
+    ENV_DIR = 'envs'.freeze
+
     # Returns the env specific tfvars file path if it exists
     def self.find_env_vars_file(config)
-      path = "#{config.module_dir}/envs/#{config.tf_env}.tfvars"
+      path = "#{config.module_dir}/#{ENV_DIR}/#{config.tf_env}.tfvars"
       VarsFile.new(path) if File.exist?(path)
     end
 
     # Returns all envs that have tfvars files
     def self.list_all_envs(config)
-      Dir.glob("#{config.module_dir}/envs/*.tfvars").map do |path|
+      Dir.glob("#{config.module_dir}/#{ENV_DIR}/*.tfvars").map do |path|
         File.basename(path, '.tfvars')
       end
     end

data/lib/yle_tf/version.rb

@@ -1,3 +1,3 @@
 class YleTf
-  VERSION = '0.1.1'.freeze
+  VERSION = '0.2.0'.freeze
 end

data/lib/yle_tf_plugins/commands/__default/command.rb

@@ -1,3 +1,4 @@
+require 'yle_tf/logger'
 require 'yle_tf/system'
 
 module YleTfPlugins
@@ -5,9 +6,16 @@ module YleTfPlugins
     class Command
       def execute(env)
         command = env[:tf_command]
-        args    = env[:tf_command_args]
 
-        YleTf::System.cmd('terraform', command, *args)
+        args = env[:tf_command_args].dup
+        args << '-no-color' if !color?(env)
+
+        YleTf::Logger.info "Running `terraform #{command}`"
+        YleTf::System.console_cmd('terraform', command, *args)
+      end
+
+      def color?(env)
+        !env[:tf_options][:no_color]
       end
     end
   end

data/lib/yle_tf_plugins/commands/help/command.rb

@@ -1,5 +1,6 @@
 require 'optparse'
 
+require 'yle_tf/system'
 require 'yle_tf/plugin'
 
 module YleTfPlugins
@@ -20,6 +21,7 @@ module YleTfPlugins
           o.on('-h', '--help', 'Prints this help')
           o.on('-v', '--version', 'Prints the version information')
           o.on('--debug', 'Print debug information')
+          o.on('--no-color', 'Do not output with colors')
           o.on('--no-hooks', 'Do not run any hooks')
           o.on('--only-hooks', 'Only run the hooks')
           o.separator ''
@@ -46,9 +48,14 @@ module YleTfPlugins
       end
 
       def terraform_help
-        `terraform help`.lines.grep(/^    /)
-      rescue Errno::ENOENT
-        '    [Terraform not found]'
+        on_error = proc do |exit_code|
+          # exit_code is nil if the command was not found
+          # Ignore other exit codes, as older Terraform versions return
+          # non-zero exit codes for the help.
+          return '    [Terraform not found]' if exit_code.nil?
+        end
+        help = YleTf::System.read_cmd('terraform', '--help', error_handler: on_error)
+        help.lines.grep(/^    /)
       end
     end
   end

data/lib/yle_tf_plugins/commands/version/command.rb

@@ -1,5 +1,4 @@
-require 'optparse'
-
+require 'yle_tf/system'
 require 'yle_tf/version'
 
 module YleTfPlugins
@@ -11,9 +10,9 @@ module YleTfPlugins
       end
 
       def terraform_version
-        `terraform version`.lines.first
-      rescue Errno::ENOENT
-        '[Terraform not found]'
+        on_error = proc { return '[Terraform not found]' }
+        v = YleTf::System.read_cmd('terraform', 'version', error_handler: on_error)
+        v.lines.first
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: yle_tf
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Yleisradio
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-31 00:00:00.000000000 Z
+date: 2017-09-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -86,11 +86,15 @@ files:
 - lib/yle_tf/config/loader.rb
 - lib/yle_tf/error.rb
 - lib/yle_tf/logger.rb
+- lib/yle_tf/logger/colorize.rb
 - lib/yle_tf/plugin.rb
 - lib/yle_tf/plugin/action_hook.rb
 - lib/yle_tf/plugin/loader.rb
 - lib/yle_tf/plugin/manager.rb
 - lib/yle_tf/system.rb
+- lib/yle_tf/system/io_handlers.rb
+- lib/yle_tf/system/output_logger.rb
+- lib/yle_tf/system/tf_hook_output_logger.rb
 - lib/yle_tf/tf_hook.rb
 - lib/yle_tf/tf_hook/runner.rb
 - lib/yle_tf/vars_file.rb