yle_tf 1.0.0 → 1.1.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4f901e67d48b00b9c2a6692677eee372f69db444952618773bdc6edac24d5fd0
-  data.tar.gz: 8f98e4fa5c53bdc58cc758aa46be51ff7867747b5fe187cb54b1a2b83e025f80
+  metadata.gz: 8203ca939d1933be0bb2aae8ba26d4a7a8b385fce0f65c52f4d707cf6260f323
+  data.tar.gz: 0e518cdaa2a4ecbdb76dae38b86b4f31a178e345c0e16fb10f469d1df8f950ab
 SHA512:
-  metadata.gz: 4aef3de1f089f711fffe0e538186516b3cd3a81c0b0f672e3302a66d2fb5c625dd22076a5f9ac7e6b228ad34ea90780f28f160ddc73709c3967717646bde56e6
-  data.tar.gz: 79508c7c0031a6515820efa3c415a131ff3e6cf34e840cd0f479436bf41a86cde065671e292a3e88a26132a47297d5a78ed6f5ccbdde5cd6a89ec92508b66f0b
+  metadata.gz: 9127c39276c80009aa7d233f7398ec0ee2ea2784eac1d9fc460c245c5f654fb36a910d4c52bd3453cd9db19595d207f4f545de2aa408e1866cb84b9a534d3cd8
+  data.tar.gz: 5c851247db9645aeef3711baa47f40ff682b21ae7030ae1a8a90b3acb6eb70dbb70a8a3f044f92b7d235f87f678a8919cc563368861042fa8c82ed48bf843997

data/bin/tf

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 # Catch Ctrl+C to avoid stack traces
 Signal.trap('INT') { abort }

data/lib/yle_tf/action/builder.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../../../vendor/middleware/builder'
 
 class YleTf

data/lib/yle_tf/action/command.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'yle_tf/logger'
 
 class YleTf
@@ -12,9 +14,9 @@ class YleTf
 
       def call(env)
         if env[:tf_options][:only_hooks]
-          Logger.debug "Skipping command #{command.class} due to `--only-hooks`"
+          Logger.debug "Skipping command #{command} due to `--only-hooks`"
         else
-          Logger.debug "Executing command #{command.class} with env: #{env.inspect}"
+          Logger.debug "Executing command #{command} with env: #{env.inspect}"
           command.new.execute(env)
         end
 

data/lib/yle_tf/action/copy_root_module.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'fileutils'
 
 require 'yle_tf/logger'

data/lib/yle_tf/action/generate_vars_file.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'yle_tf/logger'
 require 'yle_tf/vars_file'
 

data/lib/yle_tf/action/load_config.rb

@@ -1,4 +1,7 @@
+# frozen_string_literal: true
+
 require 'yle_tf/config'
+require 'yle_tf/logger'
 
 class YleTf
   module Action
@@ -8,10 +11,15 @@ class YleTf
       end
 
       def call(env)
-        env[:config] ||= Config.new(env[:tf_env])
+        env[:config] ||= load_config(env[:tf_env])
 
         @app.call(env)
       end
+
+      def load_config(tf_env)
+        Logger.debug("Initializing configuration for the #{tf_env.inspect} environment")
+        Config.load(tf_env).tap { |config| Logger.debug(config.inspect) }
+      end
     end
   end
 end

data/lib/yle_tf/action/terraform_init.rb

@@ -1,8 +1,8 @@
-require 'yle_tf/error'
+# frozen_string_literal: true
+
 require 'yle_tf/logger'
 require 'yle_tf/plugin'
 require 'yle_tf/system'
-require 'yle_tf/version_requirement'
 
 class YleTf
   module Action
@@ -10,8 +10,8 @@ class YleTf
       TF_CMD_ARGS = %w[-input=false -no-color].freeze
 
       TF_CMD_OPTS = {
-        env: { 'TF_IN_AUTOMATION' => 'true' }, # Reduces some output
-        stdout: :debug                         # Hide the output to the debug level
+        env:    { 'TF_IN_AUTOMATION' => 'true' }, # Reduces some output
+        stdout: :debug                            # Hide the output to the debug level
       }.freeze
 
       def initialize(app)
@@ -20,52 +20,30 @@ class YleTf
 
       def call(env)
         config = env[:config]
-        backend = backend_config(config)
+        backend = backend(config)
 
         Logger.info('Initializing Terraform')
         Logger.debug("Backend configuration: #{backend}")
 
-        if VersionRequirement.pre_0_9?(env[:terraform_version])
-          init_pre_0_9(backend)
-        else
-          init(backend)
-        end
+        init(backend)
 
         @app.call(env)
       end
 
-      def init_pre_0_9(backend)
-        cli_args = backend.cli_args
-        if cli_args
-          YleTf::System.cmd('terraform', 'remote', 'config', *TF_CMD_ARGS, *cli_args, TF_CMD_OPTS)
-        end
-
-        Logger.debug('Fetching Terraform modules')
-        YleTf::System.cmd('terraform', 'get', *TF_CMD_ARGS, TF_CMD_OPTS)
-      end
-
       def init(backend)
-        Logger.debug('Generating the backend configuration')
-        backend.generate_config do
-          Logger.debug('Initializing Terraform')
-          YleTf::System.cmd('terraform', 'init', *TF_CMD_ARGS, TF_CMD_OPTS)
-        end
+        Logger.debug('Configuring the backend')
+        backend.configure
+
+        Logger.debug('Initializing Terraform')
+        YleTf::System.cmd('terraform', 'init', *TF_CMD_ARGS, TF_CMD_OPTS)
       end
 
-      def backend_config(config)
+      def backend(config)
         backend_type = config.fetch('backend', 'type').downcase
-        backend_proc = backend_proc(backend_type)
+        backend_proc = Plugin.manager.backends[backend_type]
 
         klass = backend_proc.call
-        klass.new.backend_config(config)
-      end
-
-      def backend_proc(backend_type)
-        backends = Plugin.manager.backends
-        backends.fetch(backend_type.to_sym) do
-          raise Error, "Unknown backend type '#{backend_type}'. " \
-            "Supported backends: #{backends.keys.join(', ')}"
-        end
+        klass.new(config)
       end
     end
   end

data/lib/yle_tf/action/tf_hooks.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'yle_tf/logger'
 require 'yle_tf/tf_hook/runner'
 

data/lib/yle_tf/action/tmpdir.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'fileutils'
 require 'tmpdir'
 
@@ -20,7 +22,7 @@ class YleTf
           @app.call(env)
         end
       ensure
-        FileUtils.rm_r(tmpdir) if tmpdir && Dir.exist?(tmpdir)
+        FileUtils.rm_rf(tmpdir, secure: true) if tmpdir && Dir.exist?(tmpdir)
       end
 
       def tmpdir_prefix(config)

data/lib/yle_tf/action/verify_terraform_version.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+
+require 'yle_tf'
 require 'yle_tf/error'
 require 'yle_tf/logger'
 require 'yle_tf/system'
@@ -17,22 +20,32 @@ class YleTf
         raise(Error, 'Terraform not found') if !version
 
         Logger.debug("Terraform version: #{version}")
-        verify_version(env)
+
+        verify_version(version, requirement_by_yletf, required_by: 'YleTf')
+        verify_version(version, requirement_by_config(env), required_by: 'config')
 
         @app.call(env)
       end
 
       def terraform_version
         v = YleTf::System.read_cmd('terraform', 'version', error_handler: proc {})
-        Regexp.last_match(1) if v =~ /^Terraform v([^\s]+)/
+        m = /^Terraform v(?<version>[^\s]+)/.match(v)
+        m && m[:version]
       end
 
-      def verify_version(env)
-        version = env[:terraform_version]
+      def requirement_by_config(env)
         requirement = env[:config].fetch('terraform', 'version_requirement') { nil }
+        VersionRequirement.new(requirement)
+      end
+
+      def requirement_by_yletf
+        VersionRequirement.new(YleTf::TERRAFORM_VERSION_REQUIREMENT)
+      end
 
-        if !VersionRequirement.new(requirement).satisfied_by?(version)
-          raise Error, "Terraform version '#{requirement}' required, '#{version}' found"
+      def verify_version(version, requirement, **opts)
+        if !requirement.satisfied_by?(version)
+          raise Error, "Terraform version '#{requirement}' required by #{opts[:required_by]}, " \
+                       "'#{version}' found"
         end
       end
     end

data/lib/yle_tf/action/verify_tf_env.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'yle_tf/error'
 require 'yle_tf/vars_file'
 

data/lib/yle_tf/action/write_terraformrc_defaults.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'fileutils'
 require 'yle_tf/logger'
 
@@ -5,10 +7,10 @@ class YleTf
   module Action
     class WriteTerraformrcDefaults
       # Path of the Terraform CLI configuration file
-      RC_PATH = '~/.terraformrc'.freeze
+      RC_PATH = '~/.terraformrc'
 
       # Path of the plugin cache directory
-      DEFAULT_PLUGIN_CACHE_PATH = '~/.terraform.d/plugin-cache'.freeze
+      DEFAULT_PLUGIN_CACHE_PATH = '~/.terraform.d/plugin-cache'
 
       def initialize(app)
         @app = app

data/lib/yle_tf/action.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class YleTf
   module Action
     autoload :Builder, 'yle_tf/action/builder'

data/lib/yle_tf/backend.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'json'
+
+class YleTf
+  class Backend
+    BACKEND_CONFIG_FILE = '_backend.tf.json'
+
+    attr_reader :config
+
+    def initialize(config)
+      @config = config
+    end
+
+    def type
+      @type ||= config.fetch('backend', 'type')
+    end
+
+    def backend_specific_config
+      config.fetch('backend', type)
+    end
+
+    # Generate backend configuration file for Terraform
+    def configure
+      data = {
+        terraform: [{
+          backend: [to_h]
+        }]
+      }
+      File.write(BACKEND_CONFIG_FILE, JSON.pretty_generate(data))
+    end
+
+    # Returns the backend configuration as a `Hash` for Terraform
+    def to_h
+      { type => backend_specific_config }
+    end
+
+    def to_s
+      to_h.to_s
+    end
+  end
+end

data/lib/yle_tf/cli.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'yle_tf'
 
 class YleTf

data/lib/yle_tf/config/defaults.rb

@@ -1,33 +1,41 @@
+# frozen_string_literal: true
+
+require 'yle_tf/helpers/hash'
+
 class YleTf
   class Config
     module Defaults
       DEFAULT_CONFIG = {
-        'hooks' => {
-          'pre' => [],
+        'hooks'     => {
+          'pre'  => [],
           'post' => []
         },
-        'backend' => {
+        'backend'   => {
           'type' => 'file',
-          'bucket'  => nil,
-          'file'    => '<%= @module %>_<%= @env %>.tfstate',
-          'region'  => nil,
-          'encrypt' => false,
+          'file' => {
+            'path' => '<%= @module %>_<%= @env %>.tfstate'
+          },
+          's3'   => {
+            'key' => '<%= @module %>_<%= @env %>.tfstate'
+          }
         },
-        'tfvars' => {
+        'tfvars'    => {
         },
         'terraform' => {
           'version_requirement' => nil
         }
       }.freeze
 
+      # Returns deep copy of the default config Hash.
       def default_config
-        DEFAULT_CONFIG.dup
+        Helpers::Hash.deep_copy(DEFAULT_CONFIG)
       end
 
       def default_config_context
         {
-          env: tf_env,
-          module: module_dir.basename.to_s,
+          env:        tf_env,
+          module:     module_dir.basename.to_s,
+          module_dir: module_dir.to_s,
         }
       end
     end

data/lib/yle_tf/config/erb.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'erb'
 
 class YleTf

data/lib/yle_tf/config/file.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'yaml'
 
 require 'yle_tf/logger'

data/lib/yle_tf/config/loader.rb

@@ -1,11 +1,13 @@
+# frozen_string_literal: true
+
 require 'yle_tf/config/defaults'
 require 'yle_tf/config/erb'
 require 'yle_tf/config/file'
+require 'yle_tf/config/migration'
+require 'yle_tf/helpers/hash'
 require 'yle_tf/logger'
 require 'yle_tf/plugin'
 
-require_relative '../../../vendor/hash_deep_merge'
-
 class YleTf
   class Config
     class Loader
@@ -19,20 +21,88 @@ class YleTf
       end
 
       def load
-        Logger.debug('Loading default config')
-        config = default_config
-        Logger.debug(config.inspect)
+        load_sequence = %i[
+          load_default_config
+          load_plugin_configurations
+          load_config_files
+          evaluate_configuration_strings
+        ]
+        load_sequence.inject({}) { |config, method| send(method, config) }
+      end
+
+      def load_default_config(_config)
+        task('Loading default config') { default_config }
+      end
+
+      def load_plugin_configurations(config)
+        Logger.debug('Loading configuration from plugins')
+
+        plugins.inject(config) do |prev_config, plugin|
+          migrate_and_merge_configuration(prev_config, plugin.default_config,
+                                          type: 'plugin', name: plugin.to_s)
+        end
+      end
+
+      def load_config_files(config)
+        Logger.debug('Loading configuration from files')
+
+        config_files.inject(config) do |prev_config, file|
+          migrate_and_merge_configuration(prev_config, file.read,
+                                          type: 'file', name: file.name)
+        end
+      end
+
+      def evaluate_configuration_strings(config)
+        task('Evaluating the configuration strings') { eval_config(config) }
+      end
+
+      def eval_config(config)
+        case config
+        when Hash
+          config.each_with_object({}) { |(key, value), h| h[key] = eval_config(value) }
+        when Array
+          config.map { |item| eval_config(item) }
+        when String
+          Config::ERB.evaluate(config, config_context)
+        else
+          config
+        end
+      end
 
-        Logger.debug('Merging default configurations from plugins')
-        merge_plugin_configurations(config)
-        Logger.debug(config.inspect)
+      def plugins
+        Plugin.manager.registered
+      end
+
+      def config_files
+        module_dir.descend.lazy
+                  .map { |dir| dir.join('tf.yaml') }
+                  .select(&:exist?)
+                  .map { |file| Config::File.new(file) }
+      end
 
-        Logger.debug('Merging configurations from files')
-        merge_config_files(config)
-        Logger.debug(config.inspect)
+      def migrate_and_merge_configuration(prev_config, config, **opts)
+        task("- #{opts[:name]}") { config }
+        return prev_config if config.empty?
 
-        Logger.debug('Evaluating the configuration strings')
-        eval_config(config)
+        source = "#{opts[:type]}: '#{opts[:name]}'"
+        config = migrate_old_config(config, source: source)
+        deep_merge(prev_config, config, source: source)
+      end
+
+      def migrate_old_config(config, **opts)
+        task('  -> Migrating') do
+          Config::Migration.migrate_old_config(config, opts)
+        end
+      end
+
+      def deep_merge(prev_config, config, **opts)
+        task('  -> Merging') do
+          Helpers::Hash.deep_merge(prev_config, config)
+        end
+      rescue StandardError => e
+        Logger.fatal("Failed to merge configuration from #{opts[:source]}:\n" \
+                     "#{config.inspect}\ninto:\n#{prev_config.inspect}")
+        raise e
       end
 
       def config_context
@@ -54,53 +124,13 @@ class YleTf
         end
       end
 
-      def merge_plugin_configurations(config)
-        Plugin.manager.default_configs.each do |plugin_config|
-          deep_merge(
-            config, plugin_config,
-            error_msg:
-              "Failed to merge a plugin's default configuration:\n" \
-              "#{plugin_config.inspect}\ninto:\n#{config.inspect}"
-          )
-        end
-      end
-
-      def merge_config_files(config)
-        config_files do |file|
-          Logger.debug("  - #{file}")
-          deep_merge(
-            config, file.read,
-            error_msg:
-              "Failed to merge configuration from '#{file}' into:\n" \
-              "#{config.inspect}"
-          )
-        end
-      end
+      # Helper to print debug information about the task and configuration
+      # after it
+      def task(message = nil)
+        Logger.debug(message) if message
 
-      def deep_merge(config, new_config, opts = {})
-        config.deep_merge!(new_config)
-      rescue StandardError => e
-        Logger.fatal(opts[:error_msg]) if opts[:error_msg]
-        raise e
-      end
-
-      def config_files
-        module_dir.descend do |dir|
-          file = dir.join('tf.yaml')
-          yield(Config::File.new(file)) if file.exist?
-        end
-      end
-
-      def eval_config(config)
-        case config
-        when Hash
-          config.each_with_object({}) { |(key, value), h| h[key] = eval_config(value) }
-        when Array
-          config.map { |item| eval_config(item) }
-        when String
-          Config::ERB.evaluate(config, config_context)
-        else
-          config
+        yield.tap do |config|
+          Logger.debug("  #{config.inspect}")
         end
       end
     end

data/lib/yle_tf/config/migration.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+require 'yaml'
+require 'yle_tf/helpers/hash'
+require 'yle_tf/logger'
+
+class YleTf
+  class Config
+    class Migration
+      BACKEND_MIGRATIONS = {
+        'file'  => {
+          'file' => 'path'
+        },
+        's3'    => {
+          'region'  => 'region',
+          'bucket'  => 'bucket',
+          'file'    => 'key',
+          'encrypt' => 'encrypt'
+        },
+        'swift' => {
+          'region'            => 'region_name',
+          'container'         => 'container',
+          'archive_container' => 'archive_container'
+        }
+      }.freeze
+
+      include Helpers::Hash
+
+      def self.migrate_old_config(config, **opts)
+        new(config, **opts).migrated_config
+      end
+
+      attr_reader :config, :config_source
+
+      def initialize(config, **opts)
+        @config = config
+        @config_source = opts.fetch(:source)
+      end
+
+      def old_backend_config
+        config['backend']
+      end
+
+      def migrated_config
+        migrate_old_backend_config do |new_config|
+          Logger.warn("Old configuration found in #{config_source}")
+          Logger.warn("Please migrate to relevant parts of:\n" \
+                      "#{sanitize_config(new_config)}")
+          Logger.warn(
+            'See https://github.com/Yleisradio/yle_tf/wiki/Migrating-Configuration for more details'
+          )
+        end
+      end
+
+      # TODO: Remove support in v2.0
+      def migrate_old_backend_config
+        changed = false
+
+        new_config = BACKEND_MIGRATIONS.inject(config) do |prev_config, (type, keys)|
+          migrate_old_backend_config_keys(prev_config, type, keys) { changed = true }
+        end
+
+        yield(new_config) if changed
+
+        new_config
+      end
+
+      def migrate_old_backend_config_keys(config, type, keys)
+        migrated_keys = find_old_backend_config_keys(keys)
+        return config if migrated_keys.empty?
+
+        defaults = {
+          'backend' => {
+            type => {}
+          }
+        }
+        copy_with_defaults(config, defaults).tap do |new_config|
+          migrated_keys.each do |old_key, new_key|
+            new_config['backend'][type][new_key] = old_backend_config[old_key]
+          end
+
+          yield new_config
+        end
+      end
+
+      def find_old_backend_config_keys(keys)
+        return {} if !old_backend_config.is_a?(Hash)
+
+        keys.select do |old_key, _new_key|
+          old_backend_config.key?(old_key) &&
+            # Special case for 'file' as it is now used for option Hash for the
+            # 'file' backend
+            !(old_key == 'file' && old_backend_config['file'].is_a?(Hash))
+        end
+      end
+
+      def copy_with_defaults(config, defaults)
+        deep_merge(deep_copy(config), defaults)
+      end
+
+      def sanitize_config(config)
+        backend_config = config['backend'].select do |key, value|
+          key == 'type' || value.is_a?(Hash)
+        end
+
+        YAML.dump('backend' => backend_config)
+      end
+    end
+  end
+end