paramsync 0.1.0

data/lib/paramsync/diff.rb

@@ -0,0 +1,217 @@
+# This software is public domain. No rights are reserved. See LICENSE for more information.
+
+class Paramsync
+  class Diff
+    def initialize(target:, local:, remote:, mode:)
+      @target = target
+      @local = local
+      @remote = remote
+      @mode = mode
+
+      @all_keys = (@local.keys + @remote.keys).sort.uniq
+
+      @diff =
+        @all_keys.collect do |key|
+          excluded = false
+          op = :noop
+          if @remote.has_key?(key) and not @local.has_key?(key)
+            case @mode
+            when :push
+              op = @target.delete? ? :delete : :ignore
+            when :pull
+              op = :create
+            end
+          elsif @local.has_key?(key) and not @remote.has_key?(key)
+            case @mode
+            when :push
+              op = :create
+            when :pull
+              op = @target.delete? ? :delete : :ignore
+            end
+          else
+            if @remote[key] == @local[key]
+              op = :noop
+            else
+              op = :update
+            end
+          end
+
+          ssm_key = [@target.prefix, key].compact.join("/").gsub('/.', '/').squeeze("/")
+
+          if @target.exclude.include?(key) or @target.exclude.include?(ssm_key)
+            op = :ignore
+            excluded = true
+          end
+
+          filename =
+            case @target.type
+            when :dir then File.join(@target.base_path, key)
+            when :file then @target.base_path
+            end
+
+          display_filename =
+            case @target.type
+            when :dir then File.join(@target.base_path, key).trim_path
+            when :file then "#{@target.base_path.trim_path}#{':'.gray}#{key.cyan}"
+            end
+
+          OpenStruct.new(
+            op: op,
+            excluded: excluded,
+            relative_path: key,
+            filename: filename,
+            display_filename: display_filename,
+            ssm_key: ssm_key,
+            local_content: @local[key],
+            remote_content: @remote[key],
+          )
+        end
+    end
+
+    def items_to_delete
+      @diff.select { |d| d.op == :delete }
+    end
+
+    def items_to_update
+      @diff.select { |d| d.op == :update }
+    end
+
+    def items_to_create
+      @diff.select { |d| d.op == :create }
+    end
+
+    def items_to_ignore
+      @diff.select { |d| d.op == :ignore }
+    end
+
+    def items_to_exclude
+      @diff.select { |d| d.op == :ignore and d.excluded == true }
+    end
+
+    def items_to_noop
+      @diff.select { |d| d.op == :noop }
+    end
+
+    def items_to_change
+      @diff.select { |d| [:delete, :update, :create].include?(d.op) }
+    end
+
+    def final_items
+      case @mode
+      when :push then @local
+      when :pull then @remote
+      end
+    end
+
+    def any_changes?
+      self.items_to_change.count > 0
+    end
+
+    def print_report
+      puts '='*85
+      puts @target.description(@mode)
+
+      puts "  Keys scanned: #{@diff.count}"
+      if Paramsync.config.verbose?
+        puts "  Keys ignored: #{self.items_to_ignore.count}"
+        puts "  Keys in sync: #{self.items_to_noop.count}"
+      end
+
+      puts if self.any_changes?
+
+      from_content_key, to_content_key, to_path_key, to_type_display_name =
+        case @mode
+        when :push then [:local_content, :remote_content, :ssm_key, "Keys"]
+        when :pull
+          case @target.type
+          when :dir then [:remote_content, :local_content, :display_filename, "Files"]
+          when :file then [:remote_content, :local_content, :display_filename, "File entries"]
+          end
+        end
+
+      @diff.each do |item|
+        case item.op
+        when :create
+          puts "CREATE".bold.green + " #{item[to_path_key]}"
+          if(item[from_content_key][1])
+            puts '[SECURE]'
+          end
+          puts '-'*85
+          # simulate diff but without complaints about line endings
+          item[from_content_key][0].each_line do |line|
+            puts "+#{line.chomp}".green
+          end
+          puts '-'*85
+
+        when :update
+          puts "UPDATE".bold + " #{item[to_path_key]}"
+          if item[to_content_key][1] != item[from_content_key][1]
+            puts "#{item[to_content_key][1] ? '[SECURE]' : 'insecure'} => #{item[from_content_key][1] ? '[SECURE]' : 'insecure'}"
+          end
+          puts '-'*85
+          if item[to_content_key][0] != item[from_content_key][0]
+            puts Diffy::Diff.new(item[to_content_key][0], item[from_content_key][0]).to_s(:color)
+          end
+          puts '-'*85
+
+        when :delete
+          if @target.delete?
+            puts "DELETE".bold.red + " #{item[to_path_key]}"
+            puts '-'*85
+            # simulate diff but without complaints about line endings
+            item[to_content_key][0].each_line do |line|
+              puts "-#{line.chomp}".red
+            end
+            puts '-'*85
+          else
+            if Paramsync.config.verbose?
+              puts "IGNORE".bold + " #{item[to_path_key]}"
+            end
+          end
+
+        when :ignore
+          if Paramsync.config.verbose?
+            puts "IGNORE".bold + " #{item[to_path_key]}"
+          end
+
+        when :noop
+          if Paramsync.config.verbose?
+            puts "NO-OP!".bold + " #{item[to_path_key]}"
+          end
+
+        else
+          if Paramsync.config.verbose?
+            STDERR.puts "WARNING: unexpected operation '#{item.op}' for #{item[to_path_key]}"
+          end
+
+        end
+      end
+
+      if self.items_to_create.count > 0
+        puts
+        puts "#{to_type_display_name} to create: #{self.items_to_create.count}".bold
+        self.items_to_create.each do |item|
+          puts "+ #{item[to_path_key]}".green
+        end
+      end
+
+      if self.items_to_update.count > 0
+        puts
+        puts "#{to_type_display_name} to update: #{self.items_to_update.count}".bold
+        self.items_to_update.each do |item|
+          puts "~ #{item[to_path_key]}".blue
+        end
+      end
+
+      if @target.delete?
+        if self.items_to_delete.count > 0
+          puts
+          puts "#{to_type_display_name} to delete: #{self.items_to_delete.count}".bold
+          self.items_to_delete.each do |item|
+            puts "- #{item[to_path_key]}".red
+          end
+        end
+      end
+    end
+  end
+end

data/lib/paramsync/sync_target.rb

@@ -0,0 +1,191 @@
+# This software is public domain. No rights are reserved. See LICENSE for more information.
+
+class Paramsync
+  class SyncTarget
+    VALID_CONFIG_KEYS = %w( name type region prefix path exclude chomp delete erb_enabled account )
+    attr_accessor :name, :type, :region, :prefix, :path, :exclude, :erb_enabled, :account, :ssm
+
+    REQUIRED_CONFIG_KEYS = %w( prefix region account )
+    VALID_TYPES = [ :dir, :file ]
+    DEFAULT_TYPE = :dir
+
+    def initialize(config:, account:, base_dir:)
+      if not config.is_a? Hash
+        raise Paramsync::ConfigFileInvalid.new("Sync target entries must be specified as hashes")
+      end
+
+      if (config.keys - Paramsync::SyncTarget::VALID_CONFIG_KEYS) != []
+        raise Paramsync::ConfigFileInvalid.new("Only the following keys are valid in a sync target entry: #{Paramsync::SyncTarget::VALID_CONFIG_KEYS.join(", ")}")
+      end
+
+      if (Paramsync::SyncTarget::REQUIRED_CONFIG_KEYS - config.keys) != []
+        raise Paramsync::ConfigFileInvalid.new("The following keys are required in a sync target entry: #{Paramsync::SyncTarget::REQUIRED_CONFIG_KEYS.join(", ")}")
+      end
+
+      @base_dir = base_dir
+      self.region = config['region']
+      self.prefix = config['prefix']
+      self.account = config['account']
+      self.path = config['path'] || config['prefix']
+      self.name = config['name']
+      self.type = (config['type'] || Paramsync::SyncTarget::DEFAULT_TYPE).to_sym
+      unless Paramsync::SyncTarget::VALID_TYPES.include?(self.type)
+        raise Paramsync::ConfigFileInvalid.new("Sync target '#{self.name || self.path}' has type '#{self.type}'. But only the following types are valid: #{Paramsync::SyncTarget::VALID_TYPES.collect(&:to_s).join(", ")}")
+      end
+
+      if self.type == :file and File.directory?(self.base_path)
+        raise Paramsync::ConfigFileInvalid.new("Sync target '#{self.name || self.path}' has type 'file', but path '#{self.path}' is a directory.")
+      end
+
+      self.exclude = config['exclude'] || []
+      if config.has_key?('chomp')
+        @do_chomp = config['chomp'] ? true : false
+      end
+      if config.has_key?('delete')
+        @do_delete = config['delete'] ? true : false
+      else
+        @do_delete = false
+      end
+
+      self.ssm = Aws::SSM::Client.new(
+        region: region,
+        credentials: Aws::AssumeRoleCredentials.new(
+          client: Aws::STS::Client.new(region: region),
+          role_arn: account,
+          role_session_name: "paramsync"
+        ),
+      )
+      self.erb_enabled = config['erb_enabled']
+    end
+
+    def erb_enabled?
+      @erb_enabled
+    end
+
+    def chomp?
+      @do_chomp
+    end
+
+    def delete?
+      @do_delete
+    end
+
+    def description(mode = :push)
+      if mode == :pull
+        "#{self.name.nil? ? '' : self.name.bold + "\n"}#{'ssm'.cyan}:#{self.region.green}:#{self.prefix} => #{'local'.blue}:#{self.path}"
+      else
+        "#{self.name.nil? ? '' : self.name.bold + "\n"}#{'local'.blue}:#{self.path} => #{'ssm'.cyan}:#{self.region.green}:#{self.prefix}"
+      end
+    end
+
+    def clear_cache
+      @base_path = nil
+      @local_files = nil
+      @local_items = nil
+      @remote_items = nil
+    end
+
+    def base_path
+      @base_path ||= File.join(@base_dir, self.path)
+    end
+
+    def local_files
+      # see https://stackoverflow.com/questions/357754/can-i-traverse-symlinked-directories-in-ruby-with-a-glob
+      @local_files ||= Dir["#{self.base_path}/**{,/*/**}/*"].select { |f| File.file?(f) }
+    end
+
+    def local_items
+      return @local_items if not @local_items.nil?
+      @local_items = {}
+
+      case self.type
+      when :dir
+        self.local_files.each do |local_file|
+          @local_items[local_file.sub(%r{^#{self.base_path}/?}, '')] =
+            load_local_file(local_file)
+        end
+
+      when :file
+        if File.exist?(self.base_path)
+          @local_items = local_items_from_file
+        end
+      end
+      @local_items.transform_values! do |val|
+        is_kms = val.bytes[0] == 0x01
+        if is_kms
+          val = Paramsync.config.kms_client.decrypt(
+            ciphertext_blob: val
+          ).plaintext
+        end
+        [val, is_kms]
+      end
+      @local_items
+    end
+
+    def local_items_from_file
+      if erb_enabled?
+        loaded_file = YAML.load(ERB.new(File.read(self.base_path)).result)
+      else
+        loaded_file = YAML.load_file(self.base_path)
+      end
+
+      flatten_hash(nil, loaded_file)
+    end
+
+    def load_local_file(local_file)
+      file = File.read(local_file)
+
+      if self.chomp?
+        encoded_file = file.chomp.force_encoding(Encoding::ASCII_8BIT)
+      else
+        encoded_file = file.force_encoding(Encoding::ASCII_8BIT)
+      end
+
+      return ERB.new(encoded_file).result if erb_enabled?
+      encoded_file
+    end
+
+    def remote_items
+      return @remote_items if not @remote_items.nil?
+      @remote_items = {}
+
+      resp = self.ssm.get_parameters_by_path(
+        path: self.prefix,
+        recursive: true,
+        with_decryption: true
+      )
+
+      return @remote_items if resp.values.nil?
+      resp.flat_map(&:parameters).each do |param|
+        @remote_items[param.name.gsub(self.prefix, '')] = [(param.value.nil? ? '' : param.value), param.type == 'SecureString']
+      end
+
+      @remote_items
+    end
+
+    def diff(mode)
+      Paramsync::Diff.new(target: self, local: self.local_items, remote: self.remote_items, mode: mode)
+    end
+
+    private def flatten_hash(prefix, hash)
+      new_hash = {}
+
+      hash.each do |k, v|
+        if k == '_' && !prefix.nil?
+          new_key = prefix
+        else
+          new_key = [prefix, k].compact.join('.').gsub('/.', '/')
+        end
+
+        case v
+        when Hash
+          new_hash.merge!(flatten_hash(new_key, v))
+        else
+          new_hash[new_key] = v.to_s
+        end
+      end
+
+      new_hash
+    end
+  end
+end

data/lib/paramsync/version.rb

@@ -0,0 +1,5 @@
+# This software is public domain. No rights are reserved. See LICENSE for more information.
+
+class Paramsync
+  VERSION = "0.1.0"
+end

data/paramsync.gemspec

@@ -0,0 +1,31 @@
+require_relative 'lib/paramsync/version.rb'
+
+Gem::Specification.new do |s|
+  s.name = 'paramsync'
+  s.version = Paramsync::VERSION
+  s.authors = ['David Adams', 'Jacob Burroughs']
+  s.email = 'maths22@gmail.com'
+  s.date = Time.now.strftime('%Y-%m-%d')
+  s.license = 'CC0'
+  s.homepage = 'https://github.com/maths22/paramsync'
+  s.required_ruby_version = '>=2.4.0'
+
+  s.summary = 'Simple filesystem-to-aws parameter store synchronization'
+  s.description =
+    'Syncs content from the filesystem to the aws parameter store.  Derived from constancy for consul'
+
+  s.require_paths = ['lib']
+  s.files = Dir["lib/**/*.rb"] + [
+    'bin/paramsync',
+    'README.md',
+    'LICENSE',
+    'paramsync.gemspec'
+  ]
+  s.bindir = 'bin'
+  s.executables = ['paramsync']
+
+  s.add_dependency 'aws-sdk-ssm', '~> 1.89.0'
+  s.add_dependency 'aws-sdk-kms', '~> 1.37.0'
+
+  s.add_development_dependency 'rspec', '~> 3.0'
+end

metadata

@@ -0,0 +1,103 @@
+--- !ruby/object:Gem::Specification
+name: paramsync
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- David Adams
+- Jacob Burroughs
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-02-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-ssm
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.89.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.89.0
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-kms
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.37.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.37.0
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: Syncs content from the filesystem to the aws parameter store.  Derived
+  from constancy for consul
+email: maths22@gmail.com
+executables:
+- paramsync
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- bin/paramsync
+- lib/paramsync.rb
+- lib/paramsync/cli.rb
+- lib/paramsync/cli/check_command.rb
+- lib/paramsync/cli/config_command.rb
+- lib/paramsync/cli/encrypt_command.rb
+- lib/paramsync/cli/pull_command.rb
+- lib/paramsync/cli/push_command.rb
+- lib/paramsync/cli/targets_command.rb
+- lib/paramsync/config.rb
+- lib/paramsync/diff.rb
+- lib/paramsync/sync_target.rb
+- lib/paramsync/version.rb
+- paramsync.gemspec
+homepage: https://github.com/maths22/paramsync
+licenses:
+- CC0
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.4.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.4
+signing_key:
+specification_version: 4
+summary: Simple filesystem-to-aws parameter store synchronization
+test_files: []