ConfigLMM 0.3.0 → 0.5.0

data/lib/ConfigLMM/Framework/plugins/plugin.rb

@@ -3,17 +3,13 @@
 require_relative 'errors'
 require_relative 'store'
 require 'addressable/uri'
-require 'http'
-require 'fileutils'
-require 'net/ssh'
-require 'net/scp'
-require 'open3'
 
 module ConfigLMM
     module Framework
 
         class Plugin
 
+            REPOS_CACHE = '~/.cache/configlmm/repos'
 
             def self.inherited(plugin)
                 Store.registerPlugin(plugin)
@@ -48,11 +44,13 @@ module ConfigLMM
 
             attr_accessor :state
 
-            def initialize(logger, prompt, plugins)
+            def initialize(context, logger, prompt, plugins)
+                @Context = context
                 @Logger = logger
                 @Prompt = prompt
                 @Plugins = plugins
                 @Diff = {}
+                @Local = IO::Local.new(prompt, logger)
             end
 
             def id
@@ -79,6 +77,10 @@ module ConfigLMM
 
             protected
 
+            def context
+                @Context
+            end
+
             def logger
                 @Logger
             end
@@ -91,119 +93,54 @@ module ConfigLMM
                 @Plugins
             end
 
-            def shouldMatch(id, targetKey, stateKey, target, activeState)
-                if target[targetKey] != activeState[stateKey]
-                    @Diff.update({targetKey => [target[targetKey], activeState[stateKey]]})
+            def local
+                @Local
+            end
+
+            def shouldMatch(id, activeState, stateKey, target, targetKey)
+                data = nil
+                data = activeState[stateKey] if activeState.is_a?(Hash)
+                if target[targetKey] != data
+                    @Diff.update({targetKey => [data, target[targetKey]]})
                 end
             end
 
             def fileWrite(target, data, dry)
-                if dry
-                    prompt.say('Would write file ' + target)
-                else
-                    File.write(target, data)
-                end
+                local.fileWrite(target, data, dry)
             end
 
             def copy(source, target, dry)
-                if dry
-                    prompt.say('Would copy ' + source + ' to ' + target)
-                else
-                    FileUtils.cp_r(source, target, noop: dry)
-                end
+                local.copy(source, target, dry)
             end
 
             def copyNotPresent(source, target, dry)
-                if !File.exist?(target + File.basename(source))
-                    if dry
-                        prompt.say('Would copy ' + source + ' to ' + target)
-                    else
-                        FileUtils.cp_r(source, target, noop: dry)
-                    end
-                end
+                local.copyNotPresent(source, target, dry)
             end
 
-            def rm(path, dry)
-                if dry
-                    prompt.say('Would remove ' + path)
+            def rm(path, dry, ssh = nil)
+                if ssh
+                    IO::SSH.new(prompt, logger, ssh).rm(path, dry)
                 else
-                    FileUtils.rm_r(path, noop: dry)
+                    local.rm(path, dry)
                 end
             end
 
             def mkdir(target, dry)
-                if dry
-                    prompt.say('Would create ' + target)
-                else
-                    FileUtils.mkdir_p(target)
-                end
+                local.mkdir(target, dry)
             end
 
             def chown(user, group, target, dry)
-                if dry
-                    prompt.say("Would chown #{target} as #{user}:#{group}")
-                else
-                    FileUtils.chown_R(user, group, target)
-                end
-            end
-
-            def self.loadVariable(value, target)
-                variableStart = value.index('${')
-                return value unless variableStart
-                variableEnd = value.index('}', variableStart + 2)
-                variableName = value[variableStart + 2...variableEnd]
-                if variableName.start_with?('ENV:')
-                    value = value[0...variableStart].to_s + ENV[variableName[4..variableEnd]] + value[(variableEnd + 1)..-1].to_s
-                else
-                    raise 'Not implemented!'
-                end
-                value
+                local.chown(user, group, target, dry)
             end
 
-            CONFIGLMM_SECTION_BEGIN = "# -----BEGIN CONFIGLMM-----\n"
-            CONFIGLMM_SECTION_END   = "# -----END CONFIGLMM-----\n"
-
             def updateLocalFile(file, options, atTop = false, comment = '#')
-                File.write(file, '') unless File.exist?(file)
-                sectionBegin = CONFIGLMM_SECTION_BEGIN.gsub('#', comment)
-                sectionEnd = CONFIGLMM_SECTION_END.gsub('#', comment)
-                fileLines = File.read(file).lines
-                sectionBeginIndex = fileLines.index(sectionBegin)
-                sectionEndIndex = fileLines.index(sectionEnd)
-                if sectionBeginIndex.nil?
-                    linesBefore = []
-                    linesBefore = fileLines unless atTop
-                    linesBefore << "\n"
-                    linesBefore << sectionBegin
-                    linesAfter = [sectionEnd]
-                    linesAfter << "\n"
-                    linesAfter += fileLines if atTop
-                else
-                    linesBefore = fileLines[0..sectionBeginIndex]
-                    if sectionEndIndex.nil?
-                        linesAfter = [sectionEnd]
-                        linesAfter << "\n"
-                    else
-                        linesAfter = fileLines[sectionEndIndex..fileLines.length]
-                    end
-                end
-
-                fileLines = linesBefore
-                fileLines = yield(fileLines)
-                fileLines += linesAfter
-
-                fileWrite(file, fileLines.join(), options[:dry])
+                local.updateFile(file, options, atTop, comment)
             end
 
+            # DEPRECATED
             def updateRemoteFile(locationOrSSH, file, options, atTop = false, comment = '#', &block)
-
                 closure = Proc.new do |ssh|
-                    localFile = options['output'] + '/' + SecureRandom.alphanumeric(10)
-                    File.write(localFile, '')
-                    self.class.sshExec!(ssh, "touch #{file}")
-                    ssh.scp.download!(file, localFile)
-                    updateLocalFile(localFile, options, atTop, comment, &block)
-                    ssh.scp.upload!(localFile, file)
+                    IO::SSH.new(prompt, logger, ssh).updateFile(file, options, atTop, comment, &block)
                 end
 
                 if locationOrSSH.is_a?(String) || locationOrSSH.is_a?(Addressable::URI)
@@ -218,6 +155,7 @@ module ConfigLMM
                 end
             end
 
+            # DEPRECATED
             def self.filePresent?(file, ssh = nil)
                 result = self.exec("stat #{file}", ssh, true)
                 !result.start_with?('stat: cannot')
@@ -228,69 +166,38 @@ module ConfigLMM
                 self.filePresent?(file, ssh)
             end
 
-            def self.remoteFileContains?(file, content, ssh)
-                !self.sshExec!(ssh, "grep '#{content}' #{file}", true).strip.empty?
-            end
-
-            def self.uploadNotPresent(file, target, ssh)
-                target += '/' + File.basename(file)
-                if !self.remoteFilePresent?(target, ssh)
-                    ssh.scp.upload!(file, target)
-                end
-            end
-
             def self.uploadFolder(folder, target, ssh)
                 target += '/' + File.basename(folder) + '/'
-                self.sshExec!(ssh, "mkdir -p #{target}")
+                IO::SSH.exec!(ssh, "mkdir -p #{target}")
                 Dir[folder + '/*'].each do |file|
                     ssh.scp.upload!(file, target + File.basename(file), recursive: true)
                 end
             end
 
-            def self.exec(command, ssh = nil, allowFailure = false)
-                if ssh.nil?
-                    stdout, stdeerr, status = Open3.capture3(command)
-                    if !allowFailure && !status.success?
-                        $stderr.puts(stdout)
-                        $stderr.puts(stdeerr)
-                        raise Framework::PluginProcessError.new("Failed '#{command}'")
-                    end
-                    stdout + stdeerr
-                else
-                    self.sshExec!(ssh, command, allowFailure)
-                end
+            def withConnection(uri, target, &block)
+                IO::Connection.tunnel(uri, target, self.context, self.prompt, self.logger, &block)
             end
 
-            def self.toSSHparams(locationUri)
-                server = locationUri.hostname
-                params = {}
-                params[:port] = locationUri.port if locationUri.port
-                params[:user] = locationUri.user if locationUri.user
-                [server, params]
+            def ping(uri, target, &block)
+                IO::Connection.ping(uri, target, self.context, self.prompt, self.logger, &block)
             end
 
-            def self.sshStart(uri)
-                uri = Addressable::URI.parse(uri) if uri.is_a?(String)
-                server, sshParams = self.toSSHparams(uri)
-                Net::SSH.start(server, nil, sshParams) do |ssh|
-                    yield(ssh)
-                end
+            # DEPRECATED
+            def self.exec(command, ssh = nil, allowFailure = false, dry = false)
+                IO::Connection.exec(command, ssh, allowFailure, { dry: dry })
             end
 
-            def self.sshExec!(ssh, command, allowFailure = false)
-                status = {}
-                output = ''
-                channel = ssh.exec(command, status: status) do |channel, stream, data|
-                    output += data
-                end
-                channel.wait
-                if !allowFailure && !status[:exit_code].zero?
-                    $stderr.puts(output)
-                    raise Framework::PluginProcessError.new("Failed '#{command}'")
-                end
-                output
+            # DEPRECATED
+            def self.sshStart(uri, &block)
+                IO::SSH.tunnel(uri, &block)
+            end
+
+            # DEPRECATED
+            def self.sshExec!(ssh, command, allowFailure = false, dry = false)
+                IO::SSH.exec!(ssh, command, allowFailure, { dry: dry })
             end
 
+            # DEPRECATED
             def renderTemplate(template, target, outputPath, options)
                 variables = {
                     config: target,
@@ -304,6 +211,27 @@ module ConfigLMM
                 end
             end
 
+            def cleanupType(type, configs, state, context, options)
+                items = state.selectType(type)
+                items.each do |id, item|
+                    if !configs.key?(id) && item['Status'] != State::STATUS_DESTROYED && (item['Status'] != State::STATUS_DELETED || options[:destroy])
+                        begin
+                            self.withConnection(item['Config']['Location'], item) do |connection|
+                                yield(item, id, state, context, options, connection)
+                            end
+                        rescue SystemCallError => error
+                            if error.errno == Errno::EHOSTUNREACH::Errno
+                                prompt.say("#{id}: #{item[:Type].to_s} failed to connect #{item['Config'].to_h['Location'].to_s}", color: :red)
+                                prompt.say(error, color: :red)
+                                prompt.say("Skipping!", color: :red)
+                            else
+                                raise
+                            end
+                        end
+                    end
+                end
+            end
+
             def self.normalizeId(id)
                 id = id.split('::').last
                 if id.downcase.end_with?('plugin')

data/lib/ConfigLMM/Framework/plugins/store.rb

@@ -37,19 +37,19 @@ module ConfigLMM
                 end
             end
 
-            def self.boot(logger, prompt, plugins)
+            def self.boot(context, logger, prompt, plugins)
                 leafPlugins.each do |id, plugin|
-                    self.initPlugin(id, logger, prompt, plugins)
+                    self.initPlugin(id, context, logger, prompt, plugins)
                 rescue PluginLoadError => error
                     logger.warn("Plugin '#{id}' failed to load!\n#{error.message}" + (error.cause ? '  - ' : ''), error.cause)
                 end
                 true
             end
 
-            def self.initPlugin(pluginId, logger, prompt, plugins)
+            def self.initPlugin(pluginId, context, logger, prompt, plugins)
                 pluginId = pluginId.to_sym
                 raise 'Recursive/cyclic plugin' if plugins.key?(pluginId)
-                plugins[pluginId] = @@AvailablePlugins[pluginId].new(logger, prompt, plugins)
+                plugins[pluginId] = @@AvailablePlugins[pluginId].new(context, logger, prompt, plugins)
             end
 
         end

data/lib/ConfigLMM/Framework/variables.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module ConfigLMM
+    module Framework
+        class Variable
+
+            attr_reader :name
+
+            def initialize(name, args, context)
+                @name = name
+                @args = args
+                @context = context
+            end
+
+            def eval()
+                if name == 'ENV'
+                    value = ENV[@args.first]
+                    raise "Environment variable #{@args.first} not found!" if value.nil?
+                    value
+                elsif name == 'SECRET'
+                    value = @context.secrets.load(*@args)
+                    raise "Secret #{@args.join(':')} not found!" if value.nil?
+                    value
+                elsif name == 'GENERATE'
+                    secretId, secretName, length = @args
+                    raise 'Not enough arguments for GENERATE!' if secretId.to_s.empty? || secretName.to_s.empty?
+                    value = @context.secrets.load(secretId, secretName)
+                    if value.nil?
+                        length = 30 unless length
+                        value = SecureRandom.alphanumeric(length)
+                        @context.secrets.store(secretId, secretName, value)
+                    end
+                    value
+                else
+                    raise "Unsupported function #{name}!"
+                end
+            end
+
+            def to_s()
+                eval()
+            end
+        end
+
+        def Value
+            def initialize(value)
+                @value = value
+            end
+
+            def to_s()
+                @value.to_s
+            end
+        end
+
+        class Variables
+            def self.parse(value, context)
+                if value.to_s[0, 2] == '${'
+                    raise "Unterminated variable: #{value}" if value[-1] != '}'
+                    parts = value[2...-1].split(':')
+                    Variable.new(parts.shift, parts, context)
+                else
+                    Value.new(value)
+                end
+            end
+
+            def self.stringEval(data, context)
+                variableStart = data.index('${')
+                return data unless variableStart
+                variableEnd = data.index('}', variableStart + 2)
+                raise "Unterminated variable: #{data}" if variableEnd.nil?
+                parts = data[variableStart + 2...variableEnd].split(':')
+                data[0...variableStart].to_s + Variable.new(parts.shift, parts, context).to_s + data[(variableEnd + 1)..-1].to_s
+            end
+        end
+    end
+end

data/lib/ConfigLMM/Framework.rb

@@ -7,3 +7,4 @@ end
 
 require_relative 'Framework/registrator'
 require_relative 'Framework/plugins'
+require_relative 'Framework/variables'

data/lib/ConfigLMM/cli.rb

@@ -14,7 +14,7 @@ module ConfigLMM
         MissingArgument = Class.new(ArgumentError)
         InvalidOption = Class.new(ArgumentError)
 
-        #class_option :locations, type: :string, default: '', group: :configs, desc: 'Filter by config file locations'
+        class_option :locations, type: :string, default: '', group: :configs, desc: 'Filter by config file locations'
         #class_option :things, type: :string, default: '', group: :configs, desc: 'Filter things to use'
         class_option :level, type: :string, enum: ['debug', 'info', 'warn', 'error'], default: 'info', desc: 'Logging level to use'
         class_option :dry,  aliases: '-n', type: :boolean, desc: 'Only show actions without performing'
@@ -45,6 +45,8 @@ module ConfigLMM
         desc 'refresh [CONFIGS...]', 'Update local state to match deployed things'
         method_option :help,    aliases: '-h', type: :boolean,
                                 desc: 'Display usage information'
+        method_option :secrets, aliases: '-s', type: :string, default: '.secrets',
+                                desc: 'Path to the secrets provider (can be a file)'
         method_option :state,   aliases: '-s', type: :string,
                                 desc: 'Path to the state file'
         method_option :context, aliases: '-c', type: :string,
@@ -82,6 +84,8 @@ module ConfigLMM
         desc 'deploy [CONFIGS...]', 'Deploy configuration'
         method_option :help,    aliases: '-h', type: :boolean,
                                 desc: 'Display usage information'
+        method_option :secrets, aliases: '-s', type: :string, default: '.secrets',
+                                desc: 'Path to the secrets provider (can be a file)'
         method_option :state,   aliases: '-s', type: :string,
                                 desc: 'Path to the state file'
         method_option :context, aliases: '-c', type: :string,
@@ -95,10 +99,14 @@ module ConfigLMM
 
 
         desc 'cleanup [CONFIGS...]', 'In deployed infrastructure cleanup/delete unused things (eg. deployment leftover junk) (note this can be risky due to mistakes)'
-        method_option :help,  aliases: '-h', type: :boolean,
-                              desc: 'Display usage information'
-        method_option :state, aliases: '-s', type: :string,
-                              desc: 'Path to the state file'
+        method_option :help,    aliases: '-h', type: :boolean,
+                                desc: 'Display usage information'
+        method_option :secrets, aliases: '-s', type: :string, default: '.secrets',
+                                desc: 'Path to the secrets provider (can be a file)'
+        method_option :state,   aliases: '-s', type: :string,
+                                desc: 'Path to the state file'
+        method_option :destroy, aliases: '-d', type: :boolean,
+                                desc: 'Destroy all data (eg. even DB)'
         def cleanup(*configPaths)
             handleCommand(:cleanup, configPaths, options)
         end

data/lib/ConfigLMM/commands/cleanup.rb

@@ -12,6 +12,7 @@ module ConfigLMM
                     loadConfigs(plugin, config, configs)
                     plugin.cleanup(configs, state, context, options)
                 end
+                state.save
                 prompt.ok('Cleanup successful!')
             end
 

data/lib/ConfigLMM/commands/configsCommand.rb

@@ -2,11 +2,11 @@
 
 require_relative '../command'
 require_relative '../utils/filters'
-require_relative '../io/configList'
 require_relative '../context'
 require_relative '../state'
 require_relative '../Framework'
 require_relative '../LMM'
+require_relative '../io'
 require 'xdg'
 require 'tmpdir'
 
@@ -23,7 +23,7 @@ module ConfigLMM
                     config.level = @Options[:level]
                 end
 
-                @Context = Context.new(logger, prompt, ::XDG.new, @Options[:context])
+                @Context = Context.new(logger, prompt, ::XDG.new, @Options)
                 @State = State.new(logger, prompt)
                 @Diff = {}
 
@@ -31,7 +31,7 @@ module ConfigLMM
                 Framework::Registrator.registerAll(logger)
 
                 # Create Plugin instances
-                Framework::Store.boot(logger, prompt, @Plugins)
+                Framework::Store.boot(@Context, logger, prompt, @Plugins)
                 @Plugins.each do |id, plugin|
                     plugin.state = @State
                 end
@@ -43,7 +43,7 @@ module ConfigLMM
                 options = @Options.dup
                 options.delete(:locations)
                 options.delete(:things)
-                #options[:locationFilter] = Utils::Filters.parseLocationsOption(@Options[:locations], logger)
+                options[:locationFilter] = Utils::Filters.parseLocationsOption(@Options[:locations], logger)
                 #options[:thingFilter] = Utils::Filters.parseThingsOption(@Options[:things], logger)
 
                 configList = IO::ConfigList.create(@ConfigPaths, logger)
@@ -89,6 +89,10 @@ module ConfigLMM
                 elsif activeState[:Type] != singleTarget['Type'].to_s
                     raise Framework::PluginError.new("Unexpected Type #{activeState[:Type].inspect}! Wanted #{singleTarget['Type']}")
                 end
+
+                singleTarget['Location'] = '@me' unless singleTarget['Location']
+                singleTarget['SecretId'] = (singleTarget['SecretId'] || id).upcase
+
                 actionMethod = plugin.class.actionMethod(singleTarget['Type'], 'Refresh')
                 if plugin.methods.include?(:authenticate)
                     result = plugin.authenticate(actionMethod, singleTarget, state, context, options)
@@ -124,6 +128,10 @@ module ConfigLMM
                 elsif activeState[:Type] != singleTarget['Type'].to_s
                     raise Framework::PluginError.new("Unexpected Type #{activeState[:Type].inspect}! Wanted #{singleTarget['Type']}")
                 end
+
+                singleTarget['Location'] = '@me' unless singleTarget['Location']
+                singleTarget['SecretId'] = (singleTarget['SecretId'] || id).upcase
+
                 actionMethod = plugin.class.actionMethod(singleTarget['Type'], 'Deploy')
                 if plugin.methods.include?(:authenticate)
                     result = plugin.authenticate(actionMethod, singleTarget, state, context, options)
@@ -146,7 +154,7 @@ module ConfigLMM
                     end
 
                     if !options['dry']
-                     # Prevent others accessing it
+                        # Prevent others accessing it
                         FileUtils.chmod(0750, options['output'])
                     end
 
@@ -156,9 +164,34 @@ module ConfigLMM
 
                     plugin.send(actionMethod, id, singleTarget, activeState, context, options)
                 end
+                activeState['Config'] = self.class.sanitizeConfig(singleTarget)
+                activeState['Config'].delete(:Parent)
+                activeState['Config'].delete('Resources')
+                activeState['Config'] = activeState['Config'].sort.to_h
+                activeState['Status'] = State::STATUS_DEPLOYED if !activeState['Status'] || [State::STATUS_DELETED, State::STATUS_DESTROYED].include?(activeState['Status'])
                 state.save
             end
 
+            def self.sanitizeConfig(config)
+                return config unless config.is_a?(Hash)
+                newConfig = config.dup
+                config.each do |key, value|
+                    lowerkey = key.to_s.downcase
+                    if lowerkey.include?('password') || lowerkey.include?('privatekey') || lowerkey.include?('sharedkey')
+                        newConfig.delete(key)
+                    elsif value.is_a?(Hash)
+                        newConfig[key] = self.sanitizeConfig(value)
+                    elsif value.is_a?(Array)
+                        newValue = value
+                        value.each_with_index do |v, i|
+                            newValue[i] = self.sanitizeConfig(v)
+                        end
+                        newConfig[key] = newValue
+                    end
+                end
+                newConfig
+            end
+
 
             def checkDiff(id, targetName, stateName, target, activeState)
                 if target[targetName] != activeState[stateName]

data/lib/ConfigLMM/commands/diff.rb

@@ -22,22 +22,46 @@ module ConfigLMM
             end
 
             def showDiff(configDiffs)
+                level = 4
                 configDiffs.each do |id, diffs|
                     prompt.say(' ' + id + ':')
                     diffs.each do |name, diff|
-                        if diff.first.is_a?(Hash)
-                            prompt.say('     ' + name + ':')
-                            diff.first.each do |name, value|
-                                prompt.say('-      ' + name + ': ' + value, :color => :red)
+                        outputDiff(name.to_s, ' ', diff.first, diff.last, level)
+                    end
+                end
+            end
+
+            def outputDiff(name, type, old, new, level)
+                indent = ' ' * level
+                if old.is_a?(Hash) || new.is_a?(Hash)
+                    if type == '-'
+                        prompt.say('-'+ indent + name + ':', :color => :red)
+                    elsif type == '+'
+                        prompt.say('+'+ indent + name + ':', :color => :green)
+                    else
+                        prompt.say(' '+ indent + name + ':')
+                    end
+                    if !old.nil?
+                        old.each do |name, value|
+                            if value.is_a?(Hash)
+                                outputDiff(name.to_s, '-', value, nil, level + 4)
+                            else
+                                prompt.say('-' + (' ' * (level + 4)) + name.to_s + ': ' + value.to_s, :color => :red)
                             end
-                            diff.last.each do |name, value|
-                                prompt.say('+      ' + name + ': ' + value, :color => :green)
+                        end
+                    end
+                    if !new.nil?
+                        new.each do |name, value|
+                            if value.is_a?(Hash)
+                                outputDiff(name.to_s, '+', nil, value, level + 4)
+                            else
+                                prompt.say('+' + (' ' * (level + 4)) + name.to_s + ': ' + value.to_s, :color => :green)
                             end
-                        else
-                            prompt.say('-    ' + name + ': ' + diff.first, :color => :red)
-                            prompt.say('+    ' + name + ': ' + diff.last, :color => :green)
                         end
                     end
+                else
+                    prompt.say('-' + indent + name + ': ' + old.to_s, :color => :red) unless old.nil?
+                    prompt.say('+' + indent + name + ': ' + new.to_s, :color => :green) unless new.nil?
                 end
             end
         end