ConfigLMM 0.3.0 → 0.5.0

data/lib/ConfigLMM/io/http.rb

@@ -0,0 +1,78 @@
+
+require 'forwardable'
+require 'webrick'
+
+module ConfigLMM
+    module IO
+
+        class HTTPLogger
+            extend Forwardable
+
+            def_delegators :@Logger, :debug, :info, :warn, :error, :fatal
+
+            def initialize(logger, level)
+                @Logger = logger
+                @Level = level
+            end
+
+            def debug?
+                @Level == 'debug'
+            end
+
+            def <<(message)
+                @Logger.info(message)
+            end
+        end
+
+        class HTTP
+            PORT = 6582
+
+            def initialize(dir, ip, options, logger)
+                @IP = ip
+                @Logger = HTTPLogger.new(logger, options[:level])
+                @LastReadTime = nil
+                requestCallback = Proc.new do |request, response|
+                    @LastReadTime = Time.now
+                    response
+                end
+                @Server = WEBrick::HTTPServer.new(BindAddress: @IP,
+                                                  Port: PORT,
+                                                  DocumentRoot: dir,
+                                                  RequestCallback: requestCallback,
+                                                  Logger: @Logger,
+                                                  AccessLog: { @Logger => WEBrick::AccessLog::COMMON_LOG_FORMAT })
+            end
+
+            def url(path)
+                "http://#{@IP}:#{PORT}/#{path}"
+            end
+
+            def start()
+                Thread.new do
+                    @Server.start
+                end
+            end
+
+            def stop()
+                @Server.stop
+            end
+
+            def wait(timeout)
+                timeoutTime = Time.now + timeout
+                previousReadTime = @LastReadTime
+                loop do
+                    if previousReadTime != @LastReadTime
+                        previousReadTime = @LastReadTime
+                        timeoutTime = Time.now + timeout
+                    end
+                    return if Time.now >= timeoutTime
+                    sleep(0.1)
+                end
+            end
+
+            def hadRead?
+                !@LastReadTime.nil?
+            end
+        end
+    end
+end

data/lib/ConfigLMM/io/local.rb

@@ -0,0 +1,207 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'open3'
+
+module ConfigLMM
+    module IO
+        class Local
+
+            attr_reader :prompt
+            attr_reader :logger
+            attr_reader :local
+
+            def initialize(prompt, logger)
+                @prompt = prompt
+                @logger = logger
+                @local = self
+            end
+
+            def fileWrite(target, data, dry)
+                if dry
+                    prompt.say('Would write file ' + target)
+                else
+                    File.write(target, data)
+                end
+            end
+
+            def copy(source, target, dry)
+                if dry
+                    prompt.say('Would copy ' + source + ' to ' + target)
+                else
+                    FileUtils.cp_r(source, target, noop: dry)
+                end
+            end
+
+            def copyNotPresent(source, target, dry)
+                if !File.exist?(target + File.basename(source))
+                    if dry
+                        prompt.say('Would copy ' + source + ' to ' + target)
+                    else
+                        FileUtils.cp_r(source, target, noop: dry)
+                    end
+                end
+            end
+
+            def rm(path, dry)
+                if dry
+                    prompt.say('Would remove ' + path)
+                else
+                    FileUtils.rm_r(path, noop: dry)
+                end
+            end
+
+            def mkdir(target, dry)
+                if dry
+                    prompt.say('Would create ' + target)
+                else
+                    FileUtils.mkdir_p(target)
+                end
+            end
+
+            def chown(user, group, target, dry)
+                if dry
+                    prompt.say("Would chown #{target} as #{user}:#{group}")
+                else
+                    FileUtils.chown_R(user, group, target)
+                end
+            end
+
+            CONFIGLMM_SECTION_BEGIN = "# -----BEGIN CONFIGLMM-----\n"
+            CONFIGLMM_SECTION_END   = "# -----END CONFIGLMM-----\n"
+
+            def updateFile(file, options, atTop = false, comment = '#')
+                File.write(file, '') unless File.exist?(file)
+                sectionBegin = CONFIGLMM_SECTION_BEGIN.gsub('#', comment)
+                sectionEnd = CONFIGLMM_SECTION_END.gsub('#', comment)
+                fileLines = File.read(file).lines
+                sectionBeginIndex = fileLines.index(sectionBegin)
+                sectionEndIndex = fileLines.index(sectionEnd)
+                if sectionBeginIndex.nil?
+                    linesBefore = []
+                    linesBefore = fileLines unless atTop
+                    linesBefore << "\n"
+                    linesBefore << sectionBegin
+                    linesAfter = [sectionEnd]
+                    linesAfter << "\n"
+                    linesAfter += fileLines if atTop
+                else
+                    linesBefore = fileLines[0..sectionBeginIndex]
+                    if sectionEndIndex.nil?
+                        linesAfter = [sectionEnd]
+                        linesAfter << "\n"
+                    else
+                        linesAfter = fileLines[sectionEndIndex..fileLines.length]
+                    end
+                end
+
+                fileLines = linesBefore
+                fileLines = yield(fileLines)
+                fileLines += linesAfter
+
+                fileWrite(file, fileLines.join(), options[:dry])
+            end
+
+            def exec(command, allowFailure = false, options = {})
+                self.class.exec(command, allowFailure, options, self.prompt, self.logger)
+            end
+
+            def filePresent?(file, options = {})
+                result = self.exec("stat #{file}", true, options)
+                !result.start_with?('stat: cannot')
+            end
+
+            def adminExec(command, allowFailure = false, options = {})
+                if `echo $EUID`.strip == '0'
+                    self.exec(command, allowFailure, options)
+                else
+                    if options['dry']
+                        prompt.say("Would execute: sudo #{command} >/dev/null")
+                    else
+                        self.exec('sudo ' + command, false, options)
+                    end
+                end
+            end
+
+            def download(target, source, options = {})
+                copy(source, target, options[:dry])
+            end
+
+            def upload(source, target, options = {})
+                copy(source, target, options[:dry])
+            end
+
+            def uploadFolder(folder, target, options = {})
+                upload(folder, target, options)
+            end
+
+            def remoteDownload(url, targetDir, options = {})
+                filename = File.basename(Addressable::URI.parse(url).path)
+                targetFile = File.expand_path(targetDir + filename)
+                if !File.exist?(targetFile)
+                    mkdir(File.expand_path(targetDir), false)
+                    prompt.say('Downloading... ' + url)
+                    response = ::HTTP.follow.get(url)
+                    raise "Failed to download file: #{response.status}" unless response.status.success?
+                    File.open(targetFile, 'wb') do |file|
+                        response.body.each do |chunk|
+                            file.write(chunk)
+                        end
+                    end
+                end
+                targetFile
+            end
+
+            def renderTemplate(template, target, outputPath, options)
+                variables = {
+                    config: target,
+                }
+                result = template.result_with_hash(variables)
+                mkdir(File.dirname(outputPath), options['dry'])
+                if options['dry']
+                    prompt.say('Would write to ' + outputPath)
+                else
+                    File.write(outputPath, result)
+                end
+            end
+
+            def self.exec(command, allowFailure = false, options = {}, prompt = nil, logger = nil)
+                if options['dry']
+                    message = "Would execute: #{command}"
+                    if prompt
+                        prompt.say(message)
+                    else
+                        puts message
+                    end
+                    return ''
+                end
+                if options[:hide]
+                    command = ' ' + command
+                    if logger
+                        logger.debug("# **HIDDEN**")
+                    end
+                else
+                    if logger
+                        logger.debug("# #{command}")
+                    end
+                end
+                stdout, stdeerr, status = Open3.capture3(command)
+                if !allowFailure && !status.success?
+                    $stderr.puts(stdout)
+                    $stderr.puts(stdeerr)
+                    raise ExecError.new("Failed '#{command}'", command, stdout, stdeerr, status)
+                end
+                if logger
+                    logger.debug("(#{status.exitstatus})> #{stdout + stdeerr}")
+                end
+                stdout + stdeerr
+            rescue Errno::ENOENT => error
+                if !allowFailure
+                    raise ExecError.new("Failed '#{command}'", command, error, nil, nil)
+                end
+                ''
+            end
+
+        end
+    end
+end

data/lib/ConfigLMM/io/pxe.rb

@@ -0,0 +1,92 @@
+
+require_relative 'dhcp'
+require_relative 'http'
+require_relative 'tftp'
+
+require 'socket'
+
+module ConfigLMM
+    module IO
+        class PXE
+
+            def self.createMac(id)
+                '000101' + Zlib.crc32(id).to_s(16).ljust(8, '0')[2, 6]
+            end
+
+            def self.fillNetworkInfo(networkOptions, dhcp)
+                discoverMessage = dhcp.sendDiscover(self.createMac(networkOptions['ID']))
+                offerMessage = dhcp.waitOffer(discoverMessage, 10)
+                raise 'Didn\'t receive DHCP Offer' unless offerMessage
+                requestMessage = dhcp.sendRequest(discoverMessage, offerMessage)
+                ipinfo, ackMessage = dhcp.waitACK(discoverMessage, requestMessage, 5)
+                raise 'Didn\'t receive DHCP ACK' unless ackMessage
+
+                networkOptions['IP'] = ipinfo.last.ip_address
+                networkOptions['ClientIP'] = [ackMessage.yiaddr].pack('N').unpack('C4').join('.')
+
+                networkOptions['Subnet'] = '255.255.255.0'
+                subnet = ackMessage.options.find { |opt| opt.is_a?(::DHCP::SubnetMaskOption) }
+                networkOptions['Subnet'] = subnet.payload.join('.') if subnet
+
+                broadcast = ackMessage.options.find { |opt| opt.is_a?(::DHCP::BroadcastAddressOption) }
+                networkOptions['Broadcast'] = broadcast.payload.join('.') if broadcast
+
+                dns = ackMessage.options.find { |opt| opt.is_a?(::DHCP::DomainNameServerOption) }
+                networkOptions['DNS'] = dns.payload.join('.') if dns
+
+                router = ackMessage.options.find { |opt| opt.is_a?(::DHCP::RouterOption) }
+                networkOptions['Gateway'] = dns.payload.join('.') if router
+            end
+
+            def self.findMessageByIP(messages, ip)
+                if messages.length > 1
+                    keys = messages.keys.select { |ipinfo| ipinfo.last.ip_address == ip }
+                    raise "Couldn't match DHCP Discover message to respective interface" if keys.empty?
+                else
+                    keys = [messages.keys.first]
+                end
+                [keys.first, messages[keys.first]]
+            end
+
+            def self.boot(dir, uri, networkOptions, bootFileResolver, options, logger)
+                dhcp = DHCP.new(logger)
+                if networkOptions['IP'].nil?
+                    self.fillNetworkInfo(networkOptions, dhcp)
+                end
+
+                useHTTP = uri.scheme == 'pxe+http'
+                server = useHTTP ? HTTP.new(dir, networkOptions['IP'], options, logger) : TFTP.new(dir, networkOptions['IP'], logger)
+                server.start
+
+                discoverMessages = dhcp.waitDiscover(5 * 60, useHTTP)
+                raise 'Timeout while waiting for valid DHCP Discover request!' if discoverMessages.empty?
+                discoverMessageInfo = self.findMessageByIP(discoverMessages, networkOptions['IP'])
+                #offerMessage = dhcp.waitOffer(discoverMessageInfo, 5)
+                isFullDHCP = true #offerMessage.nil?
+
+                clientArch = 0x0000
+                clientArchOption = discoverMessageInfo.last.options.find { |option| option.is_a?(::DHCP::ClientSystemArchitectureOption) }
+                clientArch = clientArchOption.payload.pack('C*').unpack('n').first if clientArchOption
+                bootFile = bootFileResolver.call(clientArch)
+                bootFileSize = File.size(dir + bootFile)
+                bootFile = server.url(bootFile) if useHTTP
+
+                offerRequest = dhcp.sendOffer(discoverMessageInfo, isFullDHCP, networkOptions, bootFile, bootFileSize)
+                if isFullDHCP
+                    requestMessages = dhcp.waitRequest(offerRequest, 10)
+                    requestMessageInfo = self.findMessageByIP(requestMessages, networkOptions['IP'])
+                    raise 'Timeout while waiting for valid DHCP Request!' if requestMessages.empty?
+                    dhcp.sendACK(requestMessageInfo, offerRequest)
+                end
+                if clientArch != 0x0000 && !useHTTP
+                    proxyRequestMessage = dhcp.waitProxyRequest(20)
+                    raise 'Timeout while waiting for valid proxyDHCP Request!' unless proxyRequestMessage
+                    dhcp.sendProxyACK(proxyRequestMessage, offerRequest)
+                end
+                server.wait(3 * 60)
+                raise 'Didn\'t receive boot file read!' unless server.hadRead?
+                server.stop()
+            end
+        end
+    end
+end

data/lib/ConfigLMM/io/ssh.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+require 'net/ssh'
+require 'net/scp'
+require 'securerandom'
+
+module ConfigLMM
+    module IO
+        class SSH
+
+            attr_reader :prompt
+            attr_reader :logger
+            attr_reader :ssh
+
+            def initialize(prompt, logger, ssh)
+                @prompt = prompt
+                @logger = logger
+                @ssh = ssh
+            end
+
+            def rm(path, dry)
+                if dry
+                    prompt.say("Would remove ssh://#{ssh.transport.host}:#{ssh.transport.port}" + path)
+                else
+                    self.class.exec!(ssh, "rm -rf #{path}", false, {}, self.prompt, self.logger)
+                end
+            end
+
+            def exec(command, allowFailure = false, options = {})
+                self.class.exec!(ssh, command, allowFailure, options, self.prompt, self.logger)
+            end
+
+            def adminExec(command, allowFailure = false, options = {})
+                self.exec(command, allowFailure, options)
+            end
+
+            def updateFile(file, options, atTop = false, comment = '#', &block)
+                localFile = options['output'] + '/' + SecureRandom.alphanumeric(10)
+                File.write(localFile, '')
+                self.exec("touch #{file}", false, options)
+                self.download(file, localFile, options)
+                Local.new(self.prompt, self.logger).updateFile(localFile, options, atTop, comment, &block)
+                self.upload(localFile, file, options)
+            end
+
+            def download(source, target, options = {})
+                if options['dry']
+                    prompt.say("Would download scp -P #{ssh.transport.port} #{ssh.transport.host}:#{source} #{target}")
+                else
+                    ssh.scp.download!(source, target)
+                end
+            end
+
+            def upload(source, target, options = {})
+                if options['dry']
+                    prompt.say("Would upload scp -P #{ssh.transport.port} #{source} #{ssh.transport.host}:#{target}")
+                else
+                    ssh.scp.upload!(source, target)
+                end
+            end
+
+            def uploadFolder(folder, target, options = {})
+                target += '/' + File.basename(folder) + '/'
+                Dir[folder + '/*'].each do |file|
+                    if options['dry']
+                        prompt.say("Would upload scp -P #{ssh.transport.port} #{file} #{ssh.transport.host}:#{target + File.basename(file)}")
+                    else
+                        ssh.scp.upload!(file, target + File.basename(file), recursive: true)
+                    end
+                end
+            end
+
+            def shell(user)
+                Shell.new(self, user)
+            end
+
+            def self.tunnel(uri, &block)
+                uri = Addressable::URI.parse(uri) if uri.is_a?(String)
+                server, params = self.toParams(uri)
+                Net::SSH.start(server, nil, params, &block)
+            end
+
+            def self.ping(uri, prompt, logger)
+                server, params = self.toParams(uri)
+                options = Net::SSH.configuration_for(server, true).merge(params)
+                server = options[:host_name] || server
+                options[:timeout] = 3 unless options.key?(:timeout)
+                Net::SSH::Transport::Session.new(server, options)
+                true
+            rescue Errno::EHOSTUNREACH, Errno::ECONNREFUSED, Net::SSH::ConnectionTimeout
+                false
+            end
+
+            def self.toParams(locationUri)
+                server = locationUri.hostname
+                params = {}
+                params[:port] = locationUri.port if locationUri.port
+                params[:user] = locationUri.user if locationUri.user
+                [server, params]
+            end
+
+            def self.exec!(ssh, command, allowFailure = false, options = {}, prompt = nil, logger = nil)
+                if options['dry']
+                    message = "Would execute: ssh #{ssh.transport.host} -p #{ssh.transport.port} '#{command}'"
+                    if prompt
+                        prompt.say(message)
+                    else
+                        puts message
+                    end
+                    return ''
+                end
+                status = {}
+                output = ''
+                if options[:hide]
+                    command = ' ' + command
+                    if logger
+                        logger.debug("[#{ssh.transport.host}:#{ssh.transport.port}]# **HIDDEN**")
+                    end
+                else
+                    if logger
+                        logger.debug("[#{ssh.transport.host}:#{ssh.transport.port}]# #{command}")
+                    end
+                end
+
+                channel = ssh.exec(command, status: status) do |channel, stream, data|
+                    output += data
+                end
+                channel.wait
+                output = output.dup.force_encoding(Encoding::UTF_8)
+                if logger
+                    logger.debug("[#{ssh.transport.host}:#{ssh.transport.port}](#{status[:exit_code]})> #{output}")
+                end
+                if !allowFailure && (status[:exit_code].nil? || !status[:exit_code].zero?) && status[:exit_signal].to_i != 4 # SIGILL... Sometimes this happens for unknown reason
+                    raise ExecError.new("Failed '#{command}'", command, output, output, status)
+                end
+                output
+            end
+
+            def self.cmd(uri)
+                uri = Addressable::URI.parse(uri) if uri.is_a?(String)
+                server, sshParams = self.toParams(uri)
+                cmd = 'ssh '
+                cmd += '-p ' + sshParams[:port] if sshParams[:port]
+                cmd += sshParams[:user] + '@' if sshParams[:port]
+                cmd + server
+            end
+
+            def self.sshSuccess?(ssh, command)
+                status = {}
+                ssh.exec!(command, status)
+                status[:exit_code].zero?
+            end
+
+        end
+    end
+end

data/lib/ConfigLMM/io/tftp.rb

@@ -0,0 +1,105 @@
+
+require 'tftp'
+
+module ConfigLMM
+    module IO
+
+        class TFTPHandler < ::TFTP::Handler::Base
+
+            attr_reader :lastReadTime
+
+            def initialize(dir, opts)
+                @Dir = dir
+                @lastReadTime = nil
+                super(opts)
+            end
+
+            def processRequest(tag, req, sock, src)
+                case req
+                when ::TFTP::Packet::RRQ
+                    if !req.filename.match?(/\w[\w\-\.\/]*/)
+                        log :warn, "#{tag} #{req.filename} - File not found"
+                        sock.send(::TFTP::Packet::ERROR.new(1, 'File not found.').encode, 0)
+                        return false
+                    end
+                    filename = req.filename
+                    loop do
+                        filename = filename.gsub('//', '/')
+                        break unless filename.include?('//')
+                    end
+                    filename = filename.gsub('../', '')
+                    path = @Dir + filename
+                    if File.file?(path)
+                        mode = 'r'
+                        mode += 'b' if req.mode == :octet
+                        io = File.open(path, mode)
+                        log :debug, "#{tag} Sending #{req.filename} - #{path}"
+                        if req.options.key?('tsize')
+                            sendOACK(tag, sock, { 'tsize' => io.stat.size })
+                        end
+                        send(tag, sock, io)
+                        io.close
+                        @lastReadTime = Time.now
+                        return true
+                    else
+                        log :warn, "#{tag} #{req.filename} - File not found"
+                        sock.send(::TFTP::Packet::ERROR.new(1, 'File not found.').encode, 0)
+                    end
+                when ::TFTP::Packet::WRQ
+                    log :info, "#{tag} Denied write request for #{req.filename}"
+                    sock.send(::TFTP::Packet::ERROR.new(2, 'Access denied.').encode, 0)
+                end
+                return false
+            end
+
+            # Handle a session.
+            #
+            # Has to close the socket (and any other resources).
+            #
+            # @param tag  [String]    Tag used for logging
+            # @param req  [Packet]    The initial request packet
+            # @param sock [UDPSocket] Connected socket
+            # @param src  [UDPSource] Initial connection information
+            def run!(tag, req, sock, src)
+                processRequest(tag, req, sock, src)
+                sock.close
+            end
+        end
+
+        class TFTP
+            def initialize(dir, ip, logger, options = {})
+                @Logger = logger
+                @Handler = TFTPHandler.new(dir, { **options, logger: @Logger })
+                @Server = ::TFTP::Server::Base.new(@Handler, { **options, logger: @Logger })
+            end
+
+            def start()
+                Thread.new do
+                    @Server.run!
+                end
+                @Logger.debug('Listening on UDP port 69 (TFTP Server)')
+            end
+
+            def stop()
+                @Server.stop
+            end
+
+            def wait(timeout)
+                timeoutTime = Time.now + timeout
+                previousReadTime = @Handler.lastReadTime
+                loop do
+                    if previousReadTime != @Handler.lastReadTime
+                        previousReadTime = @Handler.lastReadTime
+                        timeoutTime = Time.now + timeout
+                    end
+                    return if Time.now >= timeoutTime
+                    sleep(0.1)
+                end
+            end
+
+            def hadRead?
+                !@Handler.lastReadTime.nil?
+            end
+        end
+    end
+end

data/lib/ConfigLMM/io.rb

@@ -1,2 +1,4 @@
 
 require_relative 'io/configList'
+require_relative 'io/connection'
+require_relative 'io/pxe'

data/lib/ConfigLMM/secrets/envStore.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module ConfigLMM
+    module Secrets
+        class EnvStore
+
+            def initialize(logger, prompt)
+                @Secrets = {}
+                @Logger = logger
+                @Prompt = prompt
+            end
+
+            def getID(id, name)
+                "#{id.upcase}_#{name.upcase.tr('@.', '_')}"
+            end
+
+            def load(id, name)
+                raise "Invalid id! #{id.inspect}" unless id
+                id = getID(id, name)
+                if @Secrets.key?(id)
+                    @Secrets[id]
+                else
+                    ENV[id]
+                end
+            end
+
+            def store(id, name, value)
+                raise "Invalid secret #{value.inspect}!" if !value.is_a?(String) || value.include?("\n")
+                id = getID(id, name)
+                @Secrets[id] = value
+            end
+
+            def print(message, value)
+                @Prompt.say(message + ': ' + value, :color => :magenta)
+            end
+
+        end
+    end
+end