canzea 0.1.0

data/lib/plan-step-class.rb

@@ -0,0 +1,136 @@
+require 'json'
+require 'logger'
+require 'trace-component'
+require 'helper-run-class'
+
+class PlanStep
+    def runPhaseInstall (role, solution, test, task)
+
+        log = Logger.new(Canzea::config[:logging_root] + '/plans.log')
+
+        # ruby plan.rb <plan description> <test run> <start>
+
+        plan = JSON.parse("{ \"plan\": [ { \"role\": \"#{role}\", \"solution\": \"#{solution}\" } ] }")
+
+        n = Worker.new
+
+
+        n.test ( test )
+
+        start = Integer(task)
+        lines = 1
+
+        cmd = "TBD"
+
+
+        begin
+
+            plan['plan'].each do |item|
+                log.info(item['solution'])
+
+                root = ENV['CANZEA_PLUGINS'] + "ike-environments/environment/production/roles/#{item['role']}/#{item['solution']}"
+                if File.exist?(root) == false
+                    puts "-- ERROR #{root} does not exist!"
+                    raise "#{root} does not exist!"
+                end
+
+                cmd = ENV['CANZEA_PLUGINS'] + "ike-environments/environment/production/roles/#{item['role']}/#{item['solution']}/install.sh"
+
+                if File.exist?(cmd)
+                    lines = n.run cmd, start, lines - 1
+
+                    log.info "#{ lines } lines read"
+                end
+
+
+            end
+        rescue => exception
+          log.error(cmd)
+          log.error(exception.to_s)
+          log.error(exception.backtrace)
+          abort()
+        end
+    end
+
+    def runPhaseConfigure (role, solution, test, task)
+
+        log = Logger.new(Dir.home() + '/plans.log')
+
+        # ruby plan.rb <plan description> <test run> <start>
+
+        plan = JSON.parse("{ \"plan\": [ { \"role\": \"#{role}\", \"solution\": \"#{solution}\" } ] }")
+
+        n = Worker.new
+
+
+        n.test ( test )
+
+        start = Integer(task)
+        lines = 1
+
+        cmd = "TBD"
+
+
+        begin
+
+            plan['plan'].each do |item|
+                log.info(item['solution'])
+
+                root = ENV['CANZEA_PLUGINS'] + "ike-environments/environment/production/roles/#{item['role']}/#{item['solution']}"
+                if File.exist?(root) == false
+                    puts "-- ERROR #{root} does not exist!"
+                    raise "#{root} does not exist!"
+                end
+
+                # Register the service with Consul, if consul is ready
+                # If metadata.json exists, then use the information to register
+                cmd = ENV['CANZEA_PLUGINS'] + "ike-environments/environment/production/roles/#{item['role']}/#{item['solution']}/metadata.json"
+                if File.exist?(cmd)
+                    md = File.read(cmd)
+                    md = JSON.parse(md)
+                    if (md['services'].size() > 0)
+                        svc = md['services'][0]
+                        adef = {"listener"=>svc['listener'], "serviceName"=>svc['name'], "tags"=>{}, "port"=>svc['port']}
+                        h = HelperRun.new
+                        h.run "consul", "add_service", JSON.generate(adef)
+                    end
+                end
+
+                envScript = ENV['CANZEA_PLUGINS'] + "ike-environments/environment/production/roles/#{item['role']}/#{item['solution']}/environment.json"
+                if File.exist?(envScript)
+                    require_relative '../init/prepare_environment'
+
+                    envPush = PrepareEnvironment.new
+                    envPush.addToEnv "#{envScript}"
+                end
+
+                cmd = ENV['CANZEA_PLUGINS'] + "ike-environments/environment/production/roles/#{item['role']}/#{item['solution']}/configure.sh"
+                if File.exist?(cmd)
+                  lines = n.run cmd, start, lines - 1
+
+                  log.info("#{ lines } lines read")
+                end
+
+                cmd = ENV['CANZEA_PLUGINS'] + "ike-environments/environment/production/roles/#{item['role']}/#{item['solution']}/enable.sh"
+                if File.exist?(cmd)
+                  lines = n.run cmd, start, lines - 1
+
+                  log.info("#{ lines } lines read")
+                end
+
+                cmd = ENV['CANZEA_PLUGINS'] + "ike-environments/environment/production/roles/#{item['role']}/#{item['solution']}/status.sh"
+                if File.exist?(cmd)
+                  lines = n.run cmd, start, lines - 1, true
+
+                  log.info("#{ lines } lines read")
+                end
+
+            end
+        rescue => exception
+          log.error(cmd)
+          log.error(exception.to_s)
+          log.error(exception.backtrace)
+          abort()
+        end
+    end
+end

data/lib/prepare-environment.rb

@@ -0,0 +1,121 @@
+require 'json'
+require 'logger'
+require 'net/http'
+require "trace-runner"
+require "registry"
+
+
+class PrepareEnvironment
+
+    def addToEnv (envFile)
+        log = Logger.new(Canzea::config[:logging_root] + '/plans.log')
+
+        r = Registry.new
+
+        # ENV["ENV_CONFIG"] instead of a file
+        file = File.read(envFile)
+
+        serviceLookups = JSON.parse(file)
+
+        serviceLookups['keyvalues'].each do |svc|
+
+          log.info("Looking up...#{svc['name']}")
+
+          s = r.getKeyValue(svc['name'])
+
+          if (s.nil? == false)
+            # puts s
+
+            key = "" + svc['name'].upcase
+            key = key.gsub(/\./, '').gsub(/-/, '_').gsub(/\//, '_')
+            log.info("Setting: " + key + " : " + s)
+            ENV.store(key, s)
+          else
+            log.warn("KEY VALUE NOT FOUND! " + svc['name'])
+            puts "-- KEY VALUE NOT FOUND! " + svc['name']
+            abort()
+          end
+        end
+
+
+        serviceLookups['services'].each do |svc|
+
+          puts "Looking up...#{svc['name']}"
+
+          url = ENV["CONSUL_URL"] + '/v1/catalog/service/' + svc['name']
+          resp = Net::HTTP.get_response(URI.parse(url))
+          service = JSON.parse(resp.body)
+
+          # s = r.getValue(svc['name'], "publicIp")
+
+          s = service[0]
+          if (s.nil? == false)
+            # puts s
+
+            key = "" + svc['name'].upcase + "_ADDRESS"
+            key = key.gsub(/\./, '')
+            log.info("Setting: " + key + " : " + s["ServiceAddress"])
+            ENV.store(key, s["ServiceAddress"])
+
+            key = "" + svc['name'].upcase + "_PORT"
+            key = key.gsub(/\./, '')
+            log.info("Setting: " + key + " : " + String(s["ServicePort"]))
+            ENV.store(key, String(s["ServicePort"]))
+
+            key = "" + svc['name'].upcase + "_URL"
+            key = key.gsub(/\./, '')
+            val = "http://" + s["ServiceAddress"] + ":" + String(s["ServicePort"])
+            log.info("Setting: " + key + " : " + val)
+            ENV.store(key, val)
+
+          else
+            log.warn("SERVICE NOT FOUND! " + svc['name'])
+            puts "-- SERVICE NOT FOUND! " + svc['name']
+            abort()
+          end
+        end
+
+        serviceLookups['secrets'].each do |svc|
+
+          # puts "Looking up secret...#{svc['name']}"
+
+          url = URI(ENV['VAULT_URL'] + '/v1/secret/' + svc['name'])
+
+          req = Net::HTTP::Get.new(url)
+          req['X-Vault-Token'] = ENV["VAULT_TOKEN"]
+          req['Content-type'] = 'application/json'
+
+          resp = Net::HTTP.start(url.hostname, url.port) {|http|
+             http.request(req)
+          }
+          # puts resp.body
+
+          if (Integer(resp.code) == 200)
+
+            data = JSON.parse(resp.body)
+
+            s = data['data']
+            s.each do |k|
+              key = "" + svc['name'].upcase + "_" + k[0].upcase
+              key = key.gsub(/\./, '').gsub(/-/, '_').gsub(/\//, '_')
+
+              log.info("Setting: " + key)
+              ENV.store(key, k[1])
+            end
+          else
+            log.warn( "SECRET NOT FOUND! " + svc['name'])
+            log.warn( "ERROR FROM VAULT " + resp.body)
+            puts "-- SECRET NOT FOUND! " + svc['name']
+            puts "-- ERROR FROM VAULT " + resp.body
+            abort()
+          end
+        end
+
+    end
+
+    def print ()
+        ENV.each_pair { |name,value|
+            puts name + " = " + value
+        }
+    end
+end

data/lib/registry.rb

@@ -0,0 +1,120 @@
+require 'patron'
+require 'json'
+require 'base64'
+
+class Registry
+
+   def getKeyValue (key)
+
+        sess = Patron::Session.new do |patron|
+            patron.timeout = 10
+            patron.base_url = ENV["CONSUL_URL"] + '/v1/kv'
+            # patron.headers = {'User-Agent' => 'IKE-Registry/1.0'}
+        end
+
+        #sess.enable_debug
+        resp = sess.get("/#{key}")
+
+        # puts resp.body
+
+        if resp.status != 200
+            abort("problem #{resp.status_line}")
+        end
+
+        result = JSON.parse(resp.body)
+        return Base64.decode64(result[0]["Value"])
+   end
+
+   def getValue (root, key)
+
+        sess = Patron::Session.new do |patron|
+            patron.timeout = 10
+            patron.base_url = ENV["CONSUL_URL"] + '/v1/kv'
+            # patron.headers = {'User-Agent' => 'IKE-Registry/1.0'}
+        end
+
+        #sess.enable_debug
+        resp = sess.get("/#{root}/#{key}")
+
+        # puts resp.body
+
+        if resp.status != 200
+            abort("problem #{resp.status_line}")
+        end
+
+        result = JSON.parse(resp.body)
+        return Base64.decode64(result[0]["Value"])
+   end
+
+
+   def exists (root)
+
+        sess = Patron::Session.new do |patron|
+            patron.timeout = 10
+            patron.base_url = ENV["CONSUL_URL"] + '/v1/kv'
+            patron.headers = {'User-Agent' => 'IKE-Registry/1.0'}
+        end
+
+        # sess.enable_debug
+        resp = sess.get("/#{root}")
+        puts resp.body
+
+        if resp.status == 200
+            abort("problem #{resp.status_line}")
+        end
+   end
+
+   def deleteDirectory (root)
+
+        sess = Patron::Session.new do |patron|
+            patron.timeout = 10
+            patron.base_url = ENV["CONSUL_URL"] + '/v1/kv'
+            patron.headers = {'User-Agent' => 'IKE-Registry/1.0'}
+        end
+
+        # sess.enable_debug
+        resp = sess.delete("/#{root}?&recurse=true")
+        puts resp.body
+
+        if resp.status != 200
+            abort("problem #{resp.status_line}")
+        end
+
+   end
+
+   def registerDirectory (root)
+
+        sess = Patron::Session.new do |patron|
+            patron.timeout = 10
+            patron.base_url = ENV["CONSUL_URL"] + '/v1/kv'
+            patron.headers = {'User-Agent' => 'IKE-Registry/1.0'}
+        end
+
+        # sess.enable_debug
+        resp = sess.put("/#{root}?dir=true", "")
+        puts resp.body
+
+        if resp.status > 202
+            abort("problem #{resp.status_line}")
+        end
+
+   end
+
+   def register (root, key, value)
+
+        sess = Patron::Session.new do |patron|
+            patron.timeout = 10
+            patron.base_url = ENV["CONSUL_URL"] + '/v1/kv'
+            patron.headers = {'User-Agent' => 'IKE-Registry/1.0'}
+        end
+
+        #sess.enable_debug
+        resp = sess.put("/#{root}/#{key}", "#{value}")
+        puts resp.body
+
+        if resp.status > 202
+            abort("problem #{resp.status_line}")
+        end
+
+    end
+end

data/lib/ssh-base-cmd-class.rb

@@ -0,0 +1,72 @@
+# canzea --role=operatingsystem --solution=centos --remote --hostname=192.34.56.119 --privateKey=/var/go/.ssh/id_rsa_digitalocean
+
+require 'net/ssh'
+require 'net/sftp'
+require 'json'
+
+class RemoteCall
+    def init (hostname, privateKey)
+        cmd1 = "yum -y install git"
+
+        cmd2 = "(rm -rf ike-environments && git clone https://IKE_CI:55665566@gitlab.com/ikethecoder/ike-environments.git)"
+
+        cmd3 = 'find /root/ike-environments -name "*.sh" -exec chmod +x {} +'
+
+        cmd4 = '(cd /root/ike-environments/environment/production && ./roles/workarounds/ruby/install.sh)'
+        cmd5 = '(cd /root/ike-environments/environment/production && ./roles/workarounds/ruby-gems/install.sh)'
+        cmd6 = '(cd /root/ike-environments/environment/production && ./roles/workarounds/sshd/configure.sh)'
+        cmd7 = '(cd /root/ike-environments/environment/production && ./roles/workarounds/sshd/enable.sh)'
+        cmd8 = '(cd /root/ike-environments/environment/production && ./roles/workarounds/canzea/install-root.sh)'
+
+        exec hostname, privateKey, cmd1
+        exec hostname, privateKey, cmd2
+        exec hostname, privateKey, cmd3
+        exec hostname, privateKey, cmd4
+        exec hostname, privateKey, cmd5
+        exec hostname, privateKey, cmd6
+        exec hostname, privateKey, cmd7
+        exec hostname, privateKey, cmd8
+    end
+
+    def run (hostname, privateKey, cmd)
+        @cmd = ". .bash_profile && cd .local/bin/ike-environments/environment/production && #{cmd}"
+        exec hostname, privateKey, @cmd
+    end
+
+    def exec (hostname, privateKey, cmd)
+
+        @username = "root"
+
+        Net::SSH.start(hostname, @username, :paranoid => false, :keys => [privateKey]) do |ssh|
+
+            chan = ssh.open_channel do |channel|
+                channel[:data] = ""
+                channel.request_pty
+                channel.env("DIGITAL_OCEAN_API_KEY", ENV['DIGITAL_OCEAN_API_KEY'])
+                channel.env("VAULT_TOKEN", ENV['VAULT_TOKEN'])
+                channel.env("CONSUL_URL", ENV['CONSUL_URL'])
+                channel.env("WORK_DIR", ENV['WORK_DIR'])
+                channel.exec(cmd) do |ch, success|
+                    abort "could not execute command" unless success
+
+                    channel.on_data do |ch, data|
+                      channel[:data] << data
+                    end
+
+                    channel.on_request("exit-status") do |ch, data|
+                        exit_code = data.read_long
+                        if (exit_code == 0)
+                        else
+                            abort(channel[:data])
+                        end
+                    end
+
+                    channel.on_close do |ch|
+                        puts channel[:data]
+                    end
+                end
+            end
+            chan.wait
+        end
+    end
+end

data/lib/template-runner.rb

@@ -0,0 +1,37 @@
+require 'json'
+require 'mustache'
+
+class Template < Mustache
+  def processAndWriteToFile (template, output, parameters)
+     contents = process template, parameters
+     File.write(output, contents)
+  end
+
+  def process (template, parameters)
+     self.template_file = template
+     ENV.each_pair do |k, v|
+       self[k] = v
+     end
+     parameters.each_pair do |k, v|
+       self[k] = v
+     end
+     return self.render
+  end
+
+  def processString (string, parameters)
+     self.template = string
+     ENV.each_pair do |k, v|
+       self[k] = v
+     end
+     parameters.each_pair do |k, v|
+       self[k] = v
+     end
+     return self.render
+  end
+end
+
+# t = Template.new
+
+# t.process ARGV[0], ARGV[1], JSON.parse(ARGV[2])
+
+# puts "Wrote to #{ARGV[1]}"

data/lib/trace-component.rb

@@ -0,0 +1,177 @@
+require 'json'
+require 'open3'
+require 'stringio'
+require 'logger'
+
+# Optionally can specify the command to continue from
+# Exit on first failure
+#
+def time_diff_milli(start, finish)
+   (finish - start) * 1000.0
+end
+
+def handleIO(stillOpen, ioArray, io, log)
+  if ioArray.include?(io)
+    begin
+      log.write(io.readpartial(4096))
+    rescue EOFError
+      stillOpen.delete_if{|s| s == io}
+    end
+  end
+end
+
+
+class Audit
+    def start (id, cmd)
+        self.log( id, cmd, "start", "", 0, "")
+    end
+
+    def complete (id, cmd, status, msecs, result)
+	    self.log( id, cmd, "complete", status, msecs, result)
+    end
+
+    def status (id, cmd, status, msecs, result)
+        data = {
+            "message" => {
+                "id" => id,
+                "cmd" => cmd,
+                "task" => "status",
+                "status" => status,
+                "elapsed" => msecs,
+                "result" => result
+            }
+        }
+        puts data.to_json
+        File.open(Canzea::config[:logging_root] + '/audit.log', 'a') { |file| file.puts(data.to_json) }
+    end
+
+    def log (id, cmd, task, status, msecs, result)
+        data = {
+            "message" => {
+                "id" => id,
+                "cmd" => cmd,
+                "task" => task,
+                "status" => status,
+                "elapsed" => msecs,
+                "result" => result
+            }
+        }
+        summary = {
+            "message" => {
+                "id" => id,
+                "cmd" => cmd,
+                "task" => task,
+                "status" => status,
+                "elapsed" => msecs
+            }
+        }
+        puts summary.to_json
+        File.open(Canzea::config[:logging_root] + '/audit.log', 'a') { |file| file.puts(data.to_json) }
+    end
+end
+
+class ManagedError < StandardError
+end
+
+class Worker
+    @test = false
+
+
+    def test (t)
+        @test = t
+    end
+
+    def run(command, start, lines, statusIndicator = false )
+        audit = Audit.new
+
+	    log = Logger.new(Canzea::config[:logging_root] + '/plans.log')
+
+        log.info("HANDLING: " + command)
+
+        File.foreach(command) { |l|
+            if ( l.start_with?('## ') )
+                log.info "#{lines + 1} Label: " + l
+            elsif ( l.start_with?('#') )
+            elsif ( /\S/ !~ l )
+            elsif ( l.chomp.end_with?(".sh") && !l.chomp.end_with?(".atomic.sh") && !l.include?("cp ") && !l.include?("chmod") )
+                log.info("#{lines + 1} RECURSE: " + l)
+
+                lines = run l.chomp, start, lines
+            else
+
+                if ( (lines + 1) < start )
+                    log.info("Skipping : #{lines + 1} #{l}")
+                    lines += 1
+                    next
+                end
+
+
+                audit.start( "#{lines + 1 }", l.chomp)
+
+                log.info("#{lines + 1} FOUND: " + l)
+
+                if @test == false
+                    t1 = Time.now
+
+                    begin
+                        puts "-- Executing: #{l}"
+                        Open3.popen3(l) {|stdin, stdout, stderr, wait_thr|
+                          pid = wait_thr.pid # pid of the started process.
+
+                          log_w = StringIO.new
+
+                          stillOpen = [stdout, stderr]
+                          while !stillOpen.empty?
+                            fhs = select(stillOpen, nil, nil, nil)
+                            handleIO(stillOpen, fhs[0], stdout, log_w)
+                            handleIO(stillOpen, fhs[0], stderr, log_w)
+                          end
+
+                          exit_status = wait_thr.value # wait for it to finish
+
+                          log_w.close_write
+
+                          output = log_w.string
+
+                          output.split(/\n/).each do | line |
+                              puts "-- #{line}"
+                          end
+                          puts "-- Exit Status = #{exit_status}"
+
+                          log.info("STDOUT #{output}")
+
+                          log.info("Exit Status = #{exit_status}")
+
+                          # if exit status is failure then exit right away
+
+                          t2 = Time.now
+
+                          msecs = time_diff_milli t1, t2
+
+                          audit.complete("#{lines + 1}", l.chomp, exit_status.exitstatus, msecs, output)
+
+                          if (statusIndicator)
+                              audit.status("#{lines + 1}", l.chomp, exit_status.exitstatus, msecs, output)
+                          end
+
+                          if exit_status.exitstatus != 0
+                            abort()
+                          end
+                        }
+                    rescue => exception
+                        msecs = time_diff_milli t1, Time.now
+                        audit.complete("#{lines + 1}", l.chomp, "-1", msecs, exception.to_s)
+                        raise
+                    end
+                else
+                    puts "-- TEST -- [#{lines + 1}] FOUND: " + l
+                end
+
+            end
+
+            lines += 1
+        }
+        return lines
+
+    end
+end