houcho 0.0.6 → 0.0.8

data/lib/houcho/outerrole.rb

@@ -0,0 +1,52 @@
+require "houcho/element"
+require "houcho/attribute"
+
+module Houcho
+  class OuterRoleExistenceException < Exception; end
+
+  class OuterRole < Element
+    include Houcho::Attribute
+
+    def initialize
+      super("outerrole")
+      @type_id = 1
+    end
+
+    def details(outer_role)
+      outer_role = outer_role.is_a?(Array) ? outer_role : [outer_role]
+      result = {}
+      outer_role.each do |role|
+        hosts = hostlist(role)
+        if !hosts.empty?
+          result[role] = {}
+          result[role]["host"] = hosts
+        end
+      end
+
+      result
+    end
+
+    def hostlist(outer_role)
+      outer_role = outer_role.is_a?(Array) ? outer_role : [outer_role]
+      hosts = []
+
+      outer_role.each do |role|
+        id = id(role)
+        hosts << @db.execute("
+          SELECT host.name
+          FROM host
+          JOIN outerrole_host ORHOST
+          ON host.id = ORHOST.host_id
+          WHERE ORHOST.outerrole_id = ?
+        ", id)
+      end
+
+      hosts.flatten.uniq.sort
+    end
+
+    private
+    def raise_target_does_not_exist(target)
+      raise OuterRoleExistenceException, "outer role does not exist - #{target}"
+    end
+  end
+end

data/lib/houcho/repository.rb

@@ -0,0 +1,59 @@
+require "houcho/config"
+require "houcho/database"
+
+module Houcho
+  class Repository
+    def self.init
+      [
+        Houcho::Config::APPROOT,
+        Houcho::Config::SPECDIR,
+        Houcho::Config::SCRIPTDIR,
+        Houcho::Config::OUTERROLESOURCEDIR,
+        Houcho::Config::CFYAMLDIR,
+        Houcho::Config::LOGDIR
+      ].each do |d|
+        Dir.mkdir(d) unless Dir.exist?(d)
+      end
+
+      File.write("#{Houcho::Config::FILE}", {
+        "ukigumo" => { "host" => "", "port" => "" },
+        "ikachan" => { "host" => "", "port" => "", "channel" => [] },
+        "git" => { "uri" => "" },
+        "rspec" => [],
+      }.to_yaml) unless File.exist?("#{Houcho::Config::FILE}")
+
+      File.write("#{Houcho::Config::SPECDIR}/spec_helper.rb", <<EOD
+require "serverspec"
+require 'pathname'
+require 'net/ssh'
+require "json"
+
+include Serverspec::Helper::Ssh
+include Serverspec::Helper::DetectOS
+include Serverspec::Helper::Attributes
+
+RSpec.configure do |c|
+  if ENV['ASK_SUDO_PASSWORD']
+    require 'highline/import'
+    c.sudo_password = ask("Enter sudo password: ") { |q| q.echo = false }
+  else
+    c.sudo_password = ENV['SUDO_PASSWORD']
+  end
+
+  c.ssh.close if c.ssh
+  c.host  = ENV['TARGET_HOST']
+  options = Net::SSH::Config.for(c.host)
+  user    = options[:user] || Etc.getlogin
+  c.ssh   = Net::SSH.start(c.host, user, options)
+
+  if ENV['TARGET_HOST_ATTR']
+    attr_set JSON.parse(ENV['TARGET_HOST_ATTR'], { :symbolize_names => true })
+  end
+end
+EOD
+      ) unless File.exist?("#{Houcho::Config::SPECDIR}/spec_helper.rb")
+
+      Houcho::Database.new.create_tables
+    end
+  end
+end

data/lib/houcho/role.rb

@@ -1,128 +1,115 @@
-module Houcho
-  module Role
-    @roles = YamlHandle::Editor.new('./role/roles.yaml')
-
-    module_function
+require "houcho/database"
+require "houcho/host"
+require "houcho/spec"
+require "houcho/outerrole"
+require "houcho/attribute"
 
-    def create(role, exists = [])
-      role   = [role] if role.class == String
-      target = role.shift
+module Houcho
+  class RoleExistenceException < Exception; end
 
-      if self.index(target)
-        exists << target
-      else
-        @roles.data[(@roles.data.keys.max||0) + 1] = target
-      end
+  class Role
+    include Houcho::Attribute
 
-      if role.size == 0
-        @roles.save_to_file
-        raise("role(#{exists.join(',')}) already exist.") if exists.size != 0
-      else
-        self.create(role, exists)
-      end
+    def initialize
+      @db = Houcho::Database.new.handle
+      @type = "role"
+      @type_id = 0
     end
 
 
-    def delete(role, errors = {exists:[], hosts:[], specs:[], cf:[]})
-      role   = [role] if role.class == String
-      target = role.shift
-      index  = self.index(target)
-      del    = true
-
-      if ! index
-        errors[:exists] << target; del = false
-      end
-      if Host.has_data?(index)
-        errors[:hosts] << target if Host.has_data?(index); del = false
-      end
-      if Spec.has_data?(index)
-        errors[:specs] << target if Spec.has_data?(index); del = false
-      end
-      if CloudForecast::Role.has_data?(index)
-        errors[:cf] << target if CloudForecast::Role.has_data?(index); del = false
-      end
-
-      @roles.data.delete(index) if del
-
-      if role.size.zero?
-        @roles.save_to_file
-        e =  []
-        e << "role(#{errors[:exists].join(',')}) does not exist" if ! errors[:exists].size.zero?
-        e << "detach host from #{errors[:hosts].join(',')} before delete" if ! errors[:hosts].size.zero?
-        e << "detach spec from #{errors[:specs].join(',')} before delete" if ! errors[:specs].size.zero?
-        e << "detach cloudforecast's role from #{errors[:cf].join(',')} before delete" if ! errors[:cf].size.zero?
-        raise("#{e.join(', ')}.") if ! e.size.zero?
+    def id(role)
+      if role.is_a?(Regexp)
+        @db.execute("SELECT id, name FROM role").map do |record|
+          record[0] if record[1] =~ role
+        end
       else
-        self.delete(role, errors)
+        @db.execute("SELECT id FROM role WHERE name = ?", role).flatten.first
       end
     end
 
 
-    def rename(role, name)
-      index = self.index(role)
-      raise("#{role} does not exist") if ! index
-      raise("#{name} already exist") if self.index(name)
-
-      @roles.data[index] = name
-      @roles.save_to_file
+    def name(id)
+      @db.execute("SELECT name FROM role WHERE id = ?", id).flatten.first
     end
 
 
-    def all
-      YamlHandle::Loader.new('./role/roles.yaml').data.values.sort
+    def exist?(role)
+      !id(role).nil?
     end
 
 
-    def details(roles)
-      result = {}
+    def create(role)
+      role = [role] unless role.is_a?(Array)
 
-      # too lengthy implementation... I think necessary to change...
-      roles = roles.map do |role|
-        if self.index(role)
-          role
-        else
-          self.indexes_regexp(Regexp.new(role)).map {|index|self.name(index)}
+      @db.transaction do
+        role.each do |r|
+          begin
+            @db.execute("INSERT INTO role(name) VALUES(?)", r)
+          rescue SQLite3::ConstraintException, "column name is not unique"
+            raise RoleExistenceException, "role already exist - #{r}"
+          end
         end
-      end.flatten.sort.uniq
-
-      roles.each do |role|
-        index = self.index(role)
-        next if ! index
+      end
+    end
 
-        hosts   = Host.elements(index)
-        specs   = Spec.elements(index)
-        cfroles = CloudForecast::Role.elements(index)
-        cfhosts = CloudForecast::Role.details(cfroles)
 
-        r         = {}
-        r['host'] = hosts   if ! hosts.empty?
-        r['spec'] = specs   if ! specs.empty?
-        r['cf']   = cfhosts if ! cfhosts.empty?
+    def delete(role)
+      role = [role] unless role.is_a?(Array)
 
-        result[role] = r
+      @db.transaction do
+        role.each do |r|
+          raise RoleExistenceException, "role does not exist - #{r}" unless exist?(r)
+          @db.execute("DELETE FROM role WHERE name = ?", r)
+        end
       end
-
-      result
     end
 
 
-    def index(role)
-      @roles.data.invert[role]
+    def rename(exist_role, name)
+      raise RoleExistenceException, "role does not exist - #{exist_role}" unless exist?(exist_role)
+      raise RoleExistenceException, "role already exist - #{name}" if exist?(name)
+      @db.execute("UPDATE role SET name = '#{name}' WHERE name = '#{exist_role}'")
     end
 
 
-    def indexes_regexp(role)
-      @roles.data.select {|index, rolename| rolename =~ role }.keys
+    def list
+      @db.execute("SELECT name FROM role").flatten
     end
 
 
-    def name(index)
-      @roles.data[index]
-    end
+    def details(role)
+      role = role.is_a?(Array) ? role : [role]
+      result = {}
+      hostobj = Host.new
+      specobj = Spec.new
+      orobj = OuterRole.new
+
+      role.each do |r|
+        id = id(r)
+        next if ! id
+        id = id.is_a?(Array) ? id : [id]
+
+        id.each do |i|
+          hosts = hostobj.list(i)
+          specs = specobj.list(i)
+          outerroles = orobj.list(i)
+          outerhosts = orobj.details(outerroles)
+
+          tmp = {}
+          tmp["host"] = hosts unless hosts.empty?
+          tmp["spec"] = specs unless specs.empty?
+          tmp["outer role"] = outerhosts unless outerhosts.empty?
+
+          result[r] = tmp
+        end
+      end
 
+      result
+    end
 
-    def exist?(role)
-      ! self.index(role).nil?
+    private
+    def raise_target_does_not_exist(target)
+      raise RoleExistenceException, "role does not exist - #{target}"
     end
   end
 end

data/lib/houcho/spec/runner.rb

@@ -1,125 +1,196 @@
+require "yaml"
+require "systemu"
+require "json"
+require "logger"
+require "parallel"
+require "houcho/role"
+require "houcho/spec"
+require "houcho/ci"
+require "houcho/config"
+
 module Houcho
-  module Spec::Runner
-    module_function
 
-    def prepare(roles, ex_hosts, hosts, specs)
-      runlist        = {}
-      spec_not_exist = []
+class Spec
+  class Runner
+    def initialize
+      @role = Houcho::Role.new
+      @host = Houcho::Host.new
+      @spec = Houcho::Spec.new
+      @outerrole = Houcho::OuterRole.new
+      @specdir = Houcho::Config::SPECDIR
+      @logger = Logger.new(Houcho::Config::SPECLOG, 10)
+    end
 
-      spec_check = Proc.new do |specs|
-        p = specs.map {|spec|'spec/' + spec + '_spec.rb'}.partition {|spes|File.exist?(spes)}
-        spec_not_exist += p[1].map {|e|e.sub(/^spec\//,'').sub(/_spec.rb$/,'')}
-        p[0]
-      end
 
-      Role.details(roles).each do |role, detail|
-        r         = {}
-        r['spec'] = spec_check.call(detail['spec']||[])
-        r['host'] = detail['host']||[]
+    def check(spec, host_count, dry_run = false, console_output = false) #dry_run for test
+      spec = spec.is_a?(Array) ? spec : [spec]
+      role = spec.map { |s| @spec.details(s)[s]["role"] }.flatten
+      host = []
 
-        (detail['cf']||{}).each do |cfrole, value|
-          r['host'] += value['host']
-        end
-        r['host'] -= ex_hosts
+      @role.details(role).each do |rolename, value|
+        host.concat(value["host"]).uniq if value["host"]
 
-        runlist[role] = r
+        if value["outer role"]
+          value["outer role"].each do |outerrolename, v|
+            host.concat(v["host"]).uniq if v["host"]
+          end
+        end
       end
 
-      m         = {}
-      m['host'] = hosts if ! hosts.empty?
-      m['spec'] = spec_check.call(specs) if ! specs.empty?
+      execute_manually(host.sample(host_count), spec, dry_run, console_output)
+    end
+
+
+    def execute_manually(host, spec, dryrun = false, console_output = false)
+      host = host.is_a?(Array) ? host : [host]
+      spec = spec.is_a?(Array) ? spec : [spec]
+
+      run(
+        { rand(36**50).to_s(36) => { "host" => host, "spec" => spec } },
+        dryrun,
+        false,
+        console_output
+      )
+    end
+
+
+    def execute_role(role, ex_host = [], dryrun = false, console_output = false)
+      role = role.is_a?(Array) ? role : [role]
+      ex_host = ex_host.is_a?(Array) ? ex_host : [ex_host]
+
+      role_valiables = {}
 
-      runlist[:'run manually'] = m if m != {}
+      @role.details(role).each do |rolename, value|
+        next unless value["spec"]
 
-      if ! spec_not_exist.empty?
-        raise "spec(#{spec_not_exist.join(',')}) file not exist in ./spec directory."
+        rv = {}
+        rv["host"] = []
+        rv["spec"] = []
+        rv["outer role"] = []
+
+        rv["spec"].concat(value["spec"]).uniq
+        rv["host"].concat(value["host"]).uniq if value["host"]
+
+        if value["outer role"]
+          rv["outer role"].concat(value["outer role"].keys).uniq
+          value["outer role"].each do |outerrolename, v|
+            rv["host"].concat(v["host"]).uniq if v["host"]
+          end
+        end
+
+        rv["host"] = rv["host"] - ex_host
+        role_valiables[rolename] = rv
       end
-      runlist
+
+      run(role_valiables, dryrun, true, console_output)
     end
 
 
-    def exec(roles, ex_hosts, hosts, specs, ci = {}, dryrun = nil)
+    def run(target, dryrun, ci = false, console_output = false)
       messages = []
-      self.prepare(roles, ex_hosts, hosts, specs).each do |role, v|
-        next if v['spec'].empty?
-        command = "parallel_rspec #{v['spec'].sort.uniq.join(' ')}"
+      failure = false
+
+      target.each do |role, v|
+        @spec.check_existence(v["spec"])
+        spec = v["spec"].map { |spec| "#{@specdir}/#{spec}_spec.rb" }
+        command = "rspec --format documentation #{spec.sort.uniq.join(" ")}"
 
         if dryrun
-          v['host'].each do |host| 
-            messages << "TARGET_HOST=#{host} #{command}"
+          v["host"].each do |host|
+            drymsg = "TARGET_HOST=#{host} #{command}"
+            puts drymsg if console_output
+            messages << drymsg
           end
           next
         end
 
-        v['host'].each do |host|
-          ENV['TARGET_HOST'] = host
-          result = systemu command
-          messages << result[1].scan(/\d* examples?, \d* failures?\n/).first.chomp + "\t#{host}, #{command}\n"
+        Parallel.each(v["host"], :in_threads => Parallel.processor_count)  do |host|
+          attr_role = @role.get_attr(role)
+          attr_host = @host.get_attr(host)
+          attr_outerrole = {}
 
-          post_result(result, role, host, command, ci) if ci != {}
-          $houcho_fail = true if result[0] != 0
+          if v["outer role"]
+            v["outer role"].each do |o|
+              attr_outerrole.merge!(@outerrole.get_attr(o))
+            end
+          end
+
+          attr = attr_role.merge(attr_outerrole)
+          attr = attr.merge(attr_host)
+
+          logmesg = ""
+          if attr != {} && attr_host == {} && v["outer role"].size > 1
+            logmsg = "might not be given the appropriate attribute value, because #{host} have no attributes and belongs to more than one outer role - #{v["outer role"].join(", ")}"
+            @logger.warn(host) { message }
+            logmsg += "\n"
+          end
+
+          result = systemu(
+            command,
+            :env => {
+              "TARGET_HOST" => host,
+              "TARGET_HOST_ATTR" => JSON.generate(attr)
+            },
+            :cwd => File.join(@specdir, "..")
+          )
+
+          @logger.info(host) { "#{result[1]}\n#{result[2]}" }
+          failure = true if result[0] != 0
+
+          msg = result[1].scan(/\d* examples?, \d* failures?\n/).first
+          msg = msg ? msg.chomp : "error"
+          msg += "\t#{host} => #{v["spec"].join(", ")}\n"
+          puts msg if console_output
+
+          if ci
+            begin
+              post_result(result, role, host, v["spec"], command, logmsg)
+            rescue
+              ci = false
+            end
+          end
         end
       end
-      messages
+
+      failure ? false : messages
     end
 
 
-    def post_result(result, role, host, command, ci)
-      conf = YAML.load_file('conf/houcho.conf')
+    private
+    def post_result(result, role, host, spec, command, message)
       result_status = result[0] == 0 ? 1 : 2
 
-      if ci[:ukigumo]
-        ukigumo_report = CI::UkigumoClient.new(conf['ukigumo']['host'], conf['ukigumo']['port']).post({
+      ukigumo = Houcho::Config::UKIGUMO
+      ikachan = Houcho::Config::IKACHAN
+      git = Houcho::Config::GIT
+
+      if ukigumo["host"] != "" && ukigumo["port"] != "" && git["uri"]
+        u = CI::UkigumoClient.new(ukigumo["host"], ukigumo["port"])
+        ukigumo_report = u.post({
           :status   => result_status,
-          :project  => role,
-          :branch   => host.gsub(/\./, '-'),
-          :repo     => conf['git']['uri'],
-          :revision => `git log spec/| grep '^commit' | head -1 | awk '{print $2}'`.chomp,
+          :project  => host.gsub(/\./, "-"),
+          :branch   => role,
+          :repo     => git["uri"],
+          :revision => spec.join(", "),
           :vc_log   => command,
-          :body     => result[1],
+          :body     => ( message || "" ) + result[1],
         })
       end
 
-      if ci[:ikachan] && result_status != 1
-        message  = "[serverspec fail]\`TARGET_HOST=#{host} #{command}\` "
-        message += JSON.parse(ukigumo_report)['report']['url'] if ukigumo_report
-        CI::IkachanClient.new(
-          conf['ikachan']['channel'],
-          conf['ikachan']['host'],
-          conf['ikachan']['port']
-        ).post(message)
-      end
-    end
+      if ikachan["host"] != "" && ikachan["port"] != "" && result_status != 1
+        message = "[serverspec fail] #{host} => #{spec.join(", ")}"
+        message += " (#{JSON.parse(ukigumo_report)["report"]["url"]})" if ukigumo_report
 
+        i = CI::IkachanClient.new(
+          ikachan["channel"],
+          ikachan["host"],
+          ikachan["port"]
+        )
 
-    def check(specs, host_count, dryrun = false) # dryrun is for test
-      specs    = specs.flatten
-      error    = []
-      messages = []
-
-      specs.each do |spec|
-        hosts   = []
-        indexes = Spec.indexes(spec)
-
-        if indexes.empty?
-          error << spec
-          next
-        end
-
-        indexes.each do |index|
-          hosts += Host.elements(index)
-          CloudForecast::Role.elements(index).each do |cfrole|
-            hosts += CloudForecast::Host.new.hosts(cfrole)
-          end
-        end
-        hosts.sample(host_count).each {|host| messages += exec([], [], [host], [spec], {}, dryrun)}
-      end
-
-      if error.empty?
-        messages
-      else
-        raise("role(#{error.join(',')}) has not attached to any roles")
+        i.post(message)
       end
     end
   end
 end
+
+end

data/lib/houcho/spec.rb

@@ -1,6 +1,75 @@
+require "houcho/role"
+require "houcho/element"
+require "houcho/config"
+
 module Houcho
-  class Spec
-    @elements = YamlHandle::Editor.new('./role/specs.yaml')
-    extend Element
+  class SpecFileException < Exception; end
+
+  class Spec < Element
+    def initialize
+      super("serverspec")
+      @specdir = Houcho::Config::SPECDIR
+    end
+
+
+    def check_existence(specs)
+      specs = [specs] unless specs.is_a?(Array)
+      files = specs.partition { |spec| File.exist?("#{@specdir}/#{spec}_spec.rb") }
+      raise SpecFileException, "No such spec file - #{files[1].join(",")}" unless files[1].empty?
+
+      files[0]
+    end
+
+
+    def attach(specs, roles)
+      specs = [specs] unless specs.is_a?(Array)
+      roles = [roles] unless roles.is_a?(Array)
+      files = check_existence(specs)
+
+      super(files, roles)
+    end
+
+
+    def rename(from, to)
+      if File.exist?("#{@specdir}/#{to}_spec.rb")
+        raise SpecFileException, "spec file already exist - #{to}"
+      end
+
+      check_existence(from)
+      File.rename("#{@specdir}/#{from}_spec.rb", "#{@specdir}/#{to}_spec.rb")
+      @db.execute("UPDATE #{@type} SET name = ? WHERE name = ?", to, from)
+    end
+
+
+    def delete(specs, force = false)
+      specs = [specs] unless specs.is_a?(Array)
+
+      detach_from_all(specs) if force
+
+      @db.transaction do
+
+      specs.each do |spec|
+        begin
+          @db.execute("DELETE FROM #{@type} WHERE name = ?", spec)
+        rescue SQLite3::ConstraintException, "foreign key constraint failed"
+          raise SpecFileException, "spec file has been attached to role - #{spec}"
+        end
+      end
+
+      end #end of transaction
+
+      begin
+        check_existence(specs).each do |spec|
+          File.delete("#{@specdir}/#{spec}_spec.rb")
+        end
+      rescue => e
+        raise e.class, "#{e.message}" unless force
+      end
+    end
+
+
+    def delete!(specs, force = true)
+      delete(specs, true)
+    end
   end
 end

data/lib/houcho/version.rb

@@ -1,3 +1,3 @@
 module Houcho
-  VERSION = '0.0.6'
+  VERSION = '0.0.8'
 end