hive-runner 2.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0a9df8820b8d030fa96b76d253723ccae821feb1
+  data.tar.gz: 7c2b1f9e1f75d269652e7e996ca515a135b92349
+SHA512:
+  metadata.gz: 9e1d1ebb42336cc59d7f5f37fe98392887cadce6c23ef9787c4b4b7767e26a74f8adcfc40f23bd56eb0ea8c8b3a242a2aa5d215ac5dcd140a77fd534b70c61bb
+  data.tar.gz: cbc8e222372513f478cf809dc297e2b1b04511e48b00b9e498dc03d2cb5f72dca396c5cf4c57f49982b7a2697fdc59ad6c05ec388838d64e177c2cb1ad2c8c7b

data/README.md

@@ -0,0 +1,120 @@
+# hive-runner
+
+Run automated jobs on devices
+
+## Quick start
+
+Install the hive-runner gem and set up your hive:
+
+    gem install hive-runner
+    hive_setup my_hive
+
+Follow the configuration instructions and, in particular, ensure that the
+`HIVE_CONFIG` variable is set.
+
+Start the Hive daemon:
+
+    hived start
+
+Determine the status of the Hive:
+
+    hived status
+
+Stop the Hive:
+
+    hived stop
+
+## Configuration file
+
+Example config file:
+
+    test:
+      controllers:
+        shell:
+          max_workers: 5
+          name_stub: SHELL_WORKER
+          queues:
+            - bash
+    
+      logging:
+        directory: logs
+        pids: pids
+        main_filename: hive.log
+    
+      timings:
+        worker_loop_interval: 5
+
+### Controllers
+
+The `controllers` section contains details about the controllers to be
+used by the hive. The name of each section indicates the controller type. Some
+of the fields in each controllers section is common to all controller types
+(see below) while some are defined for each specific controller type.
+
+Fields for all controller types are:
+
+| Field             | Content                                   |
+|-------------------|-------------------------------------------|
+| `max_workers`     | Maximum number of workers to use          |
+| `port_range_size` | Number of ports to allocate to the device |
+| `name_stub`       | Stub for name of the worker process       |
+
+### Ports
+
+| Field     | Content             |
+|-----------|---------------------|
+| `minimum` | Minimum port number |
+| `maximum` | Maximum port number |
+
+### Logging
+
+| Field           | Content                   |
+|-----------------|---------------------------|
+| `directory`     | Log file directory        |
+| `pids`          | PIDs directory            |
+| `main_filename` | Name of the main log file |
+
+### Timings
+
+The `worker_loop_interval` indicates the number of seconds to wait between each
+poll of the job queue in the worker loop.
+
+## Shell controller
+
+### Configuration
+
+The shell controller section contains the following additional field:
+
+| Field     | Content                                   |
+|-----------|-------------------------------------------|
+| `queues`  | Array of job queues for the shell workers |
+| `workers` | Number of shell workers to run            |
+
+## Setting up a new Hive Runner
+
+Check out the Hive Runner from Github:
+
+    # Using HTTPS
+    git clone https://github.com/bbc-test/hive-runner
+    # ... or using SSH
+    git clone ssh@github.com:bbc-test/hive-runner
+    # Ensure ruby gems are installed
+    cd hive-runner
+    bundle install
+
+Configure the hive, either by editing the default configuration file,
+`hive-runner/config/hive-runner.yml`, or creating a separate configuration
+file in a separate location (recommended) and ensuring that the `HIVE_CONFIG`
+environment variable is set correctly:
+
+    echo export HIVE_CONFIG=/path/to/hive-config-directory >> ~/.bashrc
+
+See the "Configuration file" above for details.
+
+Start the Hive Runner:
+
+    ./bin/hived start
+
+Ensure that your Hive Runner is running and that your workers have started:
+
+    ./bin/hived status

data/bin/hive_setup

@@ -0,0 +1,182 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'fileutils'
+require 'terminal-table'
+
+if ARGV.length < 1
+  puts "Usage:"
+  puts
+  puts "  hive_setup <directory>"
+  exit 1
+end
+
+dir = File.expand_path('', ARGV[0])
+if Dir.exists?(dir)
+  if ! Dir["#{dir}/*"].empty?
+    puts "Directory '#{dir}' exists and is not empty"
+    exit 1
+  end
+else
+  if File.exists?(dir)
+    puts "'#{dir}' exists and is not a directory"
+    exit 1
+  end
+  FileUtils.mkdir_p(dir)
+end
+
+def yn question
+  yn = 'x'
+  while yn !~ /^[yYnN]/
+    print "#{question}(y/n) "
+    yn = STDIN.gets.chomp
+  end
+  yn =~ /^[yY]/
+end
+
+# Choose options
+opt = ''
+mods = []
+while opt.upcase != 'X'
+  table = Terminal::Table.new headings: ['Device', 'Module', 'Source']
+  mods.each do |mod|
+    table.add_row [
+      mod[:device],
+      mod[:name],
+      mod.has_key?(:git_account) ?
+        "git@github.com:#{mod[:git_account]}/#{mod[:name]}" :
+        "https://rubygems.org/gems/#{mod[:name]}"
+    ]
+  end
+
+  puts ''
+  puts table
+  puts ''
+  puts '1) Add module'
+  puts 'X) Continue'
+  puts ''
+  print "> "
+  opt = STDIN.gets.chomp
+
+  case opt
+  when '1'
+    mod = {}
+    puts ''
+    print "Module name: "
+    mod[:device] = STDIN.gets.chomp
+    mod[:name] = "hive-runner-#{mod[:device]}"
+    puts ''
+    if yn "Get '#{mod[:name]}' from Github? "
+      print "Enter GIT account name: "
+      mod[:git_account] = STDIN.gets.chomp
+    end
+
+    puts ''
+    puts "Module '#{mod[:name]}'"
+    if mod.has_key?(:git_account)
+      puts "  from git@github.com:#{mod[:git_account]}/#{mod[:name]}"
+    else
+      puts "  from https://rubygems.org/gems/#{mod[:name]}"
+    end
+    puts ''
+
+    if yn "Correct? "
+      mods << mod
+    end
+  end
+end
+
+FileUtils.mkdir_p("#{dir}/config")
+FileUtils.mkdir_p("#{dir}/log")
+FileUtils.mkdir_p("#{dir}/pids")
+FileUtils.mkdir_p("#{dir}/workspaces")
+
+File.open("#{dir}/config/settings.yml", 'w') do |f|
+  f.puts "#{ENV['HIVE_ENVIRONMENT'] || 'test'}:"
+  f.puts '  daemon_name: HIVE'
+  f.puts ''
+  f.puts '  controllers:'
+  f.puts '    shell:'
+  f.puts '      # Number of shell workers to allocate'
+  f.puts '      workers: 5'
+  f.puts '      # Queue for each shell worker'
+  f.puts '      queues:'
+  f.puts '        - bash'
+  f.puts '      # Number of ports to allocate to each shell worker'
+  f.puts '      port_range_size: 50'
+  f.puts '      name_stub: SHELL_WORKER'
+  mods.each do |m|
+    f.puts "    #{m[:device]}:"
+    f.puts '      # Number of ports to allocate to each #{m[:device]} worker'
+    f.puts '      port_range_size: 50'
+    f.puts "      name_stub: #{m[:device].upcase}_WORKER"
+  end
+  f.puts ''
+  f.puts '  # Range of ports to be made available to workers'
+  f.puts '  ports:'
+  f.puts '    minimum: 4000'
+  f.puts '    maximum: 5000'
+  f.puts ''
+  f.puts '  # Logging configuration'
+  f.puts '  logging:'
+  f.puts "    directory: #{dir}/log"
+  f.puts "    pids: #{dir}/pids"
+  f.puts '    main_filename: hive.log'
+  f.puts '    main_level: INFO'
+  f.puts '    worker_level: INFO'
+  f.puts "    home: #{dir}/workspaces"
+  f.puts '    homes_to_keep: 5'
+  f.puts ''
+  f.puts '  # Timing configuration'
+  f.puts '  timings:'
+  f.puts '    worker_loop_interval: 5'
+  f.puts '    controller_loop_interval: 5'
+  f.puts ''
+  f.puts '  # Configuration for various network options'
+  f.puts '  network:'
+  f.puts '    scheduler: https://scheduler'
+  f.puts '    devicedb: https://devicedb'
+  f.puts '    cert: /path/to/certificate.pem'
+  f.puts '    cafile: /path/to/certificate-authorities.pem'
+  f.puts ''
+  f.puts '  # Configuration for diagnostic plugins'
+  f.puts '  diagnostics:'
+end
+
+File.open("#{dir}/Gemfile", 'w') do |f|
+  f.puts "source 'https://rubygems.org/'"
+  f.puts ""
+  f.puts "gem 'hive-runner'"
+  mods.each do |m|
+    source = m.has_key?(:git_account) ? ", git: 'git@github.com:#{m[:git_account]}/#{m[:name]}'" : ''
+    f.puts "gem '#{m[:name]}'#{source}"
+  end
+end
+
+print "Executing 'bundle install' ... "
+Dir.chdir dir
+if system("bundle install > bundle_install.out 2>&1")
+  print "SUCCESS\n"
+  File.delete('bundle_install.out')
+else
+  print "FAILED\n"
+  puts "See #{dir}/bundle_install.out for details"
+  exit
+end
+
+puts ''
+puts 'Configuration required:'
+puts
+puts '  * Add to config/settings.yml'
+puts '      - scheduler'
+puts '      - devicedb'
+puts '      - cert'
+puts '      - cafile'
+if mods.length > 0
+  puts '  * Configure these modules in config/settings.yml'
+  mods.each do |m|
+    puts "      - #{m[:device]}"
+  end
+end
+puts '  * Add to ~/.bashrc'
+puts "      - export HIVE_CONFIG=#{dir}/config"

data/bin/hived

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path(
+  '../../Gemfile',
+  Pathname.new(__FILE__).realpath
+)
+
+require 'rubygems'
+require 'bundler/setup'
+require 'socket'
+require 'daemons'
+
+# require 'hive' here to cause a failure if the configuration is not correct
+$LOAD_PATH << File.expand_path('../../lib', __FILE__)
+require 'hive'
+
+def test(app)
+  puts ''
+  app.default_show_status
+
+  Process.kill('USR1', app.pid.pid)
+  puts
+  server = TCPSocket.open('localhost', ENV.fetch('HIVE_COMM_PORT', 9999).to_i)
+  while line = server.gets
+    puts line.chop
+  end
+  server.close
+end
+
+Daemons.run(File.expand_path('..', __FILE__) + '/start_hive',
+            show_status_callback: :test,
+            log_dir: Hive.config.logging.directory,
+            log_output: true,
+            output_logfilename: 'hived.out')

data/bin/start_hive

@@ -0,0 +1,120 @@
+#!/usr/bin/env ruby
+
+puts "*** Starting Hive at #{Time.now} ***"
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path(
+  '../../Gemfile',
+  Pathname.new(__FILE__).realpath
+)
+
+$LOAD_PATH << File.expand_path('../../lib', __FILE__)
+
+require 'socket'
+require 'terminal-table'
+require 'hive'
+
+$PROGRAM_NAME = Hive::DAEMON_NAME
+
+# Version information
+gems = {}
+Gem::Specification.find_all_by_name(/hive-*/).each do |g|
+  gems[g.name] = g.version
+end
+
+# Communication with daemon launcher
+comm_port = ENV.fetch('HIVE_COMM_PORT', 9999).to_i
+comm = TCPServer.open(comm_port)
+Signal.trap('USR1') do
+  client = comm.accept
+  client.puts "  Controllers in use:"
+  Hive.register.controllers.each do |c|
+    client.puts "    * #{c.class.name.split('::').last.downcase}"
+  end
+
+  client.puts
+
+  client.puts "  Software versions:"
+  client.puts "    * hive-runner: #{gems['hive-runner']}"
+  gems.except('hive-runner').sort.each do |name, version|
+    client.puts "    * #{name}: #{version}"
+  end
+  
+  client.puts
+
+  # Create a hash of connected device details
+  devices = Hive.register.devices
+  device_details = {}
+  client.puts "  Total number of devices: #{devices.length}"
+  if devices.length > 0
+    devices.each do |d|
+      device_details[d.identity] = { worker: d.claimed? ? 'Claimed' : d.worker_pid }
+    end
+  end
+  client.puts ''
+
+  # Collect up the queue information per device/worker
+  device_details.each do |d, details|
+    if pid = details[:worker]
+      begin
+        queue_file = "#{Hive.config.logging.directory}/#{pid}.queues.yml"
+        details[:queues] = YAML.load( File.open(queue_file) )
+      rescue
+        details[:queues] = [ "---" ]
+      end
+    end
+  end
+
+  # Get a seperate hash of workspace details
+  workspaces_list = {}
+  Dir["#{Hive.config.logging.home}/*"].each do |d|
+    if File.directory?(d)
+      if File.exists?("#{d}/job_info")
+        File.open("#{d}/job_info") do |f|
+          if f.read =~ /^(\d*)\s*(\S*)$/
+            worker = $1 || '?'
+            state = $2 || '?'
+            if workspaces_list.has_key?(worker)
+              workspaces_list[worker][File.basename(d)] = state
+            else
+              workspaces_list[worker] = {File.basename(d) => state}
+            end
+          end
+        end
+      end
+    end
+  end
+
+  # Build the table from all that data just collected
+  table = Terminal::Table.new headings: ['Device', 'Worker', 'Job', 'Status', 'Queues']
+  device_details.each do |d, details|
+    table.add_separator
+    jobs = states = [ '---' ]
+    if workspaces_list.has_key?(details[:worker].to_s)
+      jobs = workspaces_list[details[:worker].to_s].keys
+      states = jobs.map { |key| workspaces_list[details[:worker].to_s][key] }
+      workspaces_list.delete(details[:worker].to_s)
+    end
+    table.add_row [d, details[:worker], jobs.join("\n"), states.join("\n"), details[:queues].join("\n")]
+  end
+  workspaces_list.each do |worker, list|
+    table.add_separator
+    col1 = '---'
+    col2 = worker
+    col5 = '---'
+    list.each do |job, state|
+      table.add_row [col1, col2, job, state, col5]
+      col1 = col2 = ''
+    end
+  end
+  client.puts table
+
+  client.close
+end
+
+# Initialise
+Hive.register.instantiate_controllers
+
+# Execution loop
+Hive::logger.info('*** HIVE STARTING ***')
+Hive::logger.info('Starting execution loop')
+Hive.register.run

data/lib/hive/controller/shell.rb

@@ -0,0 +1,23 @@
+require 'hive/controller'
+require 'hive/worker/shell'
+
+module Hive
+  class Controller
+    # The Shell controller
+    class Shell < Controller
+      def initialize(options)
+        Hive.logger.debug("options: #{options.inspect}")
+        @workers = options['workers'] || 0
+        super
+      end
+
+      def detect
+        Hive.logger.info('Creating shell devices')
+        (1..@workers).collect do |i|
+          Hive.logger.info("  Shell device #{i}")
+          self.create_device('id' => i)
+        end
+      end
+    end
+  end
+end

data/lib/hive/controller.rb

@@ -0,0 +1,32 @@
+require 'hive'
+
+module Hive
+  # Generic hive controller class
+  class Controller
+    attr_reader :port_range_size
+
+    class DeviceDetectionFailed < StandardError
+    end
+
+    class NoPortsAvailable < StandardError
+    end
+
+    def initialize(config = {})
+      @config = config
+      @device_class = self.class.to_s.sub('Controller', 'Device')
+      require @device_class.downcase.gsub(/::/, '/')
+      Hive.logger.info("Controller '#{self.class}' created")
+      @port_range_size = (@config.has_key?('port_range_size') ? @config['port_range_size'] : 0)
+    end
+
+    def create_device(extra_options = {})
+      object = Object
+      @device_class.split('::').each { |sub| object = object.const_get(sub) }
+      object.new(@config.merge(extra_options))
+    end
+
+    def detect
+      raise NotImplementedError, "'detect' method not defined for '#{self.class}'"
+    end
+  end
+end

data/lib/hive/device/shell.rb

@@ -0,0 +1,14 @@
+require 'hive/device'
+
+module Hive
+  class Device
+    # The Shell device
+    class Shell < Device
+      def initialize(config)
+        Hive.logger.info("    In the shell device constructor")
+        @identity = config['id']
+        super
+      end
+    end
+  end
+end

data/lib/hive/device.rb

@@ -0,0 +1,88 @@
+require 'hive'
+require 'hive/port_allocator'
+
+module Hive
+  # The generic device class
+  class Device
+    attr_reader :type
+    attr_accessor :status
+    attr_accessor :port_allocator
+
+    # Initialise the device
+    def initialize(options)
+      @worker_pid = nil
+      @options = options
+      @port_allocator = options['port_allocator'] or Hive::PortAllocator.new(ports: [])
+      @status = @options.has_key?('status') ? @options['status'] : 'none'
+      @worker_class = self.class.to_s.sub('Device', 'Worker')
+      require @worker_class.downcase.gsub(/::/, '/')
+      raise ArgumentError, "Identity not set for #{self.class} device" if ! @identity
+    end
+
+    # Start the worker process
+    def start
+      parent_pid = Process.pid
+      @worker_pid = Process.fork do
+        object = Object
+        @worker_class.split('::').each { |sub| object = object.const_get(sub) }
+        object.new(@options.merge('parent_pid' => parent_pid, 'device_identity' => self.identity, 'port_allocator' => self.port_allocator))
+      end
+      Process.detach @worker_pid
+
+      Hive.logger.info("Worker started with pid #{@worker_pid}")
+    end
+
+    # Terminate the worker process
+    def stop
+      begin
+        count = 0
+        while self.running? && count < 30 do
+          count += 1
+          Hive.logger.info("Attempting to terminate process #{@worker_pid} [#{count}]")
+          Process.kill 'TERM', @worker_pid
+          sleep 30
+        end
+        Process.kill 'KILL', @worker_pid if self.running?
+      rescue => e
+        Hive.logger.info("Process had already terminated")
+      end
+      @worker_pid = nil
+    end
+
+    # Test the state of the worker process
+    def running?
+      if @worker_pid
+        begin
+          Process.kill 0, @worker_pid
+          true
+        rescue Errno::ESRCH
+          false
+        end
+      else
+        false
+      end
+    end
+
+    # Return the worker pid, checking to see if it is running first
+    def worker_pid
+      @worker_pid = nil if ! self.running?
+      @worker_pid
+    end
+
+    # Return true if the device is claimed
+    # If the device has no status set it is assumed not to be claimed
+    def claimed?
+      @status == 'claimed'
+    end
+
+    # Test equality with another device
+    def ==(other)
+      self.identity == other.identity
+    end
+
+    # Return the unique identity of the device
+    def identity
+      "#{self.class.to_s.split('::').last}-#{@identity}"
+    end
+  end
+end

data/lib/hive/diagnostic.rb

@@ -0,0 +1,54 @@
+require 'hive'
+require 'hive/results'
+      
+module Hive
+  class Diagnostic
+    
+    class InvalidParameterError < StandardError
+      end
+
+    attr_accessor :last_run
+    attr_reader :config, :device_api
+
+    def initialize(config, options)
+      @options = options
+      @config = config
+      @serial = @options['serial']
+      @device_api = @options['device_api']
+    end
+
+    def should_run?
+      return true if @last_run == nil
+      time_now = Time.new.getutc
+      last_run_time = @last_run.timestamp
+      diff = ((time_now - last_run_time)/300).round
+      if (diff > 2 && @last_run.passed?) || diff > 1
+        true
+      else
+        false
+      end
+    end
+
+    def run
+      Hive.logger.info("Trying to run diagnostic '#{self.class}'")
+      if should_run?  
+        result = diagnose 
+        result = repair(result) if result.failed?
+        @last_run = result
+      else
+        Hive.logger.info("Diagnostic '#{self.class}' last ran less than five minutes before")
+      end
+      @last_run 
+    end
+
+    def pass(message= {}, data = {})
+      Hive.logger.info(message)
+      Hive::Results.new("pass", message, data )
+    end
+
+    def fail(message ={}, data = {})
+      Hive.logger.info(message)
+      Hive::Results.new("fail", message, data)
+    end
+  end
+end