blue_colr 0.0.6

data/README.rdoc

@@ -0,0 +1,77 @@
+= blue_colr, database-based process launcher
+
+== Overview
+
+blue_colr allows you to easily launch processes using database as a queue. It
+consists of +bluecolrd+, a deamon that executes whatever finds in a queue,
+and a DSL for enqueuing processes that enables you to easily describe the order
+and dependencies of processes.
+
+== Installation
+
+  git clone git://github.com/jablan/blue_colr.git
+  cd blue_colr
+  gem build blue_colr.gemspec
+  gem install blue_colr-0.0.6.gem
+
+You may want to install +log4r+ gem as well, as it provides more powerful logging
+features than builtin Ruby's +Logger+.
+
+== Example
+
+  require 'blue_colr'
+
+  BlueColr.start do
+    run 'echo These processes'
+    run 'echo will be ran sequentially.'
+    parallel do
+      run 'echo And these'
+      sequential do
+        run 'echo (but not'
+        run 'echo these two)'
+      end
+      run 'echo in parallel.'
+    end
+    run 'echo These will execute'
+    run 'echo after all above are finished.'
+  end
+
+Previous code will queue processes within the database, keeping them in
+dependency order. Those within +sequential+ block (and in root block, by
+default) will run each after the one before finishes. Those within +parallel+
+block will run in parallel. The commands after +parallel+ block will be executed
+after _all_ the commands in +parallel+ block are sucessfully finished.
+
+Note: the code above will not _start_ the processes by itself, but enqueue them
+to the database, by default. A separate process called +bluecolrd+ is
+used for that.
+
+== <tt>bluecolrd</tt>
+
+Blue_colr daemon is constantly running, checking the database for newly enqueued
+processes, and executing them in a subshell, observing the order.
+
+== <tt>bcrun</tt>
+
+This script is used to launch arbitrary command through blue_colr. You might want
+to do that if you want to keep track of the stuff you launch (as everything goes
+through a database table).
+
+  bcrun -c path_to_config.yaml -x "command to execute"
+
+== Enviroments
+
+An environment is something like _category_ which you assign to a set of processes
+when enqueuing them. Then you can have multiple daemons running, each one of them
+targeting specific environment. That allows easy distribution of your tasks across
+multiple machines, while keeping them synchronized, like the following scenario:
+
+* Start tasks a and b on machine X and c on machine Y
+* When all above are sucessfully done, start task d on machine Z
+
+== ToDo
+
+* Scripts to create necessarry tables
+* Proper test code
+* Examples
+

data/bin/bcrun

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+
+# utility to launch bluecolr processes
+
+require "rubygems"
+require 'blue_colr'
+
+BlueColr.log = Logger.new(STDOUT)
+cmd = ''
+
+BlueColr.custom_args do |opts|
+  opts.banner = "Usage: bcrun [options]"
+
+  opts.on("-x cmd", "--execute command", "Command to execute (enclose in quotes if contains parameters)") do |command|
+    cmd = command
+  end
+end
+
+# queuing processes sequentially
+BlueColr.launch do
+  run cmd
+end
+

data/bin/bluecolrd

@@ -0,0 +1,182 @@
+#!/usr/bin/env ruby
+
+# daemon to run blue_colr processes
+
+require "rubygems"
+require 'date'
+require 'yaml'
+begin
+  require 'log4r' # try using log4r if available
+  require 'log4r/yamlconfigurator'
+#  include Log4r
+rescue LoadError
+  require 'logger' # otherwise, use plain ruby's one
+end
+require "optparse"
+require 'sequel'
+require 'blue_colr'
+require 'fileutils'
+
+def logger(name = nil)
+  @logger[name || @environment] || @logger['default']
+end
+
+def init_logger
+  if @conf['log4r_config']
+    log_cfg = Log4r::YamlConfigurator # shorthand
+    log_cfg['ENVIRONMENT'] = @environment if @environment
+    log_cfg['LOGFILENAME'] = @log_file
+
+    # load the YAML file with this
+    log_cfg.decode_yaml(@conf['log4r_config'])
+
+    @logger = Log4r::Logger
+  else
+    @logger = {'default' => Logger.new(@log_file)}
+  end
+  logger.level = @args['debuglevel'] || Logger::WARN
+end
+
+def parse_command_line(args)
+  data = {}
+  
+  OptionParser.new do |opts|
+    opts.banner = "Usage: bluecolrd [options]"
+  
+    opts.on("-c CONFIG", "--conf CONFIG", "YAML config file") do |config|
+      data["config"] = config
+    end
+
+    opts.on("-e NAME", "--environment NAME", "Environment name (e.g. test, production etc.) to work on (default none)") do |env|
+      data["environment"] = env
+    end
+    
+    opts.on("-m COUNT", "--max-count COUNT", "Max number of simultaneous processes to start.") do |count|
+      data["max"] = count.to_i
+    end
+
+    opts.on("-l LOGFILE", "--logfile LOGFILE", "File to log to.") do |logfile|
+      data["logfile"] = logfile
+    end
+
+    opts.on("-d LEVEL", "--debuglevel LEVEL", "Debug level to use (0 - DEBUG, 1 - INFO etc).") do |level|
+      data['debuglevel'] = level.to_i
+    end
+
+    opts.on_tail('-h', '--help', 'display this help and exit') do
+      puts opts
+      return nil
+    end
+    
+    opts.parse(args)
+  end
+  
+  return data
+end
+
+# check whether it's ok to spawn another process
+def ok_to_run?
+  # check the limit of max processes, if given TODO: @pids is not used anymore, this is not working.
+  @max_processes == 0 || @pids.size < @max_processes
+#  !@args['max'] || @pids.size < @args['max']
+end
+
+def run process
+  logger.debug "Running #{process[:module_name]}"
+  script = process[:cmd]
+  logger.debug script
+  id = process[:id]
+
+  # update process item in the db
+  # set status of process_item to "running"
+  @db[:process_items].filter(:id => id).update(:status => BlueColr::STATUS_RUNNING, :started_at => Time.now)
+
+  log_path = @conf['log_path'] || '.'
+  log_path = (process[:process_from] || Time.now).strftime(log_path) # interpolate date
+
+  FileUtils.mkdir_p log_path
+  log_file = File.join(log_path, "#{id}.out")
+  # run actual command
+  Thread.new do
+    begin
+      Dir.chdir(process[:chdir]) if process[:chdir]
+      Kernel.system("#{script} >> #{log_file} 2>&1")
+      ok = $?.success?
+      exitstatus = $?.exitstatus
+    rescue
+      # do nothing, just exit with error
+      # this usually means that exec tried to execute a file that doesn't exist
+      ok = false
+      exitstatus = 99
+    end
+
+    # find corresponding process_item
+    # change its status in the DB and update ended_at timestamp
+    @db[:process_items].filter(:id => process[:id]).update(
+      :status => ok ? BlueColr::STATUS_OK : BlueColr::STATUS_ERROR,
+      :exit_code => exitstatus,
+      :ended_at => Time.now
+    )
+    logger(process[:logger]).error(@error_log_msg % process.to_hash) unless ok
+
+    logger.info "Process ended: id #{process[:id]} #{$?}"
+  end
+end
+
+# MAIN PROGRAM STARTS HERE
+
+# pid => process, hash of started processes
+@pids = {}
+
+  @args = parse_command_line(ARGV)
+
+  raise "No configuration file defined (-c <config>)." unless @args && @args["config"]
+  raise "Couldn't read #{@args["config"]} file." unless @args['config'] && @conf = YAML::load(File.new(@args["config"]).read)
+  @max_processes = @args['max'] || @conf['max_processes'] || 0 # default unlimited
+  @environment = @args['environment'] || @conf['environment'] || nil
+  @log_file = @args['logfile'] || "process_daemon_#{@environment}"
+  @error_log_msg = @conf['error_log_msg'] || 'Process failed: id %{id}'
+
+  init_logger
+
+begin
+  @db = Sequel.connect(@conf['db_url'], :logger => logger('sequel')) # try to use sequel logger, if defined
+
+  logger.info 'Starting daemon'
+
+  loop do
+    # get all pending items
+    query = "select i.id
+from process_items i
+left join process_item_dependencies d ON i.id = d.process_item_id
+left join process_items i2 ON d.depends_on_id = i2.id and i2.status NOT IN ('#{BlueColr::STATUS_OK}', '#{BlueColr::STATUS_SKIPPED}')
+where i.status = '#{BlueColr::STATUS_PENDING}' and i.environment = ?
+group by i.id
+having count(i2.id) = 0"
+    process_items = @db[query, @environment]
+
+    process_items.each do |id|
+      logger.debug "Pending item: #{id.inspect}"
+      if ok_to_run?
+        item = @db[:process_items].filter(:id => id[:id]).first
+        run(item)
+      end
+    end
+    sleep(@conf['sleep_interval'] || 10)
+  end
+
+rescue Interrupt
+  if logger
+    logger.fatal("Ctrl-C received, exiting")
+  else
+    puts "Ctrl-C received, exiting"
+  end
+  exit 1
+rescue Exception => ex
+  p ex.class
+  logger.fatal(ex.to_s) if logger
+  puts "#{ex.to_s} ==>"
+  puts ex.backtrace.join("\n")
+  exit 1
+end
+

data/lib/blue_colr.rb

@@ -0,0 +1,189 @@
+# This class provides a simple DSL for enqueuing processes to the database
+# in particular order.
+
+require 'rubygems'
+require 'date'
+require 'logger'
+require 'ostruct'
+require 'optparse'
+require 'yaml'
+require 'sequel'
+
+
+class BlueColr
+  STATUS_OK = 'ok'
+  STATUS_ERROR = 'error'
+  STATUS_PENDING = 'pending'
+  STATUS_RUNNING = 'running'
+  STATUS_PREPARING = 'preparing'
+  STATUS_SKIPPED = 'skipped'
+
+  class << self
+    attr_accessor :log, :db, :environment, :db_uri
+
+    # default options to use when launching a process - every field maps to a
+    # column in process_items table
+    def default_options
+      @default_options ||= OpenStruct.new
+    end
+
+    # local hash used to store misc runtime options
+    def options
+      @options ||= OpenStruct.new
+    end
+
+    def sequential &block
+      self.new.sequential &block
+    end
+
+    def parallel &block
+      self.new.parallel &block
+    end
+
+    # set custom commandline parameters from parent script, will be called upon
+    # command line parameter extraction
+    def custom_args &block
+      @custom_args_block = block
+    end
+
+    # launch a set of tasks, provided within a given block
+    def launch &block
+      @log ||= Logger.new('process_daemon')
+
+      unless @db # not connected
+        unless @db_uri # get the config from command line
+          @args = parse_command_line ARGV
+
+          raise "No configuration file defined (-c <config>)." if @args["config"].nil?
+          raise "Couldn't read #{@args["config"]} file." unless @args['config'] && @conf = YAML::load(File.new(@args["config"]).read)
+
+          @db_uri = @conf['db_url']
+          # setting default options that should be written along with all the records to process_items
+          if @conf['default_options']
+            @conf['default_options'].each do |k,v|
+              default_options.send("#{k}=", v)
+            end
+          end
+        end
+        @db = Sequel.connect(@db_uri, :logger => @log)
+      end
+      worker = self.new
+      db.transaction do
+        worker.instance_eval &block
+      end
+      worker
+    end
+
+    # run a set of tasks (launch it and wait until the last one finishes). exit with returned exitcode.
+    def run &block
+      worker = launch &block
+      exit worker.wait
+    end
+
+    def parse_command_line(args)
+      data = Hash.new()
+
+      OptionParser.new do |opts|
+        opts.banner = "Usage: process_daemon.rb [options]"
+
+        opts.on("-c CONFIG", "--conf CONFIG", "YAML config file") do |config|
+          data["config"] = config
+        end
+
+        # process custom args, if given
+        @custom_args_block.call(opts) if @custom_args_block
+
+        opts.on_tail('-h', '--help', 'display this help and exit') do
+          puts opts
+          exit
+#          return nil
+        end
+
+#        begin
+          opts.parse(args)
+#        rescue OptionParser::InvalidOption
+#          # do nothing
+#        end
+
+      end
+
+      return data
+    end
+  end
+
+  attr_reader :all_ids, :result
+
+  def initialize type = :sequential, waitfor = []
+    @type = type
+    @waitfor = waitfor
+    @result = []
+    @all_ids = [] # list of all ids of processes enqueued, used if waiting
+  end
+
+  def db
+    self.class.db
+  end
+
+  def log
+    self.class.log
+  end
+
+  def sequential &block
+    exec :sequential, &block
+  end
+
+  def parallel &block
+    exec :parallel, &block
+  end
+
+  def exec type = :sequential, &block
+    g = self.class.new type, @waitfor
+    g.instance_eval &block
+    ids = g.result
+    if @type == :sequential
+      @waitfor = ids
+      @result = ids
+    else
+      @result += ids
+    end
+    @result
+  end
+
+  def enqueue cmd, waitfor = [], opts = {}
+    id = nil
+    def_opts = self.class.default_options.send(:table) # convert from OpenStruct to Hash
+    id = db[:process_items].insert(def_opts.merge(opts).merge(:status => STATUS_PREPARING, :cmd => cmd, :queued_at => Time.now))
+    waitfor.each do |wid|
+      db[:process_item_dependencies].insert(:process_item_id => id, :depends_on_id => wid)
+    end
+    db[:process_items].filter(:id => id).update(:status => STATUS_PENDING)
+#    id = TaskGroup.counter
+    log.info "enqueueing #{id}: #{cmd}, waiting for #{waitfor.inspect}"
+    # remember id
+    @all_ids << id
+    id
+  end
+
+  def run cmd, opts = {}
+    id = enqueue cmd, @waitfor, opts
+    if @type == :sequential
+      @waitfor = [id]
+      @result = [id]
+    else
+      @result << id
+    end
+    @result
+  end
+
+  # wait for all enqueued processes to finish
+  def wait
+    log.info 'Waiting for all processes to finish'
+    loop do
+      failed = db[:process_items].filter(:id => @all_ids, :status => STATUS_ERROR).first
+      return failed[:exit_code] if failed
+      not_ok_count = db[:process_items].filter(:id => @all_ids).exclude(:status => STATUS_OK).count
+      return 0 if not_ok_count == 0 # all ok, finish
+      sleep 10
+    end
+  end
+end

metadata

@@ -0,0 +1,85 @@
+--- !ruby/object:Gem::Specification 
+name: blue_colr
+version: !ruby/object:Gem::Version 
+  hash: 19
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 6
+  version: 0.0.6
+platform: ruby
+authors: 
+- Mladen Jablanovic
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-07-07 00:00:00 +02:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: sequel
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+description: Blue_colr provides simple DSL to enqueue processes in given order, using database table as a queue, and a deamon to run them
+email: 
+- jablan@radioni.ca
+executables: 
+- bluecolrd
+- bcrun
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- bin/bcrun
+- bin/bluecolrd
+- lib/blue_colr.rb
+- README.rdoc
+has_rdoc: true
+homepage: http://github.com/jablan/blue_colr
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Database based process launcher
+test_files: []
+