blue_colr 0.0.9 → 0.1.0

data/README.rdoc

@@ -18,7 +18,7 @@ features than builtin Ruby's +Logger+.
 
   require 'blue_colr'
 
-  BlueColr.start do
+  BlueColr.launch do
     run 'echo These processes'
     run 'echo will be ran sequentially.'
     parallel do
@@ -43,15 +43,21 @@ Note: the code above will not _start_ the processes by itself, but enqueue them
 to the database, by default. A separate process called +bluecolrd+ is
 used for that.
 
+The following chart, generated by the same code above, is its execution sequence:
+
+http://github.com/downloads/jablan/blue_colr/readme_example.png
+
 == Requirements and Configuration
 
+In order to access the database, blue_colr requires sequel ORM library, if you
+don't have it, its gem will be installed along with blue_colr.
+
 Blue_colr uses a relational database to simulate a process queue so you will have
 to provide one. It relies on two tables, named +process_items+ and
-+process_item_dependencies+ to work. +db/+ directory contains Postgresql scripts
-for creating these two, and this should be moved to Sequel migration later.
++process_item_dependencies+ to work. +db/+ directory contains Sequel migrations
+for creating these two:
 
-In order to access the database, blue_colr requires sequel ORM library, if you
-don't have it, its gem will be installed along with blue_colr.
+  sequel -m db/ sqlite://examples/test.db
 
 Basic configuration is passed to blue_colr either by setting options from your
 code, or (if not set), blue_colr will parse your command line arguments and
@@ -77,12 +83,12 @@ when enqueuing them. Then you can have multiple daemons running, each one of the
 targeting specific environment. That allows easy distribution of your tasks across
 multiple machines, while keeping them synchronized, like the following scenario:
 
-* Start tasks a and b on machine X and c on machine Y
-* When all above are sucessfully done, start task d on machine Z
+* Start tasks +a+ and +b+ on machine +X+ and +c+ on machine +Y+
+* When all above are sucessfully done, start task +d+ on machine +Z+
 
 == ToDo
 
-* Move db table creation scripts to Sequel migration
-* Write proper tests
+* More tests
+* Better docs
 * Examples
 

data/bin/bluecolrd

@@ -23,7 +23,7 @@ def logger(name = nil)
 end
 
 def init_logger
-  if @conf['log4r_config']
+  if Module::const_defined?(:Log4r) && @conf['log4r_config']
     log_cfg = Log4r::YamlConfigurator # shorthand
     log_cfg['ENVIRONMENT'] = @environment if @environment
     log_cfg['LOGFILENAME'] = @log_file
@@ -33,7 +33,7 @@ def init_logger
 
     @logger = Log4r::Logger
   else
-    @logger = {'default' => Logger.new(@log_file)}
+    @logger = {'default' => Logger.new(@log_file || STDOUT)}
   end
   logger.level = @args['debuglevel'] || Logger::WARN
 end
@@ -82,15 +82,15 @@ def ok_to_run?
 #  !@args['max'] || @pids.size < @args['max']
 end
 
-def run process
-  logger.debug "Running #{process[:module_name]}"
+def run process, running_state
+  logger.debug "Running process ##{process[:id]}:"
   script = process[:cmd]
   logger.debug script
   id = process[:id]
 
   # update process item in the db
   # set status of process_item to "running"
-  @db[:process_items].filter(:id => id).update(:status => BlueColr::STATUS_RUNNING, :started_at => Time.now)
+  @db[:process_items].filter(:id => id).update(:status => running_state, :started_at => Time.now)
 
   log_path = @conf['log_path'] || '.'
   log_path = (process[:process_from] || Time.now).strftime(log_path) # interpolate date
@@ -111,16 +111,17 @@ def run process
       exitstatus = 99
     end
 
+    final_state = BlueColr.state_from_running(running_state, ok)
     # find corresponding process_item
     # change its status in the DB and update ended_at timestamp
     @db[:process_items].filter(:id => process[:id]).update(
-      :status => ok ? BlueColr::STATUS_OK : BlueColr::STATUS_ERROR,
+      :status => final_state,
       :exit_code => exitstatus,
       :ended_at => Time.now
     )
-    logger(process[:logger]).error(@error_log_msg % process.to_hash) unless ok
+#    logger(process[:logger]).error(@error_log_msg % process.to_hash) unless ok
 
-    logger.info "Process ended: id #{process[:id]} #{$?}"
+#    logger.info "Process ended: id #{process[:id]} #{$?}"
   end
 end
 
@@ -129,16 +130,17 @@ end
 # pid => process, hash of started processes
 @pids = {}
 
-  @args = parse_command_line(ARGV)
+@args = parse_command_line(ARGV)
 
-  raise "No configuration file defined (-c <config>)." unless @args && @args["config"]
-  raise "Couldn't read #{@args["config"]} file." unless @args['config'] && @conf = YAML::load(File.new(@args["config"]).read)
-  @max_processes = @args['max'] || @conf['max_processes'] || 0 # default unlimited
-  @environment = @args['environment'] || @conf['environment'] || nil
-  @log_file = @args['logfile'] || "process_daemon_#{@environment}"
-  @error_log_msg = @conf['error_log_msg'] || 'Process failed: id %{id}'
+raise "No configuration file defined (-c <config>)." unless @args && @args["config"]
+raise "Couldn't read #{@args["config"]} file." unless @args['config'] && @conf = YAML::load(File.new(@args["config"]).read)
+BlueColr.conf = @conf
+@max_processes = @args['max'] || @conf['max_processes'] || 0 # default unlimited
+@environment = @args['environment'] || @conf['environment'] || nil
+@log_file = @args['logfile'] || "process_daemon_#{@environment}.log"
+@error_log_msg = @conf['error_log_msg'] || 'Process failed: id %{id}'
 
-  init_logger
+init_logger
 
 begin
   @db = Sequel.connect(@conf['db_url'], :logger => logger('sequel')) # try to use sequel logger, if defined
@@ -147,34 +149,40 @@ begin
 
   loop do
     # get all pending items
-    query = "select i.id
-from process_items i
-left join process_item_dependencies d ON i.id = d.process_item_id
-left join process_items i2 ON d.depends_on_id = i2.id and i2.status NOT IN ('#{BlueColr::STATUS_OK}', '#{BlueColr::STATUS_SKIPPED}')
-where i.status = '#{BlueColr::STATUS_PENDING}' and i.environment = ?
-group by i.id
-having count(i2.id) = 0"
-    process_items = @db[query, @environment]
-
-    process_items.each do |id|
-      logger.debug "Pending item: #{id.inspect}"
+    pending_processes = @db[:process_items].filter(:status => BlueColr.get_pending_states).all
+    pending_processes = pending_processes.map do |process|
+      # get all the parents' statuses
+      parent_statuses = @db[:process_items].
+        join(:process_item_dependencies, :depends_on_id => :id).
+        filter(:process_item_id => process[:id]).
+        select(:status).
+        map{|h| h[:status]}
+
+      running_status = BlueColr.state_from_pending(process[:status], parent_statuses)
+      [process, running_status]
+    end
+
+    pending_processes.select{|_, running_status| running_status}.each do |process, running_status|
+      logger.debug "Pending item: #{process[:id]}"
       if ok_to_run?
-        item = @db[:process_items].filter(:id => id[:id]).first
-        run(item)
+#          item = @db[:process_items].filter(:id => id[:id]).first
+        run(process, running_status)
       else
         logger.debug "No available thread, waiting"
       end
-    end
-    sleep(@conf['sleep_interval'] || 10)
-  end
 
-rescue Interrupt
-  if logger
-    logger.fatal("Ctrl-C received, exiting")
-  else
-    puts "Ctrl-C received, exiting"
-  end
-  exit 1
+    end
+    Kernel.sleep 5
+#    Kernel.sleep(@conf['sleep_interval'] || 10)
+  end # loop
+
+#rescue Interrupt
+#  if logger
+#    logger.fatal("Ctrl-C received, exiting")
+#  else
+#    puts "Ctrl-C received, exiting"
+#  end
+#  exit 1
 rescue Exception => ex
   p ex.class
   logger.fatal(ex.to_s) if logger

data/lib/blue_colr/graph_output.rb

@@ -0,0 +1,74 @@
+class BlueColr
+  # this module, when included in BlueColr, generates GraphViz graph of the invoked processes,
+  # instead actually enqueueing them.
+  module GraphOutput
+
+    def self.included target
+      target.instance_eval do
+        # graph nodes are given unique ids
+        def next_id
+          @id ||= 0
+          @id += 1
+        end
+
+        # gets different color for different environments,
+        # currently cycling between couple predefined colors
+        # TODO: enable submitting color through option
+        def get_color group
+          colors = [
+            '#FFFFFF',
+            '#DDFFDD',
+            '#DDDDFF',
+            '#FFDDDD',
+            '#FFFFDD',
+            '#FFDDFF',
+            '#DDFFFF',
+          ]
+          @groups ||= []
+          @groups << group unless @groups.member? group
+          colors[@groups.index(group) % colors.length]
+        end
+
+        # override class method launch, we are creating output file here,
+        # and we don't need database
+        def launch &block
+          default_options.gv_filename ||= "output.dot"
+          worker = self.new
+          File.open(default_options.gv_filename, 'w') do |f|
+            default_options.gv_file = f
+            f.puts "digraph G {"
+            worker.instance_eval &block
+            f.puts "}"
+          end
+          worker
+        end
+
+        # override default enqueue method, as just including won't do
+        define_method :enqueue, instance_method(:graph_enqueue)
+      end
+    end
+
+    # original enqueue enqueues the process to the database,
+    # here we should just output a graph elements to the output file
+    def graph_enqueue cmd, waitfor = [], opts = {}
+      gv_file = self.class.default_options.gv_file
+      id = self.class.next_id
+      waitfor.each do |wid|
+        # output graph edges
+        gv_file.puts "  b#{wid} -> b#{id};"
+      end
+      # determine node label
+      label = opts[:label] || cmd
+      label.gsub!(/([^\\])"/, '\1""')
+      # determine node color
+      color = self.class.get_color(opts[:group] || opts[:environment])
+      # output node description
+      gv_file.puts "  b#{id} [shape=box,style=filled,fillcolor=\"#{color}\",label=\"#{label}\"];"
+      # remember id
+      @all_ids << id
+      id
+    end
+  end
+
+  include GraphOutput
+end

data/lib/blue_colr.rb

@@ -11,15 +11,72 @@ require 'sequel'
 
 
 class BlueColr
-  STATUS_OK = 'ok'
-  STATUS_ERROR = 'error'
-  STATUS_PENDING = 'pending'
-  STATUS_RUNNING = 'running'
-  STATUS_PREPARING = 'preparing'
-  STATUS_SKIPPED = 'skipped'
+#  STATUS_OK = 'ok'
+#  STATUS_ERROR = 'error'
+#  STATUS_PENDING = 'pending'
+#  STATUS_RUNNING = 'running'
+#  STATUS_PREPARING = 'preparing'
+#  STATUS_SKIPPED = 'skipped'
+
+  # default state transitions with simple state setup ('PENDING => RUNNING => OK or ERROR')
+  DEFAULT_PENDING_STATE = 'pending'
+  PREPARING_STATE = 'preparing'
+  DEFAULT_STATEMAP = {
+    'on_pending' => {
+      DEFAULT_PENDING_STATE => [
+        ['running', ['ok', 'skipped']]
+      ]
+    },
+    'on_running' => {
+      'running' => {
+        'error' => 'error',
+        'ok' => 'ok'
+      }
+    },
+    'on_restart' => {
+      'error' => 'pending',
+      'ok' => 'pending'
+    }
+  }
 
   class << self
-    attr_accessor :log, :db, :environment, :db_uri
+    attr_accessor :environment
+    attr_writer :statemap, :log, :db, :db_uri, :conf
+
+    def log
+      @log ||= Logger.new('process_daemon')
+    end
+
+    def conf
+      unless @conf
+        parse_command_line unless @args
+
+        raise "No configuration file defined (-c <config>)." if @args["config"].nil?
+        raise "Couldn't read #{@args["config"]} file." unless @args['config'] && @conf = YAML::load(File.new(@args["config"]).read)
+
+        # setting default options that should be written along with all the records to process_items
+        if @conf['default_options']
+          @conf['default_options'].each do |k,v|
+            default_options.send("#{k}=", v)
+          end
+        end
+      end
+      @conf
+    end
+
+    def db_uri
+      unless @db_uri # get the config from command line
+        @db_uri = self.conf['db_url']
+      end
+      @db_uri
+    end
+    
+    def db
+      unless @db # not connected
+        @db = Sequel.connect(self.db_uri, :logger => self.log)
+      end
+      @db
+    end
 
     # default options to use when launching a process - every field maps to a
     # column in process_items table
@@ -32,6 +89,10 @@ class BlueColr
       @options ||= OpenStruct.new
     end
 
+    def statemap
+      @statemap ||= conf['statemap'] || DEFAULT_STATEMAP
+    end
+
     def sequential &block
       self.new.sequential &block
     end
@@ -48,25 +109,7 @@ class BlueColr
 
     # launch a set of tasks, provided within a given block
     def launch &block
-      @log ||= Logger.new('process_daemon')
 
-      unless @db # not connected
-        unless @db_uri # get the config from command line
-          @args = parse_command_line ARGV
-
-          raise "No configuration file defined (-c <config>)." if @args["config"].nil?
-          raise "Couldn't read #{@args["config"]} file." unless @args['config'] && @conf = YAML::load(File.new(@args["config"]).read)
-
-          @db_uri = @conf['db_url']
-          # setting default options that should be written along with all the records to process_items
-          if @conf['default_options']
-            @conf['default_options'].each do |k,v|
-              default_options.send("#{k}=", v)
-            end
-          end
-        end
-        @db = Sequel.connect(@db_uri, :logger => @log)
-      end
       worker = self.new
       db.transaction do
         worker.instance_eval &block
@@ -80,8 +123,8 @@ class BlueColr
       exit worker.wait
     end
 
-    def parse_command_line(args)
-      data = Hash.new()
+    def parse_command_line &block
+      data = {}
 
       OptionParser.new do |opts|
         opts.banner = "Usage: process_daemon.rb [options]"
@@ -91,7 +134,7 @@ class BlueColr
         end
 
         # process custom args, if given
-        @custom_args_block.call(opts) if @custom_args_block
+        block.call(opts) if block_given?
 
         opts.on_tail('-h', '--help', 'display this help and exit') do
           puts opts
@@ -100,16 +143,53 @@ class BlueColr
         end
 
 #        begin
-          opts.parse(args)
+          opts.parse(ARGV)
 #        rescue OptionParser::InvalidOption
 #          # do nothing
 #        end
 
       end
 
-      return data
+      @args = data
     end
-  end
+
+
+    # state related methods
+    
+    # get the next state from pending, given current state and state of all "parent" processes
+    def state_from_pending current_state, parent_states
+      new_state, _ = self.statemap['on_pending'][current_state].find { |_, required_parent_states|
+        (parent_states - required_parent_states).empty?
+      }
+      new_state
+    end
+
+    # get the next state from running, given current state and whether the command has finished successfully
+    def state_from_running current_state, ok
+      self.statemap['on_running'][current_state][ok ? 'ok' : 'error']
+    end
+
+    # get the next state to get upon restart, given the current state
+    def state_on_restart current_state
+      self.statemap['on_restart'][current_state]
+    end
+
+    # get all possible pending states
+    def get_pending_states
+      self.statemap['on_pending'].map{|state, _| state}
+    end
+
+    # get all possible error states
+    def get_error_states
+      self.statemap['on_running'].map{|_, new_states| new_states['error']}
+    end
+
+    # get all possible ok states
+    def get_ok_states
+      self.statemap['on_running'].map{|_, new_states| new_states['ok']}
+    end
+
+  end # class methods
 
   attr_reader :all_ids, :result
 
@@ -151,14 +231,15 @@ class BlueColr
 
   def enqueue cmd, waitfor = [], opts = {}
     id = nil
+    opts = {status: DEFAULT_PENDING_STATE}.merge(opts)
     def_opts = self.class.default_options.send(:table) # convert from OpenStruct to Hash
-    # rejecting fields that do not match to a column in the table:
+    # rejecting fields that do not have corresponding column in the table:
     fields = def_opts.merge(opts).select{|k,_| db[:process_items].columns.member? k}
-    id = db[:process_items].insert(fields.merge(:status => STATUS_PREPARING, :cmd => cmd, :queued_at => Time.now))
+    id = db[:process_items].insert(fields.merge(:status => PREPARING_STATE, :cmd => cmd, :queued_at => Time.now))
     waitfor.each do |wid|
       db[:process_item_dependencies].insert(:process_item_id => id, :depends_on_id => wid)
     end
-    db[:process_items].filter(:id => id).update(:status => STATUS_PENDING)
+    db[:process_items].filter(:id => id).update(:status => opts[:status])
 #    id = TaskGroup.counter
     log.info "enqueueing #{id}: #{cmd}, waiting for #{waitfor.inspect}"
     # remember id
@@ -181,9 +262,9 @@ class BlueColr
   def wait
     log.info 'Waiting for all processes to finish'
     loop do
-      failed = db[:process_items].filter(:id => @all_ids, :status => STATUS_ERROR).first
+      failed = db[:process_items].filter(:id => @all_ids, :status => BlueColr.get_error_states).first
       return failed[:exit_code] if failed
-      not_ok_count = db[:process_items].filter(:id => @all_ids).exclude(:status => STATUS_OK).count
+      not_ok_count = db[:process_items].filter(:id => @all_ids).exclude(:status => BlueColr.get_ok_states).count
       return 0 if not_ok_count == 0 # all ok, finish
       sleep 10
     end

metadata

@@ -1,7 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: blue_colr
 version: !ruby/object:Gem::Version 
-  version: 0.0.9
+  hash: 27
+  prerelease: 
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
 platform: ruby
 authors: 
 - Mladen Jablanovic
@@ -9,19 +15,23 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-10-13 00:00:00 +02:00
+date: 2011-10-17 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: sequel
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
         version: "0"
-    version: 
+  type: :runtime
+  version_requirements: *id001
 description: Blue_colr provides simple DSL to enqueue processes in given order, using database table as a queue, and a deamon to run them
 email: 
 - jablan@radioni.ca
@@ -36,6 +46,7 @@ files:
 - bin/bcrun
 - bin/bluecolrd
 - lib/blue_colr.rb
+- lib/blue_colr/graph_output.rb
 - README.rdoc
 has_rdoc: true
 homepage: http://github.com/jablan/blue_colr
@@ -47,21 +58,27 @@ rdoc_options: []
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.5
+rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
 summary: Database based process launcher