rq 0.1.7

data/TODO

@@ -0,0 +1,13 @@
+---
+- re-think db schema: separate critical info/index for speed?
+- user docs
+- examples
+- installer script
+- api docs
+- rq relay mode (submit and track exit_status in local db)
+- housekeeping - make backups of queue every so often, integrity_checks at
+- scheduled intervals
+- full boolean resource monitoring and resource requests
+- in db message queue to send commands to remote nodes - ttl issues here...
+- track feeders in queue rather than using lock on local file?
+- use nodes to periodically generate stats (cache) 

data/VERSION

@@ -0,0 +1 @@
+0.1.7

data/bin/rq

@@ -0,0 +1,391 @@
+#!/usr/bin/env ruby
+#
+# rq - http://raa.ruby-lang.org/project/rq
+#
+  require 'rq'
+#
+# main program class
+#
+  class Main
+#{{{
+    include RQ
+    include RQ::Util
+    include RQ::Logging
+    include RQ::Usage
+
+    OPTSPEC =
+#{{{
+    [
+      [
+        '--priority=priority', '-p', 
+        'modes <submit> : set the job(s) priority - lowest(0) .. highest(n) - (default 0)'
+      ],
+      [
+        '--tag=tag', '-t', 
+        'modes <submit> : set the job(s) user data tag'
+      ],
+      [
+        '--infile=infile', '-i',
+        'modes <submit> : infile'
+      ],
+      [
+        '--quiet', '-q',
+        'modes <submit, feed> : do not echo submitted jobs, fail silently if
+        another process is already feeding'
+      ],
+      [
+        '--daemon', '-d',
+        'modes <feed> : spawn a daemon'
+      ],
+      [
+        '--max_feed=max_feed', '-f', 
+        'modes <feed> : the maximum number of concurrent jobs run'
+      ],
+#      [
+#        '--name=name',
+#        'set the feeder name - (default q path)'
+#      ],
+      [
+        '--retries=retries', '-r',
+        'modes <feed> : specify transaction retries'
+      ],
+      [
+        '--min_sleep=min_sleep', '-m',
+        'modes <feed> : specify min sleep'
+      ],
+      [
+        '--max_sleep=max_sleep', '-M',
+        'modes <feed> : specify max sleep'
+      ],
+      [
+        '--snapshot', '-s',
+        'operate on snapshot of queue'
+      ],
+      [
+        '--verbosity=verbostiy', '-v', 
+        '0|fatal < 1|error < 2|warn < 3|info < 4|debug - (default info)'
+      ],
+      [
+        '--log=path','-l', 
+        'set log file - (default stderr)'
+      ],
+      [
+        '--log_age=log_age',
+        'daily | weekly | monthly - what age will cause log rolling (default nil)'
+      ],
+      [
+        '--log_size=log_size',
+        'size in bytes - what size will cause log rolling (default nil)'
+      ],
+#      [
+#        '--config=path',
+#        'valid path - specify config file (default nil)'
+#      ],
+#      [
+#        '--template=[path]',
+#        'valid path - generate a template config file in path (default stdout)'
+#      ],
+      [
+        '--help', '-h', 
+        'this message'
+      ],
+    ]
+#}}}
+
+    CONFIG_DEFAULT_PATH = 'rq.conf'
+
+    CONFIG_SEARCH_PATH = %w( . ~ /dmsp/reference/etc /usr/local/etc /usr/etc /etc ) 
+
+    attr :logger
+    attr :argv
+    attr :env
+    attr :cmd
+    attr :options
+    attr :qpath
+    attr :mode
+    attr :q
+    attr :daemon
+
+    def initialize argv = ARGV, env = ENV
+#{{{
+      begin
+        @logger = Logger::new STDERR
+        @argv = mcp(argv.to_a)
+        @env = mcp(env.to_hash)
+        @cmd = ([$0] + @argv).join(' ')
+
+        parse_options
+
+        if(@options.has_key?('help') or @argv.include?('help'))
+          usage('port' => STDOUT, 'long' => true)
+          exit EXIT_SUCCESS
+        end
+
+        if(@options.has_key?('template') or (idx = @argv.index('template')))
+          gen_template(@options['template'] || @argv[idx + 1])
+          exit EXIT_SUCCESS
+        end
+
+        parse_argv
+
+        status = run
+
+        case status
+          when Integer
+            exit status
+          else
+            exit(status ? EXIT_SUCCESS : EXIT_FAILURE)
+        end
+      rescue => e
+        unless SystemExit === e
+          logerr e
+          exit EXIT_FAILURE
+        else
+          exit e.status 
+        end
+      end
+#}}}
+    end
+    def parse_argv
+  #{{{
+      @qpath = ENV['RQ_Q'] || @argv.shift
+      @mode = @argv.shift
+  #}}}
+    end
+    def run
+  #{{{
+      @qpath = Util::realpath @qpath
+
+      if @mode.nil? or @mode.strip.empty?
+        usage 'port' => STDERR, 'long' => false
+        exit EXIT_FAILURE
+      end
+
+      shortcuts = {
+        'c'  => 'create',
+        's'  => 'submit',
+        'l'  => 'list',
+        'ls' => 'list',
+        't'  => 'status',
+        'd'  => 'delete',
+        'rm' => 'delete',
+        'u'  => 'update',
+        'q'  => 'query',
+        'e'  => 'execute',
+        'C'  => 'configure',
+        'S'  => 'snapshot',
+        'L'  => 'lock',
+        'b'  => 'backup',
+        'h'  => 'help',
+        'f'  => 'feed',
+      }
+
+      if((longmode = shortcuts[@mode]))
+        @mode = longmode
+      end
+
+      begin
+        case @mode
+          when 'create'
+            create
+          when 'submit'
+            submit
+          when 'list'
+            list
+          when 'status'
+            status
+          when 'delete'
+            delete
+          when 'update'
+            update
+          when 'query'
+            query
+          when 'execute'
+            execute
+          when 'configure'
+            configure
+          when 'snapshot'
+            snapshot
+          when 'lock'
+            lock
+          when 'backup'
+            backup
+          when 'help'
+            usage 'port' => STDOUT, 'long' => true
+            exit EXIT_SUCCESS
+          when 'feed'
+            feed
+          else
+            raise "invalid mode <#{ @mode }>"
+        end
+      rescue Errno::EPIPE => e
+        raise if STDOUT.tty?
+      end
+  #}}}
+    end
+    def create 
+  #{{{
+      init_logging
+      creator = Creator::new self
+      creator.create
+  #}}}
+    end
+    def submit 
+  #{{{
+      init_logging
+      submitter = Submitter::new self
+      submitter.submit
+  #}}}
+    end
+    def list 
+  #{{{
+      init_logging
+      lister = Lister::new self
+      lister.list
+  #}}}
+    end
+    def status 
+  #{{{
+      init_logging
+      statuslister = StatusLister::new self
+      statuslister.statuslist
+  #}}}
+    end
+    def delete
+#{{{
+      init_logging
+      deleter = Deleter::new self
+      deleter.delete
+#}}}
+    end
+    def update 
+#{{{
+      init_logging
+      updater = Updater::new self
+      updater.update
+#}}}
+    end
+    def query
+#{{{
+      init_logging
+      querier = Querier::new self
+      querier.query
+#}}}
+    end
+    def execute 
+#{{{
+      init_logging
+      executor = Executor::new self
+      executor.execute
+#}}}
+    end
+    def configure 
+#{{{
+      init_logging
+      configurator = Configurator::new self
+      configurator.configure
+#}}}
+    end
+    def snapshot
+#{{{
+      init_logging
+      snapshotter = Snapshotter::new self
+      snapshotter.snapshot
+#}}}
+    end
+    def lock
+#{{{
+      init_logging
+      locker = Locker::new self
+      locker.lock
+#}}}
+    end
+    def backup 
+#{{{
+      init_logging
+      backer = Backer::new self
+      backer.backup
+#}}}
+    end
+    def feed 
+#{{{
+      feeder = Feeder::new self
+      feeder.feed
+#}}}
+    end
+    def parse_options
+#{{{
+      @op = OptionParser.new
+      @options = {}
+      OPTSPEC.each do |spec|
+        k = spec.first.gsub(%r/(?:--)|(?:=.*$)|(?:\s+)/o,'')
+        @op.def_option(*spec){|v| @options[k] = v}
+      end
+      @op.parse! @argv 
+      @options
+#}}}
+    end
+    def init_logging
+#{{{
+      log, log_age, log_size, verbosity = 
+        @options.values_at 'log', 'log_age', 'log_size', 'verbosity'
+      log_age = atoi log_age rescue nil
+      log_size = atoi log_size rescue nil
+      $logger = @logger = Logger::new(log || STDERR, log_age, log_size)
+    #
+    # hack to fix Logger sync bug
+    #
+      class << @logger; attr :logdev unless @logger.respond_to?(:logdev); end
+      @logdev = @logger.logdev.dev 
+      @logdev.sync = true
+      level = nil
+      verbosity ||= 'info'
+      verbosity =
+        case verbosity
+          when /^\s*(?:4|d|debug)\s*$/io
+            level = 'Logging::DEBUG'
+            4
+          when /^\s*(?:3|i|info)\s*$/io
+            level = 'Logging::INFO'
+            3
+          when /^\s*(?:2|w|warn)\s*$/io
+            level = 'Logging::WARN'
+            2
+          when /^\s*(?:1|e|error)\s*$/io
+            level = 'Logging::ERROR'
+            1
+          when /^\s*(?:0|f|fatal)\s*$/io
+            level = 'Logging::FATAL'
+            0
+          else
+            abort "illegal verbosity setting <#{ verbosity }>" 
+        end
+      @logger.level = 2 - ((verbosity % 5) - 2) 
+      debug {"logging level <#{ level }>"}
+      @logger
+#}}}
+    end
+    def init_config
+#{{{
+      @config =
+        if @options['config']
+          ConfigFile::new(@options['config'])
+        else
+          ConfigFile::any CONFIG_DEFAULT_PATH, CONFIG_SEARCH_PATH
+        end
+      debug { "config.path <#{ @config.path }>" }
+      @config
+#}}}
+    end
+    def gen_template template
+#{{{
+      ConfigFile::gen_template(template)
+      self
+#}}}
+    end
+#}}}
+  end
+#
+# run main program unless included as lib
+#
+  Main::new if $0 == __FILE__

data/bin/rq-0.1.7

@@ -0,0 +1,410 @@
+#!/dmsp/reference/bin/ruby
+#
+# rq - http://raa.ruby-lang.org/project/rq
+#
+  require 'rq'
+#
+# main program class
+#
+  class Main
+#{{{
+    include RQ
+    include RQ::Util
+    include RQ::Logging
+    include RQ::Usage
+
+#
+# TODO - pull out config stuff into separate class
+# TODO - add resouce manager
+#
+    CONFIG_DEFAULT_PATH = "#{ $0 }.conf"
+    CONFIG_SEARCH_PATH = %w(. ~ /usr/local/etc /usr/etc /etc) 
+
+    OPTSPEC =
+#{{{
+    [
+      [
+        '--max_feed', '-f', 
+        'the maximum number of concurrent jobs'
+      ],
+      [
+        '--priority', '-p', 
+        'set the job priority - lowest(0) .. highest(n) - (default 0)'
+      ],
+      [
+        '--name',
+        'set the feeder name - (default q path)'
+      ],
+      [
+        '--daemon', '-d',
+        'run in daemon mode'
+      ],
+      [
+        '--quiet', '-q',
+        'fail quietly'
+      ],
+      [
+        '--snapshot', '-s',
+        'operate on snapshot of queue'
+      ],
+      [
+        '--retries', '-r',
+        'retries'
+      ],
+      [
+        '--min_sleep', '-m',
+        'min sleep'
+      ],
+      [
+        '--max_sleep', '-M',
+        'max sleep'
+      ],
+      [
+        '--infile', '-i',
+        'infile'
+      ],
+      [
+        '--help', '-h', 
+        'this message'
+      ],
+      [
+        '--verbosity=verbostiy', '-v', 
+        '0|fatal < 1|error < 2|warn < 3|info < 4|debug - (default info)'
+      ],
+      [
+        '--log=path','-l', 
+        'set log file - (default stderr)'
+      ],
+      [
+        '--log_age=log_age',
+        'daily | weekly | monthly - what age will cause log rolling (default nil)'
+      ],
+      [
+        '--log_size=log_size',
+        'size in bytes - what size will cause log rolling (default nil)'
+      ],
+      [
+        '--config=path',
+        'valid path - specify config file (default nil)'
+      ],
+      [
+        '--template=[path]',
+        'valid path - generate a template config file in path (default stdout)'
+      ],
+    ]
+#}}}
+
+    attr :logger
+    attr :argv
+    attr :env
+    attr :cmd
+    attr :options
+    attr :qpath
+    attr :mode
+    attr :q
+    attr :daemon
+
+    def initialize argv = ARGV, env = ENV
+#{{{
+      begin
+        @logger = Logger::new STDERR
+        @argv = mcp(argv.to_a)
+        @env = mcp(env.to_hash)
+        @cmd = ([$0] + @argv).join(' ')
+
+        parse_options
+
+        if @options.has_key? 'help'
+          usage('port' => STDOUT, 'long' => true)
+          exit EXIT_SUCCESS
+        end
+
+        if @options.has_key? 'template' 
+          gen_template @options['template']
+          exit EXIT_SUCCESS
+        end
+
+        parse_argv
+
+        status = run
+
+        case status
+          when Integer
+            exit status
+          else
+            exit(status ? EXIT_SUCCESS : EXIT_FAILURE)
+        end
+      rescue => e
+        unless SystemExit === e
+          logerr e
+          exit EXIT_FAILURE
+        else
+          exit e.status 
+        end
+      end
+#}}}
+    end
+    def parse_argv
+  #{{{
+      @qpath = ENV['RQ_Q'] || @argv.shift
+      @mode = @argv.shift
+  #}}}
+    end
+    def run
+  #{{{
+      if @qpath == 'help'
+        usage 'port' => STDOUT, 'long' => true
+        exit EXIT_SUCCESS
+      end
+
+      @qpath = Util::realpath @qpath
+
+      if @mode.nil? or @mode.strip.empty?
+        usage 'port' => STDERR, 'long' => false
+        exit EXIT_FAILURE
+      end
+
+      shortcuts = {
+        'c'  => 'create',
+        's'  => 'submit',
+        'l'  => 'list',
+        'ls' => 'list',
+        't'  => 'status',
+        'd'  => 'delete',
+        'rm' => 'delete',
+        'u'  => 'update',
+        'q'  => 'query',
+        'e'  => 'execute',
+        'C'  => 'configure',
+        'S'  => 'snapshot',
+        'L'  => 'lock',
+        'b'  => 'backup',
+        'h'  => 'help',
+        'f'  => 'feed',
+      }
+
+      if((longmode = shortcuts[@mode]))
+        @mode = longmode
+      end
+
+      begin
+        case @mode
+          when 'create'
+            create
+          when 'submit'
+            submit
+          when 'list'
+            list
+          when 'status'
+            status
+          when 'delete'
+            delete
+          when 'update'
+            update
+          when 'query'
+            query
+          when 'execute'
+            execute
+          when 'configure'
+            configure
+          when 'snapshot'
+            snapshot
+          when 'lock'
+            lock
+          when 'backup'
+            backup
+          when 'help'
+            usage 'port' => STDOUT, 'long' => true
+            exit EXIT_SUCCESS
+          when 'feed'
+            feed
+          else
+            raise "invalid mode <#{ @mode }>"
+        end
+      rescue Errno::EPIPE => e
+        raise if STDOUT.tty?
+      end
+  #}}}
+    end
+    def create 
+  #{{{
+      init_logging
+      creator = Creator::new self
+      creator.create
+  #}}}
+    end
+    def submit 
+  #{{{
+      init_logging
+      submitter = Submitter::new self
+      submitter.submit
+  #}}}
+    end
+    def list 
+  #{{{
+      init_logging
+      lister = Lister::new self
+      lister.list
+  #}}}
+    end
+    def status 
+  #{{{
+      init_logging
+      statuslister = StatusLister::new self
+      statuslister.statuslist
+  #}}}
+    end
+    def delete
+#{{{
+      init_logging
+      deleter = Deleter::new self
+      deleter.delete
+#}}}
+    end
+    def update 
+#{{{
+      init_logging
+      updater = Updater::new self
+      updater.update
+#}}}
+    end
+    def query
+#{{{
+      init_logging
+      querier = Querier::new self
+      querier.query
+#}}}
+    end
+    def execute 
+#{{{
+      init_logging
+      executor = Executor::new self
+      executor.execute
+#}}}
+    end
+    def configure 
+#{{{
+      init_logging
+      configurator = Configurator::new self
+      configurator.configure
+#}}}
+    end
+    def snapshot
+#{{{
+      init_logging
+      snapshotter = Snapshotter::new self
+      snapshotter.snapshot
+#}}}
+    end
+    def lock
+#{{{
+      init_logging
+      locker = Locker::new self
+      locker.lock
+#}}}
+    end
+    def backup 
+#{{{
+      init_logging
+      backer = Backer::new self
+      backer.backup
+#}}}
+    end
+    def feed 
+#{{{
+      feeder = Feeder::new self
+      feeder.feed
+#}}}
+    end
+    def parse_options
+#{{{
+      @op = OptionParser.new
+      @options = {}
+      OPTSPEC.each do |spec|
+        k = spec.first.gsub(%r/(?:--)|(?:=.*$)|(?:\s+)/o,'')
+        @op.def_option(*spec){|v| @options[k] = v}
+      end
+      #begin
+        @op.parse! @argv 
+      #rescue OptionParser::InvalidOption => e
+        # preverve unknown options
+        #e.recover(argv)
+      #end
+      @options
+#}}}
+    end
+    def init_logging
+#{{{
+      log, log_age, log_size, verbosity = 
+        @options.values_at 'log', 'log_age', 'log_size', 'verbosity'
+      log_age = atoi log_age rescue nil
+      log_size = atoi log_size rescue nil
+      $logger = @logger = Logger::new(log || STDERR, log_age, log_size)
+    #
+    # hack to fix Logger sync bug
+    #
+      class << @logger; attr :logdev unless @logger.respond_to?(:logdev); end
+      @logdev = @logger.logdev.dev 
+      @logdev.sync = true
+      level = nil
+      verbosity ||= 'info'
+      verbosity =
+        case verbosity
+          when /^\s*(?:4|d|debug)\s*$/io
+            level = 'Logging::DEBUG'
+            4
+          when /^\s*(?:3|i|info)\s*$/io
+            level = 'Logging::INFO'
+            3
+          when /^\s*(?:2|w|warn)\s*$/io
+            level = 'Logging::WARN'
+            2
+          when /^\s*(?:1|e|error)\s*$/io
+            level = 'Logging::ERROR'
+            1
+          when /^\s*(?:0|f|fatal)\s*$/io
+            level = 'Logging::FATAL'
+            0
+          else
+            abort "illegal verbosity setting <#{ verbosity }>" 
+        end
+      @logger.level = 2 - ((verbosity % 5) - 2) 
+      debug {"logging level <#{ level }>"}
+      @logger
+#}}}
+    end
+    def init_config
+#{{{
+      @config =
+        if @options['config']
+          ConfigFile::new(@options['config'])
+        else
+          ConfigFile::any CONFIG_DEFAULT_PATH, CONFIG_SEARCH_PATH
+        end
+      debug { "config.path <#{ @config.path }>" }
+      debug { "config\n#{ @config.to_hash.to_yaml }\n" }
+      @config
+#}}}
+    end
+    def gen_template template
+#{{{
+      ConfigFile::gen_template(template)
+      self
+#}}}
+    end
+#}}}
+  end
+#
+# run main program unless included as lib
+#
+  Main::new if $0 == __FILE__
+#
+# the default config is stored here
+#
+__END__
+#
+# default config
+#
+
+key0 : value0
+key1 : value1