bj 0.0.1

data/lib/bj/bj.rb

@@ -0,0 +1,92 @@
+class Bj
+  module ClassMethods
+    attribute("rails_root"){ Util.const_or_env("RAILS_ROOT"){ "." } }
+    attribute("rails_env"){ Util.const_or_env("RAILS_ENV"){ "development" } }
+    attribute("database_yml"){ File.join rails_root, "config", "database.yml" }
+    attribute("configurations"){ YAML.load IO.read(database_yml) }
+    attribute("tables"){ Table.list }
+    attribute("hostname"){ Socket.gethostname }
+    attribute("logger"){ Bj::Logger.off STDERR }
+    attribute("ruby"){ Util.which_ruby }
+    attribute("script"){ Util.find_script "bj" }
+    attribute("ttl"){ Integer(Bj::Table::Config["ttl"] || (twenty_four_hours = 24 * 60 * 60)) }
+    attribute('table'){ Table }
+    attribute('config'){ table.config }
+    attribute('util'){ Util }
+    attribute('runner'){ Runner }
+    attribute('joblist'){ Joblist }
+
+    def transaction options = {}, &block
+      options.to_options!
+      options.reverse_merge! :rails_env => Bj.rails_env
+      bj_rails_env = Bj.rails_env
+      begin
+        Bj.rails_env = options[:rails_env].to_s
+        connecting(options) do
+          ActiveRecord::Base.transaction do 
+            block.call ActiveRecord::Base.connection
+          end
+        end
+      ensure
+        Bj.rails_env = bj_rails_env 
+      end
+    end
+
+    def connecting options = {}, &block
+      options.to_options!
+      rails_env = (options[:rails_env] || Bj.rails_env).to_s
+      previous = ar_connection
+      ActiveRecord::Base.connection = connections[rails_env] unless ActiveRecord::Base.connection == connections[rails_env]
+      begin
+        block.call ActiveRecord::Base.connection
+      ensure
+        ActiveRecord::Base.connection = previous unless ActiveRecord::Base.connection == previous
+      end
+    end
+
+    def connections
+      @connections ||= Hash.new do |hash, rails_env|
+        rails_env = rails_env.to_s
+        ActiveRecord::Base.establish_connection configurations[rails_env]
+        hash[rails_env] = ActiveRecord::Base.connection
+      end.merge(ar_connections)
+      @connections
+    end
+
+    def ar_connections
+      if defined?(RAILS_ENV)
+        { RAILS_ENV => ar_connection } 
+      else
+        {}
+      end
+    end
+
+    def ar_connection
+      begin
+        ActiveRecord::Base.connection
+      rescue ActiveRecord::ConnectionNotEstablished 
+        ActiveRecord::Base.establish_connection configurations[rails_env] 
+      end
+    end
+
+    def chroot options = {}, &block
+      if defined? @chrooted and @chrooted
+        return block.call(@chrooted)
+      end
+      begin
+        Dir.chdir @chrooted = rails_root do |pwd|
+          raise RailsRoot, "<#{ pwd }> is not a rails root" unless Util.valid_rails_root?(pwd)
+          block.call(@chrooted)
+        end
+      ensure
+        @chrooted = nil 
+      end
+    end
+
+    def boot
+      load File.join(rails_root, "config", "boot.rb")
+      load File.join(rails_root, "config", "environment.rb")
+    end
+  end
+  send :extend, ClassMethods
+end

data/lib/bj/errors.rb

@@ -0,0 +1,4 @@
+class Bj
+  class Error < ::StandardError; end
+  class RailsRoot < Error; end
+end

data/lib/bj/joblist.rb

@@ -0,0 +1,112 @@
+class Bj
+  class JobList < ::Array
+    module ClassMethods
+      def for jobs, options = {}
+        if Joblist === jobs
+          jobs.update options
+          return jobs
+        end
+        options.to_options!
+        jobs = [jobs].flatten.compact
+        list = []
+        jobs.each do |arg|
+          list.push(
+            case arg
+              when String
+                case arg
+                  when %r/^\d+$/
+                    job_from_id arg
+                  else
+                    job_from_string arg
+                end
+              when Hash
+                job_from_hash arg
+              when Io
+                jobs_from_io arg
+              when Fixnum, Bignum
+                job_from_number arg
+              else
+                job_from_string arg
+            end
+          )
+        end
+        list.flatten!
+        list.compact!
+        list.map!{|job| job.reverse_merge! options}
+        list
+      end
+
+      def job_from_hash arg
+        arg.to_hash.to_options!
+      end
+
+      def job_from_string arg
+        unless arg.strip.empty?
+          { :command => arg.to_s }
+        else
+          nil
+        end
+      end
+
+      def job_from_number arg
+        id = arg.to_i
+        Table::Job.find(id).to_hash
+      end
+
+      def jobs_from_io arg 
+        if arg == "-"
+          load_from_io STDIN
+        else
+          if arg.respond_to? :read
+            load_from_io arg 
+          else
+            open(arg, "r"){|fd| load_from_io fd}
+          end
+        end
+      end
+
+      def load_from_io io
+        list = []
+        io.each do |line|
+          line.strip!
+          next if line.empty?
+          list << job_from_string(line)
+        end
+        list
+      end
+
+      def jobs_from_yaml arg 
+        object =
+          if arg == "-"
+            YAML.load STDIN
+          else
+            if arg.respond_to? :read
+              YAML.load arg
+            else
+              open(arg, "r"){|fd| YAML.load fd}
+            end
+          end
+        Joblist.for object
+      end
+    end
+    send :extend, ClassMethods
+
+    module InstanceMethods
+      def update options = {}
+        options.to_options!
+        each{|job| job.update options}
+      end
+
+      def push other
+        Joblist.for(other).each do |job|
+          super job
+        end
+        self
+      end
+      alias_method "<<", "push"
+    end
+    send :include, InstanceMethods
+  end
+
+  Joblist = JobList
+end

data/lib/bj/logger.rb

@@ -0,0 +1,50 @@
+class Bj
+  class Logger < ::Logger
+    def self.new *a, &b
+      super(*a, &b).instance_eval{ @default_formatter = @formatter = Formatter.new; self }
+    end
+    def format_message(severity, datetime, progname, msg)
+      (@formatter || @default_formatter).call(severity, datetime, progname, msg)
+    end
+
+    def device
+      @logdev.instance_eval{ @dev }
+    end
+
+    def tty?
+      device.respond_to?('tty?') and device.tty?
+    end
+
+    def turn which
+      @logdev.extend OnOff unless OnOff === @logdev
+      @logdev.turn which
+    end
+
+    module OnOff
+      def turn which
+        @turned = which.to_s =~ %r/on/i ? :on : :off
+      end
+
+      def write message 
+        return message.to_s.size if @turned == :off
+        super
+      end
+    end
+
+    def on
+      turn :on
+    end
+    alias_method "on!", "on"
+    def self.on *a, &b
+      new(*a, &b).instance_eval{ turn :on; self }
+    end
+
+    def off
+      turn :off
+    end
+    alias_method "off!", "off"
+    def self.off *a, &b
+      new(*a, &b).instance_eval{ turn :off; self }
+    end
+  end
+end

data/lib/bj/runner.rb

@@ -0,0 +1,285 @@
+class Bj
+  class Runner
+    class Background
+      def self.for(*a, &b) new(*a, &b) end
+
+      attribute "command"
+      attribute "thread"
+      attribute "pid"
+
+      def initialize command
+        @command = command
+        @thread = new_thread
+      end
+
+      def inspect
+        {
+          "command" => command,
+          "pid" => pid,
+        }.inspect
+      end
+
+      def new_thread 
+        this = self
+        Thread.new do
+          Thread.current.abort_on_exception = true
+          loop do
+            cleanup = lambda{}
+
+            IO.popen command, "r+" do |pipe|
+              this.pid = pid = pipe.pid
+              cleanup = lambda do
+                cleanup = lambda{}
+                begin; Process.kill('SIGTERM', pid); rescue Exception; 42; end
+              end
+              at_exit &cleanup
+              pipe.each{|line|}
+            end
+
+            Bj.logger.error{ "#{ command } failed with #{ $?.inspect }" } unless
+              [0, 42].include?($?.exitstatus)
+
+            cleanup.call
+
+            sleep 42
+          end
+        end
+      end
+    end
+
+    module ClassMethods
+      def tickle
+        ping or start
+      end
+
+      def ping
+        pid = Bj.config[Runner.key]
+        pid ? Util.ping(pid) : nil
+      end
+
+      def key
+        "#{ Bj.rails_env }.pid"
+      end
+
+      def start options = {}
+        options.to_options!
+        background.delete Bj.rails_env if options[:force]
+        background[Bj.rails_env] ||= Background.for(command)
+      end
+
+      def background
+        @background ||= Hash.new
+      end
+
+      def command
+        "#{ Bj.ruby } #{ Bj.script } run --forever --redirect=#{ log } --rails_env=#{ Bj.rails_env } --ppid=#{ Process.pid }"
+      end
+
+      def log
+        File.join logdir, "bj.#{ Bj.hostname }.#{ Bj.rails_env }.log"
+      end
+
+      def logdir
+        File.join File.expand_path(Bj.rails_root), 'log'
+      end
+
+      def run options = {}, &block
+        new(options, &block).run
+      end 
+    end
+    send :extend, ClassMethods
+
+    module Instance_Methods
+      attribute "options"
+      attribute "block"
+
+      def initialize options = {}, &block
+        options.to_options!
+        @options, @block = options, block
+      end
+
+      def run
+        wait = options[:wait] || 42
+        limit = options[:limit]
+        forever = options[:forever]
+        ppid = options[:ppid]
+
+        limit = false if forever
+        wait = Integer wait
+        loopno = 0
+        thread = new_lifeline(ppid) if ppid
+
+        Bj.chroot do
+          register or exit!(EXIT::WARNING)
+          Bj.logger.info{ "START" }
+          fill_morgue
+          install_signal_handlers
+
+          loop do
+            loopno += 1
+            break if(limit and n > limit)
+
+            Bj.logger.debug{ "loopno #{ loopno }" }
+
+            sweep
+
+            catch :no_jobs do
+              loop do
+                job = thread = stdout = stderr = nil
+
+                Bj.transaction(options) do
+                  job = Bj::Table::Job.find :first,
+                                            :conditions => "state = 'pending'",
+                                            :order => "priority DESC, submitted_at ASC", 
+                                            :limit => 1
+                  throw :no_jobs unless job
+
+                  Bj.logger.info{ "running #{ job.id } (#{ job.command })" }
+
+                  command = job.command
+                  env = job.env || {}
+                  stdin = job.stdin || ''
+                  stdout = job.stdout || ''
+                  stderr = job.stderr || ''
+                  started_at = Time.now
+
+                  q = Queue.new
+                  thread = Thread.new do
+                    Thread.current.abort_on_exception = true
+                    systemu command, :cwd=>Bj.rails_root, :env=>env, :stdin=>stdin, :stdout=>stdout, :stderr=>stderr do |pid|
+                      q << pid
+                    end
+                  end
+                  pid = q.pop
+
+                  job.state = "running"
+                  job.runner = Bj.hostname
+                  job.pid = pid
+                  job.started_at = started_at 
+                  job.save!
+                  job.update
+                end
+
+                exit_status = thread.value
+                finished_at = Time.now
+
+                Bj.transaction(options) do
+                  job = Bj::Table::Job.find job.id 
+                  break unless job
+                  job.state = "finished"
+                  job.finished_at = finished_at 
+                  job.stdout = stdout
+                  job.stderr = stderr
+                  job.exit_status = exit_status
+                  job.save!
+                  job.update
+                  Bj.logger.info{ "exit_status #{ job.exit_status }" }
+                end
+              end
+            end
+
+            sleep wait
+
+            break unless(limit or limit == false)
+          end
+        end
+      end
+
+      def fill_morgue options = {}
+        Bj.logger.debug{ "filling the morgue..." }
+        Bj.transaction(options) do
+          now = Time.now
+          jobs = Bj::Table::Job.find :all,
+                                     :conditions => ["state = 'running' and runner = ?", Bj.hostname]
+          jobs.each do |job|
+            Bj.logger.info{ "marking #{ job.id } (#{ job.command }) as dead." }
+            job.state = 'dead'
+            job.finished_at = now
+            job.save!
+          end
+        end
+      end
+
+      def sweep options = {}
+        Bj.logger.debug{ "sweeping..." }
+        Bj.transaction(options) do
+          now = Time.now
+          too_old = now - Bj.ttl
+          jobs = Bj::Table::Job.find :all,
+                                     :conditions => ["(state = 'finished' or state = 'dead') and submitted_at < ?", too_old]
+          jobs.each do |job|
+            Bj.logger.info{ "archiving #{ job.id } (#{ job.command })." }
+            hash = job.to_hash.update(:archived_at => now)
+            Bj::Table::JobArchive.create! hash 
+            job.destroy
+          end
+        end
+      end
+
+      def new_lifeline ppid
+        Thread.new do
+          loop do
+            begin
+              Process.kill 0, ppid
+            rescue Errno::ESRCH
+              STDERR.puts "parent #{ ppid } died"
+              Kernel.exit 42
+            end
+            sleep 42 
+          end
+        end
+      end
+
+      attribute 'sighup' => false
+
+      def install_signal_handlers
+        trap('SIGHUP') do
+          if sleeping?
+            Bj.logger.debug{ "woke up!" }
+            throw :wake_up
+          else
+            sighup true
+          end
+        end
+      end
+
+      def sleep seconds
+        if sighup
+          sighup false
+          return
+        end
+        Bj.logger.debug{ "sleeping #{ seconds }..." }
+        catch :wake_up do
+          begin
+            @sleeping = true
+            Kernel.sleep seconds
+          ensure
+            @sleeping = false
+          end
+        end
+      end
+
+      def sleeping?
+        @sleeping
+      end
+
+      def register options = {}
+        Bj.transaction(options) do
+          pid = Bj.config[Runner.key]
+          if Util.alive?(pid)
+            return false
+          end
+          Bj.config[Runner.key] = Process.pid
+          at_exit{ unregister }
+        end
+      end
+
+      def unregister options = {}
+        Bj.transaction(options) do
+          Bj.config.delete Runner.key
+        end
+      end
+    end
+    send :include, Instance_Methods
+  end
+end