ambethia-bj 1.2.1

data/lib/bj/api.rb

@@ -0,0 +1,164 @@
+class Bj
+#
+# the api exposes nearly all the bj code you'll likely need, with the
+# exception of accessing the job table for searching, which is done using
+#
+# eg.
+#
+#   Bj.table.job.find :all
+#
+  module API
+  #
+  # submit jobs for background processing.  'jobs' can be a string or array of
+  # strings.  options are applied to each job in the 'jobs', and the list of
+  # submitted jobs is always returned.  options (string or symbol) can be
+  #
+  #   :rails_env => production|development|key_in_database_yml 
+  #                 when given this keyword causes bj to submit jobs to the
+  #                 specified database.  default is RAILS_ENV.
+  #
+  #   :priority => any number, including negative ones.  default is zero.
+  #
+  #   :tag => a tag added to the job.  simply makes searching easier.
+  #
+  #   :env => a hash specifying any additional environment vars the background
+  #           process should have.
+  #
+  #   :stdin => any stdin the background process should have.
+  #
+  # eg:
+  #
+  #   jobs = Bj.submit 'echo foobar', :tag => 'simple job'
+  #
+  #   jobs = Bj.submit '/bin/cat', :stdin => 'in the hat', :priority => 42
+  #
+  #   jobs = Bj.submit './script/runner ./scripts/a.rb', :rails_env => 'production'
+  #
+  #   jobs = Bj.submit './script/runner /dev/stdin', :stdin => 'p RAILS_ENV', :tag => 'dynamic ruby code'
+  #
+  #   jobs = Bj.submit array_of_commands, :priority => 451 
+  #
+  # when jobs are run, they are run in RAILS_ROOT.  various attributes are
+  # available *only* once the job has finished.  you can check whether or not
+  # a job is finished by using the #finished method, which simple does a
+  # reload and checks to see if the exit_status is non-nil.
+  #
+  # eg:
+  #
+  #   jobs = Bj.submit list_of_jobs, :tag => 'important'
+  #   ...
+  #   
+  #   jobs.each do |job|
+  #     if job.finished?
+  #       p job.exit_status
+  #       p job.stdout
+  #       p job.stderr
+  #     end
+  #   end
+  #
+  #
+    def submit jobs, options = {}, &block
+      options.to_options!
+      Bj.transaction(options) do
+        table.job.submit jobs, options, &block
+      end
+    ensure
+      Bj.runner.tickle unless options[:no_tickle]
+    end
+  #
+  # this method changes the context under which bj is operating.  a context is
+  # a RAILS_ENV.  the method accepts a block and it used to alter the
+  # behaviour of the bj lib on a global scale such that all operations,
+  # spawning of background runnner processes, etc, occur in that context.
+  #
+  # eg:
+  #
+  #  Bj.in :production do
+  #    Bj.submit './script/runner ./scripts/facebook_notification.rb'
+  #  end
+  #
+  #  Bj.in :development do
+  #    Bj.submit 'does_this_eat_memory.exe'
+  #  end
+  #
+    def in rails_env = Bj.rails_env, &block
+      transaction(:rails_env => rails_env.to_s, &block)
+    end
+  #
+  # list simply returns a list of all jobs in the job table
+  #
+    def list options = {}, &block
+      options.to_options!
+      Bj.transaction(options) do
+        options.delete :rails_env
+        table.job.find(:all, options)
+      end
+    end
+  #
+  #
+  #
+    def run options = {}
+      runner.run options
+    end
+  #
+  # generate a migration and migrate a database (production/development/etc)
+  #
+    def setup options = {}
+      options.to_options!
+      chroot do
+        generate_migration options
+        migrate options
+      end
+    end
+  #
+  # generate_migration, suprisingly, generates the single migration needed for
+  # bj.  you'll notice the the migration is very short as the migration
+  # classes themselves are inner classes of the respective bj table class.
+  # see lib/bj/table.rb for details.
+  #
+    def generate_migration options = {}
+      options.to_options!
+      options[:verbose] = true
+      chroot do
+        before = Dir.glob "./db/migrate/*"
+        n = Dir.glob("./db/migrate/*_bj_*").size
+        classname = "BjMigration#{ n }"
+        util.spawn "#{ Bj.ruby } ./script/generate migration #{ classname }", options rescue nil
+        after = Dir.glob "./db/migrate/*"
+        candidates = after - before
+        case candidates.size
+          when 0
+            false
+          when 1
+            generated = candidates.first
+            open(generated, "w"){|fd| fd.puts Bj.table.migration_code(classname)}
+            Bj.logger.info{ "generated <#{ generated }>" }
+            generated
+          else
+            raise "ambiguous migration <#{ candidates.inspect }>"
+        end
+      end
+    end
+  #
+  # migrate a database (production|development|etc)
+  #
+    def migrate options = {}
+      options.to_options!
+      options[:verbose] = true
+      chroot do
+        util.spawn "#{ Bj.rake } RAILS_ENV=#{ Bj.rails_env } db:migrate", options
+      end
+    end
+  #
+  # install plugin into this rails app 
+  #
+    def plugin options = {}
+      options.to_options!
+      options[:verbose] = true
+      chroot do
+        util.spawn "#{ Bj.ruby } ./script/plugin install http://codeforpeople.rubyforge.org/svn/rails/plugins/bj --force", options
+      end
+    end
+  end
+  send :extend, API
+end

data/lib/bj/bj.rb

@@ -0,0 +1,72 @@
+class Bj
+  module ClassMethods
+    fattr("rails_root"){ Util.const_or_env("RAILS_ROOT"){ "." } }
+    fattr("rails_env"){ Util.const_or_env("RAILS_ENV"){ "development" } }
+    fattr("database_yml"){ File.join rails_root, "config", "database.yml" }
+    fattr("configurations"){ YAML::load(ERB.new(IO.read(database_yml)).result) }
+    fattr("tables"){ Table.list }
+    fattr("hostname"){ Socket.gethostname }
+    fattr("logger"){ Bj::Logger.off STDERR }
+    fattr("ruby"){ Util.which_ruby }
+    fattr("rake"){ Util.which_rake }
+    fattr("script"){ Util.find_script "bj" }
+    fattr("ttl"){ Integer(Bj::Table::Config["ttl"] || (twenty_four_hours = 24 * 60 * 60)) }
+    fattr("table"){ Table }
+    fattr("config"){ table.config }
+    fattr("util"){ Util }
+    fattr("runner"){ Runner }
+    fattr("joblist"){ Joblist }
+    fattr("default_path"){ %w'/bin /usr/bin /usr/local/bin /opt/local/bin'.join(File::PATH_SEPARATOR) }
+
+    def transaction options = {}, &block
+      options.to_options!
+
+      cur_rails_env = Bj.rails_env.to_s
+      new_rails_env = options[:rails_env].to_s
+
+      cur_spec = configurations[cur_rails_env]
+      table.establish_connection(cur_spec) unless table.connected?
+
+      if(new_rails_env.empty? or cur_rails_env == new_rails_env) 
+        table.transaction{ block.call(table.connection) }
+      else
+        new_spec = configurations[new_rails_env]
+        table.establish_connection(new_spec)
+        Bj.rails_env = new_rails_env
+        begin
+          table.transaction{ block.call(table.connection) }
+        ensure
+          table.establish_connection(cur_spec)
+          Bj.rails_env = cur_rails_env
+        end
+      end
+    end
+
+    def chroot options = {}, &block
+      if defined? @chrooted and @chrooted
+        return(block ? block.call(@chrooted) : @chrooted)
+      end
+      if block
+        begin
+          chrooted = @chrooted
+          Dir.chdir(@chrooted = rails_root) do
+            raise RailsRoot, "<#{ Dir.pwd }> is not a rails root" unless Util.valid_rails_root?(Dir.pwd)
+            block.call(@chrooted)
+          end
+        ensure
+          @chrooted = chrooted 
+        end
+      else
+        Dir.chdir(@chrooted = rails_root)
+        raise RailsRoot, "<#{ Dir.pwd }> is not a rails root" unless Util.valid_rails_root?(Dir.pwd)
+        @chrooted
+      end
+    end
+
+    def boot
+      load File.join(rails_root, "config", "boot.rb")
+      load File.join(rails_root, "config", "environment.rb")
+    end
+  end
+  send :extend, ClassMethods
+end

data/lib/bj/errors.rb

@@ -0,0 +1,4 @@
+class Bj
+  class Error < ::StandardError; end
+  class RailsRoot < Error; end
+end

data/lib/bj/joblist.rb

@@ -0,0 +1,112 @@
+class Bj
+  class JobList < ::Array
+    module ClassMethods
+      def for jobs, options = {}
+        if Joblist === jobs
+          jobs.update options
+          return jobs
+        end
+        options.to_options!
+        jobs = [jobs].flatten.compact
+        list = []
+        jobs.each do |arg|
+          list.push(
+            case arg
+              when String
+                case arg
+                  when %r/^\d+$/
+                    job_from_id arg
+                  else
+                    job_from_string arg
+                end
+              when Hash
+                job_from_hash arg
+              when Io
+                jobs_from_io arg
+              when Fixnum, Bignum
+                job_from_number arg
+              else
+                job_from_string arg
+            end
+          )
+        end
+        list.flatten!
+        list.compact!
+        list.map!{|job| job.reverse_merge! options}
+        list
+      end
+
+      def job_from_hash arg
+        arg.to_hash.to_options!
+      end
+
+      def job_from_string arg
+        unless arg.strip.empty?
+          { :command => arg.to_s }
+        else
+          nil
+        end
+      end
+
+      def job_from_number arg
+        id = arg.to_i
+        Table::Job.find(id).to_hash
+      end
+
+      def jobs_from_io arg 
+        if arg == "-"
+          load_from_io STDIN
+        else
+          if arg.respond_to? :read
+            load_from_io arg 
+          else
+            open(arg, "r"){|fd| load_from_io fd}
+          end
+        end
+      end
+
+      def load_from_io io
+        list = []
+        io.each do |line|
+          line.strip!
+          next if line.empty?
+          list << job_from_string(line)
+        end
+        list
+      end
+
+      def jobs_from_yaml arg 
+        object =
+          if arg == "-"
+            YAML.load STDIN
+          else
+            if arg.respond_to? :read
+              YAML.load arg
+            else
+              open(arg, "r"){|fd| YAML.load fd}
+            end
+          end
+        Joblist.for object
+      end
+    end
+    send :extend, ClassMethods
+
+    module InstanceMethods
+      def update options = {}
+        options.to_options!
+        each{|job| job.update options}
+      end
+
+      def push other
+        Joblist.for(other).each do |job|
+          super job
+        end
+        self
+      end
+      alias_method "<<", "push"
+    end
+    send :include, InstanceMethods
+  end
+
+  Joblist = JobList
+end

data/lib/bj/logger.rb

@@ -0,0 +1,50 @@
+class Bj
+  class Logger < ::Logger
+    def self.new *a, &b
+      super(*a, &b).instance_eval{ @default_formatter = @formatter = Formatter.new; self }
+    end
+    def format_message(severity, datetime, progname, msg)
+      (@formatter || @default_formatter).call(severity, datetime, progname, msg)
+    end
+
+    def device
+      @logdev.instance_eval{ @dev }
+    end
+
+    def tty?
+      device.respond_to?('tty?') and device.tty?
+    end
+
+    def turn which
+      @logdev.extend OnOff unless OnOff === @logdev
+      @logdev.turn which
+    end
+
+    module OnOff
+      def turn which
+        @turned = which.to_s =~ %r/on/i ? :on : :off
+      end
+
+      def write message 
+        return message.to_s.size if @turned == :off
+        super
+      end
+    end
+
+    def on
+      turn :on
+    end
+    alias_method "on!", "on"
+    def self.on *a, &b
+      new(*a, &b).instance_eval{ turn :on; self }
+    end
+
+    def off
+      turn :off
+    end
+    alias_method "off!", "off"
+    def self.off *a, &b
+      new(*a, &b).instance_eval{ turn :off; self }
+    end
+  end
+end

data/lib/bj/runner.rb

@@ -0,0 +1,330 @@
+class Bj
+  class Runner
+    class Background
+      def self.for(*a, &b) new(*a, &b) end
+
+      fattr("command"){}
+      fattr("thread"){}
+      fattr("pid"){}
+
+      def initialize command
+        @command = command
+        @thread = new_thread
+      end
+
+      def inspect
+        {
+          "command" => command,
+          "pid" => pid,
+        }.inspect
+      end
+
+
+# TODO - auto start runner?
+
+      def new_thread 
+        this = self
+        Thread.new do
+          Thread.current.abort_on_exception = true
+          loop do
+            cleanup = lambda{}
+
+            IO.popen command, "r+" do |pipe|
+              this.pid = pid = pipe.pid
+              cleanup = lambda do
+                cleanup = lambda{}
+                begin; Process.kill(Runner.kill_signal, pid); rescue Exception; 42; end
+              end
+              at_exit &cleanup
+              Process.wait
+            end
+
+            Bj.logger.error{ "#{ command } failed with #{ $?.inspect }" } unless
+              [0, 42].include?($?.exitstatus)
+
+            cleanup.call
+
+            sleep 42
+          end
+        end
+      end
+    end
+
+    module ClassMethods
+      fattr("thread"){ Thread.current }
+      fattr("hup_signal"){ Signal.list.keys.index("HUP") ? "HUP" : "ABRT" }
+      fattr("hup_signaled"){ false }
+      fattr("kill_signal"){ "TERM" }
+      fattr("kill_signaled"){ false }
+
+      def tickle
+        return nil if Bj.config[Runner.no_tickle_key]
+        ping or start
+      end
+
+      def ping
+        begin
+          pid = nil
+          uri = nil
+          process = nil
+          Bj.transaction do
+            pid = Bj.config[Runner.key(Process.pid)] || Bj.config[Runner.key]
+            uri = Bj.config["#{ pid }.uri"]
+            process =
+              if uri
+                require "drb"
+                # DRb.start_service "druby://localhost:0"
+                DRbObject.new(nil, uri)
+              else
+                Process
+              end
+          end
+          return nil unless pid
+          pid = Integer pid
+          begin
+            process.kill Runner.hup_signal, pid
+            pid
+          rescue Exception => e
+            false
+          end
+        rescue Exception => e
+          false
+        end
+      end
+
+      def key ppid = 0
+        ppid ||= 0
+        #"#{ Bj.rails_env }.#{ ppid }.pid"
+        "runner.#{ ppid }.pid"
+      end
+
+      def no_tickle_key 
+        # "#{ Bj.rails_env }.no_tickle"
+        "no_tickle"
+      end
+
+      def start options = {}
+        options.to_options!
+        background.delete Bj.rails_env if options[:force]
+        background[Bj.rails_env] ||= Background.for(command)
+      end
+
+      def background
+        @background ||= Hash.new
+      end
+
+      def background= value
+        @background ||= value 
+      end
+
+      def command
+        "#{ Bj.ruby } " + %W[
+          #{ Bj.script }
+          run 
+          --forever 
+          --redirect=#{ log }
+          --ppid=#{ Process.pid }
+          --rails_env=#{ Bj.rails_env }
+          --rails_root=#{ Bj.rails_root }
+        ].map{|word| word.inspect}.join(" ")
+      end
+
+      def log
+        File.join logdir, "bj.#{ Bj.hostname }.#{ Bj.rails_env }.log"
+      end
+
+      def logdir
+        File.join File.expand_path(Bj.rails_root), 'log'
+      end
+
+      def run options = {}, &block
+        new(options, &block).run
+      end 
+    end
+    send :extend, ClassMethods
+
+    module Instance_Methods
+      fattr("options"){}
+      fattr("block"){}
+
+      def initialize options = {}, &block
+        options.to_options!
+        @options, @block = options, block
+      end
+
+      def run
+        wait = options[:wait] || 42
+        limit = options[:limit]
+        forever = options[:forever]
+
+        limit = false if forever
+        wait = Integer wait
+        loopno = 0
+
+        Runner.thread = Thread.current
+        Bj.chroot
+
+        register or exit!(EXIT::WARNING)
+
+        Bj.logger.info{ "STARTED" }
+        at_exit{ Bj.logger.info{ "STOPPED" } }
+
+        fill_morgue
+        install_signal_handlers
+
+        loop do
+          ping_parent
+
+          loopno += 1
+          break if(limit and loopno > limit)
+
+          archive_jobs
+
+          catch :no_jobs do
+            loop do
+              job = thread = stdout = stderr = nil
+
+              Bj.transaction(options) do
+                now = Time.now.utc
+
+                job = Bj::Table::Job.find :first,
+                                          :conditions => ["state = ? and submitted_at <= ?", "pending", now],
+                                          :order => "priority DESC, submitted_at ASC", 
+                                          :limit => 1,
+                                          :lock => true
+                throw :no_jobs unless job
+
+                Bj.logger.info{ "#{ job.title } - starting..." }
+              end
+
+              job.run!
+
+              Bj.logger.info{ "#{ job.title } - exit_status=#{ job.exit_status }" }
+            end
+          end
+
+          Runner.hup_signaled false
+          wait.times do
+            break if Runner.hup_signaled?
+            break if Runner.kill_signaled?
+            sleep 1
+          end 
+
+          break unless(limit or limit == false)
+          break if Runner.kill_signaled?
+        end
+      end
+
+      def ping_parent
+        ppid = options[:ppid]
+        return unless ppid 
+        begin
+          Process.kill 0, Integer(ppid)
+        rescue Errno::ESRCH
+          Kernel.exit 42
+        rescue Exception
+          42
+        end
+      end
+
+      def install_signal_handlers
+        Runner.hup_signaled false
+        hup_handler = nil
+        hup_handler =
+          trap Runner.hup_signal do |*a|
+            begin
+              Runner.hup_signaled true
+            rescue Exception => e
+              Bj.logger.error{ e } rescue nil
+            end
+            hup_handler.call *a rescue nil
+          end
+
+        Runner.kill_signaled false
+        kill_handler = nil
+        kill_handler =
+          trap Runner.kill_signal do |*a|
+            begin
+              Runner.kill_signaled true
+            rescue Exception => e
+              Bj.logger.error{ e } rescue nil
+            end
+            kill_handler.call *a rescue nil
+          end
+
+        begin
+          trap("INT"){ exit }
+        rescue Exception
+        end
+      end
+
+      def fill_morgue
+        Bj.transaction do
+          now = Time.now.utc
+          jobs = Bj::Table::Job.find :all,
+                                     :conditions => ["state = 'running' and runner = ?", Bj.hostname]
+          jobs.each do |job|
+            if job.is_restartable?
+              Bj.logger.info{ "#{ job.title } - found dead and bloated but resubmitted" }
+              %w[ runner pid started_at finished_at stdout stderr exit_status ].each do |column|
+                job[column] = nil
+              end
+              job.state = 'pending'
+            else
+              Bj.logger.info{ "#{ job.title } - found dead and bloated" }
+              job.state = 'dead'
+              job.finished_at = now
+            end
+            job.save!
+          end
+        end
+      end
+
+      def archive_jobs
+        Bj.transaction do
+          now = Time.now.utc
+          too_old = now - Bj.ttl
+          jobs = Bj::Table::Job.find :all,
+                                     :conditions => ["(state = 'finished' or state = 'dead') and submitted_at < ?", too_old]
+          jobs.each do |job|
+            Bj.logger.info{ "#{ job.title } - archived" }
+            hash = job.to_hash.update(:archived_at => now)
+            Bj::Table::JobArchive.create! hash 
+            job.destroy
+          end
+        end
+      end
+
+      def register
+        Bj.transaction do
+          pid = Bj.config[key]
+          return false if Util.alive?(pid)
+          Bj.config[key] = Process.pid
+          unless Bj.util.ipc_signals_supported? # not winblows
+            require "drb"
+            DRb.start_service "druby://localhost:0", Process
+            Bj.config["#{ Process.pid }.uri"] = DRb.uri 
+          end
+        end
+        at_exit{ unregister }
+        true
+      rescue Exception
+        false
+      end
+
+      def unregister
+        Bj.transaction do
+          Bj.config.delete key
+        end
+        true
+      rescue Exception
+        false
+      end
+
+      def key
+        @key ||= ( options[:ppid] ? Runner.key(options[:ppid]) : Runner.key )
+      end
+    end
+    send :include, Instance_Methods
+  end
+end