lobster 0.1.0.alpha2 → 0.1.0

data/.gitignore

@@ -0,0 +1 @@
+*.gem

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    lobster (0.1.0.alpha2)
+    lobster (0.1.0)
       daemons (>= 1.1.4)
 
 GEM

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2011 Criteo
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,42 @@
+Lobster
+=======
+
+Simple job service that was originally written to run hadoop jobs, but can be 
+used for any batch-processing scripts that you want to run.
+
+Alpha Version Warning
+---------------------
+
+Lobster has not been fully released and tested yet.
+
+How It Works
+------------
+
+### Create a `config/schedule.rb` file
+
+~~~~~ ruby
+job "my-job" do
+  cmd "runthis && runthat >> log/here.log"
+  delay 5 # minutes
+end
+~~~~~
+
+### Run Lobster
+
+- In the console with `LOBSTER_DIR=/path/to/jobs/directory lobster run`
+- As a service with `LOBSTER_DIR=/path/to/jobs/directory sudo lobster start`
+
+The job `my-job` will start in the next 5 minutes, and once it's done, 
+lobster will wait 5 minutes before starting it again.
+
+Any log in stderr/stdout will be written in `$LOBSTER_DIR/log/lobster.output` 
+along with some other useful information for monitoring.
+
+The current directory where the job command is run is `$LOBSTER_DIR`.
+
+How To Install
+--------------
+
+    gem install lobster --pre
+
+

data/bin/lobster

@@ -1,25 +1,32 @@
 #!/usr/bin/env ruby
 
 require 'lobster'
+require 'logger'
 require 'daemons'
 
 lobster_dir = File.expand_path(ENV['LOBSTER_DIR'] || '.')
-log_dir = File.join(lobster_dir, 'log')
+lobster_env = ENV['LOBSTER_ENV'] || 'development'
 
-Dir.mkdir log_dir unless Dir.exist?(log_dir) || ARGV.first == "run"
+Lobster.logger = Logger.new(STDOUT)
+config = Lobster::Configuration.new(lobster_dir, lobster_env)
 
 #puts "LOBSTER_DIR set to #{lobster_dir}"
+#puts "Config: #{config}"
+
+log_dir = config[:log_dir]
+Dir.mkdir log_dir unless Dir.exist?(log_dir) || ARGV.first == "run"
+
 
-options = {
+daemon_options = {
   :multiple => false,
-  :monitor => true,
+  :monitor => config[:monitor],
   :backtrace => true,
-  :log_dir => File.join(lobster_dir, 'log'),
-  :log_output => true
+  :log_dir => config[:log_dir],
+  :log_output => true,
+  :dir_mode => :normal,
+  :dir => config[:pid_dir]
 }
-options[:dir_mode] = :system unless ARGV.first == "run"
-options[:log_dir] = File.join(lobster_dir, 'log')
 
-Daemons.run_proc('lobster', options) do
-  Lobster::Service.start(lobster_dir)
+Daemons.run_proc('lobster', daemon_options) do
+  Lobster::Service.start(config)
 end

data/lib/lobster/configuration.rb

@@ -0,0 +1,39 @@
+require 'yaml'
+
+module Lobster
+  class Configuration
+    def initialize(dir, env)
+      @config = {
+        :monitor => true,
+        :log_dir => 'log',
+        :pid_dir => File.join('/', 'var', 'run', 'lobster'),
+        :schedule_file => File.join('config', 'schedule.rb')
+      }
+
+      config_file_path = File.join(dir, 'config', 'lobster.yml')
+      if File.exist? config_file_path
+        config_file = YAML.load_file(config_file_path)
+        @config.keys.each do |key|
+          @config[key] = config_file[env][key.to_s] || config_file[key.to_s] || @config[key]
+        end
+      end
+
+      # these keys cannot be set by the config file
+      @config[:lobster_dir] = dir
+      @config[:environment] = env
+
+      # make paths absolute
+      [:log_dir, :pid_dir, :schedule_file].each do |k|
+        @config[k] = File.absolute_path(@config[k], dir)
+      end
+    end
+    
+    def [](key)
+      @config[key]
+    end
+    
+    def to_s
+      @config.to_s
+    end
+  end
+end

data/lib/lobster/job.rb

@@ -39,7 +39,7 @@ module Lobster
     end
 
     def run(out,err,dir)
-      Lobster.logger.info "Starting job #{@name} from directory #{dir}"
+      Lobster.logger.info "Starting job #{@name}"
       begin
         @pid = spawn(@command, :out=>out, :err=>err, :chdir=>dir)
       rescue Exception => e
@@ -47,5 +47,35 @@ module Lobster
         @next_run = Time.now + 10
       end
     end
+    
+    def kill(sig)
+      if @pid
+        Lobster.logger.info "Killing job #{@name} with pid #{@pid} and all its children"
+        kill_tree sig, @pid
+      end
+    end
+
+    private
+
+    def kill_tree(sig, pid)
+      child_parent_processes = `ps -eo pid,ppid | grep #{pid}`
+      child_parent_processes = child_parent_processes.split("\n").map do |child_and_parent|
+        child_and_parent.strip.split(/\s+/).map(&:to_i)
+      end
+      child_parent_processes.each do |child, parent|
+        if parent == pid
+          kill_tree(sig, child)
+        end
+      end
+      
+      Lobster.logger.info "Killing pid #{pid}"
+      begin
+        Process.kill sig, pid
+      rescue Errno::ESRCH
+        # Process already got killed somehow
+      rescue Exception => e
+        Lobster.logger.warn "Process #{pid} exception: #{e}"
+      end
+    end
   end
 end

data/lib/lobster/service.rb

@@ -1,17 +1,16 @@
-require 'logger'
 require 'lobster/version'
 
 module Lobster
   class Service
-    def self.start(dir)
-      Lobster.logger = Logger.new(STDOUT)
-      new(dir).run
+    def self.start(config)
+      new(config).run
     end
 
-    def initialize(dir)
+    def initialize(config)
       @job_list = nil
-      @directory = dir || '.'
-      @file = File.join(@directory, 'config', 'schedule.rb')
+      @running = true
+      @sleeping = false
+      @config = config
       @poll_delay = 60
 
       rout, @wout = IO.pipe
@@ -29,37 +28,43 @@ module Lobster
         end
       end
 
-      Signal.trap "INT" do
-        Lobster.logger.info  "All jobs are getting killed."
-        exit 0
+      at_exit do
+        @running = false
+        sleep 0.01 until @sleeping # make sure no new jobs are created
+       
+        Lobster.logger.info  "Exiting, all jobs are getting killed."
+        @job_list.jobs.each_value do |job|
+          job.kill 'INT'
+        end if @job_list
       end
     end
 
     def run
       Lobster.logger.info "Lobster started version #{Lobster::VERSION}"
-      Lobster.logger.info "Schedule file: #{@file}"
-      Lobster.logger.info "Poll delay: #{@poll_delay}"
+      Lobster.logger.info "Lobster config: #{@config}"
       
-      loop do
+      while @running
+        @sleeping = false
         now = Time.now
         
-        reload_config
+        reload_schedule
 
         @job_list.jobs.each_value do |job|
           if not job.running? and now >= job.next_run
-            job.run(@wout, @werr, @directory)
+            job.run(@wout, @werr, @config[:lobster_dir])
           end
         end
+        @sleeping = true
         sleep @poll_delay
       end
     end
 
-    def reload_config
-      @job_list ||= JobList.new(@file)
+    def reload_schedule
+      @job_list ||= JobList.new(@config[:schedule_file])
       begin
         @job_list.reload
       rescue Exception => e
-        Lobster.logger.error "#{e}: error while reading config file in #{@file}, not updating"
+        Lobster.logger.error "#{e}: error while reading config file in #{@config[:schedule_file]}, not updating"
       end
     end
   end

data/lib/lobster/version.rb

@@ -1,3 +1,3 @@
 module Lobster
-  VERSION = '0.1.0.alpha2'
+  VERSION = '0.1.0'
 end

data/lib/lobster.rb

@@ -1,4 +1,5 @@
 module Lobster
+  autoload :Configuration, 'lobster/configuration'
   autoload :JobList,  'lobster/job_list'
   autoload :Job,      'lobster/job'
   autoload :Service,  'lobster/service'

data/lobster.gemspec

@@ -8,7 +8,7 @@ Gem::Specification.new do |s|
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Maxime Brugidou"]
   s.email       = ["m.brugidou@criteo.com"]
-  s.homepage    = ""
+  s.homepage    = "https://github.com/brugidou/lobster"
   s.summary     = %q{Simple loop job runner service.}
   s.description = %q{}
   s.files         = `git ls-files`.split("\n")

metadata

@@ -1,19 +1,19 @@
 --- !ruby/object:Gem::Specification
 name: lobster
 version: !ruby/object:Gem::Version
-  version: 0.1.0.alpha2
-  prerelease: 6
+  version: 0.1.0
+  prerelease: 
 platform: ruby
 authors:
 - Maxime Brugidou
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-12-19 00:00:00.000000000 Z
+date: 2011-12-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
-  requirement: &79856860 !ruby/object:Gem::Requirement
+  requirement: &80998950 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,7 +21,7 @@ dependencies:
         version: 1.1.4
   type: :runtime
   prerelease: false
-  version_requirements: *79856860
+  version_requirements: *80998950
 description: ''
 email:
 - m.brugidou@criteo.com
@@ -30,16 +30,20 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- .gitignore
 - Gemfile
 - Gemfile.lock
+- LICENSE
+- README.md
 - bin/lobster
 - lib/lobster.rb
+- lib/lobster/configuration.rb
 - lib/lobster/job.rb
 - lib/lobster/job_list.rb
 - lib/lobster/service.rb
 - lib/lobster/version.rb
 - lobster.gemspec
-homepage: ''
+homepage: https://github.com/brugidou/lobster
 licenses: []
 post_install_message: 
 rdoc_options: []
@@ -54,9 +58,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
-  - - ! '>'
+  - - ! '>='
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.11