lobster 0.1.0 → 0.1.1

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    lobster (0.1.0)
+    lobster (0.1.1.pre)
       daemons (>= 1.1.4)
 
 GEM

data/README.md

@@ -4,17 +4,29 @@ Lobster
 Simple job service that was originally written to run hadoop jobs, but can be 
 used for any batch-processing scripts that you want to run.
 
-Alpha Version Warning
----------------------
+Lobster runs as a service (daemon) and regularly checks a schedule in 
+`$LOBSTER_DIR/config/schedule.rb`. The goal is to run the scheduled jobs in 
+parallel but a job can't run twice at the same time (no overlap). The only 
+configuration needed is the delay between 2 job runs.
 
-Lobster has not been fully released and tested yet.
-
-How It Works
+Get Started
 ------------
 
+### Install
+
+    gem install lobster
+
 ### Create a `config/schedule.rb` file
 
+Run `lobsterize` in the directory where you need to setup and run your jobs 
+(defined by the LOBSTER_DIR variable)
+
+    lobsterize
+
+Then add a job in `config/schedule.rb`
+
 ~~~~~ ruby
+# config/schedule.rb
 job "my-job" do
   cmd "runthis && runthat >> log/here.log"
   delay 5 # minutes
@@ -23,20 +35,55 @@ end
 
 ### Run Lobster
 
-- In the console with `LOBSTER_DIR=/path/to/jobs/directory lobster run`
-- As a service with `LOBSTER_DIR=/path/to/jobs/directory sudo lobster start`
+Two environment variables are used (with defaults):
+
+- `LOBSTER_DIR` is the directory used for all job commands, and where the 
+  configuration/schedule files are (default: current 
+  directory)
+- `LOBSTER_ENV` is a variable used to handle different environments (default: 
+  development)
+
+You need write permission to `/var/run/lobster/` for pids, this path is 
+configurable.
+
+- `lobster run` will run in the console
+- or `lobster start` as a deamon
+
+### Configuration
+
+The lobster configuration file should be located in 
+`$LOBSTER_DIR/config/lobster.yml` and has the following layout
 
-The job `my-job` will start in the next 5 minutes, and once it's done, 
-lobster will wait 5 minutes before starting it again.
+~~~~ yaml
+# default values
 
-Any log in stderr/stdout will be written in `$LOBSTER_DIR/log/lobster.output` 
-along with some other useful information for monitoring.
+# monitor the daemon and restart it if it stops
+monitor: true
+# log directory, absolute or relative to the LOBSTER_DIR
+log_dir: log
+# pids directory, absolute or relative to the LOBSTER_DIR
+pid_dir: /var/run/lobster/
+# schedule file, absolute or relative to the LOBSTER_DIR
+schedule_file: config/schedule.rb
 
-The current directory where the job command is run is `$LOBSTER_DIR`.
+# environment overrides, the env variable LOBSTER_ENV has 
+# to be set (default: development)
+my_test_env:
+  monitor: false
+  log_dir: test_los
+my_prod_env:
+  pid_dir: /var/run/
+~~~~
 
-How To Install
---------------
+Any log in stderr/stdout will be written in the log directory as 
+`lobster.output`, the actual lobster log is in `lobster.log`
 
-    gem install lobster --pre
+Capistrano Deployment
+=====================
 
+It is very easy to have a lobster project with a proper `config/lobster.yml` to 
+handle different environments such as development, testing and production. You 
+can simply have a `schedule.rb`file per environment.
 
+Then you can use capistrano to deploy the new jobs and schedule to your servers 
+(the schedule is automatically reloaded).

data/bin/lobster

@@ -1,13 +1,12 @@
 #!/usr/bin/env ruby
 
+require 'optparse'
 require 'lobster'
-require 'logger'
 require 'daemons'
 
 lobster_dir = File.expand_path(ENV['LOBSTER_DIR'] || '.')
 lobster_env = ENV['LOBSTER_ENV'] || 'development'
 
-Lobster.logger = Logger.new(STDOUT)
 config = Lobster::Configuration.new(lobster_dir, lobster_env)
 
 #puts "LOBSTER_DIR set to #{lobster_dir}"
@@ -16,7 +15,6 @@ config = Lobster::Configuration.new(lobster_dir, lobster_env)
 log_dir = config[:log_dir]
 Dir.mkdir log_dir unless Dir.exist?(log_dir) || ARGV.first == "run"
 
-
 daemon_options = {
   :multiple => false,
   :monitor => config[:monitor],

data/bin/lobsterize

@@ -0,0 +1,80 @@
+#!/usr/bin/env ruby
+
+unless Dir.exist?('config')
+  puts 'create: config/'
+  Dir.mkdir 'config'
+end
+
+config = File.join('config', 'lobster.yml')
+schedule = File.join('config', 'schedule.rb')
+
+unless File.exist?(config)
+  puts "create: #{config}"
+  File.open(config,'w') do |f|
+    f.write <<-EOF.gsub /^\s+/,''
+      ## config/lobster.yml
+      ## configuration file for Lobster
+      ##
+      ## Default values
+      ## All paths are absolute or relative to the LOBSTER_DIR
+      ##
+      ## monitor the daemon and restart it if it stops
+      ## (requires a restart if modified)
+      #monitor: true
+      #
+      ## log directory
+      ## (requires a restart if modified)
+      #log_dir: log
+      #
+      ## pids directory
+      ## (requires a restart if modified)
+      #pid_dir: /var/run/lobster/
+      #
+      ## schedule file location
+      #schedule_file: config/schedulr.rb
+      #
+      ##
+      ## Environment values
+      ## You can use environement with the LOBSTER_ENV env variable,
+      ## the default is set to 'development'
+      ##
+      ## Example:
+      #development:
+      #  monitor: false
+      #  pid_dir: pids
+      #production:
+      #  log_dir: /var/log/lobster/
+    EOF
+  end
+end
+
+unless File.exist?(schedule)
+  puts "create: #{schedule}"
+  File.open(schedule,'w') do |f|
+    f.write <<-EOF.gsub /^\s+/,''
+      ## config/schedule.rb
+      ## schedule file for Lobster
+      ##
+      ## Example:
+      ##
+      ## Job named my_job
+      #job 'my_job' do
+      #
+      ## will just sleep for 5 seconds, this command is run from the
+      ## LOBSTER_DIR directory, so you can use relative paths from here.
+      #  command 'sleep 5'
+      #
+      ## the user my_user will be use to run the command, Lobster needs 
+      ## password-less sudo access to this user.
+      #  user 'my_user'
+      #
+      ## Lobster will wait 2 minutes between 2 runs
+      #  delay 2
+      #
+      ## end of job definition
+      #end
+    EOF
+  end
+end
+
+puts "edit your schedule and try \"lobster run\""

data/lib/lobster/configuration.rb

@@ -14,7 +14,9 @@ module Lobster
       if File.exist? config_file_path
         config_file = YAML.load_file(config_file_path)
         @config.keys.each do |key|
-          @config[key] = config_file[env][key.to_s] || config_file[key.to_s] || @config[key]
+          val = config_file[env][key.to_s] # environment-specific override
+          val = config_file[key.to_s] if val.nil?
+          @config[key] = val unless val.nil? # keep default if no config
         end
       end
 

data/lib/lobster/job.rb

@@ -11,17 +11,16 @@ module Lobster
     def reload(options)
       options[:delay] ||= 10
 
-      if options[:command] != @command
-        Lobster.logger.info "Job command updated for #{@name}, was \"#{@command}\", now \"#{options[:command]}\"" if @command
-        @command = options.delete(:command)
+      [:command, :delay, :user].each do |opt|
+        val = instance_variable_get "@#{opt}"
+        if options[opt] != val
+          Lobster.logger.info "Job #{opt} updated for #{@name}, was \"#{val}\", now \"#{options[opt]}\"" if val
+          instance_variable_set "@#{opt}", options.delete(opt)
+          # special case: reset @next_run if delay is updated
+          @next_run = nil if opt == :delay and not running?
+        end
       end
       
-      if options[:delay] != @delay
-        Lobster.logger.info "Job delay updated for #{@name}, was \"#{@delay}\", now \"#{options[:delay]}\"" if @delay
-        @delay = options.delete(:delay)
-        @next_run = nil unless running?
-      end
-
       @name ||= "<unnamed_job_#{command.hash.abs}>"
       @next_run ||= Time.now + rand(@delay*60)
     end
@@ -40,8 +39,10 @@ module Lobster
 
     def run(out,err,dir)
       Lobster.logger.info "Starting job #{@name}"
+      command_line = @user ? "sudo -nu #{@user} sh -c \"#{@command}\"" : @command
+
       begin
-        @pid = spawn(@command, :out=>out, :err=>err, :chdir=>dir)
+        @pid = spawn(command_line, :out=>out, :err=>err, :chdir=>dir)
       rescue Exception => e
         Lobster.logger.error "#{e}: error when starting job #{@name}"
         @next_run = Time.now + 10

data/lib/lobster/job_list.rb

@@ -4,14 +4,12 @@ module Lobster
 
     def initialize(file)
       @file = file
-      #@options = {}
       @current_options = nil
       @jobs = {}
     end
 
     def reload
       @new_jobs = {}
-      #@new_options = {}
 
       instance_eval(File.read(@file),@file)
 
@@ -19,12 +17,8 @@ module Lobster
       @jobs.each do |name, job|
         Lobster.logger.info "Job #{name} deleted." unless @new_jobs[name]
       end
-      #@options.each do |key, value|
-      #  Lobster.logger.info "#{key} unset." unless @new_options[key]
-      #end
 
       @jobs = @new_jobs
-      #@options = @new_options
     end
 
     def job(name)
@@ -35,21 +29,15 @@ module Lobster
       @current_options = nil
     end
 
-    def cmd(command)
-      @current_options[:command] = command
+    [:command, :delay, :user].each do |opt|
+      define_method opt do |value|
+        @current_options[opt] = value
+      end
     end
 
-    def delay(delay)
-      @current_options[:delay] = delay
+    # backward compatibility
+    def cmd(command)
+      @current_options[:command] = command
     end
-
-#    def set(option, value)
-#      Lobster.logger.info "set #{option}=#{value}" if value != @options[option]
-#      @new_options[option] = value
-#    end
-#
-#    def get(option)
-#      @options[option]
-#    end
   end
 end

data/lib/lobster/service.rb

@@ -10,6 +10,7 @@ module Lobster
       @job_list = nil
       @running = true
       @sleeping = false
+      @main_thread = Thread.current
       @config = config
       @poll_delay = 60
 
@@ -28,6 +29,16 @@ module Lobster
         end
       end
 
+      trap 'HUP' do
+        begin
+          @config = Configuration.new(@config[:lobster_dir], @config[:environment])
+        rescue Exception => e
+          Lobster.logger.error "Cannot reload conf, Exception: #{e}"
+          break
+        end
+        Lobster.logger.info "Lobster config reloaded: #{@config}"
+      end
+
       at_exit do
         @running = false
         sleep 0.01 until @sleeping # make sure no new jobs are created
@@ -36,6 +47,8 @@ module Lobster
         @job_list.jobs.each_value do |job|
           job.kill 'INT'
         end if @job_list
+
+        @main_thread.wakeup # stop sleeping and exit properly
       end
     end
 

data/lib/lobster/version.rb

@@ -1,3 +1,3 @@
 module Lobster
-  VERSION = '0.1.0'
+  VERSION = '0.1.1'
 end

data/lib/lobster.rb

@@ -1,10 +1,12 @@
+require 'logger'
+
 module Lobster
+  def self.logger
+    @@logger ||= Logger.new(STDOUT)
+  end
+
   autoload :Configuration, 'lobster/configuration'
   autoload :JobList,  'lobster/job_list'
   autoload :Job,      'lobster/job'
   autoload :Service,  'lobster/service'
-
-  class << self
-    attr_accessor :logger
-  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lobster
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-12-21 00:00:00.000000000 Z
+date: 2012-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
-  requirement: &80998950 !ruby/object:Gem::Requirement
+  requirement: &84449170 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,12 +21,13 @@ dependencies:
         version: 1.1.4
   type: :runtime
   prerelease: false
-  version_requirements: *80998950
+  version_requirements: *84449170
 description: ''
 email:
 - m.brugidou@criteo.com
 executables:
 - lobster
+- lobsterize
 extensions: []
 extra_rdoc_files: []
 files:
@@ -36,6 +37,7 @@ files:
 - LICENSE
 - README.md
 - bin/lobster
+- bin/lobsterize
 - lib/lobster.rb
 - lib/lobster/configuration.rb
 - lib/lobster/job.rb