lobster 0.1.0 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile.lock +1 -1
- data/README.md +62 -15
- data/bin/lobster +1 -3
- data/bin/lobsterize +80 -0
- data/lib/lobster/configuration.rb +3 -1
- data/lib/lobster/job.rb +11 -10
- data/lib/lobster/job_list.rb +7 -19
- data/lib/lobster/service.rb +13 -0
- data/lib/lobster/version.rb +1 -1
- data/lib/lobster.rb +6 -4
- metadata +6 -4
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -4,17 +4,29 @@ Lobster
|
|
4
4
|
Simple job service that was originally written to run hadoop jobs, but can be
|
5
5
|
used for any batch-processing scripts that you want to run.
|
6
6
|
|
7
|
-
|
8
|
-
|
7
|
+
Lobster runs as a service (daemon) and regularly checks a schedule in
|
8
|
+
`$LOBSTER_DIR/config/schedule.rb`. The goal is to run the scheduled jobs in
|
9
|
+
parallel but a job can't run twice at the same time (no overlap). The only
|
10
|
+
configuration needed is the delay between 2 job runs.
|
9
11
|
|
10
|
-
|
11
|
-
|
12
|
-
How It Works
|
12
|
+
Get Started
|
13
13
|
------------
|
14
14
|
|
15
|
+
### Install
|
16
|
+
|
17
|
+
gem install lobster
|
18
|
+
|
15
19
|
### Create a `config/schedule.rb` file
|
16
20
|
|
21
|
+
Run `lobsterize` in the directory where you need to setup and run your jobs
|
22
|
+
(defined by the LOBSTER_DIR variable)
|
23
|
+
|
24
|
+
lobsterize
|
25
|
+
|
26
|
+
Then add a job in `config/schedule.rb`
|
27
|
+
|
17
28
|
~~~~~ ruby
|
29
|
+
# config/schedule.rb
|
18
30
|
job "my-job" do
|
19
31
|
cmd "runthis && runthat >> log/here.log"
|
20
32
|
delay 5 # minutes
|
@@ -23,20 +35,55 @@ end
|
|
23
35
|
|
24
36
|
### Run Lobster
|
25
37
|
|
26
|
-
|
27
|
-
|
38
|
+
Two environment variables are used (with defaults):
|
39
|
+
|
40
|
+
- `LOBSTER_DIR` is the directory used for all job commands, and where the
|
41
|
+
configuration/schedule files are (default: current
|
42
|
+
directory)
|
43
|
+
- `LOBSTER_ENV` is a variable used to handle different environments (default:
|
44
|
+
development)
|
45
|
+
|
46
|
+
You need write permission to `/var/run/lobster/` for pids, this path is
|
47
|
+
configurable.
|
48
|
+
|
49
|
+
- `lobster run` will run in the console
|
50
|
+
- or `lobster start` as a deamon
|
51
|
+
|
52
|
+
### Configuration
|
53
|
+
|
54
|
+
The lobster configuration file should be located in
|
55
|
+
`$LOBSTER_DIR/config/lobster.yml` and has the following layout
|
28
56
|
|
29
|
-
|
30
|
-
|
57
|
+
~~~~ yaml
|
58
|
+
# default values
|
31
59
|
|
32
|
-
|
33
|
-
|
60
|
+
# monitor the daemon and restart it if it stops
|
61
|
+
monitor: true
|
62
|
+
# log directory, absolute or relative to the LOBSTER_DIR
|
63
|
+
log_dir: log
|
64
|
+
# pids directory, absolute or relative to the LOBSTER_DIR
|
65
|
+
pid_dir: /var/run/lobster/
|
66
|
+
# schedule file, absolute or relative to the LOBSTER_DIR
|
67
|
+
schedule_file: config/schedule.rb
|
34
68
|
|
35
|
-
|
69
|
+
# environment overrides, the env variable LOBSTER_ENV has
|
70
|
+
# to be set (default: development)
|
71
|
+
my_test_env:
|
72
|
+
monitor: false
|
73
|
+
log_dir: test_los
|
74
|
+
my_prod_env:
|
75
|
+
pid_dir: /var/run/
|
76
|
+
~~~~
|
36
77
|
|
37
|
-
|
38
|
-
|
78
|
+
Any log in stderr/stdout will be written in the log directory as
|
79
|
+
`lobster.output`, the actual lobster log is in `lobster.log`
|
39
80
|
|
40
|
-
|
81
|
+
Capistrano Deployment
|
82
|
+
=====================
|
41
83
|
|
84
|
+
It is very easy to have a lobster project with a proper `config/lobster.yml` to
|
85
|
+
handle different environments such as development, testing and production. You
|
86
|
+
can simply have a `schedule.rb`file per environment.
|
42
87
|
|
88
|
+
Then you can use capistrano to deploy the new jobs and schedule to your servers
|
89
|
+
(the schedule is automatically reloaded).
|
data/bin/lobster
CHANGED
@@ -1,13 +1,12 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
|
+
require 'optparse'
|
3
4
|
require 'lobster'
|
4
|
-
require 'logger'
|
5
5
|
require 'daemons'
|
6
6
|
|
7
7
|
lobster_dir = File.expand_path(ENV['LOBSTER_DIR'] || '.')
|
8
8
|
lobster_env = ENV['LOBSTER_ENV'] || 'development'
|
9
9
|
|
10
|
-
Lobster.logger = Logger.new(STDOUT)
|
11
10
|
config = Lobster::Configuration.new(lobster_dir, lobster_env)
|
12
11
|
|
13
12
|
#puts "LOBSTER_DIR set to #{lobster_dir}"
|
@@ -16,7 +15,6 @@ config = Lobster::Configuration.new(lobster_dir, lobster_env)
|
|
16
15
|
log_dir = config[:log_dir]
|
17
16
|
Dir.mkdir log_dir unless Dir.exist?(log_dir) || ARGV.first == "run"
|
18
17
|
|
19
|
-
|
20
18
|
daemon_options = {
|
21
19
|
:multiple => false,
|
22
20
|
:monitor => config[:monitor],
|
data/bin/lobsterize
ADDED
@@ -0,0 +1,80 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
unless Dir.exist?('config')
|
4
|
+
puts 'create: config/'
|
5
|
+
Dir.mkdir 'config'
|
6
|
+
end
|
7
|
+
|
8
|
+
config = File.join('config', 'lobster.yml')
|
9
|
+
schedule = File.join('config', 'schedule.rb')
|
10
|
+
|
11
|
+
unless File.exist?(config)
|
12
|
+
puts "create: #{config}"
|
13
|
+
File.open(config,'w') do |f|
|
14
|
+
f.write <<-EOF.gsub /^\s+/,''
|
15
|
+
## config/lobster.yml
|
16
|
+
## configuration file for Lobster
|
17
|
+
##
|
18
|
+
## Default values
|
19
|
+
## All paths are absolute or relative to the LOBSTER_DIR
|
20
|
+
##
|
21
|
+
## monitor the daemon and restart it if it stops
|
22
|
+
## (requires a restart if modified)
|
23
|
+
#monitor: true
|
24
|
+
#
|
25
|
+
## log directory
|
26
|
+
## (requires a restart if modified)
|
27
|
+
#log_dir: log
|
28
|
+
#
|
29
|
+
## pids directory
|
30
|
+
## (requires a restart if modified)
|
31
|
+
#pid_dir: /var/run/lobster/
|
32
|
+
#
|
33
|
+
## schedule file location
|
34
|
+
#schedule_file: config/schedulr.rb
|
35
|
+
#
|
36
|
+
##
|
37
|
+
## Environment values
|
38
|
+
## You can use environement with the LOBSTER_ENV env variable,
|
39
|
+
## the default is set to 'development'
|
40
|
+
##
|
41
|
+
## Example:
|
42
|
+
#development:
|
43
|
+
# monitor: false
|
44
|
+
# pid_dir: pids
|
45
|
+
#production:
|
46
|
+
# log_dir: /var/log/lobster/
|
47
|
+
EOF
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
unless File.exist?(schedule)
|
52
|
+
puts "create: #{schedule}"
|
53
|
+
File.open(schedule,'w') do |f|
|
54
|
+
f.write <<-EOF.gsub /^\s+/,''
|
55
|
+
## config/schedule.rb
|
56
|
+
## schedule file for Lobster
|
57
|
+
##
|
58
|
+
## Example:
|
59
|
+
##
|
60
|
+
## Job named my_job
|
61
|
+
#job 'my_job' do
|
62
|
+
#
|
63
|
+
## will just sleep for 5 seconds, this command is run from the
|
64
|
+
## LOBSTER_DIR directory, so you can use relative paths from here.
|
65
|
+
# command 'sleep 5'
|
66
|
+
#
|
67
|
+
## the user my_user will be use to run the command, Lobster needs
|
68
|
+
## password-less sudo access to this user.
|
69
|
+
# user 'my_user'
|
70
|
+
#
|
71
|
+
## Lobster will wait 2 minutes between 2 runs
|
72
|
+
# delay 2
|
73
|
+
#
|
74
|
+
## end of job definition
|
75
|
+
#end
|
76
|
+
EOF
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
puts "edit your schedule and try \"lobster run\""
|
@@ -14,7 +14,9 @@ module Lobster
|
|
14
14
|
if File.exist? config_file_path
|
15
15
|
config_file = YAML.load_file(config_file_path)
|
16
16
|
@config.keys.each do |key|
|
17
|
-
|
17
|
+
val = config_file[env][key.to_s] # environment-specific override
|
18
|
+
val = config_file[key.to_s] if val.nil?
|
19
|
+
@config[key] = val unless val.nil? # keep default if no config
|
18
20
|
end
|
19
21
|
end
|
20
22
|
|
data/lib/lobster/job.rb
CHANGED
@@ -11,17 +11,16 @@ module Lobster
|
|
11
11
|
def reload(options)
|
12
12
|
options[:delay] ||= 10
|
13
13
|
|
14
|
-
|
15
|
-
|
16
|
-
|
14
|
+
[:command, :delay, :user].each do |opt|
|
15
|
+
val = instance_variable_get "@#{opt}"
|
16
|
+
if options[opt] != val
|
17
|
+
Lobster.logger.info "Job #{opt} updated for #{@name}, was \"#{val}\", now \"#{options[opt]}\"" if val
|
18
|
+
instance_variable_set "@#{opt}", options.delete(opt)
|
19
|
+
# special case: reset @next_run if delay is updated
|
20
|
+
@next_run = nil if opt == :delay and not running?
|
21
|
+
end
|
17
22
|
end
|
18
23
|
|
19
|
-
if options[:delay] != @delay
|
20
|
-
Lobster.logger.info "Job delay updated for #{@name}, was \"#{@delay}\", now \"#{options[:delay]}\"" if @delay
|
21
|
-
@delay = options.delete(:delay)
|
22
|
-
@next_run = nil unless running?
|
23
|
-
end
|
24
|
-
|
25
24
|
@name ||= "<unnamed_job_#{command.hash.abs}>"
|
26
25
|
@next_run ||= Time.now + rand(@delay*60)
|
27
26
|
end
|
@@ -40,8 +39,10 @@ module Lobster
|
|
40
39
|
|
41
40
|
def run(out,err,dir)
|
42
41
|
Lobster.logger.info "Starting job #{@name}"
|
42
|
+
command_line = @user ? "sudo -nu #{@user} sh -c \"#{@command}\"" : @command
|
43
|
+
|
43
44
|
begin
|
44
|
-
@pid = spawn(
|
45
|
+
@pid = spawn(command_line, :out=>out, :err=>err, :chdir=>dir)
|
45
46
|
rescue Exception => e
|
46
47
|
Lobster.logger.error "#{e}: error when starting job #{@name}"
|
47
48
|
@next_run = Time.now + 10
|
data/lib/lobster/job_list.rb
CHANGED
@@ -4,14 +4,12 @@ module Lobster
|
|
4
4
|
|
5
5
|
def initialize(file)
|
6
6
|
@file = file
|
7
|
-
#@options = {}
|
8
7
|
@current_options = nil
|
9
8
|
@jobs = {}
|
10
9
|
end
|
11
10
|
|
12
11
|
def reload
|
13
12
|
@new_jobs = {}
|
14
|
-
#@new_options = {}
|
15
13
|
|
16
14
|
instance_eval(File.read(@file),@file)
|
17
15
|
|
@@ -19,12 +17,8 @@ module Lobster
|
|
19
17
|
@jobs.each do |name, job|
|
20
18
|
Lobster.logger.info "Job #{name} deleted." unless @new_jobs[name]
|
21
19
|
end
|
22
|
-
#@options.each do |key, value|
|
23
|
-
# Lobster.logger.info "#{key} unset." unless @new_options[key]
|
24
|
-
#end
|
25
20
|
|
26
21
|
@jobs = @new_jobs
|
27
|
-
#@options = @new_options
|
28
22
|
end
|
29
23
|
|
30
24
|
def job(name)
|
@@ -35,21 +29,15 @@ module Lobster
|
|
35
29
|
@current_options = nil
|
36
30
|
end
|
37
31
|
|
38
|
-
|
39
|
-
|
32
|
+
[:command, :delay, :user].each do |opt|
|
33
|
+
define_method opt do |value|
|
34
|
+
@current_options[opt] = value
|
35
|
+
end
|
40
36
|
end
|
41
37
|
|
42
|
-
|
43
|
-
|
38
|
+
# backward compatibility
|
39
|
+
def cmd(command)
|
40
|
+
@current_options[:command] = command
|
44
41
|
end
|
45
|
-
|
46
|
-
# def set(option, value)
|
47
|
-
# Lobster.logger.info "set #{option}=#{value}" if value != @options[option]
|
48
|
-
# @new_options[option] = value
|
49
|
-
# end
|
50
|
-
#
|
51
|
-
# def get(option)
|
52
|
-
# @options[option]
|
53
|
-
# end
|
54
42
|
end
|
55
43
|
end
|
data/lib/lobster/service.rb
CHANGED
@@ -10,6 +10,7 @@ module Lobster
|
|
10
10
|
@job_list = nil
|
11
11
|
@running = true
|
12
12
|
@sleeping = false
|
13
|
+
@main_thread = Thread.current
|
13
14
|
@config = config
|
14
15
|
@poll_delay = 60
|
15
16
|
|
@@ -28,6 +29,16 @@ module Lobster
|
|
28
29
|
end
|
29
30
|
end
|
30
31
|
|
32
|
+
trap 'HUP' do
|
33
|
+
begin
|
34
|
+
@config = Configuration.new(@config[:lobster_dir], @config[:environment])
|
35
|
+
rescue Exception => e
|
36
|
+
Lobster.logger.error "Cannot reload conf, Exception: #{e}"
|
37
|
+
break
|
38
|
+
end
|
39
|
+
Lobster.logger.info "Lobster config reloaded: #{@config}"
|
40
|
+
end
|
41
|
+
|
31
42
|
at_exit do
|
32
43
|
@running = false
|
33
44
|
sleep 0.01 until @sleeping # make sure no new jobs are created
|
@@ -36,6 +47,8 @@ module Lobster
|
|
36
47
|
@job_list.jobs.each_value do |job|
|
37
48
|
job.kill 'INT'
|
38
49
|
end if @job_list
|
50
|
+
|
51
|
+
@main_thread.wakeup # stop sleeping and exit properly
|
39
52
|
end
|
40
53
|
end
|
41
54
|
|
data/lib/lobster/version.rb
CHANGED
data/lib/lobster.rb
CHANGED
@@ -1,10 +1,12 @@
|
|
1
|
+
require 'logger'
|
2
|
+
|
1
3
|
module Lobster
|
4
|
+
def self.logger
|
5
|
+
@@logger ||= Logger.new(STDOUT)
|
6
|
+
end
|
7
|
+
|
2
8
|
autoload :Configuration, 'lobster/configuration'
|
3
9
|
autoload :JobList, 'lobster/job_list'
|
4
10
|
autoload :Job, 'lobster/job'
|
5
11
|
autoload :Service, 'lobster/service'
|
6
|
-
|
7
|
-
class << self
|
8
|
-
attr_accessor :logger
|
9
|
-
end
|
10
12
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: lobster
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2012-01-02 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: daemons
|
16
|
-
requirement: &
|
16
|
+
requirement: &84449170 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ! '>='
|
@@ -21,12 +21,13 @@ dependencies:
|
|
21
21
|
version: 1.1.4
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *84449170
|
25
25
|
description: ''
|
26
26
|
email:
|
27
27
|
- m.brugidou@criteo.com
|
28
28
|
executables:
|
29
29
|
- lobster
|
30
|
+
- lobsterize
|
30
31
|
extensions: []
|
31
32
|
extra_rdoc_files: []
|
32
33
|
files:
|
@@ -36,6 +37,7 @@ files:
|
|
36
37
|
- LICENSE
|
37
38
|
- README.md
|
38
39
|
- bin/lobster
|
40
|
+
- bin/lobsterize
|
39
41
|
- lib/lobster.rb
|
40
42
|
- lib/lobster/configuration.rb
|
41
43
|
- lib/lobster/job.rb
|