lobster 0.1.0.alpha2 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +1 -0
- data/Gemfile.lock +1 -1
- data/LICENSE +13 -0
- data/README.md +42 -0
- data/bin/lobster +17 -10
- data/lib/lobster/configuration.rb +39 -0
- data/lib/lobster/job.rb +31 -1
- data/lib/lobster/service.rb +23 -18
- data/lib/lobster/version.rb +1 -1
- data/lib/lobster.rb +1 -0
- data/lobster.gemspec +1 -1
- metadata +12 -8
data/.gitignore
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
*.gem
|
data/Gemfile.lock
CHANGED
data/LICENSE
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
Copyright 2011 Criteo
|
2
|
+
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
you may not use this file except in compliance with the License.
|
5
|
+
You may obtain a copy of the License at
|
6
|
+
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
See the License for the specific language governing permissions and
|
13
|
+
limitations under the License.
|
data/README.md
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
Lobster
|
2
|
+
=======
|
3
|
+
|
4
|
+
Simple job service that was originally written to run hadoop jobs, but can be
|
5
|
+
used for any batch-processing scripts that you want to run.
|
6
|
+
|
7
|
+
Alpha Version Warning
|
8
|
+
---------------------
|
9
|
+
|
10
|
+
Lobster has not been fully released and tested yet.
|
11
|
+
|
12
|
+
How It Works
|
13
|
+
------------
|
14
|
+
|
15
|
+
### Create a `config/schedule.rb` file
|
16
|
+
|
17
|
+
~~~~~ ruby
|
18
|
+
job "my-job" do
|
19
|
+
cmd "runthis && runthat >> log/here.log"
|
20
|
+
delay 5 # minutes
|
21
|
+
end
|
22
|
+
~~~~~
|
23
|
+
|
24
|
+
### Run Lobster
|
25
|
+
|
26
|
+
- In the console with `LOBSTER_DIR=/path/to/jobs/directory lobster run`
|
27
|
+
- As a service with `LOBSTER_DIR=/path/to/jobs/directory sudo lobster start`
|
28
|
+
|
29
|
+
The job `my-job` will start in the next 5 minutes, and once it's done,
|
30
|
+
lobster will wait 5 minutes before starting it again.
|
31
|
+
|
32
|
+
Any log in stderr/stdout will be written in `$LOBSTER_DIR/log/lobster.output`
|
33
|
+
along with some other useful information for monitoring.
|
34
|
+
|
35
|
+
The current directory where the job command is run is `$LOBSTER_DIR`.
|
36
|
+
|
37
|
+
How To Install
|
38
|
+
--------------
|
39
|
+
|
40
|
+
gem install lobster --pre
|
41
|
+
|
42
|
+
|
data/bin/lobster
CHANGED
@@ -1,25 +1,32 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
3
|
require 'lobster'
|
4
|
+
require 'logger'
|
4
5
|
require 'daemons'
|
5
6
|
|
6
7
|
lobster_dir = File.expand_path(ENV['LOBSTER_DIR'] || '.')
|
7
|
-
|
8
|
+
lobster_env = ENV['LOBSTER_ENV'] || 'development'
|
8
9
|
|
9
|
-
|
10
|
+
Lobster.logger = Logger.new(STDOUT)
|
11
|
+
config = Lobster::Configuration.new(lobster_dir, lobster_env)
|
10
12
|
|
11
13
|
#puts "LOBSTER_DIR set to #{lobster_dir}"
|
14
|
+
#puts "Config: #{config}"
|
15
|
+
|
16
|
+
log_dir = config[:log_dir]
|
17
|
+
Dir.mkdir log_dir unless Dir.exist?(log_dir) || ARGV.first == "run"
|
18
|
+
|
12
19
|
|
13
|
-
|
20
|
+
daemon_options = {
|
14
21
|
:multiple => false,
|
15
|
-
:monitor =>
|
22
|
+
:monitor => config[:monitor],
|
16
23
|
:backtrace => true,
|
17
|
-
:log_dir =>
|
18
|
-
:log_output => true
|
24
|
+
:log_dir => config[:log_dir],
|
25
|
+
:log_output => true,
|
26
|
+
:dir_mode => :normal,
|
27
|
+
:dir => config[:pid_dir]
|
19
28
|
}
|
20
|
-
options[:dir_mode] = :system unless ARGV.first == "run"
|
21
|
-
options[:log_dir] = File.join(lobster_dir, 'log')
|
22
29
|
|
23
|
-
Daemons.run_proc('lobster',
|
24
|
-
Lobster::Service.start(
|
30
|
+
Daemons.run_proc('lobster', daemon_options) do
|
31
|
+
Lobster::Service.start(config)
|
25
32
|
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
|
3
|
+
module Lobster
|
4
|
+
class Configuration
|
5
|
+
def initialize(dir, env)
|
6
|
+
@config = {
|
7
|
+
:monitor => true,
|
8
|
+
:log_dir => 'log',
|
9
|
+
:pid_dir => File.join('/', 'var', 'run', 'lobster'),
|
10
|
+
:schedule_file => File.join('config', 'schedule.rb')
|
11
|
+
}
|
12
|
+
|
13
|
+
config_file_path = File.join(dir, 'config', 'lobster.yml')
|
14
|
+
if File.exist? config_file_path
|
15
|
+
config_file = YAML.load_file(config_file_path)
|
16
|
+
@config.keys.each do |key|
|
17
|
+
@config[key] = config_file[env][key.to_s] || config_file[key.to_s] || @config[key]
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
# these keys cannot be set by the config file
|
22
|
+
@config[:lobster_dir] = dir
|
23
|
+
@config[:environment] = env
|
24
|
+
|
25
|
+
# make paths absolute
|
26
|
+
[:log_dir, :pid_dir, :schedule_file].each do |k|
|
27
|
+
@config[k] = File.absolute_path(@config[k], dir)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def [](key)
|
32
|
+
@config[key]
|
33
|
+
end
|
34
|
+
|
35
|
+
def to_s
|
36
|
+
@config.to_s
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
data/lib/lobster/job.rb
CHANGED
@@ -39,7 +39,7 @@ module Lobster
|
|
39
39
|
end
|
40
40
|
|
41
41
|
def run(out,err,dir)
|
42
|
-
Lobster.logger.info "Starting job #{@name}
|
42
|
+
Lobster.logger.info "Starting job #{@name}"
|
43
43
|
begin
|
44
44
|
@pid = spawn(@command, :out=>out, :err=>err, :chdir=>dir)
|
45
45
|
rescue Exception => e
|
@@ -47,5 +47,35 @@ module Lobster
|
|
47
47
|
@next_run = Time.now + 10
|
48
48
|
end
|
49
49
|
end
|
50
|
+
|
51
|
+
def kill(sig)
|
52
|
+
if @pid
|
53
|
+
Lobster.logger.info "Killing job #{@name} with pid #{@pid} and all its children"
|
54
|
+
kill_tree sig, @pid
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
private
|
59
|
+
|
60
|
+
def kill_tree(sig, pid)
|
61
|
+
child_parent_processes = `ps -eo pid,ppid | grep #{pid}`
|
62
|
+
child_parent_processes = child_parent_processes.split("\n").map do |child_and_parent|
|
63
|
+
child_and_parent.strip.split(/\s+/).map(&:to_i)
|
64
|
+
end
|
65
|
+
child_parent_processes.each do |child, parent|
|
66
|
+
if parent == pid
|
67
|
+
kill_tree(sig, child)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
Lobster.logger.info "Killing pid #{pid}"
|
72
|
+
begin
|
73
|
+
Process.kill sig, pid
|
74
|
+
rescue Errno::ESRCH
|
75
|
+
# Process already got killed somehow
|
76
|
+
rescue Exception => e
|
77
|
+
Lobster.logger.warn "Process #{pid} exception: #{e}"
|
78
|
+
end
|
79
|
+
end
|
50
80
|
end
|
51
81
|
end
|
data/lib/lobster/service.rb
CHANGED
@@ -1,17 +1,16 @@
|
|
1
|
-
require 'logger'
|
2
1
|
require 'lobster/version'
|
3
2
|
|
4
3
|
module Lobster
|
5
4
|
class Service
|
6
|
-
def self.start(
|
7
|
-
|
8
|
-
new(dir).run
|
5
|
+
def self.start(config)
|
6
|
+
new(config).run
|
9
7
|
end
|
10
8
|
|
11
|
-
def initialize(
|
9
|
+
def initialize(config)
|
12
10
|
@job_list = nil
|
13
|
-
@
|
14
|
-
@
|
11
|
+
@running = true
|
12
|
+
@sleeping = false
|
13
|
+
@config = config
|
15
14
|
@poll_delay = 60
|
16
15
|
|
17
16
|
rout, @wout = IO.pipe
|
@@ -29,37 +28,43 @@ module Lobster
|
|
29
28
|
end
|
30
29
|
end
|
31
30
|
|
32
|
-
|
33
|
-
|
34
|
-
|
31
|
+
at_exit do
|
32
|
+
@running = false
|
33
|
+
sleep 0.01 until @sleeping # make sure no new jobs are created
|
34
|
+
|
35
|
+
Lobster.logger.info "Exiting, all jobs are getting killed."
|
36
|
+
@job_list.jobs.each_value do |job|
|
37
|
+
job.kill 'INT'
|
38
|
+
end if @job_list
|
35
39
|
end
|
36
40
|
end
|
37
41
|
|
38
42
|
def run
|
39
43
|
Lobster.logger.info "Lobster started version #{Lobster::VERSION}"
|
40
|
-
Lobster.logger.info "
|
41
|
-
Lobster.logger.info "Poll delay: #{@poll_delay}"
|
44
|
+
Lobster.logger.info "Lobster config: #{@config}"
|
42
45
|
|
43
|
-
|
46
|
+
while @running
|
47
|
+
@sleeping = false
|
44
48
|
now = Time.now
|
45
49
|
|
46
|
-
|
50
|
+
reload_schedule
|
47
51
|
|
48
52
|
@job_list.jobs.each_value do |job|
|
49
53
|
if not job.running? and now >= job.next_run
|
50
|
-
job.run(@wout, @werr, @
|
54
|
+
job.run(@wout, @werr, @config[:lobster_dir])
|
51
55
|
end
|
52
56
|
end
|
57
|
+
@sleeping = true
|
53
58
|
sleep @poll_delay
|
54
59
|
end
|
55
60
|
end
|
56
61
|
|
57
|
-
def
|
58
|
-
@job_list ||= JobList.new(@
|
62
|
+
def reload_schedule
|
63
|
+
@job_list ||= JobList.new(@config[:schedule_file])
|
59
64
|
begin
|
60
65
|
@job_list.reload
|
61
66
|
rescue Exception => e
|
62
|
-
Lobster.logger.error "#{e}: error while reading config file in #{@
|
67
|
+
Lobster.logger.error "#{e}: error while reading config file in #{@config[:schedule_file]}, not updating"
|
63
68
|
end
|
64
69
|
end
|
65
70
|
end
|
data/lib/lobster/version.rb
CHANGED
data/lib/lobster.rb
CHANGED
data/lobster.gemspec
CHANGED
@@ -8,7 +8,7 @@ Gem::Specification.new do |s|
|
|
8
8
|
s.platform = Gem::Platform::RUBY
|
9
9
|
s.authors = ["Maxime Brugidou"]
|
10
10
|
s.email = ["m.brugidou@criteo.com"]
|
11
|
-
s.homepage = ""
|
11
|
+
s.homepage = "https://github.com/brugidou/lobster"
|
12
12
|
s.summary = %q{Simple loop job runner service.}
|
13
13
|
s.description = %q{}
|
14
14
|
s.files = `git ls-files`.split("\n")
|
metadata
CHANGED
@@ -1,19 +1,19 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: lobster
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.0
|
5
|
-
prerelease:
|
4
|
+
version: 0.1.0
|
5
|
+
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- Maxime Brugidou
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2011-12-
|
12
|
+
date: 2011-12-21 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: daemons
|
16
|
-
requirement: &
|
16
|
+
requirement: &80998950 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ! '>='
|
@@ -21,7 +21,7 @@ dependencies:
|
|
21
21
|
version: 1.1.4
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *80998950
|
25
25
|
description: ''
|
26
26
|
email:
|
27
27
|
- m.brugidou@criteo.com
|
@@ -30,16 +30,20 @@ executables:
|
|
30
30
|
extensions: []
|
31
31
|
extra_rdoc_files: []
|
32
32
|
files:
|
33
|
+
- .gitignore
|
33
34
|
- Gemfile
|
34
35
|
- Gemfile.lock
|
36
|
+
- LICENSE
|
37
|
+
- README.md
|
35
38
|
- bin/lobster
|
36
39
|
- lib/lobster.rb
|
40
|
+
- lib/lobster/configuration.rb
|
37
41
|
- lib/lobster/job.rb
|
38
42
|
- lib/lobster/job_list.rb
|
39
43
|
- lib/lobster/service.rb
|
40
44
|
- lib/lobster/version.rb
|
41
45
|
- lobster.gemspec
|
42
|
-
homepage:
|
46
|
+
homepage: https://github.com/brugidou/lobster
|
43
47
|
licenses: []
|
44
48
|
post_install_message:
|
45
49
|
rdoc_options: []
|
@@ -54,9 +58,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
54
58
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
55
59
|
none: false
|
56
60
|
requirements:
|
57
|
-
- - ! '
|
61
|
+
- - ! '>='
|
58
62
|
- !ruby/object:Gem::Version
|
59
|
-
version:
|
63
|
+
version: '0'
|
60
64
|
requirements: []
|
61
65
|
rubyforge_project:
|
62
66
|
rubygems_version: 1.8.11
|