eye 0.8.1 → 0.9.pre
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.rubocop.yml +21 -2
- data/.travis.yml +2 -1
- data/Gemfile +3 -0
- data/LICENSE +1 -1
- data/README.md +1 -1
- data/bin/loader_eye +1 -1
- data/examples/delayed_job.eye +2 -2
- data/examples/processes/sample.rb +1 -1
- data/examples/puma.eye +1 -1
- data/examples/rbenv.eye +1 -1
- data/examples/sidekiq.eye +1 -1
- data/examples/test.eye +1 -1
- data/examples/thin-farm.eye +1 -1
- data/examples/unicorn.eye +1 -1
- data/eye.gemspec +2 -2
- data/lib/eye.rb +2 -3
- data/lib/eye/application.rb +3 -6
- data/lib/eye/checker.rb +4 -3
- data/lib/eye/checker/children_count.rb +3 -3
- data/lib/eye/checker/socket.rb +7 -12
- data/lib/eye/child_process.rb +5 -7
- data/lib/eye/cli.rb +6 -4
- data/lib/eye/cli/commands.rb +1 -1
- data/lib/eye/cli/render.rb +6 -5
- data/lib/eye/client.rb +9 -4
- data/lib/eye/controller.rb +2 -2
- data/lib/eye/controller/{send_command.rb → apply.rb} +25 -32
- data/lib/eye/controller/commands.rb +13 -10
- data/lib/eye/controller/load.rb +49 -38
- data/lib/eye/controller/status.rb +2 -2
- data/lib/eye/dsl.rb +1 -1
- data/lib/eye/dsl/application_opts.rb +2 -2
- data/lib/eye/dsl/child_process_opts.rb +1 -1
- data/lib/eye/dsl/config_opts.rb +1 -1
- data/lib/eye/dsl/group_opts.rb +3 -3
- data/lib/eye/dsl/main.rb +3 -3
- data/lib/eye/dsl/opts.rb +17 -21
- data/lib/eye/dsl/process_opts.rb +3 -3
- data/lib/eye/dsl/validation.rb +2 -2
- data/lib/eye/group.rb +8 -114
- data/lib/eye/group/call.rb +73 -0
- data/lib/eye/group/chain.rb +19 -17
- data/lib/eye/group/data.rb +40 -0
- data/lib/eye/loader.rb +1 -1
- data/lib/eye/logger.rb +4 -4
- data/lib/eye/process.rb +2 -0
- data/lib/eye/process/commands.rb +11 -19
- data/lib/eye/process/config.rb +2 -1
- data/lib/eye/process/controller.rb +5 -12
- data/lib/eye/process/data.rb +11 -3
- data/lib/eye/process/monitor.rb +5 -5
- data/lib/eye/process/notify.rb +1 -1
- data/lib/eye/process/scheduler.rb +118 -63
- data/lib/eye/process/states.rb +10 -9
- data/lib/eye/process/states_history.rb +1 -1
- data/lib/eye/process/system.rb +1 -1
- data/lib/eye/process/trigger.rb +5 -4
- data/lib/eye/process/validate.rb +1 -1
- data/lib/eye/server.rb +22 -9
- data/lib/eye/trigger.rb +4 -7
- data/lib/eye/trigger/check_dependency.rb +1 -1
- data/lib/eye/trigger/flapping.rb +8 -10
- data/lib/eye/trigger/starting_guard.rb +7 -4
- data/lib/eye/trigger/stop_children.rb +1 -1
- data/lib/eye/trigger/wait_dependency.rb +2 -2
- data/lib/eye/utils.rb +19 -9
- metadata +10 -10
- data/lib/eye/reason.rb +0 -26
- data/lib/eye/utils/celluloid_chain.rb +0 -73
data/lib/eye/process/config.rb
CHANGED
@@ -24,7 +24,7 @@ module Eye::Process::Config
|
|
24
24
|
|
25
25
|
auto_update_pidfile_grace: 30.seconds,
|
26
26
|
revert_fuckup_pidfile_grace: 120.seconds
|
27
|
-
}
|
27
|
+
}.freeze
|
28
28
|
|
29
29
|
def prepare_config(new_config)
|
30
30
|
h = DEFAULTS.merge(new_config)
|
@@ -46,6 +46,7 @@ module Eye::Process::Config
|
|
46
46
|
h[:stdall] = Eye::System.normalized_file(h[:stdall], h[:working_dir]) if h[:stdall]
|
47
47
|
|
48
48
|
h[:environment] = Eye::System.prepare_env(h)
|
49
|
+
h[:stop_signals] = [:TERM, 0.5, :KILL] unless h[:stop_command] || h[:stop_signals]
|
49
50
|
|
50
51
|
h
|
51
52
|
end
|
@@ -1,8 +1,7 @@
|
|
1
1
|
module Eye::Process::Controller
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
end
|
3
|
+
# scheduled actions
|
4
|
+
# :update_config, :start, :stop, :restart, :unmonitor, :monitor, :break_chain, :delete, :signal, :user_command
|
6
5
|
|
7
6
|
def start
|
8
7
|
if load_external_pid_file == :ok
|
@@ -26,12 +25,10 @@ module Eye::Process::Controller
|
|
26
25
|
def monitor
|
27
26
|
if self[:auto_start]
|
28
27
|
start
|
28
|
+
elsif load_external_pid_file == :ok
|
29
|
+
switch :already_running
|
29
30
|
else
|
30
|
-
|
31
|
-
switch :already_running
|
32
|
-
else
|
33
|
-
schedule :unmonitor, Eye::Reason.new(:'not found')
|
34
|
-
end
|
31
|
+
schedule command: :unmonitor, reason: 'not found'
|
35
32
|
end
|
36
33
|
end
|
37
34
|
|
@@ -62,8 +59,4 @@ module Eye::Process::Controller
|
|
62
59
|
end
|
63
60
|
end
|
64
61
|
|
65
|
-
def freeze
|
66
|
-
scheduler_freeze
|
67
|
-
end
|
68
|
-
|
69
62
|
end
|
data/lib/eye/process/data.rb
CHANGED
@@ -9,7 +9,7 @@ module Eye::Process::Data
|
|
9
9
|
end
|
10
10
|
|
11
11
|
def group_name
|
12
|
-
|
12
|
+
self[:group] == '__default__' ? nil : self[:group]
|
13
13
|
end
|
14
14
|
|
15
15
|
def group_name_pure
|
@@ -46,13 +46,21 @@ module Eye::Process::Data
|
|
46
46
|
|
47
47
|
h[:debug] = debug_data if opts[:debug]
|
48
48
|
h[:procline] = Eye::SystemResources.args(self.pid) if opts[:procline]
|
49
|
-
h[:current_command] =
|
49
|
+
h[:current_command] = scheduler_current_command if scheduler_current_command
|
50
50
|
|
51
51
|
h
|
52
52
|
end
|
53
53
|
|
54
54
|
def debug_data
|
55
|
-
{ queue: scheduler_actions_list, watchers: @watchers.keys }
|
55
|
+
{ queue: scheduler_actions_list, watchers: @watchers.keys, timers: timers_data }
|
56
|
+
end
|
57
|
+
|
58
|
+
def timers_data
|
59
|
+
if actor = Thread.current[:celluloid_actor]
|
60
|
+
actor.timers.timers.map(&:interval)
|
61
|
+
end
|
62
|
+
rescue
|
63
|
+
[]
|
56
64
|
end
|
57
65
|
|
58
66
|
def sub_object?(obj)
|
data/lib/eye/process/monitor.rb
CHANGED
@@ -39,7 +39,7 @@ private
|
|
39
39
|
notify :info, 'crashed!'
|
40
40
|
clear_pid_file(true) if control_pid?
|
41
41
|
|
42
|
-
switch :crashed,
|
42
|
+
switch :crashed, reason: :crashed
|
43
43
|
end
|
44
44
|
end
|
45
45
|
|
@@ -83,7 +83,7 @@ private
|
|
83
83
|
def check_identity
|
84
84
|
if compare_identity == :fail
|
85
85
|
notify :info, 'crashed by identity!'
|
86
|
-
switch :crashed,
|
86
|
+
switch :crashed, reason: :crashed_by_identity
|
87
87
|
clear_pid_file if self[:clear_pid]
|
88
88
|
false
|
89
89
|
else
|
@@ -101,13 +101,13 @@ private
|
|
101
101
|
warn 'check crashed: process is down'
|
102
102
|
|
103
103
|
if self[:restore_in]
|
104
|
-
|
104
|
+
schedule in: self[:restore_in].to_f, command: :restore, reason: :crashed
|
105
105
|
else
|
106
|
-
schedule :restore,
|
106
|
+
schedule command: :restore, reason: :crashed
|
107
107
|
end
|
108
108
|
else
|
109
109
|
warn 'check crashed: process without keep_alive'
|
110
|
-
schedule :unmonitor,
|
110
|
+
schedule command: :unmonitor, reason: :crashed
|
111
111
|
end
|
112
112
|
end
|
113
113
|
|
data/lib/eye/process/notify.rb
CHANGED
@@ -5,7 +5,7 @@ module Eye::Process::Notify
|
|
5
5
|
# 2) checker bounded to restart process [:warn]
|
6
6
|
# 3) flapping + switch to unmonitored [:error]
|
7
7
|
|
8
|
-
LEVELS = { debug: 0, info: 1, warn: 2, error: 3, fatal: 4 }
|
8
|
+
LEVELS = { debug: 0, info: 1, warn: 2, error: 3, fatal: 4 }.freeze
|
9
9
|
|
10
10
|
def notify(level, msg)
|
11
11
|
# logging it
|
@@ -1,11 +1,54 @@
|
|
1
1
|
module Eye::Process::Scheduler
|
2
2
|
|
3
|
-
#
|
4
|
-
|
5
|
-
|
3
|
+
# Call params:
|
4
|
+
# :command
|
5
|
+
# :args
|
6
|
+
# :by
|
7
|
+
# :reason
|
8
|
+
# :block
|
9
|
+
# :signal
|
10
|
+
# :at
|
11
|
+
# :in
|
12
|
+
|
13
|
+
# :update_config, :start, :stop, :restart, :unmonitor, :monitor, :break_chain, :delete, :signal, :user_command
|
14
|
+
def send_call(call)
|
15
|
+
user_schedule(call)
|
16
|
+
end
|
17
|
+
|
18
|
+
def user_schedule(call)
|
19
|
+
call[:by] ||= :user
|
20
|
+
internal_schedule(call)
|
21
|
+
end
|
22
|
+
|
23
|
+
# 2 Forms of schedule:
|
24
|
+
# schedule command: 'bla', args: [1, 2, 3]
|
25
|
+
# schedule :bla, 1, 2, 3
|
26
|
+
def schedule(*args)
|
27
|
+
if args[0].is_a?(Hash)
|
28
|
+
internal_schedule(args[0])
|
29
|
+
else
|
30
|
+
internal_schedule(command: args[0], args: args[1..-1])
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def internal_schedule(call)
|
35
|
+
if interval = call[:in]
|
36
|
+
debug { "schedule_in #{interval} :#{call[:command]}" }
|
37
|
+
after(interval.to_f) do
|
38
|
+
debug { "scheduled_in #{interval} :#{call[:command]}" }
|
39
|
+
call[:in] = nil
|
40
|
+
internal_schedule(call)
|
41
|
+
end
|
42
|
+
return
|
43
|
+
end
|
44
|
+
|
45
|
+
call[:at] ||= Time.now.to_i
|
46
|
+
command = call[:command]
|
47
|
+
|
48
|
+
@scheduler_freeze = false if call[:freeze] == false
|
6
49
|
|
7
|
-
if scheduler_freeze
|
8
|
-
warn ":#{command} ignoring to schedule, because scheduler is
|
50
|
+
if @scheduler_freeze
|
51
|
+
warn ":#{command} ignoring to schedule, because scheduler is freezed"
|
9
52
|
return
|
10
53
|
end
|
11
54
|
|
@@ -14,95 +57,107 @@ module Eye::Process::Scheduler
|
|
14
57
|
return
|
15
58
|
end
|
16
59
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
if reason.class == Eye::Reason
|
22
|
-
# for auto reasons
|
23
|
-
# skip already running commands and all in chain
|
24
|
-
scheduler.add_wo_dups_current(:scheduled_action, command, args: args, reason: reason, block: block)
|
60
|
+
if filter_call(call)
|
61
|
+
info "schedule :#{command} (#{reason_from_call(call)})"
|
62
|
+
scheduler_add(call)
|
25
63
|
else
|
26
|
-
|
27
|
-
# skip only for last in chain
|
28
|
-
scheduler.add_wo_dups(:scheduled_action, command, args: args, reason: reason, block: block)
|
64
|
+
info "not scheduled: #{command} (#{reason_from_call(call)})"
|
29
65
|
end
|
30
|
-
end
|
31
66
|
|
32
|
-
|
33
|
-
debug { "schedule_in #{interval} :#{command} #{args}" }
|
34
|
-
after(interval.to_f) do
|
35
|
-
debug { "scheduled_in #{interval} :#{command} #{args}" }
|
36
|
-
schedule(command, *args, &block)
|
37
|
-
end
|
67
|
+
@scheduler_freeze = true if call[:freeze] == true
|
38
68
|
end
|
39
69
|
|
40
|
-
def
|
41
|
-
|
42
|
-
|
70
|
+
def scheduler_consume(call)
|
71
|
+
args = call[:args]
|
72
|
+
reason_str = reason_from_call(call)
|
73
|
+
info "=> #{call[:command]} #{args.present? ? args.join(',') : nil} (#{reason_str})"
|
74
|
+
send(call[:command], *args, &call[:block])
|
43
75
|
|
44
|
-
|
45
|
-
|
46
|
-
@last_scheduled_reason = reason
|
47
|
-
@last_scheduled_at = Time.now
|
76
|
+
history_cmd = is_a?(Eye::ChildProcess) ? "#{call[:command]}_child" : call[:command]
|
77
|
+
scheduler_history.push(history_cmd, reason_str, call[:at])
|
48
78
|
|
49
|
-
|
50
|
-
|
51
|
-
|
79
|
+
rescue Object => ex
|
80
|
+
raise(ex) if ex.class == Celluloid::TaskTerminated
|
81
|
+
log_ex(ex)
|
52
82
|
|
53
|
-
|
83
|
+
ensure
|
84
|
+
# signaling to waiter
|
85
|
+
call[:signal].try :signal
|
54
86
|
|
55
|
-
|
56
|
-
parent.schedule_history.push("#{command}_child", reason, @last_scheduled_at.to_i)
|
57
|
-
end
|
87
|
+
info "<= #{call[:command]}"
|
58
88
|
end
|
59
89
|
|
60
|
-
def
|
61
|
-
|
62
|
-
|
63
|
-
|
90
|
+
def filter_call(call)
|
91
|
+
# for auto reasons, compare call with current @scheduled_call
|
92
|
+
return false if call[:by] != :user && equal_action_call?(@scheduled_call, call)
|
93
|
+
|
94
|
+
# check any equal call in queue scheduler_calls
|
95
|
+
!scheduler_calls.any? { |c| equal_action_call?(c, call) }
|
64
96
|
end
|
65
97
|
|
66
|
-
def
|
67
|
-
|
98
|
+
def equal_action_call?(call1, call2)
|
99
|
+
call1 && call2 && (call1[:command] == call2[:command]) && (call1[:args] == call2[:args]) && (call1[:block] == call2[:block])
|
68
100
|
end
|
69
101
|
|
70
|
-
def
|
71
|
-
|
102
|
+
def scheduler_calls
|
103
|
+
@scheduler_calls ||= []
|
72
104
|
end
|
73
105
|
|
74
|
-
def
|
75
|
-
|
76
|
-
base.execute_block_on_receiver :schedule
|
106
|
+
def scheduler_history
|
107
|
+
@scheduler_history ||= Eye::Process::StatesHistory.new(50)
|
77
108
|
end
|
78
109
|
|
79
|
-
|
80
|
-
|
110
|
+
def scheduler_add(call)
|
111
|
+
scheduler_calls << call
|
112
|
+
ensure_scheduler_process
|
113
|
+
end
|
81
114
|
|
82
|
-
def
|
83
|
-
@
|
115
|
+
def ensure_scheduler_process
|
116
|
+
unless @scheduler_running
|
117
|
+
@scheduler_running = true
|
118
|
+
async.scheduler_run
|
119
|
+
end
|
84
120
|
end
|
85
121
|
|
86
|
-
def
|
87
|
-
@
|
122
|
+
def scheduler_run
|
123
|
+
while @scheduled_call = scheduler_calls.shift
|
124
|
+
@scheduler_running = true
|
125
|
+
@last_scheduled_call = @scheduled_call
|
126
|
+
scheduler_consume(@scheduled_call)
|
127
|
+
end
|
128
|
+
@scheduler_running = false
|
88
129
|
end
|
89
130
|
|
90
|
-
def
|
91
|
-
|
131
|
+
def scheduler_commands_list
|
132
|
+
scheduler_calls.map { |c| c[:command] }
|
92
133
|
end
|
93
134
|
|
94
|
-
def
|
95
|
-
|
135
|
+
def scheduler_clear_pending_list
|
136
|
+
scheduler_calls.clear
|
96
137
|
end
|
97
138
|
|
98
|
-
|
139
|
+
def scheduler_last_command
|
140
|
+
@last_scheduled_call.try :[], :command
|
141
|
+
end
|
142
|
+
|
143
|
+
def scheduler_last_reason
|
144
|
+
reason_from_call(@last_scheduled_call)
|
145
|
+
end
|
99
146
|
|
100
|
-
def
|
101
|
-
@
|
147
|
+
def scheduler_current_command
|
148
|
+
@scheduled_call.try :[], :command
|
102
149
|
end
|
103
150
|
|
104
|
-
|
105
|
-
|
151
|
+
private
|
152
|
+
|
153
|
+
def reason_from_call(call)
|
154
|
+
return unless call
|
155
|
+
|
156
|
+
if msg = call[:reason]
|
157
|
+
msg.to_s
|
158
|
+
elsif call[:by]
|
159
|
+
"#{call[:command]} by #{call[:by]}"
|
160
|
+
end
|
106
161
|
end
|
107
162
|
|
108
163
|
end
|
data/lib/eye/process/states.rb
CHANGED
@@ -1,13 +1,13 @@
|
|
1
|
-
require '
|
2
|
-
require '
|
1
|
+
require 'state_machines'
|
2
|
+
require 'state_machines/version'
|
3
3
|
|
4
4
|
class Eye::Process
|
5
5
|
|
6
|
-
class StateError <
|
6
|
+
class StateError < RuntimeError; end
|
7
7
|
|
8
8
|
# do transition
|
9
|
-
def switch(name,
|
10
|
-
@
|
9
|
+
def switch(name, call = {})
|
10
|
+
@state_call = @last_scheduled_call ? @last_scheduled_call.merge(call) : call
|
11
11
|
self.send("#{name}!")
|
12
12
|
end
|
13
13
|
|
@@ -71,7 +71,7 @@ class Eye::Process
|
|
71
71
|
|
72
72
|
def on_crashed
|
73
73
|
self.pid = nil
|
74
|
-
schedule :check_crash,
|
74
|
+
schedule command: :check_crash, reason: :crashed
|
75
75
|
end
|
76
76
|
|
77
77
|
def on_unmonitored
|
@@ -79,9 +79,10 @@ class Eye::Process
|
|
79
79
|
end
|
80
80
|
|
81
81
|
def log_transition(transition)
|
82
|
-
if transition.to_name != transition.from_name || @
|
83
|
-
|
84
|
-
|
82
|
+
if transition.to_name != transition.from_name || @state_call[:by] == :user
|
83
|
+
reason_str = reason_from_call(@state_call)
|
84
|
+
@states_history.push transition.to_name, reason_str
|
85
|
+
info "switch :#{transition.event} [:#{transition.from_name} => :#{transition.to_name}] #{reason_str}"
|
85
86
|
end
|
86
87
|
end
|
87
88
|
|
@@ -13,7 +13,7 @@ class Eye::Process::StatesHistory < Eye::Utils::Tail
|
|
13
13
|
tm = [tm, from_time].max if from_time
|
14
14
|
tm = tm.to_f
|
15
15
|
if block
|
16
|
-
self.each { |s|
|
16
|
+
self.each { |s| yield(s) if s[:at] >= tm }
|
17
17
|
else
|
18
18
|
self.select { |s| s[:at] >= tm }.map { |c| c[:state] }
|
19
19
|
end
|
data/lib/eye/process/system.rb
CHANGED
@@ -48,7 +48,7 @@ module Eye::Process::System
|
|
48
48
|
if (id1 - st1).abs > self[:check_identity_grace]
|
49
49
|
args = Eye::SystemResources.args(pid)
|
50
50
|
msg = "pid_file: '#{Eye::Utils.human_time2(id)}', process: '#{Eye::Utils.human_time2(st)}' (#{args})"
|
51
|
-
res =
|
51
|
+
res = id1 < st1 ? :fail : :touched
|
52
52
|
warn "compare_identity: #{res}, #{msg}"
|
53
53
|
res
|
54
54
|
else
|
data/lib/eye/process/trigger.rb
CHANGED
@@ -9,7 +9,7 @@ module Eye::Process::Trigger
|
|
9
9
|
end
|
10
10
|
|
11
11
|
def check_triggers(transition)
|
12
|
-
self.triggers.each { |trigger| trigger.notify(transition,
|
12
|
+
self.triggers.each { |trigger| trigger.notify(transition, @state_call) }
|
13
13
|
end
|
14
14
|
|
15
15
|
# conditional start, used in triggers, to start only from unmonitored state, and only if special reason
|
@@ -19,9 +19,10 @@ module Eye::Process::Trigger
|
|
19
19
|
return
|
20
20
|
end
|
21
21
|
|
22
|
-
|
23
|
-
|
24
|
-
|
22
|
+
state_by = @state_call.try(:[], :by)
|
23
|
+
current_by = @scheduled_call.try(:[], :by)
|
24
|
+
if state_by && current_by && state_by != current_by
|
25
|
+
warn "skip, state_by(#{state_by}) != current_by(#{current_by})"
|
25
26
|
return
|
26
27
|
end
|
27
28
|
|