ey_stonith 0.1.4 → 0.1.5.pre
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/ey-monitor +2 -5
- data/bin/{ey-monitor-reset → stonith} +1 -1
- data/bin/{ey-monitor-stop → stonith-check} +1 -1
- data/bin/stonith-claim +5 -0
- data/bin/{ey-monitor-resume → stonith-cron} +1 -1
- data/bin/stonith-notify +5 -0
- data/bin/stonith-reset +5 -0
- data/bin/stonith-resume +5 -0
- data/bin/stonith-status +5 -0
- data/bin/{ey-monitor-status → stonith-stop} +1 -1
- data/bin/stonith-takeover +5 -0
- data/lib/ey_stonith/address_stealer.rb +1 -6
- data/lib/ey_stonith/awsm_notifier.rb +28 -29
- data/lib/ey_stonith/check_recorder.rb +19 -17
- data/lib/ey_stonith/commands/abstract.rb +94 -0
- data/lib/ey_stonith/commands/check.rb +58 -0
- data/lib/ey_stonith/commands/claim.rb +113 -0
- data/lib/ey_stonith/commands/commands.rb +26 -0
- data/lib/ey_stonith/commands/cron.rb +40 -0
- data/lib/ey_stonith/commands/help.rb +16 -0
- data/lib/ey_stonith/commands/not_found.rb +11 -0
- data/lib/ey_stonith/commands/notify.rb +85 -0
- data/lib/ey_stonith/commands/reset.rb +21 -0
- data/lib/ey_stonith/commands/resume.rb +19 -0
- data/lib/ey_stonith/commands/status.rb +23 -0
- data/lib/ey_stonith/commands/stop.rb +21 -0
- data/lib/ey_stonith/commands/takeover.rb +106 -0
- data/lib/ey_stonith/commands.rb +40 -0
- data/lib/ey_stonith/config.rb +107 -14
- data/lib/ey_stonith/data.rb +5 -1
- data/lib/ey_stonith/database.rb +28 -6
- data/lib/ey_stonith/history.rb +1 -1
- data/lib/ey_stonith.rb +2 -8
- metadata +57 -61
- data/lib/ey_stonith/abstract_master.rb +0 -15
- data/lib/ey_stonith/box.rb +0 -61
- data/lib/ey_stonith/cli.rb +0 -138
- data/lib/ey_stonith/local_master.rb +0 -28
- data/lib/ey_stonith/master.rb +0 -37
- data/lib/ey_stonith/meta_data.rb +0 -11
- data/lib/ey_stonith/slave.rb +0 -41
metadata
CHANGED
@@ -1,12 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ey_stonith
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
prerelease:
|
4
|
+
prerelease: true
|
5
5
|
segments:
|
6
6
|
- 0
|
7
7
|
- 1
|
8
|
-
-
|
9
|
-
|
8
|
+
- 5
|
9
|
+
- pre
|
10
|
+
version: 0.1.5.pre
|
10
11
|
platform: ruby
|
11
12
|
authors:
|
12
13
|
- Ezra Zygmuntowicz
|
@@ -16,25 +17,24 @@ autorequire:
|
|
16
17
|
bindir: bin
|
17
18
|
cert_chain: []
|
18
19
|
|
19
|
-
date: 2010-
|
20
|
+
date: 2010-04-14 00:00:00 -07:00
|
20
21
|
default_executable:
|
21
22
|
dependencies:
|
22
23
|
- !ruby/object:Gem::Dependency
|
23
|
-
|
24
|
-
|
25
|
-
requirement: &id001 !ruby/object:Gem::Requirement
|
24
|
+
type: :runtime
|
25
|
+
version_requirements: &id001 !ruby/object:Gem::Requirement
|
26
26
|
requirements:
|
27
27
|
- - ">="
|
28
28
|
- !ruby/object:Gem::Version
|
29
29
|
segments:
|
30
30
|
- 0
|
31
31
|
version: "0"
|
32
|
-
|
33
|
-
|
34
|
-
- !ruby/object:Gem::Dependency
|
35
|
-
name: fog
|
32
|
+
requirement: *id001
|
33
|
+
name: json
|
36
34
|
prerelease: false
|
37
|
-
|
35
|
+
- !ruby/object:Gem::Dependency
|
36
|
+
type: :runtime
|
37
|
+
version_requirements: &id002 !ruby/object:Gem::Requirement
|
38
38
|
requirements:
|
39
39
|
- - "="
|
40
40
|
- !ruby/object:Gem::Version
|
@@ -43,38 +43,12 @@ dependencies:
|
|
43
43
|
- 0
|
44
44
|
- 58
|
45
45
|
version: 0.0.58
|
46
|
-
|
47
|
-
|
48
|
-
- !ruby/object:Gem::Dependency
|
49
|
-
name: eventmachine
|
46
|
+
requirement: *id002
|
47
|
+
name: fog
|
50
48
|
prerelease: false
|
51
|
-
requirement: &id003 !ruby/object:Gem::Requirement
|
52
|
-
requirements:
|
53
|
-
- - ">="
|
54
|
-
- !ruby/object:Gem::Version
|
55
|
-
segments:
|
56
|
-
- 0
|
57
|
-
version: "0"
|
58
|
-
type: :runtime
|
59
|
-
version_requirements: *id003
|
60
49
|
- !ruby/object:Gem::Dependency
|
61
|
-
name: em-http-request
|
62
|
-
prerelease: false
|
63
|
-
requirement: &id004 !ruby/object:Gem::Requirement
|
64
|
-
requirements:
|
65
|
-
- - ~>
|
66
|
-
- !ruby/object:Gem::Version
|
67
|
-
segments:
|
68
|
-
- 0
|
69
|
-
- 2
|
70
|
-
- 7
|
71
|
-
version: 0.2.7
|
72
50
|
type: :runtime
|
73
|
-
version_requirements:
|
74
|
-
- !ruby/object:Gem::Dependency
|
75
|
-
name: redis
|
76
|
-
prerelease: false
|
77
|
-
requirement: &id005 !ruby/object:Gem::Requirement
|
51
|
+
version_requirements: &id003 !ruby/object:Gem::Requirement
|
78
52
|
requirements:
|
79
53
|
- - ~>
|
80
54
|
- !ruby/object:Gem::Version
|
@@ -83,42 +57,62 @@ dependencies:
|
|
83
57
|
- 1
|
84
58
|
- 2
|
85
59
|
version: 0.1.2
|
86
|
-
|
87
|
-
|
60
|
+
requirement: *id003
|
61
|
+
name: redis
|
62
|
+
prerelease: false
|
88
63
|
description: Shoot The Other Node In The Head
|
89
64
|
email: awsmdev@engineyard.com
|
90
65
|
executables:
|
91
66
|
- ey-monitor
|
92
|
-
-
|
93
|
-
-
|
94
|
-
-
|
95
|
-
-
|
67
|
+
- stonith
|
68
|
+
- stonith-check
|
69
|
+
- stonith-claim
|
70
|
+
- stonith-cron
|
71
|
+
- stonith-notify
|
72
|
+
- stonith-reset
|
73
|
+
- stonith-resume
|
74
|
+
- stonith-status
|
75
|
+
- stonith-stop
|
76
|
+
- stonith-takeover
|
96
77
|
extensions: []
|
97
78
|
|
98
79
|
extra_rdoc_files:
|
99
80
|
- README.rdoc
|
100
81
|
- LICENSE
|
101
82
|
files:
|
102
|
-
- lib/ey_stonith/abstract_master.rb
|
103
83
|
- lib/ey_stonith/address_stealer.rb
|
104
84
|
- lib/ey_stonith/awsm_notifier.rb
|
105
|
-
- lib/ey_stonith/box.rb
|
106
85
|
- lib/ey_stonith/check_recorder.rb
|
107
|
-
- lib/ey_stonith/
|
86
|
+
- lib/ey_stonith/commands/abstract.rb
|
87
|
+
- lib/ey_stonith/commands/check.rb
|
88
|
+
- lib/ey_stonith/commands/claim.rb
|
89
|
+
- lib/ey_stonith/commands/commands.rb
|
90
|
+
- lib/ey_stonith/commands/cron.rb
|
91
|
+
- lib/ey_stonith/commands/help.rb
|
92
|
+
- lib/ey_stonith/commands/not_found.rb
|
93
|
+
- lib/ey_stonith/commands/notify.rb
|
94
|
+
- lib/ey_stonith/commands/reset.rb
|
95
|
+
- lib/ey_stonith/commands/resume.rb
|
96
|
+
- lib/ey_stonith/commands/status.rb
|
97
|
+
- lib/ey_stonith/commands/stop.rb
|
98
|
+
- lib/ey_stonith/commands/takeover.rb
|
99
|
+
- lib/ey_stonith/commands.rb
|
108
100
|
- lib/ey_stonith/config.rb
|
109
101
|
- lib/ey_stonith/data.rb
|
110
102
|
- lib/ey_stonith/database.rb
|
111
103
|
- lib/ey_stonith/history.rb
|
112
|
-
- lib/ey_stonith/local_master.rb
|
113
|
-
- lib/ey_stonith/master.rb
|
114
|
-
- lib/ey_stonith/meta_data.rb
|
115
|
-
- lib/ey_stonith/slave.rb
|
116
104
|
- lib/ey_stonith.rb
|
117
105
|
- bin/ey-monitor
|
118
|
-
- bin/
|
119
|
-
- bin/
|
120
|
-
- bin/
|
121
|
-
- bin/
|
106
|
+
- bin/stonith
|
107
|
+
- bin/stonith-check
|
108
|
+
- bin/stonith-claim
|
109
|
+
- bin/stonith-cron
|
110
|
+
- bin/stonith-notify
|
111
|
+
- bin/stonith-reset
|
112
|
+
- bin/stonith-resume
|
113
|
+
- bin/stonith-status
|
114
|
+
- bin/stonith-stop
|
115
|
+
- bin/stonith-takeover
|
122
116
|
- README.rdoc
|
123
117
|
- LICENSE
|
124
118
|
has_rdoc: true
|
@@ -139,11 +133,13 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
139
133
|
version: "0"
|
140
134
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
141
135
|
requirements:
|
142
|
-
- - "
|
136
|
+
- - ">"
|
143
137
|
- !ruby/object:Gem::Version
|
144
138
|
segments:
|
145
|
-
-
|
146
|
-
|
139
|
+
- 1
|
140
|
+
- 3
|
141
|
+
- 1
|
142
|
+
version: 1.3.1
|
147
143
|
requirements: []
|
148
144
|
|
149
145
|
rubyforge_project:
|
@@ -1,15 +0,0 @@
|
|
1
|
-
require 'json'
|
2
|
-
require 'json/ext'
|
3
|
-
module EY
|
4
|
-
module Stonith
|
5
|
-
class AbstractMaster
|
6
|
-
private
|
7
|
-
|
8
|
-
# ips are elastic, do not cache!
|
9
|
-
def public_ip() meta_data('public-ipv4') end
|
10
|
-
def local_hostname() @local_hostname ||= meta_data("local-hostname") end
|
11
|
-
|
12
|
-
def meta_data(key) EY::Stonith.meta_data[key] end
|
13
|
-
end
|
14
|
-
end
|
15
|
-
end
|
data/lib/ey_stonith/box.rb
DELETED
@@ -1,61 +0,0 @@
|
|
1
|
-
module EY
|
2
|
-
module Stonith
|
3
|
-
class Box
|
4
|
-
def initialize(config, history)
|
5
|
-
EY::Stonith.logger.info "Booting Stonith"
|
6
|
-
|
7
|
-
@config, @history = config, history
|
8
|
-
database = Database.new(@config)
|
9
|
-
@master = Master.new(database, @config.master_hostname_from_dna)
|
10
|
-
|
11
|
-
trap("HUP") { stop }
|
12
|
-
trap("USR1") do
|
13
|
-
stop
|
14
|
-
EM.next_tick { start }
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
def start
|
19
|
-
if @master.local?
|
20
|
-
@local_master = LocalMaster.new(@config.notify_uri, @config.cloud_credentials)
|
21
|
-
@master.update @local_master.data
|
22
|
-
@history << :claim
|
23
|
-
else
|
24
|
-
@slave = Slave.new(self, @config.monitor_heartbeat)
|
25
|
-
@slave.monitor!(@master)
|
26
|
-
@history << :monitor
|
27
|
-
end
|
28
|
-
end
|
29
|
-
|
30
|
-
def try_takeover(master_unchanged)
|
31
|
-
@master.with_locked_data do |data|
|
32
|
-
if master_unchanged.call(data.key)
|
33
|
-
begin
|
34
|
-
Stonith.logger.info("Locked! Taking over.")
|
35
|
-
@slave = nil
|
36
|
-
@history << :takeover
|
37
|
-
@local_master = LocalMaster.new(@config.notify_uri, @config.cloud_credentials)
|
38
|
-
new_data = @local_master.takeover!(data.instance_id, data.ip)
|
39
|
-
@master.update new_data
|
40
|
-
ensure
|
41
|
-
@history << :done
|
42
|
-
end
|
43
|
-
else
|
44
|
-
Stonith.logger.info("Failed to grab lock, relenting.")
|
45
|
-
@slave.start!
|
46
|
-
@history << :relent
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
private
|
52
|
-
|
53
|
-
def stop
|
54
|
-
EM.next_tick {
|
55
|
-
@slave.stop! if @slave
|
56
|
-
@history << :stop
|
57
|
-
}
|
58
|
-
end
|
59
|
-
end
|
60
|
-
end
|
61
|
-
end
|
data/lib/ey_stonith/cli.rb
DELETED
@@ -1,138 +0,0 @@
|
|
1
|
-
require 'eventmachine'
|
2
|
-
require 'optparse'
|
3
|
-
require 'pathname'
|
4
|
-
|
5
|
-
module EY
|
6
|
-
module Stonith
|
7
|
-
class CLI
|
8
|
-
DEFAULT_FILES = {
|
9
|
-
:ey => "/etc/.ey-cloud.yml",
|
10
|
-
:redis => "/etc/redis.yml",
|
11
|
-
:dna => "/etc/chef/dna.json",
|
12
|
-
:history => "/var/run/ey-monitor.history",
|
13
|
-
:pid_path => "/var/run/ey-monitor.pid",
|
14
|
-
:log_path => "/var/log/ey-monitor.log",
|
15
|
-
}
|
16
|
-
|
17
|
-
def self.commands
|
18
|
-
@commands ||= Hash.new do |commands, command|
|
19
|
-
lambda {
|
20
|
-
puts "Command not found: #{command}"
|
21
|
-
puts @parser
|
22
|
-
exit(1)
|
23
|
-
}
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
def self.command(cmd,&block)
|
28
|
-
commands[cmd.to_sym] = block
|
29
|
-
end
|
30
|
-
|
31
|
-
command :start do
|
32
|
-
with_pid do
|
33
|
-
Stonith.meta_data = @options[:meta_data] if @options[:meta_data]
|
34
|
-
Stonith.logger = Logger.new(@config.log_path)
|
35
|
-
Stonith.logger.level = Logger::INFO
|
36
|
-
begin
|
37
|
-
EM.run { Box.new(@config, @history).start }
|
38
|
-
rescue Exception => e
|
39
|
-
Stonith.logger.info "#{e.class.to_s}: #{e.message}\n#{e.backtrace.join("\n")}"
|
40
|
-
end
|
41
|
-
end
|
42
|
-
end
|
43
|
-
|
44
|
-
command :stop do
|
45
|
-
system("kill -HUP #{get_pid}")
|
46
|
-
sleep(1) until @history.last == "stop"
|
47
|
-
puts "takeover" if @history.include?(:takeover)
|
48
|
-
end
|
49
|
-
|
50
|
-
command :resume do
|
51
|
-
system("kill -USR1 #{get_pid}")
|
52
|
-
end
|
53
|
-
|
54
|
-
command :status do
|
55
|
-
puts @history
|
56
|
-
end
|
57
|
-
|
58
|
-
command :reset do
|
59
|
-
Database.new(@config).reset
|
60
|
-
@history.reset
|
61
|
-
end
|
62
|
-
|
63
|
-
def initialize(command, argv)
|
64
|
-
parse!(argv)
|
65
|
-
@config = Config.new(@options[:ey], @options[:redis], @options[:dna], @options[:history], @options[:pid_path], @options[:log_path])
|
66
|
-
cmd = command ? command.to_sym : :start
|
67
|
-
@history = History.new(@config.history_path)
|
68
|
-
instance_eval(&self.class.commands[cmd.to_sym])
|
69
|
-
end
|
70
|
-
|
71
|
-
def with_pid
|
72
|
-
@config.pid_path.open('w') { |file| file << $$ }
|
73
|
-
yield
|
74
|
-
@config.pid_path.delete
|
75
|
-
end
|
76
|
-
|
77
|
-
def get_pid
|
78
|
-
@config.pid_path.read
|
79
|
-
end
|
80
|
-
|
81
|
-
def parse!(argv)
|
82
|
-
@options = DEFAULT_FILES
|
83
|
-
parser.parse!(argv)
|
84
|
-
end
|
85
|
-
|
86
|
-
def parser
|
87
|
-
@parser ||= OptionParser.new do |parser|
|
88
|
-
parser.banner = <<-USAGE
|
89
|
-
Usage: ey-monitor [FLAGS] [COMMAND]
|
90
|
-
|
91
|
-
COMMANDS
|
92
|
-
start Begin monitoring (default).
|
93
|
-
stop Safely stop running ey-monitor agents.
|
94
|
-
Reports shutdown status to stdout.
|
95
|
-
Exits with non-zero status if there is a problem.
|
96
|
-
|
97
|
-
USAGE
|
98
|
-
|
99
|
-
parser.separator "FLAGS"
|
100
|
-
|
101
|
-
parser.on('-e', '--ey [FILE]', "Path to the ey-cloud.yml file (default #{@options[:ey]})") do |ey|
|
102
|
-
@options[:ey] = Pathname.new(ey)
|
103
|
-
end
|
104
|
-
|
105
|
-
parser.on('-r', '--redis [FILE]', "Path to the redis.yml file (default #{@options[:redis]})") do |redis|
|
106
|
-
@options[:redis] = Pathname.new(redis)
|
107
|
-
end
|
108
|
-
|
109
|
-
parser.on('-d', '--dna [FILE]', "Path to the chef/dna.json file (default #{@options[:dna]})") do |dna|
|
110
|
-
@options[:dna] = Pathname.new(dna)
|
111
|
-
end
|
112
|
-
|
113
|
-
parser.on('-m', '--meta-data [FILE]', "Mocked AWS meta_data yaml file") do |path|
|
114
|
-
@options[:meta_data] = YAML.load_file(Pathname.new(path))
|
115
|
-
end
|
116
|
-
|
117
|
-
parser.on('-t', '--history [FILE]', "Location of stonith history file (default #{@options[:history]}") do |path|
|
118
|
-
@options[:history] = Pathname.new(path)
|
119
|
-
end
|
120
|
-
|
121
|
-
parser.on('-p', '--pid [FILE]', "Location of stonith pid file (default #{@options[:pid_path]}") do |path|
|
122
|
-
@options[:pid_path] = Pathname.new(path)
|
123
|
-
end
|
124
|
-
|
125
|
-
parser.on('-l', '--log [FILE]', "Location of stonith log file (default #{@options[:log_path]}") do |path|
|
126
|
-
@options[:log_path] = Pathname.new(path)
|
127
|
-
end
|
128
|
-
|
129
|
-
parser.on_tail("-h", "--help", "Show this message") do
|
130
|
-
puts parser
|
131
|
-
exit
|
132
|
-
end
|
133
|
-
end
|
134
|
-
end
|
135
|
-
end
|
136
|
-
end
|
137
|
-
end
|
138
|
-
|
@@ -1,28 +0,0 @@
|
|
1
|
-
require 'open-uri'
|
2
|
-
require 'fog'
|
3
|
-
|
4
|
-
module EY
|
5
|
-
module Stonith
|
6
|
-
class LocalMaster < AbstractMaster
|
7
|
-
def initialize(notify_uri, cloud_credentials)
|
8
|
-
@notify_uri, @cloud_credentials = notify_uri, cloud_credentials
|
9
|
-
end
|
10
|
-
alias hostname local_hostname
|
11
|
-
|
12
|
-
def takeover!(master_instance_id, master_ip)
|
13
|
-
address = AddressStealer.new(master_instance_id, master_ip, @cloud_credentials)
|
14
|
-
address.associate(instance_id)
|
15
|
-
AwsmNotifier.new(instance_id, @notify_uri, @cloud_credentials)
|
16
|
-
data(address.ip)
|
17
|
-
end
|
18
|
-
|
19
|
-
def data(ip = public_ip)
|
20
|
-
Data.new(hostname, instance_id, ip)
|
21
|
-
end
|
22
|
-
|
23
|
-
private
|
24
|
-
|
25
|
-
def instance_id() @instance_id ||= EY::Stonith.meta_data["instance-id"] end
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
data/lib/ey_stonith/master.rb
DELETED
@@ -1,37 +0,0 @@
|
|
1
|
-
require 'ey_stonith'
|
2
|
-
require 'eventmachine'
|
3
|
-
require 'em-http'
|
4
|
-
|
5
|
-
module EY
|
6
|
-
module Stonith
|
7
|
-
class Master < AbstractMaster
|
8
|
-
def initialize(database, hostname_from_dna)
|
9
|
-
@database, @hostname_from_dna = database, hostname_from_dna
|
10
|
-
end
|
11
|
-
|
12
|
-
def check(good, bad)
|
13
|
-
@database.with_data do |data|
|
14
|
-
http = EM::HttpRequest.new("http://#{data.hostname}/haproxy/monitor").get :timeout => 10
|
15
|
-
http.callback {
|
16
|
-
unless http.response_header.status == 200
|
17
|
-
bad.call data.key
|
18
|
-
else
|
19
|
-
good.call data.key
|
20
|
-
end
|
21
|
-
}
|
22
|
-
http.errback { bad.call data.key }
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
|
-
def local?() hostname == local_hostname end
|
27
|
-
def hostname() hostname_from_db || hostname_from_dna end
|
28
|
-
def with_locked_data(&block) @database.with_locked_data(&block) end
|
29
|
-
def update(data) @database.set(data) end
|
30
|
-
|
31
|
-
private
|
32
|
-
|
33
|
-
attr_reader :hostname_from_dna
|
34
|
-
def hostname_from_db() @database.with_data { |data| data.hostname } end
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|
data/lib/ey_stonith/meta_data.rb
DELETED
data/lib/ey_stonith/slave.rb
DELETED
@@ -1,41 +0,0 @@
|
|
1
|
-
module EY
|
2
|
-
module Stonith
|
3
|
-
class Slave
|
4
|
-
def initialize(box, heartbeat)
|
5
|
-
@box, @heartbeat = box, heartbeat
|
6
|
-
end
|
7
|
-
|
8
|
-
def monitor!(master)
|
9
|
-
return if @checking
|
10
|
-
Stonith.logger.info "Monitoring started"
|
11
|
-
@check_recorder = CheckRecorder.new
|
12
|
-
@checking = EM.add_periodic_timer(@heartbeat) { check(master) }
|
13
|
-
end
|
14
|
-
|
15
|
-
def stop!
|
16
|
-
if @checking
|
17
|
-
Stonith.logger.info "Stopped: Not monitoring until SIGUSR1 received"
|
18
|
-
@checking.cancel
|
19
|
-
@checking = nil
|
20
|
-
end
|
21
|
-
end
|
22
|
-
|
23
|
-
private
|
24
|
-
|
25
|
-
def check(master)
|
26
|
-
good = lambda { |master_key| @check_recorder.good_check!(master_key) }
|
27
|
-
bad = lambda { |master_key|
|
28
|
-
@check_recorder.bad_check!(master_key)
|
29
|
-
takeover! if @check_recorder.limit_exceeded?
|
30
|
-
}
|
31
|
-
master.check(good, bad)
|
32
|
-
end
|
33
|
-
|
34
|
-
def takeover!
|
35
|
-
Stonith.logger.info("Trying to grab the lock for takeover")
|
36
|
-
stop!
|
37
|
-
@box.try_takeover(@check_recorder.method(:checking_key?))
|
38
|
-
end
|
39
|
-
end
|
40
|
-
end
|
41
|
-
end
|