mcrain 0.6.0 → 0.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +7 -3
- data/README.md +66 -26
- data/exe/mcrain +25 -8
- data/lib/mcrain.rb +8 -0
- data/lib/mcrain/base.rb +10 -0
- data/lib/mcrain/client_provider.rb +6 -1
- data/lib/mcrain/docker_machine.rb +18 -0
- data/lib/mcrain/hbase.rb +107 -0
- data/lib/mcrain/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7fd5573092d72cb073836c6bc403daac785784c5
|
4
|
+
data.tar.gz: ea9a1ad0b4adc2cfb38519716bd053f7717db63a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c7f2572fcdd11ed35c39e2bd93f0b0fde6bc4f91db24c728413b62f30b94145e3db50e29eca631b25ada4c9801630ef4e413d83f6a6dd86cb15b8509fd261ae5
|
7
|
+
data.tar.gz: 76d8ffe6bb962d0d326d6115efbd914c06b7b62625ea8ad8cf844948de87b1176a292cd68dcdd578208a64e2684f55b60b3509fcf90367cd5ca5526237293fd9
|
data/Gemfile
CHANGED
@@ -15,13 +15,17 @@ group :development do
|
|
15
15
|
gem "riak-client"
|
16
16
|
|
17
17
|
# for mysql
|
18
|
-
gem "mysql2"
|
18
|
+
gem "mysql2", platform: "ruby"
|
19
|
+
# gem "jdbc-mysql", platform: "jruby" # this is not supported yet.
|
20
|
+
|
21
|
+
# for HBase
|
22
|
+
gem "hbase-jruby", platform: "jruby"
|
19
23
|
end
|
20
24
|
|
21
25
|
group :development do
|
22
26
|
gem "pry"
|
23
|
-
gem "pry-byebug"
|
24
|
-
gem "pry-stack_explorer"
|
27
|
+
gem "pry-byebug", platform: "ruby"
|
28
|
+
gem "pry-stack_explorer", platform: "ruby"
|
25
29
|
gem "simplecov"
|
26
30
|
gem "fuubar"
|
27
31
|
gem "parallel_tests"
|
data/README.md
CHANGED
@@ -59,34 +59,23 @@ Or install it yourself as:
|
|
59
59
|
|
60
60
|
$ gem install mcrain
|
61
61
|
|
62
|
-
### with
|
62
|
+
### with middleware clients
|
63
63
|
|
64
|
-
|
65
|
-
|
66
|
-
gem '
|
67
|
-
|
68
|
-
|
69
|
-
|
64
|
+
middleware | client gem (CRuby) | client gem (JRuby)
|
65
|
+
-----------|---------------------|------------------
|
66
|
+
MySQL | `gem 'mysql2'` | N/A
|
67
|
+
Redis | `gem 'redis'` | (the same as CRuby)
|
68
|
+
RabbitMQ | `gem 'rabbitmq_http_api_client', '>= 1.6.0'` | (the same as CRuby)
|
69
|
+
Riak | `gem 'docker-api', '~> 1.21.1'; gem 'riak-client'` | (the same as CRuby)
|
70
|
+
HBase | N/A | `gem 'hbase-jruby'`
|
70
71
|
|
71
|
-
Add this line also to your application's Gemfile
|
72
|
-
```ruby
|
73
|
-
gem 'rabbitmq_http_api_client', '>= 1.6.0'
|
74
|
-
```
|
75
|
-
|
76
|
-
### with riak
|
77
|
-
|
78
|
-
Add this line also to your application's Gemfile
|
79
|
-
```ruby
|
80
|
-
gem 'docker-api', '~> 1.21.1'
|
81
|
-
gem 'riak-client'
|
82
|
-
```
|
83
72
|
|
84
73
|
## Usage
|
85
74
|
|
86
75
|
### redis in code
|
87
76
|
|
88
77
|
```ruby
|
89
|
-
Mcrain
|
78
|
+
Mcrain::Redis.new.start do |s|
|
90
79
|
c = s.client # Redis::Client object
|
91
80
|
c.ping
|
92
81
|
end
|
@@ -95,7 +84,7 @@ end
|
|
95
84
|
### rabbitmq in code
|
96
85
|
|
97
86
|
```ruby
|
98
|
-
Mcrain
|
87
|
+
Mcrain::Rabbitmq.new.start do |s|
|
99
88
|
c = s.client # RabbitMQ::HTTP::Client object
|
100
89
|
c.list_nodes
|
101
90
|
end
|
@@ -103,12 +92,10 @@ end
|
|
103
92
|
|
104
93
|
### riak in code
|
105
94
|
|
106
|
-
Mcrain::Riak uses [docker-riak](https://github.com/hectcastro/docker-riak).
|
107
|
-
So set the path to `Mcrain.docker_riak_path` .
|
95
|
+
Mcrain::Riak uses [hectcastro/docker-riak](https://github.com/hectcastro/docker-riak).
|
108
96
|
|
109
97
|
```ruby
|
110
|
-
Mcrain.
|
111
|
-
Mcrain[:riak].start do |s|
|
98
|
+
Mcrain::Riak.new.start do |s|
|
112
99
|
c = s.client # Riak::Client object
|
113
100
|
obj = c.bucket("bucket1").get_or_new("foo")
|
114
101
|
obj.data = data
|
@@ -116,6 +103,28 @@ Mcrain[:riak].start do |s|
|
|
116
103
|
end
|
117
104
|
```
|
118
105
|
|
106
|
+
### hbase in code
|
107
|
+
|
108
|
+
Mcrain::Hbase uses [nerdammer/hbase](https://hub.docker.com/r/nerdammer/hbase).
|
109
|
+
|
110
|
+
Add a line like this to `/etc/hosts`
|
111
|
+
|
112
|
+
| | |
|
113
|
+
|------|-------|
|
114
|
+
| With docker toolbox | `192.168.99.100 docker-host1` |
|
115
|
+
| Without docker toolbox | `127.0.0.1 docker-host1` |
|
116
|
+
|
117
|
+
|
118
|
+
```ruby
|
119
|
+
Mcrain::Hbase.new.start do |s|
|
120
|
+
c = s.client # HBase object defined by hbase-jruby
|
121
|
+
c.list
|
122
|
+
c[:my_table].create! :f
|
123
|
+
c[:my_table].put 100, 'f:a' => 1, 'f:b' => 'two', 'f:c' => 3.14
|
124
|
+
c[:my_table].get(100).double('f:c') #=> 3.14
|
125
|
+
end
|
126
|
+
```
|
127
|
+
|
119
128
|
|
120
129
|
### redis in terminal
|
121
130
|
|
@@ -146,7 +155,6 @@ OK
|
|
146
155
|
### riak in terminal
|
147
156
|
|
148
157
|
```
|
149
|
-
$ export DOCKER_RIAK_PATH=/path/to/docker-riak
|
150
158
|
$ mcrain start riak
|
151
159
|
To connect:
|
152
160
|
require 'riak'
|
@@ -170,6 +178,38 @@ $ mcrain stop riak 5
|
|
170
178
|
OK
|
171
179
|
```
|
172
180
|
|
181
|
+
|
182
|
+
|
183
|
+
### hbase in terminal
|
184
|
+
|
185
|
+
Add a line like this to `/etc/hosts`
|
186
|
+
|
187
|
+
| | |
|
188
|
+
|------|-------|
|
189
|
+
| With docker toolbox | `192.168.99.100 docker-host1` |
|
190
|
+
| Without docker toolbox | `127.0.0.1 docker-host1` |
|
191
|
+
|
192
|
+
|
193
|
+
```
|
194
|
+
$ mcrain start hbase
|
195
|
+
(snip)
|
196
|
+
To connect:
|
197
|
+
$CLASSPATH << "/Users/akima/.mcrain/hbase/hbase-client-dep-1.0.jar"
|
198
|
+
$LOAD_PATH << "hbase-jruby/lib"
|
199
|
+
require 'hbase-jruby'
|
200
|
+
client = HBase.new(*[{"hbase.zookeeper.quorum"=>"192.168.99.100", "hbase.zookeeper.property.clientPort"=>54489, "hbase.master.port"=>60000, "hbase.master.info.port"=>54490, "hbase.regionserver.port"=>60020, "hbase.regionserver.info.port"=>54491}])
|
201
|
+
OK
|
202
|
+
|
203
|
+
$ mcrain stop riak
|
204
|
+
(snip)
|
205
|
+
86a8dd6c13cd2c346fe9111e16f97265cb4fdb67cc67873c495622a28f0c1062
|
206
|
+
OK
|
207
|
+
```
|
208
|
+
|
209
|
+
use irb or something in JRuby.
|
210
|
+
|
211
|
+
|
212
|
+
|
173
213
|
## Mcrain.before_setup
|
174
214
|
|
175
215
|
Use Mcrain.before_setup hook if you don't want your test or spec always works with mcrain.
|
data/exe/mcrain
CHANGED
@@ -24,7 +24,7 @@ MESSAGE
|
|
24
24
|
exit(1)
|
25
25
|
end
|
26
26
|
|
27
|
-
unless
|
27
|
+
unless verbose
|
28
28
|
require 'logger'
|
29
29
|
Mcrain.logger = Logger.new("/dev/null")
|
30
30
|
end
|
@@ -38,10 +38,31 @@ begin
|
|
38
38
|
options = {cluster_size: cluster_size, automatic_clustering: cluster_size > 1}
|
39
39
|
end
|
40
40
|
|
41
|
+
cid_filepath = ".mcrain.#{service}.cid"
|
41
42
|
case action
|
42
|
-
when "start"
|
43
|
-
|
44
|
-
|
43
|
+
when "start" then
|
44
|
+
if File.exist?(cid_filepath)
|
45
|
+
$stderr.puts("\e[31m#{service} is already run on docker. Check #{cid_filepath}\e[0m")
|
46
|
+
exit(1)
|
47
|
+
else
|
48
|
+
server = Mcrain.class_for(service.to_sym).new(options)
|
49
|
+
server.start
|
50
|
+
open(cid_filepath, "w"){|f| f.puts(server.container.id)}
|
51
|
+
puts "To connect:\n#{server.client_script}"
|
52
|
+
end
|
53
|
+
when "stop" then
|
54
|
+
if File.readable?(cid_filepath)
|
55
|
+
cid = File.read(cid_filepath).strip
|
56
|
+
if system("docker kill #{cid} && docker rm #{cid} >/dev/null")
|
57
|
+
File.delete(cid_filepath)
|
58
|
+
else
|
59
|
+
$stderr.puts("\e[31mFailed to stop or rm container #{cid}. `rm #{cid_filepath}` unless the container exists.\e[0m")
|
60
|
+
exit(1)
|
61
|
+
end
|
62
|
+
else
|
63
|
+
$stderr.puts("\e[31m#{service} container is not found.\e[0m")
|
64
|
+
exit(1)
|
65
|
+
end
|
45
66
|
when "pull" then
|
46
67
|
services = (service == "all") ? Mcrain.images.keys : [service]
|
47
68
|
services.map(&:to_sym).map{|s| Mcrain.class_for(s)}.each(&:pull_image)
|
@@ -49,10 +70,6 @@ begin
|
|
49
70
|
raise "Unknown action: #{action.inspect}"
|
50
71
|
end
|
51
72
|
|
52
|
-
case action
|
53
|
-
when "start" then
|
54
|
-
puts "To connect:\nrequire '#{server.client_require}'\nclient = #{server.client_script}"
|
55
|
-
end
|
56
73
|
rescue => e
|
57
74
|
$stderr.puts "\e[31m[#{e.class}] #{e.message}\e[0m"
|
58
75
|
$stderr.puts e.backtrace.join("\n ") if verbose
|
data/lib/mcrain.rb
CHANGED
@@ -36,6 +36,7 @@ module Mcrain
|
|
36
36
|
redis: "redis:2.8.19",
|
37
37
|
rabbitmq: "rabbitmq:3.4.4-management",
|
38
38
|
riak: "hectcastro/riak",
|
39
|
+
hbase: "nerdammer/hbase:latest",
|
39
40
|
}.freeze
|
40
41
|
|
41
42
|
def images
|
@@ -64,6 +65,11 @@ module Mcrain
|
|
64
65
|
end
|
65
66
|
|
66
67
|
attr_accessor :before_setup
|
68
|
+
|
69
|
+
attr_writer :home_dir
|
70
|
+
def home_dir
|
71
|
+
@home_dir ||= (ENV['MCRAIN_HOME'] || File.join(ENV['HOME'], '.mcrain'))
|
72
|
+
end
|
67
73
|
end
|
68
74
|
|
69
75
|
autoload :Base, 'mcrain/base'
|
@@ -75,9 +81,11 @@ module Mcrain
|
|
75
81
|
autoload :Redis, 'mcrain/redis'
|
76
82
|
autoload :Rabbitmq, 'mcrain/rabbitmq'
|
77
83
|
autoload :Mysql, 'mcrain/mysql'
|
84
|
+
autoload :Hbase, 'mcrain/hbase'
|
78
85
|
|
79
86
|
register :riak, "Mcrain::Riak"
|
80
87
|
register :redis, "Mcrain::Redis"
|
81
88
|
register :rabbitmq, "Mcrain::Rabbitmq"
|
82
89
|
register :mysql, "Mcrain::Mysql"
|
90
|
+
register :hbase, "Mcrain::Hbase"
|
83
91
|
end
|
data/lib/mcrain/base.rb
CHANGED
@@ -42,11 +42,13 @@ module Mcrain
|
|
42
42
|
end
|
43
43
|
|
44
44
|
def setup
|
45
|
+
logger.info("#{self.class.name}#setup STARTED")
|
45
46
|
return false if Mcrain.before_setup && !Mcrain.before_setup.call(self)
|
46
47
|
Timeout.timeout(30) do
|
47
48
|
DockerMachine.setup_docker_options
|
48
49
|
container.start!
|
49
50
|
end
|
51
|
+
logger.info("#{self.class.name}#setup COMPLETED")
|
50
52
|
return container
|
51
53
|
end
|
52
54
|
|
@@ -72,6 +74,7 @@ module Mcrain
|
|
72
74
|
# ポートはdockerがまずLISTENしておいて、その後コンテナ内のミドルウェアが起動するので、
|
73
75
|
# 実際にそのAPIを叩いてみて例外が起きないことを確認します。
|
74
76
|
def wait
|
77
|
+
logger.info("#{self.class.name}#wait STARTED")
|
75
78
|
Timeout.timeout(30) do
|
76
79
|
begin
|
77
80
|
wait_for_ready
|
@@ -81,6 +84,7 @@ module Mcrain
|
|
81
84
|
retry
|
82
85
|
end
|
83
86
|
end
|
87
|
+
logger.info("#{self.class.name}#wait COMPLETED")
|
84
88
|
end
|
85
89
|
|
86
90
|
def wait_for_ready
|
@@ -101,5 +105,11 @@ module Mcrain
|
|
101
105
|
container.remove unless ENV['MCRAIN_KEEP_CONTAINERS'] =~ /true|yes|on|1/i
|
102
106
|
end
|
103
107
|
|
108
|
+
class << self
|
109
|
+
def work_dir
|
110
|
+
File.join(Mcrain.home_dir, server_name.to_s)
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
104
114
|
end
|
105
115
|
end
|
@@ -8,6 +8,7 @@ module Mcrain
|
|
8
8
|
def build_client
|
9
9
|
require client_require
|
10
10
|
yield if block_given?
|
11
|
+
logger.debug("#{self.class.name}#build_client call: #{client_instantiation_script}")
|
11
12
|
client_class.new(*client_init_args)
|
12
13
|
end
|
13
14
|
|
@@ -23,9 +24,13 @@ module Mcrain
|
|
23
24
|
raise NotImplementedError
|
24
25
|
end
|
25
26
|
|
27
|
+
def client_instantiation_script
|
28
|
+
"#{client_class.name}.new(*#{client_init_args.inspect})"
|
29
|
+
end
|
30
|
+
|
26
31
|
def client_script
|
27
32
|
client
|
28
|
-
"#{
|
33
|
+
"require '#{client_require}'\nclient = #{client_instantiation_script}"
|
29
34
|
end
|
30
35
|
|
31
36
|
end
|
@@ -12,12 +12,30 @@ require 'docker'
|
|
12
12
|
module Mcrain
|
13
13
|
module DockerMachine
|
14
14
|
|
15
|
+
DEFAULT_HOSTNAME = 'docker-host1'.freeze
|
16
|
+
|
15
17
|
class << self
|
16
18
|
attr_accessor :certs_dir
|
17
19
|
|
18
20
|
def docker_machine_name
|
19
21
|
ENV['DOCKER_MACHINE_NAME']
|
20
22
|
end
|
23
|
+
|
24
|
+
def docker_hostname
|
25
|
+
ENV['MCRAIN_DOCKER_HOSTNAME'] || DEFAULT_HOSTNAME
|
26
|
+
end
|
27
|
+
|
28
|
+
def docker_hostname!
|
29
|
+
name = docker_hostname
|
30
|
+
return name if valid_docker_hostname?
|
31
|
+
raise "hostname: #{name.inspect} is not valid. Plase set it with docker IP address to /etc/hosts or export MCRAIN_DOCKER_HOSTNAME=(valid-hostname) ."
|
32
|
+
end
|
33
|
+
|
34
|
+
def valid_docker_hostname?
|
35
|
+
return !!IPSocket.getaddress(docker_hostname)
|
36
|
+
rescue SocketError
|
37
|
+
return false
|
38
|
+
end
|
21
39
|
end
|
22
40
|
self.certs_dir = File.expand_path('.docker/machine/certs', ENV["HOME"])
|
23
41
|
|
data/lib/mcrain/hbase.rb
ADDED
@@ -0,0 +1,107 @@
|
|
1
|
+
require 'mcrain'
|
2
|
+
|
3
|
+
require 'fileutils'
|
4
|
+
|
5
|
+
module Mcrain
|
6
|
+
# Mcrain::Hbase can't start 2 containers concurrently.
|
7
|
+
#
|
8
|
+
# The zookeeper in a container of nerdammer/hbase has static configuration
|
9
|
+
# for client in same network, so the clients outside of container's network
|
10
|
+
# can't get the correct configuration from the zookeepr.
|
11
|
+
# So Mcrain::Hbase uses static port mapping for 60000 and 60020.
|
12
|
+
class Hbase < Base
|
13
|
+
self.server_name = :hbase
|
14
|
+
|
15
|
+
DEFAULT_CLIENT_DEP_JAR_URL = "https://github.com/junegunn/hbase-client-dep/releases/download/1.0.0/hbase-client-dep-1.0.jar"
|
16
|
+
attr_writer :client_dep_jar_url
|
17
|
+
def client_dep_jar_url
|
18
|
+
@client_dep_jar_url ||= DEFAULT_CLIENT_DEP_JAR_URL
|
19
|
+
end
|
20
|
+
|
21
|
+
attr_writer :client_dep_jar_path
|
22
|
+
def client_dep_jar_path
|
23
|
+
@client_dep_jar_path = File.join(self.class.work_dir, File.basename(client_dep_jar_url))
|
24
|
+
end
|
25
|
+
|
26
|
+
def client_require
|
27
|
+
'hbase-jruby'
|
28
|
+
end
|
29
|
+
|
30
|
+
def client_class
|
31
|
+
::HBase
|
32
|
+
end
|
33
|
+
|
34
|
+
self.port = 60000 # hbase.master.port # 60000
|
35
|
+
|
36
|
+
def port # hbase.master.port
|
37
|
+
60000 # static port number which is defined by the container
|
38
|
+
end
|
39
|
+
|
40
|
+
def regionserver_port # hbase.regionserver.port
|
41
|
+
60020 # static port number which is defined by the container
|
42
|
+
end
|
43
|
+
|
44
|
+
# https://blog.cloudera.com/blog/2013/07/guide-to-using-apache-hbase-ports/
|
45
|
+
PORT_DEFS = {
|
46
|
+
'hbase.zookeeper.property.clientPort' => {method: :zookeeper_port , default: 2181},
|
47
|
+
'hbase.master.port' => {method: :port , default: 60000},
|
48
|
+
'hbase.master.info.port' => {method: :master_info_port , default: 60010},
|
49
|
+
'hbase.regionserver.port' => {method: :regionserver_port , default: 60020},
|
50
|
+
'hbase.regionserver.info.port' => {method: :regionserver_info_port, default: 60030},
|
51
|
+
}.freeze
|
52
|
+
|
53
|
+
PORT_DEFS.each do |key, d|
|
54
|
+
mname = d[:method]
|
55
|
+
next if instance_methods.include?(mname)
|
56
|
+
module_eval("def #{mname}; @#{mname} ||= find_portno; end", __FILE__, __LINE__)
|
57
|
+
end
|
58
|
+
|
59
|
+
def client_init_args
|
60
|
+
options = {'hbase.zookeeper.quorum' => host}
|
61
|
+
PORT_DEFS.each do |key, d|
|
62
|
+
options[key] = send(d[:method])
|
63
|
+
end
|
64
|
+
return [options]
|
65
|
+
end
|
66
|
+
|
67
|
+
def download_jar
|
68
|
+
logger.debug("#{self.class.name}#download_jar STARTED")
|
69
|
+
FileUtils.mkdir_p(File.dirname(client_dep_jar_path))
|
70
|
+
LoggerPipe.run(Mcrain.logger, "curl -L -o #{client_dep_jar_path} #{client_dep_jar_url}")
|
71
|
+
logger.debug("#{self.class.name}#download_jar COMPLETED")
|
72
|
+
end
|
73
|
+
|
74
|
+
def build_client
|
75
|
+
logger.debug("#{self.class.name}#build_client STARTED")
|
76
|
+
download_jar unless File.exist?(client_dep_jar_path)
|
77
|
+
$CLASSPATH << client_dep_jar_path
|
78
|
+
$LOAD_PATH << 'hbase-jruby/lib'
|
79
|
+
r = super
|
80
|
+
logger.debug("#{self.class.name}#build_client COMPLETED")
|
81
|
+
return r
|
82
|
+
end
|
83
|
+
|
84
|
+
def client_script
|
85
|
+
[
|
86
|
+
"$CLASSPATH << #{client_dep_jar_path.inspect}",
|
87
|
+
'$LOAD_PATH << "hbase-jruby/lib"',
|
88
|
+
super,
|
89
|
+
].join("\n")
|
90
|
+
end
|
91
|
+
|
92
|
+
def wait_for_ready
|
93
|
+
c = client
|
94
|
+
c.tables
|
95
|
+
end
|
96
|
+
|
97
|
+
def build_docker_options
|
98
|
+
r = super
|
99
|
+
r["Hostname"] = Mcrain::DockerMachine.docker_hostname!
|
100
|
+
PORT_DEFS.each do |key, d|
|
101
|
+
r['HostConfig']['PortBindings']["#{d[:default]}/tcp"] = [{ 'HostPort' => send(d[:method]).to_s }]
|
102
|
+
end
|
103
|
+
return r
|
104
|
+
end
|
105
|
+
|
106
|
+
end
|
107
|
+
end
|
data/lib/mcrain/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mcrain
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.7.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- akm
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-11-
|
11
|
+
date: 2015-11-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: logger_pipe
|
@@ -125,6 +125,7 @@ files:
|
|
125
125
|
- lib/mcrain/client_provider.rb
|
126
126
|
- lib/mcrain/container_controller.rb
|
127
127
|
- lib/mcrain/docker_machine.rb
|
128
|
+
- lib/mcrain/hbase.rb
|
128
129
|
- lib/mcrain/mysql.rb
|
129
130
|
- lib/mcrain/rabbitmq.rb
|
130
131
|
- lib/mcrain/redis.rb
|