mcrain 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f9ac1799c54714be216cebb06654ced585cd3e87
-  data.tar.gz: 4dd2d1b9b29596535a0868fd1725beb406decac3
+  metadata.gz: 7fd5573092d72cb073836c6bc403daac785784c5
+  data.tar.gz: ea9a1ad0b4adc2cfb38519716bd053f7717db63a
 SHA512:
-  metadata.gz: 8ab8fa189190d3eebb1a292775f9e1dc9f29f381d428fbd6b4922615f9cff4ceedf3e5f3178026ffed8435950bceb6e9e4d504506d1890fa0a6433dccde90592
-  data.tar.gz: 4af72bce3122ac9fff4d33e5677e7ed274e4aee34d2796d24b58d398273231d03c3b1244a16414fa34ca0d4c810d7c46dc1aefbc383762dc3ae42833b4db083d
+  metadata.gz: c7f2572fcdd11ed35c39e2bd93f0b0fde6bc4f91db24c728413b62f30b94145e3db50e29eca631b25ada4c9801630ef4e413d83f6a6dd86cb15b8509fd261ae5
+  data.tar.gz: 76d8ffe6bb962d0d326d6115efbd914c06b7b62625ea8ad8cf844948de87b1176a292cd68dcdd578208a64e2684f55b60b3509fcf90367cd5ca5526237293fd9

data/Gemfile

@@ -15,13 +15,17 @@ group :development do
   gem "riak-client"
 
   # for mysql
-  gem "mysql2"
+  gem "mysql2", platform: "ruby"
+  # gem "jdbc-mysql", platform: "jruby" # this is not supported yet.
+
+  # for HBase
+  gem "hbase-jruby", platform: "jruby"
 end
 
 group :development do
   gem "pry"
-  gem "pry-byebug"
-  gem "pry-stack_explorer"
+  gem "pry-byebug", platform: "ruby"
+  gem "pry-stack_explorer", platform: "ruby"
   gem "simplecov"
   gem "fuubar"
   gem "parallel_tests"

data/README.md

@@ -59,34 +59,23 @@ Or install it yourself as:
 
     $ gem install mcrain
 
-### with redis
+### with middleware clients
 
-Add this line also to your application's Gemfile
-```ruby
-gem 'redis'
-```
-
-### with rabbitmq
+middleware | client gem (CRuby)  | client gem (JRuby)
+-----------|---------------------|------------------
+MySQL      | `gem 'mysql2'`      | N/A
+Redis      | `gem 'redis'`       | (the same as CRuby)
+RabbitMQ   | `gem 'rabbitmq_http_api_client', '>= 1.6.0'`       | (the same as CRuby)
+Riak       | `gem 'docker-api', '~> 1.21.1'; gem 'riak-client'` | (the same as CRuby)
+HBase      | N/A                 | `gem 'hbase-jruby'`
 
-Add this line also to your application's Gemfile
-```ruby
-gem 'rabbitmq_http_api_client', '>= 1.6.0'
-```
-
-### with riak
-
-Add this line also to your application's Gemfile
-```ruby
-gem 'docker-api', '~> 1.21.1'
-gem 'riak-client'
-```
 
 ## Usage
 
 ### redis in code
 
 ```ruby
-Mcrain[:redis].start do |s|
+Mcrain::Redis.new.start do |s|
   c = s.client # Redis::Client object
   c.ping
 end
@@ -95,7 +84,7 @@ end
 ### rabbitmq in code
 
 ```ruby
-Mcrain[:rabbitmq].start do |s|
+Mcrain::Rabbitmq.new.start do |s|
   c = s.client # RabbitMQ::HTTP::Client object
   c.list_nodes
 end
@@ -103,12 +92,10 @@ end
 
 ### riak in code
 
-Mcrain::Riak uses [docker-riak](https://github.com/hectcastro/docker-riak).
-So set the path to `Mcrain.docker_riak_path` .
+Mcrain::Riak uses [hectcastro/docker-riak](https://github.com/hectcastro/docker-riak).
 
 ```ruby
-Mcrain.docker_riak_path = "path/to/docker-riak"
-Mcrain[:riak].start do |s|
+Mcrain::Riak.new.start do |s|
   c = s.client # Riak::Client object
   obj = c.bucket("bucket1").get_or_new("foo")
   obj.data = data
@@ -116,6 +103,28 @@ Mcrain[:riak].start do |s|
 end
 ```
 
+### hbase in code
+
+Mcrain::Hbase uses [nerdammer/hbase](https://hub.docker.com/r/nerdammer/hbase).
+
+Add a line like this to `/etc/hosts`
+
+|      |       |
+|------|-------|
+| With docker toolbox    | `192.168.99.100 docker-host1` |
+| Without docker toolbox | `127.0.0.1 docker-host1` |
+
+
+```ruby
+Mcrain::Hbase.new.start do |s|
+  c = s.client # HBase object defined by hbase-jruby
+  c.list
+  c[:my_table].create! :f
+  c[:my_table].put 100, 'f:a' => 1, 'f:b' => 'two', 'f:c' => 3.14
+  c[:my_table].get(100).double('f:c') #=> 3.14
+end
+```
+
 
 ### redis in terminal
 
@@ -146,7 +155,6 @@ OK
 ### riak in terminal
 
 ```
-$ export DOCKER_RIAK_PATH=/path/to/docker-riak
 $ mcrain start riak
 To connect:
 require 'riak'
@@ -170,6 +178,38 @@ $ mcrain stop riak 5
 OK
 ```
 
+
+
+### hbase in terminal
+
+Add a line like this to `/etc/hosts`
+
+|      |       |
+|------|-------|
+| With docker toolbox    | `192.168.99.100 docker-host1` |
+| Without docker toolbox | `127.0.0.1 docker-host1` |
+
+
+```
+$ mcrain start hbase
+(snip)
+To connect:
+$CLASSPATH << "/Users/akima/.mcrain/hbase/hbase-client-dep-1.0.jar"
+$LOAD_PATH << "hbase-jruby/lib"
+require 'hbase-jruby'
+client = HBase.new(*[{"hbase.zookeeper.quorum"=>"192.168.99.100", "hbase.zookeeper.property.clientPort"=>54489, "hbase.master.port"=>60000, "hbase.master.info.port"=>54490, "hbase.regionserver.port"=>60020, "hbase.regionserver.info.port"=>54491}])
+OK
+
+$ mcrain stop riak
+(snip)
+86a8dd6c13cd2c346fe9111e16f97265cb4fdb67cc67873c495622a28f0c1062
+OK
+```
+
+use irb or something in JRuby.
+
+
+
 ## Mcrain.before_setup
 
 Use Mcrain.before_setup hook if you don't want your test or spec always works with mcrain.

data/exe/mcrain

@@ -24,7 +24,7 @@ MESSAGE
   exit(1)
 end
 
-unless args.include?("-v") || args.include?("--verbose")
+unless verbose
   require 'logger'
   Mcrain.logger = Logger.new("/dev/null")
 end
@@ -38,10 +38,31 @@ begin
     options = {cluster_size: cluster_size, automatic_clustering: cluster_size > 1}
   end
 
+  cid_filepath = ".mcrain.#{service}.cid"
   case action
-  when "start", "stop" then
-    server = Mcrain.lookup(service.to_sym, options)
-    server.send(action)
+  when "start" then
+    if File.exist?(cid_filepath)
+      $stderr.puts("\e[31m#{service} is already run on docker. Check #{cid_filepath}\e[0m")
+      exit(1)
+    else
+      server = Mcrain.class_for(service.to_sym).new(options)
+      server.start
+      open(cid_filepath, "w"){|f| f.puts(server.container.id)}
+      puts "To connect:\n#{server.client_script}"
+    end
+  when "stop" then
+    if File.readable?(cid_filepath)
+      cid = File.read(cid_filepath).strip
+      if system("docker kill #{cid} && docker rm #{cid} >/dev/null")
+        File.delete(cid_filepath)
+      else
+        $stderr.puts("\e[31mFailed to stop or rm container #{cid}. `rm #{cid_filepath}` unless the container exists.\e[0m")
+        exit(1)
+      end
+    else
+      $stderr.puts("\e[31m#{service} container is not found.\e[0m")
+      exit(1)
+    end
   when "pull" then
     services = (service == "all") ? Mcrain.images.keys : [service]
     services.map(&:to_sym).map{|s| Mcrain.class_for(s)}.each(&:pull_image)
@@ -49,10 +70,6 @@ begin
     raise "Unknown action: #{action.inspect}"
   end
 
-  case action
-  when "start" then
-    puts "To connect:\nrequire '#{server.client_require}'\nclient = #{server.client_script}"
-  end
 rescue => e
   $stderr.puts "\e[31m[#{e.class}] #{e.message}\e[0m"
   $stderr.puts e.backtrace.join("\n  ") if verbose

data/lib/mcrain.rb

@@ -36,6 +36,7 @@ module Mcrain
       redis: "redis:2.8.19",
       rabbitmq: "rabbitmq:3.4.4-management",
       riak: "hectcastro/riak",
+      hbase: "nerdammer/hbase:latest",
     }.freeze
 
     def images
@@ -64,6 +65,11 @@ module Mcrain
     end
 
     attr_accessor :before_setup
+
+    attr_writer :home_dir
+    def home_dir
+      @home_dir ||= (ENV['MCRAIN_HOME'] || File.join(ENV['HOME'], '.mcrain'))
+    end
   end
 
   autoload :Base, 'mcrain/base'
@@ -75,9 +81,11 @@ module Mcrain
   autoload :Redis, 'mcrain/redis'
   autoload :Rabbitmq, 'mcrain/rabbitmq'
   autoload :Mysql, 'mcrain/mysql'
+  autoload :Hbase, 'mcrain/hbase'
 
   register :riak, "Mcrain::Riak"
   register :redis, "Mcrain::Redis"
   register :rabbitmq, "Mcrain::Rabbitmq"
   register :mysql, "Mcrain::Mysql"
+  register :hbase, "Mcrain::Hbase"
 end

data/lib/mcrain/base.rb

@@ -42,11 +42,13 @@ module Mcrain
     end
 
     def setup
+      logger.info("#{self.class.name}#setup STARTED")
       return false if Mcrain.before_setup && !Mcrain.before_setup.call(self)
       Timeout.timeout(30) do
         DockerMachine.setup_docker_options
         container.start!
       end
+      logger.info("#{self.class.name}#setup COMPLETED")
       return container
     end
 
@@ -72,6 +74,7 @@ module Mcrain
     # ポートはdockerがまずLISTENしておいて、その後コンテナ内のミドルウェアが起動するので、
     # 実際にそのAPIを叩いてみて例外が起きないことを確認します。
     def wait
+      logger.info("#{self.class.name}#wait STARTED")
       Timeout.timeout(30) do
         begin
           wait_for_ready
@@ -81,6 +84,7 @@ module Mcrain
           retry
         end
       end
+      logger.info("#{self.class.name}#wait COMPLETED")
     end
 
     def wait_for_ready
@@ -101,5 +105,11 @@ module Mcrain
       container.remove unless ENV['MCRAIN_KEEP_CONTAINERS'] =~ /true|yes|on|1/i
     end
 
+    class << self
+      def work_dir
+        File.join(Mcrain.home_dir, server_name.to_s)
+      end
+    end
+
   end
 end

data/lib/mcrain/client_provider.rb

@@ -8,6 +8,7 @@ module Mcrain
     def build_client
       require client_require
       yield if block_given?
+      logger.debug("#{self.class.name}#build_client call: #{client_instantiation_script}")
       client_class.new(*client_init_args)
     end
 
@@ -23,9 +24,13 @@ module Mcrain
       raise NotImplementedError
     end
 
+    def client_instantiation_script
+      "#{client_class.name}.new(*#{client_init_args.inspect})"
+    end
+
     def client_script
       client
-      "#{client_class.name}.new(*#{client_init_args.inspect})"
+      "require '#{client_require}'\nclient = #{client_instantiation_script}"
     end
 
   end

data/lib/mcrain/docker_machine.rb

@@ -12,12 +12,30 @@ require 'docker'
 module Mcrain
   module DockerMachine
 
+    DEFAULT_HOSTNAME = 'docker-host1'.freeze
+
     class << self
       attr_accessor :certs_dir
 
       def docker_machine_name
         ENV['DOCKER_MACHINE_NAME']
       end
+
+      def docker_hostname
+        ENV['MCRAIN_DOCKER_HOSTNAME'] || DEFAULT_HOSTNAME
+      end
+
+      def docker_hostname!
+        name = docker_hostname
+        return name if valid_docker_hostname?
+        raise "hostname: #{name.inspect} is not valid. Plase set it with docker IP address to /etc/hosts or export MCRAIN_DOCKER_HOSTNAME=(valid-hostname) ."
+      end
+
+      def valid_docker_hostname?
+        return !!IPSocket.getaddress(docker_hostname)
+      rescue SocketError
+        return false
+      end
     end
     self.certs_dir = File.expand_path('.docker/machine/certs', ENV["HOME"])
 

data/lib/mcrain/hbase.rb

@@ -0,0 +1,107 @@
+require 'mcrain'
+
+require 'fileutils'
+
+module Mcrain
+  # Mcrain::Hbase can't start 2 containers concurrently.
+  #
+  # The zookeeper in a container of nerdammer/hbase has static configuration
+  # for client in same network, so the clients outside of container's network
+  # can't get the correct configuration from the zookeepr.
+  # So Mcrain::Hbase uses static port mapping for 60000 and 60020.
+  class Hbase < Base
+    self.server_name = :hbase
+
+    DEFAULT_CLIENT_DEP_JAR_URL = "https://github.com/junegunn/hbase-client-dep/releases/download/1.0.0/hbase-client-dep-1.0.jar"
+    attr_writer :client_dep_jar_url
+    def client_dep_jar_url
+      @client_dep_jar_url ||= DEFAULT_CLIENT_DEP_JAR_URL
+    end
+
+    attr_writer :client_dep_jar_path
+    def client_dep_jar_path
+      @client_dep_jar_path = File.join(self.class.work_dir, File.basename(client_dep_jar_url))
+    end
+
+    def client_require
+      'hbase-jruby'
+    end
+
+    def client_class
+      ::HBase
+    end
+
+    self.port = 60000 # hbase.master.port # 60000
+
+    def port # hbase.master.port
+      60000 # static port number which is defined by the container
+    end
+
+    def regionserver_port # hbase.regionserver.port
+      60020 # static port number which is defined by the container
+    end
+
+    # https://blog.cloudera.com/blog/2013/07/guide-to-using-apache-hbase-ports/
+    PORT_DEFS = {
+      'hbase.zookeeper.property.clientPort' => {method: :zookeeper_port , default:  2181},
+      'hbase.master.port'            => {method: :port                  , default: 60000},
+      'hbase.master.info.port'       => {method: :master_info_port      , default: 60010},
+      'hbase.regionserver.port'      => {method: :regionserver_port     , default: 60020},
+      'hbase.regionserver.info.port' => {method: :regionserver_info_port, default: 60030},
+    }.freeze
+
+    PORT_DEFS.each do |key, d|
+      mname = d[:method]
+      next if instance_methods.include?(mname)
+      module_eval("def #{mname}; @#{mname} ||= find_portno; end", __FILE__, __LINE__)
+    end
+
+    def client_init_args
+      options = {'hbase.zookeeper.quorum' => host}
+      PORT_DEFS.each do |key, d|
+        options[key] = send(d[:method])
+      end
+      return [options]
+    end
+
+    def download_jar
+      logger.debug("#{self.class.name}#download_jar STARTED")
+      FileUtils.mkdir_p(File.dirname(client_dep_jar_path))
+      LoggerPipe.run(Mcrain.logger, "curl -L -o #{client_dep_jar_path} #{client_dep_jar_url}")
+      logger.debug("#{self.class.name}#download_jar COMPLETED")
+    end
+
+    def build_client
+      logger.debug("#{self.class.name}#build_client STARTED")
+      download_jar unless File.exist?(client_dep_jar_path)
+      $CLASSPATH << client_dep_jar_path
+      $LOAD_PATH << 'hbase-jruby/lib'
+      r = super
+      logger.debug("#{self.class.name}#build_client COMPLETED")
+      return r
+    end
+
+    def client_script
+      [
+        "$CLASSPATH << #{client_dep_jar_path.inspect}",
+        '$LOAD_PATH << "hbase-jruby/lib"',
+        super,
+      ].join("\n")
+    end
+
+    def wait_for_ready
+      c = client
+      c.tables
+    end
+
+    def build_docker_options
+      r = super
+      r["Hostname"] = Mcrain::DockerMachine.docker_hostname!
+      PORT_DEFS.each do |key, d|
+        r['HostConfig']['PortBindings']["#{d[:default]}/tcp"] = [{ 'HostPort' => send(d[:method]).to_s }]
+      end
+      return r
+    end
+
+  end
+end

data/lib/mcrain/version.rb

@@ -1,3 +1,3 @@
 module Mcrain
-  VERSION = "0.6.0"
+  VERSION = "0.7.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mcrain
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - akm
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-11-16 00:00:00.000000000 Z
+date: 2015-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logger_pipe
@@ -125,6 +125,7 @@ files:
 - lib/mcrain/client_provider.rb
 - lib/mcrain/container_controller.rb
 - lib/mcrain/docker_machine.rb
+- lib/mcrain/hbase.rb
 - lib/mcrain/mysql.rb
 - lib/mcrain/rabbitmq.rb
 - lib/mcrain/redis.rb