kumogata 0.3.13 → 0.3.14

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZDVkOTFjYWZhMzc2OTdhYjlkM2RlYWY3ZDI4MjcyYzNjYmZlN2NhYQ==
+    NWY1ZGU1ZTExOTU3MWMyMmYwNGViNGVkNTc0ZGI3OTkxMzI2NTRkNA==
   data.tar.gz: !binary |-
-    MmYyNjA2YjZkZTZjNzJkZDljOWQzNjU2NzNkMDgwYjQ3MzBiN2Q5Nw==
+    NjE4MjUwNWQ5ZjQ0Njg5YzUzNDYxZDQwOTIzNTdjZmFlZGViM2RjNQ==
 SHA512:
   metadata.gz: !binary |-
-    NDBjZDk1YTY2YTZkMDM1YjkzMDE1ZDU3ZDdiOTZlNTFhYWY4NGQ3YmY1OGNk
-    YmMwZDc5Yzc3MmI0MTExNzU0ZTNhYTQ0MTI2ZTFlZWJhYTg5MzcyZjY2ZjJm
-    NmZhYjQ2ZWMyMjU3OGUxMTA3MThlNzdmNjk4ODhlYjQ2YmE5NmQ=
+    YjE1YzNkYWJmMTcwZTNiMjdmMjZhMWUwNDc2ZDJkOWQzYjgwOTRlMDY3NTM4
+    NWQzZTYxM2U5OTY1NjgwMGE3MWI5YjRmYmRmZjI3OGM4MTJlMzdhOTJkZmFm
+    OGIzNTc3NTliMjYzMjVlNzYxOWE5Y2Y2YTJlMDZlYjNjMDVjMDc=
   data.tar.gz: !binary |-
-    MTRiY2EzMDIyYjBkZDdhNmYyOTgwZTM1ZDg3ZWEzMjc3MGViMmY0NjhlMzE1
-    NTExMjFmNmI4ZmUzM2M2MWFkZmNlY2MyODI3ODY1MjI0Yzk4Y2JiZDZiMDFm
-    NWM0MmVkMjljMGNjYTNmNDViNWRhYjIyNDYwNDgzMzAxNDNmNTI=
+    Y2ZiMzE5MmM3ZDU0MGJlZjJjNGVlYjRhOWI3MGNlZmU2YTFlOTc3NTU5YzM2
+    N2JiMzk1ZTJmNzllNDM5NzMyOWIzZmQ3NWU2NWQ1Mzk3NDMzMWY3YTQxZjMz
+    ZGJiNDNmNDE1OWMyNDU2MDU1Yzg5Y2Q4NDUxMTYxZjM4MGNhYzY=

data/README.md

@@ -5,8 +5,8 @@
 
 Kumogata is a tool for [AWS CloudFormation](https://aws.amazon.com/cloudformation/).
 
-[![Gem Version](https://badge.fury.io/rb/kumogata.png?201403111456)](http://badge.fury.io/rb/kumogata)
-[![Build Status](https://drone.io/github.com/winebarrel/kumogata/status.png?201403111456)](https://drone.io/github.com/winebarrel/kumogata/latest)
+[![Gem Version](https://badge.fury.io/rb/kumogata.png?201403121243)](http://badge.fury.io/rb/kumogata)
+[![Build Status](https://drone.io/github.com/winebarrel/kumogata/status.png?201403121243)](https://drone.io/github.com/winebarrel/kumogata/latest)
 
 It can define a template in Ruby DSL, such as:
 

data/lib/kumogata.rb

@@ -19,6 +19,7 @@ require 'set'
 require 'singleton'
 require 'strscan'
 require 'term/ansicolor'
+require 'thread'
 require 'uuidtools'
 
 require 'kumogata/argument_parser'
@@ -29,4 +30,5 @@ require 'kumogata/ext/json_ext'
 require 'kumogata/ext/string_ext'
 require 'kumogata/logger'
 require 'kumogata/post_processing'
+require 'kumogata/string_stream'
 require 'kumogata/utils'

data/lib/kumogata/post_processing.rb

@@ -135,13 +135,22 @@ class Kumogata::PostProcessing
     connect_tries = (ssh['connect_tries'] || 36).to_i
     retry_interval = (ssh['retry_interval'] || 5).to_i
 
+    stderr_orig = nil
+
     begin
-      retryable(:tries => connect_tries, :on => Net::SSH::Disconnect, :sleep => retry_interval) do
-        Net::SSH.start(*args) {|ssh| ssh_exec!(ssh, command) }
+      stderr_orig = STDERR.dup
+      STDERR.reopen('/dev/null', 'w')
+
+      begin
+        retryable(:tries => connect_tries, :on => Net::SSH::Disconnect, :sleep => retry_interval) do
+          Net::SSH.start(*args) {|ssh| ssh_exec!(ssh, command) }
+        end
+      rescue Net::SSH::HostKeyMismatch => e
+        e.remember_host!
+        retry
       end
-    rescue Net::SSH::HostKeyMismatch => e
-      e.remember_host!
-      retry
+    ensure
+      STDERR.reopen(stderr_orig)
     end
   end
 
@@ -151,6 +160,9 @@ class Kumogata::PostProcessing
     exit_code = nil
     #exit_signal = nil
 
+    stdout_stream = create_stdout_stream
+    stderr_stream = create_stderr_stream
+
     ssh.open_channel do |channel|
       channel.exec(command) do |ch, success|
         unless success
@@ -158,10 +170,12 @@ class Kumogata::PostProcessing
         end
 
         channel.on_data do |ch, data|
+          stdout_stream.push data
           stdout_data << data
         end
 
         channel.on_extended_data do |ch, type, data|
+          stderr_stream.push data
           stderr_data << data
         end
 
@@ -177,13 +191,57 @@ class Kumogata::PostProcessing
 
     ssh.loop
 
+    stdout_stream.close
+    stderr_stream.close
+
     #[stdout_data, stderr_data, exit_code, exit_signal]
     [stdout_data, stderr_data, exit_code]
   end
 
   def run_shell_command(command, outputs)
     command = evaluate_command_template(command, outputs)
-    Open3.capture3(command)
+
+    stdout_data = ''
+    stderr_data = ''
+    exit_code = nil
+
+    Open3.popen3(command) do |stdin, stdout, stderr, wait_thr|
+      mutex = Mutex.new
+
+      th_out = Thread.start do
+        stdout_stream = create_stdout_stream
+
+        stdout.each_line do |line|
+          mutex.synchronize do
+            stdout_stream.push line
+          end
+
+          stdout_data << line
+        end
+
+        stdout_stream.close
+      end
+
+      th_err = Thread.start do
+        stderr_stream = create_stderr_stream
+
+        stderr.each_line do |line|
+          mutex.synchronize do
+            stderr_stream.push line
+          end
+          stderr_data << line
+        end
+
+        stderr_stream.close
+      end
+
+      th_out.join
+      th_err.join
+      exit_code = wait_thr.value
+    end
+
+    #[stdout_data, stderr_data, exit_code, exit_signal]
+    [stdout_data, stderr_data, exit_code]
   end
 
   def validate_command_template(name, command, outputs)
@@ -232,17 +290,25 @@ Command: #{name.intense_blue}
     EOS
   end
 
-  def print_command_result(out, err, status)
+  def create_stdout_stream
+    Kumogata::StringStream.new do |line|
+      puts '1> '.intense_green + line
+      $stdout.flush
+    end
+  end
+
+  def create_stderr_stream
+    Kumogata::StringStream.new do |line|
+      puts '2> '.intense_red + line
+      $stdout.flush
+    end
+  end
+
+  def print_command_result(out, err, status) # XXX:
     status = status.to_i
-    dspout = (out || '').lines.map {|i| "1> ".intense_green + i }.join.chomp
-    dsperr = (err || '').lines.map {|i| "2> ".intense_red + i }.join.chomp
 
     puts <<-EOS
-Status: #{status.zero? ? status : status.to_s.red}#{
-  dspout.empty? ? '' : ("\n---\n" + dspout)
-}#{
-  dsperr.empty? ? '' : ("\n---\n" + dsperr)
-}
+Status: #{status.zero? ? status : status.to_s.red}
     EOS
   end
 

data/lib/kumogata/string_stream.rb

@@ -0,0 +1,39 @@
+class Kumogata::StringStream
+  def initialize(&block)
+    @buf = StringScanner.new('')
+    @block = block
+
+    @fiber = Fiber.new do
+      self.run
+    end
+
+    # Step to `yield`
+    @fiber.resume
+  end
+
+  def run
+    loop do
+      chunk = Fiber.yield
+      break unless chunk
+
+      @buf << chunk.to_s
+      self.each_line
+    end
+  end
+
+  def each_line
+    while (line = @buf.scan_until(/(\r\n|\r|\n)/))
+      @block.call(line.chomp)
+    end
+  end
+
+  def push(chunk)
+    @fiber.resume(chunk)
+  end
+
+  def close
+    self.each_line
+    @block.call(@buf.rest) if @buf.rest?
+    @fiber.resume
+  end
+end

data/lib/kumogata/version.rb

@@ -1,3 +1,3 @@
 module Kumogata
-  VERSION = '0.3.13'
+  VERSION = '0.3.14'
 end

data/spec/kumogata_create_spec.rb

@@ -146,9 +146,13 @@ end
       process_status1 = make_double('process_status1') {|obj| obj.should_receive(:to_i).and_return(0) }
       process_status2 = make_double('process_status2') {|obj| obj.should_receive(:to_i).and_return(0) }
 
-      Open3.should_receive(:capture3).with("echo ap-northeast-1b\necho ap-northeast-1\n")
+      client.instance_variable_get(:@post_processing)
+           .should_receive(:run_shell_command)
+           .with("      echo <%= Key \"AZ\" %>\n      echo <%= Key \"Region\" %>\n", {"AZ"=>"ap-northeast-1b", "Region"=>"ap-northeast-1"})
            .and_return(["ap-northeast-1b\nap-northeast-1\n", "", process_status1])
-      Open3.should_receive(:capture3).with("echo ap-northeast-1\necho ap-northeast-1b\n")
+      client.instance_variable_get(:@post_processing)
+           .should_receive(:run_shell_command)
+           .with("      echo <%= Key \"Region\" %>\n      echo <%= Key \"AZ\" %>\n", {"AZ"=>"ap-northeast-1b", "Region"=>"ap-northeast-1"})
            .and_return(["ap-northeast-1\nap-northeast-1b\n", "", process_status2])
 
       client.instance_variable_get(:@post_processing)
@@ -240,7 +244,9 @@ end
 
       cf.should_receive(:stacks).twice { stacks }
 
-      Net::SSH.should_receive(:start).with("127.0.0.1", "ec2-user")
+      client.instance_variable_get(:@post_processing)
+           .should_receive(:run_ssh_command)
+           .with({"host"=>"<%= Key \"PublicIp\" %>", "user"=>"ec2-user"}, "      ls\n", {"PublicIp"=>"127.0.0.1"})
            .and_return(["file1\nfile2\n", "", 0])
 
       client.instance_variable_get(:@post_processing)
@@ -345,9 +351,13 @@ end
       process_status1 = make_double('process_status1') {|obj| obj.should_receive(:to_i).and_return(0) }
       process_status2 = make_double('process_status2') {|obj| obj.should_receive(:to_i).and_return(0) }
 
-      Open3.should_receive(:capture3).with("echo ap-northeast-1b\necho ap-northeast-1\n")
+      client.instance_variable_get(:@post_processing)
+           .should_receive(:run_shell_command)
+           .with("      echo <%= Key \"AZ\" %>\n      echo <%= Key \"Region\" %>\n", {"AZ"=>"ap-northeast-1b", "Region"=>"ap-northeast-1"})
            .and_return(["ap-northeast-1b\nap-northeast-1\n", "", process_status1])
-      Open3.should_receive(:capture3).with("echo ap-northeast-1\necho ap-northeast-1b\n")
+      client.instance_variable_get(:@post_processing)
+           .should_receive(:run_shell_command)
+           .with("      echo <%= Key \"Region\" %>\n      echo <%= Key \"AZ\" %>\n", {"AZ"=>"ap-northeast-1b", "Region"=>"ap-northeast-1"})
            .and_return(["ap-northeast-1\nap-northeast-1b\n", "", process_status2])
 
       client.instance_variable_get(:@post_processing)

data/spec/kumogata_update_spec.rb

@@ -141,9 +141,13 @@ end
       process_status1 = make_double('process_status1') {|obj| obj.should_receive(:to_i).and_return(0) }
       process_status2 = make_double('process_status2') {|obj| obj.should_receive(:to_i).and_return(0) }
 
-      Open3.should_receive(:capture3).with("echo ap-northeast-1b\necho ap-northeast-1\n")
+      client.instance_variable_get(:@post_processing)
+           .should_receive(:run_shell_command)
+           .with("      echo <%= Key \"AZ\" %>\n      echo <%= Key \"Region\" %>\n", {"AZ"=>"ap-northeast-1b", "Region"=>"ap-northeast-1"})
            .and_return(["ap-northeast-1b\nap-northeast-1\n", "", process_status1])
-      Open3.should_receive(:capture3).with("echo ap-northeast-1\necho ap-northeast-1b\n")
+      client.instance_variable_get(:@post_processing)
+           .should_receive(:run_shell_command)
+           .with("      echo <%= Key \"Region\" %>\n      echo <%= Key \"AZ\" %>\n", {"AZ"=>"ap-northeast-1b", "Region"=>"ap-northeast-1"})
            .and_return(["ap-northeast-1\nap-northeast-1b\n", "", process_status2])
 
       client.instance_variable_get(:@post_processing)
@@ -232,7 +236,9 @@ end
 
       cf.should_receive(:stacks) { stacks }
 
-      Net::SSH.should_receive(:start).with("127.0.0.1", "ec2-user")
+      client.instance_variable_get(:@post_processing)
+           .should_receive(:run_ssh_command)
+           .with({"host"=>"<%= Key \"PublicIp\" %>", "user"=>"ec2-user"}, "      ls\n", {"PublicIp"=>"127.0.0.1"})
            .and_return(["file1\nfile2\n", "", 0])
 
       client.instance_variable_get(:@post_processing)
@@ -336,9 +342,13 @@ end
       process_status1 = make_double('process_status1') {|obj| obj.should_receive(:to_i).and_return(0) }
       process_status2 = make_double('process_status2') {|obj| obj.should_receive(:to_i).and_return(0) }
 
-      Open3.should_receive(:capture3).with("echo ap-northeast-1b\necho ap-northeast-1\n")
+      client.instance_variable_get(:@post_processing)
+           .should_receive(:run_shell_command)
+           .with("      echo <%= Key \"AZ\" %>\n      echo <%= Key \"Region\" %>\n", {"AZ"=>"ap-northeast-1b", "Region"=>"ap-northeast-1"})
            .and_return(["ap-northeast-1b\nap-northeast-1\n", "", process_status1])
-      Open3.should_receive(:capture3).with("echo ap-northeast-1\necho ap-northeast-1b\n")
+      client.instance_variable_get(:@post_processing)
+           .should_receive(:run_shell_command)
+           .with("      echo <%= Key \"Region\" %>\n      echo <%= Key \"AZ\" %>\n", {"AZ"=>"ap-northeast-1b", "Region"=>"ap-northeast-1"})
            .and_return(["ap-northeast-1\nap-northeast-1b\n", "", process_status2])
 
       client.instance_variable_get(:@post_processing)

data/spec/string_stream_spec.rb

@@ -0,0 +1,123 @@
+describe Kumogata::StringStream do
+  it 'pass the line ("\n")' do
+    lines = []
+
+    sstream = Kumogata::StringStream.new do |line|
+      lines << line
+    end
+
+    sstream.push("chunk1")
+    sstream.push("chunk2\n")
+    sstream.push("chunk3")
+    sstream.push("chunk4")
+    sstream.push("chunk5\n")
+    sstream.push("\n")
+    sstream.push("\n")
+    sstream.push("chunk6")
+    sstream.push("chunk7")
+    sstream.close
+
+    expect(lines).to eq([
+      "chunk1chunk2",
+      "chunk3chunk4chunk5",
+      "",
+      "",
+      "chunk6chunk7",
+    ])
+  end
+
+  it 'pass the line ("\r")' do
+    lines = []
+
+    sstream = Kumogata::StringStream.new do |line|
+      lines << line
+    end
+
+    sstream.push("chunk1")
+    sstream.push("chunk2\r")
+    sstream.push("chunk3")
+    sstream.push("chunk4")
+    sstream.push("chunk5\r")
+    sstream.push("\r")
+    sstream.push("\r")
+    sstream.push("chunk6")
+    sstream.push("chunk7")
+    sstream.close
+
+    expect(lines).to eq([
+      "chunk1chunk2",
+      "chunk3chunk4chunk5",
+      "",
+      "",
+      "chunk6chunk7",
+    ])
+  end
+
+  it 'pass the line ("\r\n")' do
+    lines = []
+
+    sstream = Kumogata::StringStream.new do |line|
+      lines << line
+    end
+
+    sstream.push("chunk1")
+    sstream.push("chunk2\r\n")
+    sstream.push("chunk3")
+    sstream.push("chunk4")
+    sstream.push("chunk5\r\n")
+    sstream.push("\r\n")
+    sstream.push("\r\n")
+    sstream.push("chunk6")
+    sstream.push("chunk7")
+    sstream.close
+
+    expect(lines).to eq([
+      "chunk1chunk2",
+      "chunk3chunk4chunk5",
+      "",
+      "",
+      "chunk6chunk7",
+    ])
+  end
+
+  it 'pass the line ("\n" / "\r" / "\r\n")' do
+    lines = []
+
+    sstream = Kumogata::StringStream.new do |line|
+      lines << line
+    end
+
+    sstream.push("chunk1")
+    sstream.push("chunk2\n")
+    sstream.push("chunk3")
+    sstream.push("chunk4")
+    sstream.push("chunk5\r")
+    sstream.push("\r\n")
+    sstream.push("\n")
+    sstream.push("chunk6")
+    sstream.push("chunk7")
+    sstream.push("chunk1")
+    sstream.push("chunk2\r")
+    sstream.push("chunk3")
+    sstream.push("chunk4")
+    sstream.push("chunk5\n\r")
+    sstream.push("\n")
+    sstream.push("\r")
+    sstream.push("chunk6")
+    sstream.push("chunk7")
+    sstream.close
+
+    expect(lines).to eq([
+      "chunk1chunk2",
+      "chunk3chunk4chunk5",
+      "",
+      "",
+      "chunk6chunk7chunk1chunk2",
+      "chunk3chunk4chunk5",
+      "",
+      "",
+      "",
+      "chunk6chunk7",
+    ])
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kumogata
 version: !ruby/object:Gem::Version
-  version: 0.3.13
+  version: 0.3.14
 platform: ruby
 authors:
 - Genki Sugawara
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-11 00:00:00.000000000 Z
+date: 2014-03-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -231,6 +231,7 @@ files:
 - lib/kumogata/ext/string_ext.rb
 - lib/kumogata/logger.rb
 - lib/kumogata/post_processing.rb
+- lib/kumogata/string_stream.rb
 - lib/kumogata/utils.rb
 - lib/kumogata/version.rb
 - packer/CentOS-6.4-x86_64-with_Updates-Asia_Pacific.json
@@ -250,6 +251,7 @@ files:
 - spec/kumogata_update_spec.rb
 - spec/kumogata_validate_spec.rb
 - spec/spec_helper.rb
+- spec/string_stream_spec.rb
 homepage: https://github.com/winebarrel/kumogata
 licenses:
 - MIT
@@ -290,3 +292,4 @@ test_files:
 - spec/kumogata_update_spec.rb
 - spec/kumogata_validate_spec.rb
 - spec/spec_helper.rb
+- spec/string_stream_spec.rb