blower 4.8 → 5.0a1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 64719d3d0e7dadbea696e04130f8a96141c3567c195eb456a42e2c0805924f78
-  data.tar.gz: b73b17da21d56ff6a7ed17da0d64906ae8639201d29136a1c8f0f90ed03f04a6
+  metadata.gz: 7e86119a63baca69801f4556a37de41ace91bcb5a877d3c82e6fe15cb569f512
+  data.tar.gz: ef8affaeb03b9eaf5cba800f18bf43a80bcb2a86d59de734744926eb9441e7c8
 SHA512:
-  metadata.gz: 8f3f6f1b20a32c90e11a8ef22ef5f01454b876342ccd4a82a063be3d9fcd6e3dcb1be1fc52634022305e604b6e868b3d4a2f15d8366b14fb7f903f3930a9d692
-  data.tar.gz: '0970f7c8a32e89e0f549a318542b22969264d03d638560bfec66c748b240cef36746aecb5bf07045e3067f43e6ccda2db9ffa407a52e1622f0daaf286e2fc680'
+  metadata.gz: 27bba3e972d839c4f1b0a0469e0abff4a6ad007cfc6d87e03596604e2a4025db007ccd2b3e679901440134ad598d09312af448b1f880a7ae50e282ce498b45ee
+  data.tar.gz: b3e698f5285034a22b44e66dd1bfff3b987dbae6af6bc5f840a43fe1bef1cafc3cc1644b19fd69a8c1b211ec979f308e6fa1135905830ed796970d9cd3ba5909

data/bin/blow

@@ -30,6 +30,7 @@ if File.directory?(File.join(Dir.pwd, "lib"))
 end
 
 context.run "Blowfile", optional: true
+
 begin
   until ARGV.empty?
     context.run ARGV.shift

data/lib/blower.rb

@@ -8,5 +8,7 @@ require 'blower/logger'
 require 'blower/errors'
 require 'blower/context'
 require 'blower/host'
+require 'blower/local'
+require 'blower/target'
 require 'blower/plugin'
 require 'blower/version'

data/lib/blower/context.rb

@@ -1,3 +1,4 @@
+require 'colorize'
 require 'find'
 require 'erb'
 require 'json'
@@ -32,6 +33,9 @@ module Blower
     # The target hosts.
     attr_accessor :hosts
 
+    # The failed hosts.
+    attr_accessor :failures
+
     # Username override. If not-nil, this user is used for all remote accesses.
     attr_accessor :user
 
@@ -45,6 +49,7 @@ module Blower
       @path = path
       @data = {}
       @hosts = []
+      @failures = []
     end
 
     # Return a context variable.
@@ -115,12 +120,20 @@ module Blower
     # @raise Whatever the task itself raises.
     # @return The result of evaluating the task file.
     def run (task, optional: false, quiet: false, once: nil)
+      @run_cache ||= {}
       once once, quiet: quiet do
         log.info "run #{task}", quiet: quiet do
           file = find_task(task)
-          code = File.read(file)
-          let :@file => file do
-            instance_eval(code, file)
+          if @run_cache.has_key? file
+            log.info "*cached*"
+            @run_cache[file]
+          else
+            @run_cache[file] = begin
+              code = File.read(file)
+              let :@file => file do
+                instance_eval(code, file)
+              end
+            end
           end
         end
       end
@@ -136,7 +149,7 @@ module Blower
     # @macro onceable
     def sh (command, as: user, on: hosts, quiet: false, once: nil)
       self.once once, quiet: quiet do
-        log.info "sh #{command}", quiet: quiet do
+        log.info "sh: #{command}", quiet: quiet do
           hash_map(hosts) do |host|
             host.sh command, as: as, quiet: quiet
           end
@@ -282,7 +295,7 @@ module Blower
 
       def to_s
         map do |host, data|
-          "#{host.name.blue} (#{host.address.green})\n" + data.strip.to_s.gsub(/^/, "  ")
+          "#{host.name.blue}\n" + data.strip.to_s.gsub(/^/, "  ")
         end.join("\n")
       end
 
@@ -302,40 +315,80 @@ module Blower
     end
 
     def hash_map (hosts = self.hosts)
-      HostHash.new.tap do |result|
-        each(hosts) do |host|
-          result[host] = yield(host)
+      hh = HostHash.new.tap do |result|
+        each(hosts) do |host, i|
+          result[host] = yield(host, i)
         end
       end
+      if @singular
+        hh.values.first
+      else
+        hh
+      end
+    end
+
+    def singularly (flag = true)
+      was, @singular = @singular, flag
+      yield
+    ensure
+      @singular = was
+    end
+
+    def on_one (host = self.hosts.sample, serial: true)
+      ret = nil
+      each([host], serial: serial) do |host, i|
+        on host do
+          singularly do
+            ret = yield host, i
+          end
+        end
+      end
+      ret
     end
 
     def on_each (hosts = self.hosts, serial: true)
-      each(hosts, serial: serial) do |host|
+      each(hosts, serial: serial) do |host, i|
         on host do
-          yield host
+          singularly do
+            yield host, i
+          end
         end
       end
     end
 
+    def locally (&block)
+      on Local.new("<local>") do
+        singularly &block
+      end
+    end
+
     def each (hosts = self.hosts, serial: false)
       fail "No hosts" if hosts.empty?
-      if false
-        [hosts].flatten.each do |host|
-          yield host
-        end
-      else
-        threads = [hosts].flatten.map do |host|
-          Thread.new do
-            begin
-              yield host
-            rescue => e
-              host.log.error e.full_message
-              hosts.delete host
-            end
+      q = (@threads || serial) && Queue.new
+      if q && serial
+        q.push nil
+      elsif q
+        @threads.times { q.push nil }
+      end
+      indent = Thread.current[:indent]
+      i = -1
+      threads = [hosts].flatten.map.with_index do |host|
+        Thread.new do
+          Thread.current[:indent] = indent
+          begin
+            q.pop if q
+            yield host, i += 1
+          rescue => e
+            host.log.error e.message
+            hosts.delete host
+            @failures |= [host]
+          ensure
+            q.push nil if q
+            sleep @delay if @delay
           end
         end
-        threads.each(&:join)
       end
+      threads.each(&:join)
       fail "No hosts remaining" if hosts.empty?
     end
 

data/lib/blower/host.rb

@@ -2,7 +2,6 @@ require 'net/ssh'
 require 'net/ssh/gateway'
 require 'net/scp'
 require 'monitor'
-require 'colorize'
 require 'base64'
 require 'timeout'
 
@@ -127,7 +126,7 @@ module Blower
               end
               ch.on_extended_data do |_, _, data|
                 log.trace "received #{data.bytesize} bytes stderr", quiet: quiet
-                output << data.colorize(:red)
+                output << data
               end
               ch.on_request("exit-status") do |_, data|
                 result = data.read_long

data/lib/blower/local.rb

@@ -0,0 +1,41 @@
+require 'net/ssh'
+require 'net/ssh/gateway'
+require 'net/scp'
+require 'monitor'
+require 'base64'
+require 'timeout'
+
+module Blower
+
+  class Local
+    include MonitorMixin
+    extend Forwardable
+
+    attr_reader :name, :data
+
+    def_delegators :data, :[], :[]=
+
+    def initialize (name, proxy: nil)
+      @name, @proxy = name, proxy
+      @data = {}
+    end
+
+    # Represent the host as a string.
+    def to_s
+      @name
+    end
+
+    def sh (command, as: nil, quiet: false)
+      command = "#{@proxy} #{command.shellescape}" if @proxy
+      IO.popen(command).read
+    end
+
+    # Produce a Logger prefixed with the host name.
+    # @api private
+    def log
+      @log ||= Logger.instance.with_prefix("on #{name}: ")
+    end
+
+  end
+
+end

data/lib/blower/logger.rb

@@ -1,3 +1,4 @@
+require 'colorize'
 require "singleton"
 
 module Blower
@@ -33,12 +34,11 @@ module Blower
       attr_accessor :indent
     end
 
-    self.indent = 0
-
     self.level = :info
 
     def initialize (prefix = "")
       @prefix = prefix
+      thread[:indent] = 0
       super()
     end
 
@@ -49,10 +49,10 @@ module Blower
 
     # Yield with a temporarily incremented indent counter
     def with_indent ()
-      Logger.indent += 1
+      thread[:indent] += 1
       yield
     ensure
-      Logger.indent -= 1
+      thread[:indent] -= 1
     end
 
     # Display a log message. The block, if specified, is executed in an indented region after the log message is shown.
@@ -64,9 +64,10 @@ module Blower
     def log (level, message, quiet: false, &block)
       if !quiet && (LEVELS.index(level) >= LEVELS.index(Logger.level))
         synchronize do
+          message = message.to_s.colorize(COLORS[level]) if level
           message = message.to_s.colorize(COLORS[level]) if level
           message.split("\n").each do |line|
-            STDERR.puts "  " * Logger.indent + @prefix + line
+            STDERR.puts "  " * thread[:indent] + @prefix + line
           end
         end
         with_indent(&block) if block
@@ -95,6 +96,10 @@ module Blower
     define_helper :error
     define_helper :fatal
 
+    def thread
+      Thread.current
+    end
+
   end
 
 end

data/lib/blower/target.rb

@@ -0,0 +1,106 @@
+require 'monitor'
+require 'base64'
+require 'timeout'
+require 'open3'
+
+module Blower
+
+  class Target
+    include MonitorMixin
+    extend Forwardable
+
+    attr_reader :name, :data
+
+    def_delegators :data, :[], :[]=
+
+    def initialize (name, ssh: "ssh", scp: "scp")
+      @name, @ssh, @scp = name, ssh, scp
+      @data = {}
+      super()
+    end
+
+    # Represent the host as a string.
+    def to_s
+      @name
+    end
+
+    # Copy files or directories to the host.
+    # @api private
+    def cp (froms, to, as: nil, quiet: false, delete: false)
+      as ||= @user
+      output = ""
+      synchronize do
+        [froms].flatten.each do |from|
+          if from.is_a?(String)
+            to += "/" if to[-1] != "/" && from.is_a?(Array)
+            command = ["rsync", "-e", @ssh, "-r"]
+            if File.exist?(".blowignore")
+              command += ["--exclude-from", ".blowignore"]
+            end
+            command += ["--delete"] if delete
+            command += [*from, ":#{to}"]
+            log.trace command.shelljoin, quiet: quiet
+            IO.popen(command, in: :close, err: %i(child out)) do |io|
+              until io.eof?
+                begin
+                  output << io.read_nonblock(100)
+                rescue IO::WaitReadable
+                  IO.select([io])
+                  retry
+                end
+              end
+              io.close
+              if !$?.success?
+                log.fatal "exit status #{$?.exitstatus}: #{command}", quiet: quiet
+                log.fatal output, quiet: quiet
+                fail "failed to copy files"
+              end
+            end
+          elsif from.respond_to?(:read)
+            cmd = "echo #{Base64.strict_encode64(from.read).shellescape} | base64 -d > #{to.shellescape}"
+            sh cmd, quiet: quiet
+          else
+            fail "Don't know how to copy a #{from.class}: #{from}"
+          end
+        end
+      end
+      true
+    end
+
+    def sh (command, as: nil, quiet: false)
+      marker, output = SecureRandom.hex(32), nil
+      ssh do |i, o, _|
+        i.puts "echo #{marker}"
+        i.puts "sh -c #{command.shellescape} 2>&1"
+        i.puts "STATUS_#{marker}=$?"
+        i.puts "echo #{marker}"
+        i.flush
+        o.readline("#{marker}\n")
+        output = o.readline("#{marker}\n")[0..-(marker.length + 2)]
+        i.puts "echo $STATUS_#{marker}"
+        status = o.readline.to_i
+        if status != 0
+          fail FailedCommand, output
+        end
+        output
+      end
+    end
+
+    # Produce a Logger prefixed with the host name.
+    # @api private
+    def log
+      @log ||= Logger.instance.with_prefix("on #{name}: ")
+    end
+
+    private
+
+    def ssh
+      unless @wait
+        @stdin, @stdout, @stderr, @wait = Open3.popen3(@ssh)
+      end
+      yield @stdin, @stdout, @stderr
+    end
+
+  end
+
+end

data/lib/blower/version.rb

@@ -1,3 +1,3 @@
 module Blower
-  VERSION = "4.8"
+  VERSION = "5.0a1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: blower
 version: !ruby/object:Gem::Version
-  version: '4.8'
+  version: 5.0a1
 platform: ruby
 authors:
 - Nathan Baum
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-27 00:00:00.000000000 Z
+date: 2019-02-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: net-ssh
@@ -120,8 +120,10 @@ files:
 - lib/blower/context.rb
 - lib/blower/errors.rb
 - lib/blower/host.rb
+- lib/blower/local.rb
 - lib/blower/logger.rb
 - lib/blower/plugin.rb
+- lib/blower/target.rb
 - lib/blower/util.rb
 - lib/blower/version.rb
 homepage: http://www.github.org/nbaum/blower
@@ -139,12 +141,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 3.0.2
 signing_key: 
 specification_version: 4
 summary: Really simple server orchestration