gina-conveyor 0.2.4 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b5e7532c41370b5895baf8dc2776eaef2531ce99
-  data.tar.gz: a2c2b104cd9ebbc166bc5d397cc4b0a93d660c13
+  metadata.gz: 08fc3174aa43d6d5eb64bfec83f61eadf806f0b3
+  data.tar.gz: fb2d81a0e75ffc1a620630831f9f8f1e9b5d37af
 SHA512:
-  metadata.gz: 3f310031700a8186a14dce0d7130898d3ada6bdaf0e27ea9db5d1223e654657ae7123d2d612bcf5961f7c8d48bd66ee9206042e0e1b1994a57d3517bc4bce22d
-  data.tar.gz: baacd19e877b28d60c7672b9f08123cadbb48bae2d694ba4b035a775cfd2d300a4e52f1778fb1dce8de6d312d4933bfffc6ab33c04f11b72de35e8592fed6920
+  metadata.gz: 589d9f31d1621aea95ac881210703e72a56165c8d53c7c59e472e2e4fb9c1677b3109d32fae779dfe7abf52ef21ab478b7fc81549b5a59bf6091352f96acb81d
+  data.tar.gz: 718daf0c2872ed754c6dfd24207dfac7ef6981678aa718c639c5a93e06be39916e38aa3a40107d5f82738047565964085fb3344c561831893e388df60b0b83da

data/.gitignore

@@ -6,4 +6,5 @@ log/*
 .DS_Store
 .workers
 pkg
-.conveyor
+.conveyor
+data_test

data/Gemfile.lock

@@ -1,22 +1,23 @@
 PATH
   remote: .
   specs:
-    gina-conveyor (0.2.3)
+    gina-conveyor (1.0.0)
       activemodel (~> 4.1.0)
       activesupport (~> 4.1.0)
       em-websocket (~> 0.5.1)
       eventmachine (~> 1.0.0)
       listen (~> 2.10)
+      mixlib-shellout (~> 2.2.1)
       rainbow (~> 2.0.0)
       rb-readline (~> 0.5.0)
 
 GEM
   remote: http://rubygems.org/
   specs:
-    activemodel (4.1.12)
-      activesupport (= 4.1.12)
+    activemodel (4.1.13)
+      activesupport (= 4.1.13)
       builder (~> 3.1)
-    activesupport (4.1.12)
+    activesupport (4.1.13)
       i18n (~> 0.6, >= 0.6.9)
       json (~> 1.7, >= 1.7.7)
       minitest (~> 5.1)
@@ -28,9 +29,9 @@ GEM
     em-websocket (0.5.1)
       eventmachine (>= 0.12.9)
       http_parser.rb (~> 0.6.0)
-    eventmachine (1.0.7)
+    eventmachine (1.0.8)
     ffi (1.9.10)
-    hitimes (1.2.2)
+    hitimes (1.2.3)
     http_parser.rb (0.6.0)
     i18n (0.7.0)
     json (1.8.3)
@@ -38,15 +39,16 @@ GEM
       celluloid (~> 0.16.0)
       rb-fsevent (>= 0.9.3)
       rb-inotify (>= 0.9)
-    minitest (5.7.0)
+    minitest (5.8.1)
+    mixlib-shellout (2.2.1)
     rainbow (2.0.0)
     rake (10.3.2)
-    rb-fsevent (0.9.5)
+    rb-fsevent (0.9.6)
     rb-inotify (0.9.5)
       ffi (>= 0.5.0)
     rb-readline (0.5.3)
     thread_safe (0.3.5)
-    timers (4.0.1)
+    timers (4.0.4)
       hitimes
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
@@ -57,3 +59,6 @@ PLATFORMS
 DEPENDENCIES
   gina-conveyor!
   rake
+
+BUNDLED WITH
+   1.10.6

data/bin/conveyor

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+require 'rubygems'
 require 'conveyor'
 
 Conveyor.start

data/conveyor.gemspec

@@ -27,4 +27,5 @@ Gem::Specification.new do |gem|
   gem.add_dependency('rb-readline', '~> 0.5.0')
   gem.add_dependency('eventmachine', '~> 1.0.0')
   gem.add_dependency('em-websocket', '~> 0.5.1')
+  gem.add_dependency('mixlib-shellout', '~> 2.2.1')
 end

data/lib/conveyor.rb

@@ -16,6 +16,7 @@ require 'fileutils'
 require 'eventmachine'
 require 'em-websocket'
 require "conveyor/version"
+require "mixlib/shellout"
 
 module Conveyor
   extend ActiveSupport::Autoload

data/lib/conveyor/foreman.rb

@@ -31,8 +31,9 @@ module Conveyor
         "worker_defs" => File.expand_path('.workers', Dir.pwd),
         "logfile" => File.expand_path('log/conveyor.log', Dir.pwd),
         "threadpool" => 5,
+        "command_timeout" => 600,
         "websocket" => {
-          "disabled" => false,
+          "disabled" => true,
           "host" => "0.0.0.0",
           "port" => 9876
         }

data/lib/conveyor/output/email.rb

@@ -11,11 +11,11 @@ module Conveyor
         def reset!
           @msg_queue = []
         end
-        
+
         def warning(*msg)
           # Do nothing
         end
-        
+
         def error(*msg)
           @msg_queue ||= []
           options = msg.extract_options!
@@ -23,20 +23,20 @@ module Conveyor
           msg =  msg.join("\n") if msg.class == Array
           @msg_queue << msg
         end
-        
+
         def write(msgtype, *msg)
           if respond_to?(msgtype)
             self.send(msgtype, *msg)
           end
         end
-        
+
         def mail
           return if @msg_queue.nil? || @msg_queue.empty?
-          puts "Sending email to #{Conveyor::Foreman.instance.notify_list}"
-          puts @msg_queue
+          # puts "Sending email to #{Conveyor::Foreman.instance.notify_list}"
+          # puts @msg_queue
           reset!
         end
       end
     end
   end
-end
+end

data/lib/conveyor/version.rb

@@ -1,3 +1,3 @@
 module Conveyor
-  VERSION = "0.2.4"
+  VERSION = "1.0.0"
 end

data/lib/conveyor/websocket.rb

@@ -2,13 +2,10 @@ module Conveyor
   class Websocket
     class << self
       def start
-        if config[:disable]
-          fm.info "Websocket disabled"
-          return
-        end
+        return if config[:disabled]
 
         fm.info "Starting websocket on #{config[:host]}:#{config[:port]}", :color => :green
-        
+
         EventMachine::start_server(config[:host], config[:port],
           EventMachine::WebSocket::Connection, config) do |ws|
           ws.onopen {
@@ -34,4 +31,4 @@ module Conveyor
       end
     end
   end
-end
+end

data/lib/conveyor/worker.rb

@@ -7,7 +7,7 @@ module Conveyor
   class Worker
     include Conveyor::Output
     include Conveyor::Workers::Syntax
-    
+
     attr_accessor :filename
     attr_reader :status
     attr_reader :worker_def
@@ -32,13 +32,13 @@ module Conveyor
       dir = File.dirname(worker_def)
       File.expand_path(File.basename(worker_def, '.worker') + '.log', dir)
     end
-    
+
     # Catch any calls to error and set the status fail flags
     def error(*msg)
       opts = msg.extract_options!
       unless msg.flatten.empty?
         @status.fail!
-        msg.unshift("Error encountered in #{worker_def}")
+        # msg.unshift("Error encountered in #{worker_def}")
         super(*msg, opts)
       end
       @status.success?
@@ -58,14 +58,14 @@ module Conveyor
         if @status.success?
           info "Completed #{@filename}, #{@elapsed}s elapsed", :color => :green
         else
-          error "Error(s) encountered in #{@filename}", :color => :red
+          error "Error encountered in #{@worker_def} -> #{@filename}", :color => :red
         end
         send_notifications
       end
     end
-  
+
     protected
-    
+
     def create_dirs_for_cmd(src, dest)
       if src.is_a?(Array) || Array.wrap(src).count > 1 || dest.last == ?/
         mkdir(dest)
@@ -73,34 +73,34 @@ module Conveyor
         mkdir(File.dirname(dest))
       end
     end
-    
+
     def verified_cmd(cmd, src, dest, &block)
       create_dirs_for_cmd(src, dest)
       dest = File.expand_path(dest)
-      
+
       Array.wrap(src).each do |s|
         run "#{cmd} #{s} #{dest}"
         sync
-        
+
         if block_given?
           result = yield(s, dest)
           @status.fail! unless result
         end
       end
     end
-    
+
     def verified_move(src, dest)
       verified_cmd(:mv, src, dest) do |src,dest|
         verify_move(src,dest)
       end
     end
-        
+
     def verified_copy(src, dest)
       verified_cmd(:cp, src, dest) do |src,dest|
         verify_copy(src,dest)
       end
     end
-    
+
     def verify_copy(src, dest)
       if File.directory? dest
         File.exists?(File.join(dest, File.basename(src)))
@@ -108,7 +108,7 @@ module Conveyor
         File.exists?(dest)
       end
     end
-    
+
     def verify_move(src, dest)
       if File.directory? dest
         File.exists?(File.join(dest, File.basename(src))) && !File.exists?(src)

data/lib/conveyor/workers/syntax.rb

@@ -1,18 +1,18 @@
 module Conveyor
   module Workers
     module Syntax
-      
+
       # Will return a string of any method returned that isn't handled
       # ex: match extension txt => match(extension("foo"))
       def method_missing(method, value = nil)
         return method.to_s
-      end      
-      
+      end
+
       # Returns a recursive file glob string for the passed in string
       def file(glob)
         "**/#{glob}"
       end
-    
+
       # Returns an extension glob string for passed in string
       def extension(glob)
         "*.#{glob}"
@@ -22,7 +22,7 @@ module Conveyor
       def any
         '*'
       end
-      
+
       # Returns a list of files that have the same basename but different extension
       # in the same directory
       def like(name)
@@ -34,7 +34,7 @@ module Conveyor
       def filename
         @filename
       end
-      
+
       # Which directories to watch for file change events.
       def watch(*args, &block)
         yield
@@ -44,42 +44,49 @@ module Conveyor
       def match(glob, &block)
         yield @filename
       end
-      
+
       # Run the system sync command
       def sync
         run 'sync', :quiet => true
       end
-    
+
       # Change current working directory, optionally takes a block
       # NOTE: Consider removing this as it can cause problems with threaded workers
       def chdir(dir, &block)
         Dir.chdir(File.expand_path(dir), &block)
       end
-      
-      # Run the system command, and make sure that any errors are caught and returne
+
+      # Run the system command, and make sure that any errors are caught
       # up the status change
-      def run(*cmd)
-        opts = cmd.extract_options!
+      def run(*params)
+        opts = params.extract_options!
+        command = Array.wrap(params).join(' ')
+        info command unless opts[:quiet]
+
         begin
-          info cmd.join(' ') unless opts[:quiet]
-          output,err,thr = Open3.capture3(Array.wrap(cmd).join(' '))
-          info output.chomp unless output.chomp.length == 0
-  				if thr.success?
-  					if err.chomp.length > 0
+          cmdrunner = Mixlib::ShellOut.new(command, timeout: Conveyor::Foreman.instance.config[:command_timeout] || 600)
+          cmdrunner.run_command()
+
+          info cmdrunner.stdout.chomp unless cmdrunner.stdout.chomp.length == 0
+
+  				if cmdrunner.error!
+          	error "Error running: `#{command}`", cmdrunner.stderr.chomp
+          	@status.fail!
+  				else
+  					if cmdrunner.stderr.chomp.length > 0
   						warning "Error output recieved, but no error code recieved"
-  						warning err.chomp
+  						warning cmdrunner.stderr.chomp
   					end
-  				else
-          	error "Error running: `#{cmd.join(' ')}`", err.chomp 
-          	@status.fail!
   				end
-        
-          return thr.success?
+
+          return !cmdrunner.error!
+        rescue Mixlib::ShellOut::CommandTimeout => e
+          error e.class, "Timeout running: #{command}"
         rescue => e
           error e.class, e.message, e.backtrace.join("\n")
         end
       end
-      
+
       # Deletes passed in files
       def delete(files)
         # sync before we delete
@@ -90,10 +97,10 @@ module Conveyor
           error "#{f} wasn't removed" if File.exists?(f)
         end
       end
-    
+
       # Create a new directory
       def mkdir(dir)
-        FileUtils.mkdir_p(File.expand_path(dir))      
+        FileUtils.mkdir_p(File.expand_path(dir))
         @status.fail! unless File.exists?(File.expand_path(dir))
       end
 
@@ -101,22 +108,22 @@ module Conveyor
       def copy(src = [], dest = nil)
         destination = dest unless dest.nil?
         source = src unless src.empty?
-      
+
         if source && destination
           verified_copy(source, destination)
-        end      
+        end
       end
-    
+
       # Move files to destination
       def move(src=[], dest = nil)
         destination = dest unless dest.nil?
         source = src unless src.empty?
-      
+
         if source && destination
           verified_move(source, destination)
         end
       end
-  
+
       # Scp files to destination
       # See: man scp
       def scp(src, dest)
@@ -124,4 +131,4 @@ module Conveyor
       end
     end
   end
-end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gina-conveyor
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 1.0.0
 platform: ruby
 authors:
 - Will Fisher
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-24 00:00:00.000000000 Z
+date: 2015-09-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -122,6 +122,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.5.1
+- !ruby/object:Gem::Dependency
+  name: mixlib-shellout
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.2.1
 description: Conveyor is used for shuffling data around
 email:
 - will@gina.alaska.edu