naf 2.1.3 → 2.1.4

data/app/models/logical/naf/log_parser/runner.rb

@@ -52,7 +52,7 @@ module Logical::Naf
 
       def get_invocation_id(uuid)
         if invocations_ids[uuid].blank?
-          @invocations_ids[uuid] = ::Naf::MachineRunnerInvocation.find_by_uuid(uuid).id
+          @invocations_ids[uuid] = ::Naf::MachineRunnerInvocation.find_by_uuid(uuid).try(:id)
         end
 
         invocations_ids[uuid]

data/app/models/naf/application_type.rb

@@ -7,6 +7,7 @@ module Naf
                     :invocation_method
 
     SCRIPT_RUNNER = "#{Gem.ruby} #{Rails.root}/script/rails runner"
+    JOB_LOGGER = "#{Rails.root}/script/rails runner ::Process::Naf::Logger::JobLog.run"
 
     #---------------------
     # *** Associations ***
@@ -32,9 +33,9 @@ module Naf
       self.send(invocation_method.to_sym, job)
     end
 
-    def invoke(job, command)
-      ENV['NAF_JOB_ID'] = job.id.to_s
-      Open4::popen4(command)
+    def invoke(job, job_command)
+      command = job_command + " 2>&1 | #{JOB_LOGGER} >> #{LOGGING_ROOT_DIRECTORY}/naf/crash.log 2>&1"
+      Process.spawn({ 'NAF_JOB_ID' => job.id.to_s }, command)
     end
 
     def rails_invocator(job)

data/app/models/process/naf/logger/base.rb

@@ -0,0 +1,22 @@
+require 'open4'
+
+module Process::Naf::Logger
+  class Base < ::Af::Application
+
+    def work
+      log_file = ::Logical::Naf::LogFile.new(file_path)
+      log_file.open
+
+      while $stdin.gets
+        begin
+          log_file << $_.rstrip
+        ensure
+          log_file.write
+        end
+      end
+
+      log_file.close
+    end
+
+  end
+end

data/app/models/process/naf/logger/job_log.rb

@@ -0,0 +1,9 @@
+module Process::Naf::Logger
+  class JobLog < Base
+
+    def file_path
+      "#{::Naf::PREFIX_PATH}/#{::Naf.schema_name}/jobs/#{ENV['NAF_JOB_ID']}/"
+    end
+
+  end
+end

data/app/models/process/naf/logger/runner_log.rb

@@ -0,0 +1,13 @@
+module Process::Naf::Logger
+  class RunnerLog < Base
+
+    opt :invocation_uuid,
+        "unique identifer used for runner logs",
+        default: `uuidgen`
+
+    def file_path
+      "#{::Naf::PREFIX_PATH}/#{::Naf.schema_name}/runners/#{@invocation_uuid}/"
+    end
+
+  end
+end

data/app/models/process/naf/runner.rb

@@ -184,7 +184,6 @@ module Process::Naf
       logger.info escape_html("working: #{machine}")
 
       @children = {}
-      @threads = {}
 
       at_exit {
         ::Af::Application.singleton.emergency_teardown
@@ -349,11 +348,6 @@ module Process::Naf
           logger.info { escape_html("cleaning up dead child: #{child_job.reload}") }
           finish_job(child_job,
                      { exit_status: (status && status.exitstatus), termination_signal: (status && status.termsig) })
-
-          thread = @threads.delete(pid)
-          logger.detail escape_html("cleaning up threads: #{thread.inspect}")
-          logger.detail escape_html("thread list: #{Thread.list}")
-          thread.join
         else
           # this can happen if the child is sigstopped
           logger.warn escape_html("child waited for did not exit: #{child_job}, status: #{status.inspect}")
@@ -382,13 +376,8 @@ module Process::Naf
 
           logger.info escape_html("starting new job : #{running_job.inspect}")
 
-          # fork and run
-          pid, stdin, stdout, stderr = running_job.historical_job.spawn
-          stdin.close
-
-          # Reset NAF_JOB_ID
-          ENV.delete('NAF_JOB_ID')
-          if pid
+          pid = running_job.historical_job.spawn
+          if pid.present?
             @children[pid] = running_job
             running_job.pid = pid
             running_job.historical_job.pid = pid
@@ -397,17 +386,6 @@ module Process::Naf
             logger.info escape_html("job started : #{running_job}")
             running_job.save!
             running_job.historical_job.save!
-
-            # Spawn a thread to output the log of each job to files.
-            #
-            # Make sure not to execute any database calls inside this
-            # block, as it will start an ActiveRecord connection for each
-            # thread and eventually raise a ConnetionTimeoutError, resulting
-            # the runner to exit.
-            thread = Thread.new do
-              log_output_until_job_finishes(running_job.id, stdout, stderr)
-            end
-            @threads[pid] = thread
           else
             # should never get here (well, hopefully)
             logger.error escape_html("#{machine}: failed to execute #{running_job}")
@@ -425,55 +403,6 @@ module Process::Naf
       logger.debug_gross "done starting jobs"
     end
 
-    def log_output_until_job_finishes(job_id, stdout, stderr)
-      log_file = ::Logical::Naf::LogFile.new("#{::Naf::PREFIX_PATH}/#{::Naf.schema_name}/jobs/#{job_id}")
-      log_file.open
-
-      # Continue reading logs from stdout/stderror until it reaches end of file
-      while true
-        read_pipes = []
-        read_pipes << stdout if stdout
-        read_pipes << stderr if stderr
-        return if (read_pipes.length == 0)
-
-        error_pipes = read_pipes.clone
-        read_array, write_array, error_array = Kernel.select(read_pipes, nil, error_pipes, 1)
-
-        unless error_array.blank?
-          logger.error escape_html("job(#{job_id}): select returned error for #{error_pipes.inspect} (read_pipes: #{read_pipes.inspect})")
-          # XXX we should probably close the errored FDs
-        end
-
-        unless read_array.blank?
-          begin
-            for r in read_array do
-              begin
-                # Parse each log line into JSON
-                r.read_nonblock(10240).split("\n").each do |log|
-                  log_file << log.rstrip
-                end
-              rescue Errno::EAGAIN => e
-                logger.error 'EAGAIN'
-                logger.error "#{e.inspect}"
-                logger.error "#{e.message}"
-              rescue Errno::EWOULDBLOCK => ew
-                logger.error 'EWOULDBLOCK'
-                logger.error "#{ew.inspect}"
-                logger.error "#{ew.message}"
-              rescue EOFError => eof
-                stdout = nil if r == stdout
-                stderr = nil if r == stderr
-              end
-            end
-          ensure
-            log_file.write
-          end
-        end
-      end
-
-      log_file.close
-    end
-
     # XXX update_all doesn't support "from_partition" so we have this helper
     def update_historical_job(updates, historical_job_id)
       updates[:updated_at] = Time.zone.now

data/bin/naf

@@ -6,7 +6,7 @@ if action == 'runner'
   if option == 'up'
     puts "Bringing up the runner(s)..."
     `screen -d -m bash -c 'source /root/.bash_profile && cd /root/current && a=\`uuidgen\` && script/rails runner ::Process::Naf::Runner.run --invocation-uuid $a 2>&1 |
-      script/rails runner ::Process::Naf::RunnerLog.run --invocation-uuid $a'`
+      script/rails runner ::Process::Naf::Logger::RunnerLog.run --invocation-uuid $a'`
 
   elsif option == 'status'
     num_runners = Integer(`ps -ef | grep Process::Naf::Runner.run | grep -v grep | grep -v uuidgen | wc -l`.strip)

data/lib/naf/version.rb

@@ -1,3 +1,3 @@
 module Naf
-  VERSION = "2.1.3"
+  VERSION = "2.1.4"
 end

data/naf.gemspec

@@ -8,7 +8,7 @@ Gem::Specification.new do |s|
   s.name        = "naf"
   s.version     = Naf::VERSION
   s.license     = 'New BSD License'
-  s.date        = '2014-02-20'
+  s.date        = '2014-02-25'
   s.summary     = "Creates infrastructure for a customizable and robust Postgres-backed script scheduling/running"
   s.description = "A cloud based distributed cron, application framework and operations console. Naf works as a distributed script running " +
                   "system that provides scheduling, logging, alarming, machine redundancy, and the ability to set constraint during script execution"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: naf
 version: !ruby/object:Gem::Version
-  version: 2.1.3
+  version: 2.1.4
   prerelease: 
 platform: ruby
 authors:
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-20 00:00:00.000000000 Z
+date: 2014-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -320,10 +320,12 @@ files:
 - app/models/process/naf/janitor.rb
 - app/models/process/naf/log_archiver.rb
 - app/models/process/naf/log_reader.rb
+- app/models/process/naf/logger/base.rb
+- app/models/process/naf/logger/job_log.rb
+- app/models/process/naf/logger/runner_log.rb
 - app/models/process/naf/machine_manager.rb
 - app/models/process/naf/machine_upgrader.rb
 - app/models/process/naf/runner.rb
-- app/models/process/naf/runner_log.rb
 - app/views/naf/affinities/_form.html.erb
 - app/views/naf/affinities/edit.html.erb
 - app/views/naf/affinities/index.html.erb

data/app/models/process/naf/runner_log.rb

@@ -1,26 +0,0 @@
-require 'open4'
-
-module Process::Naf
-  class RunnerLog < ::Af::Application
-
-    opt :invocation_uuid,
-        "unique identifer used for runner logs",
-        default: `uuidgen`
-
-    def work
-      log_file = ::Logical::Naf::LogFile.new("#{::Naf::PREFIX_PATH}/#{::Naf.schema_name}/runners/#{@invocation_uuid}/")
-      log_file.open
-
-      while $stdin.gets
-        begin
-          log_file << $_.rstrip
-        ensure
-          log_file.write
-        end
-      end
-
-      log_file.close
-    end
-
-  end
-end