tumugi 0.6.1 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d097fbd84b2e0cbe35d75c307cfaabf6f3627aee
-  data.tar.gz: 3d08f180dac47101cc308b7574d4188365b9f1bc
+  metadata.gz: e012701afe6c29a35dd59f9c380c7a2e8b350c76
+  data.tar.gz: 2a2e156e12b4977a3148a8e6d8fd5329d7691083
 SHA512:
-  metadata.gz: 53f05ad5df3653c4effe4d22a4eb49f82c8f862c2fa81572f837ddbd0af5a1ed56e1ecdf6acd1b19d661aa8699bc1640488fbd2095075ee1f09d336a19be0157
-  data.tar.gz: 7f70098d0f37930fd9053753dbd339edd42812b30f44edd7f0ce1c4408133c584a38c83e964fc19ac8e0cbc6b5e5896ec8880981f4fcc4fcd20399b8c9a21f92
+  metadata.gz: 3fa3d8aa768b6a70da18cd765cacbb8dedf2186d9b64d9784dcefe03cd13e422e24021a92a408904385861c29f986a748ecaf8780bd4455427cb584da6f5e531
+  data.tar.gz: 219ba5614c9726fbd5c06bac22c735ed6056f7d8a0a808519bbd4040056a4736be36506303390b6d3d54667c47c5bfa5e68975298437c81da583941d9c7143cd

data/CHANGELOG.md

@@ -1,10 +1,26 @@
 # Change Log
 
-## [0.6.1](https://github.com/tumugi/tumugi/tree/0.6.1) (2016-07-11)
-[Full Changelog](https://github.com/tumugi/tumugi/compare/v0.6.0...0.6.1)
+## [v0.6.2](https://github.com/tumugi/tumugi/tree/v0.6.2) (2016-08-02)
+[Full Changelog](https://github.com/tumugi/tumugi/compare/v0.6.1...v0.6.2)
+
+**Implemented enhancements:**
+
+- Improve Log output [\#105](https://github.com/tumugi/tumugi/issues/105)
+
+**Fixed bugs:**
+
+- tumugi should abort when config file not found [\#108](https://github.com/tumugi/tumugi/issues/108)
+
+**Merged pull requests:**
+
+- Improve and change logging [\#107](https://github.com/tumugi/tumugi/pull/107) ([hakobera](https://github.com/hakobera))
+
+## [v0.6.1](https://github.com/tumugi/tumugi/tree/v0.6.1) (2016-07-10)
+[Full Changelog](https://github.com/tumugi/tumugi/compare/v0.6.0...v0.6.1)
 
 **Merged pull requests:**
 
+- Prepare release for 0.6.1 [\#103](https://github.com/tumugi/tumugi/pull/103) ([hakobera](https://github.com/hakobera))
 - Suppress unusable info log [\#102](https://github.com/tumugi/tumugi/pull/102) ([hakobera](https://github.com/hakobera))
 
 ## [v0.6.0](https://github.com/tumugi/tumugi/tree/v0.6.0) (2016-07-09)

data/lib/tumugi.rb

@@ -1,8 +1,8 @@
 require 'tumugi/workflow'
 require 'tumugi/config'
 require 'tumugi/error'
-require 'tumugi/logger'
 require 'tumugi/version'
+require 'tumugi/logger/logger'
 
 module Tumugi
   class << self

data/lib/tumugi/cli.rb

@@ -9,7 +9,7 @@ module Tumugi
     class << self
       def common_options
         option :file, aliases: '-f', desc: 'Workflow file name', required: true
-        option :config, aliases: '-c', desc: 'Configuration file name', default: 'tumugi_config.rb'
+        option :config, aliases: '-c', desc: 'Configuration file name'
         option :params, aliases: '-p', type: :hash, desc: 'Task parameters'
         option :quiet, type: :boolean, desc: 'Suppress log', default: false
         option :verbose, type: :boolean, desc: 'Show verbose log', default: false
@@ -53,39 +53,45 @@ module Tumugi
     private
 
     def execute(command, task, options)
+      args = { task: task, options: options }
       success = Tumugi.workflow.execute(command, task, options)
       unless success
-        raise Thor::Error, "execute finished, but return it's failed"
+        raise Thor::Error, "execute finished, but failed"
       end
-      logger.info "status: success, command: #{command}, task: #{task}, options: #{options}"
+      logger.info "status: success, command: #{command}, args: #{args}"
     rescue => e
-      logger.error "#{command} command failed"
-      logger.error e.message
-      if options[:verbose]
-        e.backtrace.each { |line| logger.debug line }
-      else
-        logger.error "If you want to know more detail, run with '--verbose' option"
-      end
-      logger.error "status: failed, command: #{command}, task: #{task}, options: #{options}"
-      raise Thor::Error.new("tumugi #{command} failed, please check log")
+      handle_error(command, e, args)
     end
 
     def generate_plugin(name, options)
+      args = { name: name, options: options }
       Tumugi::Command::New.new.execute(name, options)
-      logger.info "status: success, command: new, name: #{name}, options: #{options}"
+      logger.info "status: success, command: new, args: #{args}"
     rescue => e
+      handle_error("new", e, args)
+    end
+
+    private
+
+    def handle_error(command, e, args)
+      logger.error "#{command} command failed"
       logger.error e.message
+      reason = e
+      if e.is_a?(Tumugi::TumugiError) && !e.reason.nil?
+        reason = e.reason
+        logger.error reason.message
+      end
       if options[:verbose]
-        e.backtrace.each { |line| logger.debug line }
+        logger.error { reason.backtrace.join("\n") }
       else
         logger.error "If you want to know more detail, run with '--verbose' option"
       end
-      logger.error "status: failed, command: new, name: #{name}, options: #{options}"
+      logger.error "status: failed, command: #{command}, args: #{args}"
       raise Thor::Error.new("tumugi new failed, please check log")
     end
 
     def logger
-      Tumugi::Logger.instance
+      @logger ||= Tumugi::ScopedLogger.new("tumugi-main")
     end
   end
 end

data/lib/tumugi/command/new.rb

@@ -47,7 +47,7 @@ module Tumugi
       end
 
       def logger
-        Tumugi::Logger.instance
+        @logger ||= Tumugi::ScopedLogger.new("tumugi-new")
       end
 
       private

data/lib/tumugi/command/run.rb

@@ -6,7 +6,7 @@ module Tumugi
     class Run
       def execute(dag, options={})
         worker_num = options[:workers] || Tumugi.config.workers
-        executor = Tumugi::Executor::LocalExecutor.new(dag, logger, worker_num: worker_num)
+        executor = Tumugi::Executor::LocalExecutor.new(dag, worker_num: worker_num)
         result = start(executor)
         show_result_report(dag)
         result
@@ -15,9 +15,9 @@ module Tumugi
       private
 
       def start(executor)
-        logger.info "start workflow: #{Tumugi.workflow.id}"
+        logger.info "workflow_start: #{Tumugi.workflow.id}"
         result = executor.execute
-        logger.info "end workflow: #{Tumugi.workflow.id}"
+        logger.info "workflow_end: #{Tumugi.workflow.id}"
         result
       end
 
@@ -28,7 +28,7 @@ module Tumugi
       end
 
       def logger
-        Tumugi::Logger.instance
+        @logger ||= Tumugi::ScopedLogger.new("tumugi-run")
       end
     end
   end

data/lib/tumugi/command/show.rb

@@ -47,7 +47,7 @@ module Tumugi
       end
 
       def logger
-        Tumugi::Logger.instance
+        @logger ||= Tumugi::ScopedLogger.new("tumugi-show")
       end
     end
   end

data/lib/tumugi/config.rb

@@ -11,7 +11,7 @@ module Tumugi
 
     def self.register_section(name, *args)
       @@sections[name] = Struct.new(camelize(name), *args)
-      logger.debug { "registered config section '#{name}' with '#{args}'" }
+      logger.info { "registered config section '#{name}' with '#{args}'" }
     end
 
     def self.camelize(term)
@@ -22,7 +22,7 @@ module Tumugi
     end
 
     def self.logger
-      Tumugi::Logger.instance
+      @logger ||= Tumugi::ScopedLogger.new("tumugi-config")
     end
 
     def initialize

data/lib/tumugi/executor/local_executor.rb

@@ -7,10 +7,9 @@ require 'tumugi/error'
 module Tumugi
   module Executor
     class LocalExecutor
-      def initialize(dag, logger=nil, worker_num: 1)
+      def initialize(dag, worker_num: 1)
         @dag = dag
         @main_task = dag.tsort.last
-        @logger = logger || Tumugi::Logger.instance
         @options = { worker_num: worker_num }
         @mutex = Mutex.new
       end
@@ -28,17 +27,17 @@ module Tumugi
 
           Concurrent::Future.execute(executor: pool) do
             if !task.runnable?(Time.now)
-              debug { "not_runnable: #{task.id}" }
+              logger.trace { "task_cannot_run: #{task.id}" }
               enqueue_task(task)
             else
               begin
-                info "start: #{task.id}"
+                logger.info { "task_start: #{task.id}" }
                 task.trigger!(:start)
                 MuchTimeout.optional_timeout(task_timeout(task), Tumugi::TimeoutError) do
                   task.run
                 end
                 task.trigger!(:complete)
-                info "#{task.state}: #{task.id}"
+                logger.info { "task_#{task.state}: #{task.id}" }
               rescue => e
                 handle_error(task, e)
               end
@@ -69,7 +68,7 @@ module Tumugi
       def dequeue_task
         loop do
           task = @mutex.synchronize {
-            debug { "queue: #{@queue.map(&:id)}" }
+            logger.trace { "task_queue_dump: #{@queue.map(&:id)}" } unless @queue.empty?
             @queue.shift
           }
 
@@ -80,14 +79,14 @@ module Tumugi
               sleep(0.1)
             end
           else
-            debug { "dequeue: #{task.id}" }
+            logger.trace { "task_queue_dequeue: #{task.id}" }
 
             if task.requires_failed?
               task.trigger!(:requires_fail)
-              info "#{task.state}: #{task.id} has failed requires task"
+              logger.info { "task_#{task.state}: #{task.id} has failed requires task" }
             elsif task.completed?
               task.trigger!(:skip)
-              info "#{task.state}: #{task.id} is already completed"
+              logger.info { "task_#{task.state}: #{task.id} is already completed" }
             else
               break task
             end
@@ -96,30 +95,26 @@ module Tumugi
       end
 
       def enqueue_task(task)
-        debug { "enqueue: #{task.id}" }
+        logger.trace { "task_queue_enqueue: #{task.id}" }
         @mutex.synchronize { @queue.push(task) }
       end
 
       def handle_error(task, err)
         if task.retry
           task.trigger!(:pend)
-          @logger.error "#{err.class}: '#{err.message}' - #{task.tries} tries and wait #{task.retry_interval} seconds until the next try."
+          logger.error { "#{err.class}: '#{err.message}' - #{task.tries} tries and wait #{task.retry_interval} seconds until the next try." }
           enqueue_task(task)
         else
           task.trigger!(:fail)
-          @logger.error "#{err.class}: '#{err.message}' - #{task.tries} tries and reached max retry count, so task #{task.id} failed."
-          info "#{task.state}: #{task.id}"
-          @logger.error "#{err.message}"
-          @logger.debug { err.backtrace.join("\n") }
+          logger.error { "#{err.class}: '#{err.message}' - #{task.tries} tries and reached max retry count, so task #{task.id} failed." }
+          logger.error { "#{err.message}" }
+          logger.error { err.backtrace.join("\n") }
         end
+        logger.info { "task_#{task.state}: error handeling done for #{task.id}" }
       end
 
-      def info(message)
-        @logger.info "#{message}, thread: #{Thread.current.object_id}"
-      end
-
-      def debug(&block)
-        @logger.debug { "#{block.call}, thread: #{Thread.current.object_id}" }
+      def logger
+        @logger ||= Tumugi::ScopedLogger.new("tumugi-executor")
       end
     end
   end

data/lib/tumugi/{logger.rb → logger/logger.rb}

@@ -7,7 +7,9 @@ module Tumugi
   class Logger
     include Singleton
     extend Forwardable
-    def_delegators :@logger, :debug, :error, :fatal, :info, :warn, :level
+    def_delegators :@logger,  :debug, :debug?, :error, :error?,
+                              :fatal, :fatal?, :info, :info?,
+                              :warn, :warn?, :level
     attr_accessor :workflow_id
 
     def initialize
@@ -35,21 +37,16 @@ module Tumugi
     private
 
     def text_formatter
-      Proc.new { |severity, datetime, progname, msg|
-        if !workflow_id.nil?
-          "#{datetime} #{severity} [#{workflow_id}] #{msg}\n"
-        else
-          "#{datetime} #{severity} #{msg}\n"
-        end
+      Proc.new { |level, datetime, program_name, msg|
+        "#{datetime} [#{level}]#{' (' + program_name + ')' unless program_name.nil?} #{msg}\n"
       }
     end
 
     def json_formatter
-      Proc.new { |severity, datetime, progname, msg|
-        hash = { time: datetime, severity: severity, message: msg }
-        if !workflow_id.nil?
-          hash[:workflow] = workflow_id
-        end
+      Proc.new { |level, datetime, program_name, msg|
+        hash = { time: datetime, level: level, message: msg }
+        hash[:program_name] = program_name unless program_name.nil?
+        hash[:workflow] = workflow_id unless workflow_id.nil?
         "#{JSON.generate(hash)}\n"
       }
     end

data/lib/tumugi/logger/scoped_logger.rb

@@ -0,0 +1,43 @@
+require 'tumugi/logger/logger'
+require 'forwardable'
+
+module Tumugi
+  class ScopedLogger
+    extend Forwardable
+    def_delegators :@logger, :init, :verbose!, :quiet!, :workflow_id, :workflow_id=,
+                             :debug?, :error?, :fatal?, :info?, :warn?, :level
+
+    def initialize(scope)
+      @scope = scope
+      @logger = Tumugi::Logger.instance
+    end
+
+    [:debug, :error, :fatal, :info, :warn].each do |level|
+      class_eval "def #{level}(msg=nil, &block); log(:#{level}, msg, &block) end", __FILE__, __LINE__
+    end
+
+    def trace(msg=nil, &block)
+      if ENV.key?("TUMUGI_DEBUG")
+        log(:debug, msg, &block)
+      end
+    end
+
+    private
+
+    def log(level, msg=nil, &block)
+      if block_given?
+        @logger.send(level, progname, &block)
+      else
+        @logger.send(level, progname) { msg }
+      end
+    end
+
+    def progname
+      if @scope.is_a?(Proc)
+        @scope.call
+      else
+        @scope.to_s
+      end
+    end
+  end
+end

data/lib/tumugi/mixin/parameterizable.rb

@@ -1,5 +1,5 @@
 require 'tumugi/error'
-require 'tumugi/logger'
+require 'tumugi/logger/logger'
 require 'tumugi/parameter/parameter_proxy'
 
 module Tumugi

data/lib/tumugi/target.rb

@@ -11,7 +11,7 @@ module Tumugi
     end
 
     def logger
-      @logger ||= Tumugi::Logger.instance
+      @logger ||= Tumugi::ScopedLogger.new(self.class.name)
     end
 
     def log(msg)

data/lib/tumugi/task.rb

@@ -1,3 +1,4 @@
+require 'tumugi/logger/scoped_logger'
 require 'tumugi/mixin/listable'
 require 'tumugi/mixin/task_helper'
 require 'tumugi/mixin/parameterizable'
@@ -50,7 +51,7 @@ module Tumugi
     end
 
     def logger
-      @logger ||= Tumugi::Logger.instance
+      @logger ||= Tumugi::ScopedLogger.new(->{"Thread-#{Thread.list.index {|t| t == Thread.current}}: #{id}"})
     end
 
     def log(msg)

data/lib/tumugi/task_definition.rb

@@ -1,6 +1,6 @@
-require 'tumugi/logger'
 require 'tumugi/plugin'
 require 'tumugi/task'
+require 'tumugi/logger/logger'
 require 'tumugi/mixin/listable'
 require 'tumugi/mixin/task_helper'
 

data/lib/tumugi/version.rb

@@ -1,3 +1,3 @@
 module Tumugi
-  VERSION = "0.6.1"
+  VERSION = "0.6.2"
 end

data/lib/tumugi/workflow.rb

@@ -13,6 +13,8 @@ module Tumugi
     attr_reader :id
     attr_accessor :params
 
+    DEFAULT_CONFIG_FILE = "tumugi_config.rb"
+
     def initialize
       @id = SecureRandom.uuid
       @tasks = {}
@@ -42,13 +44,13 @@ module Tumugi
 
     def load_workflow_file(file)
       unless File.exist?(file)
-        raise Tumugi::TumugiError, "Workflow file '#{file}' not exist."
+        raise Tumugi::TumugiError, "Workflow file '#{file}' does not exist"
       end
 
       begin
         logger.info "Load workflow from #{file}"
         load(file, true)
-      rescue LoadError => e
+      rescue Exception => e
         raise Tumugi::TumugiError.new("Workflow file load error: #{file}", e)
       end
     end
@@ -67,7 +69,7 @@ module Tumugi
     end
 
     def logger
-      @logger ||= Tumugi::Logger.instance
+      @logger ||= Tumugi::ScopedLogger.new("tumugi-workflow")
     end
 
     def setup_logger(command, options)
@@ -84,12 +86,23 @@ module Tumugi
 
     def load_config(options)
       config_file = options[:config]
-      if config_file && File.exist?(config_file) && File.extname(config_file) == '.rb'
+
+      if config_file && !File.exist?(config_file)
+        raise Tumugi::TumugiError, "Config file '#{config_file}' does not exist"
+      end
+
+      if !config_file && File.exist?(DEFAULT_CONFIG_FILE)
+        config_file = DEFAULT_CONFIG_FILE
+      end
+
+      if config_file && File.exist?(config_file)
         logger.info "Load config from #{config_file}"
-        load(config_file)
+        begin
+          load(config_file)
+        rescue Exception => e
+          raise Tumugi::TumugiError.new("Config file load error: #{config_file}", e)
+        end
       end
-    rescue LoadError => e
-      raise Tumugi::TumugiError.new("Config file load error: #{config_file}", e)
     end
 
     def set_params(options)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tumugi
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.6.2
 platform: ruby
 authors:
 - Kazuyuki Honda
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-07-10 00:00:00.000000000 Z
+date: 2016-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -219,7 +219,8 @@ files:
 - lib/tumugi/error.rb
 - lib/tumugi/executor/local_executor.rb
 - lib/tumugi/file_system.rb
-- lib/tumugi/logger.rb
+- lib/tumugi/logger/logger.rb
+- lib/tumugi/logger/scoped_logger.rb
 - lib/tumugi/mixin/listable.rb
 - lib/tumugi/mixin/parameterizable.rb
 - lib/tumugi/mixin/task_helper.rb