dronejob 1.1.0 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 49ed6bec795138cb9ab9c9eee02fd9a15b4da8e7
-  data.tar.gz: 888255bd00bd54f4970c627128a45f8778622ba1
+  metadata.gz: 4945a0a8896b7c9bf4ce1f0672c086e0a9dfe956
+  data.tar.gz: 1b41875621ed5d31aad837edd0354c089743b9db
 SHA512:
-  metadata.gz: cf0137e6014b5ebe002413dbc17d9fbec26d5bdec0e47688ffd529927fcd46deb2ac6c7360d7879369f472ae937b6c5cbe0104c9f31ee6042ac18fef54094698
-  data.tar.gz: 29dfcbb9bf3b7e9b91caff7540af886ad6e4ec81a95cbf486dd93da8d641e00dc16ec2f6b8d64bede9f44e0ba0adda41ea1c73f41da678d9c6eb1f0a1cd6f9f8
+  metadata.gz: 69466465ebaf6db5b03d0b77c461e4421385bb0abde2c90f2addb337017b6deb69850ca0a69ca6b56a01d38812fa918cc3f9f88b5d890d6073eb9671bc068fb5
+  data.tar.gz: 41c3a2ab4bb915018763550db1652621b0c2100f2b492c0cd93fb8bd1dbd435cb22ed21c8db59834c562d5ca8d71d409cc59a6b71322d11ccffce58dbba18d2b

data/.gitignore

@@ -3,3 +3,4 @@ app/*
 tmp/*
 plugins/*
 /archive/*
+.bundle/

data/Gemfile.lock

@@ -1,16 +1,15 @@
 PATH
   remote: .
   specs:
-    dronejob (1.1.0)
+    dronejob (1.1.1)
       activejob (~> 4.2)
-      bundler (~> 1.11)
+      bundler (>= 1.3.0, < 2.0)
       git (~> 1.2)
       haml (~> 4.0)
       open_uri_redirections (~> 0.2)
-      rake (~> 10.5)
       rubyzip (~> 1.1)
-      sidekiq (~> 4.1.0)
-      sidekiq-status (~> 0.6.0)
+      sidekiq (~> 4.1)
+      sidekiq-status (~> 0.6)
       thor (~> 0.19)
       typhoeus (~> 1.0)
 
@@ -26,7 +25,7 @@ GEM
       minitest (~> 5.1)
       thread_safe (~> 0.3, >= 0.3.4)
       tzinfo (~> 1.1)
-    coderay (1.1.0)
+    coderay (1.1.1)
     concurrent-ruby (1.0.1)
     connection_pool (2.2.0)
     diff-lcs (1.2.5)
@@ -47,13 +46,12 @@ GEM
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
       slop (~> 3.4)
-    rake (10.5.0)
     redis (3.3.0)
     rspec (3.4.0)
       rspec-core (~> 3.4.0)
       rspec-expectations (~> 3.4.0)
       rspec-mocks (~> 3.4.0)
-    rspec-core (3.4.2)
+    rspec-core (3.4.4)
       rspec-support (~> 3.4.0)
     rspec-expectations (3.4.0)
       diff-lcs (>= 1.2.0, < 2.0)
@@ -87,4 +85,4 @@ DEPENDENCIES
   rspec (~> 3.3)
 
 BUNDLED WITH
-   1.11.2
+   1.12.1

data/README.md

@@ -1,4 +1,4 @@
-# Drone Factory
+# DroneJob
 
 Scalable worker factory for ruby
 
@@ -10,19 +10,6 @@ Install gems:
 
 ## Usage
 
+List dronejob commands:
 
-### Logging
-
-
-```ruby
-# Create stdout logger
-require "logging"
-Logging.color_scheme("bright", levels: { debug: :blue, info: :green, warn: :yellow, error: :red, fatal: [:white, :on_red] }, date: :blue, ndc: :blue, logger: :cyan, message: :black)
-Dronejob::Base.logger = Logging::Logger.new("Dronejob").add_appenders(Logging::Appenders::Stdout.new("stdout", {layout: Logging.layouts.pattern({pattern: '[%d] %-5l %c{1} %x %m\n', color_scheme: 'bright'})}))
-	
-# Create gcl logger
-require "logging"
-require "logging/appenders/gcl"
-Logging.init(Logging::Appenders::GoogleCloudLogging::SEVERITY_NAMES)
-Dronejob::Base.logger = Logging::Logger.new("Dronejob").add_appenders(Logging::Appenders::GoogleCloudLogging.new("gcl", {project_id: "project-id", log_name: "dronejob", resource_type: "gce_instance", resource_labels: {instance_id: "instance-id", zone: "us-central1-b"}, buffer_size: '3', immediate_at: 'error, fatal'}))
-```
+	$ bundle exec dronejob

data/bin/dronejob

@@ -1,7 +1,5 @@
 #!/usr/bin/env ruby
 $:.push("lib/")
 require "thor"
-
-# initialize dronejob
 require "dronejob/command"
 Dronejob::Command.start

data/dronejob.gemspec

@@ -14,17 +14,16 @@ Gem::Specification.new do |s|
   s.description = "Scalable worker factory for ruby"
   s.required_ruby_version = '~> 2.0'
   s.required_rubygems_version = '~> 2.4'
-  s.add_runtime_dependency "bundler", "~> 1.11"
-  s.add_runtime_dependency "rake", "~> 10.5"
+  s.add_dependency "bundler", '>= 1.3.0', '< 2.0'
   s.add_runtime_dependency "git", "~> 1.2"
   s.add_runtime_dependency "thor", "~> 0.19"
-  s.add_runtime_dependency "sidekiq", "~> 4.1.0"
   s.add_runtime_dependency "haml", "~> 4.0"
-  s.add_runtime_dependency "sidekiq-status", "~> 0.6.0"
-  s.add_runtime_dependency "open_uri_redirections", "~> 0.2"
+  s.add_runtime_dependency "sidekiq", "~> 4.1"
+  s.add_runtime_dependency "sidekiq-status", "~> 0.6"
   s.add_runtime_dependency "rubyzip", "~> 1.1"
   s.add_runtime_dependency "activejob", "~> 4.2"
   s.add_runtime_dependency "typhoeus", "~> 1.0"
+  s.add_runtime_dependency "open_uri_redirections", "~> 0.2"
   s.add_development_dependency "rspec", "~> 3.3"
   s.add_development_dependency "pry", "~> 0.10"
   s.files        = `git ls-files`.split("\n")

data/lib/dronejob/base.rb

@@ -1,5 +1,6 @@
 require "active_job"
 require 'dronejob/modules/attr_store'
+require 'dronejob/modules/callbacks'
 require 'dronejob/modules/core'
 require 'dronejob/modules/git'
 require 'dronejob/modules/log'
@@ -7,7 +8,6 @@ require 'dronejob/modules/queue'
 require 'dronejob/modules/options'
 require 'dronejob/modules/params'
 require 'dronejob/modules/phases'
-require 'dronejob/modules/stateful'
 require 'dronejob/modules/workspace'
 
 module Dronejob
@@ -15,6 +15,7 @@ module Dronejob
     DRONEJOB_TYPE = "base"
     queue_as :dronejob
     include Modules::AttrStore
+    include Modules::Callbacks
     include Modules::Core
     include Modules::Git
     include Modules::Log
@@ -22,9 +23,22 @@ module Dronejob
     include Modules::Options
     include Modules::Params
     include Modules::Phases
-    include Modules::Stateful
     include Modules::Workspace
     
+    def uuid
+      @uuid ||= "#{self.class.name.split('::').last.underscore}_#{SecureRandom.uuid}"
+    end
+    
+    def title
+      self.class.name
+    end
+    
+    rescue_from(Exception) do |exception|
+      publish_error(exception)
+      error(exception)
+      Kernel.exit!
+    end
+    
     ActiveSupport.run_load_hooks(:dronejob, self)
   end
 end

data/lib/dronejob/modules/attr_store.rb

@@ -7,7 +7,7 @@ module Dronejob
         attr_accessor :variables
       end
 
-      module ClassMethods
+      class_methods do
         def attr_store(*args)
           @attr_stores ||= [:dronejob_completed]
           @attr_stores.push(*args)

data/lib/dronejob/modules/callbacks.rb

@@ -0,0 +1,31 @@
+module Dronejob
+  module Modules
+    module Callbacks
+      extend ActiveSupport::Concern
+      include ActiveSupport::Callbacks
+      
+      included do
+        define_callbacks :phase
+        define_callbacks :job
+      end
+
+      class_methods do
+        def before_phase(*methods, &block)
+          set_callback(:phase, :before, *methods, &block)
+        end
+        
+        def after_phase(*methods, &block)
+          set_callback(:phase, :after, *methods, &block)
+        end
+        
+        def before_job(*methods, &block)
+          set_callback(:job, :before, *methods, &block)
+        end
+        
+        def after_job(*methods, &block)
+          set_callback(:job, :after, *methods, &block)
+        end
+      end
+    end
+  end
+end

data/lib/dronejob/modules/core.rb

@@ -10,53 +10,9 @@ module Dronejob
       def perform(params)
         set_parameters(params)
         validate_parameters!
-        create_working_dir
-        
-        if self.class.stateful?
-          load_variables
-          git_init(@working_dir.to_s)
-          git_collect_commits
-          git_commit("start")
-          if param(:from)
-            info("starting from phase #{param(:from)}")
-            phase = prev_phase(param(:from))
-            raise("phase '#{param(:from)}' not found!") if phase.nil? or @commits[phase.to_s].nil?
-            git_reset(phase.to_s)
-          else
-            git_reset
-          end
-          git_collect_commits
-          git_clean
+        run_callbacks :job do
+          run_phases
         end
-        
-        # Run through phases
-        each_phase do |phase, phase_config|
-          begin
-            phase_result = public_send(phase)
-          rescue Exception => e
-            error(e.message)
-            error(e.backtrace.join("\n"))
-            delete_working_dir if self.class.cleanup_on_error?
-            publish_status({title: self.title, description: e.message, phase: phase, backtrace: e.backtrace}, "failed") if param(:notify) and !param(:run)
-            return false
-          end
-          
-          publish_status({title: self.title, description: phase_config[:title], phase: phase}, "working") if phase_config[:notify] and param(:notify) and !param(:run)
-          git_commit(phase) if self.class.stateful?
-        end
-        @phase = "complete"
-
-        publish_status({title: self.title, description: "Job Completed!", phase: @phase}, "complete") if param(:notify) and !param(:run)
-        
-        delete_working_dir if self.class.cleanup_on_complete?
-      end
-      
-      def uuid
-        @uuid ||= "#{self.class.name.split('::').last.underscore}_#{SecureRandom.uuid}"
-      end
-      
-      def title
-        self.class.name
       end
       
       def shell

data/lib/dronejob/modules/git.rb

@@ -7,6 +7,22 @@ module Dronejob
       
       included do
         attr_reader :git, :commits
+        set_callback :job, :before, :git_init_job, if: :stateful?
+        set_callback(:phase, :after, if: :stateful?){git_commit(@phase)}
+      end
+      
+      class_methods do        
+        def stateful(value=true)
+          @stateful = value
+        end
+        
+        def stateful?
+          !!@stateful
+        end
+      end
+      
+      def stateful?
+        self.class.stateful?
       end
       
       def git_init(path)
@@ -46,6 +62,28 @@ module Dronejob
         end
       end
       
+      def git_init_job
+        if self.class.stateful?
+          load_variables
+          git_init(@working_dir.to_s)
+          git_collect_commits
+          git_commit("start")
+          if param(:from) == "start"
+            info("starting from scratch")
+            git_reset("start")
+          elsif param(:from)
+            info("starting from phase #{param(:from)}")
+            phase = prev_phase(param(:from))
+            raise("phase '#{param(:from)}' not found!") if phase.nil? or @commits[phase.to_s].nil?
+            git_reset(phase.to_s)
+          else
+            git_reset
+          end
+          git_collect_commits
+          git_clean
+        end
+      end
+      
     end
   end
 end

data/lib/dronejob/modules/log.rb

@@ -9,8 +9,7 @@ module Dronejob
         cattr_accessor(:logger) { ActiveSupport::TaggedLogging.new(ActiveSupport::Logger.new(STDOUT)) }
       end
       
-      module ClassMethods
-        
+      class_methods do
         def log(level, message, phase=:global)
           if !logger.nil?
             logger.send(level, message)
@@ -41,7 +40,26 @@ module Dronejob
       end
       
       def error(message)
-        log(:error, message)
+        if Exception === message
+          say(" ERROR ", ["\e[38;5;15m", Thor::Shell::Color::BOLD, Thor::Shell::Color::ON_RED], false)
+          say(" #{message.class}: ", [Thor::Shell::Color::BOLD], false)
+          say("#{message.message}", [Thor::Shell::Color::BOLD, Thor::Shell::Color::RED])
+          backtrace(message.backtrace)
+        else
+          log(:error, message)
+        end
+      end
+      
+      def backtrace(lines)
+        lines.each do |line|
+          line.gsub!("#{Dir.pwd}/", '')
+          (file, lineno, info) = line.split(":")
+          if /ruby\/?gems/.match(line).nil?
+            say(" #{lineno.rjust(5, ' ')} ", ["\e[48;5;255m", "\e[38;5;245m"], false)
+            say(" #{file} ", [Thor::Shell::Color::RED], false)
+            say(info, ["\e[38;5;245m"], true)
+          end
+        end
       end
       
       def error!(message)

data/lib/dronejob/modules/options.rb

@@ -7,7 +7,7 @@ module Dronejob
         attr_reader :options
       end
 
-      module ClassMethods
+      class_methods do
         def options
           if @options.nil?
             yaml_file = File.join("dronejob.yml")

data/lib/dronejob/modules/params.rb

@@ -3,7 +3,7 @@ module Dronejob
     module Params
       extend ActiveSupport::Concern
 
-      module ClassMethods
+      class_methods do
         def param(key, options)
           @params ||= {}
           @params[key] = options

data/lib/dronejob/modules/phases.rb

@@ -2,12 +2,14 @@ module Dronejob
   module Modules
     module Phases
       extend ActiveSupport::Concern
+      include ActiveSupport::Callbacks
 
       included do
         attr_accessor :phase
+        set_callback(:phase, :around){|r, block| logger.tagged(@phase, &block)}
       end
 
-      module ClassMethods
+      class_methods do
         def phase(key, options)
           @phases ||= {}
           @phases[key] = options
@@ -18,42 +20,46 @@ module Dronejob
         end
       end
       
-      def each_phase(&block)
-        self.class.phases.each do |phase, config|
-          fail "Phase not found: '#{phase}'" unless self.respond_to?(phase)
+      def log_phase
+        
+      end
+      
+      def run_phases
+        self.class.phases.each do |phase, phase_config|
           @phase = phase
-          before_phase(phase, config) if self.respond_to?(:before_phase)
-          
-          logger.tagged(phase) do
-            if completed_phase?(phase) and !config[:always_run]
+          @phase_config = phase_config
+
+          # initialize phase
+          fail "Phase not found: '#{phase}'" unless self.respond_to?(phase)
+          run_callbacks :phase do
+            if completed_phase?(phase) and !@phase_config[:always_run]
               info("already completed")
             elsif skip_phase?(phase)
               info("skipping...")
             else
-              info "running phase #{phase}"
+              info("running phase #{phase}")
               # Require libraries
-              if !config[:require].nil?
-                config[:require].each do |library|
+              if !@phase_config[:require].nil?
+                @phase_config[:require].each do |library|
                   require library
                 end
               end
 
               # Include helpers
-              if !config[:helpers].nil?
-                config[:helpers].each do |helper|
+              if !@phase_config[:helpers].nil?
+                @phase_config[:helpers].each do |helper|
                   helper_path = "helpers/#{helper}"
                   require "./app/#{helper_path}"
                   self.class.send(:include, helper_path.camelcase.constantize)
                 end
               end
-              
-              block.call(phase, config)
+            
+              # OLD CODE FROM CORE
+              public_send(phase)
             end
           end
-          
-          after_phase(phase, config) if self.respond_to?(:after_phase)
         end
-        
+        @phase = "complete"
         @dronejob_completed = true
         git_commit("dronejob_completed") if self.class.stateful?
       end

data/lib/dronejob/modules/queue.rb

@@ -2,12 +2,14 @@ module Dronejob
   module Modules
     module Queue
       extend ActiveSupport::Concern
-
+      
       included do
         attr_accessor :queue_adapter
+        set_callback :phase, :before, :notify_before_phase
+        set_callback :job, :after, :notify_after_job
       end
       
-      module ClassMethods
+      class_methods do
         def run_worker!
           ActiveJob::Base.queue_adapter = self.queue_adapter
           require 'sidekiq/cli'
@@ -28,6 +30,27 @@ module Dronejob
         data[:secret_key] = option(:secret_key) if backend
         Redis.new(url: option(:redis_url)).publish("processes_#{backend ? 'backend' : data[:user_id]}", data.to_json)
       end
+      
+      def publish_error(e)
+        publish_status({title: self.title, description: e.message, phase: @phase, backtrace: e.backtrace, progress: 100}, "failed") if should_notify?
+      end
+      
+      def notify_before_phase
+        publish_status({title: self.title, description: @phase_config[:title], phase: @phase, progress: @phase_config[:progress]}, "working") if @phase_config[:notify] and should_notify?
+      end
+      
+      def notify_after_job
+        publish_status({title: self.title, description: "Job Completed!", phase: @phase, progress: 100}, "complete") if should_notify?
+      end
+      
+      def should_notify?
+        param(:notify) and !param(:run)
+      end
+      
+      def publish_progress(progress, description=nil)
+        publish_status({title: self.title, description: description, phase: @phase, progress: progress}, "working") if should_notify?
+      end
+      
     end
   end
 end

data/lib/dronejob/modules/workspace.rb

@@ -5,7 +5,13 @@ module Dronejob
     module Workspace
       extend ActiveSupport::Concern
 
-      module ClassMethods
+      included do
+        cattr_accessor(:jobs_path) { "app/jobs" }
+        cattr_accessor(:output_path) { "tmp/jobs" }
+        set_callback :job, :after, :delete_working_dir
+      end
+      
+      class_methods do
         def cleanup_on_complete(value=true)
           @cleanup_on_complete = value
         end
@@ -22,12 +28,6 @@ module Dronejob
           !!@cleanup_on_error
         end
       end
-
-      included do
-        attr_accessor :working_dir
-        cattr_accessor(:jobs_path) { "app/jobs" }
-        cattr_accessor(:output_path) { "tmp/jobs" }
-      end
       
       def file(path, &block)
         f = working_dir.file(path)
@@ -42,14 +42,16 @@ module Dronejob
         working_dir.dir(path)
       end
 
-      def create_working_dir
-        @working_dir = WorkspaceDir.new(File.join(self.class.output_path, uuid))
-        @working_dir.create
+      def working_dir
+        if !@working_dir
+          @working_dir = WorkspaceDir.new(File.join(self.class.output_path, uuid))
+          @working_dir.create
+        end
         @working_dir
       end
       
       def delete_working_dir
-        working_dir.delete!
+        working_dir.delete! if self.class.cleanup_on_complete? && @working_dir
       end
       
       def asset_file(path)

data/lib/dronejob/version.rb

@@ -1,3 +1,3 @@
 module Dronejob
-  VERSION = "1.1.0"
+  VERSION = "1.1.1"
 end

metadata

@@ -1,43 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: dronejob
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.1
 platform: ruby
 authors:
 - Tobias Strebitzer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-30 00:00:00.000000000 Z
+date: 2016-05-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.11'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.11'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
+        version: 1.3.0
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '10.5'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.5'
+        version: 1.3.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: git
   requirement: !ruby/object:Gem::Requirement
@@ -66,20 +58,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.19'
-- !ruby/object:Gem::Dependency
-  name: sidekiq
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 4.1.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 4.1.0
 - !ruby/object:Gem::Dependency
   name: haml
   requirement: !ruby/object:Gem::Requirement
@@ -95,33 +73,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '4.0'
 - !ruby/object:Gem::Dependency
-  name: sidekiq-status
+  name: sidekiq
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: '4.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: '4.1'
 - !ruby/object:Gem::Dependency
-  name: open_uri_redirections
+  name: sidekiq-status
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.6'
 - !ruby/object:Gem::Dependency
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
@@ -164,6 +142,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: open_uri_redirections
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -211,6 +203,7 @@ files:
 - lib/dronejob/command.rb
 - lib/dronejob/loader.rb
 - lib/dronejob/modules/attr_store.rb
+- lib/dronejob/modules/callbacks.rb
 - lib/dronejob/modules/core.rb
 - lib/dronejob/modules/git.rb
 - lib/dronejob/modules/log.rb
@@ -218,7 +211,6 @@ files:
 - lib/dronejob/modules/params.rb
 - lib/dronejob/modules/phases.rb
 - lib/dronejob/modules/queue.rb
-- lib/dronejob/modules/stateful.rb
 - lib/dronejob/modules/workspace.rb
 - lib/dronejob/railtie.rb
 - lib/dronejob/railties/dronejob.rake

data/lib/dronejob/modules/stateful.rb

@@ -1,18 +0,0 @@
-module Dronejob
-  module Modules
-    module Stateful
-      extend ActiveSupport::Concern
-
-      module ClassMethods
-        def stateful(value=true)
-          @stateful = value
-        end
-        
-        def stateful?
-          !!@stateful
-        end
-      end
-          
-    end
-  end
-end