upperkut 0.1.4 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7793f7bbdc5c1ca3301ac58060ebcccb74b04abb
-  data.tar.gz: 23c894883a95c00b113f92a9275565d6fa1bc35c
+  metadata.gz: 86cdd27d527086fd910ae6065716239959e2948b
+  data.tar.gz: 2556072af6d5b2ba93dffdcccb3c2c8f7dd7da22
 SHA512:
-  metadata.gz: 30932fa2ac443396621e73252f71c665b9d5e1ec2659aa8e37325faf30bf318340694c63d66f274ab5346b38ab36657e8dac5211a2dac272abe05976e30e8b57
-  data.tar.gz: ab3d183ba7731e30c1010dcd7a032dc28ba12b017c75521021565232e427eaebd007195f12484c2e2054274250792486e3832bbdfe7a3616237ccae738755ef2
+  metadata.gz: 2d758d766b00c8a094d1a83a97a1632abfdbc6406867c47379e54876a837232c526be01bb4e1aea723852c6e0af8f248afae584f3ad053dbd8311f7bd1aec019
+  data.tar.gz: ced16ccf12783e7a23f4cb13405da13a9c1a860b6e0b607a1822ac9feb6f3545fb2cada50a930c9df04fe8e973cbef4962d5e7f773feb2457848565138cdc90d

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.1
+before_install: gem install bundler -v 1.16.1

data/Gemfile

@@ -6,7 +6,4 @@ git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 gemspec
 
 gem 'fakeredis'
-gem 'fivemat'
 gem 'pry'
-gem 'rspec_junit_formatter'
-gem 'simplecov', require: false

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    upperkut (0.1.4)
+    upperkut (0.3.0)
       redis (>= 3.3.5, < 5)
 
 GEM
@@ -9,11 +9,8 @@ GEM
   specs:
     coderay (1.1.2)
     diff-lcs (1.3)
-    docile (1.3.1)
     fakeredis (0.7.0)
       redis (>= 3.2, < 5.0)
-    fivemat (1.3.6)
-    json (2.1.0)
     method_source (0.9.0)
     pry (0.11.3)
       coderay (~> 1.1.0)
@@ -33,13 +30,6 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.7.0)
     rspec-support (3.7.1)
-    rspec_junit_formatter (0.4.1)
-      rspec-core (>= 2, < 4, != 2.12.0)
-    simplecov (0.16.1)
-      docile (~> 1.1)
-      json (>= 1.8, < 3)
-      simplecov-html (~> 0.10.0)
-    simplecov-html (0.10.2)
 
 PLATFORMS
   ruby
@@ -47,12 +37,9 @@ PLATFORMS
 DEPENDENCIES
   bundler (~> 1.16)
   fakeredis
-  fivemat
   pry
   rake (~> 10.0)
   rspec (~> 3.0)
-  rspec_junit_formatter
-  simplecov
   upperkut!
 
 BUNDLED WITH

data/README.md

@@ -1,9 +1,5 @@
 # Upperkut
 
-[![CircleCI](https://circleci.com/gh/ResultadosDigitais/upperkut/tree/master.svg?style=svg&circle-token=693e512de6985be3b3db12279ba6ed508fb5c6f6)](https://circleci.com/gh/ResultadosDigitais/upperkut/tree/master)
-[![Maintainability](https://api.codeclimate.com/v1/badges/ece40319b0db03af891d/maintainability)](https://codeclimate.com/repos/5b318a7c6d37b70272008676/maintainability)
-[![Test Coverage](https://api.codeclimate.com/v1/badges/ece40319b0db03af891d/test_coverage)](https://codeclimate.com/repos/5b318a7c6d37b70272008676/test_coverage)
-
 Batch background processing tool.
 
 ## Installation

data/lib/upperkut/batch_execution.rb

@@ -0,0 +1,29 @@
+require_relative 'logging'
+
+module Upperkut
+  class BatchExecution
+    def initialize(worker, logger = Upperkut::Logging.logger)
+      @worker = worker
+      @logger = logger
+    end
+
+    def execute
+      worker_instance = @worker.new
+      items  = @worker.fetch_items.collect! do |item|
+        item['body']
+      end
+
+      worker_instance.perform(items.dup)
+    rescue Exception => ex
+      @worker.push_items(items)
+
+      @logger.info(
+        action: :requeue,
+        ex: ex,
+        item_size: items.size
+      )
+
+      raise ex
+    end
+  end
+end

data/lib/upperkut/cli.rb

@@ -1,14 +1,15 @@
 require 'optparse'
 require_relative '../upperkut'
 require_relative 'manager'
+require_relative 'logging'
 
 module Upperkut
   class CLI
     def initialize(args = ARGV)
       @options = {}
-      parse_options(args)
+      @logger = Upperkut::Logging.logger
 
-      STDOUT.puts @options.inspect
+      parse_options(args)
     end
 
     def start
@@ -16,8 +17,16 @@ module Upperkut
         require file
       end
 
+      if log_level = @options[:log_level]
+        @logger.level = log_level
+      end
+
+      @options[:logger] = @logger
+
       manager = Manager.new(@options)
 
+      @logger.info(@options)
+
       r, w = IO.pipe
       signals = %w[INT TERM]
 
@@ -34,7 +43,10 @@ module Upperkut
           handle_signal(signal)
         end
       rescue Interrupt
-        puts 'Shutting down'
+        @logger.info(
+          'Stopping managers, wait for 5 seconds and them kill processors'
+        )
+
         manager.stop
         sleep(5)
         manager.kill
@@ -64,6 +76,9 @@ module Upperkut
         o.on('-c', '--concurrency INT', 'Numbers of threads to spawn') do |arg|
           @options[:concurrency] = Integer(arg)
         end
+        o.on('-l', '--log-level LEVEL', 'Log level') do |arg|
+          @options[:log_level] = arg.to_i
+        end
       end.parse!(args)
     end
   end

data/lib/upperkut/logging.rb

@@ -0,0 +1,38 @@
+require 'logger'
+require 'time'
+require 'socket'
+
+module Upperkut
+  module Logging
+
+    class DefaultFormatter < Logger::Formatter
+      def call(severity, time, program_name, message)
+        "upperkut: #{time.utc.iso8601(3)} hostname=#{Socket.gethostname} "\
+        "pid=#{::Process.pid} severity=#{severity} #{format_message(message)}\n"
+      end
+
+      private
+
+      def format_message(message)
+        return "msg=#{message} " unless message.is_a?(Hash)
+
+        message.each_with_object('') do |(k,v), memo|
+          memo << "#{k}=#{v}\s"
+          memo
+        end
+
+      end
+    end
+
+    def self.initialize_logger
+      logger = Logger.new($stdout)
+      logger.level     = Logger::INFO
+      logger.formatter = DefaultFormatter.new
+      logger
+    end
+
+    def self.logger
+      @logger ||= initialize_logger
+    end
+  end
+end

data/lib/upperkut/manager.rb

@@ -4,19 +4,23 @@ require_relative 'processor'
 module Upperkut
   class Manager
     attr_accessor :worker, :redis
-    attr_reader :stopped
+    attr_reader :stopped, :logger
 
     def initialize(opts = {})
       self.worker = opts.fetch(:worker).constantize
       self.redis  = worker.setup.redis
       @concurrency = opts.fetch(:concurrency, 25)
+      @logger = opts.fetch(:logger, Upperkut::Logging.logger)
+
       @stopped = false
       @processors = []
     end
 
     def run
       @concurrency.times do
-        @processors << Processor.new(self).run
+        processor = Processor.new(self)
+        @processors << processor
+        processor.run
       end
     end
 
@@ -27,5 +31,14 @@ module Upperkut
     def kill
       @processors.each(&:kill)
     end
+
+    def notify_killed_processor(processor)
+      @processors.delete(processor)
+      return if @stopped
+
+      processor = Processor.new(self)
+      @processors << processor
+      processor.run
+    end
   end
 end

data/lib/upperkut/processor.rb

@@ -1,20 +1,34 @@
+require_relative 'batch_execution'
+
 module Upperkut
   class Processor
     def initialize(manager)
       @manager = manager
       @worker  = @manager.worker
+      @logger =  @manager.logger
+
       @sleeping_time = 0
     end
 
     def run
       @thread ||= Thread.new do
-        process
+        begin
+          process
+        rescue Exception => e
+          @logger.debug(
+            action: :processor_killed,
+            reason: e
+          )
+
+          @manager.notify_killed_processor(self)
+        end
       end
     end
 
     def kill
       return unless @thread
       @thread.raise Upperkut::Shutdown
+      @thread.value # wait
     end
 
     private
@@ -28,6 +42,7 @@ module Upperkut
         end
 
         @sleeping_time += sleep(@worker.setup.polling_interval)
+        @logger.debug(sleeping_time: @sleeping_time)
       end
     end
 
@@ -43,13 +58,7 @@ module Upperkut
     end
 
     def process_batch
-      @sleeping_time = 0
-      @worker.new.process
-    rescue Exception => ex
-      # Add to retry_queue
-      # if retry_limit is reached
-      # send to dead
-      raise ex
+      BatchExecution.new(@worker, @logger).execute
     end
   end
 end

data/lib/upperkut/version.rb

@@ -1,3 +1,3 @@
 module Upperkut
-  VERSION = '0.1.4'.freeze
+  VERSION = '0.3.0'.freeze
 end

data/lib/upperkut/worker.rb

@@ -9,13 +9,6 @@ module Upperkut
       base.extend(ClassMethods)
     end
 
-    def process
-      items = self.class.fetch_items.collect! do |item|
-        item['body']
-      end
-      perform(items)
-    end
-
     module ClassMethods
       extend Forwardable
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: upperkut
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.3.0
 platform: ruby
 authors:
 - Nando Sousa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-26 00:00:00.000000000 Z
+date: 2018-06-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -80,9 +80,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ".circleci/config.yml"
 - ".gitignore"
 - ".rspec"
+- ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
 - Gemfile.lock
@@ -92,8 +92,10 @@ files:
 - bin/upperkut
 - examples/basic.rb
 - lib/upperkut.rb
+- lib/upperkut/batch_execution.rb
 - lib/upperkut/cli.rb
 - lib/upperkut/core_ext.rb
+- lib/upperkut/logging.rb
 - lib/upperkut/manager.rb
 - lib/upperkut/processor.rb
 - lib/upperkut/strategy.rb
@@ -121,7 +123,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.14.1
+rubygems_version: 2.6.14
 signing_key: 
 specification_version: 4
 summary: Batch background processing tool

data/.circleci/config.yml

@@ -1,63 +0,0 @@
-# Ruby CircleCI 2.0 configuration file
-#
-# Check https://circleci.com/docs/2.0/language-ruby/ for more details
-#
-version: 2
-jobs:
-  build:
-    docker:
-      - image: circleci/ruby:2.4.4
-        environment:
-          CC_TEST_REPORTER_ID: 03ab83a772148a577d29d4acf438d7ebdc95c632224122d0ba8dbb291eedebe6
-          COVERAGE: true
-    steps:
-      - checkout
-
-      # Download and cache dependencies
-      - restore_cache:
-          keys:
-          - v1-dependencies-{{ checksum "Gemfile.lock" }}
-          # fallback to using the latest cache if no exact match is found
-          - v1-dependencies-
-
-      - run:
-          name: install dependencies
-          command: |
-            bundle install --jobs=4 --retry=3 --path vendor/bundle
-
-      - save_cache:
-          paths:
-            - ./vendor/bundle
-          key: v1-dependencies-{{ checksum "Gemfile.lock" }}
-
-      - run:
-          name: Setup Code Climate test-reporter
-          command: |
-            curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
-            chmod +x ./cc-test-reporter
-            ./cc-test-reporter before-build
-
-      # run tests!
-      - run:
-          name: run tests
-          command: |
-            mkdir /tmp/test-results
-            TEST_FILES="$(circleci tests glob "spec/**/*_spec.rb" | circleci tests split --split-by=timings)"
-
-            bundle exec rspec --format progress \
-                            --format RspecJunitFormatter \
-                            --out /tmp/test-results/rspec.xml \
-                            --format progress \
-                            $TEST_FILES
-      - run:
-          name: Submit coverage
-          command: |
-            ./cc-test-reporter after-build --coverage-input-type simplecov --exit-code $?
-
-      # collect reports
-      - store_test_results:
-          path: /tmp/test-results
-
-      - store_artifacts:
-          path: /tmp/test-results
-          destination: test-results