upperkut 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 91c4f06d58263b93f03a4c21abb41cb855a2ec79
-  data.tar.gz: e69b6d2c5fc335e34a49814af7bfa6e4225cd985
+  metadata.gz: 7ec2e461543486232e5fef688a00cbcef1a605fa
+  data.tar.gz: 6a4dbf806dcf5f8e9833b2e8062dcb39b8d6f184
 SHA512:
-  metadata.gz: 503f1a51933df504062ae31e285e1471c857d3f52b61cb0310abfe43f60e54c4eee2f6cc670677d652cd49c868db503ba79f4b2c849b31dc1d5c896b76a9e81c
-  data.tar.gz: f20ef3064c8d3d861abba7de081399f67d91613e7c33b7ed6aad658dfc5155784a52bbfd33f8f2a2253ff3aa753885ddd293b763ce806a111a10b64b86fec143
+  metadata.gz: c67e94633c0f33071724af9f939bc0b0ba3db29262c97526a386daba2cb66a0c89101c34ab42f284ee9bacd7b43a6664b55ffa5fbf79e91fbc7fe112e1a0d196
+  data.tar.gz: 06ceffd02b4bb83bdf1e2dfad936def7dbad1d6d7b2cab2c6d7faf965c119f7bda3e1bc7d7b4957ed732049f0b7851a887a281b8b2ee90683d40def39b64ba3a

data/.gitignore

@@ -9,3 +9,4 @@
 
 # rspec failure tracking
 .rspec_status
+*.gem

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    upperkut (0.1.0)
-      redis
+    upperkut (0.1.1)
+      redis (>= 3.3.5, < 5)
 
 GEM
   remote: https://rubygems.org/

data/README.md

@@ -1,8 +1,6 @@
 # Upperkut
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/upperkut`. To experiment with that code, run `bin/console` for an interactive prompt.
-
-TODO: Delete this and the text above, and describe your gem
+Batch background processing tool.
 
 ## Installation
 
@@ -21,8 +19,50 @@ Or install it yourself as:
     $ gem install upperkut
 
 ## Usage
-
-TODO: Write usage instructions here
+Examples:
+                                                                                                        
+1) Create a Worker class and the define how to process the batch;
+  ```ruby                                                                                                      
+  class MyWorker
+    include Upperkut::Worker
+                                                                                                        
+    # This is optional
+                                                                                                        
+    setup_upperkut do |s|
+      # Define which redis instance you want to use
+      s.redis = Redis.new(url: ENV['ANOTHER_REDIS_INSTANCE_URL'])
+                                                                                                        
+      # Define the amount of items must be accumulated
+      s.batch_size = 2_000 # The default value is 1_000
+                                                                                                        
+      # How frequent the Processor should hit redis looking for elegible
+      # batch. The default value is 5. You can also set the env
+      # UPPERKUT_POLLING_INTERVAL.
+      s.polling_interval = 4
+                                                                                                        
+      # How long the Processor should wait to process batch even though
+      # the amount of items did not reached the batch_size.
+      s.max_wait = 300
+    end
+                                                                                                        
+    def perform(batch_items)
+      SidekiqJobA.perform_async(batch_items)
+      SidekiqJobB.perform_async(batch_items)
+                                                                                                        
+      process_metrics(batch_items)
+    end
+  end
+  ```
+                                                                                                        
+2) Start pushings items;
+  ```ruby                                                                                                
+  Myworker.push([{'id' => SecureRandom.uuid}, 'name' => 'Robert C Hall',  'action' => 'EMAIL_OPENNED'])
+  ```
+                                                                                                        
+3) Start Upperkut;
+  ```bash
+  $ bundle exec upperkut --worker MyWorker --concurrency 10
+  ```
 
 ## Development
 

data/bin/upperkut

@@ -1,13 +1,11 @@
 #!/usr/bin/env ruby
 
 require_relative '../lib/upperkut/cli'
-require 'pry'
 
 begin
   cli = Upperkut::CLI.new(ARGV)
   cli.start
 rescue => e
-  raise e
   STDERR.puts e.message
   STDERR.puts e.backtrace.join('\n')
   exit 1

data/lib/upperkut/cli.rb

@@ -7,6 +7,8 @@ module Upperkut
     def initialize(args = ARGV)
       @options = {}
       parse_options(args)
+
+      STDOUT.puts @options.inspect
     end
 
     def start
@@ -20,12 +22,13 @@ module Upperkut
       signals = %w(INT TERM)
 
       signals.each do |signal|
-        w.puts(signal)
+        trap signal do
+          w.puts(signal)
+        end
       end
 
       begin
         manager.run
-
         while readable_io = IO.select([r])
           signal = readable_io.first[0].gets.strip
           handle_signal(signal)
@@ -33,21 +36,22 @@ module Upperkut
       rescue Interrupt
         puts 'Shutting down'
         manager.stop
+        sleep(5)
+        manager.kill
         exit(0)
       end
     end
 
     private
 
-   def handle_signal(sig)
-      Upperkut.logger.debug "Got #{sig} signal"
+    def handle_signal(sig)
       case sig
       when 'INT'
         raise Interrupt
       when 'TERM'
         raise Interrupt
       end
-   end
+    end
 
     def parse_options(args)
       OptionParser.new do |o|
@@ -57,6 +61,9 @@ module Upperkut
         o.on('-r', '--require FILE', 'Indicate a file to be required') do |arg|
           @options[:file] = arg
         end
+        o.on('-c', '--concurrency INT', 'Numbers of threads to spawn') do |arg|
+          @options[:concurrency] = Integer(arg)
+        end
 
       end.parse!(args)
     end

data/lib/upperkut/manager.rb

@@ -10,16 +10,25 @@ module Upperkut
     def initialize(opts = {})
       self.worker = opts.fetch(:worker).constantize
       self.redis  = worker.setup.redis
-
+      @concurrency = opts.fetch(:concurrency, 25)
       @stopped = false
+      @processors = []
     end
 
     def run
-      Processor.new(self).process
+      @concurrency.times do
+        @processors << Processor.new(self).run
+      end
     end
 
     def stop
       @stopped = true
     end
+
+    def kill
+      @processors.each do |processor|
+        processor.kill
+      end
+    end
   end
 end

data/lib/upperkut/processor.rb

@@ -7,6 +7,18 @@ module Upperkut
       @sleeping_time = 0
     end
 
+    def run
+      @thread ||= Thread.new do
+        process
+      end
+    end
+    def kill
+      return if !@thread
+      @thread.raise Upperkut::Shutdown
+    end
+
+    private
+
     def process
       loop do
         if should_process?
@@ -15,18 +27,18 @@ module Upperkut
           next
         end
 
-        puts "sleeping for #{@worker.setup.polling_interval} seconds"
         @sleeping_time += sleep(@worker.setup.polling_interval)
       end
     end
 
-    private
-
     def should_process?
+      buffer_size = @worker.size
+
       return false if @manager.stopped
+      return false if buffer_size == 0
 
       # TODO: rename #setup by config
-      @worker.size >= @worker.setup.batch_size ||
+      buffer_size >= @worker.setup.batch_size ||
         @sleeping_time >= @worker.setup.max_wait
     end
 

data/lib/upperkut/strategy.rb

@@ -32,6 +32,15 @@ module Upperkut
       redis.llen(key)
     end
 
+    def latency
+      item = redis.lrange(key, -1, -1)
+      item = decode_json_items(item).first
+      return 0 unless item
+      now = Time.now.to_f
+      lat = now - item.fetch('enqueued_at', Time.now).to_f
+      lat
+    end
+
     private
 
     def key

data/lib/upperkut/util.rb

@@ -12,12 +12,17 @@ module Upperkut
       klass_name
     end
 
-    def decode_json_items(items)
-      items.collect {|i| JSON.parse(i) }
+    def encode_json_items(items)
+      items = items.collect do |i|
+        JSON.generate(
+          'enqueued_at' => Time.now.to_i,
+          'body'        => i
+        )
+      end
     end
 
-    def encode_json_items(items)
-      items.collect {|i| JSON.generate(i) }
+    def decode_json_items(items)
+      items.collect {|i| JSON.parse(i) }
     end
   end
 end

data/lib/upperkut/version.rb

@@ -1,3 +1,3 @@
 module Upperkut
-  VERSION = "0.1.0"
+  VERSION = '0.1.2'
 end

data/lib/upperkut/worker.rb

@@ -11,7 +11,9 @@ module Upperkut
     end
 
     def process
-      items = self.class.fetch_items
+      items = self.class.fetch_items.collect! do |item|
+        item['body']
+      end
       perform(items)
     end
 
@@ -19,7 +21,7 @@ module Upperkut
       extend Forwardable
 
       def_delegators :setup, :strategy
-      def_delegators :strategy, :push_items, :fetch_items, :size
+      def_delegators :strategy, :push_items, :size, :latency
 
       def push_items(items)
         strategy.push_items(items)

data/lib/upperkut.rb

@@ -1,7 +1,52 @@
-require_relative "upperkut/version"
+require_relative 'upperkut/version'
 require_relative 'upperkut/worker'
 require 'redis'
 
+# Public: Upperkut is a batch background processing tool for Ruby.
+#
+# Examples:
+#
+# 1) Create a Worker class and the define how to process the batch;
+#
+#   class MyWorker
+#     include Upperkut::Worker
+#
+#     # This is optional
+#
+#     setup_upperkut do |s|
+#       # Define which redis instance you want to use
+#       s.redis = Redis.new(url: ENV['ANOTHER_REDIS_INSTANCE_URL'])
+#
+#       # Define the amount of items must be accumulated
+#       s.batch_size = 2_000 # The default value is 1_000
+#
+#       # How frequent the Processor should hit redis looking for elegible
+#       # batch. The default value is 5. You can also set the env
+#       # UPPERKUT_POLLING_INTERVAL.
+#       s.polling_interval = 4
+#
+#       # How long the Processor should wait to process batch even though
+#       # the amount of items did not reached the batch_size.
+#       s.max_wait = 300
+#     end
+#
+#     def perform(batch_items)
+#       SidekiqJobA.perform_async(batch_items)
+#       SidekiqJobB.perform_async(batch_items)
+#
+#       process_metrics(batch_items)
+#     end
+#   end
+#
+# 2) Start pushings items;
+#
+#   Myworker.push([{'id' => SecureRandom.uuid}, 'name' => 'Robert C Hall',  'action' => 'EMAIL_OPENNED'])
+#
+# 3) Start Upperkut;
+#
+#   $ bundle exec upperkut -worker MyWorker --concurrency 10
+#
+# 4) That's it :)
 module Upperkut
   class Configuration
     attr_accessor :batch_size, :redis, :strategy, :max_wait, :polling_interval
@@ -15,4 +60,6 @@ module Upperkut
       end
     end
   end
+
+  class Shutdown < StandardError ; end
 end

data/upperkut.gemspec

@@ -1,30 +1,29 @@
 
-lib = File.expand_path("../lib", __FILE__)
+lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require "upperkut/version"
+require 'upperkut/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "upperkut"
+  spec.name          = 'upperkut'
   spec.version       = Upperkut::VERSION
-  spec.authors       = ["Nando Sousa"]
-  spec.email         = ["nandosousafr@gmail.com"]
+  spec.authors       = ['Nando Sousa']
+  spec.email         = ['nandosousafr@gmail.com']
 
   spec.summary       = %q{Batch background processing tool}
   spec.description   = %q{Batch background processing tool}
-  spec.homepage      = "http://shipit.resultadosdigitais.com.br/open-source/"
-  spec.license       = "MIT"
-
-  #spec.metadata["allowed_push_host"] = "all"
+  spec.homepage      = 'http://shipit.resultadosdigitais.com.br/open-source/'
+  spec.license       = 'MIT'
 
   spec.files         = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
   end
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
+  spec.executables   = ['upperkut']
+  spec.require_paths = ['lib']
+
+  spec.required_ruby_version = '>= 2.2.2'
 
-  spec.add_dependency 'redis'
-  spec.add_development_dependency "bundler", "~> 1.16"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_dependency 'redis', '>= 3.3.5', '< 5'
+  spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
 end

metadata

@@ -1,12 +1,12 @@
 --- !ruby/object:Gem::Specification
 name: upperkut
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - Nando Sousa
 autorequire: 
-bindir: exe
+bindir: bin
 cert_chain: []
 date: 2018-06-24 00:00:00.000000000 Z
 dependencies:
@@ -16,14 +16,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.3.5
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.3.5
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -69,7 +75,8 @@ dependencies:
 description: Batch background processing tool
 email:
 - nandosousafr@gmail.com
-executables: []
+executables:
+- upperkut
 extensions: []
 extra_rdoc_files: []
 files:
@@ -106,7 +113,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.2.2
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="