upperkut 1.0.3 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 731a0ea07d1228421c4c5a306f3de6217e11f2a594e25e28ebf333f85056c9cc
-  data.tar.gz: 35680bb770644d47331488900de47a8ba1f2848d90cd8437511c1809ed15be32
+  metadata.gz: 961d354c22fd90d48b03c46718e3eb47aae82a0316fdac64ddd7d54c256aa04b
+  data.tar.gz: ee3d2b465a554d2cd3ec0b7ba8e42609cc2b57c60c971c878b0154bd75c1ba1d
 SHA512:
-  metadata.gz: 0521a72596013f4e2333bf28e2e189df95dd36a781375a90e45f0f59ee59c39d1026d80822fceb798cf9fb929335a82019f50ffe701cbd48d8dfe89daafa0b06
-  data.tar.gz: 3e0f9d1e0449aa4d6e70d76e9ea954b29624fca7efe549812d5aad3f76d653ef4fc787e70e371bd2c1b06a51e366fc8e0406f351cf9275fa008a948ee724e08d
+  metadata.gz: ec26662055db6114e55f6794116439d43268989871e47b77bdeb411ccc57d4062a0fda6a162be6fa3e93001ccd8c9cd8d935e82c49c162f9d3faa8e0618e0bf6
+  data.tar.gz: 1dff38c89064fe6cb9c1b94c3d4018ed71344ca95c034ec5f65129887da216f56917b35b03f507fb31a3374e05bb30006d327f759835d0ce5d90f688501b8e3c

data/.circleci/config.yml

@@ -0,0 +1,65 @@
+# Ruby CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-ruby/ for more details
+#
+version: 2
+jobs:
+  build:
+    docker:
+      - image: cimg/ruby:3.2
+        environment:
+          CC_TEST_REPORTER_ID: 03ab83a772148a577d29d4acf438d7ebdc95c632224122d0ba8dbb291eedebe6
+          COVERAGE: true
+          REDIS_URL: redis://localhost:6379
+      - image: redis:5.0
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+          - v1-dependencies-{{ checksum "Gemfile.lock" }}
+          # fallback to using the latest cache if no exact match is found
+          - v1-dependencies-
+
+      - run:
+          name: install dependencies
+          command: |
+            bundle install --jobs=4 --retry=3 --path vendor/bundle
+
+      - save_cache:
+          paths:
+            - ./vendor/bundle
+          key: v1-dependencies-{{ checksum "Gemfile.lock" }}
+
+      - run:
+          name: Setup Code Climate test-reporter
+          command: |
+            curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
+            chmod +x ./cc-test-reporter
+            ./cc-test-reporter before-build
+
+      # run tests!
+      - run:
+          name: run tests
+          command: |
+            mkdir /tmp/test-results
+            TEST_FILES="$(circleci tests glob "spec/**/*_spec.rb" | circleci tests split --split-by=timings)"
+
+            bundle exec rspec --format progress \
+                            --format RspecJunitFormatter \
+                            --out /tmp/test-results/rspec.xml \
+                            --format progress \
+                            $TEST_FILES
+      - run:
+          name: Submit coverage
+          command: |
+            ./cc-test-reporter after-build --coverage-input-type simplecov --exit-code $?
+
+      # collect reports
+      - store_test_results:
+          path: /tmp/test-results
+
+      - store_artifacts:
+          path: /tmp/test-results
+          destination: test-results

data/.codeclimate.yml

@@ -0,0 +1,15 @@
+version: "2"
+plugins:
+  brakeman:
+    enabled: false
+  eslint:
+    enabled: false
+  csslint:
+    enabled: false
+  rubocop:
+    enabled: true
+    channel: rubocop-0-63
+  reek:
+    enabled: true
+  bundler-audit:
+    enabled: true

data/.github/dependabot.yml

@@ -0,0 +1,12 @@
+version: 2
+updates:
+- package-ecosystem: bundler
+  directory: "/"
+  schedule:
+    interval: daily
+    time: "06:00"
+    timezone: Etc/UCT
+  pull-request-branch-name:
+    separator: "-"
+  open-pull-requests-limit: 10
+  rebase-strategy: disabled

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+*.gem

data/.rspec

@@ -0,0 +1,4 @@
+--format documentation
+--color
+--require spec_helper
+--order random

data/.rubocop.yml

@@ -0,0 +1,73 @@
+AllCops:
+  Exclude:
+    - 'vendor/**/*'
+    - 'spec/fixtures/**/*'
+    - 'tmp/**/*'
+  TargetRubyVersion: 3.2
+
+Naming/PredicateName:
+  MethodDefinitionMacros:
+    - define_method
+    - define_singleton_method
+    - def_node_matcher
+    - def_node_search
+
+Style/FrozenStringLiteralComment:
+  EnforcedStyle: always
+
+Style/TrailingCommaInHashLiteral:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Metrics/LineLength:
+  Max: 120
+
+Metrics/MethodLength:
+  Max: 20
+
+Layout/EndOfLine:
+  EnforcedStyle: lf
+
+Layout/ClassStructure:
+  Enabled: true
+  Categories:
+    module_inclusion:
+      - include
+      - prepend
+      - extend
+  ExpectedOrder:
+      - module_inclusion
+      - constants
+      - public_class_methods
+      - initializer
+      - instance_methods
+      - protected_methods
+      - private_methods
+
+Layout/TrailingWhitespace:
+  AllowInHeredoc: true
+
+Lint/AmbiguousBlockAssociation:
+  Exclude:
+    - 'spec/**/*.rb'
+
+Lint/InterpolationCheck:
+  Exclude:
+    - 'spec/**/*.rb'
+
+Lint/UselessAccessModifier:
+  MethodCreatingMethods:
+    - 'def_matcher'
+    - 'def_node_matcher'
+
+Metrics/BlockLength:
+  Exclude:
+    - 'Rakefile'
+    - '**/*.rake'
+    - 'spec/**/*.rb'
+
+Metrics/ModuleLength:
+  Exclude:
+    - 'spec/**/*.rb'

data/CHANGELOG.md

@@ -0,0 +1,45 @@
+# Upperkut changes
+0.8.x
+--------
+- Added exponential backoff when push_items #57
+- Introducing Item to avoid losing enqueued at and report wrong latency
+  metrics #56 thanks to @jeangnc
+
+0.7.x
+---------
+- Fix logging timeout message #54 by @jeanmatheussouto
+- Add handle_error method #44
+- Added Datahog Middleware (#42)
+- Added Priority Queue (#39) thanks to @jeangnc and @jeanmatheussouto
+- Added Scheduled Queue Implementation thanks to @rodrigo-araujo #38
+- Added Datahog middleware #42 by @gabriel-augusto
+- Added redis to CI #40 by #henrich-m
+- Specs improvements #34 and #35, #37 by @gabriel-augusto
+- Enable Rubocop #32 by @henrich-m
+- Added codeclimate #31 by @henrich-m
+- Extract Buffered Queue behavior to its own strategy #29
+
+0.6.x
+---------
+- Set redis url when env var REDIS_URL is set thanks to @lucaskds #22
+- Enable users to pass their own connection pool also thanks to @lucaskds #21
+- Sets default concurrency to 1
+- Creates connection pools by default;
+- Simplifies API to enable extension via Strategy;
+
+0.5.x
+----------
+- Introducing client middlewares.
+- Fix issue that prevented worker to run.
+- Fix thread-unsafe code that was overwriting worker configurations when
+multiples workers were configured in the same process.
+
+0.4.x
+-----------
+
+- Change redis versioning policy #17
+- Fixed error that prevented users from start upperkut from non Rails apps.
+- Added integration with Rollbar.
+- Added backtrace to logs when some error occurs.
+- Fixed NewRelic trace args
+

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at nandosousafr@gmail.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/Dockerfile

@@ -0,0 +1,7 @@
+FROM ruby:3.2
+
+WORKDIR /code
+COPY . .
+
+RUN gem install bundler
+RUN bundle install

data/Gemfile

@@ -0,0 +1,11 @@
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in upperkut.gemspec
+gemspec
+
+gem 'fivemat'
+gem 'pry'
+gem 'rspec_junit_formatter'
+gem 'simplecov', '< 0.23', require: false

data/Gemfile.lock

@@ -0,0 +1,61 @@
+PATH
+  remote: .
+  specs:
+    upperkut (1.0.4)
+      connection_pool (~> 2.2, >= 2.2.2)
+      redis (>= 4.1.0, < 6.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    coderay (1.1.3)
+    connection_pool (2.3.0)
+    diff-lcs (1.5.0)
+    docile (1.4.0)
+    fivemat (1.3.7)
+    method_source (1.0.0)
+    pry (0.14.2)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rake (13.0.6)
+    redis (5.0.6)
+      redis-client (>= 0.9.0)
+    redis-client (0.12.1)
+      connection_pool
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.1)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.4)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.0)
+    rspec_junit_formatter (0.6.0)
+      rspec-core (>= 2, < 4, != 2.12.0)
+    simplecov (0.22.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.4)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (>= 1.16)
+  fivemat
+  pry
+  rake (~> 13.0)
+  rspec (~> 3.0)
+  rspec_junit_formatter
+  simplecov (< 0.23)
+  upperkut!
+
+BUNDLED WITH
+   2.4.1

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Nando Sousa
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Makefile

@@ -0,0 +1,4 @@
+bash:
+	docker-compose run gem bash
+specs:
+	docker-compose run gem bundle exec rspec

data/README.md

@@ -0,0 +1,162 @@
+# Upperkut
+
+[![CircleCI](https://circleci.com/gh/ResultadosDigitais/upperkut/tree/master.svg?style=svg&circle-token=693e512de6985be3b3db12279ba6ed508fb5c6f6)](https://circleci.com/gh/ResultadosDigitais/upperkut/tree/master)
+[![Maintainability](https://api.codeclimate.com/v1/badges/ece40319b0db03af891d/maintainability)](https://codeclimate.com/repos/5b318a7c6d37b70272008676/maintainability)
+[![Test Coverage](https://api.codeclimate.com/v1/badges/ece40319b0db03af891d/test_coverage)](https://codeclimate.com/repos/5b318a7c6d37b70272008676/test_coverage)
+
+[[Docs]](https://www.rubydoc.info/gems/upperkut/0.7.2/Upperkut)
+
+Background processing framework for Ruby applications.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'upperkut'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install upperkut
+
+## Usage
+
+### Example 1 - Buffered Queue:
+
+1) Create a Worker class and the define how to process the batch;
+  ```ruby
+  class MyWorker
+    include Upperkut::Worker
+
+    def perform(batch_items)
+      heavy_processing(batch_items)
+      process_metrics(batch_items)
+    end
+  end
+  ```
+
+2) Start pushing items;
+  ```ruby
+  MyWorker.push_items(
+    [
+      {
+        'id' => SecureRandom.uuid,
+        'name' => 'Robert C Hall',
+        'action' => 'EMAIL_OPENNED'
+      }
+    ]
+  )
+  ```
+
+3) Start Upperkut;
+  ```bash
+  $ bundle exec upperkut --worker MyWorker --concurrency 10
+  ```
+
+### Example 2 - Scheduled Queue:
+
+1) Create a Worker class and the define how to process the batch;
+  ```ruby
+  require 'upperkut/strategies/scheduled_queue'
+  class MyWorker
+    include Upperkut::Worker
+
+    setup_upperkut do |config|
+      config.strategy = Upperkut::Strategies::ScheduledQueue.new(self)
+    end
+
+    def perform(batch_items)
+      heavy_processing(batch_items)
+      process_metrics(batch_items)
+    end
+  end
+  ```
+
+2) Start pushing items with `timestamp` parameter;
+  ```ruby
+  # timestamp is 'Thu, 10 May 2019 23:43:58 GMT'
+  MyWorker.push_items(
+    [
+      {
+        'timestamp' => '1557531838',
+        'id' => SecureRandom.uuid,
+        'name' => 'Robert C Hall',
+        'action' => 'SEND_NOTIFICATION'
+      }
+    ]
+  )
+  ```
+
+3) Start Upperkut;
+  ```bash
+  $ bundle exec upperkut --worker MyWorker --concurrency 10
+  ```
+
+### Example 3 - Priority Queue:
+
+Note: priority queues requires redis 5.0.0+ as it uses ZPOP* commands.
+
+1) Create a Worker class and the define how to process the batch;
+  ```ruby
+  require 'upperkut/strategies/priority_queue'
+
+  class MyWorker
+    include Upperkut::Worker
+
+    setup_upperkut do |config|
+      config.strategy = Upperkut::Strategies::PriorityQueue.new(
+        self,
+        priority_key: -> { |item| item['tenant_id'] }
+      )
+    end
+
+    def perform(items)
+      items.each do |item|
+        puts "event dispatched: #{item.inspect}"
+      end
+    end
+  end
+  ```
+
+2) So you can enqueue items from different tenants;
+  ```ruby
+  MyWorker.push_items(
+    [
+      { 'tenant_id' => 1, 'id' => 1 },
+      { 'tenant_id' => 1, 'id' => 2 },
+      { 'tenant_id' => 1, 'id' => 3 },
+      { 'tenant_id' => 2, 'id' => 4 },
+      { 'tenant_id' => 3, 'id' => 5 },
+    ]
+  )
+  ```
+
+  The code above will enqueue items as follows `1, 4, 5, 2, 3`
+
+3) Start Upperkut;
+  ```bash
+  $ bundle exec upperkut --worker MyWorker --concurrency 10
+  ```
+
+## Development
+
+After checking out the repo, run `bundle install` to install dependencies. Then, run `bundle exec rspec` to run the tests.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/ResultadosDigitais/upperkut. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the Upperkut project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/ResultadosDigitais/upperkut/blob/master/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,6 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/catalog-info.yaml

@@ -0,0 +1,15 @@
+apiVersion: backstage.io/v1alpha1
+kind: Component
+metadata:
+  name: upperkut
+  description: Ferramenta de processamento em segundo plano para Ruby.
+  annotations:
+    circleci.com/project-slug: github/ResultadosDigitais/upperkut
+    github.com/project-slug: ResultadosDigitais/upperkut
+  links:
+    - url: https://opensource.resultadosdigitais.com.br/
+      title: Projetos open source da RD.
+spec:
+  type: library
+  owner: devtools
+  lifecycle: production

data/docker-compose.yml

@@ -0,0 +1,18 @@
+services:
+  gem:
+    build: .
+    volumes:
+      - .:/code
+    environment:
+      - REDIS_URL=redis://redis:6379
+    depends_on:
+      - redis
+  redis:
+    image: redis:5.0.4-alpine
+    command: redis-server --save "" --appendonly yes --appendfsync everysec
+    ports:
+      - 6379:6379
+    volumes:
+      - redis-data:/data
+volumes:
+  redis-data:

data/examples/basic.rb

@@ -0,0 +1,12 @@
+require_relative '../lib/upperkut/worker'
+
+class MyWorker
+  include Upperkut::Worker
+
+  def perform(items)
+    puts 'starting performing'
+    exec_time = rand(9000..50_000)
+    sleep (exec_time.to_f / 1000.to_f)
+    puts "performed #{items.size} items in #{exec_time.to_f / 1000.to_f} ms"
+  end
+end

data/examples/priority_worker.rb

@@ -0,0 +1,21 @@
+require_relative '../lib/upperkut/worker'
+require_relative '../lib/upperkut/strategies/priority_queue'
+
+class PriorityWorker
+  include Upperkut::Worker
+
+  setup_upperkut do |config|
+    config.strategy = Upperkut::Strategies::PriorityQueue.new(
+      self,
+      priority_key: -> { |item| item['tenant_id'] },
+      batch_size: 1
+    )
+  end
+
+  def perform(items)
+    items.each do |item|
+      puts "event dispatched: #{item.inspect}"
+      sleep 1
+    end
+  end
+end