bg_results 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: cebb816663b5bd96fa1ff2c765736ed93b6547dcdd565a4c382f252e5288c6d5
+  data.tar.gz: d3a31791a234e31a826d792ac857bf3a12380cf3d490c0bf7fcec975d08e7189
+SHA512:
+  metadata.gz: 109900b1428de3d74f39d3fd122b062acafb7cd8fa59f9404129aaba3b3bd3f62d4218dd4cba5d6b837f8f015b192ba0970baebb61c7211b1061721faab16fa6
+  data.tar.gz: 58fed360f9505a4c6347aa0586b58b348422537ecefcc2c6123eb34c8ec0135759b8876ce55bf3be5ee4a42f257e1cc4ce0cac2e514d44ce4a45912288e1dc97

data/.gitignore

@@ -0,0 +1,8 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.1
+before_install: gem install bundler -v 1.16.0

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in bg_results.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,29 @@
+PATH
+  remote: .
+  specs:
+    bg_results (0.1.0)
+      connection_pool (~> 2.0)
+      redis
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    connection_pool (2.2.2)
+    fakeredis (0.7.0)
+      redis (>= 3.2, < 5.0)
+    minitest (5.11.3)
+    rake (10.5.0)
+    redis (4.0.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bg_results!
+  bundler (~> 1.16)
+  fakeredis (~> 0.7)
+  minitest (~> 5.0)
+  rake (~> 10.0)
+
+BUNDLED WITH
+   1.17.1

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Jason
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,99 @@
+# BgResults
+
+A simple tool for collecting results from background job batches.
+Results returned from the jobs are stored in a Redis hash.
+
+Currently Sidekiq is supported using the 3rd party batch gem, but it should work with Sidekiq pro as well. Worker modules for other queue systems can easily be added to support different providers.
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'bg_results'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install bg_results
+
+## Usage
+
+### Configure Redis
+
+A Redis connection pool is maintained to support flexibility from queue provider.
+
+Redis can be configured using the `REDIS_URL` environment variable.
+
+Otherwise you can supply your own connection pool or use Sidekiq's.
+
+```ruby
+BgResults.redis_pool = your_pool || Sidekiq.redis_pool
+```
+### Data
+
+Results are stored in redis as JSON strings. Any datatype valid in JSON will work.
+
+### Workers
+To enable results in a sidekiq worker `prepend BgResults::Workers::Sidekiq` and the return value of `perform` will be collected.
+```ruby
+class ResultsJobThe
+  include Sidekiq::Worker
+  prepend BgResults::Worker::Sidekiq
+
+  def perform
+    result = background_task
+  end
+end
+```
+
+###  Collect results
+Use the results batch object to access background results in the batch callbacks. Results are identified by batch_id.
+
+```ruby
+def on_success status, options
+  batch_res = BgResults::Batch.new status.bid
+end
+```
+
+#### All in one
+Return all results in a hash with mapping `{job_id => result}`.
+
+```ruby
+results = batch_res.results
+```
+
+#### Scan
+Collect results in batches with redis scan.
+```ruby
+batch_res.results_each do |job_id, result|
+  puts result
+end
+```
+
+#### Lifetime
+Results are persisted for up to 24 hours after first access to allow for callback retry and debugging.
+
+In order to prevent redis from filling up with large results they should be cleared manually upon success.
+```ruby
+batch_res.clear_results!
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/bg_results.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+task :default => :test

data/bg_results.gemspec

@@ -0,0 +1,40 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "bg_results/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "bg_results"
+  spec.version       = BgResults::VERSION
+  spec.authors       = ["Jason"]
+  spec.email         = ["jabrady42@gmail.com"]
+
+  spec.summary       = %q{A tool for handling results from background workers}
+  spec.description   = %q{A tool for handling results from background workers}
+  spec.homepage      = "https://github.com/jbrady42/bg_results"
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata["allowed_push_host"] = "https://rubygems.org"
+  else
+    raise "RubyGems 2.0 or newer is required to protect against " \
+      "public gem pushes."
+  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "redis"
+  spec.add_dependency "connection_pool", "~>2.0"
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "fakeredis", "~> 0.7"
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "bg_results"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/bg_results.rb

@@ -0,0 +1,23 @@
+require "connection_pool"
+require "redis"
+require "json"
+
+require "bg_results/version"
+require "bg_results/batch"
+require "bg_results/worker/sidekiq"
+
+module BgResults
+  def self.redis_pool
+    @redis ||= ConnectionPool.new(size: 5, timeout: 5) { Redis.new }
+  end
+
+  def self.redis_pool= pool
+    @redis = pool
+  end
+
+  def self.redis
+    redis_pool.with do |redis|
+      yield redis
+    end
+  end
+end

data/lib/bg_results/batch.rb

@@ -0,0 +1,67 @@
+module BgResults
+  class Batch
+    RESULT_TTL = 86400 # 1 day
+    def initialize bid
+      @bid = bid
+      @key = "RESULTS-#{@bid}"
+    end
+
+    def results
+      data, _ = BgResults.redis do |conn|
+        conn.multi do
+          conn.hgetall @key
+          conn.expire @key, RESULT_TTL
+        end
+      end
+      parse_results data
+    end
+
+    def results_in_batches
+      count = 100
+      cursor = 0
+      loop do
+        cursor, data = scan cursor, count
+        data = parse_results data.to_h
+        yield data
+        break if cursor == 0
+      end
+      set_results_expire
+    end
+
+    def results_each
+      return unless block_given?
+      results_in_batches do |batch|
+        batch.each do |jid, res|
+          yield jid, res
+        end
+      end
+    end
+
+    def set_results_expire exp=RESULT_TTL
+      BgResults.redis do |conn|
+        conn.expire @key, exp
+      end
+    end
+
+    def clear_results!
+      BgResults.redis do |conn|
+        conn.del @key
+      end
+    end
+
+  private
+    def scan cursor, count
+      new_cursor, data = BgResults.redis do |conn|
+        conn.hscan @key, cursor, count: count
+      end
+      [new_cursor.to_i, data]
+    end
+
+    def parse_results data
+      data = data.map do |k,v|
+        payload = JSON.parse v
+        [k, payload["result"]]
+      end.to_h
+    end
+  end
+end

data/lib/bg_results/version.rb

@@ -0,0 +1,3 @@
+module BgResults
+  VERSION = "0.1.0"
+end

data/lib/bg_results/worker/sidekiq.rb

@@ -0,0 +1,15 @@
+module BgResults
+  module Worker
+    module Sidekiq
+      def perform *args
+        result = super
+        if bid && jid
+          data = {result: result}.to_json
+          BgResults.redis do |conn|
+            conn.hset "RESULTS-#{bid}", jid, data
+          end
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,143 @@
+--- !ruby/object:Gem::Specification
+name: bg_results
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jason
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2019-10-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: connection_pool
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: fakeredis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+description: A tool for handling results from background workers
+email:
+- jabrady42@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bg_results.gemspec
+- bin/console
+- bin/setup
+- lib/bg_results.rb
+- lib/bg_results/batch.rb
+- lib/bg_results/version.rb
+- lib/bg_results/worker/sidekiq.rb
+homepage: https://github.com/jbrady42/bg_results
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.8
+signing_key: 
+specification_version: 4
+summary: A tool for handling results from background workers
+test_files: []