distributed_resque_worker 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 215e94466ab47726a61e6a2b82cc4f341a70d0fa7b003365ba183b44b8082e78
+  data.tar.gz: 87649120c3f63c429e41aaaf745cfdb5517abd829826617eb12ee5b4919e1a7e
+SHA512:
+  metadata.gz: b314648eaa5206f2b949355a8a8cd90d0be7cbbf182003f85d9e9bab2304235135e0de04242216c1bc77683a9d806b4886cf0feffb6b4eb5da79ef842b5631b3
+  data.tar.gz: 989d6433ab4767c92387f65bad4522c57504444aef6adfc0e93b8cebd2726495a1262ee0ec9bff5dc0d604af7361436ee4c114a0e77c1d10acec58d368f2a967

data/.circleci/config.yml

@@ -0,0 +1,59 @@
+version: 2
+jobs:
+  build:
+    working_directory: ~/YourMechanic/distributed_resque_worker
+    parallelism: 1
+    shell: /bin/bash --login
+
+    environment:
+      CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
+      CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
+    docker:
+      - image: circleci/ruby:2.3.8
+        environment:
+          RAILS_ENV: test
+      - image: circleci/postgres:9.5.15
+        environment:
+      - image: redis
+    steps:
+      - checkout
+      - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS
+      - restore_cache:
+          keys:
+          # This branch if available
+          - v2-dep-{{ .Branch }}-
+          # Default branch if not
+          - v2-dep-master-
+          # Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly
+          - v2-dep-
+      - run:
+          name: Install bundler for distributed_resque_worker
+          working_directory: ~/YourMechanic/distributed_resque_worker
+          command: gem install bundler:1.17.3
+      - run:
+          name: Install Gems for distributed_resque_worker
+          working_directory: ~/YourMechanic/distributed_resque_worker
+          command: bundle install
+      # Save dependency cache
+      - save_cache:
+          key: v2-dep-{{ .Branch }}-{{ epoch }}
+          paths:
+          - vendor/bundle
+          - ~/virtualenvs
+          - ~/.m2
+          - ~/.ivy2
+          - ~/.bundle
+      - run:
+          working_directory: ~/YourMechanic/distributed_resque_worker
+          command: bundle exec rubocop
+      - run:
+          working_directory: ~/YourMechanic/distributed_resque_worker
+          command: bundle exec rspec --colour --drb --profile -fd --format progress $(circleci tests glob "spec/*_spec.rb" | circleci tests split)
+      - store_test_results:
+          path: /tmp/circleci-test-results
+      - store_artifacts:
+          path: /tmp/circleci-artifacts
+      - store_artifacts:
+          path: /tmp/circleci-test-results
+      - store_artifacts:
+          path: distributed_resque_worker/coverage

data/.gitignore

@@ -0,0 +1,56 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/*
+/log/*
+# Used by dotenv library to load environment variables.
+# .env
+
+# Ignore Byebug command history file.
+.byebug_history
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+#
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+# Used by RuboCop. Remote config files pulled in from inherit_from directive.
+# .rubocop-https?--*

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rspec_status

@@ -0,0 +1,6 @@
+example_id                                    | status | run_time        |
+--------------------------------------------- | ------ | --------------- |
+./spec/distributed_resque_worker_spec.rb[1:1] | passed | 0.00282 seconds |
+./spec/distributed_resque_worker_spec.rb[1:2] | passed | 1.04 seconds    |
+./spec/distributed_resque_worker_spec.rb[1:3] | passed | 10 seconds      |
+./spec/distributed_resque_worker_spec.rb[1:4] | passed | 0.01556 seconds |

data/.rubocop.yml

@@ -0,0 +1,18 @@
+inherit_from: .rubocop_todo.yml
+
+Style/ModuleFunction:
+  Exclude:
+    - 'spec/**/*'
+
+Style/NumericLiterals:
+  MinDigits: 20
+Metrics/MethodLength:
+  Max: 100
+Metrics/BlockLength:
+  Max: 10
+Layout/LineLength:
+  Max: 80
+Metrics/AbcSize:
+  Max: 25
+Metrics/ClassLength:
+  Max: 150

data/.rubocop_todo.yml

@@ -0,0 +1,7 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2021-04-08 16:20:29 +0530 using RuboCop version 0.81.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.

data/.ruby-version

@@ -0,0 +1 @@
+2.3.8

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at trupti@yourmechanic.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in distributed_resque_worker.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,99 @@
+PATH
+  remote: .
+  specs:
+    distributed_resque_worker (1.0.0)
+      aws-sdk (= 1.11.1)
+      resque (~> 2.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.4.0)
+    ast (2.4.2)
+    aws-sdk (1.11.1)
+      json (~> 1.4)
+      nokogiri (>= 1.4.4)
+      uuidtools (~> 2.1)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    diff-lcs (1.4.4)
+    hashdiff (0.3.7)
+    jaro_winkler (1.5.4)
+    json (1.8.6)
+    mini_portile2 (2.4.0)
+    mono_logger (1.1.1)
+    multi_json (1.15.0)
+    mustermann (1.1.1)
+      ruby2_keywords (~> 0.0.1)
+    nokogiri (1.10.10)
+      mini_portile2 (~> 2.4.0)
+    parallel (1.19.2)
+    parser (3.0.1.0)
+      ast (~> 2.4.1)
+    rack (2.2.3)
+    rack-protection (2.1.0)
+      rack
+    rainbow (3.0.0)
+    rake (10.5.0)
+    redis (4.4.0)
+    redis-namespace (1.6.0)
+      redis (>= 3.0.4)
+    resque (2.0.0)
+      mono_logger (~> 1.0)
+      multi_json (~> 1.0)
+      redis-namespace (~> 1.6)
+      sinatra (>= 0.9.2)
+      vegas (~> 0.1.2)
+    rexml (3.2.5)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.2)
+    rubocop (0.81.0)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.7.0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      rexml
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    ruby-progressbar (1.11.0)
+    ruby2_keywords (0.0.5)
+    safe_yaml (1.0.4)
+    sinatra (2.1.0)
+      mustermann (~> 1.0)
+      rack (~> 2.2)
+      rack-protection (= 2.1.0)
+      tilt (~> 2.0)
+    tilt (2.0.10)
+    unicode-display_width (1.7.0)
+    uuidtools (2.2.0)
+    vegas (0.1.11)
+      rack (>= 1.0.0)
+    webmock (3.4.2)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.17.3)
+  distributed_resque_worker!
+  rake (~> 10.0)
+  rspec (~> 3.0)
+  rubocop (~> 0.48)
+  webmock (~> 3.4.2)
+
+BUNDLED WITH
+   1.17.3

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 YourMechanic
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,91 @@
+# distributed_resque_worker
+Distributed Background Worker to process a large workload
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/distributed_resque_worker`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'distributed_resque_worker', git: 'https://github.com/YourMechanic/distributed_resque_worker.git', :branch => 'master'
+```
+And then execute:
+    $ bundle install
+
+Add the following in routes.rb
+    require 'resque/server'
+    mount Resque::Server.new, at: '/resque'
+
+
+Setup Redis: Add the following in the config/initializers/resque.yml
+```
+development: localhost:6379
+test: localhost:6379
+staging: <%= ENV['REDIS_URL'] %>
+production: <%= ENV['REDIS_URL'] %>
+```
+
+Configure Resque: Add the following in the config/initializers/resque.rb
+```
+resque_config = YAML.load(File.read(Rails.root.join('config/resque.yml')))
+Resque.redis = resque_config[Rails.env]
+
+Resque.logger = Logger.new(Rails.root.join('log', "#{Rails.env}_resque.log"))
+Resque.logger.level = Logger::INFO
+```
+
+In your Rakefile, or some other file in lib/tasks (ex: lib/tasks/resque.rake), load the resque rake tasks (For Rails 3+):
+```
+require 'resque/tasks'
+task 'resque:setup' => :environment
+```
+
+## Usage
+Can use the following code to chunk the work and run on 4 workers using DistributedResqueWorker and you also need to add a function that workers will run with their chunk of work
+```
+class SomeClass
+    ...
+    def some_method
+        opts = {address:'', requestor: 'trupti', emails: [], requested_date: '', ...}
+        input = (1..4).to_a
+
+        worker = DistributedResqueWorker::ResqueWorker.new(
+            self,
+            'bucket_name',
+            'root_dir_to_stor_tmp_files'
+        )
+        worker.chunk_work_and_enqueue(input, __callee__, opts)
+    end
+    ...
+
+    def self.some_method_chunk(work_chunk, output_filepath, opts = {})
+        # some code to work on the work_chunk and add data into output_filepath
+    end
+
+    def self.some_method_post(work_chunk, output_filepath, opts = {})
+        # some code to either send email/slack message the final file s3 link
+    end
+end
+```
+
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/distributed_resque_worker. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the DistributedResqueWorker project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/distributed_resque_worker/blob/master/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+require 'rubocop/rake_task'
+require 'resque'
+require 'resque/tasks'
+
+RSpec::Core::RakeTask.new(:spec)
+RuboCop::RakeTask.new
+
+task 'resque:setup' do
+  require_relative 'lib/distributed_resque_worker'
+end
+
+task default: %i[spec rubocop resque]

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'distributed_resque_worker'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/distributed_resque_worker.gemspec

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'distributed_resque_worker/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'distributed_resque_worker'
+  spec.version       = DistributedResqueWorker::VERSION
+  spec.authors       = ['TruptiHosmani']
+  spec.email         = ['dev@yourmechanic.com']
+
+  spec.summary       = 'Gem for downloadable reports using Resque'
+  spec.description   = 'Downloadable reports which uses background workers'\
+                       'and then emails an S3 link for CSV to the requestor'
+  spec.homepage      = 'https://github.com/YourMechanic/distributed_resque_worker'
+  spec.license       = 'MIT'
+  spec.required_ruby_version = Gem::Requirement.new('>= 2.3')
+  spec.metadata['allowed_push_host'] = 'https://rubygems.org'
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = 'https://github.com/YourMechanic/distributed_resque_worker'
+
+  # Specify which files should be added to the gem when it is released.
+  # `git ls-files -z` loads the files in RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      f.match(%r{\A(?:test|spec|features)/})
+    end
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.17.3'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rubocop', '~> 0.48'
+  spec.add_development_dependency 'webmock', '~> 3.4.2'
+  spec.add_dependency 'aws-sdk', '1.11.1'
+  spec.add_dependency 'resque', '~> 2.0'
+end

data/lib/distributed_resque_worker.rb

@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+require 'distributed_resque_worker/version'
+require 'csv'
+require 'resque'
+
+require_relative 'distributed_resque_worker/aws_helper'
+require_relative 'distributed_resque_worker/resque_tester'
+
+# DistributedResqueWorker
+module DistributedResqueWorker
+  # ResqueFailure
+  module ResqueFailure
+    def on_failure_logging(error, *args)
+      msg = "Performing #{self} caused an exception #{error} with args #{args}."
+      Resque.logger.info msg
+    end
+  end
+
+  # ResqueWorker
+  class ResqueWorker
+    CHUNK_SIZE = 100
+    extend ResqueFailure
+
+    def initialize(queue_name, bucket, root)
+      @queue = "#{queue_name}_#{Time.now.to_i}_#{Random.rand(1000000)}".to_sym
+      @bucket = bucket
+      @root = root
+      FileUtils.mkdir_p("#{root}/tmp/#{@queue}")
+    end
+
+    def chunk_work_and_enqueue(work_list, method, opts)
+      total_jobs = (work_list.length.to_f / CHUNK_SIZE).ceil
+      total_jobs = 1 if total_jobs.zero?
+      Resque.logger.info("total_jobs #{total_jobs}")
+      ResqueWorker.resque_redis.redis.set(
+        ResqueWorker.resque_worker_redis_key(@queue), total_jobs
+      )
+      work_list.each_slice(CHUNK_SIZE).each_with_index do |chunk, index|
+        details = { work_name: @queue, chunk: chunk, index: index,
+                    type: 'processor', root: @root.to_s,
+                    bucket: @bucket, method: method, opts: opts }
+        Resque.enqueue_to(@queue, ResqueWorker, details)
+      end
+    end
+
+    def self.perform(args)
+      redis_key = resque_worker_redis_key(args['work_name'])
+      no_jobs = resque_redis.redis.get(redis_key)
+      Resque.logger.info("No of jobs remaining => #{no_jobs}")
+      if args['type'] == 'processor'
+        process_chunk(args)
+        if resque_redis.redis.get(redis_key).to_i.zero?
+          enqueue_post_processor(args)
+        end
+      elsif args['type'] == 'post_processor'
+        post_processing(args)
+      end
+    end
+
+    class << self
+      def resque_redis
+        Resque.redis
+      end
+
+      def resque_worker_redis_key(work_name)
+        "DistributedResqueWorker:#{work_name}"
+      end
+
+      def merge_intermediate_files(work_name, final_file)
+        files = "tmp/#{work_name}/#{work_name}_*.csv"
+        cmd = "awk '(NR == 1) || (FNR > 1)' #{files} > #{final_file}"
+        system(`#{cmd}`)
+      end
+
+      def enqueue_post_processor(args)
+        Resque.logger.info('start enqueue_post_processor')
+        input = args.symbolize_keys!
+        details = { type: 'post_processor', work_name: input[:work_name],
+                    bucket: input[:bucket], method: input[:method],
+                    root: input[:root], opts: input[:opts] }
+        Resque.enqueue_to(input[:work_name], ResqueWorker, details)
+        Resque.logger.info('finished enqueue_post_processor')
+      end
+
+      def process_chunk(args)
+        input = args.symbolize_keys!
+        method_chunk = "#{input[:method]}_chunk".to_sym
+        worker_class = input[:work_name].split('_').first
+        worker = worker_class.constantize
+        path = "#{input[:work_name]}/#{input[:work_name]}_#{input[:index]}.csv"
+        filename = "#{input[:root]}/tmp/#{path}"
+        worker.send(method_chunk, input[:chunk], filename, input[:opts])
+        store_to_s3_delete_local_copy(path, filename, input[:bucket])
+        resque_redis.redis.decr(resque_worker_redis_key(input[:work_name]))
+      end
+
+      def store_to_s3_delete_local_copy(path, filename, bucket)
+        s3_name = "resque_worker/#{path}"
+        begin
+          AwsHelper.s3_store_file(s3_name, filename, bucket)
+          File.delete(filename)
+        rescue StandardError
+          Resque.logger.error($ERROR_INFO)
+          nil
+        end
+      end
+
+      def post_processing(args)
+        input = args.symbolize_keys!
+        work_name = input[:work_name]
+        root = input[:root]
+        final_tmp_file = "#{root}/tmp/#{work_name}/#{work_name}_final.csv"
+        Resque.logger.info("start post_processing #{input}")
+        begin
+          download_intermediate_files(work_name, input[:bucket], root)
+          delete_intermediate_s3_files(work_name, input[:bucket])
+          merge_intermediate_files(work_name, final_tmp_file)
+          upload_final_file_to_s3_and_send(input, final_tmp_file)
+        rescue StandardError
+          Resque.logger.error($ERROR_INFO)
+          nil
+        end
+        Resque.logger.info('finished post_processing ')
+      end
+
+      def download_intermediate_files(work_name, bucket, root)
+        aws_bucket = AwsHelper.bucket(bucket)
+        folder = "resque_worker/#{work_name}/"
+        s3_object = aws_bucket.objects.with_prefix(folder)
+        s3_file_names = s3_object.collect(&:key)
+        s3_file_names.each do |filename|
+          local_file_name = filename.split('/')
+          next unless local_file_name[2]
+
+          download_file_path = "#{root}/tmp/#{work_name}/#{local_file_name[2]}"
+          Resque.logger.info("download_file_path #{download_file_path} ")
+          AwsHelper.s3_download_file(filename, download_file_path, bucket)
+        end
+      end
+
+      def delete_intermediate_s3_files(work_name, bucket)
+        aws_bucket = AwsHelper.bucket(bucket)
+        folder = "resque_worker/#{work_name}/"
+        s3_object = aws_bucket.objects.with_prefix(folder)
+        s3_file_names = s3_object.collect(&:key)
+        s3_file_names.each do |item|
+          AwsHelper.s3_delete(item, bucket)
+        end
+      end
+
+      def upload_final_file_to_s3_and_send(input, final_tmp_file)
+        work_name = input[:work_name]
+        s3_name = "resque_worker/#{work_name}/#{work_name}_final.csv"
+        final_file_link = AwsHelper.s3_store_file(s3_name, final_tmp_file,
+                                                  input[:bucket])
+        method_post = "#{input[:method]}_post".to_sym
+        worker_class = input[:work_name].split('_').first
+        worker = worker_class.constantize
+        worker.send(method_post, final_file_link, input[:opts])
+        clean_up(work_name, input[:root])
+      end
+
+      def clean_up(queue_name, root)
+        FileUtils.remove_dir("#{root}/tmp/#{queue_name}")
+
+        delete_queue(queue_name)
+        Resque.logger.info('Cleanup Done!')
+      end
+
+      def delete_queue(queue_name)
+        Resque.queues.each do |queue|
+          Resque.remove_queue(queue) if queue == queue_name
+        end
+      end
+    end
+  end
+end

data/lib/distributed_resque_worker/aws_helper.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require 'aws-sdk'
+
+# AwsHelper
+module AwsHelper
+  module_function
+
+  def bucket(bucket)
+    @bucket = AWS::S3.new.buckets[bucket]
+  end
+
+  def s3_store_file(name, file, bucket_name, opt = {})
+    # Stream the content for storage
+    File.open(file, 'rb') do |f|
+      return s3_store(name, f, bucket_name, opt)
+    end
+  end
+
+  def s3_store(name, content, bucket_name, opt = {})
+    run_with_retry do
+      obj = s3_get_object(name, bucket_name)
+      obj.write(content, opt)
+      obj.exists? ? s3_get_object_url(name, bucket_name) : nil
+    end
+  end
+
+  def run_with_retry
+    maxtry = 3
+    ntry = 0
+    begin
+      yield
+    rescue AWS::S3::Errors::RequestTimeout
+      ntry += 1
+      if ntry > maxtry
+        Resque.logger.info($ERROR_INFO)
+        raise
+      end
+      print "Error: #{$ERROR_INFO}, retrying\n"
+      @bucket = nil # So that we create a new bucket
+      retry
+    end
+  end
+
+  def doomsday
+    expiration = Time.zone.now + 20.years
+    # TODO: ::Update after AWS changes this limit. Will likely be a while
+    # since it depends on global transition to 64-bit systems
+    #
+    # AWS sets 01/20/2038 as an upper limit threshold on expiration date
+    # due to https://en.wikipedia.org/wiki/Year_2038_problem
+    aws_max_date = Time.zone.parse('2038-01-18')
+    expiration = aws_max_date if expiration > aws_max_date
+    expiration
+  end
+
+  def s3_get_object_url(name, bucket_name, _opt = {})
+    obj = s3_get_object(name, bucket_name)
+    return nil unless obj&.exists?
+
+    secure = true
+    obj.url_for(:read, secure: secure, expires: doomsday).to_s
+  end
+
+  def s3_delete(name, bucket_name)
+    run_with_retry { s3_get_object(name, bucket_name).delete }
+  end
+
+  def s3_get_object(name, bucket_name)
+    bucket = bucket(bucket_name)
+    bucket.objects[name]
+  end
+
+  def s3_download_file(name, filename, bucket_name)
+    run_with_retry do
+      data = s3_get_object(name, bucket_name).read
+      File.open(filename, 'wb') do |file|
+        file.write(data)
+      end
+      nil
+    end
+  end
+
+  CONTENT_TYPE_TO_EXT = {
+    'audio/amr' => '.amr',
+    'audio/acc' => '.mp4',
+    'audio/mp4' => '.mp4',
+    'audio/mpeg' => '.mp3',
+    'audio/ogg' => '.ogg',
+    'image/jpeg' => '.jpg',
+    'image/png' => '.png',
+    'image/gif' => '.gif',
+    'text/plain' => '.txt',
+    'text/rtf' => '.rtf',
+    'application/zip' => '.zip',
+    'application/pdf' => '.pdf',
+    'application/msword' => '.doc'
+  }.freeze
+
+  def content_type(ext)
+    CONTENT_TYPE_TO_EXT.each do |ct, cext|
+      return ct if ext == cext
+    end
+  end
+
+  def content_ext(content_type)
+    CONTENT_TYPE_TO_EXT[content_type] || ''
+  end
+end

data/lib/distributed_resque_worker/resque_tester.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative '../distributed_resque_worker'
+
+# ResqueTester
+class ResqueTester
+  def self.run(bucket)
+    input = (1..4).to_a
+    worker = DistributedResqueWorker::ResqueWorker.new(
+      'ResqueTester',
+      bucket,
+      Dir.pwd
+    )
+    worker.chunk_work_and_enqueue(input, __callee__, {})
+  end
+
+  def self.run_chunk(chunk, filename, _opts = {})
+    time = chunk.first
+    Resque.logger.info('start run_chunk')
+    sleep(time)
+    Resque.logger.info('end run_chunk')
+    CSV.open(filename, 'wb') do |csv|
+      csv << ['id']
+    end
+  end
+
+  def self.run_post(_chunk, _filename, _opts = {})
+    Resque.logger.info('start run_post')
+  end
+end

data/lib/distributed_resque_worker/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module DistributedResqueWorker
+  VERSION = '1.0.0'
+end

metadata

@@ -0,0 +1,166 @@
+--- !ruby/object:Gem::Specification
+name: distributed_resque_worker
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- TruptiHosmani
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2021-08-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.17.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.17.3
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.48'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.48'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.4.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.4.2
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.11.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.11.1
+- !ruby/object:Gem::Dependency
+  name: resque
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+description: Downloadable reports which uses background workersand then emails an
+  S3 link for CSV to the requestor
+email:
+- dev@yourmechanic.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".circleci/config.yml"
+- ".gitignore"
+- ".rspec"
+- ".rspec_status"
+- ".rubocop.yml"
+- ".rubocop_todo.yml"
+- ".ruby-version"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- distributed_resque_worker.gemspec
+- lib/distributed_resque_worker.rb
+- lib/distributed_resque_worker/aws_helper.rb
+- lib/distributed_resque_worker/resque_tester.rb
+- lib/distributed_resque_worker/version.rb
+- tmp/test.log
+homepage: https://github.com/YourMechanic/distributed_resque_worker
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/YourMechanic/distributed_resque_worker
+  source_code_uri: https://github.com/YourMechanic/distributed_resque_worker
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.3'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.8
+signing_key: 
+specification_version: 4
+summary: Gem for downloadable reports using Resque
+test_files: []