distributed_resque_worker 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 215e94466ab47726a61e6a2b82cc4f341a70d0fa7b003365ba183b44b8082e78
4
+ data.tar.gz: 87649120c3f63c429e41aaaf745cfdb5517abd829826617eb12ee5b4919e1a7e
5
+ SHA512:
6
+ metadata.gz: b314648eaa5206f2b949355a8a8cd90d0be7cbbf182003f85d9e9bab2304235135e0de04242216c1bc77683a9d806b4886cf0feffb6b4eb5da79ef842b5631b3
7
+ data.tar.gz: 989d6433ab4767c92387f65bad4522c57504444aef6adfc0e93b8cebd2726495a1262ee0ec9bff5dc0d604af7361436ee4c114a0e77c1d10acec58d368f2a967
@@ -0,0 +1,59 @@
1
+ version: 2
2
+ jobs:
3
+ build:
4
+ working_directory: ~/YourMechanic/distributed_resque_worker
5
+ parallelism: 1
6
+ shell: /bin/bash --login
7
+
8
+ environment:
9
+ CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
10
+ CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
11
+ docker:
12
+ - image: circleci/ruby:2.3.8
13
+ environment:
14
+ RAILS_ENV: test
15
+ - image: circleci/postgres:9.5.15
16
+ environment:
17
+ - image: redis
18
+ steps:
19
+ - checkout
20
+ - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS
21
+ - restore_cache:
22
+ keys:
23
+ # This branch if available
24
+ - v2-dep-{{ .Branch }}-
25
+ # Default branch if not
26
+ - v2-dep-master-
27
+ # Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly
28
+ - v2-dep-
29
+ - run:
30
+ name: Install bundler for distributed_resque_worker
31
+ working_directory: ~/YourMechanic/distributed_resque_worker
32
+ command: gem install bundler:1.17.3
33
+ - run:
34
+ name: Install Gems for distributed_resque_worker
35
+ working_directory: ~/YourMechanic/distributed_resque_worker
36
+ command: bundle install
37
+ # Save dependency cache
38
+ - save_cache:
39
+ key: v2-dep-{{ .Branch }}-{{ epoch }}
40
+ paths:
41
+ - vendor/bundle
42
+ - ~/virtualenvs
43
+ - ~/.m2
44
+ - ~/.ivy2
45
+ - ~/.bundle
46
+ - run:
47
+ working_directory: ~/YourMechanic/distributed_resque_worker
48
+ command: bundle exec rubocop
49
+ - run:
50
+ working_directory: ~/YourMechanic/distributed_resque_worker
51
+ command: bundle exec rspec --colour --drb --profile -fd --format progress $(circleci tests glob "spec/*_spec.rb" | circleci tests split)
52
+ - store_test_results:
53
+ path: /tmp/circleci-test-results
54
+ - store_artifacts:
55
+ path: /tmp/circleci-artifacts
56
+ - store_artifacts:
57
+ path: /tmp/circleci-test-results
58
+ - store_artifacts:
59
+ path: distributed_resque_worker/coverage
data/.gitignore ADDED
@@ -0,0 +1,56 @@
1
+ *.gem
2
+ *.rbc
3
+ /.config
4
+ /coverage/
5
+ /InstalledFiles
6
+ /pkg/
7
+ /spec/reports/
8
+ /spec/examples.txt
9
+ /test/tmp/
10
+ /test/version_tmp/
11
+ /tmp/*
12
+ /log/*
13
+ # Used by dotenv library to load environment variables.
14
+ # .env
15
+
16
+ # Ignore Byebug command history file.
17
+ .byebug_history
18
+
19
+ ## Specific to RubyMotion:
20
+ .dat*
21
+ .repl_history
22
+ build/
23
+ *.bridgesupport
24
+ build-iPhoneOS/
25
+ build-iPhoneSimulator/
26
+
27
+ ## Specific to RubyMotion (use of CocoaPods):
28
+ #
29
+ # We recommend against adding the Pods directory to your .gitignore. However
30
+ # you should judge for yourself, the pros and cons are mentioned at:
31
+ # https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
32
+ #
33
+ # vendor/Pods/
34
+
35
+ ## Documentation cache and generated files:
36
+ /.yardoc/
37
+ /_yardoc/
38
+ /doc/
39
+ /rdoc/
40
+
41
+ ## Environment normalization:
42
+ /.bundle/
43
+ /vendor/bundle
44
+ /lib/bundler/man/
45
+
46
+ # for a library or gem, you might want to ignore these files since the code is
47
+ # intended to run in multiple environments; otherwise, check them in:
48
+ # Gemfile.lock
49
+ # .ruby-version
50
+ # .ruby-gemset
51
+
52
+ # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
53
+ .rvmrc
54
+
55
+ # Used by RuboCop. Remote config files pulled in from inherit_from directive.
56
+ # .rubocop-https?--*
data/.rspec ADDED
@@ -0,0 +1,3 @@
1
+ --format documentation
2
+ --color
3
+ --require spec_helper
data/.rspec_status ADDED
@@ -0,0 +1,6 @@
1
+ example_id | status | run_time |
2
+ --------------------------------------------- | ------ | --------------- |
3
+ ./spec/distributed_resque_worker_spec.rb[1:1] | passed | 0.00282 seconds |
4
+ ./spec/distributed_resque_worker_spec.rb[1:2] | passed | 1.04 seconds |
5
+ ./spec/distributed_resque_worker_spec.rb[1:3] | passed | 10 seconds |
6
+ ./spec/distributed_resque_worker_spec.rb[1:4] | passed | 0.01556 seconds |
data/.rubocop.yml ADDED
@@ -0,0 +1,18 @@
1
+ inherit_from: .rubocop_todo.yml
2
+
3
+ Style/ModuleFunction:
4
+ Exclude:
5
+ - 'spec/**/*'
6
+
7
+ Style/NumericLiterals:
8
+ MinDigits: 20
9
+ Metrics/MethodLength:
10
+ Max: 100
11
+ Metrics/BlockLength:
12
+ Max: 10
13
+ Layout/LineLength:
14
+ Max: 80
15
+ Metrics/AbcSize:
16
+ Max: 25
17
+ Metrics/ClassLength:
18
+ Max: 150
data/.rubocop_todo.yml ADDED
@@ -0,0 +1,7 @@
1
+ # This configuration was generated by
2
+ # `rubocop --auto-gen-config`
3
+ # on 2021-04-08 16:20:29 +0530 using RuboCop version 0.81.0.
4
+ # The point is for the user to remove these configuration records
5
+ # one by one as the offenses are removed from the code base.
6
+ # Note that changes in the inspected code, or installation of new
7
+ # versions of RuboCop, may require this file to be generated again.
data/.ruby-version ADDED
@@ -0,0 +1 @@
1
+ 2.3.8
@@ -0,0 +1,74 @@
1
+ # Contributor Covenant Code of Conduct
2
+
3
+ ## Our Pledge
4
+
5
+ In the interest of fostering an open and welcoming environment, we as
6
+ contributors and maintainers pledge to making participation in our project and
7
+ our community a harassment-free experience for everyone, regardless of age, body
8
+ size, disability, ethnicity, gender identity and expression, level of experience,
9
+ nationality, personal appearance, race, religion, or sexual identity and
10
+ orientation.
11
+
12
+ ## Our Standards
13
+
14
+ Examples of behavior that contributes to creating a positive environment
15
+ include:
16
+
17
+ * Using welcoming and inclusive language
18
+ * Being respectful of differing viewpoints and experiences
19
+ * Gracefully accepting constructive criticism
20
+ * Focusing on what is best for the community
21
+ * Showing empathy towards other community members
22
+
23
+ Examples of unacceptable behavior by participants include:
24
+
25
+ * The use of sexualized language or imagery and unwelcome sexual attention or
26
+ advances
27
+ * Trolling, insulting/derogatory comments, and personal or political attacks
28
+ * Public or private harassment
29
+ * Publishing others' private information, such as a physical or electronic
30
+ address, without explicit permission
31
+ * Other conduct which could reasonably be considered inappropriate in a
32
+ professional setting
33
+
34
+ ## Our Responsibilities
35
+
36
+ Project maintainers are responsible for clarifying the standards of acceptable
37
+ behavior and are expected to take appropriate and fair corrective action in
38
+ response to any instances of unacceptable behavior.
39
+
40
+ Project maintainers have the right and responsibility to remove, edit, or
41
+ reject comments, commits, code, wiki edits, issues, and other contributions
42
+ that are not aligned to this Code of Conduct, or to ban temporarily or
43
+ permanently any contributor for other behaviors that they deem inappropriate,
44
+ threatening, offensive, or harmful.
45
+
46
+ ## Scope
47
+
48
+ This Code of Conduct applies both within project spaces and in public spaces
49
+ when an individual is representing the project or its community. Examples of
50
+ representing a project or community include using an official project e-mail
51
+ address, posting via an official social media account, or acting as an appointed
52
+ representative at an online or offline event. Representation of a project may be
53
+ further defined and clarified by project maintainers.
54
+
55
+ ## Enforcement
56
+
57
+ Instances of abusive, harassing, or otherwise unacceptable behavior may be
58
+ reported by contacting the project team at trupti@yourmechanic.com. All
59
+ complaints will be reviewed and investigated and will result in a response that
60
+ is deemed necessary and appropriate to the circumstances. The project team is
61
+ obligated to maintain confidentiality with regard to the reporter of an incident.
62
+ Further details of specific enforcement policies may be posted separately.
63
+
64
+ Project maintainers who do not follow or enforce the Code of Conduct in good
65
+ faith may face temporary or permanent repercussions as determined by other
66
+ members of the project's leadership.
67
+
68
+ ## Attribution
69
+
70
+ This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
71
+ available at [http://contributor-covenant.org/version/1/4][version]
72
+
73
+ [homepage]: http://contributor-covenant.org
74
+ [version]: http://contributor-covenant.org/version/1/4/
data/Gemfile ADDED
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ source 'https://rubygems.org'
4
+
5
+ git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
6
+
7
+ # Specify your gem's dependencies in distributed_resque_worker.gemspec
8
+ gemspec
data/Gemfile.lock ADDED
@@ -0,0 +1,99 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ distributed_resque_worker (1.0.0)
5
+ aws-sdk (= 1.11.1)
6
+ resque (~> 2.0)
7
+
8
+ GEM
9
+ remote: https://rubygems.org/
10
+ specs:
11
+ addressable (2.4.0)
12
+ ast (2.4.2)
13
+ aws-sdk (1.11.1)
14
+ json (~> 1.4)
15
+ nokogiri (>= 1.4.4)
16
+ uuidtools (~> 2.1)
17
+ crack (0.4.3)
18
+ safe_yaml (~> 1.0.0)
19
+ diff-lcs (1.4.4)
20
+ hashdiff (0.3.7)
21
+ jaro_winkler (1.5.4)
22
+ json (1.8.6)
23
+ mini_portile2 (2.4.0)
24
+ mono_logger (1.1.1)
25
+ multi_json (1.15.0)
26
+ mustermann (1.1.1)
27
+ ruby2_keywords (~> 0.0.1)
28
+ nokogiri (1.10.10)
29
+ mini_portile2 (~> 2.4.0)
30
+ parallel (1.19.2)
31
+ parser (3.0.1.0)
32
+ ast (~> 2.4.1)
33
+ rack (2.2.3)
34
+ rack-protection (2.1.0)
35
+ rack
36
+ rainbow (3.0.0)
37
+ rake (10.5.0)
38
+ redis (4.4.0)
39
+ redis-namespace (1.6.0)
40
+ redis (>= 3.0.4)
41
+ resque (2.0.0)
42
+ mono_logger (~> 1.0)
43
+ multi_json (~> 1.0)
44
+ redis-namespace (~> 1.6)
45
+ sinatra (>= 0.9.2)
46
+ vegas (~> 0.1.2)
47
+ rexml (3.2.5)
48
+ rspec (3.10.0)
49
+ rspec-core (~> 3.10.0)
50
+ rspec-expectations (~> 3.10.0)
51
+ rspec-mocks (~> 3.10.0)
52
+ rspec-core (3.10.1)
53
+ rspec-support (~> 3.10.0)
54
+ rspec-expectations (3.10.1)
55
+ diff-lcs (>= 1.2.0, < 2.0)
56
+ rspec-support (~> 3.10.0)
57
+ rspec-mocks (3.10.2)
58
+ diff-lcs (>= 1.2.0, < 2.0)
59
+ rspec-support (~> 3.10.0)
60
+ rspec-support (3.10.2)
61
+ rubocop (0.81.0)
62
+ jaro_winkler (~> 1.5.1)
63
+ parallel (~> 1.10)
64
+ parser (>= 2.7.0.1)
65
+ rainbow (>= 2.2.2, < 4.0)
66
+ rexml
67
+ ruby-progressbar (~> 1.7)
68
+ unicode-display_width (>= 1.4.0, < 2.0)
69
+ ruby-progressbar (1.11.0)
70
+ ruby2_keywords (0.0.5)
71
+ safe_yaml (1.0.4)
72
+ sinatra (2.1.0)
73
+ mustermann (~> 1.0)
74
+ rack (~> 2.2)
75
+ rack-protection (= 2.1.0)
76
+ tilt (~> 2.0)
77
+ tilt (2.0.10)
78
+ unicode-display_width (1.7.0)
79
+ uuidtools (2.2.0)
80
+ vegas (0.1.11)
81
+ rack (>= 1.0.0)
82
+ webmock (3.4.2)
83
+ addressable (>= 2.3.6)
84
+ crack (>= 0.3.2)
85
+ hashdiff
86
+
87
+ PLATFORMS
88
+ ruby
89
+
90
+ DEPENDENCIES
91
+ bundler (~> 1.17.3)
92
+ distributed_resque_worker!
93
+ rake (~> 10.0)
94
+ rspec (~> 3.0)
95
+ rubocop (~> 0.48)
96
+ webmock (~> 3.4.2)
97
+
98
+ BUNDLED WITH
99
+ 1.17.3
data/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2021 YourMechanic
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,91 @@
1
+ # distributed_resque_worker
2
+ Distributed Background Worker to process a large workload
3
+
4
+ Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/distributed_resque_worker`. To experiment with that code, run `bin/console` for an interactive prompt.
5
+
6
+ TODO: Delete this and the text above, and describe your gem
7
+
8
+ ## Installation
9
+
10
+ Add this line to your application's Gemfile:
11
+
12
+ ```ruby
13
+ gem 'distributed_resque_worker', git: 'https://github.com/YourMechanic/distributed_resque_worker.git', :branch => 'master'
14
+ ```
15
+ And then execute:
16
+ $ bundle install
17
+
18
+ Add the following in routes.rb
19
+ require 'resque/server'
20
+ mount Resque::Server.new, at: '/resque'
21
+
22
+
23
+ Setup Redis: Add the following in the config/initializers/resque.yml
24
+ ```
25
+ development: localhost:6379
26
+ test: localhost:6379
27
+ staging: <%= ENV['REDIS_URL'] %>
28
+ production: <%= ENV['REDIS_URL'] %>
29
+ ```
30
+
31
+ Configure Resque: Add the following in the config/initializers/resque.rb
32
+ ```
33
+ resque_config = YAML.load(File.read(Rails.root.join('config/resque.yml')))
34
+ Resque.redis = resque_config[Rails.env]
35
+
36
+ Resque.logger = Logger.new(Rails.root.join('log', "#{Rails.env}_resque.log"))
37
+ Resque.logger.level = Logger::INFO
38
+ ```
39
+
40
+ In your Rakefile, or some other file in lib/tasks (ex: lib/tasks/resque.rake), load the resque rake tasks (For Rails 3+):
41
+ ```
42
+ require 'resque/tasks'
43
+ task 'resque:setup' => :environment
44
+ ```
45
+
46
+ ## Usage
47
+ Can use the following code to chunk the work and run on 4 workers using DistributedResqueWorker and you also need to add a function that workers will run with their chunk of work
48
+ ```
49
+ class SomeClass
50
+ ...
51
+ def some_method
52
+ opts = {address:'', requestor: 'trupti', emails: [], requested_date: '', ...}
53
+ input = (1..4).to_a
54
+
55
+ worker = DistributedResqueWorker::ResqueWorker.new(
56
+ self,
57
+ 'bucket_name',
58
+ 'root_dir_to_stor_tmp_files'
59
+ )
60
+ worker.chunk_work_and_enqueue(input, __callee__, opts)
61
+ end
62
+ ...
63
+
64
+ def self.some_method_chunk(work_chunk, output_filepath, opts = {})
65
+ # some code to work on the work_chunk and add data into output_filepath
66
+ end
67
+
68
+ def self.some_method_post(work_chunk, output_filepath, opts = {})
69
+ # some code to either send email/slack message the final file s3 link
70
+ end
71
+ end
72
+ ```
73
+
74
+
75
+ ## Development
76
+
77
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
78
+
79
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
80
+
81
+ ## Contributing
82
+
83
+ Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/distributed_resque_worker. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
84
+
85
+ ## License
86
+
87
+ The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
88
+
89
+ ## Code of Conduct
90
+
91
+ Everyone interacting in the DistributedResqueWorker project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/distributed_resque_worker/blob/master/CODE_OF_CONDUCT.md).
data/Rakefile ADDED
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'bundler/gem_tasks'
4
+ require 'rspec/core/rake_task'
5
+ require 'rubocop/rake_task'
6
+ require 'resque'
7
+ require 'resque/tasks'
8
+
9
+ RSpec::Core::RakeTask.new(:spec)
10
+ RuboCop::RakeTask.new
11
+
12
+ task 'resque:setup' do
13
+ require_relative 'lib/distributed_resque_worker'
14
+ end
15
+
16
+ task default: %i[spec rubocop resque]
data/bin/console ADDED
@@ -0,0 +1,15 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ require 'bundler/setup'
5
+ require 'distributed_resque_worker'
6
+
7
+ # You can add fixtures and/or initialization code here to make experimenting
8
+ # with your gem easier. You can also use a different console, if you like.
9
+
10
+ # (If you use this, don't forget to add pry to your Gemfile!)
11
+ # require "pry"
12
+ # Pry.start
13
+
14
+ require 'irb'
15
+ IRB.start(__FILE__)
data/bin/setup ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+
3
+ lib = File.expand_path('lib', __dir__)
4
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
+ require 'distributed_resque_worker/version'
6
+
7
+ Gem::Specification.new do |spec|
8
+ spec.name = 'distributed_resque_worker'
9
+ spec.version = DistributedResqueWorker::VERSION
10
+ spec.authors = ['TruptiHosmani']
11
+ spec.email = ['dev@yourmechanic.com']
12
+
13
+ spec.summary = 'Gem for downloadable reports using Resque'
14
+ spec.description = 'Downloadable reports which uses background workers'\
15
+ 'and then emails an S3 link for CSV to the requestor'
16
+ spec.homepage = 'https://github.com/YourMechanic/distributed_resque_worker'
17
+ spec.license = 'MIT'
18
+ spec.required_ruby_version = Gem::Requirement.new('>= 2.3')
19
+ spec.metadata['allowed_push_host'] = 'https://rubygems.org'
20
+
21
+ spec.metadata['homepage_uri'] = spec.homepage
22
+ spec.metadata['source_code_uri'] = 'https://github.com/YourMechanic/distributed_resque_worker'
23
+
24
+ # Specify which files should be added to the gem when it is released.
25
+ # `git ls-files -z` loads the files in RubyGem that have been added into git.
26
+ spec.files = Dir.chdir(File.expand_path(__dir__)) do
27
+ `git ls-files -z`.split("\x0").reject do |f|
28
+ f.match(%r{\A(?:test|spec|features)/})
29
+ end
30
+ end
31
+ spec.bindir = 'exe'
32
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
33
+ spec.require_paths = ['lib']
34
+
35
+ spec.add_development_dependency 'bundler', '~> 1.17.3'
36
+ spec.add_development_dependency 'rake', '~> 10.0'
37
+ spec.add_development_dependency 'rspec', '~> 3.0'
38
+ spec.add_development_dependency 'rubocop', '~> 0.48'
39
+ spec.add_development_dependency 'webmock', '~> 3.4.2'
40
+ spec.add_dependency 'aws-sdk', '1.11.1'
41
+ spec.add_dependency 'resque', '~> 2.0'
42
+ end
@@ -0,0 +1,178 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'distributed_resque_worker/version'
4
+ require 'csv'
5
+ require 'resque'
6
+
7
+ require_relative 'distributed_resque_worker/aws_helper'
8
+ require_relative 'distributed_resque_worker/resque_tester'
9
+
10
+ # DistributedResqueWorker
11
+ module DistributedResqueWorker
12
+ # ResqueFailure
13
+ module ResqueFailure
14
+ def on_failure_logging(error, *args)
15
+ msg = "Performing #{self} caused an exception #{error} with args #{args}."
16
+ Resque.logger.info msg
17
+ end
18
+ end
19
+
20
+ # ResqueWorker
21
+ class ResqueWorker
22
+ CHUNK_SIZE = 100
23
+ extend ResqueFailure
24
+
25
+ def initialize(queue_name, bucket, root)
26
+ @queue = "#{queue_name}_#{Time.now.to_i}_#{Random.rand(1000000)}".to_sym
27
+ @bucket = bucket
28
+ @root = root
29
+ FileUtils.mkdir_p("#{root}/tmp/#{@queue}")
30
+ end
31
+
32
+ def chunk_work_and_enqueue(work_list, method, opts)
33
+ total_jobs = (work_list.length.to_f / CHUNK_SIZE).ceil
34
+ total_jobs = 1 if total_jobs.zero?
35
+ Resque.logger.info("total_jobs #{total_jobs}")
36
+ ResqueWorker.resque_redis.redis.set(
37
+ ResqueWorker.resque_worker_redis_key(@queue), total_jobs
38
+ )
39
+ work_list.each_slice(CHUNK_SIZE).each_with_index do |chunk, index|
40
+ details = { work_name: @queue, chunk: chunk, index: index,
41
+ type: 'processor', root: @root.to_s,
42
+ bucket: @bucket, method: method, opts: opts }
43
+ Resque.enqueue_to(@queue, ResqueWorker, details)
44
+ end
45
+ end
46
+
47
+ def self.perform(args)
48
+ redis_key = resque_worker_redis_key(args['work_name'])
49
+ no_jobs = resque_redis.redis.get(redis_key)
50
+ Resque.logger.info("No of jobs remaining => #{no_jobs}")
51
+ if args['type'] == 'processor'
52
+ process_chunk(args)
53
+ if resque_redis.redis.get(redis_key).to_i.zero?
54
+ enqueue_post_processor(args)
55
+ end
56
+ elsif args['type'] == 'post_processor'
57
+ post_processing(args)
58
+ end
59
+ end
60
+
61
+ class << self
62
+ def resque_redis
63
+ Resque.redis
64
+ end
65
+
66
+ def resque_worker_redis_key(work_name)
67
+ "DistributedResqueWorker:#{work_name}"
68
+ end
69
+
70
+ def merge_intermediate_files(work_name, final_file)
71
+ files = "tmp/#{work_name}/#{work_name}_*.csv"
72
+ cmd = "awk '(NR == 1) || (FNR > 1)' #{files} > #{final_file}"
73
+ system(`#{cmd}`)
74
+ end
75
+
76
+ def enqueue_post_processor(args)
77
+ Resque.logger.info('start enqueue_post_processor')
78
+ input = args.symbolize_keys!
79
+ details = { type: 'post_processor', work_name: input[:work_name],
80
+ bucket: input[:bucket], method: input[:method],
81
+ root: input[:root], opts: input[:opts] }
82
+ Resque.enqueue_to(input[:work_name], ResqueWorker, details)
83
+ Resque.logger.info('finished enqueue_post_processor')
84
+ end
85
+
86
+ def process_chunk(args)
87
+ input = args.symbolize_keys!
88
+ method_chunk = "#{input[:method]}_chunk".to_sym
89
+ worker_class = input[:work_name].split('_').first
90
+ worker = worker_class.constantize
91
+ path = "#{input[:work_name]}/#{input[:work_name]}_#{input[:index]}.csv"
92
+ filename = "#{input[:root]}/tmp/#{path}"
93
+ worker.send(method_chunk, input[:chunk], filename, input[:opts])
94
+ store_to_s3_delete_local_copy(path, filename, input[:bucket])
95
+ resque_redis.redis.decr(resque_worker_redis_key(input[:work_name]))
96
+ end
97
+
98
+ def store_to_s3_delete_local_copy(path, filename, bucket)
99
+ s3_name = "resque_worker/#{path}"
100
+ begin
101
+ AwsHelper.s3_store_file(s3_name, filename, bucket)
102
+ File.delete(filename)
103
+ rescue StandardError
104
+ Resque.logger.error($ERROR_INFO)
105
+ nil
106
+ end
107
+ end
108
+
109
+ def post_processing(args)
110
+ input = args.symbolize_keys!
111
+ work_name = input[:work_name]
112
+ root = input[:root]
113
+ final_tmp_file = "#{root}/tmp/#{work_name}/#{work_name}_final.csv"
114
+ Resque.logger.info("start post_processing #{input}")
115
+ begin
116
+ download_intermediate_files(work_name, input[:bucket], root)
117
+ delete_intermediate_s3_files(work_name, input[:bucket])
118
+ merge_intermediate_files(work_name, final_tmp_file)
119
+ upload_final_file_to_s3_and_send(input, final_tmp_file)
120
+ rescue StandardError
121
+ Resque.logger.error($ERROR_INFO)
122
+ nil
123
+ end
124
+ Resque.logger.info('finished post_processing ')
125
+ end
126
+
127
+ def download_intermediate_files(work_name, bucket, root)
128
+ aws_bucket = AwsHelper.bucket(bucket)
129
+ folder = "resque_worker/#{work_name}/"
130
+ s3_object = aws_bucket.objects.with_prefix(folder)
131
+ s3_file_names = s3_object.collect(&:key)
132
+ s3_file_names.each do |filename|
133
+ local_file_name = filename.split('/')
134
+ next unless local_file_name[2]
135
+
136
+ download_file_path = "#{root}/tmp/#{work_name}/#{local_file_name[2]}"
137
+ Resque.logger.info("download_file_path #{download_file_path} ")
138
+ AwsHelper.s3_download_file(filename, download_file_path, bucket)
139
+ end
140
+ end
141
+
142
+ def delete_intermediate_s3_files(work_name, bucket)
143
+ aws_bucket = AwsHelper.bucket(bucket)
144
+ folder = "resque_worker/#{work_name}/"
145
+ s3_object = aws_bucket.objects.with_prefix(folder)
146
+ s3_file_names = s3_object.collect(&:key)
147
+ s3_file_names.each do |item|
148
+ AwsHelper.s3_delete(item, bucket)
149
+ end
150
+ end
151
+
152
+ def upload_final_file_to_s3_and_send(input, final_tmp_file)
153
+ work_name = input[:work_name]
154
+ s3_name = "resque_worker/#{work_name}/#{work_name}_final.csv"
155
+ final_file_link = AwsHelper.s3_store_file(s3_name, final_tmp_file,
156
+ input[:bucket])
157
+ method_post = "#{input[:method]}_post".to_sym
158
+ worker_class = input[:work_name].split('_').first
159
+ worker = worker_class.constantize
160
+ worker.send(method_post, final_file_link, input[:opts])
161
+ clean_up(work_name, input[:root])
162
+ end
163
+
164
+ def clean_up(queue_name, root)
165
+ FileUtils.remove_dir("#{root}/tmp/#{queue_name}")
166
+
167
+ delete_queue(queue_name)
168
+ Resque.logger.info('Cleanup Done!')
169
+ end
170
+
171
+ def delete_queue(queue_name)
172
+ Resque.queues.each do |queue|
173
+ Resque.remove_queue(queue) if queue == queue_name
174
+ end
175
+ end
176
+ end
177
+ end
178
+ end
@@ -0,0 +1,109 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aws-sdk'
4
+
5
+ # AwsHelper
6
+ module AwsHelper
7
+ module_function
8
+
9
+ def bucket(bucket)
10
+ @bucket = AWS::S3.new.buckets[bucket]
11
+ end
12
+
13
+ def s3_store_file(name, file, bucket_name, opt = {})
14
+ # Stream the content for storage
15
+ File.open(file, 'rb') do |f|
16
+ return s3_store(name, f, bucket_name, opt)
17
+ end
18
+ end
19
+
20
+ def s3_store(name, content, bucket_name, opt = {})
21
+ run_with_retry do
22
+ obj = s3_get_object(name, bucket_name)
23
+ obj.write(content, opt)
24
+ obj.exists? ? s3_get_object_url(name, bucket_name) : nil
25
+ end
26
+ end
27
+
28
+ def run_with_retry
29
+ maxtry = 3
30
+ ntry = 0
31
+ begin
32
+ yield
33
+ rescue AWS::S3::Errors::RequestTimeout
34
+ ntry += 1
35
+ if ntry > maxtry
36
+ Resque.logger.info($ERROR_INFO)
37
+ raise
38
+ end
39
+ print "Error: #{$ERROR_INFO}, retrying\n"
40
+ @bucket = nil # So that we create a new bucket
41
+ retry
42
+ end
43
+ end
44
+
45
+ def doomsday
46
+ expiration = Time.zone.now + 20.years
47
+ # TODO: ::Update after AWS changes this limit. Will likely be a while
48
+ # since it depends on global transition to 64-bit systems
49
+ #
50
+ # AWS sets 01/20/2038 as an upper limit threshold on expiration date
51
+ # due to https://en.wikipedia.org/wiki/Year_2038_problem
52
+ aws_max_date = Time.zone.parse('2038-01-18')
53
+ expiration = aws_max_date if expiration > aws_max_date
54
+ expiration
55
+ end
56
+
57
+ def s3_get_object_url(name, bucket_name, _opt = {})
58
+ obj = s3_get_object(name, bucket_name)
59
+ return nil unless obj&.exists?
60
+
61
+ secure = true
62
+ obj.url_for(:read, secure: secure, expires: doomsday).to_s
63
+ end
64
+
65
+ def s3_delete(name, bucket_name)
66
+ run_with_retry { s3_get_object(name, bucket_name).delete }
67
+ end
68
+
69
+ def s3_get_object(name, bucket_name)
70
+ bucket = bucket(bucket_name)
71
+ bucket.objects[name]
72
+ end
73
+
74
+ def s3_download_file(name, filename, bucket_name)
75
+ run_with_retry do
76
+ data = s3_get_object(name, bucket_name).read
77
+ File.open(filename, 'wb') do |file|
78
+ file.write(data)
79
+ end
80
+ nil
81
+ end
82
+ end
83
+
84
+ CONTENT_TYPE_TO_EXT = {
85
+ 'audio/amr' => '.amr',
86
+ 'audio/acc' => '.mp4',
87
+ 'audio/mp4' => '.mp4',
88
+ 'audio/mpeg' => '.mp3',
89
+ 'audio/ogg' => '.ogg',
90
+ 'image/jpeg' => '.jpg',
91
+ 'image/png' => '.png',
92
+ 'image/gif' => '.gif',
93
+ 'text/plain' => '.txt',
94
+ 'text/rtf' => '.rtf',
95
+ 'application/zip' => '.zip',
96
+ 'application/pdf' => '.pdf',
97
+ 'application/msword' => '.doc'
98
+ }.freeze
99
+
100
+ def content_type(ext)
101
+ CONTENT_TYPE_TO_EXT.each do |ct, cext|
102
+ return ct if ext == cext
103
+ end
104
+ end
105
+
106
+ def content_ext(content_type)
107
+ CONTENT_TYPE_TO_EXT[content_type] || ''
108
+ end
109
+ end
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../distributed_resque_worker'
4
+
5
+ # ResqueTester
6
+ class ResqueTester
7
+ def self.run(bucket)
8
+ input = (1..4).to_a
9
+ worker = DistributedResqueWorker::ResqueWorker.new(
10
+ 'ResqueTester',
11
+ bucket,
12
+ Dir.pwd
13
+ )
14
+ worker.chunk_work_and_enqueue(input, __callee__, {})
15
+ end
16
+
17
+ def self.run_chunk(chunk, filename, _opts = {})
18
+ time = chunk.first
19
+ Resque.logger.info('start run_chunk')
20
+ sleep(time)
21
+ Resque.logger.info('end run_chunk')
22
+ CSV.open(filename, 'wb') do |csv|
23
+ csv << ['id']
24
+ end
25
+ end
26
+
27
+ def self.run_post(_chunk, _filename, _opts = {})
28
+ Resque.logger.info('start run_post')
29
+ end
30
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module DistributedResqueWorker
4
+ VERSION = '1.0.0'
5
+ end
data/tmp/test.log ADDED
File without changes
metadata ADDED
@@ -0,0 +1,166 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: distributed_resque_worker
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.0.0
5
+ platform: ruby
6
+ authors:
7
+ - TruptiHosmani
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2021-08-13 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: 1.17.3
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: 1.17.3
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '10.0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '10.0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rspec
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '3.0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '3.0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: rubocop
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '0.48'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '0.48'
69
+ - !ruby/object:Gem::Dependency
70
+ name: webmock
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: 3.4.2
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: 3.4.2
83
+ - !ruby/object:Gem::Dependency
84
+ name: aws-sdk
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - '='
88
+ - !ruby/object:Gem::Version
89
+ version: 1.11.1
90
+ type: :runtime
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - '='
95
+ - !ruby/object:Gem::Version
96
+ version: 1.11.1
97
+ - !ruby/object:Gem::Dependency
98
+ name: resque
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - "~>"
102
+ - !ruby/object:Gem::Version
103
+ version: '2.0'
104
+ type: :runtime
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - "~>"
109
+ - !ruby/object:Gem::Version
110
+ version: '2.0'
111
+ description: Downloadable reports which uses background workersand then emails an
112
+ S3 link for CSV to the requestor
113
+ email:
114
+ - dev@yourmechanic.com
115
+ executables: []
116
+ extensions: []
117
+ extra_rdoc_files: []
118
+ files:
119
+ - ".circleci/config.yml"
120
+ - ".gitignore"
121
+ - ".rspec"
122
+ - ".rspec_status"
123
+ - ".rubocop.yml"
124
+ - ".rubocop_todo.yml"
125
+ - ".ruby-version"
126
+ - CODE_OF_CONDUCT.md
127
+ - Gemfile
128
+ - Gemfile.lock
129
+ - LICENSE
130
+ - README.md
131
+ - Rakefile
132
+ - bin/console
133
+ - bin/setup
134
+ - distributed_resque_worker.gemspec
135
+ - lib/distributed_resque_worker.rb
136
+ - lib/distributed_resque_worker/aws_helper.rb
137
+ - lib/distributed_resque_worker/resque_tester.rb
138
+ - lib/distributed_resque_worker/version.rb
139
+ - tmp/test.log
140
+ homepage: https://github.com/YourMechanic/distributed_resque_worker
141
+ licenses:
142
+ - MIT
143
+ metadata:
144
+ allowed_push_host: https://rubygems.org
145
+ homepage_uri: https://github.com/YourMechanic/distributed_resque_worker
146
+ source_code_uri: https://github.com/YourMechanic/distributed_resque_worker
147
+ post_install_message:
148
+ rdoc_options: []
149
+ require_paths:
150
+ - lib
151
+ required_ruby_version: !ruby/object:Gem::Requirement
152
+ requirements:
153
+ - - ">="
154
+ - !ruby/object:Gem::Version
155
+ version: '2.3'
156
+ required_rubygems_version: !ruby/object:Gem::Requirement
157
+ requirements:
158
+ - - ">="
159
+ - !ruby/object:Gem::Version
160
+ version: '0'
161
+ requirements: []
162
+ rubygems_version: 3.0.8
163
+ signing_key:
164
+ specification_version: 4
165
+ summary: Gem for downloadable reports using Resque
166
+ test_files: []