sidekiq_publisher 0.1.0.rc0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +4 -0
- data/Gemfile +10 -0
- data/LICENSE.txt +22 -0
- data/README.md +110 -0
- data/lib/active_job/queue_adapters/sidekiq_publisher_adapter.rb +40 -0
- data/lib/generators/sidekiq_publisher/install_generator.rb +25 -0
- data/lib/generators/sidekiq_publisher/templates/create_sidekiq_publisher_jobs.rb +20 -0
- data/lib/sidekiq_publisher/client.rb +32 -0
- data/lib/sidekiq_publisher/job.rb +72 -0
- data/lib/sidekiq_publisher/publisher.rb +68 -0
- data/lib/sidekiq_publisher/railtie.rb +13 -0
- data/lib/sidekiq_publisher/runner.rb +46 -0
- data/lib/sidekiq_publisher/tasks.rake +7 -0
- data/lib/sidekiq_publisher/version.rb +5 -0
- data/lib/sidekiq_publisher/worker.rb +21 -0
- data/lib/sidekiq_publisher.rb +34 -0
- data/log/.gitkeep +0 -0
- data/sidekiq_publisher.gemspec +64 -0
- metadata +287 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA1:
|
|
3
|
+
metadata.gz: 769acfc86ebd02bbae960c1469ff957aea13caee
|
|
4
|
+
data.tar.gz: 8b0584f6454472e1a8d8d57eec2f649afd73725f
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: ea23052c691e985e08a692f3b559d0c450421cc0c17221f5701c0fef06196224a41d062b9d82fdab372fef6f9bc2a6794c9cce2c0b7abb7ea46702343d7757b8
|
|
7
|
+
data.tar.gz: '0979fc3625949cd6d3df4831616b355cf6e1d273b51c2cd441e373fdbd1e7280dabc311425dcf962efbfb65fc35bbaec664fc25b6b197d52d85a4daefdfeba98'
|
data/CHANGELOG.md
ADDED
data/Gemfile
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# TODO: this should be changed back to rubygems once the activerecord-postgres_pub_sub is public
|
|
4
|
+
source "https://ezcater.jfrog.io/ezcater/api/gems/ezcater-gem-source"
|
|
5
|
+
|
|
6
|
+
# override the :github shortcut to be secure by using HTTPS
|
|
7
|
+
git_source(:github) { |repo_name| "https://github.com/#{repo_name}.git" }
|
|
8
|
+
|
|
9
|
+
# Specify your gem's dependencies in sidekiq_publisher.gemspec
|
|
10
|
+
gemspec
|
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2017 ezCater, Inc
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
|
22
|
+
|
data/README.md
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
# sidekiq_publisher
|
|
2
|
+
|
|
3
|
+
This gem provides support to enqueue jobs for Sidekiq by first staging the job
|
|
4
|
+
in Postgres and relying on a separate process to communicate with Sidekiq/Redis.
|
|
5
|
+
|
|
6
|
+
The publisher process is alerted that a job is available to be published using
|
|
7
|
+
Postgres NOTIFY/LISTEN.
|
|
8
|
+
|
|
9
|
+
This approach has the benefit that jobs can be published as part of a transaction
|
|
10
|
+
that modifies the system of record for the application. It also allows jobs to
|
|
11
|
+
be created even when Sidekiq/Redis is temporarily unavailable. The separate
|
|
12
|
+
publisher process handles retries and ensure that each job is delivered to Sidekiq.
|
|
13
|
+
|
|
14
|
+
## Installation
|
|
15
|
+
|
|
16
|
+
Add this line to your application's Gemfile:
|
|
17
|
+
|
|
18
|
+
```ruby
|
|
19
|
+
gem "sidekiq_publisher"
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
And then execute:
|
|
23
|
+
|
|
24
|
+
$ bundle
|
|
25
|
+
|
|
26
|
+
Or install it yourself as:
|
|
27
|
+
|
|
28
|
+
$ gem install sidekiq_publisher
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
Run the generator to create migrations for the jobs table and notifications:
|
|
32
|
+
|
|
33
|
+
$ rails generate sidekiq_publisher:install
|
|
34
|
+
|
|
35
|
+
## Configuration
|
|
36
|
+
|
|
37
|
+
This gem uses the following configuration:
|
|
38
|
+
|
|
39
|
+
* **logger**: the logger for this gem to use.
|
|
40
|
+
* **exception_reporter**: a Proc that will be called with an exception
|
|
41
|
+
* **batch_size**: the maximum number of jobs that will be enqueued to Sidekiq
|
|
42
|
+
together
|
|
43
|
+
* **job_retention_period**: the duration that published jobs will be kept in
|
|
44
|
+
Postgres after they have been enqueued to Sidekiq
|
|
45
|
+
|
|
46
|
+
## Usage
|
|
47
|
+
|
|
48
|
+
### ActiveJob Adapter
|
|
49
|
+
|
|
50
|
+
This gem includes an adapter to use `SidekiqPublisher` with `ActiveJob`. This
|
|
51
|
+
adapter must be explicitly required:
|
|
52
|
+
|
|
53
|
+
```ruby
|
|
54
|
+
require "active_job/queue_adapters/sidekiq_publisher_adapter"
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
The adapter can also be required via your Gemfile:
|
|
58
|
+
|
|
59
|
+
```ruby
|
|
60
|
+
gem "sidekiq_publisher", require: ["sidekiq_publisher", "active_job/queue_adapters/sidekiq_publisher_adapter"]
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
The adapter to use with `ActiveJob` must be specified in Rails configuration
|
|
64
|
+
|
|
65
|
+
```ruby
|
|
66
|
+
# application.rb
|
|
67
|
+
config.active_job.queue_adapter = :sidekiq_publisher
|
|
68
|
+
|
|
69
|
+
# or directly in configuration
|
|
70
|
+
Rails.application.config.active_job.queue_adapter = :sidekiq_publisher
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
### SidekiqPublisher::Worker
|
|
74
|
+
|
|
75
|
+
Sidekiq workers are usually defined by including `Sidekiq::Worker` in a class.
|
|
76
|
+
|
|
77
|
+
To use the `SidekiqPublisher`, this can be replaced by including
|
|
78
|
+
`SidekiqPublisher::Worker`. The usual `perform_async`, etc methods will be
|
|
79
|
+
available on the class but jobs will be staged in the Postgres table.
|
|
80
|
+
|
|
81
|
+
### Running
|
|
82
|
+
|
|
83
|
+
The publisher process that pulls the job data from postgres and puts them into redis
|
|
84
|
+
can be run with a rake task that is added via railtie for Rails applications:
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
bundle exec rake sidekiq_publisher:publish
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
## Development
|
|
91
|
+
|
|
92
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then,
|
|
93
|
+
run `rake spec` to run the tests. You can also run `bin/console` for an
|
|
94
|
+
interactive prompt that will allow you to experiment.
|
|
95
|
+
|
|
96
|
+
To install this gem onto your local machine, run `bundle exec rake install`.
|
|
97
|
+
|
|
98
|
+
To release a new version, update the version number in `version.rb`, and then
|
|
99
|
+
run `bundle exec rake release`, which will create a git tag for the version,
|
|
100
|
+
push git commits and tags, and push the `.gem` file to
|
|
101
|
+
[rubygems.org](https://rubygems.org).
|
|
102
|
+
|
|
103
|
+
## Contributing
|
|
104
|
+
|
|
105
|
+
Bug reports and pull requests are welcome on GitHub at
|
|
106
|
+
https://github.com/ezcater/sidekiq_publisher.## License
|
|
107
|
+
|
|
108
|
+
The gem is available as open source under the terms of the
|
|
109
|
+
[MIT License](http://opensource.org/licenses/MIT).
|
|
110
|
+
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "active_job/queue_adapters/sidekiq_adapter"
|
|
4
|
+
|
|
5
|
+
module ActiveJob
|
|
6
|
+
module QueueAdapters
|
|
7
|
+
# To use SidekiqPublisher set the queue_adapter config to +:sidekiq_publisher+.
|
|
8
|
+
# Rails.application.config.active_job.queue_adapter = :sidekiq_publisher
|
|
9
|
+
class SidekiqPublisherAdapter
|
|
10
|
+
JOB_WRAPPER_CLASS = ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper.to_s.freeze
|
|
11
|
+
|
|
12
|
+
def enqueue(job)
|
|
13
|
+
internal_enqueue(job)
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def enqueue_at(job, timestamp)
|
|
17
|
+
internal_enqueue(job, timestamp)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
private
|
|
21
|
+
|
|
22
|
+
def internal_enqueue(job, timestamp = nil)
|
|
23
|
+
job.provider_job_id = SidekiqPublisher::Job.generate_sidekiq_jid
|
|
24
|
+
attributes = job_attributes(job)
|
|
25
|
+
attributes[:run_at] = timestamp if timestamp.present?
|
|
26
|
+
SidekiqPublisher::Job.create!(attributes).job_id
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def job_attributes(job)
|
|
30
|
+
{
|
|
31
|
+
job_id: job.provider_job_id,
|
|
32
|
+
job_class: JOB_WRAPPER_CLASS,
|
|
33
|
+
wrapped: job.class.to_s,
|
|
34
|
+
queue: job.queue_name,
|
|
35
|
+
args: [job.serialize],
|
|
36
|
+
}
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/generators"
|
|
4
|
+
require "rails/generators/migration"
|
|
5
|
+
require "rails/generators/active_record"
|
|
6
|
+
|
|
7
|
+
module SidekiqPublisher
|
|
8
|
+
class InstallGenerator < Rails::Generators::Base
|
|
9
|
+
include ActiveRecord::Generators::Migration
|
|
10
|
+
|
|
11
|
+
TEMPLATE_FILE = "create_sidekiq_publisher_jobs.rb"
|
|
12
|
+
|
|
13
|
+
source_paths << File.join(__dir__, "templates")
|
|
14
|
+
|
|
15
|
+
def create_migration_file
|
|
16
|
+
migration_template(TEMPLATE_FILE, "db/migrate/#{TEMPLATE_FILE}")
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def generate_notify_trigger
|
|
20
|
+
invoke "active_record:postgres_pub_sub:notify_on_insert",
|
|
21
|
+
[],
|
|
22
|
+
model_name: "SidekiqPublisher::Job"
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class CreateSidekiqPublisherJobs < ActiveRecord::Migration[5.1]
|
|
4
|
+
def change
|
|
5
|
+
# rubocop:disable Rails/CreateTableWithTimestamps
|
|
6
|
+
create_table(:sidekiq_publisher_jobs, id: :bigserial) do |t|
|
|
7
|
+
t.string :job_id, null: false
|
|
8
|
+
t.string :job_class, null: false
|
|
9
|
+
t.string :queue
|
|
10
|
+
t.string :wrapped
|
|
11
|
+
t.json :args, null: false
|
|
12
|
+
t.float :run_at
|
|
13
|
+
t.timestamp :published_at
|
|
14
|
+
t.timestamp :created_at, null: false
|
|
15
|
+
end
|
|
16
|
+
# rubocop:enable Rails/CreateTableWithTimestamps
|
|
17
|
+
|
|
18
|
+
add_index(:sidekiq_publisher_jobs, :published_at)
|
|
19
|
+
end
|
|
20
|
+
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "sidekiq"
|
|
4
|
+
|
|
5
|
+
module SidekiqPublisher
|
|
6
|
+
class Client < Sidekiq::Client
|
|
7
|
+
def bulk_push(items)
|
|
8
|
+
payloads = items.map do |item|
|
|
9
|
+
normed = normalize_item(item)
|
|
10
|
+
process_single(item["class"], normed)
|
|
11
|
+
end.compact
|
|
12
|
+
|
|
13
|
+
pushed = 0
|
|
14
|
+
with_connection do |conn|
|
|
15
|
+
conn.multi do
|
|
16
|
+
payloads.each do |payload|
|
|
17
|
+
atomic_push(conn, [payload])
|
|
18
|
+
pushed += 1
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
pushed
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
private
|
|
27
|
+
|
|
28
|
+
def with_connection(&blk)
|
|
29
|
+
@redis_pool.with(&blk)
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "active_record"
|
|
4
|
+
|
|
5
|
+
module SidekiqPublisher
|
|
6
|
+
class Job < ActiveRecord::Base
|
|
7
|
+
self.table_name = "sidekiq_publisher_jobs"
|
|
8
|
+
|
|
9
|
+
BATCH_KEYS = %i(id job_id job_class args run_at queue wrapped).freeze
|
|
10
|
+
|
|
11
|
+
before_create :ensure_job_id
|
|
12
|
+
before_save :ensure_string_job_class
|
|
13
|
+
|
|
14
|
+
validates :job_class, presence: true
|
|
15
|
+
validates :args, exclusion: { in: [nil] }
|
|
16
|
+
|
|
17
|
+
scope :unpublished, -> { where(published_at: nil) }
|
|
18
|
+
scope :published, -> { where.not(published_at: nil) }
|
|
19
|
+
scope :purgeable, -> { where("published_at < ?", Time.now.utc - job_retention_period) }
|
|
20
|
+
|
|
21
|
+
def self.generate_sidekiq_jid
|
|
22
|
+
SecureRandom.hex(12)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def self.job_retention_period
|
|
26
|
+
SidekiqPublisher.job_retention_period
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def self.published!(ids)
|
|
30
|
+
where(id: ids).update_all(published_at: Time.now.utc)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def self.purge_expired_published!
|
|
34
|
+
SidekiqPublisher.logger.info("#{name} purging expired published jobs.")
|
|
35
|
+
count = purgeable.delete_all
|
|
36
|
+
SidekiqPublisher.logger.info("#{name} purged #{count} expired published jobs.")
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def self.unpublished_batches(batch_size: SidekiqPublisher.batch_size)
|
|
40
|
+
unpublished.in_batches(of: batch_size, load: false) do |relation|
|
|
41
|
+
batch = relation.pluck(*BATCH_KEYS)
|
|
42
|
+
yield batch.map { |values| Hash[BATCH_KEYS.zip(values)] }
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# TODO: this method was just for testing and may be removed
|
|
47
|
+
def publish
|
|
48
|
+
Sidekiq::Client.push(sidekiq_item)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def sidekiq_item
|
|
52
|
+
{
|
|
53
|
+
"jid" => job_id,
|
|
54
|
+
"class" => job_class.constantize,
|
|
55
|
+
"args" => args,
|
|
56
|
+
"at" => run_at,
|
|
57
|
+
"queue" => queue,
|
|
58
|
+
"wrapped" => wrapped,
|
|
59
|
+
}.tap(&:compact!)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
private
|
|
63
|
+
|
|
64
|
+
def ensure_job_id
|
|
65
|
+
self.job_id ||= self.class.generate_sidekiq_jid
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def ensure_string_job_class
|
|
69
|
+
self.job_class = job_class.to_s
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "sidekiq_publisher/client"
|
|
4
|
+
|
|
5
|
+
module SidekiqPublisher
|
|
6
|
+
class Publisher
|
|
7
|
+
extend PrivateAttr
|
|
8
|
+
|
|
9
|
+
private_attr_reader :client
|
|
10
|
+
|
|
11
|
+
def initialize
|
|
12
|
+
@client = SidekiqPublisher::Client.new
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def publish
|
|
16
|
+
Job.unpublished_batches do |batch|
|
|
17
|
+
items = batch.map do |job|
|
|
18
|
+
{
|
|
19
|
+
"jid" => job[:job_id],
|
|
20
|
+
"class" => job[:job_class].constantize,
|
|
21
|
+
"args" => job[:args],
|
|
22
|
+
"at" => job[:run_at],
|
|
23
|
+
"queue" => job[:queue],
|
|
24
|
+
"wrapped" => job[:wrapped],
|
|
25
|
+
}.tap(&:compact!)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
publish_batch(batch, items)
|
|
29
|
+
end
|
|
30
|
+
purge_expired_published_jobs
|
|
31
|
+
rescue StandardError => ex
|
|
32
|
+
failure_warning(__method__, ex)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
private
|
|
36
|
+
|
|
37
|
+
def publish_batch(batch, items)
|
|
38
|
+
pushed_count = client.bulk_push(items)
|
|
39
|
+
published_count = update_jobs_as_published!(batch)
|
|
40
|
+
rescue StandardError => ex
|
|
41
|
+
failure_warning(__method__, ex)
|
|
42
|
+
ensure
|
|
43
|
+
update_jobs_as_published!(batch) if pushed_count.present? && published_count.nil?
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def update_jobs_as_published!(jobs)
|
|
47
|
+
Job.published!(jobs.map { |job| job[:id] })
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def purge_expired_published_jobs
|
|
51
|
+
Job.purge_expired_published! if perform_purge?
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def perform_purge?
|
|
55
|
+
rand(100).zero?
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def failure_warning(method, ex)
|
|
59
|
+
logger.warn("#{self.class.name}: msg=\"#{method} failed\" error=#{ex.class} error_msg=#{ex.message.inspect}\n"\
|
|
60
|
+
"#{ex.backtrace.join("\n")}")
|
|
61
|
+
SidekiqPublisher.exception_reporter&.call(ex)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def logger
|
|
65
|
+
SidekiqPublisher.logger
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SidekiqPublisher
|
|
4
|
+
class Railtie < Rails::Railtie
|
|
5
|
+
rake_tasks do
|
|
6
|
+
load "sidekiq_publisher/tasks.rake"
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
initializer "sidekiq_publisher.configure" do
|
|
10
|
+
SidekiqPublisher.logger = Rails.logger
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
end
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "activerecord-postgres_pub_sub"
|
|
4
|
+
|
|
5
|
+
module SidekiqPublisher
|
|
6
|
+
class Runner
|
|
7
|
+
extend PrivateAttr
|
|
8
|
+
|
|
9
|
+
LISTENER_TIMEOUT_SECONDS = 60
|
|
10
|
+
CHANNEL_NAME = "sidekiq_publisher_job"
|
|
11
|
+
|
|
12
|
+
private_attr_reader :publisher
|
|
13
|
+
|
|
14
|
+
def self.run
|
|
15
|
+
new.run
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def initialize
|
|
19
|
+
@publisher = Publisher.new
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def run
|
|
23
|
+
ActiveRecord::PostgresPubSub::Listener.listen(
|
|
24
|
+
CHANNEL_NAME,
|
|
25
|
+
listen_timeout: LISTENER_TIMEOUT_SECONDS
|
|
26
|
+
) do |listener|
|
|
27
|
+
listener.on_start { publisher.publish }
|
|
28
|
+
listener.on_notify { publisher.publish }
|
|
29
|
+
listener.on_timeout { listener_timeout }
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
private
|
|
34
|
+
|
|
35
|
+
def listener_timeout
|
|
36
|
+
if Job.unpublished.exists?
|
|
37
|
+
SidekiqPublisher.logger&.warn(
|
|
38
|
+
"#{self.class.name}: msg='publishing pending jobs at timeout'"
|
|
39
|
+
)
|
|
40
|
+
publisher.publish
|
|
41
|
+
else
|
|
42
|
+
Job.purge_expired_published!
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SidekiqPublisher
|
|
4
|
+
module Worker
|
|
5
|
+
def self.included(base)
|
|
6
|
+
base.include(Sidekiq::Worker)
|
|
7
|
+
base.extend(ClassMethods)
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
module ClassMethods
|
|
11
|
+
def client_push(item)
|
|
12
|
+
SidekiqPublisher::Job.create!(
|
|
13
|
+
job_class: item["class"].to_s,
|
|
14
|
+
args: item["args"],
|
|
15
|
+
run_at: item["at"],
|
|
16
|
+
queue: item["queue"]
|
|
17
|
+
)
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "private_attr"
|
|
4
|
+
require "sidekiq_publisher/version"
|
|
5
|
+
require "sidekiq_publisher/job"
|
|
6
|
+
require "sidekiq_publisher/worker"
|
|
7
|
+
require "sidekiq_publisher/publisher"
|
|
8
|
+
require "sidekiq_publisher/runner"
|
|
9
|
+
require "sidekiq_publisher/railtie" if defined?(Rails)
|
|
10
|
+
|
|
11
|
+
module SidekiqPublisher
|
|
12
|
+
DEFAULT_BATCH_SIZE = 100
|
|
13
|
+
DEFAULT_JOB_RETENTION_PERIOD = 1.day.freeze
|
|
14
|
+
|
|
15
|
+
class << self
|
|
16
|
+
attr_accessor :logger, :exception_reporter
|
|
17
|
+
attr_writer :batch_size, :job_retention_period
|
|
18
|
+
|
|
19
|
+
def batch_size
|
|
20
|
+
@batch_size || DEFAULT_BATCH_SIZE
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def job_retention_period
|
|
24
|
+
@job_retention_period || DEFAULT_JOB_RETENTION_PERIOD
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# For test purposes
|
|
28
|
+
def reset!
|
|
29
|
+
@batch_size = nil
|
|
30
|
+
@job_retention_period = nil
|
|
31
|
+
@exception_reporter = nil
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
data/log/.gitkeep
ADDED
|
File without changes
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
lib = File.expand_path("../lib", __FILE__)
|
|
4
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
5
|
+
require "sidekiq_publisher/version"
|
|
6
|
+
|
|
7
|
+
Gem::Specification.new do |spec|
|
|
8
|
+
spec.name = "sidekiq_publisher"
|
|
9
|
+
spec.version = SidekiqPublisher::VERSION
|
|
10
|
+
spec.authors = ["ezCater, Inc"]
|
|
11
|
+
spec.email = ["engineering@ezcater.com"]
|
|
12
|
+
|
|
13
|
+
spec.summary = "Publisher for enqueuing jobs to Sidekiq"
|
|
14
|
+
spec.description = spec.summary
|
|
15
|
+
spec.homepage = "https://github.com/ezcater/sidekiq_publisher"
|
|
16
|
+
|
|
17
|
+
spec.license = "MIT"
|
|
18
|
+
|
|
19
|
+
# Set "allowed_push_post" to control where this gem can be published.
|
|
20
|
+
if spec.respond_to?(:metadata)
|
|
21
|
+
spec.metadata["allowed_push_host"] = "https://rubygems.org"
|
|
22
|
+
|
|
23
|
+
else
|
|
24
|
+
raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
|
|
25
|
+
end
|
|
26
|
+
# rubocop:enable Style/GuardClause
|
|
27
|
+
|
|
28
|
+
excluded_files = %w(.circleci/config.yml
|
|
29
|
+
.github/PULL_REQUEST_TEMPLATE.md
|
|
30
|
+
.gitignore
|
|
31
|
+
.rspec
|
|
32
|
+
.rubocop.yml
|
|
33
|
+
.ruby-gemset
|
|
34
|
+
.ruby-version
|
|
35
|
+
.travis.yml
|
|
36
|
+
bin/console
|
|
37
|
+
bin/setup
|
|
38
|
+
Rakefile)
|
|
39
|
+
|
|
40
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
|
41
|
+
f.match(/^(test|spec|features)\//)
|
|
42
|
+
end - excluded_files
|
|
43
|
+
spec.bindir = "bin"
|
|
44
|
+
spec.executables = []
|
|
45
|
+
spec.require_paths = ["lib"]
|
|
46
|
+
|
|
47
|
+
spec.add_development_dependency "activejob"
|
|
48
|
+
spec.add_development_dependency "bundler", "~> 1.12"
|
|
49
|
+
spec.add_development_dependency "database_cleaner"
|
|
50
|
+
spec.add_development_dependency "ezcater_matchers" # TODO: this is a private gem
|
|
51
|
+
spec.add_development_dependency "ezcater_rubocop", "0.52.7"
|
|
52
|
+
spec.add_development_dependency "factory_bot"
|
|
53
|
+
spec.add_development_dependency "overcommit"
|
|
54
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
|
55
|
+
spec.add_development_dependency "redis-namespace"
|
|
56
|
+
spec.add_development_dependency "rspec", "~> 3.4"
|
|
57
|
+
spec.add_development_dependency "rspec_junit_formatter", "0.2.2"
|
|
58
|
+
spec.add_development_dependency "shoulda-matchers"
|
|
59
|
+
spec.add_development_dependency "simplecov"
|
|
60
|
+
|
|
61
|
+
spec.add_runtime_dependency "activerecord-postgres_pub_sub"
|
|
62
|
+
spec.add_runtime_dependency "private_attr"
|
|
63
|
+
spec.add_runtime_dependency "sidekiq", "~> 5.0.4"
|
|
64
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,287 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: sidekiq_publisher
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.0.rc0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- ezCater, Inc
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: bin
|
|
10
|
+
cert_chain: []
|
|
11
|
+
date: 2018-05-16 00:00:00.000000000 Z
|
|
12
|
+
dependencies:
|
|
13
|
+
- !ruby/object:Gem::Dependency
|
|
14
|
+
name: activejob
|
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
|
16
|
+
requirements:
|
|
17
|
+
- - ">="
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: '0'
|
|
20
|
+
type: :development
|
|
21
|
+
prerelease: false
|
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
23
|
+
requirements:
|
|
24
|
+
- - ">="
|
|
25
|
+
- !ruby/object:Gem::Version
|
|
26
|
+
version: '0'
|
|
27
|
+
- !ruby/object:Gem::Dependency
|
|
28
|
+
name: bundler
|
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
|
30
|
+
requirements:
|
|
31
|
+
- - "~>"
|
|
32
|
+
- !ruby/object:Gem::Version
|
|
33
|
+
version: '1.12'
|
|
34
|
+
type: :development
|
|
35
|
+
prerelease: false
|
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
37
|
+
requirements:
|
|
38
|
+
- - "~>"
|
|
39
|
+
- !ruby/object:Gem::Version
|
|
40
|
+
version: '1.12'
|
|
41
|
+
- !ruby/object:Gem::Dependency
|
|
42
|
+
name: database_cleaner
|
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
|
44
|
+
requirements:
|
|
45
|
+
- - ">="
|
|
46
|
+
- !ruby/object:Gem::Version
|
|
47
|
+
version: '0'
|
|
48
|
+
type: :development
|
|
49
|
+
prerelease: false
|
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
51
|
+
requirements:
|
|
52
|
+
- - ">="
|
|
53
|
+
- !ruby/object:Gem::Version
|
|
54
|
+
version: '0'
|
|
55
|
+
- !ruby/object:Gem::Dependency
|
|
56
|
+
name: ezcater_matchers
|
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
|
58
|
+
requirements:
|
|
59
|
+
- - ">="
|
|
60
|
+
- !ruby/object:Gem::Version
|
|
61
|
+
version: '0'
|
|
62
|
+
type: :development
|
|
63
|
+
prerelease: false
|
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
65
|
+
requirements:
|
|
66
|
+
- - ">="
|
|
67
|
+
- !ruby/object:Gem::Version
|
|
68
|
+
version: '0'
|
|
69
|
+
- !ruby/object:Gem::Dependency
|
|
70
|
+
name: ezcater_rubocop
|
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
|
72
|
+
requirements:
|
|
73
|
+
- - '='
|
|
74
|
+
- !ruby/object:Gem::Version
|
|
75
|
+
version: 0.52.7
|
|
76
|
+
type: :development
|
|
77
|
+
prerelease: false
|
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
79
|
+
requirements:
|
|
80
|
+
- - '='
|
|
81
|
+
- !ruby/object:Gem::Version
|
|
82
|
+
version: 0.52.7
|
|
83
|
+
- !ruby/object:Gem::Dependency
|
|
84
|
+
name: factory_bot
|
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
|
86
|
+
requirements:
|
|
87
|
+
- - ">="
|
|
88
|
+
- !ruby/object:Gem::Version
|
|
89
|
+
version: '0'
|
|
90
|
+
type: :development
|
|
91
|
+
prerelease: false
|
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
93
|
+
requirements:
|
|
94
|
+
- - ">="
|
|
95
|
+
- !ruby/object:Gem::Version
|
|
96
|
+
version: '0'
|
|
97
|
+
- !ruby/object:Gem::Dependency
|
|
98
|
+
name: overcommit
|
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
|
100
|
+
requirements:
|
|
101
|
+
- - ">="
|
|
102
|
+
- !ruby/object:Gem::Version
|
|
103
|
+
version: '0'
|
|
104
|
+
type: :development
|
|
105
|
+
prerelease: false
|
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
107
|
+
requirements:
|
|
108
|
+
- - ">="
|
|
109
|
+
- !ruby/object:Gem::Version
|
|
110
|
+
version: '0'
|
|
111
|
+
- !ruby/object:Gem::Dependency
|
|
112
|
+
name: rake
|
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
|
114
|
+
requirements:
|
|
115
|
+
- - "~>"
|
|
116
|
+
- !ruby/object:Gem::Version
|
|
117
|
+
version: '10.0'
|
|
118
|
+
type: :development
|
|
119
|
+
prerelease: false
|
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
121
|
+
requirements:
|
|
122
|
+
- - "~>"
|
|
123
|
+
- !ruby/object:Gem::Version
|
|
124
|
+
version: '10.0'
|
|
125
|
+
- !ruby/object:Gem::Dependency
|
|
126
|
+
name: redis-namespace
|
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
|
128
|
+
requirements:
|
|
129
|
+
- - ">="
|
|
130
|
+
- !ruby/object:Gem::Version
|
|
131
|
+
version: '0'
|
|
132
|
+
type: :development
|
|
133
|
+
prerelease: false
|
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
135
|
+
requirements:
|
|
136
|
+
- - ">="
|
|
137
|
+
- !ruby/object:Gem::Version
|
|
138
|
+
version: '0'
|
|
139
|
+
- !ruby/object:Gem::Dependency
|
|
140
|
+
name: rspec
|
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
|
142
|
+
requirements:
|
|
143
|
+
- - "~>"
|
|
144
|
+
- !ruby/object:Gem::Version
|
|
145
|
+
version: '3.4'
|
|
146
|
+
type: :development
|
|
147
|
+
prerelease: false
|
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
149
|
+
requirements:
|
|
150
|
+
- - "~>"
|
|
151
|
+
- !ruby/object:Gem::Version
|
|
152
|
+
version: '3.4'
|
|
153
|
+
- !ruby/object:Gem::Dependency
|
|
154
|
+
name: rspec_junit_formatter
|
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
|
156
|
+
requirements:
|
|
157
|
+
- - '='
|
|
158
|
+
- !ruby/object:Gem::Version
|
|
159
|
+
version: 0.2.2
|
|
160
|
+
type: :development
|
|
161
|
+
prerelease: false
|
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
163
|
+
requirements:
|
|
164
|
+
- - '='
|
|
165
|
+
- !ruby/object:Gem::Version
|
|
166
|
+
version: 0.2.2
|
|
167
|
+
- !ruby/object:Gem::Dependency
|
|
168
|
+
name: shoulda-matchers
|
|
169
|
+
requirement: !ruby/object:Gem::Requirement
|
|
170
|
+
requirements:
|
|
171
|
+
- - ">="
|
|
172
|
+
- !ruby/object:Gem::Version
|
|
173
|
+
version: '0'
|
|
174
|
+
type: :development
|
|
175
|
+
prerelease: false
|
|
176
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
177
|
+
requirements:
|
|
178
|
+
- - ">="
|
|
179
|
+
- !ruby/object:Gem::Version
|
|
180
|
+
version: '0'
|
|
181
|
+
- !ruby/object:Gem::Dependency
|
|
182
|
+
name: simplecov
|
|
183
|
+
requirement: !ruby/object:Gem::Requirement
|
|
184
|
+
requirements:
|
|
185
|
+
- - ">="
|
|
186
|
+
- !ruby/object:Gem::Version
|
|
187
|
+
version: '0'
|
|
188
|
+
type: :development
|
|
189
|
+
prerelease: false
|
|
190
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
191
|
+
requirements:
|
|
192
|
+
- - ">="
|
|
193
|
+
- !ruby/object:Gem::Version
|
|
194
|
+
version: '0'
|
|
195
|
+
- !ruby/object:Gem::Dependency
|
|
196
|
+
name: activerecord-postgres_pub_sub
|
|
197
|
+
requirement: !ruby/object:Gem::Requirement
|
|
198
|
+
requirements:
|
|
199
|
+
- - ">="
|
|
200
|
+
- !ruby/object:Gem::Version
|
|
201
|
+
version: '0'
|
|
202
|
+
type: :runtime
|
|
203
|
+
prerelease: false
|
|
204
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
205
|
+
requirements:
|
|
206
|
+
- - ">="
|
|
207
|
+
- !ruby/object:Gem::Version
|
|
208
|
+
version: '0'
|
|
209
|
+
- !ruby/object:Gem::Dependency
|
|
210
|
+
name: private_attr
|
|
211
|
+
requirement: !ruby/object:Gem::Requirement
|
|
212
|
+
requirements:
|
|
213
|
+
- - ">="
|
|
214
|
+
- !ruby/object:Gem::Version
|
|
215
|
+
version: '0'
|
|
216
|
+
type: :runtime
|
|
217
|
+
prerelease: false
|
|
218
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
219
|
+
requirements:
|
|
220
|
+
- - ">="
|
|
221
|
+
- !ruby/object:Gem::Version
|
|
222
|
+
version: '0'
|
|
223
|
+
- !ruby/object:Gem::Dependency
|
|
224
|
+
name: sidekiq
|
|
225
|
+
requirement: !ruby/object:Gem::Requirement
|
|
226
|
+
requirements:
|
|
227
|
+
- - "~>"
|
|
228
|
+
- !ruby/object:Gem::Version
|
|
229
|
+
version: 5.0.4
|
|
230
|
+
type: :runtime
|
|
231
|
+
prerelease: false
|
|
232
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
233
|
+
requirements:
|
|
234
|
+
- - "~>"
|
|
235
|
+
- !ruby/object:Gem::Version
|
|
236
|
+
version: 5.0.4
|
|
237
|
+
description: Publisher for enqueuing jobs to Sidekiq
|
|
238
|
+
email:
|
|
239
|
+
- engineering@ezcater.com
|
|
240
|
+
executables: []
|
|
241
|
+
extensions: []
|
|
242
|
+
extra_rdoc_files: []
|
|
243
|
+
files:
|
|
244
|
+
- CHANGELOG.md
|
|
245
|
+
- Gemfile
|
|
246
|
+
- LICENSE.txt
|
|
247
|
+
- README.md
|
|
248
|
+
- lib/active_job/queue_adapters/sidekiq_publisher_adapter.rb
|
|
249
|
+
- lib/generators/sidekiq_publisher/install_generator.rb
|
|
250
|
+
- lib/generators/sidekiq_publisher/templates/create_sidekiq_publisher_jobs.rb
|
|
251
|
+
- lib/sidekiq_publisher.rb
|
|
252
|
+
- lib/sidekiq_publisher/client.rb
|
|
253
|
+
- lib/sidekiq_publisher/job.rb
|
|
254
|
+
- lib/sidekiq_publisher/publisher.rb
|
|
255
|
+
- lib/sidekiq_publisher/railtie.rb
|
|
256
|
+
- lib/sidekiq_publisher/runner.rb
|
|
257
|
+
- lib/sidekiq_publisher/tasks.rake
|
|
258
|
+
- lib/sidekiq_publisher/version.rb
|
|
259
|
+
- lib/sidekiq_publisher/worker.rb
|
|
260
|
+
- log/.gitkeep
|
|
261
|
+
- sidekiq_publisher.gemspec
|
|
262
|
+
homepage: https://github.com/ezcater/sidekiq_publisher
|
|
263
|
+
licenses:
|
|
264
|
+
- MIT
|
|
265
|
+
metadata:
|
|
266
|
+
allowed_push_host: https://rubygems.org
|
|
267
|
+
post_install_message:
|
|
268
|
+
rdoc_options: []
|
|
269
|
+
require_paths:
|
|
270
|
+
- lib
|
|
271
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
272
|
+
requirements:
|
|
273
|
+
- - ">="
|
|
274
|
+
- !ruby/object:Gem::Version
|
|
275
|
+
version: '0'
|
|
276
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
277
|
+
requirements:
|
|
278
|
+
- - ">"
|
|
279
|
+
- !ruby/object:Gem::Version
|
|
280
|
+
version: 1.3.1
|
|
281
|
+
requirements: []
|
|
282
|
+
rubyforge_project:
|
|
283
|
+
rubygems_version: 2.6.11
|
|
284
|
+
signing_key:
|
|
285
|
+
specification_version: 4
|
|
286
|
+
summary: Publisher for enqueuing jobs to Sidekiq
|
|
287
|
+
test_files: []
|