que 0.11.3 → 2.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (114) hide show
  1. checksums.yaml +5 -5
  2. data/.github/workflows/tests.yml +51 -0
  3. data/.gitignore +2 -0
  4. data/.ruby-version +1 -0
  5. data/CHANGELOG.md +502 -97
  6. data/Dockerfile +20 -0
  7. data/LICENSE.txt +1 -1
  8. data/README.md +205 -59
  9. data/auto/dev +21 -0
  10. data/auto/pre-push-hook +30 -0
  11. data/auto/psql +9 -0
  12. data/auto/test +5 -0
  13. data/auto/test-postgres-14 +17 -0
  14. data/bin/que +8 -81
  15. data/docker-compose.yml +47 -0
  16. data/docs/README.md +881 -0
  17. data/lib/que/active_job/extensions.rb +114 -0
  18. data/lib/que/active_record/connection.rb +51 -0
  19. data/lib/que/active_record/model.rb +48 -0
  20. data/lib/que/command_line_interface.rb +259 -0
  21. data/lib/que/connection.rb +198 -0
  22. data/lib/que/connection_pool.rb +78 -0
  23. data/lib/que/job.rb +210 -103
  24. data/lib/que/job_buffer.rb +255 -0
  25. data/lib/que/job_methods.rb +176 -0
  26. data/lib/que/listener.rb +176 -0
  27. data/lib/que/locker.rb +507 -0
  28. data/lib/que/metajob.rb +47 -0
  29. data/lib/que/migrations/4/down.sql +48 -0
  30. data/lib/que/migrations/4/up.sql +267 -0
  31. data/lib/que/migrations/5/down.sql +73 -0
  32. data/lib/que/migrations/5/up.sql +76 -0
  33. data/lib/que/migrations/6/down.sql +8 -0
  34. data/lib/que/migrations/6/up.sql +8 -0
  35. data/lib/que/migrations/7/down.sql +5 -0
  36. data/lib/que/migrations/7/up.sql +13 -0
  37. data/lib/que/migrations.rb +37 -18
  38. data/lib/que/poller.rb +274 -0
  39. data/lib/que/rails/railtie.rb +12 -0
  40. data/lib/que/result_queue.rb +35 -0
  41. data/lib/que/sequel/model.rb +52 -0
  42. data/lib/que/utils/assertions.rb +62 -0
  43. data/lib/que/utils/constantization.rb +19 -0
  44. data/lib/que/utils/error_notification.rb +68 -0
  45. data/lib/que/utils/freeze.rb +20 -0
  46. data/lib/que/utils/introspection.rb +50 -0
  47. data/lib/que/utils/json_serialization.rb +21 -0
  48. data/lib/que/utils/logging.rb +79 -0
  49. data/lib/que/utils/middleware.rb +46 -0
  50. data/lib/que/utils/queue_management.rb +18 -0
  51. data/lib/que/utils/ruby2_keywords.rb +19 -0
  52. data/lib/que/utils/transactions.rb +34 -0
  53. data/lib/que/version.rb +5 -1
  54. data/lib/que/worker.rb +145 -149
  55. data/lib/que.rb +103 -159
  56. data/que.gemspec +17 -4
  57. data/scripts/docker-entrypoint +14 -0
  58. data/scripts/test +6 -0
  59. metadata +59 -95
  60. data/.rspec +0 -2
  61. data/.travis.yml +0 -17
  62. data/Gemfile +0 -24
  63. data/docs/advanced_setup.md +0 -106
  64. data/docs/customizing_que.md +0 -200
  65. data/docs/error_handling.md +0 -47
  66. data/docs/inspecting_the_queue.md +0 -114
  67. data/docs/logging.md +0 -50
  68. data/docs/managing_workers.md +0 -80
  69. data/docs/migrating.md +0 -30
  70. data/docs/multiple_queues.md +0 -27
  71. data/docs/shutting_down_safely.md +0 -7
  72. data/docs/using_plain_connections.md +0 -41
  73. data/docs/using_sequel.md +0 -31
  74. data/docs/writing_reliable_jobs.md +0 -117
  75. data/lib/generators/que/install_generator.rb +0 -24
  76. data/lib/generators/que/templates/add_que.rb +0 -13
  77. data/lib/que/adapters/active_record.rb +0 -54
  78. data/lib/que/adapters/base.rb +0 -127
  79. data/lib/que/adapters/connection_pool.rb +0 -16
  80. data/lib/que/adapters/pg.rb +0 -21
  81. data/lib/que/adapters/pond.rb +0 -16
  82. data/lib/que/adapters/sequel.rb +0 -20
  83. data/lib/que/railtie.rb +0 -16
  84. data/lib/que/rake_tasks.rb +0 -59
  85. data/lib/que/sql.rb +0 -152
  86. data/spec/adapters/active_record_spec.rb +0 -152
  87. data/spec/adapters/connection_pool_spec.rb +0 -22
  88. data/spec/adapters/pg_spec.rb +0 -41
  89. data/spec/adapters/pond_spec.rb +0 -22
  90. data/spec/adapters/sequel_spec.rb +0 -57
  91. data/spec/gemfiles/Gemfile1 +0 -18
  92. data/spec/gemfiles/Gemfile2 +0 -18
  93. data/spec/spec_helper.rb +0 -118
  94. data/spec/support/helpers.rb +0 -19
  95. data/spec/support/jobs.rb +0 -35
  96. data/spec/support/shared_examples/adapter.rb +0 -37
  97. data/spec/support/shared_examples/multi_threaded_adapter.rb +0 -46
  98. data/spec/travis.rb +0 -23
  99. data/spec/unit/connection_spec.rb +0 -14
  100. data/spec/unit/customization_spec.rb +0 -251
  101. data/spec/unit/enqueue_spec.rb +0 -245
  102. data/spec/unit/helper_spec.rb +0 -12
  103. data/spec/unit/logging_spec.rb +0 -101
  104. data/spec/unit/migrations_spec.rb +0 -84
  105. data/spec/unit/pool_spec.rb +0 -365
  106. data/spec/unit/run_spec.rb +0 -14
  107. data/spec/unit/states_spec.rb +0 -50
  108. data/spec/unit/stats_spec.rb +0 -46
  109. data/spec/unit/transaction_spec.rb +0 -36
  110. data/spec/unit/work_spec.rb +0 -407
  111. data/spec/unit/worker_spec.rb +0 -167
  112. data/tasks/benchmark.rb +0 -3
  113. data/tasks/rspec.rb +0 -14
  114. data/tasks/safe_shutdown.rb +0 -67
data/Dockerfile ADDED
@@ -0,0 +1,20 @@
1
+ FROM ruby:3.1.1-slim-buster@sha256:2ada3e4fe7b1703c9333ad4eb9fc12c1d4d60bce0f981281b2151057e928d9ad AS base
2
+
3
+ # Install libpq-dev in our base layer, as it's needed in all environments
4
+ RUN apt-get update \
5
+ && apt-get install -y libpq-dev \
6
+ && rm -rf /var/lib/apt/lists/*
7
+
8
+ ENV RUBY_BUNDLER_VERSION 2.3.7
9
+ RUN gem install bundler -v $RUBY_BUNDLER_VERSION
10
+
11
+ ENV BUNDLE_PATH /usr/local/bundle
12
+
13
+ ENV RUBYOPT=-W:deprecated
14
+
15
+ FROM base AS dev-environment
16
+
17
+ # Install build-essential and git, as we'd need them for building gems that have native code components
18
+ RUN apt-get update \
19
+ && apt-get install -y build-essential git \
20
+ && rm -rf /var/lib/apt/lists/*
data/LICENSE.txt CHANGED
@@ -1,4 +1,4 @@
1
- Copyright (c) 2013 Chris Hanks
1
+ Copyright (c) 2013-2017 Chris Hanks
2
2
 
3
3
  MIT License
4
4
 
data/README.md CHANGED
@@ -1,81 +1,104 @@
1
- # Que
1
+ # Que ![tests](https://github.com/que-rb/que/workflows/tests/badge.svg)
2
2
 
3
- **TL;DR: Que is a high-performance alternative to DelayedJob or QueueClassic that improves the reliability of your application by protecting your jobs with the same [ACID guarantees](https://en.wikipedia.org/wiki/ACID) as the rest of your data.**
3
+ **This README and the rest of the docs on the master branch all refer to Que 2.x. For older versions, please refer to the docs on the respective branches: [1.x](https://github.com/que-rb/que/tree/1.x), or [0.x](https://github.com/que-rb/que/tree/0.x).**
4
+
5
+ *TL;DR: Que is a high-performance job queue that improves the reliability of your application by protecting your jobs with the same [ACID guarantees](https://en.wikipedia.org/wiki/ACID) as the rest of your data.*
4
6
 
5
7
  Que ("keɪ", or "kay") is a queue for Ruby and PostgreSQL that manages jobs using [advisory locks](http://www.postgresql.org/docs/current/static/explicit-locking.html#ADVISORY-LOCKS), which gives it several advantages over other RDBMS-backed queues:
6
8
 
7
- * **Concurrency** - Workers don't block each other when trying to lock jobs, as often occurs with "SELECT FOR UPDATE"-style locking. This allows for very high throughput with a large number of workers.
8
- * **Efficiency** - Locks are held in memory, so locking a job doesn't incur a disk write. These first two points are what limit performance with other queues - all workers trying to lock jobs have to wait behind one that's persisting its UPDATE on a locked_at column to disk (and the disks of however many other servers your database is synchronously replicating to). Under heavy load, Que's bottleneck is CPU, not I/O.
9
- * **Safety** - If a Ruby process dies, the jobs it's working won't be lost, or left in a locked or ambiguous state - they immediately become available for any other worker to pick up.
9
+ - **Concurrency** - Workers don't block each other when trying to lock jobs, as often occurs with "SELECT FOR UPDATE"-style locking. This allows for very high throughput with a large number of workers.
10
+ - **Efficiency** - Locks are held in memory, so locking a job doesn't incur a disk write. These first two points are what limit performance with other queues. Under heavy load, Que's bottleneck is CPU, not I/O.
11
+ - **Safety** - If a Ruby process dies, the jobs it's working won't be lost, or left in a locked or ambiguous state - they immediately become available for any other worker to pick up.
10
12
 
11
13
  Additionally, there are the general benefits of storing jobs in Postgres, alongside the rest of your data, rather than in Redis or a dedicated queue:
12
14
 
13
- * **Transactional Control** - Queue a job along with other changes to your database, and it'll commit or rollback with everything else. If you're using ActiveRecord or Sequel, Que can piggyback on their connections, so setup is simple and jobs are protected by the transactions you're already using.
14
- * **Atomic Backups** - Your jobs and data can be backed up together and restored as a snapshot. If your jobs relate to your data (and they usually do), there's no risk of jobs falling through the cracks during a recovery.
15
- * **Fewer Dependencies** - If you're already using Postgres (and you probably should be), a separate queue is another moving part that can break.
16
- * **Security** - Postgres' support for SSL connections keeps your data safe in transport, for added protection when you're running workers on cloud platforms that you can't completely control.
15
+ - **Transactional Control** - Queue a job along with other changes to your database, and it'll commit or rollback with everything else. If you're using ActiveRecord or Sequel, Que can piggyback on their connections, so setup is simple and jobs are protected by the transactions you're already using.
16
+ - **Atomic Backups** - Your jobs and data can be backed up together and restored as a snapshot. If your jobs relate to your data (and they usually do), there's no risk of jobs falling through the cracks during a recovery.
17
+ - **Fewer Dependencies** - If you're already using Postgres (and you probably should be), a separate queue is another moving part that can break.
18
+ - **Security** - Postgres' support for SSL connections keeps your data safe in transport, for added protection when you're running workers on cloud platforms that you can't completely control.
17
19
 
18
- Que's primary goal is reliability. You should be able to leave your application running indefinitely without worrying about jobs being lost due to a lack of transactional support, or left in limbo due to a crashing process. Que does everything it can to ensure that jobs you queue are performed exactly once (though the occasional repetition of a job can be impossible to avoid - see the docs on [how to write a reliable job](https://github.com/chanks/que/blob/master/docs/writing_reliable_jobs.md)).
20
+ Que's primary goal is reliability. You should be able to leave your application running indefinitely without worrying about jobs being lost due to a lack of transactional support, or left in limbo due to a crashing process. Que does everything it can to ensure that jobs you queue are performed exactly once (though the occasional repetition of a job can be impossible to avoid - see the docs on [how to write a reliable job](/docs/README.md#writing-reliable-jobs)).
19
21
 
20
- Que's secondary goal is performance. It won't be able to match the speed or throughput of a dedicated queue, or maybe even a Redis-backed queue, but it should be fast enough for most use cases. In [benchmarks of RDBMS queues](https://github.com/chanks/queue-shootout) using PostgreSQL 9.3 on a AWS c3.8xlarge instance, Que approaches 10,000 jobs per second, or about twenty times the throughput of DelayedJob or QueueClassic. You are encouraged to try things out on your own production hardware, though.
22
+ Que's secondary goal is performance. The worker process is multithreaded, so that a single process can run many jobs simultaneously.
21
23
 
22
- Que also includes a worker pool, so that multiple threads can process jobs in the same process. It can even do this in the background of your web process - if you're running on Heroku, for example, you don't need to run a separate worker dyno.
24
+ Compatibility:
23
25
 
24
- Que is tested on Ruby 2.0, Rubinius and JRuby (with the `jruby-pg` gem, which is [not yet functional with ActiveRecord](https://github.com/chanks/que/issues/4#issuecomment-29561356)). It requires Postgres 9.2+ for the JSON datatype.
26
+ - MRI Ruby 2.7+ (for Ruby 3, Que 2+ is required)
27
+ - PostgreSQL 9.5+
28
+ - Rails 6.0+ (optional)
25
29
 
26
30
  **Please note** - Que's job table undergoes a lot of churn when it is under high load, and like any heavily-written table, is susceptible to bloat and slowness if Postgres isn't able to clean it up. The most common cause of this is long-running transactions, so it's recommended to try to keep all transactions against the database housing Que's job table as short as possible. This is good advice to remember for any high-activity database, but bears emphasizing when using tables that undergo a lot of writes.
27
31
 
28
-
29
32
  ## Installation
30
33
 
31
34
  Add this line to your application's Gemfile:
32
35
 
33
- gem 'que'
36
+ ```ruby
37
+ gem 'que'
38
+ ```
34
39
 
35
40
  And then execute:
36
41
 
37
- $ bundle
42
+ ```bash
43
+ bundle
44
+ ```
38
45
 
39
46
  Or install it yourself as:
40
47
 
41
- $ gem install que
42
-
48
+ ```bash
49
+ gem install que
50
+ ```
43
51
 
44
52
  ## Usage
45
53
 
46
- The following assumes you're using Rails 4.0 and ActiveRecord. *Que hasn't been tested with versions of Rails before 4.0, and may or may not work with them.* See the [/docs directory](https://github.com/chanks/que/blob/master/docs) for instructions on using Que [outside of Rails](https://github.com/chanks/que/blob/master/docs/advanced_setup.md), and with [Sequel](https://github.com/chanks/que/blob/master/docs/using_sequel.md) or [no ORM](https://github.com/chanks/que/blob/master/docs/using_plain_connections.md), among other things.
54
+ First, create the queue schema in a migration. For example:
47
55
 
48
- First, generate and run a migration for the job table.
56
+ ```ruby
57
+ class CreateQueSchema < ActiveRecord::Migration[6.0]
58
+ def up
59
+ # Whenever you use Que in a migration, always specify the version you're
60
+ # migrating to. If you're unsure what the current version is, check the
61
+ # changelog.
62
+ Que.migrate!(version: 6)
63
+ end
49
64
 
50
- $ bin/rails generate que:install
51
- $ bin/rake db:migrate
65
+ def down
66
+ # Migrate to version 0 to remove Que entirely.
67
+ Que.migrate!(version: 0)
68
+ end
69
+ end
70
+ ```
52
71
 
53
72
  Create a class for each type of job you want to run:
54
73
 
55
-
56
- ``` ruby
74
+ ```ruby
57
75
  # app/jobs/charge_credit_card.rb
58
76
  class ChargeCreditCard < Que::Job
59
77
  # Default settings for this job. These are optional - without them, jobs
60
78
  # will default to priority 100 and run immediately.
61
- @priority = 10
62
- @run_at = proc { 1.minute.from_now }
79
+ self.run_at = proc { 1.minute.from_now }
80
+
81
+ # We use the Linux priority scale - a lower number is more important.
82
+ self.priority = 10
63
83
 
64
- def run(user_id, options)
84
+ def run(credit_card_id, user_id:)
65
85
  # Do stuff.
66
- user = User[user_id]
67
- card = CreditCard[options[:credit_card_id]]
86
+ user = User.find(user_id)
87
+ card = CreditCard.find(credit_card_id)
68
88
 
69
- ActiveRecord::Base.transaction do
89
+ User.transaction do
70
90
  # Write any changes you'd like to the database.
71
- user.update_attributes :charged_at => Time.now
91
+ user.update charged_at: Time.now
72
92
 
73
93
  # It's best to destroy the job in the same transaction as any other
74
- # changes you make. Que will destroy the job for you after the run
75
- # method if you don't do it yourself, but if your job writes to the
76
- # DB but doesn't destroy the job in the same transaction, it's
77
- # possible that the job could be repeated in the event of a crash.
94
+ # changes you make. Que will mark the job as destroyed for you after the
95
+ # run method if you don't do it yourself, but if your job writes to the DB
96
+ # but doesn't destroy the job in the same transaction, it's possible that
97
+ # the job could be repeated in the event of a crash.
78
98
  destroy
99
+
100
+ # If you'd rather leave the job record in the database to maintain a job
101
+ # history, simply replace the `destroy` call with a `finish` call.
79
102
  end
80
103
  end
81
104
  end
@@ -83,57 +106,180 @@ end
83
106
 
84
107
  Queue your job. Again, it's best to do this in a transaction with other changes you're making. Also note that any arguments you pass will be serialized to JSON and back again, so stick to simple types (strings, integers, floats, hashes, and arrays).
85
108
 
86
- ``` ruby
87
- ActiveRecord::Base.transaction do
109
+ ```ruby
110
+ CreditCard.transaction do
88
111
  # Persist credit card information
89
112
  card = CreditCard.create(params[:credit_card])
90
- ChargeCreditCard.enqueue(current_user.id, :credit_card_id => card.id)
113
+ ChargeCreditCard.enqueue(card.id, user_id: current_user.id)
91
114
  end
92
115
  ```
93
116
 
94
117
  You can also add options to run the job after a specific time, or with a specific priority:
95
118
 
96
- ``` ruby
97
- # The default priority is 100, and a lower number means a higher priority. 5 would be very important.
98
- ChargeCreditCard.enqueue current_user.id, :credit_card_id => card.id, :run_at => 1.day.from_now, :priority => 5
119
+ ```ruby
120
+ ChargeCreditCard.enqueue(card.id, user_id: current_user.id, job_options: { run_at: 1.day.from_now, priority: 5 })
121
+ ```
122
+
123
+ [Learn more about job options](docs/README.md#job-options).
124
+
125
+ ## Running the Que Worker
126
+
127
+ In order to process jobs, you must start a separate worker process outside of your main server.
128
+
129
+ ```bash
130
+ bundle exec que
99
131
  ```
100
132
 
101
- To determine what happens when a job is queued, you can set Que's mode. There are a few options for the mode:
133
+ Try running `que -h` to get a list of runtime options:
134
+
135
+ ```
136
+ $ que -h
137
+ usage: que [options] [file/to/require] ...
138
+ -h, --help Show this help text.
139
+ -i, --poll-interval [INTERVAL] Set maximum interval between polls for available jobs, in seconds (default: 5)
140
+ ...
141
+ ```
142
+
143
+ You may need to pass que a file path to require so that it can load your app. Que will automatically load `config/environment.rb` if it exists, so you shouldn't need an argument if you're using Rails.
144
+
145
+ ## Additional Rails-specific Setup
146
+
147
+ If you're using ActiveRecord to dump your database's schema, please [set your schema_format to :sql](http://guides.rubyonrails.org/migrations.html#types-of-schema-dumps) so that Que's table structure is managed correctly. This is a good idea regardless, as the `:ruby` schema format doesn't support many of PostgreSQL's advanced features.
148
+
149
+ Pre-1.0, the default queue name needed to be configured in order for Que to work out of the box with Rails. As of 1.0 the default queue name is now 'default', as Rails expects, but when Rails enqueues some types of jobs it may try to use another queue name that isn't worked by default. You can either:
150
+
151
+ - [Configure Rails](https://guides.rubyonrails.org/configuring.html) to send all internal job types to the 'default' queue by adding the following to `config/application.rb`:
152
+
153
+ ```ruby
154
+ config.action_mailer.deliver_later_queue_name = :default
155
+ config.action_mailbox.queues.incineration = :default
156
+ config.action_mailbox.queues.routing = :default
157
+ config.active_storage.queues.analysis = :default
158
+ config.active_storage.queues.purge = :default
159
+ ```
160
+
161
+ - [Tell que](/docs#multiple-queues) to work all of these queues (less efficient because it requires polling all of them):
162
+
163
+ ```bash
164
+ que -q default -q mailers -q action_mailbox_incineration -q action_mailbox_routing -q active_storage_analysis -q active_storage_purge
165
+ ```
166
+
167
+ Also, if you would like to integrate Que with Active Job, you can do it by setting the adapter in `config/application.rb` or in a specific environment by setting it in `config/environments/production.rb`, for example:
168
+
169
+ ```ruby
170
+ config.active_job.queue_adapter = :que
171
+ ```
172
+
173
+ Que will automatically use the database configuration of your rails application, so there is no need to configure anything else.
174
+
175
+ You can then write your jobs as usual following the [Active Job documentation](https://guides.rubyonrails.org/active_job_basics.html). However, be aware that you'll lose the ability to finish the job in the same transaction as other database operations. That happens because Active Job is a generic background job framework that doesn't benefit from the database integration Que provides.
176
+
177
+ If you later decide to switch a job from Active Job to Que to have transactional integrity you can easily change the corresponding job class to inherit from `Que::Job` and follow the usage guidelines in the previous section.
102
178
 
103
- * `Que.mode = :off` - In this mode, queueing a job will simply insert it into the database - the current process will make no effort to run it. You should use this if you want to use a dedicated process to work tasks (there's an executable included that will do this, `que`). This is the default when running `bin/rails console`.
104
- * `Que.mode = :async` - In this mode, a pool of background workers is spun up, each running in their own thread. See the docs for [more information on managing workers](https://github.com/chanks/que/blob/master/docs/managing_workers.md).
105
- * `Que.mode = :sync` - In this mode, any jobs you queue will be run in the same thread, synchronously (that is, `MyJob.enqueue` runs the job and won't return until it's completed). This makes your application's behavior easier to test, so it's the default in the test environment.
179
+ ## Testing
106
180
 
107
- **If you're using ActiveRecord to dump your database's schema, [set your schema_format to :sql](http://guides.rubyonrails.org/migrations.html#types-of-schema-dumps) so that Que's table structure is managed correctly.** (You can use schema_format as :ruby if you want but keep in mind this is highly advised against, as some parts of Que will not work.)
181
+ There are a couple ways to do testing. You may want to set `Que::Job.run_synchronously = true`, which will cause JobClass.enqueue to simply execute the job's logic synchronously, as if you'd run JobClass.run(*your_args). Or, you may want to leave it disabled so you can assert on the job state once they are stored in the database.
108
182
 
183
+ ## Documentation
184
+
185
+ **For full documentation, see [here](docs/README.md)**.
109
186
 
110
187
  ## Related Projects
111
188
 
112
- * [que-web](https://github.com/statianzo/que-web) is a Sinatra-based UI for inspecting your job queue.
113
- * [que-testing](https://github.com/statianzo/que-testing) allows making assertions on enqueued jobs.
114
- * [que-go](https://github.com/bgentry/que-go) is a port of Que for the Go programming language. It uses the same table structure, so that you can use the same job queue from Ruby and Go applications.
115
- * [wisper-que](https://github.com/joevandyk/wisper-que) adds support for Que to [wisper](https://github.com/krisleech/wisper).
189
+ These projects are tested to be compatible with Que 1.x:
116
190
 
117
- If you have a project that uses or relates to Que, feel free to submit a PR adding it to the list!
191
+ - [que-web](https://github.com/statianzo/que-web) is a Sinatra-based UI for inspecting your job queue.
192
+ - [que-scheduler](https://github.com/hlascelles/que-scheduler) lets you schedule tasks using a cron style config file.
193
+ - [que-locks](https://github.com/airhorns/que-locks) lets you lock around job execution for so only one job runs at once for a set of arguments.
118
194
 
195
+ If you have a project that uses or relates to Que, feel free to submit a PR adding it to the list!
119
196
 
120
197
  ## Community and Contributing
121
198
 
122
- * For bugs in the library, please feel free to [open an issue](https://github.com/chanks/que/issues/new).
123
- * For general discussion and questions/concerns that don't relate to obvious bugs, try posting on the [que-talk Google Group](https://groups.google.com/forum/#!forum/que-talk).
124
- * For contributions, pull requests submitted via Github are welcome.
125
-
126
- Regarding contributions, one of the project's priorities is to keep Que as simple, lightweight and dependency-free as possible, and pull requests that change too much or wouldn't be useful to the majority of Que's users have a good chance of being rejected. If you're thinking of submitting a pull request that adds a new feature, consider starting a discussion in [que-talk](https://groups.google.com/forum/#!forum/que-talk) first about what it would do and how it would be implemented. If it's a sufficiently large feature, or if most of Que's users wouldn't find it useful, it may be best implemented as a standalone gem, like some of the related projects above.
199
+ - For feature suggestions or bugs in the library, please feel free to [open an issue](https://github.com/que-rb/que/issues/new).
200
+ - For general discussion and questions/concerns that don't relate to obvious bugs, join our [Discord Server](https://discord.gg/B3EW32H).
201
+ - For contributions, pull requests submitted via Github are welcome.
127
202
 
203
+ Regarding contributions, one of the project's priorities is to keep Que as simple, lightweight and dependency-free as possible, and pull requests that change too much or wouldn't be useful to the majority of Que's users have a good chance of being rejected. If you're thinking of submitting a pull request that adds a new feature, consider starting a discussion in an issue first about what it would do and how it would be implemented. If it's a sufficiently large feature, or if most of Que's users wouldn't find it useful, it may be best implemented as a standalone gem, like some of the related projects above.
128
204
 
129
205
  ### Specs
130
206
 
131
- A note on running specs - Que's worker system is multithreaded and therefore prone to race conditions (especially on interpreters without a global lock, like Rubinius or JRuby). As such, if you've touched that code, a single spec run passing isn't a guarantee that any changes you've made haven't introduced bugs. One thing I like to do before pushing changes is rerun the specs many times and watching for hangs. You can do this from the command line with something like:
207
+ A note on running specs - Que's worker system is multithreaded and therefore prone to race conditions. As such, if you've touched that code, a single spec run passing isn't a guarantee that any changes you've made haven't introduced bugs. One thing I like to do before pushing changes is rerun the specs many times and watching for hangs. You can do this from the command line with something like:
132
208
 
133
- for i in {1..1000}; do bundle exec rspec -b --seed $i; done
209
+ ```bash
210
+ for i in {1..1000}; do SEED=$i bundle exec rake; done
211
+ ```
134
212
 
135
213
  This will iterate the specs one thousand times, each with a different ordering. If the specs hang, note what the seed number was on that iteration. For example, if the previous specs finished with a "Randomized with seed 328", you know that there's a hang with seed 329, and you can narrow it down to a specific spec with:
136
214
 
137
- for i in {1..1000}; do LOG_SPEC=true bundle exec rspec -b --seed 329; done
215
+ ```bash
216
+ for i in {1..1000}; do LOG_SPEC=true SEED=328 bundle exec rake; done
217
+ ```
138
218
 
139
219
  Note that we iterate because there's no guarantee that the hang would reappear with a single additional run, so we need to rerun the specs until it reappears. The LOG_SPEC parameter will output the name and file location of each spec before it is run, so you can easily tell which spec is hanging, and you can continue narrowing things down from there.
220
+
221
+ Another helpful technique is to replace an `it` spec declaration with `hit` - this will run that particular spec 100 times during the run.
222
+
223
+ #### With Docker
224
+
225
+ We've provided a Dockerised environment to avoid the need to manually: install Ruby, install the gem bundle, set up Postgres, and connect to the database.
226
+
227
+ To run the specs using this environment, run:
228
+
229
+ ```bash
230
+ ./auto/test
231
+ ```
232
+
233
+ To get a shell in the environment, run:
234
+
235
+ ```bash
236
+ ./auto/dev
237
+ ```
238
+
239
+ The [Docker Compose config](docker-compose.yml) provides a convenient way to inject your local shell aliases into the Docker container. Simply create a file containing your alias definitions (or which sources them from other files) at `~/.docker-rc.d/.docker-bashrc`, and they will be available inside the container.
240
+
241
+ #### Without Docker
242
+
243
+ You'll need to have Postgres running. Assuming you have it running on port 5697, with a `que-test` database, and a username & password of `que`, you can run:
244
+
245
+ ```bash
246
+ DATABASE_URL=postgres://que:que@localhost:5697/que-test bundle exec rake
247
+ ```
248
+
249
+ If you don't already have Postgres, you could use Docker Compose to run just the database:
250
+
251
+ ```bash
252
+ docker compose up -d db
253
+ ```
254
+
255
+ If you want to try a different version of Postgres, e.g. 12:
256
+
257
+ ```bash
258
+ export POSTGRES_VERSION=12
259
+ ```
260
+
261
+ ### Git pre-push hook
262
+
263
+ So we can avoid breaking the build, we've created Git pre-push hooks to verify everything is ok before pushing.
264
+
265
+ To set up the pre-push hook locally, run:
266
+
267
+ ```bash
268
+ echo -e "#\!/bin/bash\n\$(dirname \$0)/../../auto/pre-push-hook" > .git/hooks/pre-push
269
+ chmod +x .git/hooks/pre-push
270
+ ```
271
+
272
+ ### Release process
273
+
274
+ The process for releasing a new version of the gem is:
275
+
276
+ - Merge PR(s)
277
+ - Git pull locally
278
+ - Update the version number, bundle install, and commit
279
+ - Update `CHANGELOG.md`, and commit
280
+ - Tag the commit with the version number, prefixed by `v`
281
+ - Git push to master
282
+ - Git push the tag
283
+ - Publish the new version of the gem to RubyGems: `gem build -o que.gem && gem push que.gem`
284
+ - Create a GitHub release - rather than describe anything there, link to the heading for the release in `CHANGELOG.md`
285
+ - Post on the Que Discord in `#announcements`
data/auto/dev ADDED
@@ -0,0 +1,21 @@
1
+ #!/bin/bash
2
+ #
3
+ # Operate in development environment
4
+
5
+ set -Eeuo pipefail
6
+
7
+ cd "$(dirname "$0")/.."
8
+
9
+ docker compose build dev
10
+
11
+ # Delete containers and DB volume afterwards on CI
12
+ if [[ "${CI-}" == "true" ]]; then
13
+ trap '{
14
+ echo "Stopping containers..."
15
+ docker compose down
16
+ docker volume rm -f que_db-data
17
+ }' EXIT
18
+ fi
19
+
20
+ set -x
21
+ docker compose run --rm dev "${@-bash}"
@@ -0,0 +1,30 @@
1
+ #!/bin/bash
2
+
3
+ set -Eeuo pipefail
4
+
5
+ cd "$(dirname "$0")/.."
6
+
7
+ green='\e[32m'; blue='\e[36m'; red='\e[31m'; bold='\e[1m'; reset='\e[0m'
8
+ coloured-arrow() { printf "$bold$1==> $2$reset\n"; }
9
+ success() { coloured-arrow "$green" "$1"; }
10
+ info() { coloured-arrow "$blue" "$1"; }
11
+ err() { coloured-arrow "$red" "$1"; exit 1; }
12
+
13
+ info 'Running pre-push hook...'
14
+
15
+ on-exit() {
16
+ [[ -n "${succeeded-}" ]] || err 'Pre-push checks failed'
17
+ }
18
+ trap on-exit EXIT
19
+
20
+ info 'Checking for uncommitted changes...'
21
+ [[ -z $(git status -s) ]] || err 'ERROR: You have uncommited changes'
22
+
23
+ info 'Checking bundle...'
24
+ bundle check --dry-run || bundle install
25
+
26
+ info 'Specs...'
27
+ auto/test
28
+
29
+ succeeded=true
30
+ success 'All pre-push checks passed! =)'
data/auto/psql ADDED
@@ -0,0 +1,9 @@
1
+ #!/bin/bash
2
+ #
3
+ # Open a database shell
4
+
5
+ set -Eeuo pipefail
6
+
7
+ cd "$(dirname "$0")/.."
8
+
9
+ docker compose run --rm pg-dev psql "${@-}"
data/auto/test ADDED
@@ -0,0 +1,5 @@
1
+ #!/bin/bash
2
+
3
+ set -Eeuo pipefail
4
+
5
+ "$(dirname "$0")"/dev ./scripts/test "$@"
@@ -0,0 +1,17 @@
1
+ #!/bin/bash
2
+
3
+ set -Eeuo pipefail
4
+
5
+ export POSTGRES_VERSION=14
6
+
7
+ delete_db() {
8
+ docker compose down
9
+ docker volume rm -f que_db-data
10
+ }
11
+
12
+ trap delete_db EXIT
13
+
14
+ # pre-test cleanup is necessary as the existing db container will be used if it's running (potentially with the wrong PG version)
15
+ delete_db
16
+ "$(dirname "$0")"/test "$@"
17
+ delete_db
data/bin/que CHANGED
@@ -1,87 +1,14 @@
1
1
  #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
2
3
 
3
- require 'optparse'
4
- require 'ostruct'
5
- require 'logger'
4
+ require 'que/command_line_interface'
6
5
 
7
6
  $stdout.sync = true
8
7
 
9
- options = OpenStruct.new
8
+ exit_code =
9
+ Que::CommandLineInterface.parse(
10
+ args: ARGV.dup,
11
+ output: $stdout,
12
+ )
10
13
 
11
- OptionParser.new do |opts|
12
- opts.banner = 'usage: que [options] file/to/require ...'
13
-
14
- opts.on('-w', '--worker-count [COUNT]', Integer, "Set number of workers in process (default: 4)") do |worker_count|
15
- options.worker_count = worker_count
16
- end
17
-
18
- opts.on('-i', '--wake-interval [INTERVAL]', Float, "Set maximum interval between polls of the job queue (in seconds) (default: 0.1)") do |wake_interval|
19
- options.wake_interval = wake_interval
20
- end
21
-
22
- opts.on('-l', '--log-level [LEVEL]', String, "Set level of Que's logger (debug, info, warn, error, fatal) (default: info)") do |log_level|
23
- options.log_level = log_level
24
- end
25
-
26
- opts.on('-q', '--queue-name [NAME]', String, "Set the name of the queue to work jobs from (default: the default queue)") do |queue_name|
27
- options.queue_name = queue_name
28
- end
29
-
30
- opts.on('-v', '--version', "Show Que version") do
31
- $stdout.puts "Que version #{Que::Version}"
32
- exit 0
33
- end
34
-
35
- opts.on('-h', '--help', "Show help text") do
36
- $stdout.puts opts
37
- exit 0
38
- end
39
- end.parse!(ARGV)
40
-
41
- if ARGV.length.zero?
42
- $stdout.puts <<-OUTPUT
43
- You didn't include any Ruby files to require!
44
- Que needs to be able to load your application before it can process jobs.
45
- (Hint: If you're using Rails, try `que ./config/environment.rb`)
46
- (Or use `que -h` for a list of options)
47
- OUTPUT
48
- exit 1
49
- end
50
-
51
- ARGV.each do |file|
52
- begin
53
- require file
54
- rescue LoadError
55
- $stdout.puts "Could not load file '#{file}'"
56
- end
57
- end
58
-
59
- Que.logger ||= Logger.new(STDOUT)
60
-
61
- begin
62
- if log_level = (options.log_level || ENV['QUE_LOG_LEVEL'])
63
- Que.logger.level = Logger.const_get(log_level.upcase)
64
- end
65
- rescue NameError
66
- $stdout.puts "Bad logging level: #{log_level}"
67
- exit 1
68
- end
69
-
70
- Que.queue_name = options.queue_name || ENV['QUE_QUEUE'] || Que.queue_name || nil
71
- Que.worker_count = (options.worker_count || ENV['QUE_WORKER_COUNT'] || Que.worker_count || 4).to_i
72
- Que.wake_interval = (options.wake_interval || ENV['QUE_WAKE_INTERVAL'] || Que.wake_interval || 0.1).to_f
73
- Que.mode = :async
74
-
75
- stop = false
76
- %w(INT TERM).each { |signal| trap(signal) { stop = true } }
77
-
78
- loop do
79
- sleep 0.01
80
- break if stop
81
- end
82
-
83
- $stdout.puts
84
- $stdout.puts "Finishing Que's current jobs before exiting..."
85
- Que.worker_count = 0
86
- Que.mode = :off
87
- $stdout.puts "Que's jobs finished, exiting..."
14
+ exit(exit_code)
@@ -0,0 +1,47 @@
1
+ version: "3.7"
2
+
3
+ services:
4
+
5
+ dev:
6
+ build:
7
+ context: .
8
+ target: dev-environment
9
+ depends_on:
10
+ - db
11
+ volumes:
12
+ - .:/work
13
+ - ruby-3.1.1-gem-cache:/usr/local/bundle
14
+ - ~/.docker-rc.d/:/.docker-rc.d/:ro
15
+ working_dir: /work
16
+ entrypoint: /work/scripts/docker-entrypoint
17
+ command: bash
18
+ environment:
19
+ DATABASE_URL: postgres://que:que@db/que-test
20
+ USE_RAILS: ~
21
+
22
+ db:
23
+ image: "postgres:${POSTGRES_VERSION-13}"
24
+ volumes:
25
+ - db-data:/var/lib/postgresql/data
26
+ environment:
27
+ POSTGRES_USER: que
28
+ POSTGRES_PASSWORD: que
29
+ POSTGRES_DB: que-test
30
+ ports:
31
+ - 5697:5432
32
+
33
+ pg-dev:
34
+ image: "postgres:${POSTGRES_VERSION-13}"
35
+ depends_on:
36
+ - db
37
+ entrypoint: []
38
+ command: psql
39
+ environment:
40
+ PGHOST: db
41
+ PGUSER: que
42
+ PGPASSWORD: que
43
+ PGDATABASE: que-test
44
+
45
+ volumes:
46
+ db-data: ~
47
+ ruby-3.1.1-gem-cache: ~