que 1.0.0.beta → 1.0.0.beta5

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 9bbd70fe386be02949156458e095b91fc7af2001
4
- data.tar.gz: d7029485d2eb220cad37d8b41ca8021a6089e934
2
+ SHA256:
3
+ metadata.gz: 90ae5c7c65d93f4ea3833d80351c2e0d3ed3f706ba00411937e8aba2137a41d7
4
+ data.tar.gz: 58aba934ee2735522e602dd5777de666c6acbee1cb531d9918ce6f2b75f3b693
5
5
  SHA512:
6
- metadata.gz: 94f36a7474bf38f0a06839f4ce7afc904bb855e828aff90891ccdfbf87f77e017bd823f3b92ed0bbce3ade75b931d4158c70ee1df3fd4d52d48554a8f7e2c74e
7
- data.tar.gz: e90625eb0de12115e779cd2f497db6b4348e26736959f6c4d7b8bd40b58b51c9b086c6f8acaf4569f31b120733648e6bb84cfe6a52dc1628167efead9a016250
6
+ metadata.gz: 1cbfa5f3c6d13868897ac9f1fcf079aaeaa29c9eadfd231044a484878fbf8e99f4b159a4458690027b181b29c5da3abafcc1c9de2d33b3c487b1fac4a15c7e50
7
+ data.tar.gz: 1ed4b71f51cd72e3968de6ec745e2de592dc021e9d334526630b3011d3f5526388b8562da4bf513066cb68d4dee033043115027ddfc0e9bef5e816ed2c35ede5
@@ -0,0 +1,43 @@
1
+ name: tests
2
+
3
+ on: [push, pull_request]
4
+
5
+ jobs:
6
+ test:
7
+ runs-on: ubuntu-latest
8
+ strategy:
9
+ matrix:
10
+ ruby_version: [2.5.x, 2.6.x, 2.7.x]
11
+ gemfile: ["4.2", "5.2", "6.0"]
12
+ postgres_version: [9, 10, 11, 12]
13
+ exclude:
14
+ - { gemfile: "4.2", ruby_version: "2.7.x" }
15
+ services:
16
+ db:
17
+ image: postgres:${{ matrix.postgres_version }}
18
+ env:
19
+ POSTGRES_HOST_AUTH_METHOD: trust
20
+ ports: ['5432:5432']
21
+ options: >-
22
+ --health-cmd pg_isready
23
+ --health-interval 10s
24
+ --health-timeout 5s
25
+ --health-retries 5
26
+ steps:
27
+ - uses: actions/checkout@v1
28
+ - name: Set up Ruby
29
+ uses: actions/setup-ruby@v1
30
+ with:
31
+ ruby-version: ${{ matrix.ruby_version }}
32
+ - name: Test with Rake
33
+ env:
34
+ PGHOST: 127.0.0.1
35
+ PGUSER: postgres
36
+ BUNDLE_GEMFILE: spec/gemfiles/Gemfile.${{ matrix.gemfile }}
37
+ run: |
38
+ sudo apt-get -yqq install libpq-dev postgresql-client
39
+ createdb que-test
40
+ gem install bundler
41
+ bundle install --jobs 4 --retry 3
42
+ USE_RAILS=true bundle exec rake test
43
+ bundle exec rake test
@@ -0,0 +1,137 @@
1
+ ### 1.0.0.beta3 (2018-05-18)
2
+
3
+ * Added support for customizing log levels for `job_worked` events (#217).
4
+
5
+ * Began logging all `job_errored` events at the `ERROR` log level.
6
+
7
+ * Fixed the Railtie when running in test mode (#214).
8
+
9
+ * Tweaked the meanings of worker-priorities and worker-count options in the CLI, to better support use cases with low worker counts (#216).
10
+
11
+ ### 1.0.0.beta2 (2018-04-13)
12
+
13
+ * Fixed an incompatibility that caused the new locker to hang when using Rails in development mode (#213).
14
+
15
+ * Fixed a bug with setting the log level via the CLI when the configured logger was based on a callable (#210).
16
+
17
+ * Renamed Que.middleware to Que.job_middleware.
18
+
19
+ * Added Que.sql_middleware.
20
+
21
+ * Officially added support for Ruby 2.5.
22
+
23
+ * Internal cleanup and renamings.
24
+
25
+ ### 1.0.0.beta (2017-10-25)
26
+
27
+ * **A schema upgrade to version 4 will be required for this release.** See [the migration doc](https://github.com/que-rb/que/blob/master/docs/migrating.md) for information if you're upgrading from a previous release.
28
+
29
+ * Please note that this migration requires a rewrite of the jobs table, which makes it O(n) with the size of the table. If you have a very large backlog of jobs you may want to schedule downtime for this migration.
30
+
31
+ * Que's implementation has been changed from one in which worker threads hold their own PG connections and lock their own jobs to one in which a single thread (and PG connection) locks jobs through LISTEN/NOTIFY and batch polling, and passes jobs along to worker threads. This has many benefits, including:
32
+
33
+ * Jobs queued for immediate processing can be actively distributed to workers with LISTEN/NOTIFY, which is more efficient than having workers repeatedly poll for new jobs.
34
+
35
+ * When polling is necessary (to pick up jobs that are scheduled for the future or that need to be retried due to errors), jobs can be locked and fetched in batches, rather than one at a time.
36
+
37
+ * Individual workers no longer need to monopolize their own (usually idle) connections while working jobs, so Ruby processes will require fewer Postgres connections.
38
+
39
+ * PgBouncer or another external connection pool can be used for workers' connections (though not for the connection used to lock and listen for jobs).
40
+
41
+ * Other features introduced in this version include:
42
+
43
+ * Much better support for all versions of ActiveJob.
44
+
45
+ * In particular, you may (optionally) include `Que::ActiveJob::JobExtensions` into `ApplicationJob` to get support for all of Que's job helper methods.
46
+
47
+ * Custom middleware that wrap running jobs are now supported.
48
+
49
+ * Support for categorizing jobs with tags.
50
+
51
+ * Support for configuring a `maximum_retry_count` on individual job classes.
52
+
53
+ * Job configuration options are now inheritable, so job class hierarchies are more useful.
54
+
55
+ * There are now built-in models for ActiveRecord and Sequel to allow inspecting the queue easily.
56
+
57
+ * Jobs that have finished working may optionally be retained in the database indefinitely.
58
+
59
+ * To keep a job record, replace the `destroy` calls in your jobs with `finish`. `destroy` will still delete records entirely, for jobs that you don't want to keep.
60
+
61
+ * If you don't resolve a job yourself one way or another, Que will still `destroy` the job for you by default.
62
+
63
+ * Finished jobs have a timestamp set in the finished_at column.
64
+
65
+ * Jobs that have errored too many times will now be marked as expired, and won't be retried again.
66
+
67
+ * You can configure a maximum_retry_count in your job classes, to set the threshold at which a job will be marked expired. The default is 15.
68
+
69
+ * To manually mark a job as expired (and keep it in the database but not try to run it again) you can call `expire` helper in your job.
70
+
71
+ * You can now set job priority thresholds for individual workers, to ensure that there will always be space available for high-priority jobs.
72
+
73
+ * `Que.job_states` returns a list of locked jobs and the hostname/pid of the Ruby processes that have locked them.
74
+
75
+ * `Que.connection_proc=` has been added, to allow for the easy integration of custom connection pools.
76
+
77
+ * In keeping with semantic versioning, the major version is being bumped since the new implementation requires some backwards-incompatible changes. These changes include:
78
+
79
+ * Support for MRI Rubies before 2.2 has been dropped.
80
+
81
+ * Support for Postgres versions before 9.5 has been dropped (JSONB and upsert support is required).
82
+
83
+ * JRuby support has been dropped. It will be reintroduced whenever the jruby-pg gem is production-ready.
84
+
85
+ * The `que:work` rake task has been removed. Use the `que` executable instead.
86
+
87
+ * Therefore, configuring workers using QUE_* environment variables is no longer supported. Please pass the appropriate options to the `que` executable instead.
88
+
89
+ * The `mode` setter has been removed.
90
+
91
+ * To run jobs synchronously when they are enqueued (the old `:sync` behavior) you can set `Que.run_synchronously = true`.
92
+
93
+ * To start up the worker pool (the old :async behavior) you should use the `que` executable to start up a worker process. There's no longer a supported API for running workers outside of the `que` executable.
94
+
95
+ * The following methods are not meaningful under the new implementation and have been removed:
96
+
97
+ * The `Que.wake_interval` getter and setter.
98
+
99
+ * The `Que.worker_count` getter and setter.
100
+
101
+ * `Que.wake!`
102
+
103
+ * `Que.wake_all!`
104
+
105
+ * Since Que needs a dedicated Postgres connection to manage job locks, running Que through a single PG connection is no longer supported.
106
+
107
+ * It's not clear that anyone ever actually did this.
108
+
109
+ * `Que.worker_states` has been removed, as the connection that locks a job is no longer the one that the job is using to run. Its functionality has been partially replaced with `Que.job_states`.
110
+
111
+ * When using Rails, for simplicity, job attributes and keys in argument hashes are now converted to symbols when retrieved from the database, rather than being converted to instances of HashWithIndifferentAccess.
112
+
113
+ * Arguments passed to jobs are now deep-frozen, to prevent unexpected behavior when the args are mutated and the job is reenqueued.
114
+
115
+ * Since JSONB is now used to store arguments, the order of argument hashes is no longer maintained.
116
+
117
+ * It wouldn't have been a good idea to rely on this anyway.
118
+
119
+ * Calling Que.log() directly is no longer supported/recommended.
120
+
121
+ * Features marked as deprecated in the final 0.x releases have been removed.
122
+
123
+ * Finally, if you've built up your own tooling and customizations around Que, you may need to be aware of some DB schema changes made in the migration to schema version #4.
124
+
125
+ * The `job_id` column has been renamed `id` and is now the primary key. This makes it easier to manage the queue using an ActiveRecord model.
126
+
127
+ * Finished jobs are now kept in the DB, unless you explicitly call `destroy`. If you want to query the DB for only jobs that haven't finished yet, add a `WHERE finished_at IS NULL` condition to your query, or use the not_finished scope on one of the provided ORM models.
128
+
129
+ * There is now an `expired_at` timestamp column, which is set when a job reaches its maximum number of retries and will not be attempted again.
130
+
131
+ * Due to popular demand, the default queue name is now "default" rather than an empty string. The migration will move pending jobs under the "" queue to the "default" queue.
132
+
133
+ * The `last_error` column has been split in two, to `last_error_message` and `last_error_backtrace`. These two columns are now limited to 500 and 10,000 characters, respectively. The migration will split old error data correctly, and truncate it if necessary.
134
+
135
+ * Names for queues and job classes are now limited to 500 characters, which is still far longer than either of these values should reasonably be.
136
+
137
+ * There is now a `data` JSONB column which is used to support various ways of organizing jobs (setting tags on them, etc).
data/CHANGELOG.md CHANGED
@@ -1,6 +1,6 @@
1
- ### 1.0.0.beta (2017-10-25)
1
+ ### 1.0.0.beta2 (2018-04-13)
2
2
 
3
- * **A schema upgrade to version 4 will be required for this release.** See [the migration doc](https://github.com/chanks/que/blob/master/docs/migrating.md) for information if you're upgrading from a previous release.
3
+ * **A schema upgrade to version 4 will be required for this release.** See [the migration doc](https://github.com/que-rb/que/blob/master/docs/migrating.md) for information if you're upgrading from a previous release.
4
4
 
5
5
  * Please note that this migration requires a rewrite of the jobs table, which makes it O(n) with the size of the table. If you have a very large backlog of jobs you may want to schedule downtime for this migration.
6
6
 
@@ -20,7 +20,7 @@
20
20
 
21
21
  * In particular, you may (optionally) include `Que::ActiveJob::JobExtensions` into `ApplicationJob` to get support for all of Que's job helper methods.
22
22
 
23
- * Custom middleware that wrap running jobs are now supported.
23
+ * Custom middleware that wrap running jobs and executing SQL statements are now supported.
24
24
 
25
25
  * Support for categorizing jobs with tags.
26
26
 
@@ -68,11 +68,7 @@
68
68
 
69
69
  * To start up the worker pool (the old :async behavior) you should use the `que` executable to start up a worker process. There's no longer a supported API for running workers outside of the `que` executable.
70
70
 
71
- * The way Que uses prepared statements internally has changed. This shouldn't affect anyone's use of Que, except that the `disable_prepared_statements` configuration option is no longer necessary and has been removed.
72
-
73
- * Specifically, while Que previously used prepared statements for most of its built-in queries, now only the polling query uses it, due to its complexity. Since the polling query is only run through a dedicated connection, it's no longer possible for prepared statements to conflict with external connection pools, which was the reason that `disable_prepared_statements` was supported in the first place.
74
-
75
- * In addition to `Que.disable_prepared_statements=`, the following methods are not meaningful under the new implementation and have been removed:
71
+ * The following methods are not meaningful under the new implementation and have been removed:
76
72
 
77
73
  * The `Que.wake_interval` getter and setter.
78
74
 
@@ -116,6 +112,32 @@
116
112
 
117
113
  * There is now a `data` JSONB column which is used to support various ways of organizing jobs (setting tags on them, etc).
118
114
 
115
+ For a detailed list of the changes between each beta release of 1.0.0, see [the beta Changelog](CHANGELOG.1.0.beta.md).
116
+
117
+ ### 0.14.3 (2018-03-02)
118
+
119
+ * Recorded errors now always include the error class, so that empty error messages can still be helpful. ( joehorsnell)
120
+
121
+ * Recorded error messages are now truncated to the first 500 characters.
122
+
123
+ ### 0.14.2 (2018-01-05)
124
+
125
+ * Deprecate the Que.disable_prepared_statements= accessors.
126
+
127
+ * Add Que.use_prepared_statements= configuration accessors.
128
+
129
+ * Update the generated Rails migration to declare a version. (NARKOZ)
130
+
131
+ ### 0.14.1 (2017-12-14)
132
+
133
+ * Fix a bug with typecasting boolean values on Rails 5+.
134
+
135
+ ### 0.14.0 (2017-08-11)
136
+
137
+ * Fix incompatibility with Rails 5.1.
138
+
139
+ * Drop support for waking an in-process worker when an ActiveRecord transaction commits.
140
+
119
141
  ### 0.13.1 (2017-07-05)
120
142
 
121
143
  * Fix issue that caused error stacktraces to not be persisted in most cases.
@@ -250,7 +272,7 @@
250
272
 
251
273
  ### 0.6.0 (2014-02-04)
252
274
 
253
- * **A schema upgrade to version 3 is required for this release.** See [the migration doc](https://github.com/chanks/que/blob/master/docs/migrating.md) for information if you're upgrading from a previous release.
275
+ * **A schema upgrade to version 3 is required for this release.** See [the migration doc](https://github.com/que-rb/que/blob/master/docs/migrating.md) for information if you're upgrading from a previous release.
254
276
 
255
277
  * You can now run a job's logic directly (without enqueueing it) like `MyJob.run(arg1, arg2, other_arg: arg3)`. This is useful when a job class encapsulates logic that you want to invoke without involving the entire queue.
256
278
 
@@ -262,11 +284,11 @@
262
284
 
263
285
  * Log lines now include the machine's hostname, since a pid alone may not uniquely identify a process.
264
286
 
265
- * Multiple queues are now supported. See [the docs](https://github.com/chanks/que/blob/master/docs/multiple_queues.md) for details. (chanks, joevandyk)
287
+ * Multiple queues are now supported. See [the docs](https://github.com/que-rb/que/blob/master/docs/multiple_queues.md) for details. (chanks, joevandyk)
266
288
 
267
289
  * Rubinius 2.2 is now supported. (brixen)
268
290
 
269
- * Job classes may now define their own logic for determining the retry interval when a job raises an error. See [error handling](https://github.com/chanks/que/blob/master/docs/error_handling.md) for more information.
291
+ * Job classes may now define their own logic for determining the retry interval when a job raises an error. See [error handling](https://github.com/que-rb/que/blob/master/docs/error_handling.md) for more information.
270
292
 
271
293
  ### 0.5.0 (2014-01-14)
272
294
 
@@ -280,7 +302,7 @@
280
302
 
281
303
  * Added a migration system to make it easier to change the schema when updating Que. You can now write, for example, `Que.migrate!(version: 2)` in your migrations. Migrations are run transactionally.
282
304
 
283
- * The logging format has changed to be more easily machine-readable. You can also now customize the logging format by assigning a callable to Que.log_formatter=. See the new doc on [logging](https://github.com/chanks/que/blob/master/docs/logging.md)) for details. The default logger level is INFO - for less critical information (such as when no jobs were found to be available or when a job-lock race condition has been detected and avoided) you can set the QUE_LOG_LEVEL environment variable to DEBUG.
305
+ * The logging format has changed to be more easily machine-readable. You can also now customize the logging format by assigning a callable to Que.log_formatter=. See the new doc on [logging](https://github.com/que-rb/que/blob/master/docs/logging.md)) for details. The default logger level is INFO - for less critical information (such as when no jobs were found to be available or when a job-lock race condition has been detected and avoided) you can set the QUE_LOG_LEVEL environment variable to DEBUG.
284
306
 
285
307
  * MultiJson is now a soft dependency. Que will use it if it is available, but it is not required.
286
308
 
data/README.md CHANGED
@@ -1,6 +1,8 @@
1
- # Que
1
+ # Que ![tests](https://github.com/que-rb/que/workflows/tests/badge.svg)
2
2
 
3
- **TL;DR: Que is a high-performance job queue that improves the reliability of your application by protecting your jobs with the same [ACID guarantees](https://en.wikipedia.org/wiki/ACID) as the rest of your data.**
3
+ **This README and the rest of the docs on the master branch all refer to Que 1.0, which is currently in beta. If you're using version 0.x, please refer to the docs on [the 0.x branch](https://github.com/que-rb/que/tree/0.x).**
4
+
5
+ *TL;DR: Que is a high-performance job queue that improves the reliability of your application by protecting your jobs with the same [ACID guarantees](https://en.wikipedia.org/wiki/ACID) as the rest of your data.*
4
6
 
5
7
  Que ("keɪ", or "kay") is a queue for Ruby and PostgreSQL that manages jobs using [advisory locks](http://www.postgresql.org/docs/current/static/explicit-locking.html#ADVISORY-LOCKS), which gives it several advantages over other RDBMS-backed queues:
6
8
 
@@ -15,7 +17,7 @@ Additionally, there are the general benefits of storing jobs in Postgres, alongs
15
17
  * **Fewer Dependencies** - If you're already using Postgres (and you probably should be), a separate queue is another moving part that can break.
16
18
  * **Security** - Postgres' support for SSL connections keeps your data safe in transport, for added protection when you're running workers on cloud platforms that you can't completely control.
17
19
 
18
- Que's primary goal is reliability. You should be able to leave your application running indefinitely without worrying about jobs being lost due to a lack of transactional support, or left in limbo due to a crashing process. Que does everything it can to ensure that jobs you queue are performed exactly once (though the occasional repetition of a job can be impossible to avoid - see the docs on [how to write a reliable job](https://github.com/chanks/que/blob/master/docs/writing_reliable_jobs.md)).
20
+ Que's primary goal is reliability. You should be able to leave your application running indefinitely without worrying about jobs being lost due to a lack of transactional support, or left in limbo due to a crashing process. Que does everything it can to ensure that jobs you queue are performed exactly once (though the occasional repetition of a job can be impossible to avoid - see the docs on [how to write a reliable job](/docs/README.md#writing-reliable-jobs)).
19
21
 
20
22
  Que's secondary goal is performance. The worker process is multithreaded, so that a single process can run many jobs simultaneously.
21
23
 
@@ -112,23 +114,81 @@ You can also add options to run the job after a specific time, or with a specifi
112
114
  ``` ruby
113
115
  ChargeCreditCard.enqueue card.id, user_id: current_user.id, run_at: 1.day.from_now, priority: 5
114
116
  ```
117
+ ## Running the Que Worker
118
+ In order to process jobs, you must start a separate worker process outside of your main server.
119
+
120
+ ```
121
+ bundle exec que
122
+ ```
123
+
124
+ Try running `que -h` to get a list of runtime options:
125
+ ```
126
+ $ que -h
127
+ usage: que [options] [file/to/require] ...
128
+ -h, --help Show this help text.
129
+ -i, --poll-interval [INTERVAL] Set maximum interval between polls for available jobs, in seconds (default: 5)
130
+ ...
131
+ ```
132
+
133
+ You may need to pass que a file path to require so that it can load your app. Que will automatically load `config/environment.rb` if it exists, so you shouldn't need an argument if you're using Rails.
134
+
135
+ ## Additional Rails-specific Setup
136
+
137
+ If you're using ActiveRecord to dump your database's schema, please [set your schema_format to :sql](http://guides.rubyonrails.org/migrations.html#types-of-schema-dumps) so that Que's table structure is managed correctly. This is a good idea regardless, as the `:ruby` schema format doesn't support many of PostgreSQL's advanced features.
138
+
139
+ Pre-1.0, the default queue name needed to be configured in order for Que to work out of the box with Rails. In 1.0 the default queue name is now 'default', as Rails expects, but when Rails enqueues some types of jobs it may try to use another queue name that isn't worked by default. You can either:
140
+
141
+ * [Configure Rails](https://guides.rubyonrails.org/configuring.html) to send all internal job types to the 'default' queue by adding the following to `config/application.rb`:
142
+ ```ruby
143
+ config.action_mailer.deliver_later_queue_name = :default
144
+ config.action_mailbox.queues.incineration = :default
145
+ config.action_mailbox.queues.routing = :default
146
+ config.active_storage.queues.analysis = :default
147
+ config.active_storage.queues.purge = :default
148
+ ```
149
+
150
+ * [Tell que](/docs#multiple-queues) to work all of these queues (less efficient because it requires polling all of them):
151
+ ```
152
+ que -q default -q mailers -q action_mailbox_incineration -q action_mailbox_routing -q active_storage_analysis -q active_storage_purge
153
+ ```
154
+
155
+ Also, if you would like to integrate Que with Active Job, you can do it by setting the adapter in `config/application.rb` or in a specific environment by setting it in `config/environments/production.rb`, for example:
156
+ ```ruby
157
+ config.active_job.queue_adapter = :que
158
+ ```
159
+
160
+ Que will automatically use the database configuration of your rails application, so there is no need to configure anything else.
161
+
162
+ You can then write your jobs as usual following the [Active Job documentation](https://guides.rubyonrails.org/active_job_basics.html). However, be aware that you'll lose the ability to finish the job in the same transaction as other database operations. That happens because Active Job is a generic background job framework that doesn't benefit from the database integration Que provides.
163
+
164
+ If you later decide to switch a job from Active Job to Que to have transactional integrity you can easily change the corresponding job class to inherit from `Que::Job` and follow the usage guidelines in the previous section.
115
165
 
116
166
  ## Testing
117
167
 
118
168
  There are a couple ways to do testing. You may want to set `Que::Job.run_synchronously = true`, which will cause JobClass.enqueue to simply execute the job's logic synchronously, as if you'd run JobClass.run(*your_args). Or, you may want to leave it disabled so you can assert on the job state once they are stored in the database.
119
169
 
120
- **If you're using ActiveRecord to dump your database's schema, please [set your schema_format to :sql](http://guides.rubyonrails.org/migrations.html#types-of-schema-dumps) so that Que's table structure is managed correctly.** This is a good idea regardless, as the `:ruby` schema format doesn't support many of PostgreSQL's advanced features.
170
+ ## Documentation
171
+
172
+ **For full documentation, see [here](docs/README.md)**.
173
+
174
+ ## Related Projects
121
175
 
176
+ These projects are tested to be compatible with Que 1.x:
177
+
178
+ - [que-web](https://github.com/statianzo/que-web) is a Sinatra-based UI for inspecting your job queue.
179
+ - [que-scheduler](https://github.com/hlascelles/que-scheduler) lets you schedule tasks using a cron style config file.
180
+ - [que-locks](https://github.com/airhorns/que-locks) lets you lock around job execution for so only one job runs at once for a set of arguments.
181
+
182
+ If you have a project that uses or relates to Que, feel free to submit a PR adding it to the list!
122
183
 
123
184
  ## Community and Contributing
124
185
 
125
- * For bugs in the library, please feel free to [open an issue](https://github.com/chanks/que/issues/new).
126
- * For general discussion and questions/concerns that don't relate to obvious bugs, try posting on the [que-talk Google Group](https://groups.google.com/forum/#!forum/que-talk).
186
+ * For bugs in the library, please feel free to [open an issue](https://github.com/que-rb/que/issues/new).
187
+ * For general discussion and questions/concerns that don't relate to obvious bugs, join our [Discord Server](https://discord.gg/B3EW32H).
127
188
  * For contributions, pull requests submitted via Github are welcome.
128
189
 
129
190
  Regarding contributions, one of the project's priorities is to keep Que as simple, lightweight and dependency-free as possible, and pull requests that change too much or wouldn't be useful to the majority of Que's users have a good chance of being rejected. If you're thinking of submitting a pull request that adds a new feature, consider starting a discussion in [que-talk](https://groups.google.com/forum/#!forum/que-talk) first about what it would do and how it would be implemented. If it's a sufficiently large feature, or if most of Que's users wouldn't find it useful, it may be best implemented as a standalone gem, like some of the related projects above.
130
191
 
131
-
132
192
  ### Specs
133
193
 
134
194
  A note on running specs - Que's worker system is multithreaded and therefore prone to race conditions. As such, if you've touched that code, a single spec run passing isn't a guarantee that any changes you've made haven't introduced bugs. One thing I like to do before pushing changes is rerun the specs many times and watching for hangs. You can do this from the command line with something like:
@@ -18,12 +18,14 @@ module Que
18
18
  default_require_file: RAILS_ENVIRONMENT_FILE
19
19
  )
20
20
 
21
- options = {}
22
- queues = []
23
- log_level = 'info'
24
- log_internals = false
25
- poll_interval = 5
26
- connection_url = nil
21
+ options = {}
22
+ queues = []
23
+ log_level = 'info'
24
+ log_internals = false
25
+ poll_interval = 5
26
+ connection_url = nil
27
+ worker_count = nil
28
+ worker_priorities = nil
27
29
 
28
30
  parser =
29
31
  OptionParser.new do |opts|
@@ -58,6 +60,26 @@ module Que
58
60
  log_level = l
59
61
  end
60
62
 
63
+ opts.on(
64
+ '-p',
65
+ '--worker-priorities [LIST]',
66
+ Array,
67
+ "List of priorities to assign to workers (default: 10,30,50,any,any,any)",
68
+ ) do |priority_array|
69
+ worker_priorities =
70
+ priority_array.map do |p|
71
+ case p
72
+ when /\Aany\z/i
73
+ nil
74
+ when /\A\d+\z/
75
+ Integer(p)
76
+ else
77
+ output.puts "Invalid priority option: '#{p}'. Please use an integer or the word 'any'."
78
+ return 1
79
+ end
80
+ end
81
+ end
82
+
61
83
  opts.on(
62
84
  '-q',
63
85
  '--queue-name [NAME]',
@@ -69,6 +91,15 @@ module Que
69
91
  queues << queue_name
70
92
  end
71
93
 
94
+ opts.on(
95
+ '-w',
96
+ '--worker-count [COUNT]',
97
+ Integer,
98
+ "Set number of workers in process (default: 6)",
99
+ ) do |w|
100
+ worker_count = w
101
+ end
102
+
72
103
  opts.on(
73
104
  '-v',
74
105
  '--version',
@@ -79,21 +110,12 @@ module Que
79
110
  return 0
80
111
  end
81
112
 
82
- opts.on(
83
- '-w',
84
- '--worker-count [COUNT]',
85
- Integer,
86
- "Set number of workers in process (default: 6)",
87
- ) do |w|
88
- options[:worker_count] = w
89
- end
90
-
91
113
  opts.on(
92
114
  '--connection-url [URL]',
93
115
  String,
94
116
  "Set a custom database url to connect to for locking purposes.",
95
117
  ) do |url|
96
- connection_url = url
118
+ options[:connection_url] = url
97
119
  end
98
120
 
99
121
  opts.on(
@@ -107,19 +129,19 @@ module Que
107
129
  opts.on(
108
130
  '--maximum-buffer-size [SIZE]',
109
131
  Integer,
110
- "Set maximum number of jobs to be cached in this process " \
111
- "awaiting a worker (default: 8)",
132
+ "Set maximum number of jobs to be locked and held in this " \
133
+ "process awaiting a worker (default: 8)",
112
134
  ) do |s|
113
- options[:maximum_queue_size] = s
135
+ options[:maximum_buffer_size] = s
114
136
  end
115
137
 
116
138
  opts.on(
117
139
  '--minimum-buffer-size [SIZE]',
118
140
  Integer,
119
- "Set minimum number of jobs to be cached in this process " \
120
- "awaiting a worker (default: 2)",
141
+ "Set minimum number of jobs to be locked and held in this " \
142
+ "process awaiting a worker (default: 2)",
121
143
  ) do |s|
122
- options[:minimum_queue_size] = s
144
+ options[:minimum_buffer_size] = s
123
145
  end
124
146
 
125
147
  opts.on(
@@ -130,19 +152,21 @@ module Que
130
152
  ) do |p|
131
153
  options[:wait_period] = p
132
154
  end
133
-
134
- opts.on(
135
- '--worker-priorities [LIST]',
136
- Array,
137
- "List of priorities to assign to workers, " \
138
- "unspecified workers take jobs of any priority (default: 10,30,50)",
139
- ) do |p|
140
- options[:worker_priorities] = p.map(&:to_i)
141
- end
142
155
  end
143
156
 
144
157
  parser.parse!(args)
145
158
 
159
+ options[:worker_priorities] =
160
+ if worker_count && worker_priorities
161
+ worker_priorities.values_at(0...worker_count)
162
+ elsif worker_priorities
163
+ worker_priorities
164
+ elsif worker_count
165
+ Array.new(worker_count) { nil }
166
+ else
167
+ [10, 30, 50, nil, nil, nil]
168
+ end
169
+
146
170
  if args.length.zero?
147
171
  if File.exist?(default_require_file)
148
172
  args << default_require_file
@@ -159,8 +183,8 @@ OUTPUT
159
183
  args.each do |file|
160
184
  begin
161
185
  require file
162
- rescue LoadError
163
- output.puts "Could not load file '#{file}'"
186
+ rescue LoadError => e
187
+ output.puts "Could not load file '#{file}': #{e}"
164
188
  return 1
165
189
  end
166
190
  end
@@ -172,7 +196,7 @@ OUTPUT
172
196
  end
173
197
 
174
198
  begin
175
- Que.logger.level = Logger.const_get(log_level.upcase)
199
+ Que.get_logger.level = Logger.const_get(log_level.upcase)
176
200
  rescue NameError
177
201
  output.puts "Unsupported logging level: #{log_level} (try debug, info, warn, error, or fatal)"
178
202
  return 1
@@ -195,19 +219,6 @@ OUTPUT
195
219
 
196
220
  options[:poll_interval] = poll_interval
197
221
 
198
- if connection_url
199
- uri = URI.parse(connection_url)
200
-
201
- options[:connection] =
202
- PG::Connection.open(
203
- host: uri.host,
204
- user: uri.user,
205
- password: uri.password,
206
- port: uri.port || 5432,
207
- dbname: uri.path[1..-1],
208
- )
209
- end
210
-
211
222
  locker =
212
223
  begin
213
224
  Que::Locker.new(options)
@@ -221,13 +232,14 @@ OUTPUT
221
232
  $stop_que_executable = false
222
233
  %w[INT TERM].each { |signal| trap(signal) { $stop_que_executable = true } }
223
234
 
235
+ output.puts "Que waiting for jobs..."
236
+
224
237
  loop do
225
238
  sleep 0.01
226
239
  break if $stop_que_executable || locker.stopping?
227
240
  end
228
241
 
229
- output.puts ''
230
- output.puts "Finishing Que's current jobs before exiting..."
242
+ output.puts "\nFinishing Que's current jobs before exiting..."
231
243
 
232
244
  locker.stop!
233
245