que 0.14.3 → 1.0.0.beta

Sign up to get free protection for your applications and to get access to all the features.
Files changed (102) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +2 -0
  3. data/CHANGELOG.md +108 -14
  4. data/LICENSE.txt +1 -1
  5. data/README.md +49 -45
  6. data/bin/command_line_interface.rb +239 -0
  7. data/bin/que +8 -82
  8. data/docs/README.md +2 -0
  9. data/docs/active_job.md +6 -0
  10. data/docs/advanced_setup.md +7 -64
  11. data/docs/command_line_interface.md +45 -0
  12. data/docs/error_handling.md +65 -18
  13. data/docs/inspecting_the_queue.md +30 -80
  14. data/docs/job_helper_methods.md +27 -0
  15. data/docs/logging.md +3 -22
  16. data/docs/managing_workers.md +6 -61
  17. data/docs/middleware.md +15 -0
  18. data/docs/migrating.md +4 -7
  19. data/docs/multiple_queues.md +8 -4
  20. data/docs/shutting_down_safely.md +1 -1
  21. data/docs/using_plain_connections.md +39 -15
  22. data/docs/using_sequel.md +5 -3
  23. data/docs/writing_reliable_jobs.md +15 -24
  24. data/lib/que.rb +98 -182
  25. data/lib/que/active_job/extensions.rb +97 -0
  26. data/lib/que/active_record/connection.rb +51 -0
  27. data/lib/que/active_record/model.rb +48 -0
  28. data/lib/que/connection.rb +179 -0
  29. data/lib/que/connection_pool.rb +78 -0
  30. data/lib/que/job.rb +107 -156
  31. data/lib/que/job_cache.rb +240 -0
  32. data/lib/que/job_methods.rb +168 -0
  33. data/lib/que/listener.rb +176 -0
  34. data/lib/que/locker.rb +466 -0
  35. data/lib/que/metajob.rb +47 -0
  36. data/lib/que/migrations.rb +24 -17
  37. data/lib/que/migrations/4/down.sql +48 -0
  38. data/lib/que/migrations/4/up.sql +265 -0
  39. data/lib/que/poller.rb +267 -0
  40. data/lib/que/rails/railtie.rb +14 -0
  41. data/lib/que/result_queue.rb +35 -0
  42. data/lib/que/sequel/model.rb +51 -0
  43. data/lib/que/utils/assertions.rb +62 -0
  44. data/lib/que/utils/constantization.rb +19 -0
  45. data/lib/que/utils/error_notification.rb +68 -0
  46. data/lib/que/utils/freeze.rb +20 -0
  47. data/lib/que/utils/introspection.rb +50 -0
  48. data/lib/que/utils/json_serialization.rb +21 -0
  49. data/lib/que/utils/logging.rb +78 -0
  50. data/lib/que/utils/middleware.rb +33 -0
  51. data/lib/que/utils/queue_management.rb +18 -0
  52. data/lib/que/utils/transactions.rb +34 -0
  53. data/lib/que/version.rb +1 -1
  54. data/lib/que/worker.rb +128 -167
  55. data/que.gemspec +13 -2
  56. metadata +37 -80
  57. data/.rspec +0 -2
  58. data/.travis.yml +0 -64
  59. data/Gemfile +0 -24
  60. data/docs/customizing_que.md +0 -200
  61. data/lib/generators/que/install_generator.rb +0 -24
  62. data/lib/generators/que/templates/add_que.rb +0 -13
  63. data/lib/que/adapters/active_record.rb +0 -40
  64. data/lib/que/adapters/base.rb +0 -133
  65. data/lib/que/adapters/connection_pool.rb +0 -16
  66. data/lib/que/adapters/pg.rb +0 -21
  67. data/lib/que/adapters/pond.rb +0 -16
  68. data/lib/que/adapters/sequel.rb +0 -20
  69. data/lib/que/railtie.rb +0 -16
  70. data/lib/que/rake_tasks.rb +0 -59
  71. data/lib/que/sql.rb +0 -170
  72. data/spec/adapters/active_record_spec.rb +0 -175
  73. data/spec/adapters/connection_pool_spec.rb +0 -22
  74. data/spec/adapters/pg_spec.rb +0 -41
  75. data/spec/adapters/pond_spec.rb +0 -22
  76. data/spec/adapters/sequel_spec.rb +0 -57
  77. data/spec/gemfiles/Gemfile.current +0 -19
  78. data/spec/gemfiles/Gemfile.old +0 -19
  79. data/spec/gemfiles/Gemfile.older +0 -19
  80. data/spec/gemfiles/Gemfile.oldest +0 -19
  81. data/spec/spec_helper.rb +0 -129
  82. data/spec/support/helpers.rb +0 -25
  83. data/spec/support/jobs.rb +0 -35
  84. data/spec/support/shared_examples/adapter.rb +0 -42
  85. data/spec/support/shared_examples/multi_threaded_adapter.rb +0 -46
  86. data/spec/unit/configuration_spec.rb +0 -31
  87. data/spec/unit/connection_spec.rb +0 -14
  88. data/spec/unit/customization_spec.rb +0 -251
  89. data/spec/unit/enqueue_spec.rb +0 -245
  90. data/spec/unit/helper_spec.rb +0 -12
  91. data/spec/unit/logging_spec.rb +0 -101
  92. data/spec/unit/migrations_spec.rb +0 -84
  93. data/spec/unit/pool_spec.rb +0 -365
  94. data/spec/unit/run_spec.rb +0 -14
  95. data/spec/unit/states_spec.rb +0 -50
  96. data/spec/unit/stats_spec.rb +0 -46
  97. data/spec/unit/transaction_spec.rb +0 -36
  98. data/spec/unit/work_spec.rb +0 -596
  99. data/spec/unit/worker_spec.rb +0 -167
  100. data/tasks/benchmark.rb +0 -3
  101. data/tasks/rspec.rb +0 -14
  102. data/tasks/safe_shutdown.rb +0 -67
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA256:
3
- metadata.gz: 73f8f07b0a406399663e08cbfd7faddaf5239b5ac08d29320eb8651e76dd1671
4
- data.tar.gz: 6fc8445aed22c679a323ac22941e9f9037f668840a97fc16b7915850d494f668
2
+ SHA1:
3
+ metadata.gz: 9bbd70fe386be02949156458e095b91fc7af2001
4
+ data.tar.gz: d7029485d2eb220cad37d8b41ca8021a6089e934
5
5
  SHA512:
6
- metadata.gz: 6020097fb14b30ce4775a34124ef7e40d52d8f56381286bab98e2e670626fac61c137f685f6ba87443143d52eee4e366bed96177bcb4e196b72c84546c022e07
7
- data.tar.gz: 916cbe011fc1d742b1559997ad71dbc9822bf68c56ce629d2cf7e01a2bd218dd6b626eb09ac873da0be1e3c770fcfe75a245361e418c01787fc10d7c85b4e1f2
6
+ metadata.gz: 94f36a7474bf38f0a06839f4ce7afc904bb855e828aff90891ccdfbf87f77e017bd823f3b92ed0bbce3ade75b931d4158c70ee1df3fd4d52d48554a8f7e2c74e
7
+ data.tar.gz: e90625eb0de12115e779cd2f497db6b4348e26736959f6c4d7b8bd40b58b51c9b086c6f8acaf4569f31b120733648e6bb84cfe6a52dc1628167efead9a016250
data/.gitignore CHANGED
@@ -12,6 +12,8 @@ lib/bundler/man
12
12
  pkg
13
13
  rdoc
14
14
  spec/reports
15
+ spec/temp/*.rb
15
16
  test/tmp
16
17
  test/version_tmp
17
18
  tmp
19
+ vendor/*
@@ -1,26 +1,120 @@
1
- ### 0.14.3 (2018-03-02)
1
+ ### 1.0.0.beta (2017-10-25)
2
2
 
3
- * Recorded errors now always include the error class, so that empty error messages can still be helpful. ( joehorsnell)
3
+ * **A schema upgrade to version 4 will be required for this release.** See [the migration doc](https://github.com/chanks/que/blob/master/docs/migrating.md) for information if you're upgrading from a previous release.
4
4
 
5
- * Recorded error messages are now truncated to the first 500 characters.
5
+ * Please note that this migration requires a rewrite of the jobs table, which makes it O(n) with the size of the table. If you have a very large backlog of jobs you may want to schedule downtime for this migration.
6
6
 
7
- ### 0.14.2 (2018-01-05)
7
+ * Que's implementation has been changed from one in which worker threads hold their own PG connections and lock their own jobs to one in which a single thread (and PG connection) locks jobs through LISTEN/NOTIFY and batch polling, and passes jobs along to worker threads. This has many benefits, including:
8
8
 
9
- * Deprecate the Que.disable_prepared_statements= accessors.
9
+ * Jobs queued for immediate processing can be actively distributed to workers with LISTEN/NOTIFY, which is more efficient than having workers repeatedly poll for new jobs.
10
10
 
11
- * Add Que.use_prepared_statements= configuration accessors.
11
+ * When polling is necessary (to pick up jobs that are scheduled for the future or that need to be retried due to errors), jobs can be locked and fetched in batches, rather than one at a time.
12
12
 
13
- * Update the generated Rails migration to declare a version. (NARKOZ)
13
+ * Individual workers no longer need to monopolize their own (usually idle) connections while working jobs, so Ruby processes will require fewer Postgres connections.
14
14
 
15
- ### 0.14.1 (2017-12-14)
15
+ * PgBouncer or another external connection pool can be used for workers' connections (though not for the connection used to lock and listen for jobs).
16
16
 
17
- * Fix a bug with typecasting boolean values on Rails 5+.
17
+ * Other features introduced in this version include:
18
18
 
19
- ### 0.14.0 (2017-08-11)
19
+ * Much better support for all versions of ActiveJob.
20
20
 
21
- * Fix incompatibility with Rails 5.1.
21
+ * In particular, you may (optionally) include `Que::ActiveJob::JobExtensions` into `ApplicationJob` to get support for all of Que's job helper methods.
22
22
 
23
- * Drop support for waking an in-process worker when an ActiveRecord transaction commits.
23
+ * Custom middleware that wrap running jobs are now supported.
24
+
25
+ * Support for categorizing jobs with tags.
26
+
27
+ * Support for configuring a `maximum_retry_count` on individual job classes.
28
+
29
+ * Job configuration options are now inheritable, so job class hierarchies are more useful.
30
+
31
+ * There are now built-in models for ActiveRecord and Sequel to allow inspecting the queue easily.
32
+
33
+ * Jobs that have finished working may optionally be retained in the database indefinitely.
34
+
35
+ * To keep a job record, replace the `destroy` calls in your jobs with `finish`. `destroy` will still delete records entirely, for jobs that you don't want to keep.
36
+
37
+ * If you don't resolve a job yourself one way or another, Que will still `destroy` the job for you by default.
38
+
39
+ * Finished jobs have a timestamp set in the finished_at column.
40
+
41
+ * Jobs that have errored too many times will now be marked as expired, and won't be retried again.
42
+
43
+ * You can configure a maximum_retry_count in your job classes, to set the threshold at which a job will be marked expired. The default is 15.
44
+
45
+ * To manually mark a job as expired (and keep it in the database but not try to run it again) you can call `expire` helper in your job.
46
+
47
+ * You can now set job priority thresholds for individual workers, to ensure that there will always be space available for high-priority jobs.
48
+
49
+ * `Que.job_states` returns a list of locked jobs and the hostname/pid of the Ruby processes that have locked them.
50
+
51
+ * `Que.connection_proc=` has been added, to allow for the easy integration of custom connection pools.
52
+
53
+ * In keeping with semantic versioning, the major version is being bumped since the new implementation requires some backwards-incompatible changes. These changes include:
54
+
55
+ * Support for MRI Rubies before 2.2 has been dropped.
56
+
57
+ * Support for Postgres versions before 9.5 has been dropped (JSONB and upsert support is required).
58
+
59
+ * JRuby support has been dropped. It will be reintroduced whenever the jruby-pg gem is production-ready.
60
+
61
+ * The `que:work` rake task has been removed. Use the `que` executable instead.
62
+
63
+ * Therefore, configuring workers using QUE_* environment variables is no longer supported. Please pass the appropriate options to the `que` executable instead.
64
+
65
+ * The `mode` setter has been removed.
66
+
67
+ * To run jobs synchronously when they are enqueued (the old `:sync` behavior) you can set `Que.run_synchronously = true`.
68
+
69
+ * To start up the worker pool (the old :async behavior) you should use the `que` executable to start up a worker process. There's no longer a supported API for running workers outside of the `que` executable.
70
+
71
+ * The way Que uses prepared statements internally has changed. This shouldn't affect anyone's use of Que, except that the `disable_prepared_statements` configuration option is no longer necessary and has been removed.
72
+
73
+ * Specifically, while Que previously used prepared statements for most of its built-in queries, now only the polling query uses it, due to its complexity. Since the polling query is only run through a dedicated connection, it's no longer possible for prepared statements to conflict with external connection pools, which was the reason that `disable_prepared_statements` was supported in the first place.
74
+
75
+ * In addition to `Que.disable_prepared_statements=`, the following methods are not meaningful under the new implementation and have been removed:
76
+
77
+ * The `Que.wake_interval` getter and setter.
78
+
79
+ * The `Que.worker_count` getter and setter.
80
+
81
+ * `Que.wake!`
82
+
83
+ * `Que.wake_all!`
84
+
85
+ * Since Que needs a dedicated Postgres connection to manage job locks, running Que through a single PG connection is no longer supported.
86
+
87
+ * It's not clear that anyone ever actually did this.
88
+
89
+ * `Que.worker_states` has been removed, as the connection that locks a job is no longer the one that the job is using to run. Its functionality has been partially replaced with `Que.job_states`.
90
+
91
+ * When using Rails, for simplicity, job attributes and keys in argument hashes are now converted to symbols when retrieved from the database, rather than being converted to instances of HashWithIndifferentAccess.
92
+
93
+ * Arguments passed to jobs are now deep-frozen, to prevent unexpected behavior when the args are mutated and the job is reenqueued.
94
+
95
+ * Since JSONB is now used to store arguments, the order of argument hashes is no longer maintained.
96
+
97
+ * It wouldn't have been a good idea to rely on this anyway.
98
+
99
+ * Calling Que.log() directly is no longer supported/recommended.
100
+
101
+ * Features marked as deprecated in the final 0.x releases have been removed.
102
+
103
+ * Finally, if you've built up your own tooling and customizations around Que, you may need to be aware of some DB schema changes made in the migration to schema version #4.
104
+
105
+ * The `job_id` column has been renamed `id` and is now the primary key. This makes it easier to manage the queue using an ActiveRecord model.
106
+
107
+ * Finished jobs are now kept in the DB, unless you explicitly call `destroy`. If you want to query the DB for only jobs that haven't finished yet, add a `WHERE finished_at IS NULL` condition to your query, or use the not_finished scope on one of the provided ORM models.
108
+
109
+ * There is now an `expired_at` timestamp column, which is set when a job reaches its maximum number of retries and will not be attempted again.
110
+
111
+ * Due to popular demand, the default queue name is now "default" rather than an empty string. The migration will move pending jobs under the "" queue to the "default" queue.
112
+
113
+ * The `last_error` column has been split in two, to `last_error_message` and `last_error_backtrace`. These two columns are now limited to 500 and 10,000 characters, respectively. The migration will split old error data correctly, and truncate it if necessary.
114
+
115
+ * Names for queues and job classes are now limited to 500 characters, which is still far longer than either of these values should reasonably be.
116
+
117
+ * There is now a `data` JSONB column which is used to support various ways of organizing jobs (setting tags on them, etc).
24
118
 
25
119
  ### 0.13.1 (2017-07-05)
26
120
 
@@ -158,7 +252,7 @@
158
252
 
159
253
  * **A schema upgrade to version 3 is required for this release.** See [the migration doc](https://github.com/chanks/que/blob/master/docs/migrating.md) for information if you're upgrading from a previous release.
160
254
 
161
- * You can now run a job's logic directly (without enqueueing it) like `MyJob.run(arg1, arg2, :other_arg => arg3)`. This is useful when a job class encapsulates logic that you want to invoke without involving the entire queue.
255
+ * You can now run a job's logic directly (without enqueueing it) like `MyJob.run(arg1, arg2, other_arg: arg3)`. This is useful when a job class encapsulates logic that you want to invoke without involving the entire queue.
162
256
 
163
257
  * You can now check the current version of Que's database schema with `Que.db_version`.
164
258
 
@@ -184,7 +278,7 @@
184
278
 
185
279
  * The default priority for jobs is now 100 (it was 1 before). Like always (and like delayed_job), a lower priority means it's more important. You can migrate the schema version to 2 to set the new default value on the que_jobs table, though it's only necessary if you're doing your own INSERTs - if you use `MyJob.queue`, it's already taken care of.
186
280
 
187
- * Added a migration system to make it easier to change the schema when updating Que. You can now write, for example, `Que.migrate!(:version => 2)` in your migrations. Migrations are run transactionally.
281
+ * Added a migration system to make it easier to change the schema when updating Que. You can now write, for example, `Que.migrate!(version: 2)` in your migrations. Migrations are run transactionally.
188
282
 
189
283
  * The logging format has changed to be more easily machine-readable. You can also now customize the logging format by assigning a callable to Que.log_formatter=. See the new doc on [logging](https://github.com/chanks/que/blob/master/docs/logging.md)) for details. The default logger level is INFO - for less critical information (such as when no jobs were found to be available or when a job-lock race condition has been detected and avoided) you can set the QUE_LOG_LEVEL environment variable to DEBUG.
190
284
 
@@ -1,4 +1,4 @@
1
- Copyright (c) 2013 Chris Hanks
1
+ Copyright (c) 2013-2017 Chris Hanks
2
2
 
3
3
  MIT License
4
4
 
data/README.md CHANGED
@@ -1,11 +1,11 @@
1
1
  # Que
2
2
 
3
- **TL;DR: Que is a high-performance alternative to DelayedJob or QueueClassic that improves the reliability of your application by protecting your jobs with the same [ACID guarantees](https://en.wikipedia.org/wiki/ACID) as the rest of your data.**
3
+ **TL;DR: Que is a high-performance job queue that improves the reliability of your application by protecting your jobs with the same [ACID guarantees](https://en.wikipedia.org/wiki/ACID) as the rest of your data.**
4
4
 
5
5
  Que ("keɪ", or "kay") is a queue for Ruby and PostgreSQL that manages jobs using [advisory locks](http://www.postgresql.org/docs/current/static/explicit-locking.html#ADVISORY-LOCKS), which gives it several advantages over other RDBMS-backed queues:
6
6
 
7
7
  * **Concurrency** - Workers don't block each other when trying to lock jobs, as often occurs with "SELECT FOR UPDATE"-style locking. This allows for very high throughput with a large number of workers.
8
- * **Efficiency** - Locks are held in memory, so locking a job doesn't incur a disk write. These first two points are what limit performance with other queues - all workers trying to lock jobs have to wait behind one that's persisting its UPDATE on a locked_at column to disk (and the disks of however many other servers your database is synchronously replicating to). Under heavy load, Que's bottleneck is CPU, not I/O.
8
+ * **Efficiency** - Locks are held in memory, so locking a job doesn't incur a disk write. These first two points are what limit performance with other queues. Under heavy load, Que's bottleneck is CPU, not I/O.
9
9
  * **Safety** - If a Ruby process dies, the jobs it's working won't be lost, or left in a locked or ambiguous state - they immediately become available for any other worker to pick up.
10
10
 
11
11
  Additionally, there are the general benefits of storing jobs in Postgres, alongside the rest of your data, rather than in Redis or a dedicated queue:
@@ -17,11 +17,12 @@ Additionally, there are the general benefits of storing jobs in Postgres, alongs
17
17
 
18
18
  Que's primary goal is reliability. You should be able to leave your application running indefinitely without worrying about jobs being lost due to a lack of transactional support, or left in limbo due to a crashing process. Que does everything it can to ensure that jobs you queue are performed exactly once (though the occasional repetition of a job can be impossible to avoid - see the docs on [how to write a reliable job](https://github.com/chanks/que/blob/master/docs/writing_reliable_jobs.md)).
19
19
 
20
- Que's secondary goal is performance. It won't be able to match the speed or throughput of a dedicated queue, or maybe even a Redis-backed queue, but it should be fast enough for most use cases. In [benchmarks of RDBMS queues](https://github.com/chanks/queue-shootout) using PostgreSQL 9.3 on a AWS c3.8xlarge instance, Que approaches 10,000 jobs per second, or about twenty times the throughput of DelayedJob or QueueClassic. You are encouraged to try things out on your own production hardware, though.
20
+ Que's secondary goal is performance. The worker process is multithreaded, so that a single process can run many jobs simultaneously.
21
21
 
22
- Que also includes a worker pool, so that multiple threads can process jobs in the same process. It can even do this in the background of your web process - if you're running on Heroku, for example, you don't need to run a separate worker dyno.
23
-
24
- Que is tested on Ruby 2.0, Rubinius and JRuby (with the `jruby-pg` gem, which is [not yet functional with ActiveRecord](https://github.com/chanks/que/issues/4#issuecomment-29561356)). It requires Postgres 9.2+ for the JSON datatype.
22
+ Compatibility:
23
+ - MRI Ruby 2.2+
24
+ - PostgreSQL 9.5+
25
+ - Rails 4.1+ (optional)
25
26
 
26
27
  **Please note** - Que's job table undergoes a lot of churn when it is under high load, and like any heavily-written table, is susceptible to bloat and slowness if Postgres isn't able to clean it up. The most common cause of this is long-running transactions, so it's recommended to try to keep all transactions against the database housing Que's job table as short as possible. This is good advice to remember for any high-activity database, but bears emphasizing when using tables that undergo a lot of writes.
27
28
 
@@ -43,39 +44,54 @@ Or install it yourself as:
43
44
 
44
45
  ## Usage
45
46
 
46
- The following assumes you're using Rails 4.0 and ActiveRecord. *Que hasn't been tested with versions of Rails before 4.0, and may or may not work with them.* See the [/docs directory](https://github.com/chanks/que/blob/master/docs) for instructions on using Que [outside of Rails](https://github.com/chanks/que/blob/master/docs/advanced_setup.md), and with [Sequel](https://github.com/chanks/que/blob/master/docs/using_sequel.md) or [no ORM](https://github.com/chanks/que/blob/master/docs/using_plain_connections.md), among other things.
47
+ First, create the queue schema in a migration. For example:
47
48
 
48
- First, generate and run a migration for the job table.
49
+ ``` ruby
50
+ class CreateQueSchema < ActiveRecord::Migration[5.0]
51
+ def up
52
+ # Whenever you use Que in a migration, always specify the version you're
53
+ # migrating to. If you're unsure what the current version is, check the
54
+ # changelog.
55
+ Que.migrate!(version: 4)
56
+ end
49
57
 
50
- $ bin/rails generate que:install
51
- $ bin/rake db:migrate
58
+ def down
59
+ # Migrate to version 0 to remove Que entirely.
60
+ Que.migrate!(version: 0)
61
+ end
62
+ end
63
+ ```
52
64
 
53
65
  Create a class for each type of job you want to run:
54
66
 
55
-
56
67
  ``` ruby
57
68
  # app/jobs/charge_credit_card.rb
58
69
  class ChargeCreditCard < Que::Job
59
70
  # Default settings for this job. These are optional - without them, jobs
60
71
  # will default to priority 100 and run immediately.
61
- @priority = 10
62
- @run_at = proc { 1.minute.from_now }
72
+ self.run_at = proc { 1.minute.from_now }
63
73
 
64
- def run(user_id, options)
74
+ # We use the Linux priority scale - a lower number is more important.
75
+ self.priority = 10
76
+
77
+ def run(credit_card_id, user_id:)
65
78
  # Do stuff.
66
- user = User[user_id]
67
- card = CreditCard[options[:credit_card_id]]
79
+ user = User.find(user_id)
80
+ card = CreditCard.find(credit_card_id)
68
81
 
69
- ActiveRecord::Base.transaction do
82
+ User.transaction do
70
83
  # Write any changes you'd like to the database.
71
- user.update_attributes :charged_at => Time.now
84
+ user.update charged_at: Time.now
72
85
 
73
86
  # It's best to destroy the job in the same transaction as any other
74
- # changes you make. Que will destroy the job for you after the run
75
- # method if you don't do it yourself, but if your job writes to the
76
- # DB but doesn't destroy the job in the same transaction, it's
77
- # possible that the job could be repeated in the event of a crash.
87
+ # changes you make. Que will mark the job as destroyed for you after the
88
+ # run method if you don't do it yourself, but if your job writes to the DB
89
+ # but doesn't destroy the job in the same transaction, it's possible that
90
+ # the job could be repeated in the event of a crash.
78
91
  destroy
92
+
93
+ # If you'd rather leave the job record in the database to maintain a job
94
+ # history, simply replace the `destroy` call with a `finish` call.
79
95
  end
80
96
  end
81
97
  end
@@ -84,38 +100,24 @@ end
84
100
  Queue your job. Again, it's best to do this in a transaction with other changes you're making. Also note that any arguments you pass will be serialized to JSON and back again, so stick to simple types (strings, integers, floats, hashes, and arrays).
85
101
 
86
102
  ``` ruby
87
- ActiveRecord::Base.transaction do
103
+ CreditCard.transaction do
88
104
  # Persist credit card information
89
105
  card = CreditCard.create(params[:credit_card])
90
- ChargeCreditCard.enqueue(current_user.id, :credit_card_id => card.id)
106
+ ChargeCreditCard.enqueue(card.id, user_id: current_user.id)
91
107
  end
92
108
  ```
93
109
 
94
110
  You can also add options to run the job after a specific time, or with a specific priority:
95
111
 
96
112
  ``` ruby
97
- # The default priority is 100, and a lower number means a higher priority. 5 would be very important.
98
- ChargeCreditCard.enqueue current_user.id, :credit_card_id => card.id, :run_at => 1.day.from_now, :priority => 5
113
+ ChargeCreditCard.enqueue card.id, user_id: current_user.id, run_at: 1.day.from_now, priority: 5
99
114
  ```
100
115
 
101
- To determine what happens when a job is queued, you can set Que's mode. There are a few options for the mode:
102
-
103
- * `Que.mode = :off` - In this mode, queueing a job will simply insert it into the database - the current process will make no effort to run it. You should use this if you want to use a dedicated process to work tasks (there's an executable included that will do this, `que`). This is the default when running `bin/rails console`.
104
- * `Que.mode = :async` - In this mode, a pool of background workers is spun up, each running in their own thread. See the docs for [more information on managing workers](https://github.com/chanks/que/blob/master/docs/managing_workers.md).
105
- * `Que.mode = :sync` - In this mode, any jobs you queue will be run in the same thread, synchronously (that is, `MyJob.enqueue` runs the job and won't return until it's completed). This makes your application's behavior easier to test, so it's the default in the test environment.
106
-
107
- **If you're using ActiveRecord to dump your database's schema, [set your schema_format to :sql](http://guides.rubyonrails.org/migrations.html#types-of-schema-dumps) so that Que's table structure is managed correctly.** (You can use schema_format as :ruby if you want but keep in mind this is highly advised against, as some parts of Que will not work.)
116
+ ## Testing
108
117
 
118
+ There are a couple ways to do testing. You may want to set `Que::Job.run_synchronously = true`, which will cause JobClass.enqueue to simply execute the job's logic synchronously, as if you'd run JobClass.run(*your_args). Or, you may want to leave it disabled so you can assert on the job state once they are stored in the database.
109
119
 
110
- ## Related Projects
111
-
112
- * [que-web](https://github.com/statianzo/que-web) is a Sinatra-based UI for inspecting your job queue.
113
- * [que-testing](https://github.com/statianzo/que-testing) allows making assertions on enqueued jobs.
114
- * [que-scheduler](https://github.com/hlascelles/que-scheduler) is a cron scheduler for Que jobs that runs on Que itself.
115
- * [que-go](https://github.com/bgentry/que-go) is a port of Que for the Go programming language. It uses the same table structure, so that you can use the same job queue from Ruby and Go applications.
116
- * [wisper-que](https://github.com/joevandyk/wisper-que) adds support for Que to [wisper](https://github.com/krisleech/wisper).
117
-
118
- If you have a project that uses or relates to Que, feel free to submit a PR adding it to the list!
120
+ **If you're using ActiveRecord to dump your database's schema, please [set your schema_format to :sql](http://guides.rubyonrails.org/migrations.html#types-of-schema-dumps) so that Que's table structure is managed correctly.** This is a good idea regardless, as the `:ruby` schema format doesn't support many of PostgreSQL's advanced features.
119
121
 
120
122
 
121
123
  ## Community and Contributing
@@ -129,12 +131,14 @@ Regarding contributions, one of the project's priorities is to keep Que as simpl
129
131
 
130
132
  ### Specs
131
133
 
132
- A note on running specs - Que's worker system is multithreaded and therefore prone to race conditions (especially on interpreters without a global lock, like Rubinius or JRuby). As such, if you've touched that code, a single spec run passing isn't a guarantee that any changes you've made haven't introduced bugs. One thing I like to do before pushing changes is rerun the specs many times and watching for hangs. You can do this from the command line with something like:
134
+ A note on running specs - Que's worker system is multithreaded and therefore prone to race conditions. As such, if you've touched that code, a single spec run passing isn't a guarantee that any changes you've made haven't introduced bugs. One thing I like to do before pushing changes is rerun the specs many times and watching for hangs. You can do this from the command line with something like:
133
135
 
134
- for i in {1..1000}; do bundle exec rspec -b --seed $i; done
136
+ for i in {1..1000}; do SEED=$i bundle exec rake; done
135
137
 
136
138
  This will iterate the specs one thousand times, each with a different ordering. If the specs hang, note what the seed number was on that iteration. For example, if the previous specs finished with a "Randomized with seed 328", you know that there's a hang with seed 329, and you can narrow it down to a specific spec with:
137
139
 
138
- for i in {1..1000}; do LOG_SPEC=true bundle exec rspec -b --seed 329; done
140
+ for i in {1..1000}; do LOG_SPEC=true SEED=328 bundle exec rake; done
139
141
 
140
142
  Note that we iterate because there's no guarantee that the hang would reappear with a single additional run, so we need to rerun the specs until it reappears. The LOG_SPEC parameter will output the name and file location of each spec before it is run, so you can easily tell which spec is hanging, and you can continue narrowing things down from there.
143
+
144
+ Another helpful technique is to replace an `it` spec declaration with `hit` - this will run that particular spec 100 times during the run.
@@ -0,0 +1,239 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'optparse'
4
+ require 'logger'
5
+ require 'uri'
6
+
7
+ module Que
8
+ module CommandLineInterface
9
+ # Have a sensible default require file for Rails.
10
+ RAILS_ENVIRONMENT_FILE = './config/environment.rb'
11
+
12
+ class << self
13
+ # Need to rely on dependency injection a bit to make this method cleanly
14
+ # testable :/
15
+ def parse(
16
+ args:,
17
+ output:,
18
+ default_require_file: RAILS_ENVIRONMENT_FILE
19
+ )
20
+
21
+ options = {}
22
+ queues = []
23
+ log_level = 'info'
24
+ log_internals = false
25
+ poll_interval = 5
26
+ connection_url = nil
27
+
28
+ parser =
29
+ OptionParser.new do |opts|
30
+ opts.banner = 'usage: que [options] [file/to/require] ...'
31
+
32
+ opts.on(
33
+ '-h',
34
+ '--help',
35
+ "Show this help text.",
36
+ ) do
37
+ output.puts opts.help
38
+ return 0
39
+ end
40
+
41
+ opts.on(
42
+ '-i',
43
+ '--poll-interval [INTERVAL]',
44
+ Float,
45
+ "Set maximum interval between polls for available jobs, " \
46
+ "in seconds (default: 5)",
47
+ ) do |i|
48
+ poll_interval = i
49
+ end
50
+
51
+ opts.on(
52
+ '-l',
53
+ '--log-level [LEVEL]',
54
+ String,
55
+ "Set level at which to log to STDOUT " \
56
+ "(debug, info, warn, error, fatal) (default: info)",
57
+ ) do |l|
58
+ log_level = l
59
+ end
60
+
61
+ opts.on(
62
+ '-q',
63
+ '--queue-name [NAME]',
64
+ String,
65
+ "Set a queue name to work jobs from. " \
66
+ "Can be passed multiple times. " \
67
+ "(default: the default queue only)",
68
+ ) do |queue_name|
69
+ queues << queue_name
70
+ end
71
+
72
+ opts.on(
73
+ '-v',
74
+ '--version',
75
+ "Print Que version and exit.",
76
+ ) do
77
+ require 'que'
78
+ output.puts "Que version #{Que::VERSION}"
79
+ return 0
80
+ end
81
+
82
+ opts.on(
83
+ '-w',
84
+ '--worker-count [COUNT]',
85
+ Integer,
86
+ "Set number of workers in process (default: 6)",
87
+ ) do |w|
88
+ options[:worker_count] = w
89
+ end
90
+
91
+ opts.on(
92
+ '--connection-url [URL]',
93
+ String,
94
+ "Set a custom database url to connect to for locking purposes.",
95
+ ) do |url|
96
+ connection_url = url
97
+ end
98
+
99
+ opts.on(
100
+ '--log-internals',
101
+ "Log verbosely about Que's internal state. " \
102
+ "Only recommended for debugging issues",
103
+ ) do |l|
104
+ log_internals = true
105
+ end
106
+
107
+ opts.on(
108
+ '--maximum-buffer-size [SIZE]',
109
+ Integer,
110
+ "Set maximum number of jobs to be cached in this process " \
111
+ "awaiting a worker (default: 8)",
112
+ ) do |s|
113
+ options[:maximum_queue_size] = s
114
+ end
115
+
116
+ opts.on(
117
+ '--minimum-buffer-size [SIZE]',
118
+ Integer,
119
+ "Set minimum number of jobs to be cached in this process " \
120
+ "awaiting a worker (default: 2)",
121
+ ) do |s|
122
+ options[:minimum_queue_size] = s
123
+ end
124
+
125
+ opts.on(
126
+ '--wait-period [PERIOD]',
127
+ Float,
128
+ "Set maximum interval between checks of the in-memory job queue, " \
129
+ "in milliseconds (default: 50)",
130
+ ) do |p|
131
+ options[:wait_period] = p
132
+ end
133
+
134
+ opts.on(
135
+ '--worker-priorities [LIST]',
136
+ Array,
137
+ "List of priorities to assign to workers, " \
138
+ "unspecified workers take jobs of any priority (default: 10,30,50)",
139
+ ) do |p|
140
+ options[:worker_priorities] = p.map(&:to_i)
141
+ end
142
+ end
143
+
144
+ parser.parse!(args)
145
+
146
+ if args.length.zero?
147
+ if File.exist?(default_require_file)
148
+ args << default_require_file
149
+ else
150
+ output.puts <<-OUTPUT
151
+ You didn't include any Ruby files to require!
152
+ Que needs to be able to load your application before it can process jobs.
153
+ (Or use `que -h` for a list of options)
154
+ OUTPUT
155
+ return 1
156
+ end
157
+ end
158
+
159
+ args.each do |file|
160
+ begin
161
+ require file
162
+ rescue LoadError
163
+ output.puts "Could not load file '#{file}'"
164
+ return 1
165
+ end
166
+ end
167
+
168
+ Que.logger ||= Logger.new(STDOUT)
169
+
170
+ if log_internals
171
+ Que.internal_logger = Que.logger
172
+ end
173
+
174
+ begin
175
+ Que.logger.level = Logger.const_get(log_level.upcase)
176
+ rescue NameError
177
+ output.puts "Unsupported logging level: #{log_level} (try debug, info, warn, error, or fatal)"
178
+ return 1
179
+ end
180
+
181
+ if queues.any?
182
+ queues_hash = {}
183
+
184
+ queues.each do |queue|
185
+ name, interval = queue.split('=')
186
+ p = interval ? Float(interval) : poll_interval
187
+
188
+ Que.assert(p > 0.01) { "Poll intervals can't be less than 0.01 seconds!" }
189
+
190
+ queues_hash[name] = p
191
+ end
192
+
193
+ options[:queues] = queues_hash
194
+ end
195
+
196
+ options[:poll_interval] = poll_interval
197
+
198
+ if connection_url
199
+ uri = URI.parse(connection_url)
200
+
201
+ options[:connection] =
202
+ PG::Connection.open(
203
+ host: uri.host,
204
+ user: uri.user,
205
+ password: uri.password,
206
+ port: uri.port || 5432,
207
+ dbname: uri.path[1..-1],
208
+ )
209
+ end
210
+
211
+ locker =
212
+ begin
213
+ Que::Locker.new(options)
214
+ rescue => e
215
+ output.puts(e.message)
216
+ return 1
217
+ end
218
+
219
+ # It's a bit sloppy to use a global for this when a local variable would
220
+ # do, but we want to stop the locker from the CLI specs, so...
221
+ $stop_que_executable = false
222
+ %w[INT TERM].each { |signal| trap(signal) { $stop_que_executable = true } }
223
+
224
+ loop do
225
+ sleep 0.01
226
+ break if $stop_que_executable || locker.stopping?
227
+ end
228
+
229
+ output.puts ''
230
+ output.puts "Finishing Que's current jobs before exiting..."
231
+
232
+ locker.stop!
233
+
234
+ output.puts "Que's jobs finished, exiting..."
235
+ return 0
236
+ end
237
+ end
238
+ end
239
+ end