milvaion-worker 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +28 -0
- data/lib/milvaion/errors.rb +18 -0
- data/lib/milvaion/processor.rb +35 -14
- data/lib/milvaion/worker/version.rb +1 -1
- data/milvaion-worker.gemspec +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 266e15bbebb185e836cc5afd47172a786b60f99ecd8732c1f4a9d7c2f4d0cd96
|
|
4
|
+
data.tar.gz: 13faf93abe68c3bcd80ae783a9700195b78d52af149c924756195e7fae61febb
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4fb026579a1e823bd97453f3a9fca12b1e1677f4c0814a4cf510048ad1258f65563d1a1c2c1069b06366d29b224077b8cedda64b6eb0fa229f020ff880ad6473
|
|
7
|
+
data.tar.gz: 7c5a25ea52e8bddb64ca040eeb6c1121ab87d81a05ddba92205e20d4c7c7e9a01d4b8ca32ed58f38520f3ae27f8fb899997e464a19dd2d40dec033fe5d20d642
|
data/README.md
CHANGED
|
@@ -98,10 +98,38 @@ How the handler ends decides the outcome:
|
|
|
98
98
|
| returns a value | `Completed`. A string is stored as-is; anything else is stored as JSON. `nil` stores `Job {name} completed successfully`. |
|
|
99
99
|
| raises | Retried with exponential backoff. After `max_retries`, the occurrence is `Failed` and the job goes to the dead letter queue. |
|
|
100
100
|
| raises `Milvaion::PermanentJobError` | `Failed` without retrying, and the job goes to the dead letter queue. |
|
|
101
|
+
| raises `Milvaion::RetryAfterError` | `Failed` without retrying or dead-lettering; the status carries the requested retry time (see below). |
|
|
101
102
|
| runs past its timeout | `TimedOut` |
|
|
102
103
|
| is cancelled from the dashboard | `Cancelled` |
|
|
103
104
|
| is still running when the worker stops | The job goes back to the queue and runs again. |
|
|
104
105
|
|
|
106
|
+
### Retry at a specific time
|
|
107
|
+
|
|
108
|
+
When a retry now cannot succeed but a retry later will, for example a YouTube API quota that resets at
|
|
109
|
+
11:00 UTC, raise `Milvaion::RetryAfterError` with the time to run again. Any `Time` works; it is converted
|
|
110
|
+
to UTC.
|
|
111
|
+
|
|
112
|
+
```ruby
|
|
113
|
+
worker.job("UploadVideo") do |ctx|
|
|
114
|
+
upload(ctx.data)
|
|
115
|
+
rescue QuotaExceeded
|
|
116
|
+
now = Time.now.utc
|
|
117
|
+
reset = Time.utc(now.year, now.month, now.day, 11)
|
|
118
|
+
reset += 86_400 if reset <= now
|
|
119
|
+
raise Milvaion::RetryAfterError.new(reset, "YouTube quota exhausted until #{reset.iso8601}")
|
|
120
|
+
end
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
The worker does not retry the job and does not send it to the dead letter queue. It reports the
|
|
124
|
+
occurrence as `Failed` with `retryAfter` set and logs a warning with the requested time.
|
|
125
|
+
|
|
126
|
+
- In a workflow step, Milvaion runs the step again at that time. The workflow's `MaxRetryAfter` setting
|
|
127
|
+
caps how many times this can happen (default 10), and these retries do not count against `MaxStepRetries`.
|
|
128
|
+
- In a standalone job, the occurrence ends as `Failed` and is not retried.
|
|
129
|
+
|
|
130
|
+
Only the error the handler raises counts: a `RetryAfterError` that is only the `cause` of another error is
|
|
131
|
+
treated as an ordinary failure.
|
|
132
|
+
|
|
105
133
|
### Cancellation and timeouts in Ruby threads
|
|
106
134
|
|
|
107
135
|
Jobs run on a thread pool, so blocking code does not stall other jobs. Ruby has no safe way to stop a thread from outside: `Thread#raise` and `Thread#kill` can interrupt a job in the middle of an `ensure` block or while it holds a lock. So cancellation and timeouts are **cooperative** by default:
|
data/lib/milvaion/errors.rb
CHANGED
|
@@ -8,6 +8,24 @@ module Milvaion
|
|
|
8
8
|
# A job that raises it is not retried and goes straight to the dead letter queue.
|
|
9
9
|
class PermanentJobError < Error; end
|
|
10
10
|
|
|
11
|
+
# Asks Milvaion to run the job again at a given time instead of retrying now, for example when an
|
|
12
|
+
# API quota resets at 11:00 UTC. The worker does not retry it and does not dead-letter it: the
|
|
13
|
+
# occurrence ends as Failed with retryAfter set. Milvaion reruns workflow steps at that time; a
|
|
14
|
+
# standalone job ends there as Failed.
|
|
15
|
+
class RetryAfterError < Error
|
|
16
|
+
# The requested time, in UTC.
|
|
17
|
+
attr_reader :retry_at
|
|
18
|
+
|
|
19
|
+
# retry_at: a Time (or anything responding to to_time, such as a DateTime). It is converted to UTC.
|
|
20
|
+
def initialize(retry_at, message = nil)
|
|
21
|
+
time = retry_at.is_a?(Time) ? retry_at : retry_at&.to_time
|
|
22
|
+
raise ArgumentError, "retry_at must be a Time" unless time.is_a?(Time)
|
|
23
|
+
|
|
24
|
+
@retry_at = time.getutc
|
|
25
|
+
super(message || "Job asked to be retried at #{Milvaion.iso(@retry_at)}.")
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
11
29
|
# RabbitMQ or Redis could not be reached, or a message was not confirmed by the broker.
|
|
12
30
|
class ConnectionError < Error; end
|
|
13
31
|
|
data/lib/milvaion/processor.rb
CHANGED
|
@@ -395,6 +395,11 @@ module Milvaion
|
|
|
395
395
|
def finish_failed(delivery, consumer, registered, run, error, timing)
|
|
396
396
|
exception = Milvaion.format_error(error)
|
|
397
397
|
|
|
398
|
+
if error.is_a?(RetryAfterError)
|
|
399
|
+
finish_retry_after(delivery, run, error, exception, timing)
|
|
400
|
+
return
|
|
401
|
+
end
|
|
402
|
+
|
|
398
403
|
if error.is_a?(PermanentJobError)
|
|
399
404
|
run.log_error("Permanent job failure (will not retry): #{error.message}", error)
|
|
400
405
|
run.log("Error", "❌ Permanent failure - job will not be retried. Moving to Dead Letter Queue (DLQ).",
|
|
@@ -430,6 +435,20 @@ module Milvaion
|
|
|
430
435
|
dead_letter(delivery, run, final, registered.max_retries, registered.max_retries)
|
|
431
436
|
end
|
|
432
437
|
|
|
438
|
+
# The job asked to run again at a given time: no local retry and no dead letter. One Failed status
|
|
439
|
+
# carries retryAfter; Milvaion reruns workflow steps then, a standalone job ends here.
|
|
440
|
+
def finish_retry_after(delivery, run, error, exception, timing)
|
|
441
|
+
at = Milvaion.iso(error.retry_at)
|
|
442
|
+
|
|
443
|
+
run.log_error("Job execution failed: #{error.message}", error)
|
|
444
|
+
run.log("Warning", "⏰ Job asked to be retried at #{at}. Milvaion reruns workflow steps at that time; a standalone job ends here as Failed.",
|
|
445
|
+
{ "RetryAfter" => at }, "Retry")
|
|
446
|
+
run.flush_logs
|
|
447
|
+
run.status(Status::FAILED, start: timing[:start], end_time: timing[:end], duration_ms: timing[:duration_ms],
|
|
448
|
+
exception: "[RETRY AFTER #{at}] #{exception}", retry_after: error.retry_at)
|
|
449
|
+
settle { delivery.ack.call }
|
|
450
|
+
end
|
|
451
|
+
|
|
433
452
|
# Waits out the backoff, republishes the job with a higher retry count and acks the original.
|
|
434
453
|
# The original stays unacked meanwhile, so a crash during the wait gets it redelivered. A broker
|
|
435
454
|
# that schedules retries itself (Kafka) gets the retry right away instead.
|
|
@@ -590,20 +609,22 @@ module Milvaion
|
|
|
590
609
|
|
|
591
610
|
# Publishes a status update. Final statuses are retried, because losing one leaves the
|
|
592
611
|
# occurrence running until the zombie detector gives up on it.
|
|
593
|
-
def status(status, start: nil, end_time: nil, duration_ms: nil, result: nil, exception: nil, reliable: true)
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
612
|
+
def status(status, start: nil, end_time: nil, duration_ms: nil, result: nil, exception: nil, retry_after: nil, reliable: true)
|
|
613
|
+
message = {
|
|
614
|
+
"occurrenceId" => occurrence_id,
|
|
615
|
+
"jobId" => job["id"],
|
|
616
|
+
"workerId" => @p.options.worker_id,
|
|
617
|
+
"instanceId" => @p.options.instance_id,
|
|
618
|
+
"status" => status,
|
|
619
|
+
"startTime" => Milvaion.iso(start),
|
|
620
|
+
"endTime" => Milvaion.iso(end_time),
|
|
621
|
+
"durationMs" => duration_ms,
|
|
622
|
+
"result" => result,
|
|
623
|
+
"exception" => exception,
|
|
624
|
+
"messageTimestamp" => Milvaion.utc_now
|
|
625
|
+
}
|
|
626
|
+
message["retryAfter"] = Milvaion.iso(retry_after.getutc) if retry_after
|
|
627
|
+
body = JSON.generate(message)
|
|
607
628
|
|
|
608
629
|
delays = reliable ? Processor::STATUS_RETRY_DELAYS : []
|
|
609
630
|
attempt = 0
|
data/milvaion-worker.gemspec
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
Gem::Specification.new do |spec|
|
|
4
4
|
spec.name = "milvaion-worker"
|
|
5
5
|
# Kept equal to Milvaion::Worker::VERSION by a spec; the publish workflow checks it against the tag.
|
|
6
|
-
spec.version = "1.
|
|
6
|
+
spec.version = "1.2.0"
|
|
7
7
|
spec.authors = ["Milvasoft"]
|
|
8
8
|
spec.summary = "Ruby worker SDK for the Milvaion distributed job scheduler."
|
|
9
9
|
spec.description = "Run Milvaion jobs in Ruby: registration, heartbeats, live logs, retries with " \
|