flexiq 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +302 -0
- data/lib/flexiq/backoff.rb +21 -0
- data/lib/flexiq/batch_item_result.rb +21 -0
- data/lib/flexiq/cbor/decoder.rb +169 -0
- data/lib/flexiq/cbor/encoder.rb +112 -0
- data/lib/flexiq/client.rb +196 -0
- data/lib/flexiq/deadline.rb +24 -0
- data/lib/flexiq/debounce.rb +20 -0
- data/lib/flexiq/edge_condition.rb +24 -0
- data/lib/flexiq/enqueue_options.rb +38 -0
- data/lib/flexiq/enqueue_request.rb +25 -0
- data/lib/flexiq/enqueue_result.rb +13 -0
- data/lib/flexiq/errors.rb +21 -0
- data/lib/flexiq/job.rb +60 -0
- data/lib/flexiq/job_page.rb +19 -0
- data/lib/flexiq/job_status.rb +41 -0
- data/lib/flexiq/job_transition.rb +25 -0
- data/lib/flexiq/payload.rb +55 -0
- data/lib/flexiq/queue_stats.rb +12 -0
- data/lib/flexiq/reason.rb +52 -0
- data/lib/flexiq/rpc_error.rb +117 -0
- data/lib/flexiq/task_error.rb +50 -0
- data/lib/flexiq/transition_kind.rb +27 -0
- data/lib/flexiq/transport.rb +216 -0
- data/lib/flexiq/version.rb +6 -0
- data/lib/flexiq/watch/by_ids.rb +33 -0
- data/lib/flexiq/watch/by_queue.rb +33 -0
- data/lib/flexiq/watch.rb +132 -0
- data/lib/flexiq/watch_items.rb +13 -0
- data/lib/flexiq/wire/bytes.rb +23 -0
- data/lib/flexiq/wire/duration.rb +41 -0
- data/lib/flexiq/wire/int64.rb +20 -0
- data/lib/flexiq/wire/path.rb +20 -0
- data/lib/flexiq/wire/sse.rb +91 -0
- data/lib/flexiq/wire/timestamp.rb +27 -0
- data/lib/flexiq/workflow_constructs.rb +66 -0
- data/lib/flexiq/workflow_edge.rb +21 -0
- data/lib/flexiq/workflow_graph.rb +50 -0
- data/lib/flexiq/workflow_node.rb +54 -0
- data/lib/flexiq/workflow_node_state.rb +17 -0
- data/lib/flexiq/workflow_node_status.rb +31 -0
- data/lib/flexiq/workflow_run.rb +24 -0
- data/lib/flexiq/workflow_run_view.rb +19 -0
- data/lib/flexiq/workflow_state.rb +31 -0
- data/lib/flexiq.rb +53 -0
- metadata +92 -0
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FlexiQ
|
|
4
|
+
# A producer client: submit work and workflows, read them back, list, cancel and count jobs.
|
|
5
|
+
#
|
|
6
|
+
# client = FlexiQ::Client.new("https://flexiq.internal:50051", token: ENV.fetch("FLEXIQ_TOKEN"))
|
|
7
|
+
# job = client.enqueue("send_receipt", args: [{ "order_id" => "o-1" }], queue: "emails").job
|
|
8
|
+
# client.get_job(job.id).status # => :pending
|
|
9
|
+
#
|
|
10
|
+
# The namespace is the token's, fixed when it was minted; no call can name another.
|
|
11
|
+
# Methods raise RPCError when the server refuses and TransportError when no answer arrived.
|
|
12
|
+
# Thread-safe; calls on one client are serialised over one connection, and each watch holds a
|
|
13
|
+
# connection of its own.
|
|
14
|
+
class Client
|
|
15
|
+
# See Transport#initialize for the keyword options (TLS, timeouts, user agent).
|
|
16
|
+
def initialize(url, token:, transport: nil, **)
|
|
17
|
+
@transport = transport || Transport.new(url, token: token, **)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Submits one job. Keywords other than `args:` / `kwargs:` are EnqueueOptions.
|
|
21
|
+
#
|
|
22
|
+
# Not idempotent. If this raises TransportError, or RPCError with code UNAVAILABLE,
|
|
23
|
+
# DEADLINE_EXCEEDED or CANCELLED, the job may exist; resend only with `unique_key` set.
|
|
24
|
+
def enqueue(task_name, args: [], kwargs: {}, **options)
|
|
25
|
+
request = EnqueueRequest.new(task_name: task_name, args: args, kwargs: kwargs, options: options)
|
|
26
|
+
EnqueueResult.from_json(@transport.post("/v1/jobs", request.to_wire))
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Submits several jobs in one call; returns one BatchItemResult per request, in order.
|
|
30
|
+
#
|
|
31
|
+
# Not atomic. An item's error is that item alone. When the whole call raises instead,
|
|
32
|
+
# nothing was enqueued and `RPCError#batch_index` names the item that failed.
|
|
33
|
+
def enqueue_batch(requests)
|
|
34
|
+
items = requests.map { |request| coerce_request(request).to_wire }
|
|
35
|
+
response = @transport.post("/v1/jobs:batchEnqueue", { "items" => items })
|
|
36
|
+
batch_results(response["results"], items.length)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Reads one job. The payload and result are left out unless asked for: they are the largest
|
|
40
|
+
# things a job carries. A job in another namespace reads as JOB_NOT_FOUND.
|
|
41
|
+
def get_job(job_id, include_payload: false, include_result: false)
|
|
42
|
+
fetch_job(job_id, include_payload: include_payload, include_result: include_result)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Reads one page of jobs, newest first. Every filter is optional; `status` is a
|
|
46
|
+
# FlexiQ::JobStatus symbol. A grant narrowed to queues needs a `queue:` it reaches, one
|
|
47
|
+
# narrowed to tasks a `task_name:`, or the call raises RPCError SCOPE_DENIED: a listing is
|
|
48
|
+
# refused, never filtered silently.
|
|
49
|
+
def list_jobs(status: nil, queue: nil, task_name: nil, page_size: nil, page_token: nil)
|
|
50
|
+
query = {
|
|
51
|
+
"status" => status.nil? ? nil : JobStatus.dump(status), "queue" => queue, "taskName" => task_name,
|
|
52
|
+
"pageSize" => page_size, "pageToken" => page_token
|
|
53
|
+
}
|
|
54
|
+
JobPage.from_json(@transport.get("/v1/jobs", query))
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Every job `list_jobs` reaches with these filters, one page at a time. Without a block,
|
|
58
|
+
# returns an Enumerator; a page is fetched only when iteration reaches it.
|
|
59
|
+
def each_job(status: nil, queue: nil, task_name: nil, page_size: nil, &block)
|
|
60
|
+
return enum_for(:each_job, status: status, queue: queue, task_name: task_name, page_size: page_size) unless block
|
|
61
|
+
|
|
62
|
+
filters = { status: status, queue: queue, task_name: task_name, page_size: page_size }
|
|
63
|
+
|
|
64
|
+
# Not `loop`: it would swallow a StopIteration the caller's block raised.
|
|
65
|
+
page = nil
|
|
66
|
+
until page&.last_page?
|
|
67
|
+
page = list_jobs(**filters, page_token: page&.next_page_token)
|
|
68
|
+
page.jobs.each(&block)
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Requests cancellation and returns the job as the call left it. Idempotent.
|
|
73
|
+
def cancel_job(job_id)
|
|
74
|
+
job_from(@transport.post("/v1/jobs/#{Wire::Path.segment(job_id)}:cancel"))
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Counts jobs by state in `queue`, or across the namespace when `queue` is nil.
|
|
78
|
+
def queue_stats(queue = nil)
|
|
79
|
+
path = queue.nil? ? "/v1/stats" : "/v1/queues/#{Wire::Path.segment(queue)}/stats"
|
|
80
|
+
QueueStats.from_json(@transport.get(path))
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Follows jobs by id until each is `terminal?`. Yields a `:snapshot` JobTransition (or
|
|
84
|
+
# JobNotFound) per id, then live transitions. At most 100 ids. Enumerator without a block.
|
|
85
|
+
#
|
|
86
|
+
# A dropped stream reopens on the unfinished ids only.
|
|
87
|
+
def watch_jobs(job_ids, &block)
|
|
88
|
+
return enum_for(:watch_jobs, job_ids) unless block
|
|
89
|
+
|
|
90
|
+
Watch::ByIds.new(@transport, job_ids).run(&block)
|
|
91
|
+
nil
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# Follows `queue` (nil = "default") until the block breaks. Yields a WatchCheckpoint, then
|
|
95
|
+
# live JobTransitions; no snapshot. Pass the last `cursor` back as `resume_cursor:` to replay.
|
|
96
|
+
#
|
|
97
|
+
# A dropped stream resumes from its last cursor; an expired one yields a WatchGap.
|
|
98
|
+
# Sees only transitions the reached server process handles.
|
|
99
|
+
def watch_queue(queue = nil, resume_cursor: nil, &block)
|
|
100
|
+
return enum_for(:watch_queue, queue, resume_cursor: resume_cursor) unless block
|
|
101
|
+
|
|
102
|
+
Watch::ByQueue.new(@transport, queue, resume_cursor).run(&block)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Blocks until the job is finished; returns it with its result. Watches, never polls.
|
|
106
|
+
# Raises WaitTimeoutError after `timeout` seconds (nil: no limit), RPCError JOB_NOT_FOUND
|
|
107
|
+
# for an id this token cannot see.
|
|
108
|
+
def wait(job_id, timeout:)
|
|
109
|
+
deadline = Deadline.within(timeout)
|
|
110
|
+
Watch::ByIds.new(@transport, [job_id], deadline: deadline).run { nil }
|
|
111
|
+
read_finished(job_id, deadline)
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Submits a static workflow graph (a FlexiQ::WorkflowGraph) and returns the new run's id.
|
|
115
|
+
# The server enqueues one job per node, chained by the edges; any worker with workflow
|
|
116
|
+
# tracking enabled advances the run. `params_json` is JSON text the run carries, never a call.
|
|
117
|
+
#
|
|
118
|
+
# Static graphs only: a node setting gate, cache, fan_out, fan_in or sub_workflow is refused
|
|
119
|
+
# with WORKFLOW_CONSTRUCT_UNSUPPORTED, naming one node (`RPCError#workflow_construct`).
|
|
120
|
+
# Every submission is version 1 of `name`; a different graph under a used name is refused.
|
|
121
|
+
#
|
|
122
|
+
# Not idempotent, and there is no `unique_key`: a run cannot be found by name, so a call that
|
|
123
|
+
# raised TransportError, or RPCError UNAVAILABLE / DEADLINE_EXCEEDED / CANCELLED, may have
|
|
124
|
+
# submitted a run whose id is lost. A retry submits a second run; never retry blind.
|
|
125
|
+
def submit_workflow(name, graph, params_json: nil)
|
|
126
|
+
raise ArgumentError, "a workflow name must be a non-empty String" unless name.is_a?(String) && !name.empty?
|
|
127
|
+
raise ArgumentError, "graph must be a FlexiQ::WorkflowGraph, got #{graph.class}" unless graph.is_a?(WorkflowGraph)
|
|
128
|
+
unless params_json.nil? || params_json.is_a?(String)
|
|
129
|
+
raise ArgumentError, "params_json must be JSON text (a String), got #{params_json.class}"
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
body = { "name" => name, "graph" => graph.validate!.to_wire, "paramsJson" => params_json }.compact
|
|
133
|
+
run_id = @transport.post("/v1/workflows", body)["runId"]
|
|
134
|
+
raise TransportError, "the server answered a submission without a run id" if run_id.to_s.empty?
|
|
135
|
+
|
|
136
|
+
run_id
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
# Reads a workflow run and every node it has, as a WorkflowRunView. A run in another
|
|
140
|
+
# namespace, or one whose jobs a narrowed grant does not all reach, reads as NOT_FOUND.
|
|
141
|
+
def get_workflow_run(run_id)
|
|
142
|
+
WorkflowRunView.from_json(@transport.get("/v1/workflows/#{Wire::Path.segment(run_id)}"))
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# Closes the underlying connection; the next call reopens it.
|
|
146
|
+
def close = @transport.close
|
|
147
|
+
|
|
148
|
+
private
|
|
149
|
+
|
|
150
|
+
# The job is done, so only the read can fail; retry what clears, within the deadline.
|
|
151
|
+
# The deadline also caps this read, so a stalled answer cannot outlast `wait`'s timeout.
|
|
152
|
+
def read_finished(job_id, deadline)
|
|
153
|
+
backoff = Backoff.new
|
|
154
|
+
begin
|
|
155
|
+
fetch_job(job_id, include_result: true, deadline: deadline)
|
|
156
|
+
rescue TransportError, RPCError => e
|
|
157
|
+
raise if e.is_a?(RPCError) && !e.retryable?
|
|
158
|
+
raise WaitTimeoutError, "gave up after #{deadline.seconds}s reading job #{job_id}" if deadline&.expired?
|
|
159
|
+
|
|
160
|
+
backoff.pause(deadline)
|
|
161
|
+
retry
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def fetch_job(job_id, include_payload: false, include_result: false, deadline: nil)
|
|
166
|
+
query = { "includePayload" => include_payload || nil, "includeResult" => include_result || nil }
|
|
167
|
+
job_from(@transport.get("/v1/jobs/#{Wire::Path.segment(job_id)}", query, deadline: deadline))
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
def coerce_request(request)
|
|
171
|
+
case request
|
|
172
|
+
when EnqueueRequest then request
|
|
173
|
+
when Hash then EnqueueRequest.new(**request)
|
|
174
|
+
else raise ArgumentError, "a batch item must be an EnqueueRequest or a Hash, got #{request.class}"
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
# A short or long answer cannot be paired with its requests; guessing would attribute an
|
|
179
|
+
# outcome to the wrong job, or report an item that never landed as handled.
|
|
180
|
+
def batch_results(results, expected)
|
|
181
|
+
unless results.is_a?(Array) && results.length == expected
|
|
182
|
+
got = results.is_a?(Array) ? results.length : "no"
|
|
183
|
+
raise TransportError, "the batch answered #{got} result(s) for #{expected} item(s)"
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
results.each_with_index.map { |result, index| BatchItemResult.from_json(result, index) }
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
def job_from(response)
|
|
190
|
+
json = response["job"]
|
|
191
|
+
raise TransportError, "the server answered without a job" unless json.is_a?(Hash)
|
|
192
|
+
|
|
193
|
+
Job.from_json(json)
|
|
194
|
+
end
|
|
195
|
+
end
|
|
196
|
+
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FlexiQ
|
|
4
|
+
# A monotonic-clock deadline, immune to wall-clock steps.
|
|
5
|
+
Deadline = Data.define(:at, :seconds) do
|
|
6
|
+
# nil for nil (no limit).
|
|
7
|
+
def self.within(seconds)
|
|
8
|
+
return nil if seconds.nil?
|
|
9
|
+
raise ArgumentError, "a timeout must be a positive number of seconds, got #{seconds.inspect}" unless
|
|
10
|
+
seconds.is_a?(Numeric) && seconds.positive?
|
|
11
|
+
|
|
12
|
+
new(at: now + seconds, seconds: seconds)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def self.now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
16
|
+
|
|
17
|
+
def remaining = [at - self.class.now, 0].max
|
|
18
|
+
|
|
19
|
+
def expired? = remaining.zero?
|
|
20
|
+
|
|
21
|
+
# `limit`, capped at the time left.
|
|
22
|
+
def cap(limit) = [limit, remaining].min
|
|
23
|
+
end
|
|
24
|
+
end
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FlexiQ
|
|
4
|
+
# Collapses a burst of enqueues sharing `key` into one job. Durations are seconds.
|
|
5
|
+
Debounce = Data.define(:key, :window, :max_wait, :replace_payload, :max_pending) do
|
|
6
|
+
def initialize(key:, window: nil, max_wait: nil, replace_payload: false, max_pending: nil)
|
|
7
|
+
super
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def to_wire
|
|
11
|
+
{
|
|
12
|
+
"key" => key,
|
|
13
|
+
"window" => window && Wire::Duration.dump(window),
|
|
14
|
+
"maxWait" => max_wait && Wire::Duration.dump(max_wait),
|
|
15
|
+
"replacePayload" => replace_payload || nil,
|
|
16
|
+
"maxPending" => max_pending && Wire::Int64.dump(max_pending)
|
|
17
|
+
}.compact
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FlexiQ
|
|
4
|
+
# `EdgeCondition` as symbols: how a node's inbound edges gate it on its predecessors' outcome.
|
|
5
|
+
# nil (unset) and `:always` both mean no filter.
|
|
6
|
+
module EdgeCondition
|
|
7
|
+
WIRE_NAME = {
|
|
8
|
+
on_success: "EDGE_CONDITION_ON_SUCCESS",
|
|
9
|
+
on_failure: "EDGE_CONDITION_ON_FAILURE",
|
|
10
|
+
always: "EDGE_CONDITION_ALWAYS"
|
|
11
|
+
}.freeze
|
|
12
|
+
|
|
13
|
+
module_function
|
|
14
|
+
|
|
15
|
+
# The wire name; nil stays nil so the field is omitted. A String passes as-is.
|
|
16
|
+
def dump(condition)
|
|
17
|
+
case condition
|
|
18
|
+
when nil, String then condition
|
|
19
|
+
when Symbol then WIRE_NAME.fetch(condition) { raise ArgumentError, "unknown edge condition #{condition.inspect}" }
|
|
20
|
+
else raise ArgumentError, "an edge condition must be a Symbol or a String, got #{condition.class}"
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FlexiQ
|
|
4
|
+
# Per-job settings for an enqueue. Every field is optional; an unset one takes the server's
|
|
5
|
+
# default. Times are `Time`, durations are seconds (Integer, Float or Rational).
|
|
6
|
+
#
|
|
7
|
+
# `unique_key` dedupes against the *active* job only — once it finishes the key is free again —
|
|
8
|
+
# so it is the value to reuse when resending a write whose outcome is unknown.
|
|
9
|
+
EnqueueOptions = Data.define(
|
|
10
|
+
:queue, :priority, :max_retries, :scheduled_at, :timeout, :unique_key, :metadata,
|
|
11
|
+
:notes, :depends_on, :expires_at, :result_ttl, :debounce
|
|
12
|
+
) do
|
|
13
|
+
def initialize(queue: nil, priority: nil, max_retries: nil, scheduled_at: nil, timeout: nil,
|
|
14
|
+
unique_key: nil, metadata: nil, notes: nil, depends_on: [], expires_at: nil,
|
|
15
|
+
result_ttl: nil, debounce: nil)
|
|
16
|
+
debounce = Debounce.new(**debounce) if debounce.is_a?(Hash)
|
|
17
|
+
super
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# The proto3 JSON `EnqueueOptions` object, unset fields omitted.
|
|
21
|
+
def to_wire
|
|
22
|
+
{
|
|
23
|
+
"queue" => queue,
|
|
24
|
+
"priority" => priority,
|
|
25
|
+
"maxRetries" => max_retries,
|
|
26
|
+
"scheduledAt" => scheduled_at && Wire::Timestamp.dump(scheduled_at),
|
|
27
|
+
"timeout" => timeout && Wire::Duration.dump(timeout),
|
|
28
|
+
"uniqueKey" => unique_key,
|
|
29
|
+
"metadata" => metadata,
|
|
30
|
+
"notes" => notes,
|
|
31
|
+
"dependsOn" => depends_on.empty? ? nil : depends_on.map(&:to_s),
|
|
32
|
+
"expiresAt" => expires_at && Wire::Timestamp.dump(expires_at),
|
|
33
|
+
"resultTtl" => result_ttl && Wire::Duration.dump(result_ttl),
|
|
34
|
+
"debounce" => debounce&.to_wire
|
|
35
|
+
}.compact
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FlexiQ
|
|
4
|
+
# One job to submit: a task name, its call arguments and its options.
|
|
5
|
+
#
|
|
6
|
+
# The arguments travel as the `raw` CBOR envelope, so the bytes — and any `auto:` key derived
|
|
7
|
+
# from them — match every other FlexiQ runtime's. Prefer one Hash argument (`args: [{...}]`);
|
|
8
|
+
# it maps onto every runtime's handler-binding model.
|
|
9
|
+
EnqueueRequest = Data.define(:task_name, :args, :kwargs, :options) do
|
|
10
|
+
def initialize(task_name:, args: [], kwargs: {}, options: EnqueueOptions.new)
|
|
11
|
+
raise ArgumentError, "task_name must be a non-empty String" unless task_name.is_a?(String) && !task_name.empty?
|
|
12
|
+
|
|
13
|
+
options = EnqueueOptions.new(**options) if options.is_a?(Hash)
|
|
14
|
+
super
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# The proto3 JSON `EnqueueRequest` object.
|
|
18
|
+
def to_wire
|
|
19
|
+
wire = { "taskName" => task_name, "raw" => Wire::Bytes.dump(Payload.encode_call(args, kwargs)) }
|
|
20
|
+
opts = options.to_wire
|
|
21
|
+
wire["options"] = opts unless opts.empty?
|
|
22
|
+
wire
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FlexiQ
|
|
4
|
+
# What an accepted enqueue returned. `deduplicated` is true when a live job already held the
|
|
5
|
+
# `unique_key` and that job came back instead of a new one.
|
|
6
|
+
EnqueueResult = Data.define(:job, :deduplicated) do
|
|
7
|
+
def self.from_json(json)
|
|
8
|
+
new(job: json["job"] && Job.from_json(json["job"]), deduplicated: json["deduplicated"] == true)
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
alias_method :deduplicated?, :deduplicated
|
|
12
|
+
end
|
|
13
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FlexiQ
|
|
4
|
+
# Root of every error this gem raises, so one `rescue FlexiQ::Error` covers them all.
|
|
5
|
+
class Error < StandardError; end
|
|
6
|
+
|
|
7
|
+
# The client was built with settings it cannot honour (no token, plaintext without opt-in, …).
|
|
8
|
+
class ConfigurationError < Error; end
|
|
9
|
+
|
|
10
|
+
# A value could not be written as, or read from, the CBOR payload envelope.
|
|
11
|
+
class CodecError < Error; end
|
|
12
|
+
|
|
13
|
+
# The request never produced a FlexiQ answer: DNS, connect, TLS, timeout, or a non-FlexiQ body.
|
|
14
|
+
#
|
|
15
|
+
# On a write this does not mean the write failed — the connection may have dropped after
|
|
16
|
+
# the server committed it. Retry a write only with `unique_key` set, reusing the same value.
|
|
17
|
+
class TransportError < Error; end
|
|
18
|
+
|
|
19
|
+
# Client#wait timed out before the job finished. The job itself is untouched.
|
|
20
|
+
class WaitTimeoutError < Error; end
|
|
21
|
+
end
|
data/lib/flexiq/job.rb
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FlexiQ
|
|
4
|
+
# One job as the server reports it.
|
|
5
|
+
#
|
|
6
|
+
# Timestamps are `Time` (UTC), durations are seconds as `Rational`, and `payload` / `result` are
|
|
7
|
+
# the raw envelope bytes — nil unless requested on `Client#get_job` and present. `status` is a
|
|
8
|
+
# Symbol from FlexiQ::JobStatus, or the raw wire value when this build does not know it.
|
|
9
|
+
Job = Data.define(
|
|
10
|
+
:id, :queue, :task_name, :status, :priority, :created_at, :scheduled_at,
|
|
11
|
+
:retry_count, :max_retries, :timeout, :cancel_requested, :has_deps, :namespace,
|
|
12
|
+
:started_at, :completed_at, :payload, :result, :error, :progress, :metadata, :notes,
|
|
13
|
+
:unique_key, :expires_at, :result_ttl, :debounce_key, :enqueued_by
|
|
14
|
+
) do
|
|
15
|
+
# Builds a Job from its proto3 JSON object; unknown keys are ignored as "not this build".
|
|
16
|
+
def self.from_json(json)
|
|
17
|
+
new(
|
|
18
|
+
id: json["id"], queue: json["queue"], task_name: json["taskName"],
|
|
19
|
+
status: JobStatus.load(json["status"]), priority: json["priority"] || 0,
|
|
20
|
+
created_at: Wire::Timestamp.load(json["createdAt"]),
|
|
21
|
+
scheduled_at: Wire::Timestamp.load(json["scheduledAt"]),
|
|
22
|
+
retry_count: json["retryCount"] || 0, max_retries: json["maxRetries"] || 0,
|
|
23
|
+
timeout: Wire::Duration.load(json["timeout"]),
|
|
24
|
+
cancel_requested: json["cancelRequested"] == true, has_deps: json["hasDeps"] == true,
|
|
25
|
+
namespace: json["namespace"],
|
|
26
|
+
started_at: Wire::Timestamp.load(json["startedAt"]),
|
|
27
|
+
completed_at: Wire::Timestamp.load(json["completedAt"]),
|
|
28
|
+
payload: Wire::Bytes.load(json["payload"]), result: Wire::Bytes.load(json["result"]),
|
|
29
|
+
error: json["error"], progress: json["progress"], metadata: json["metadata"], notes: json["notes"],
|
|
30
|
+
unique_key: json["uniqueKey"], expires_at: Wire::Timestamp.load(json["expiresAt"]),
|
|
31
|
+
result_ttl: Wire::Duration.load(json["resultTtl"]), debounce_key: json["debounceKey"],
|
|
32
|
+
enqueued_by: json["enqueuedBy"]
|
|
33
|
+
)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Finished for good. An unrecognised status is never terminal.
|
|
37
|
+
def terminal? = JobStatus.terminal?(status)
|
|
38
|
+
|
|
39
|
+
def known_status? = JobStatus.known?(status)
|
|
40
|
+
|
|
41
|
+
def cancel_requested? = cancel_requested
|
|
42
|
+
|
|
43
|
+
# `[args, kwargs]` from the payload. Requires `get_job(..., include_payload: true)`.
|
|
44
|
+
def decode_payload
|
|
45
|
+
raise CodecError, "job #{id} carries no payload; request it with include_payload: true" if payload.nil?
|
|
46
|
+
|
|
47
|
+
Payload.decode_call(payload)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# The value the task returned. Requires `get_job(..., include_result: true)`.
|
|
51
|
+
def decode_result
|
|
52
|
+
raise CodecError, "job #{id} carries no result; request it with include_result: true" if result.nil?
|
|
53
|
+
|
|
54
|
+
Payload.decode_result(result)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# The failure as a TaskError, or nil when the job has not failed.
|
|
58
|
+
def task_error = error && TaskError.parse(error)
|
|
59
|
+
end
|
|
60
|
+
end
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FlexiQ
|
|
4
|
+
# One page of `Client#list_jobs`, newest first. Rows never carry payload or result.
|
|
5
|
+
#
|
|
6
|
+
# `next_page_token` is opaque: pass it back as `page_token:` and read nothing out of it.
|
|
7
|
+
# It is nil on the last page.
|
|
8
|
+
JobPage = Data.define(:jobs, :next_page_token) do
|
|
9
|
+
def self.from_json(json)
|
|
10
|
+
jobs = json.fetch("jobs", [])
|
|
11
|
+
raise TransportError, "the server answered a listing whose jobs are not a list" unless jobs.is_a?(Array)
|
|
12
|
+
|
|
13
|
+
token = json["nextPageToken"]
|
|
14
|
+
new(jobs: jobs.map { |job| Job.from_json(job) }, next_page_token: token.nil? || token.empty? ? nil : token)
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def last_page? = next_page_token.nil?
|
|
18
|
+
end
|
|
19
|
+
end
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FlexiQ
|
|
4
|
+
# `JobStatus` as Ruby symbols.
|
|
5
|
+
#
|
|
6
|
+
# A value this build does not recognise is kept as the server sent it (a String name, or an
|
|
7
|
+
# Integer) and is never terminal: a newer server may grow a state, and reading it as finished
|
|
8
|
+
# would have a poller stop watching a job that is still running.
|
|
9
|
+
module JobStatus
|
|
10
|
+
BY_WIRE_NAME = {
|
|
11
|
+
"JOB_STATUS_PENDING" => :pending,
|
|
12
|
+
"JOB_STATUS_RUNNING" => :running,
|
|
13
|
+
"JOB_STATUS_COMPLETE" => :complete,
|
|
14
|
+
"JOB_STATUS_FAILED" => :failed,
|
|
15
|
+
"JOB_STATUS_DEAD" => :dead,
|
|
16
|
+
"JOB_STATUS_CANCELLED" => :cancelled
|
|
17
|
+
}.freeze
|
|
18
|
+
|
|
19
|
+
KNOWN = BY_WIRE_NAME.values.freeze
|
|
20
|
+
TERMINAL = %i[complete dead cancelled].freeze
|
|
21
|
+
WIRE_NAME = BY_WIRE_NAME.invert.freeze
|
|
22
|
+
|
|
23
|
+
module_function
|
|
24
|
+
|
|
25
|
+
def load(value) = BY_WIRE_NAME.fetch(value, value)
|
|
26
|
+
|
|
27
|
+
# The wire name for a request filter. A String passes as-is, so a state this build does not
|
|
28
|
+
# know can still be named; an unknown Symbol is a typo and is refused before the call.
|
|
29
|
+
def dump(status)
|
|
30
|
+
case status
|
|
31
|
+
when String then status
|
|
32
|
+
when Symbol then WIRE_NAME.fetch(status) { raise ArgumentError, "unknown job status #{status.inspect}" }
|
|
33
|
+
else raise ArgumentError, "a job status must be a Symbol or a String, got #{status.class}"
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def known?(status) = KNOWN.include?(status)
|
|
38
|
+
|
|
39
|
+
def terminal?(status) = TERMINAL.include?(status)
|
|
40
|
+
end
|
|
41
|
+
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FlexiQ
|
|
4
|
+
# A job's state, or a change to it. `kind`: TransitionKind; `status`: JobStatus; `attempt`:
|
|
5
|
+
# retry count. `cursor` is nil on an id watch.
|
|
6
|
+
JobTransition = Data.define(
|
|
7
|
+
:job_id, :queue, :task_name, :kind, :status, :attempt, :time, :terminal,
|
|
8
|
+
:error, :reason, :timed_out, :wake_at, :cursor
|
|
9
|
+
) do
|
|
10
|
+
def self.from_json(json, cursor: nil)
|
|
11
|
+
new(
|
|
12
|
+
job_id: json["jobId"], queue: json["queue"], task_name: json["taskName"],
|
|
13
|
+
kind: TransitionKind.load(json["kind"]), status: JobStatus.load(json["status"]),
|
|
14
|
+
attempt: Wire::Int64.load(json["attempt"]) || 0, time: Wire::Timestamp.load(json["time"]),
|
|
15
|
+
terminal: json["terminal"] == true, error: json["error"], reason: json["reason"],
|
|
16
|
+
timed_out: json["timedOut"] == true, wake_at: Wire::Timestamp.load(json["wakeAt"]), cursor: cursor
|
|
17
|
+
)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# The job is finished. Use this, never `status`: `:failed` may still retry or dead-letter.
|
|
21
|
+
def terminal? = terminal
|
|
22
|
+
|
|
23
|
+
def timed_out? = timed_out
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FlexiQ
|
|
4
|
+
# The payload envelope: one codec tag byte, then that codec's body.
|
|
5
|
+
#
|
|
6
|
+
# A call body (`Job#payload`) is a two-element CBOR array `[args, kwargs]`.
|
|
7
|
+
# A result body (`Job#result`) is a bare CBOR value with no wrapper.
|
|
8
|
+
module Payload
|
|
9
|
+
NATIVE = 0x00
|
|
10
|
+
MSGPACK = 0x01
|
|
11
|
+
CBOR_TAG = 0x02
|
|
12
|
+
|
|
13
|
+
CODEC_NAMES = { NATIVE => "language-native", MSGPACK => "MessagePack" }.freeze
|
|
14
|
+
private_constant :CODEC_NAMES
|
|
15
|
+
|
|
16
|
+
module_function
|
|
17
|
+
|
|
18
|
+
# Encodes a call as `0x02 ++ CBOR([args, kwargs])`.
|
|
19
|
+
def encode_call(args = [], kwargs = {})
|
|
20
|
+
raise ArgumentError, "args must be an Array, got #{args.class}" unless args.is_a?(Array)
|
|
21
|
+
raise ArgumentError, "kwargs must be a Hash, got #{kwargs.class}" unless kwargs.is_a?(Hash)
|
|
22
|
+
|
|
23
|
+
CBOR_TAG.chr.b << CBOR::Encoder.encode([args, kwargs])
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# Encodes a task's return value as `0x02 ++ CBOR(value)`, the form `decode_result` reads.
|
|
27
|
+
def encode_result(value) = CBOR_TAG.chr.b << CBOR::Encoder.encode(value)
|
|
28
|
+
|
|
29
|
+
# Decodes a call body into `[args, kwargs]`.
|
|
30
|
+
def decode_call(bytes)
|
|
31
|
+
body = decode(bytes)
|
|
32
|
+
unless body.is_a?(Array) && body.length == 2 && body[0].is_a?(Array) && body[1].is_a?(Hash)
|
|
33
|
+
raise CodecError, "a call body must be a two-element [args, kwargs] array"
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
body
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Decodes a result body into the value the task returned.
|
|
40
|
+
def decode_result(bytes) = decode(bytes)
|
|
41
|
+
|
|
42
|
+
# Strips the tag and decodes the CBOR body. Untagged payloads are never sniffed:
|
|
43
|
+
# a raw CBOR or MessagePack body can begin with any byte.
|
|
44
|
+
def decode(bytes)
|
|
45
|
+
raise CodecError, "an empty payload carries no codec tag" if bytes.nil? || bytes.empty?
|
|
46
|
+
|
|
47
|
+
tag = bytes.getbyte(0)
|
|
48
|
+
return CBOR::Decoder.decode(bytes.byteslice(1..)) if tag == CBOR_TAG
|
|
49
|
+
|
|
50
|
+
name = CODEC_NAMES.fetch(tag, "reserved")
|
|
51
|
+
raise CodecError, format("payload codec tag 0x%<tag>02x (%<name>s) is not readable by this client",
|
|
52
|
+
tag: tag, name: name)
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FlexiQ
|
|
4
|
+
# Job counts by state, for one queue or the whole namespace.
|
|
5
|
+
QueueStats = Data.define(:pending, :running, :completed, :failed, :dead, :cancelled) do
|
|
6
|
+
def self.from_json(json)
|
|
7
|
+
new(**members.to_h { |name| [name, Wire::Int64.load(json[name.to_s]) || 0] })
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def total = to_h.values.sum
|
|
11
|
+
end
|
|
12
|
+
end
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FlexiQ
|
|
4
|
+
# The closed list of `google.rpc.ErrorInfo.reason` values. Branch on these, never on a
|
|
5
|
+
# message: messages are for humans and may be reworded in any release.
|
|
6
|
+
#
|
|
7
|
+
# A newer server may send a reason missing here; it arrives verbatim on `RPCError#reason`.
|
|
8
|
+
module Reason
|
|
9
|
+
UNAUTHENTICATED = "UNAUTHENTICATED"
|
|
10
|
+
SCOPE_DENIED = "SCOPE_DENIED"
|
|
11
|
+
INVALID_REQUEST = "INVALID_REQUEST"
|
|
12
|
+
MALFORMED_PAYLOAD = "MALFORMED_PAYLOAD"
|
|
13
|
+
NO_SUCH_METHOD = "NO_SUCH_METHOD"
|
|
14
|
+
JOB_NOT_FOUND = "JOB_NOT_FOUND"
|
|
15
|
+
DEAD_LETTER_NOT_FOUND = "DEAD_LETTER_NOT_FOUND"
|
|
16
|
+
PERIODIC_TASK_NOT_FOUND = "PERIODIC_TASK_NOT_FOUND"
|
|
17
|
+
WORKER_NOT_FOUND = "WORKER_NOT_FOUND"
|
|
18
|
+
TOKEN_NOT_FOUND = "TOKEN_NOT_FOUND"
|
|
19
|
+
DEPENDENCY_NOT_FOUND = "DEPENDENCY_NOT_FOUND"
|
|
20
|
+
QUEUE_FULL = "QUEUE_FULL"
|
|
21
|
+
RATE_LIMITED = "RATE_LIMITED"
|
|
22
|
+
TASK_NOT_REGISTERED = "TASK_NOT_REGISTERED"
|
|
23
|
+
WORKFLOW_CONSTRUCT_UNSUPPORTED = "WORKFLOW_CONSTRUCT_UNSUPPORTED"
|
|
24
|
+
CONTRACT_TOO_OLD = "CONTRACT_TOO_OLD"
|
|
25
|
+
JOB_TIMEOUT = "JOB_TIMEOUT"
|
|
26
|
+
CLAIM_LOST = "CLAIM_LOST"
|
|
27
|
+
JOB_CANCELLED = "JOB_CANCELLED"
|
|
28
|
+
STEP_DIVERGED = "STEP_DIVERGED"
|
|
29
|
+
STEP_LIMIT_EXCEEDED = "STEP_LIMIT_EXCEEDED"
|
|
30
|
+
STEP_REFUSED = "STEP_REFUSED"
|
|
31
|
+
LOCK_HELD = "LOCK_HELD"
|
|
32
|
+
SETTING_CONFLICT = "SETTING_CONFLICT"
|
|
33
|
+
WATCH_LIMIT = "WATCH_LIMIT"
|
|
34
|
+
WATCH_OVERFLOW = "WATCH_OVERFLOW"
|
|
35
|
+
WATCH_CURSOR_EXPIRED = "WATCH_CURSOR_EXPIRED"
|
|
36
|
+
SHUTTING_DOWN = "SHUTTING_DOWN"
|
|
37
|
+
STORAGE_UNAVAILABLE = "STORAGE_UNAVAILABLE"
|
|
38
|
+
STORAGE_CONSTRAINT = "STORAGE_CONSTRAINT"
|
|
39
|
+
SERVER_MISCONFIGURED = "SERVER_MISCONFIGURED"
|
|
40
|
+
INTERNAL = "INTERNAL"
|
|
41
|
+
UNKNOWN = "UNKNOWN"
|
|
42
|
+
|
|
43
|
+
# Conditions that clear on their own. Says nothing about whether a write is safe to resend.
|
|
44
|
+
TRANSIENT = [
|
|
45
|
+
QUEUE_FULL, RATE_LIMITED, STORAGE_UNAVAILABLE, LOCK_HELD,
|
|
46
|
+
SETTING_CONFLICT, WATCH_LIMIT, WATCH_OVERFLOW, SHUTTING_DOWN
|
|
47
|
+
].freeze
|
|
48
|
+
|
|
49
|
+
# The domain every FlexiQ ErrorInfo carries; details from any other domain are ignored.
|
|
50
|
+
DOMAIN = "flexiq.byteveda.org"
|
|
51
|
+
end
|
|
52
|
+
end
|