tina4ruby 3.11.13 → 3.11.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +80 -80
- data/LICENSE.txt +21 -21
- data/README.md +137 -137
- data/exe/tina4ruby +5 -5
- data/lib/tina4/ai.rb +696 -696
- data/lib/tina4/api.rb +189 -189
- data/lib/tina4/auth.rb +305 -305
- data/lib/tina4/auto_crud.rb +244 -244
- data/lib/tina4/cache.rb +154 -154
- data/lib/tina4/cli.rb +1449 -1449
- data/lib/tina4/constants.rb +46 -46
- data/lib/tina4/container.rb +74 -74
- data/lib/tina4/cors.rb +74 -74
- data/lib/tina4/crud.rb +692 -692
- data/lib/tina4/database/sqlite3_adapter.rb +165 -165
- data/lib/tina4/database.rb +625 -625
- data/lib/tina4/database_result.rb +208 -208
- data/lib/tina4/debug.rb +8 -8
- data/lib/tina4/dev.rb +14 -14
- data/lib/tina4/dev_admin.rb +935 -935
- data/lib/tina4/dev_mailbox.rb +191 -191
- data/lib/tina4/drivers/firebird_driver.rb +124 -110
- data/lib/tina4/drivers/mongodb_driver.rb +561 -561
- data/lib/tina4/drivers/mssql_driver.rb +112 -112
- data/lib/tina4/drivers/mysql_driver.rb +90 -90
- data/lib/tina4/drivers/odbc_driver.rb +191 -191
- data/lib/tina4/drivers/postgres_driver.rb +116 -106
- data/lib/tina4/drivers/sqlite_driver.rb +122 -122
- data/lib/tina4/env.rb +95 -95
- data/lib/tina4/error_overlay.rb +252 -252
- data/lib/tina4/events.rb +109 -109
- data/lib/tina4/field_types.rb +154 -154
- data/lib/tina4/frond.rb +2025 -2025
- data/lib/tina4/gallery/auth/meta.json +1 -1
- data/lib/tina4/gallery/auth/src/routes/api/gallery_auth.rb +114 -114
- data/lib/tina4/gallery/database/meta.json +1 -1
- data/lib/tina4/gallery/database/src/routes/api/gallery_db.rb +43 -43
- data/lib/tina4/gallery/error-overlay/meta.json +1 -1
- data/lib/tina4/gallery/error-overlay/src/routes/api/gallery_crash.rb +17 -17
- data/lib/tina4/gallery/orm/meta.json +1 -1
- data/lib/tina4/gallery/orm/src/routes/api/gallery_products.rb +16 -16
- data/lib/tina4/gallery/queue/meta.json +1 -1
- data/lib/tina4/gallery/queue/src/routes/api/gallery_queue.rb +325 -325
- data/lib/tina4/gallery/rest-api/meta.json +1 -1
- data/lib/tina4/gallery/rest-api/src/routes/api/gallery_hello.rb +14 -14
- data/lib/tina4/gallery/templates/meta.json +1 -1
- data/lib/tina4/gallery/templates/src/routes/gallery_page.rb +12 -12
- data/lib/tina4/gallery/templates/src/templates/gallery_page.twig +257 -257
- data/lib/tina4/graphql.rb +966 -966
- data/lib/tina4/health.rb +39 -39
- data/lib/tina4/html_element.rb +170 -170
- data/lib/tina4/job.rb +80 -80
- data/lib/tina4/localization.rb +168 -168
- data/lib/tina4/log.rb +203 -203
- data/lib/tina4/mcp.rb +696 -696
- data/lib/tina4/messenger.rb +587 -587
- data/lib/tina4/metrics.rb +793 -793
- data/lib/tina4/middleware.rb +445 -445
- data/lib/tina4/migration.rb +451 -451
- data/lib/tina4/orm.rb +790 -790
- data/lib/tina4/public/css/tina4.css +2463 -2463
- data/lib/tina4/public/css/tina4.min.css +1 -1
- data/lib/tina4/public/images/logo.svg +5 -5
- data/lib/tina4/public/js/frond.min.js +2 -2
- data/lib/tina4/public/js/tina4-dev-admin.js +565 -565
- data/lib/tina4/public/js/tina4-dev-admin.min.js +480 -480
- data/lib/tina4/public/js/tina4.min.js +92 -92
- data/lib/tina4/public/js/tina4js.min.js +48 -48
- data/lib/tina4/public/swagger/index.html +90 -90
- data/lib/tina4/public/swagger/oauth2-redirect.html +63 -63
- data/lib/tina4/query_builder.rb +380 -380
- data/lib/tina4/queue.rb +366 -366
- data/lib/tina4/queue_backends/kafka_backend.rb +80 -80
- data/lib/tina4/queue_backends/lite_backend.rb +298 -298
- data/lib/tina4/queue_backends/mongo_backend.rb +126 -126
- data/lib/tina4/queue_backends/rabbitmq_backend.rb +73 -73
- data/lib/tina4/rack_app.rb +817 -817
- data/lib/tina4/rate_limiter.rb +130 -130
- data/lib/tina4/request.rb +268 -255
- data/lib/tina4/response.rb +346 -346
- data/lib/tina4/response_cache.rb +551 -551
- data/lib/tina4/router.rb +406 -406
- data/lib/tina4/scss/tina4css/_alerts.scss +34 -34
- data/lib/tina4/scss/tina4css/_badges.scss +22 -22
- data/lib/tina4/scss/tina4css/_buttons.scss +69 -69
- data/lib/tina4/scss/tina4css/_cards.scss +49 -49
- data/lib/tina4/scss/tina4css/_forms.scss +156 -156
- data/lib/tina4/scss/tina4css/_grid.scss +81 -81
- data/lib/tina4/scss/tina4css/_modals.scss +84 -84
- data/lib/tina4/scss/tina4css/_nav.scss +149 -149
- data/lib/tina4/scss/tina4css/_reset.scss +94 -94
- data/lib/tina4/scss/tina4css/_tables.scss +54 -54
- data/lib/tina4/scss/tina4css/_typography.scss +55 -55
- data/lib/tina4/scss/tina4css/_utilities.scss +197 -197
- data/lib/tina4/scss/tina4css/_variables.scss +117 -117
- data/lib/tina4/scss/tina4css/base.scss +1 -1
- data/lib/tina4/scss/tina4css/colors.scss +48 -48
- data/lib/tina4/scss/tina4css/tina4.scss +17 -17
- data/lib/tina4/scss_compiler.rb +178 -178
- data/lib/tina4/seeder.rb +567 -567
- data/lib/tina4/service_runner.rb +303 -303
- data/lib/tina4/session.rb +297 -297
- data/lib/tina4/session_handlers/database_handler.rb +72 -72
- data/lib/tina4/session_handlers/file_handler.rb +67 -67
- data/lib/tina4/session_handlers/mongo_handler.rb +49 -49
- data/lib/tina4/session_handlers/redis_handler.rb +43 -43
- data/lib/tina4/session_handlers/valkey_handler.rb +43 -43
- data/lib/tina4/shutdown.rb +84 -84
- data/lib/tina4/sql_translation.rb +158 -158
- data/lib/tina4/swagger.rb +124 -124
- data/lib/tina4/template.rb +894 -894
- data/lib/tina4/templates/base.twig +26 -26
- data/lib/tina4/templates/errors/302.twig +14 -14
- data/lib/tina4/templates/errors/401.twig +9 -9
- data/lib/tina4/templates/errors/403.twig +29 -29
- data/lib/tina4/templates/errors/404.twig +29 -29
- data/lib/tina4/templates/errors/500.twig +38 -38
- data/lib/tina4/templates/errors/502.twig +9 -9
- data/lib/tina4/templates/errors/503.twig +12 -12
- data/lib/tina4/templates/errors/base.twig +37 -37
- data/lib/tina4/test_client.rb +159 -159
- data/lib/tina4/testing.rb +340 -340
- data/lib/tina4/validator.rb +174 -174
- data/lib/tina4/version.rb +1 -1
- data/lib/tina4/webserver.rb +312 -312
- data/lib/tina4/websocket.rb +343 -343
- data/lib/tina4/websocket_backplane.rb +190 -190
- data/lib/tina4/wsdl.rb +564 -564
- data/lib/tina4.rb +458 -458
- data/lib/tina4ruby.rb +4 -4
- metadata +3 -3
data/lib/tina4/queue.rb
CHANGED
|
@@ -1,366 +1,366 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
require "json"
|
|
3
|
-
require "securerandom"
|
|
4
|
-
require_relative "job"
|
|
5
|
-
|
|
6
|
-
module Tina4
|
|
7
|
-
# Queue — unified wrapper for queue management operations.
|
|
8
|
-
# Auto-detects backend from TINA4_QUEUE_BACKEND env var.
|
|
9
|
-
#
|
|
10
|
-
# Usage:
|
|
11
|
-
# # Auto-detect from env (default: lite/file backend)
|
|
12
|
-
# queue = Queue.new(topic: "tasks")
|
|
13
|
-
#
|
|
14
|
-
# # Explicit backend
|
|
15
|
-
# queue = Queue.new(topic: "tasks", backend: :rabbitmq)
|
|
16
|
-
#
|
|
17
|
-
# # Or pass a backend instance directly (legacy)
|
|
18
|
-
# queue = Queue.new(topic: "tasks", backend: my_backend)
|
|
19
|
-
class Queue
|
|
20
|
-
attr_reader :topic, :max_retries
|
|
21
|
-
|
|
22
|
-
def initialize(topic:, backend: nil, max_retries: 3)
|
|
23
|
-
@topic = topic
|
|
24
|
-
@max_retries = max_retries
|
|
25
|
-
@backend = resolve_backend_arg(backend)
|
|
26
|
-
end
|
|
27
|
-
|
|
28
|
-
# Push a job onto the queue. Returns the Job.
|
|
29
|
-
# priority: higher-priority messages are dequeued first (default 0).
|
|
30
|
-
# delay_seconds: delay before the message becomes available (default 0).
|
|
31
|
-
def push(payload, priority: 0, delay_seconds: 0)
|
|
32
|
-
available_at = delay_seconds > 0 ? Time.now + delay_seconds : nil
|
|
33
|
-
message = Job.new(topic: @topic, payload: payload, priority: priority, available_at: available_at, queue: self)
|
|
34
|
-
@backend.enqueue(message)
|
|
35
|
-
message
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
# Pop the next available job. Returns Job or nil.
|
|
39
|
-
def pop # -> Job|None
|
|
40
|
-
@backend.dequeue(@topic)
|
|
41
|
-
end
|
|
42
|
-
|
|
43
|
-
# Pop up to count jobs at once. Returns a partial batch if fewer available.
|
|
44
|
-
def pop_batch(count)
|
|
45
|
-
if @backend.respond_to?(:dequeue_batch)
|
|
46
|
-
@backend.dequeue_batch(@topic, count)
|
|
47
|
-
else
|
|
48
|
-
jobs = []
|
|
49
|
-
count.times do
|
|
50
|
-
job = @backend.dequeue(@topic)
|
|
51
|
-
break if job.nil?
|
|
52
|
-
jobs << job
|
|
53
|
-
end
|
|
54
|
-
jobs
|
|
55
|
-
end
|
|
56
|
-
end
|
|
57
|
-
|
|
58
|
-
# Clear all pending jobs from this queue's topic. Returns count removed.
|
|
59
|
-
def clear # -> int
|
|
60
|
-
return 0 unless @backend.respond_to?(:clear)
|
|
61
|
-
@backend.clear(@topic)
|
|
62
|
-
end
|
|
63
|
-
|
|
64
|
-
# Get jobs that failed but are still eligible for retry (under max_retries).
|
|
65
|
-
def failed # -> list[dict]
|
|
66
|
-
return [] unless @backend.respond_to?(:failed)
|
|
67
|
-
@backend.failed(@topic, max_retries: @max_retries)
|
|
68
|
-
end
|
|
69
|
-
|
|
70
|
-
# Retry a specific failed job by ID, or all dead-letter jobs if no id given.
|
|
71
|
-
# Returns true if re-queued.
|
|
72
|
-
def retry(job_id = nil, delay_seconds: 0) # -> bool
|
|
73
|
-
return false unless @backend.respond_to?(:retry_job)
|
|
74
|
-
@backend.retry_job(@topic, job_id: job_id, delay_seconds: delay_seconds)
|
|
75
|
-
end
|
|
76
|
-
|
|
77
|
-
# Get dead letter jobs — messages that exceeded max retries.
|
|
78
|
-
# Pass max_retries to override the queue's default.
|
|
79
|
-
def dead_letters(max_retries: nil) # -> list[dict]
|
|
80
|
-
return [] unless @backend.respond_to?(:dead_letters)
|
|
81
|
-
@backend.dead_letters(@topic, max_retries: max_retries || @max_retries)
|
|
82
|
-
end
|
|
83
|
-
|
|
84
|
-
# Delete messages by status (completed, failed, dead).
|
|
85
|
-
def purge(status, max_retries: nil) # -> int
|
|
86
|
-
return 0 unless @backend.respond_to?(:purge)
|
|
87
|
-
@backend.purge(@topic, status)
|
|
88
|
-
end
|
|
89
|
-
|
|
90
|
-
# Re-queue failed messages (under max_retries) back to pending.
|
|
91
|
-
# Returns the number of jobs re-queued.
|
|
92
|
-
def retry_failed(max_retries: nil) # -> int
|
|
93
|
-
return 0 unless @backend.respond_to?(:retry_failed)
|
|
94
|
-
@backend.retry_failed(@topic, max_retries: max_retries || @max_retries)
|
|
95
|
-
end
|
|
96
|
-
|
|
97
|
-
# Produce a message onto a topic. Convenience wrapper around push().
|
|
98
|
-
def produce(topic, payload, priority: 0, delay_seconds: 0)
|
|
99
|
-
available_at = delay_seconds > 0 ? Time.now + delay_seconds : nil
|
|
100
|
-
message = Job.new(topic: topic, payload: payload, priority: priority, available_at: available_at, queue: self)
|
|
101
|
-
@backend.enqueue(message)
|
|
102
|
-
message
|
|
103
|
-
end
|
|
104
|
-
|
|
105
|
-
# Consume jobs from a topic using an Enumerator (yield pattern).
|
|
106
|
-
#
|
|
107
|
-
# Usage:
|
|
108
|
-
# queue.consume("emails") do |job|
|
|
109
|
-
# process(job)
|
|
110
|
-
# end
|
|
111
|
-
#
|
|
112
|
-
# # Consume a specific job by ID:
|
|
113
|
-
# queue.consume("emails", id: "abc-123") do |job|
|
|
114
|
-
# process(job)
|
|
115
|
-
# end
|
|
116
|
-
#
|
|
117
|
-
# # Or as an enumerator:
|
|
118
|
-
# queue.consume("emails").each { |job| process(job) }
|
|
119
|
-
#
|
|
120
|
-
# Consume jobs from a topic using a long-running generator.
|
|
121
|
-
#
|
|
122
|
-
# Polls the queue continuously. When empty, sleeps for poll_interval
|
|
123
|
-
# seconds before polling again. No external while-loop or sleep needed.
|
|
124
|
-
#
|
|
125
|
-
# queue.consume("emails") { |job| process(job) }
|
|
126
|
-
# queue.consume("emails", poll_interval: 5) { |job| process(job) }
|
|
127
|
-
# queue.consume("emails", id: "abc-123") { |job| process(job) }
|
|
128
|
-
#
|
|
129
|
-
def consume(topic = nil, id: nil, poll_interval: 1.0, iterations: 0, batch_size: 1, &block)
|
|
130
|
-
topic ||= @topic
|
|
131
|
-
|
|
132
|
-
if id
|
|
133
|
-
# Single job by ID — no polling
|
|
134
|
-
job = pop_by_id(topic, id)
|
|
135
|
-
if job
|
|
136
|
-
block_given? ? yield(job) : (return Enumerator.new { |y| y << job })
|
|
137
|
-
end
|
|
138
|
-
return block_given? ? nil : Enumerator.new { |_| }
|
|
139
|
-
end
|
|
140
|
-
|
|
141
|
-
# poll_interval=0 → single-pass drain (returns when empty)
|
|
142
|
-
# poll_interval>0 → long-running poll (sleeps when empty, never returns)
|
|
143
|
-
# iterations>0 → stop after consuming N jobs
|
|
144
|
-
if block_given?
|
|
145
|
-
consumed = 0
|
|
146
|
-
if batch_size > 1
|
|
147
|
-
loop do
|
|
148
|
-
jobs = pop_batch(batch_size)
|
|
149
|
-
if jobs.empty?
|
|
150
|
-
break if poll_interval <= 0
|
|
151
|
-
sleep(poll_interval)
|
|
152
|
-
next
|
|
153
|
-
end
|
|
154
|
-
yield jobs
|
|
155
|
-
consumed += jobs.length
|
|
156
|
-
break if iterations > 0 && consumed >= iterations
|
|
157
|
-
end
|
|
158
|
-
else
|
|
159
|
-
loop do
|
|
160
|
-
job = @backend.dequeue(topic)
|
|
161
|
-
if job.nil?
|
|
162
|
-
break if poll_interval <= 0
|
|
163
|
-
sleep(poll_interval)
|
|
164
|
-
next
|
|
165
|
-
end
|
|
166
|
-
yield job
|
|
167
|
-
consumed += 1
|
|
168
|
-
break if iterations > 0 && consumed >= iterations
|
|
169
|
-
end
|
|
170
|
-
end
|
|
171
|
-
else
|
|
172
|
-
Enumerator.new do |yielder|
|
|
173
|
-
consumed = 0
|
|
174
|
-
loop do
|
|
175
|
-
job = @backend.dequeue(topic)
|
|
176
|
-
if job.nil?
|
|
177
|
-
break if poll_interval <= 0
|
|
178
|
-
sleep(poll_interval)
|
|
179
|
-
next
|
|
180
|
-
end
|
|
181
|
-
yielder << job
|
|
182
|
-
consumed += 1
|
|
183
|
-
break if iterations > 0 && consumed >= iterations
|
|
184
|
-
end
|
|
185
|
-
end
|
|
186
|
-
end
|
|
187
|
-
end
|
|
188
|
-
|
|
189
|
-
# Pop a specific job by ID from the queue.
|
|
190
|
-
def pop_by_id(id)
|
|
191
|
-
return nil unless @backend.respond_to?(:find_by_id)
|
|
192
|
-
@backend.find_by_id(@topic, id)
|
|
193
|
-
end
|
|
194
|
-
|
|
195
|
-
# Get the number of messages by status.
|
|
196
|
-
# status: "pending" (default) counts pending messages in the topic queue.
|
|
197
|
-
# status: "failed" or "dead" counts messages in the dead_letter directory.
|
|
198
|
-
def size(status: "pending")
|
|
199
|
-
case status.to_s
|
|
200
|
-
when "pending"
|
|
201
|
-
@backend.size(@topic)
|
|
202
|
-
when "failed", "dead"
|
|
203
|
-
if @backend.respond_to?(:dead_letter_count)
|
|
204
|
-
@backend.dead_letter_count(@topic)
|
|
205
|
-
else
|
|
206
|
-
0
|
|
207
|
-
end
|
|
208
|
-
else
|
|
209
|
-
@backend.size(@topic)
|
|
210
|
-
end
|
|
211
|
-
end
|
|
212
|
-
|
|
213
|
-
# Get the topic name this queue was constructed with.
|
|
214
|
-
def get_topic
|
|
215
|
-
@topic
|
|
216
|
-
end
|
|
217
|
-
|
|
218
|
-
# Consume all available jobs and pass each to handler, then stop.
|
|
219
|
-
#
|
|
220
|
-
# Simpler alternative to consume() for drain-and-exit use cases.
|
|
221
|
-
#
|
|
222
|
-
# queue.process { |job| handle(job); job.complete }
|
|
223
|
-
# queue.process(topic: "emails", max_jobs: 10) { |job| ... }
|
|
224
|
-
#
|
|
225
|
-
def process(topic: nil, max_jobs: nil, batch_size: 1, &handler)
|
|
226
|
-
raise ArgumentError, "block required" unless block_given?
|
|
227
|
-
drain_topic = topic || @topic
|
|
228
|
-
processed = 0
|
|
229
|
-
loop do
|
|
230
|
-
break if max_jobs && processed >= max_jobs
|
|
231
|
-
if batch_size > 1
|
|
232
|
-
remaining = max_jobs ? [batch_size, max_jobs - processed].min : batch_size
|
|
233
|
-
jobs = @backend.respond_to?(:dequeue_batch) ?
|
|
234
|
-
@backend.dequeue_batch(drain_topic, remaining) :
|
|
235
|
-
(1..remaining).map { @backend.dequeue(drain_topic) }.compact
|
|
236
|
-
break if jobs.empty?
|
|
237
|
-
begin
|
|
238
|
-
handler.call(jobs)
|
|
239
|
-
rescue => e
|
|
240
|
-
jobs.each { |job| job.fail(e.message) }
|
|
241
|
-
end
|
|
242
|
-
processed += jobs.length
|
|
243
|
-
else
|
|
244
|
-
job = @backend.dequeue(drain_topic)
|
|
245
|
-
break if job.nil?
|
|
246
|
-
begin
|
|
247
|
-
handler.call(job)
|
|
248
|
-
rescue => e
|
|
249
|
-
job.fail(e.message)
|
|
250
|
-
end
|
|
251
|
-
processed += 1
|
|
252
|
-
end
|
|
253
|
-
end
|
|
254
|
-
end
|
|
255
|
-
|
|
256
|
-
# Get the underlying backend instance.
|
|
257
|
-
def backend
|
|
258
|
-
@backend
|
|
259
|
-
end
|
|
260
|
-
|
|
261
|
-
# Resolve the default backend from env vars.
|
|
262
|
-
def self.resolve_backend(name = nil)
|
|
263
|
-
chosen = name || ENV.fetch("TINA4_QUEUE_BACKEND", "file").downcase.strip
|
|
264
|
-
|
|
265
|
-
case chosen.to_s
|
|
266
|
-
when "lite", "file", "default"
|
|
267
|
-
Tina4::QueueBackends::LiteBackend.new
|
|
268
|
-
when "rabbitmq"
|
|
269
|
-
config = resolve_rabbitmq_config
|
|
270
|
-
Tina4::QueueBackends::RabbitmqBackend.new(config)
|
|
271
|
-
when "kafka"
|
|
272
|
-
config = resolve_kafka_config
|
|
273
|
-
Tina4::QueueBackends::KafkaBackend.new(config)
|
|
274
|
-
when "mongodb", "mongo"
|
|
275
|
-
config = resolve_mongo_config
|
|
276
|
-
Tina4::QueueBackends::MongoBackend.new(config)
|
|
277
|
-
else
|
|
278
|
-
raise ArgumentError, "Unknown queue backend: #{chosen.inspect}. Use 'lite', 'rabbitmq', 'kafka', or 'mongodb'."
|
|
279
|
-
end
|
|
280
|
-
end
|
|
281
|
-
|
|
282
|
-
private
|
|
283
|
-
|
|
284
|
-
def resolve_backend_arg(backend)
|
|
285
|
-
# If a backend instance is passed directly (legacy), use it
|
|
286
|
-
return backend if backend && !backend.is_a?(Symbol) && !backend.is_a?(String)
|
|
287
|
-
# If a symbol or string name is passed, resolve it
|
|
288
|
-
Queue.resolve_backend(backend)
|
|
289
|
-
end
|
|
290
|
-
|
|
291
|
-
def self.resolve_rabbitmq_config
|
|
292
|
-
config = {}
|
|
293
|
-
url = ENV["TINA4_QUEUE_URL"]
|
|
294
|
-
if url
|
|
295
|
-
config = parse_amqp_url(url)
|
|
296
|
-
end
|
|
297
|
-
config[:host] ||= ENV.fetch("TINA4_RABBITMQ_HOST", "localhost")
|
|
298
|
-
config[:port] ||= (ENV["TINA4_RABBITMQ_PORT"] || 5672).to_i
|
|
299
|
-
config[:username] ||= ENV.fetch("TINA4_RABBITMQ_USERNAME", "guest")
|
|
300
|
-
config[:password] ||= ENV.fetch("TINA4_RABBITMQ_PASSWORD", "guest")
|
|
301
|
-
config[:vhost] ||= ENV.fetch("TINA4_RABBITMQ_VHOST", "/")
|
|
302
|
-
config
|
|
303
|
-
end
|
|
304
|
-
|
|
305
|
-
def self.resolve_kafka_config
|
|
306
|
-
config = {}
|
|
307
|
-
url = ENV["TINA4_QUEUE_URL"]
|
|
308
|
-
if url
|
|
309
|
-
config[:brokers] = url.sub("kafka://", "")
|
|
310
|
-
end
|
|
311
|
-
brokers = ENV["TINA4_KAFKA_BROKERS"]
|
|
312
|
-
config[:brokers] = brokers if brokers
|
|
313
|
-
config[:brokers] ||= "localhost:9092"
|
|
314
|
-
config[:group_id] = ENV.fetch("TINA4_KAFKA_GROUP_ID", "tina4_consumer_group")
|
|
315
|
-
config
|
|
316
|
-
end
|
|
317
|
-
|
|
318
|
-
def self.resolve_mongo_config
|
|
319
|
-
config = {}
|
|
320
|
-
uri = ENV["TINA4_MONGO_URI"]
|
|
321
|
-
config[:uri] = uri if uri
|
|
322
|
-
config[:host] = ENV.fetch("TINA4_MONGO_HOST", "localhost") unless uri
|
|
323
|
-
config[:port] = (ENV["TINA4_MONGO_PORT"] || 27017).to_i unless uri
|
|
324
|
-
username = ENV["TINA4_MONGO_USERNAME"]
|
|
325
|
-
password = ENV["TINA4_MONGO_PASSWORD"]
|
|
326
|
-
config[:username] = username if username
|
|
327
|
-
config[:password] = password if password
|
|
328
|
-
config[:db] = ENV.fetch("TINA4_MONGO_DB", "tina4")
|
|
329
|
-
config[:collection] = ENV.fetch("TINA4_MONGO_COLLECTION", "tina4_queue")
|
|
330
|
-
config
|
|
331
|
-
end
|
|
332
|
-
|
|
333
|
-
def self.parse_amqp_url(url)
|
|
334
|
-
config = {}
|
|
335
|
-
url = url.sub("amqp://", "").sub("amqps://", "")
|
|
336
|
-
|
|
337
|
-
if url.include?("@")
|
|
338
|
-
creds, rest = url.split("@", 2)
|
|
339
|
-
if creds.include?(":")
|
|
340
|
-
config[:username], config[:password] = creds.split(":", 2)
|
|
341
|
-
else
|
|
342
|
-
config[:username] = creds
|
|
343
|
-
end
|
|
344
|
-
else
|
|
345
|
-
rest = url
|
|
346
|
-
end
|
|
347
|
-
|
|
348
|
-
if rest.include?("/")
|
|
349
|
-
hostport, vhost = rest.split("/", 2)
|
|
350
|
-
config[:vhost] = vhost.start_with?("/") ? vhost : "/#{vhost}" if vhost && !vhost.empty?
|
|
351
|
-
else
|
|
352
|
-
hostport = rest
|
|
353
|
-
end
|
|
354
|
-
|
|
355
|
-
if hostport.include?(":")
|
|
356
|
-
host, port = hostport.split(":", 2)
|
|
357
|
-
config[:host] = host
|
|
358
|
-
config[:port] = port.to_i
|
|
359
|
-
elsif hostport && !hostport.empty?
|
|
360
|
-
config[:host] = hostport
|
|
361
|
-
end
|
|
362
|
-
|
|
363
|
-
config
|
|
364
|
-
end
|
|
365
|
-
end
|
|
366
|
-
end
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require "json"
|
|
3
|
+
require "securerandom"
|
|
4
|
+
require_relative "job"
|
|
5
|
+
|
|
6
|
+
module Tina4
|
|
7
|
+
# Queue — unified wrapper for queue management operations.
|
|
8
|
+
# Auto-detects backend from TINA4_QUEUE_BACKEND env var.
|
|
9
|
+
#
|
|
10
|
+
# Usage:
|
|
11
|
+
# # Auto-detect from env (default: lite/file backend)
|
|
12
|
+
# queue = Queue.new(topic: "tasks")
|
|
13
|
+
#
|
|
14
|
+
# # Explicit backend
|
|
15
|
+
# queue = Queue.new(topic: "tasks", backend: :rabbitmq)
|
|
16
|
+
#
|
|
17
|
+
# # Or pass a backend instance directly (legacy)
|
|
18
|
+
# queue = Queue.new(topic: "tasks", backend: my_backend)
|
|
19
|
+
class Queue
|
|
20
|
+
attr_reader :topic, :max_retries
|
|
21
|
+
|
|
22
|
+
def initialize(topic:, backend: nil, max_retries: 3)
|
|
23
|
+
@topic = topic
|
|
24
|
+
@max_retries = max_retries
|
|
25
|
+
@backend = resolve_backend_arg(backend)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Push a job onto the queue. Returns the Job.
|
|
29
|
+
# priority: higher-priority messages are dequeued first (default 0).
|
|
30
|
+
# delay_seconds: delay before the message becomes available (default 0).
|
|
31
|
+
def push(payload, priority: 0, delay_seconds: 0)
|
|
32
|
+
available_at = delay_seconds > 0 ? Time.now + delay_seconds : nil
|
|
33
|
+
message = Job.new(topic: @topic, payload: payload, priority: priority, available_at: available_at, queue: self)
|
|
34
|
+
@backend.enqueue(message)
|
|
35
|
+
message
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Pop the next available job. Returns Job or nil.
|
|
39
|
+
def pop # -> Job|None
|
|
40
|
+
@backend.dequeue(@topic)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Pop up to count jobs at once. Returns a partial batch if fewer available.
|
|
44
|
+
def pop_batch(count)
|
|
45
|
+
if @backend.respond_to?(:dequeue_batch)
|
|
46
|
+
@backend.dequeue_batch(@topic, count)
|
|
47
|
+
else
|
|
48
|
+
jobs = []
|
|
49
|
+
count.times do
|
|
50
|
+
job = @backend.dequeue(@topic)
|
|
51
|
+
break if job.nil?
|
|
52
|
+
jobs << job
|
|
53
|
+
end
|
|
54
|
+
jobs
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Clear all pending jobs from this queue's topic. Returns count removed.
|
|
59
|
+
def clear # -> int
|
|
60
|
+
return 0 unless @backend.respond_to?(:clear)
|
|
61
|
+
@backend.clear(@topic)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Get jobs that failed but are still eligible for retry (under max_retries).
|
|
65
|
+
def failed # -> list[dict]
|
|
66
|
+
return [] unless @backend.respond_to?(:failed)
|
|
67
|
+
@backend.failed(@topic, max_retries: @max_retries)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Retry a specific failed job by ID, or all dead-letter jobs if no id given.
|
|
71
|
+
# Returns true if re-queued.
|
|
72
|
+
def retry(job_id = nil, delay_seconds: 0) # -> bool
|
|
73
|
+
return false unless @backend.respond_to?(:retry_job)
|
|
74
|
+
@backend.retry_job(@topic, job_id: job_id, delay_seconds: delay_seconds)
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Get dead letter jobs — messages that exceeded max retries.
|
|
78
|
+
# Pass max_retries to override the queue's default.
|
|
79
|
+
def dead_letters(max_retries: nil) # -> list[dict]
|
|
80
|
+
return [] unless @backend.respond_to?(:dead_letters)
|
|
81
|
+
@backend.dead_letters(@topic, max_retries: max_retries || @max_retries)
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Delete messages by status (completed, failed, dead).
|
|
85
|
+
def purge(status, max_retries: nil) # -> int
|
|
86
|
+
return 0 unless @backend.respond_to?(:purge)
|
|
87
|
+
@backend.purge(@topic, status)
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# Re-queue failed messages (under max_retries) back to pending.
|
|
91
|
+
# Returns the number of jobs re-queued.
|
|
92
|
+
def retry_failed(max_retries: nil) # -> int
|
|
93
|
+
return 0 unless @backend.respond_to?(:retry_failed)
|
|
94
|
+
@backend.retry_failed(@topic, max_retries: max_retries || @max_retries)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# Produce a message onto a topic. Convenience wrapper around push().
|
|
98
|
+
def produce(topic, payload, priority: 0, delay_seconds: 0)
|
|
99
|
+
available_at = delay_seconds > 0 ? Time.now + delay_seconds : nil
|
|
100
|
+
message = Job.new(topic: topic, payload: payload, priority: priority, available_at: available_at, queue: self)
|
|
101
|
+
@backend.enqueue(message)
|
|
102
|
+
message
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Consume jobs from a topic using an Enumerator (yield pattern).
|
|
106
|
+
#
|
|
107
|
+
# Usage:
|
|
108
|
+
# queue.consume("emails") do |job|
|
|
109
|
+
# process(job)
|
|
110
|
+
# end
|
|
111
|
+
#
|
|
112
|
+
# # Consume a specific job by ID:
|
|
113
|
+
# queue.consume("emails", id: "abc-123") do |job|
|
|
114
|
+
# process(job)
|
|
115
|
+
# end
|
|
116
|
+
#
|
|
117
|
+
# # Or as an enumerator:
|
|
118
|
+
# queue.consume("emails").each { |job| process(job) }
|
|
119
|
+
#
|
|
120
|
+
# Consume jobs from a topic using a long-running generator.
|
|
121
|
+
#
|
|
122
|
+
# Polls the queue continuously. When empty, sleeps for poll_interval
|
|
123
|
+
# seconds before polling again. No external while-loop or sleep needed.
|
|
124
|
+
#
|
|
125
|
+
# queue.consume("emails") { |job| process(job) }
|
|
126
|
+
# queue.consume("emails", poll_interval: 5) { |job| process(job) }
|
|
127
|
+
# queue.consume("emails", id: "abc-123") { |job| process(job) }
|
|
128
|
+
#
|
|
129
|
+
def consume(topic = nil, id: nil, poll_interval: 1.0, iterations: 0, batch_size: 1, &block)
|
|
130
|
+
topic ||= @topic
|
|
131
|
+
|
|
132
|
+
if id
|
|
133
|
+
# Single job by ID — no polling
|
|
134
|
+
job = pop_by_id(topic, id)
|
|
135
|
+
if job
|
|
136
|
+
block_given? ? yield(job) : (return Enumerator.new { |y| y << job })
|
|
137
|
+
end
|
|
138
|
+
return block_given? ? nil : Enumerator.new { |_| }
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
# poll_interval=0 → single-pass drain (returns when empty)
|
|
142
|
+
# poll_interval>0 → long-running poll (sleeps when empty, never returns)
|
|
143
|
+
# iterations>0 → stop after consuming N jobs
|
|
144
|
+
if block_given?
|
|
145
|
+
consumed = 0
|
|
146
|
+
if batch_size > 1
|
|
147
|
+
loop do
|
|
148
|
+
jobs = pop_batch(batch_size)
|
|
149
|
+
if jobs.empty?
|
|
150
|
+
break if poll_interval <= 0
|
|
151
|
+
sleep(poll_interval)
|
|
152
|
+
next
|
|
153
|
+
end
|
|
154
|
+
yield jobs
|
|
155
|
+
consumed += jobs.length
|
|
156
|
+
break if iterations > 0 && consumed >= iterations
|
|
157
|
+
end
|
|
158
|
+
else
|
|
159
|
+
loop do
|
|
160
|
+
job = @backend.dequeue(topic)
|
|
161
|
+
if job.nil?
|
|
162
|
+
break if poll_interval <= 0
|
|
163
|
+
sleep(poll_interval)
|
|
164
|
+
next
|
|
165
|
+
end
|
|
166
|
+
yield job
|
|
167
|
+
consumed += 1
|
|
168
|
+
break if iterations > 0 && consumed >= iterations
|
|
169
|
+
end
|
|
170
|
+
end
|
|
171
|
+
else
|
|
172
|
+
Enumerator.new do |yielder|
|
|
173
|
+
consumed = 0
|
|
174
|
+
loop do
|
|
175
|
+
job = @backend.dequeue(topic)
|
|
176
|
+
if job.nil?
|
|
177
|
+
break if poll_interval <= 0
|
|
178
|
+
sleep(poll_interval)
|
|
179
|
+
next
|
|
180
|
+
end
|
|
181
|
+
yielder << job
|
|
182
|
+
consumed += 1
|
|
183
|
+
break if iterations > 0 && consumed >= iterations
|
|
184
|
+
end
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
# Pop a specific job by ID from the queue.
|
|
190
|
+
def pop_by_id(id)
|
|
191
|
+
return nil unless @backend.respond_to?(:find_by_id)
|
|
192
|
+
@backend.find_by_id(@topic, id)
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
# Get the number of messages by status.
|
|
196
|
+
# status: "pending" (default) counts pending messages in the topic queue.
|
|
197
|
+
# status: "failed" or "dead" counts messages in the dead_letter directory.
|
|
198
|
+
def size(status: "pending")
|
|
199
|
+
case status.to_s
|
|
200
|
+
when "pending"
|
|
201
|
+
@backend.size(@topic)
|
|
202
|
+
when "failed", "dead"
|
|
203
|
+
if @backend.respond_to?(:dead_letter_count)
|
|
204
|
+
@backend.dead_letter_count(@topic)
|
|
205
|
+
else
|
|
206
|
+
0
|
|
207
|
+
end
|
|
208
|
+
else
|
|
209
|
+
@backend.size(@topic)
|
|
210
|
+
end
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
# Get the topic name this queue was constructed with.
|
|
214
|
+
def get_topic
|
|
215
|
+
@topic
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
# Consume all available jobs and pass each to handler, then stop.
|
|
219
|
+
#
|
|
220
|
+
# Simpler alternative to consume() for drain-and-exit use cases.
|
|
221
|
+
#
|
|
222
|
+
# queue.process { |job| handle(job); job.complete }
|
|
223
|
+
# queue.process(topic: "emails", max_jobs: 10) { |job| ... }
|
|
224
|
+
#
|
|
225
|
+
def process(topic: nil, max_jobs: nil, batch_size: 1, &handler)
|
|
226
|
+
raise ArgumentError, "block required" unless block_given?
|
|
227
|
+
drain_topic = topic || @topic
|
|
228
|
+
processed = 0
|
|
229
|
+
loop do
|
|
230
|
+
break if max_jobs && processed >= max_jobs
|
|
231
|
+
if batch_size > 1
|
|
232
|
+
remaining = max_jobs ? [batch_size, max_jobs - processed].min : batch_size
|
|
233
|
+
jobs = @backend.respond_to?(:dequeue_batch) ?
|
|
234
|
+
@backend.dequeue_batch(drain_topic, remaining) :
|
|
235
|
+
(1..remaining).map { @backend.dequeue(drain_topic) }.compact
|
|
236
|
+
break if jobs.empty?
|
|
237
|
+
begin
|
|
238
|
+
handler.call(jobs)
|
|
239
|
+
rescue => e
|
|
240
|
+
jobs.each { |job| job.fail(e.message) }
|
|
241
|
+
end
|
|
242
|
+
processed += jobs.length
|
|
243
|
+
else
|
|
244
|
+
job = @backend.dequeue(drain_topic)
|
|
245
|
+
break if job.nil?
|
|
246
|
+
begin
|
|
247
|
+
handler.call(job)
|
|
248
|
+
rescue => e
|
|
249
|
+
job.fail(e.message)
|
|
250
|
+
end
|
|
251
|
+
processed += 1
|
|
252
|
+
end
|
|
253
|
+
end
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
# Get the underlying backend instance.
|
|
257
|
+
def backend
|
|
258
|
+
@backend
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
# Resolve the default backend from env vars.
|
|
262
|
+
def self.resolve_backend(name = nil)
|
|
263
|
+
chosen = name || ENV.fetch("TINA4_QUEUE_BACKEND", "file").downcase.strip
|
|
264
|
+
|
|
265
|
+
case chosen.to_s
|
|
266
|
+
when "lite", "file", "default"
|
|
267
|
+
Tina4::QueueBackends::LiteBackend.new
|
|
268
|
+
when "rabbitmq"
|
|
269
|
+
config = resolve_rabbitmq_config
|
|
270
|
+
Tina4::QueueBackends::RabbitmqBackend.new(config)
|
|
271
|
+
when "kafka"
|
|
272
|
+
config = resolve_kafka_config
|
|
273
|
+
Tina4::QueueBackends::KafkaBackend.new(config)
|
|
274
|
+
when "mongodb", "mongo"
|
|
275
|
+
config = resolve_mongo_config
|
|
276
|
+
Tina4::QueueBackends::MongoBackend.new(config)
|
|
277
|
+
else
|
|
278
|
+
raise ArgumentError, "Unknown queue backend: #{chosen.inspect}. Use 'lite', 'rabbitmq', 'kafka', or 'mongodb'."
|
|
279
|
+
end
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
private
|
|
283
|
+
|
|
284
|
+
def resolve_backend_arg(backend)
|
|
285
|
+
# If a backend instance is passed directly (legacy), use it
|
|
286
|
+
return backend if backend && !backend.is_a?(Symbol) && !backend.is_a?(String)
|
|
287
|
+
# If a symbol or string name is passed, resolve it
|
|
288
|
+
Queue.resolve_backend(backend)
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
def self.resolve_rabbitmq_config
|
|
292
|
+
config = {}
|
|
293
|
+
url = ENV["TINA4_QUEUE_URL"]
|
|
294
|
+
if url
|
|
295
|
+
config = parse_amqp_url(url)
|
|
296
|
+
end
|
|
297
|
+
config[:host] ||= ENV.fetch("TINA4_RABBITMQ_HOST", "localhost")
|
|
298
|
+
config[:port] ||= (ENV["TINA4_RABBITMQ_PORT"] || 5672).to_i
|
|
299
|
+
config[:username] ||= ENV.fetch("TINA4_RABBITMQ_USERNAME", "guest")
|
|
300
|
+
config[:password] ||= ENV.fetch("TINA4_RABBITMQ_PASSWORD", "guest")
|
|
301
|
+
config[:vhost] ||= ENV.fetch("TINA4_RABBITMQ_VHOST", "/")
|
|
302
|
+
config
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
def self.resolve_kafka_config
|
|
306
|
+
config = {}
|
|
307
|
+
url = ENV["TINA4_QUEUE_URL"]
|
|
308
|
+
if url
|
|
309
|
+
config[:brokers] = url.sub("kafka://", "")
|
|
310
|
+
end
|
|
311
|
+
brokers = ENV["TINA4_KAFKA_BROKERS"]
|
|
312
|
+
config[:brokers] = brokers if brokers
|
|
313
|
+
config[:brokers] ||= "localhost:9092"
|
|
314
|
+
config[:group_id] = ENV.fetch("TINA4_KAFKA_GROUP_ID", "tina4_consumer_group")
|
|
315
|
+
config
|
|
316
|
+
end
|
|
317
|
+
|
|
318
|
+
def self.resolve_mongo_config
|
|
319
|
+
config = {}
|
|
320
|
+
uri = ENV["TINA4_MONGO_URI"]
|
|
321
|
+
config[:uri] = uri if uri
|
|
322
|
+
config[:host] = ENV.fetch("TINA4_MONGO_HOST", "localhost") unless uri
|
|
323
|
+
config[:port] = (ENV["TINA4_MONGO_PORT"] || 27017).to_i unless uri
|
|
324
|
+
username = ENV["TINA4_MONGO_USERNAME"]
|
|
325
|
+
password = ENV["TINA4_MONGO_PASSWORD"]
|
|
326
|
+
config[:username] = username if username
|
|
327
|
+
config[:password] = password if password
|
|
328
|
+
config[:db] = ENV.fetch("TINA4_MONGO_DB", "tina4")
|
|
329
|
+
config[:collection] = ENV.fetch("TINA4_MONGO_COLLECTION", "tina4_queue")
|
|
330
|
+
config
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
def self.parse_amqp_url(url)
|
|
334
|
+
config = {}
|
|
335
|
+
url = url.sub("amqp://", "").sub("amqps://", "")
|
|
336
|
+
|
|
337
|
+
if url.include?("@")
|
|
338
|
+
creds, rest = url.split("@", 2)
|
|
339
|
+
if creds.include?(":")
|
|
340
|
+
config[:username], config[:password] = creds.split(":", 2)
|
|
341
|
+
else
|
|
342
|
+
config[:username] = creds
|
|
343
|
+
end
|
|
344
|
+
else
|
|
345
|
+
rest = url
|
|
346
|
+
end
|
|
347
|
+
|
|
348
|
+
if rest.include?("/")
|
|
349
|
+
hostport, vhost = rest.split("/", 2)
|
|
350
|
+
config[:vhost] = vhost.start_with?("/") ? vhost : "/#{vhost}" if vhost && !vhost.empty?
|
|
351
|
+
else
|
|
352
|
+
hostport = rest
|
|
353
|
+
end
|
|
354
|
+
|
|
355
|
+
if hostport.include?(":")
|
|
356
|
+
host, port = hostport.split(":", 2)
|
|
357
|
+
config[:host] = host
|
|
358
|
+
config[:port] = port.to_i
|
|
359
|
+
elsif hostport && !hostport.empty?
|
|
360
|
+
config[:host] = hostport
|
|
361
|
+
end
|
|
362
|
+
|
|
363
|
+
config
|
|
364
|
+
end
|
|
365
|
+
end
|
|
366
|
+
end
|