pgbus 0.19.4 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/README.md +97 -7
- data/Rakefile +6 -1
- data/app/controllers/pgbus/dashboard_controller.rb +2 -0
- data/app/controllers/pgbus/workflow_definitions_controller.rb +28 -0
- data/app/controllers/pgbus/workflows_controller.rb +197 -0
- data/app/frontend/pgbus/modules/charts.js +148 -4
- data/app/frontend/pgbus/modules/charts.min.js +1 -1
- data/app/frontend/pgbus/modules/charts.min.js.map +1 -1
- data/app/frontend/pgbus/style.css +1 -1
- data/app/helpers/pgbus/button_helper.rb +4 -2
- data/app/helpers/pgbus/workflow_insights_helper.rb +129 -0
- data/app/helpers/pgbus/workflows_helper.rb +165 -0
- data/app/models/pgbus/blocked_execution.rb +12 -0
- data/app/models/pgbus/uniqueness_key.rb +88 -3
- data/app/models/pgbus/workflow_entry.rb +172 -0
- data/app/models/pgbus/workflow_event.rb +43 -0
- data/app/models/pgbus/workflow_signal.rb +77 -0
- data/app/models/pgbus/workflow_step.rb +149 -0
- data/app/views/layouts/pgbus/application.html.erb +2 -0
- data/app/views/pgbus/dashboard/_stats_cards.html.erb +17 -0
- data/app/views/pgbus/workflows/_actions.html.erb +59 -0
- data/app/views/pgbus/workflows/_events_table.html.erb +24 -0
- data/app/views/pgbus/workflows/_filters.html.erb +29 -0
- data/app/views/pgbus/workflows/_signal_form.html.erb +31 -0
- data/app/views/pgbus/workflows/_signals_table.html.erb +40 -0
- data/app/views/pgbus/workflows/_status_filter.html.erb +12 -0
- data/app/views/pgbus/workflows/_steps_table.html.erb +79 -0
- data/app/views/pgbus/workflows/_timeline.html.erb +49 -0
- data/app/views/pgbus/workflows/_workflows_table.html.erb +59 -0
- data/app/views/pgbus/workflows/attention.html.erb +88 -0
- data/app/views/pgbus/workflows/definitions/_by_revision.html.erb +52 -0
- data/app/views/pgbus/workflows/definitions/_duration_table.html.erb +18 -0
- data/app/views/pgbus/workflows/definitions/_funnel.html.erb +49 -0
- data/app/views/pgbus/workflows/definitions/_pipeline.html.erb +16 -0
- data/app/views/pgbus/workflows/definitions/_step_card.html.erb +31 -0
- data/app/views/pgbus/workflows/definitions/_throughput_table.html.erb +32 -0
- data/app/views/pgbus/workflows/definitions/_window.html.erb +17 -0
- data/app/views/pgbus/workflows/definitions/index.html.erb +71 -0
- data/app/views/pgbus/workflows/definitions/not_found.html.erb +5 -0
- data/app/views/pgbus/workflows/definitions/show.html.erb +80 -0
- data/app/views/pgbus/workflows/index.html.erb +15 -0
- data/app/views/pgbus/workflows/not_found.html.erb +5 -0
- data/app/views/pgbus/workflows/show.html.erb +93 -0
- data/config/i18n-tasks.yml +6 -0
- data/config/locales/da.yml +445 -0
- data/config/locales/de.yml +445 -0
- data/config/locales/en.yml +445 -0
- data/config/locales/es.yml +445 -0
- data/config/locales/fi.yml +445 -0
- data/config/locales/fr.yml +445 -0
- data/config/locales/it.yml +445 -0
- data/config/locales/ja.yml +445 -0
- data/config/locales/nb.yml +445 -0
- data/config/locales/nl.yml +445 -0
- data/config/locales/pt.yml +445 -0
- data/config/locales/sv.yml +445 -0
- data/config/routes.rb +26 -0
- data/lib/generators/pgbus/add_workflow_signals_generator.rb +56 -0
- data/lib/generators/pgbus/add_workflows_generator.rb +54 -0
- data/lib/generators/pgbus/templates/add_workflow_signals.rb.erb +51 -0
- data/lib/generators/pgbus/templates/add_workflows.rb.erb +150 -0
- data/lib/generators/pgbus/templates/migration.rb.erb +127 -0
- data/lib/pgbus/active_job/adapter.rb +81 -3
- data/lib/pgbus/active_job/delivery.rb +63 -0
- data/lib/pgbus/active_job/executor.rb +20 -8
- data/lib/pgbus/concurrency/blocked_execution.rb +14 -6
- data/lib/pgbus/configuration/workflow_settings.rb +99 -0
- data/lib/pgbus/configuration.rb +64 -4
- data/lib/pgbus/dead_letter_header.rb +9 -1
- data/lib/pgbus/generators/migration_detector.rb +40 -0
- data/lib/pgbus/instrumentation.rb +33 -0
- data/lib/pgbus/integrations/appsignal/dashboard.json +157 -1
- data/lib/pgbus/integrations/appsignal/subscriber.rb +16 -1
- data/lib/pgbus/mcp/rack_app.rb +7 -2
- data/lib/pgbus/mcp/runner.rb +2 -1
- data/lib/pgbus/mcp/serialization.rb +25 -0
- data/lib/pgbus/mcp/server.rb +53 -10
- data/lib/pgbus/mcp/tools/deploy_tasks_tool.rb +1 -10
- data/lib/pgbus/mcp/tools/workflow_cancel_tool.rb +23 -0
- data/lib/pgbus/mcp/tools/workflow_definitions_tool.rb +36 -0
- data/lib/pgbus/mcp/tools/workflow_detail_tool.rb +38 -0
- data/lib/pgbus/mcp/tools/workflow_resume_at_tool.rb +32 -0
- data/lib/pgbus/mcp/tools/workflow_resume_tool.rb +24 -0
- data/lib/pgbus/mcp/tools/workflow_run_now_tool.rb +23 -0
- data/lib/pgbus/mcp/tools/workflow_signal_tool.rb +77 -0
- data/lib/pgbus/mcp/tools/workflow_skip_step_tool.rb +23 -0
- data/lib/pgbus/mcp/tools/workflow_stats_tool.rb +42 -0
- data/lib/pgbus/mcp/tools/workflows_attention_tool.rb +42 -0
- data/lib/pgbus/mcp/tools/workflows_tool.rb +60 -0
- data/lib/pgbus/mcp/workflow_insights_payload.rb +15 -0
- data/lib/pgbus/mcp/workflow_payloads.rb +53 -0
- data/lib/pgbus/mcp/write_tool.rb +134 -0
- data/lib/pgbus/mcp.rb +15 -0
- data/lib/pgbus/metrics/backends/statsd.rb +6 -0
- data/lib/pgbus/metrics/subscriber.rb +14 -1
- data/lib/pgbus/metrics/workflow_metrics.rb +75 -0
- data/lib/pgbus/process/consumer.rb +2 -1
- data/lib/pgbus/process/dispatcher.rb +39 -3
- data/lib/pgbus/process/worker.rb +10 -1
- data/lib/pgbus/recurring/schedule.rb +1 -1
- data/lib/pgbus/streams/presence.rb +65 -36
- data/lib/pgbus/uniqueness.rb +23 -6
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source/workflow_actions.rb +109 -0
- data/lib/pgbus/web/data_source/workflow_insights.rb +99 -0
- data/lib/pgbus/web/data_source/workflow_list.rb +291 -0
- data/lib/pgbus/web/data_source.rb +126 -15
- data/lib/pgbus/web/workflow_action_plan.rb +62 -0
- data/lib/pgbus/web/workflow_insights_queries.rb +250 -0
- data/lib/pgbus/web/workflow_insights_report.rb +171 -0
- data/lib/pgbus/web/workflow_payload.rb +36 -0
- data/lib/pgbus/web/workflow_query.rb +82 -0
- data/lib/pgbus/web/workflow_state.rb +89 -0
- data/lib/pgbus/web/workflow_timeline.rb +130 -0
- data/lib/pgbus/web/workflow_window.rb +51 -0
- data/lib/pgbus/workflow/batch_finished_job.rb +21 -0
- data/lib/pgbus/workflow/dsl.rb +105 -0
- data/lib/pgbus/workflow/error_capture.rb +63 -0
- data/lib/pgbus/workflow/exception_policy.rb +170 -0
- data/lib/pgbus/workflow/fan_out.rb +157 -0
- data/lib/pgbus/workflow/fan_out_definition.rb +41 -0
- data/lib/pgbus/workflow/flow_control.rb +94 -0
- data/lib/pgbus/workflow/instance.rb +183 -0
- data/lib/pgbus/workflow/outside_verbs.rb +138 -0
- data/lib/pgbus/workflow/payload.rb +53 -0
- data/lib/pgbus/workflow/registry.rb +53 -0
- data/lib/pgbus/workflow/removed_step.rb +14 -0
- data/lib/pgbus/workflow/runner.rb +218 -0
- data/lib/pgbus/workflow/signal_matcher.rb +53 -0
- data/lib/pgbus/workflow/signals.rb +132 -0
- data/lib/pgbus/workflow/states.rb +119 -0
- data/lib/pgbus/workflow/step_context.rb +58 -0
- data/lib/pgbus/workflow/step_definition.rb +133 -0
- data/lib/pgbus/workflow/step_execution.rb +242 -0
- data/lib/pgbus/workflow/step_job.rb +28 -0
- data/lib/pgbus/workflow/step_outcome.rb +151 -0
- data/lib/pgbus/workflow/steps.rb +103 -0
- data/lib/pgbus/workflow/sweep.rb +156 -0
- data/lib/pgbus/workflow/test_helpers.rb +152 -0
- data/lib/pgbus/workflow/wait_timeout.rb +94 -0
- data/lib/pgbus/workflow/waiting.rb +82 -0
- data/lib/pgbus/workflow.rb +183 -0
- data/sig/pgbus/configuration.rbs +23 -0
- metadata +92 -1
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
class Configuration
|
|
5
|
+
# Workflows (issue #540): durable multi-step Pgbus::Workflow runs.
|
|
6
|
+
#
|
|
7
|
+
# config.workflow_queue = "workflows" # where StepJob messages go
|
|
8
|
+
# config.workflow_retention = 30.days # finished/cancelled runs kept this long; nil keeps them
|
|
9
|
+
# config.workflow_sweep_interval = 5.minutes # how often the dispatcher repairs stuck steps
|
|
10
|
+
# config.workflow_stall_threshold = 1.hour # an in_progress step older than this is stalled
|
|
11
|
+
# config.workflow_stall_action = :pause # :pause, :reattempt or :cancel a stalled step
|
|
12
|
+
# config.workflow_max_payload_size = 131_072 # bytes: arguments and step outputs
|
|
13
|
+
# config.workflow_error_backtrace_lines = 50 # backtrace lines kept on a failed step
|
|
14
|
+
# config.workflow_waiting_visibility_threshold = 7.days # a step waiting this long without a
|
|
15
|
+
# # wait_timeout: counts as stuck (waiting_overdue)
|
|
16
|
+
# config.workflow_subject_url = ->(type, id) { "/admin/#{type.underscore.pluralize}/#{id}" }
|
|
17
|
+
# # the dashboard's subject link (nil: plain text)
|
|
18
|
+
module WorkflowSettings
|
|
19
|
+
WORKFLOW_STALL_ACTIONS = %i[pause reattempt cancel].freeze
|
|
20
|
+
|
|
21
|
+
attr_reader :workflow_queue, :workflow_retention, :workflow_sweep_interval, :workflow_stall_threshold,
|
|
22
|
+
:workflow_stall_action, :workflow_waiting_visibility_threshold
|
|
23
|
+
attr_accessor :workflow_max_payload_size, :workflow_error_backtrace_lines, :workflow_subject_url
|
|
24
|
+
|
|
25
|
+
def workflow_queue=(value)
|
|
26
|
+
name = value.to_s
|
|
27
|
+
raise Pgbus::ConfigurationError, "workflow_queue must be a non-empty queue name" if value.nil? || name.strip.empty?
|
|
28
|
+
|
|
29
|
+
@workflow_queue = name
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def workflow_retention=(value)
|
|
33
|
+
@workflow_retention = coerce_duration!(value, :workflow_retention)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def workflow_sweep_interval=(value)
|
|
37
|
+
@workflow_sweep_interval = coerce_duration!(value, :workflow_sweep_interval)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def workflow_stall_threshold=(value)
|
|
41
|
+
@workflow_stall_threshold = coerce_duration!(value, :workflow_stall_threshold)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def workflow_waiting_visibility_threshold=(value)
|
|
45
|
+
@workflow_waiting_visibility_threshold = coerce_duration!(value, :workflow_waiting_visibility_threshold)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def workflow_stall_action=(value)
|
|
49
|
+
action = value.respond_to?(:to_sym) ? value.to_sym : value
|
|
50
|
+
unless WORKFLOW_STALL_ACTIONS.include?(action)
|
|
51
|
+
raise Pgbus::ConfigurationError,
|
|
52
|
+
"workflow_stall_action must be one of #{WORKFLOW_STALL_ACTIONS.map(&:inspect).join(", ")}, got #{value.inspect}"
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
@workflow_stall_action = action
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
private
|
|
59
|
+
|
|
60
|
+
def initialize_workflow_settings
|
|
61
|
+
@workflow_queue = "default"
|
|
62
|
+
@workflow_retention = 30 * 24 * 3600 # 30 days
|
|
63
|
+
@workflow_sweep_interval = 300 # 5 minutes
|
|
64
|
+
@workflow_stall_threshold = 3600 # 1 hour
|
|
65
|
+
@workflow_stall_action = :pause
|
|
66
|
+
@workflow_max_payload_size = 131_072 # 128 KB
|
|
67
|
+
@workflow_error_backtrace_lines = 50
|
|
68
|
+
@workflow_waiting_visibility_threshold = 7 * 24 * 3600 # 7 days
|
|
69
|
+
@workflow_subject_url = nil
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def validate_workflow_settings!
|
|
73
|
+
{ workflow_sweep_interval: workflow_sweep_interval, workflow_stall_threshold: workflow_stall_threshold,
|
|
74
|
+
workflow_waiting_visibility_threshold: workflow_waiting_visibility_threshold,
|
|
75
|
+
workflow_retention: workflow_retention }.each do |name, value|
|
|
76
|
+
next if value.nil? && name == :workflow_retention
|
|
77
|
+
next if value.is_a?(Numeric) && value.positive? && value.finite?
|
|
78
|
+
|
|
79
|
+
raise Pgbus::ConfigurationError, "#{name} must be a finite number of seconds > 0"
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
begin
|
|
83
|
+
QueueNameValidator.validate!(queue_name(workflow_queue))
|
|
84
|
+
rescue ArgumentError => e
|
|
85
|
+
raise Pgbus::ConfigurationError, "workflow_queue #{workflow_queue.inspect}: #{e.message}"
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
{ workflow_max_payload_size: workflow_max_payload_size,
|
|
89
|
+
workflow_error_backtrace_lines: workflow_error_backtrace_lines }.each do |name, value|
|
|
90
|
+
raise Pgbus::ConfigurationError, "#{name} must be a positive integer" unless value.is_a?(Integer) && value.positive?
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
return if workflow_subject_url.nil? || workflow_subject_url.respond_to?(:call)
|
|
94
|
+
|
|
95
|
+
raise Pgbus::ConfigurationError, "workflow_subject_url must respond to #call (a Proc/lambda) or be nil"
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
data/lib/pgbus/configuration.rb
CHANGED
|
@@ -7,12 +7,14 @@ module Pgbus
|
|
|
7
7
|
include PresenceSettings
|
|
8
8
|
include StreamAuthorization
|
|
9
9
|
include DeploySettings
|
|
10
|
+
include WorkflowSettings
|
|
10
11
|
|
|
11
12
|
# Connection settings
|
|
12
13
|
attr_accessor :database_url, :connection_params, :pool_size, :pool_timeout
|
|
13
14
|
|
|
14
15
|
# Queue settings
|
|
15
|
-
attr_accessor :default_queue
|
|
16
|
+
attr_accessor :default_queue
|
|
17
|
+
attr_reader :queue_prefix # rubocop:disable Style/AccessorGrouping -- the writer normalizes (see queue_prefix=)
|
|
16
18
|
|
|
17
19
|
# Worker settings
|
|
18
20
|
attr_accessor :polling_interval, :prefetch_limit, :execution_mode
|
|
@@ -215,10 +217,16 @@ module Pgbus
|
|
|
215
217
|
|
|
216
218
|
# Web dashboard
|
|
217
219
|
attr_accessor :web_auth, :web_refresh_interval, :web_per_page, :web_live_updates, :web_data_source,
|
|
218
|
-
:insights_default_minutes, :base_controller_class, :return_to_app_url,
|
|
220
|
+
:insights_default_minutes, :base_controller_class, :return_to_app_url, :web_actor,
|
|
219
221
|
:metrics_enabled,
|
|
220
222
|
:web_filter_parameters, :web_filter_sensitive
|
|
221
223
|
|
|
224
|
+
# The MCP server's workflow write tools (issue #550): pgbus_workflow_resume,
|
|
225
|
+
# _cancel, _skip_step, _resume_at, _run_now and _signal. Off by default, and
|
|
226
|
+
# then they are never built, so tools/list cannot show them. Read once, when
|
|
227
|
+
# Pgbus::MCP::Server.build runs (pgbus mcp, Pgbus::MCP.rack_app).
|
|
228
|
+
attr_accessor :mcp_write_tools
|
|
229
|
+
|
|
222
230
|
# HTTP health endpoints (liveness/readiness for orchestrators like
|
|
223
231
|
# Kubernetes). `health_port` nil (default) leaves the standalone server in
|
|
224
232
|
# the supervisor disabled — mount Pgbus::Web::HealthApp in the host Rails
|
|
@@ -365,6 +373,7 @@ module Pgbus
|
|
|
365
373
|
@recurring_enabled = true
|
|
366
374
|
@recurring_execution_retention = 7 * 24 * 3600 # 7 days
|
|
367
375
|
initialize_deploy_settings
|
|
376
|
+
initialize_workflow_settings
|
|
368
377
|
|
|
369
378
|
@zombie_detection = true
|
|
370
379
|
|
|
@@ -380,7 +389,13 @@ module Pgbus
|
|
|
380
389
|
@require_primary = false
|
|
381
390
|
@connection_guc_mode = :options
|
|
382
391
|
|
|
392
|
+
@mcp_write_tools = false
|
|
393
|
+
|
|
383
394
|
@web_auth = nil
|
|
395
|
+
# ->(request) { String or nil }: who an operator action on the dashboard
|
|
396
|
+
# is recorded as ("dashboard:<name>"; nil → "dashboard"). web_auth only
|
|
397
|
+
# answers allow/deny, so the name comes from here.
|
|
398
|
+
@web_actor = nil
|
|
384
399
|
@web_refresh_interval = 5000
|
|
385
400
|
@web_per_page = 25
|
|
386
401
|
@web_live_updates = true
|
|
@@ -583,6 +598,25 @@ module Pgbus
|
|
|
583
598
|
@finalized = false
|
|
584
599
|
end
|
|
585
600
|
|
|
601
|
+
# Stored the way the queue tables spell it ("my-app" → "my_app"). Table
|
|
602
|
+
# names are unchanged (queue_name normalizes the joined name the same
|
|
603
|
+
# way), but code that strips the prefix off a table name to get its queue
|
|
604
|
+
# back (the dashboard, the worker, a job's uniqueness lock queue) then
|
|
605
|
+
# finds it. A value normalize cannot spell (nothing table-safe left, or
|
|
606
|
+
# too long) is kept as given, as before: validate! rejects one that is
|
|
607
|
+
# too long, and queue_name drops one with nothing table-safe in it.
|
|
608
|
+
def queue_prefix=(value)
|
|
609
|
+
@queue_prefix = if value.is_a?(String) && !value.empty?
|
|
610
|
+
begin
|
|
611
|
+
QueueNameValidator.normalize(value)
|
|
612
|
+
rescue ArgumentError
|
|
613
|
+
value
|
|
614
|
+
end
|
|
615
|
+
else
|
|
616
|
+
value
|
|
617
|
+
end
|
|
618
|
+
end
|
|
619
|
+
|
|
586
620
|
def queue_name(name)
|
|
587
621
|
full = "#{queue_prefix}_#{name}"
|
|
588
622
|
QueueNameValidator.normalize(full)
|
|
@@ -941,12 +975,14 @@ module Pgbus
|
|
|
941
975
|
end
|
|
942
976
|
|
|
943
977
|
raise Pgbus::ConfigurationError, "require_primary must be true or false" unless [true, false].include?(require_primary)
|
|
978
|
+
raise Pgbus::ConfigurationError, "mcp_write_tools must be true or false" unless [true, false].include?(mcp_write_tools)
|
|
944
979
|
|
|
945
980
|
validate_job_path_gaps!
|
|
946
981
|
validate_streams!
|
|
947
982
|
validate_metrics_backend!
|
|
948
983
|
validate_fair_share!
|
|
949
984
|
validate_deploy_settings!
|
|
985
|
+
validate_workflow_settings!
|
|
950
986
|
|
|
951
987
|
self
|
|
952
988
|
end
|
|
@@ -986,6 +1022,27 @@ module Pgbus
|
|
|
986
1022
|
raise Pgbus::ConfigurationError, "capsule processes must be a positive Integer (got #{value.inspect})"
|
|
987
1023
|
end
|
|
988
1024
|
|
|
1025
|
+
# A prefix with a leading, trailing or doubled underscore is kept as
|
|
1026
|
+
# written (its tables already exist under it), but queue_name collapses
|
|
1027
|
+
# the separator after it only for a queue name that needs normalizing:
|
|
1028
|
+
# "myapp_" gives myapp__orders yet myapp_orders_handler for
|
|
1029
|
+
# "orders-handler". A job's own release and the dashboard derive the same
|
|
1030
|
+
# lock queue for it and still match, but the reaper cannot resolve that
|
|
1031
|
+
# name back to the table, so a lock orphaned there is never reclaimed and
|
|
1032
|
+
# has to be released by key. Legal, but warned.
|
|
1033
|
+
def warn_queue_prefix_underscores
|
|
1034
|
+
return unless queue_prefix.match?(/\A_|_\z|__/)
|
|
1035
|
+
|
|
1036
|
+
suggested = queue_prefix.squeeze("_").delete_prefix("_").delete_suffix("_")
|
|
1037
|
+
suggestion = suggested.empty? ? "" : "; prefer #{suggested.inspect} for new deployments"
|
|
1038
|
+
Pgbus.logger.warn do
|
|
1039
|
+
"[Pgbus] queue_prefix #{queue_prefix.inspect} has a leading, trailing or doubled underscore — a queue " \
|
|
1040
|
+
"whose name needs normalizing (\"orders-handler\") gets a table spelled unlike its queue, so the " \
|
|
1041
|
+
"reaper cannot find that table: a uniqueness lock orphaned there (a worker crash before its release) " \
|
|
1042
|
+
"is never reclaimed and has to be released by key#{suggestion}"
|
|
1043
|
+
end
|
|
1044
|
+
end
|
|
1045
|
+
|
|
989
1046
|
# Buffered claims are kept invisible only by the visibility heartbeat.
|
|
990
1047
|
# Without it a message buffered longer than visibility_timeout can be
|
|
991
1048
|
# claimed by another worker while it still waits here. Legal, but warned.
|
|
@@ -1059,11 +1116,14 @@ module Pgbus
|
|
|
1059
1116
|
{ queue_prefix: queue_prefix, default_queue: default_queue }.each do |name, value|
|
|
1060
1117
|
raise Pgbus::ConfigurationError, "#{name} must be a non-empty String" unless value.is_a?(String) && !value.empty?
|
|
1061
1118
|
end
|
|
1119
|
+
warn_queue_prefix_underscores
|
|
1062
1120
|
|
|
1063
1121
|
# web_auth gates the dashboard; a non-callable value silently leaves it
|
|
1064
1122
|
# open (mirror streams_presence_member).
|
|
1065
|
-
|
|
1066
|
-
|
|
1123
|
+
{ web_auth: web_auth, web_actor: web_actor }.each do |name, value|
|
|
1124
|
+
next if value.nil? || value.respond_to?(:call)
|
|
1125
|
+
|
|
1126
|
+
raise Pgbus::ConfigurationError, "#{name} must respond to #call (a Proc/lambda) or be nil"
|
|
1067
1127
|
end
|
|
1068
1128
|
|
|
1069
1129
|
raise Pgbus::ConfigurationError, "error_reporters must be an Array" unless error_reporters.is_a?(Array)
|
|
@@ -30,7 +30,13 @@ module Pgbus
|
|
|
30
30
|
|
|
31
31
|
# error: FailedEventRecorder.last_error's Hash, or nil when no failure
|
|
32
32
|
# was recorded. Returns the merged headers as a JSON string.
|
|
33
|
-
|
|
33
|
+
# source_msg_id / source_enqueued_at: the message's id in source_queue and
|
|
34
|
+
# when it was sent (the copy gets a new id and enqueued_at in the DLQ).
|
|
35
|
+
# A discard of the copy uses both to release only the uniqueness lock the
|
|
36
|
+
# original still holds: bound to it, or unbound and acquired before its
|
|
37
|
+
# send, never a lock it handed to a retry after (issue #564).
|
|
38
|
+
def build(existing:, reason:, source:, source_queue:, attempts:, max_retries:, error: nil, source_msg_id: nil,
|
|
39
|
+
source_enqueued_at: nil, now: Time.now.utc)
|
|
34
40
|
headers = decode(existing, warn: true)
|
|
35
41
|
retries = count(headers.delete(RETRIES_KEY))
|
|
36
42
|
block = {
|
|
@@ -42,6 +48,8 @@ module Pgbus
|
|
|
42
48
|
"max_retries" => max_retries,
|
|
43
49
|
"dead_lettered_at" => iso(now)
|
|
44
50
|
}
|
|
51
|
+
block["source_msg_id"] = source_msg_id if source_msg_id
|
|
52
|
+
block["source_enqueued_at"] = iso(source_enqueued_at) if source_enqueued_at
|
|
45
53
|
block["retries_from_dlq"] = retries if retries.positive?
|
|
46
54
|
block.merge!(error_fields(error)) if error
|
|
47
55
|
JSON.generate(headers.merge(KEY => block))
|
|
@@ -57,6 +57,22 @@ module Pgbus
|
|
|
57
57
|
pgbus_batches
|
|
58
58
|
].freeze
|
|
59
59
|
|
|
60
|
+
# The three tables pgbus:add_workflows creates (issue #540).
|
|
61
|
+
WORKFLOW_TABLES = %w[pgbus_workflows pgbus_workflow_steps pgbus_workflow_events].freeze
|
|
62
|
+
|
|
63
|
+
# The waiting-step columns pgbus:add_workflow_signals adds to
|
|
64
|
+
# pgbus_workflow_steps (issue #541), with the pgbus_workflow_signals table.
|
|
65
|
+
WORKFLOW_WAITING_COLUMNS = %w[batch_id wait_for signal_id waiting_since timeout_at].freeze
|
|
66
|
+
WORKFLOW_WAITING_INDEXES = Ractor.make_shareable(
|
|
67
|
+
[
|
|
68
|
+
%w[pgbus_workflow_steps idx_pgbus_workflow_steps_batch],
|
|
69
|
+
%w[pgbus_workflow_steps idx_pgbus_workflow_steps_timeout],
|
|
70
|
+
%w[pgbus_workflow_signals idx_pgbus_workflow_signals_idempotency],
|
|
71
|
+
%w[pgbus_workflow_signals idx_pgbus_workflow_signals_pending],
|
|
72
|
+
%w[pgbus_workflow_signals idx_pgbus_workflow_signals_claimed]
|
|
73
|
+
]
|
|
74
|
+
)
|
|
75
|
+
|
|
60
76
|
# generator_key → Rails generator name. Passed to Thor's invoke.
|
|
61
77
|
#
|
|
62
78
|
# Note: uniqueness_keys invokes add_uniqueness_keys, which only
|
|
@@ -78,6 +94,8 @@ module Pgbus
|
|
|
78
94
|
add_outbox: "pgbus:add_outbox",
|
|
79
95
|
add_recurring: "pgbus:add_recurring",
|
|
80
96
|
add_deploy_tasks: "pgbus:add_deploy_tasks",
|
|
97
|
+
add_workflows: "pgbus:add_workflows",
|
|
98
|
+
add_workflow_signals: "pgbus:add_workflow_signals",
|
|
81
99
|
add_failed_events_index: "pgbus:add_failed_events_index",
|
|
82
100
|
add_processed_event_completion: "pgbus:add_processed_event_completion",
|
|
83
101
|
add_batch_executions: "pgbus:add_batch_executions",
|
|
@@ -101,6 +119,8 @@ module Pgbus
|
|
|
101
119
|
add_outbox: "outbox entries table (transactional outbox)",
|
|
102
120
|
add_recurring: "recurring tasks + executions tables",
|
|
103
121
|
add_deploy_tasks: "deploy tasks + runs tables (one-time jobs run after a rollover)",
|
|
122
|
+
add_workflows: "workflow tables (durable multi-step workflows: runs, step attempts, audit trail)",
|
|
123
|
+
add_workflow_signals: "waiting workflow steps (fan_out, wait_for: signals, wait timeouts)",
|
|
104
124
|
add_failed_events_index: "unique index on pgbus_failed_events (queue_name, msg_id)",
|
|
105
125
|
add_processed_event_completion: "completed_at on pgbus_processed_events (two-phase idempotency claim)",
|
|
106
126
|
add_batch_executions: "batch execution-row tracking (self-healing completion, on_failure rename)",
|
|
@@ -130,6 +150,7 @@ module Pgbus
|
|
|
130
150
|
*outbox_migrations,
|
|
131
151
|
*recurring_migrations,
|
|
132
152
|
*deploy_tasks_migrations,
|
|
153
|
+
*workflows_migrations,
|
|
133
154
|
*failed_events_index_migrations,
|
|
134
155
|
*processed_event_completion_migrations,
|
|
135
156
|
*batch_executions_migrations,
|
|
@@ -226,6 +247,25 @@ module Pgbus
|
|
|
226
247
|
[:add_deploy_tasks]
|
|
227
248
|
end
|
|
228
249
|
|
|
250
|
+
# The add_workflows template uses if_not_exists throughout, so a
|
|
251
|
+
# half-present set of the three tables is safe to queue again.
|
|
252
|
+
#
|
|
253
|
+
# add_workflows carries the waiting-step schema too, so a database with
|
|
254
|
+
# no steps table needs only that one; a steps table without the columns
|
|
255
|
+
# (or a missing signals table) predates waiting steps.
|
|
256
|
+
def workflows_migrations
|
|
257
|
+
migrations = []
|
|
258
|
+
migrations << :add_workflows unless WORKFLOW_TABLES.all? { |table| table_exists?(table) }
|
|
259
|
+
return migrations unless table_exists?("pgbus_workflow_steps")
|
|
260
|
+
|
|
261
|
+
columns = column_names("pgbus_workflow_steps")
|
|
262
|
+
waiting = WORKFLOW_WAITING_COLUMNS.all? { |column| columns.include?(column) } &&
|
|
263
|
+
table_exists?("pgbus_workflow_signals") &&
|
|
264
|
+
WORKFLOW_WAITING_INDEXES.all? { |table, index| index_exists?(table, index) }
|
|
265
|
+
migrations << :add_workflow_signals unless waiting
|
|
266
|
+
migrations
|
|
267
|
+
end
|
|
268
|
+
|
|
229
269
|
def failed_events_index_migrations
|
|
230
270
|
# pgbus_failed_events is created by pgbus:install, but the
|
|
231
271
|
# unique (queue_name, msg_id) index was added later via its own
|
|
@@ -54,6 +54,39 @@ module Pgbus
|
|
|
54
54
|
# pgbus.batch_sweep — dispatcher stalled-batch sweep
|
|
55
55
|
# payload: stalled_for, stale_executions, orphan_rows,
|
|
56
56
|
# started_batches, finished_batches
|
|
57
|
+
# pgbus.workflow.step — a workflow step attempt ended (body, guard, stall)
|
|
58
|
+
# payload: workflow_class, workflow_id, step_name, attempt,
|
|
59
|
+
# outcome (completed | reattempted | skipped | paused |
|
|
60
|
+
# cancelled | finished_early), status (the step row's),
|
|
61
|
+
# error_class (nil without an error), duration (seconds
|
|
62
|
+
# the body ran; 0.0 for a guard; nil for a stall),
|
|
63
|
+
# pickup_latency (seconds from scheduled_at to the claim)
|
|
64
|
+
# pgbus.workflow_finished — a workflow run finished
|
|
65
|
+
# payload: workflow_class, workflow_id
|
|
66
|
+
# pgbus.workflow_paused — a workflow run paused (an exception, a stall, pause!)
|
|
67
|
+
# payload: workflow_class, workflow_id, step_name,
|
|
68
|
+
# error_class (nil for a manual pause)
|
|
69
|
+
# pgbus.workflow_cancelled — a workflow run was cancelled
|
|
70
|
+
# payload: workflow_class, workflow_id, step_name
|
|
71
|
+
# pgbus.workflow.stall_detected — a step was stalled on the redelivery of the message that
|
|
72
|
+
# claimed it (detection: "redelivery": its worker died, or
|
|
73
|
+
# its heartbeat lapsed) or after it outlived its stall_at
|
|
74
|
+
# (detection: "sweep")
|
|
75
|
+
# payload: workflow_class, workflow_id, step_name, attempt,
|
|
76
|
+
# detection
|
|
77
|
+
# pgbus.workflow.signal — signal! stored a signal on a run (or found it stored:
|
|
78
|
+
# a redelivery with the same idempotency key)
|
|
79
|
+
# payload: workflow_class, workflow_id, name, duplicate,
|
|
80
|
+
# woke (true when it woke the step waiting for it)
|
|
81
|
+
# pgbus.workflow.wait_timed_out — a waiting step's wait_timeout: passed; its on_timeout
|
|
82
|
+
# applied (the sweep, or TestHelpers#expire_wait); under
|
|
83
|
+
# notify the step keeps waiting, once per sweep that finds it late
|
|
84
|
+
# payload: workflow_class, workflow_id, step_name, attempt,
|
|
85
|
+
# action (pause! | cancel! | skip! | run | notify)
|
|
86
|
+
# pgbus.workflow.gauge — emitted by each dispatcher workflow sweep
|
|
87
|
+
# payload: statuses ([{ workflow_class:, status:, count: }]),
|
|
88
|
+
# stuck ([{ workflow_class:, reason:, count: }]), zero-filled
|
|
89
|
+
# across every known class; reasons include waiting_overdue
|
|
57
90
|
#
|
|
58
91
|
module Instrumentation
|
|
59
92
|
module_function
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
],
|
|
5
5
|
"dashboard": {
|
|
6
6
|
"title": "Pgbus",
|
|
7
|
-
"description": "PostgreSQL-native job processing and event bus built on PGMQ.\nMonitor job throughput, queue depth, latency, dead letter queues,\nworker processes, event handlers, and stream broadcasts.\n",
|
|
7
|
+
"description": "PostgreSQL-native job processing and event bus built on PGMQ.\nMonitor job throughput, queue depth, latency, dead letter queues,\nworker processes, event handlers, and stream broadcasts.\nWorkflow step outcomes and run times, stuck workflows and late waits.\n",
|
|
8
8
|
"visuals": [
|
|
9
9
|
{
|
|
10
10
|
"type": "timeseries",
|
|
@@ -393,6 +393,162 @@
|
|
|
393
393
|
"tags": []
|
|
394
394
|
}
|
|
395
395
|
]
|
|
396
|
+
},
|
|
397
|
+
{
|
|
398
|
+
"type": "timeseries",
|
|
399
|
+
"display": "LINE",
|
|
400
|
+
"title": "Workflow steps per outcome",
|
|
401
|
+
"description": "Workflow step attempts by class, step and outcome (completed, reattempted, skipped, paused, cancelled, finished_early).",
|
|
402
|
+
"line_label": "%workflow_class% - %step% - %outcome%",
|
|
403
|
+
"format": "number",
|
|
404
|
+
"draw_null_as_zero": true,
|
|
405
|
+
"metrics": [
|
|
406
|
+
{
|
|
407
|
+
"name": "pgbus_workflow_steps_total",
|
|
408
|
+
"fields": [
|
|
409
|
+
{
|
|
410
|
+
"field": "COUNTER"
|
|
411
|
+
}
|
|
412
|
+
],
|
|
413
|
+
"tags": [
|
|
414
|
+
{
|
|
415
|
+
"key": "workflow_class",
|
|
416
|
+
"value": "*"
|
|
417
|
+
},
|
|
418
|
+
{
|
|
419
|
+
"key": "step",
|
|
420
|
+
"value": "*"
|
|
421
|
+
},
|
|
422
|
+
{
|
|
423
|
+
"key": "outcome",
|
|
424
|
+
"value": "*"
|
|
425
|
+
}
|
|
426
|
+
]
|
|
427
|
+
}
|
|
428
|
+
]
|
|
429
|
+
},
|
|
430
|
+
{
|
|
431
|
+
"type": "timeseries",
|
|
432
|
+
"display": "LINE",
|
|
433
|
+
"title": "Workflow step run time (seconds)",
|
|
434
|
+
"description": "Mean and p95 run time of each workflow step's body, in seconds.",
|
|
435
|
+
"line_label": "%workflow_class% - %step% - %outcome%",
|
|
436
|
+
"format": "number",
|
|
437
|
+
"draw_null_as_zero": true,
|
|
438
|
+
"metrics": [
|
|
439
|
+
{
|
|
440
|
+
"name": "pgbus_workflow_step_duration_seconds",
|
|
441
|
+
"fields": [
|
|
442
|
+
{
|
|
443
|
+
"field": "MEAN"
|
|
444
|
+
},
|
|
445
|
+
{
|
|
446
|
+
"field": "P95"
|
|
447
|
+
}
|
|
448
|
+
],
|
|
449
|
+
"tags": [
|
|
450
|
+
{
|
|
451
|
+
"key": "workflow_class",
|
|
452
|
+
"value": "*"
|
|
453
|
+
},
|
|
454
|
+
{
|
|
455
|
+
"key": "step",
|
|
456
|
+
"value": "*"
|
|
457
|
+
},
|
|
458
|
+
{
|
|
459
|
+
"key": "outcome",
|
|
460
|
+
"value": "*"
|
|
461
|
+
}
|
|
462
|
+
]
|
|
463
|
+
}
|
|
464
|
+
]
|
|
465
|
+
},
|
|
466
|
+
{
|
|
467
|
+
"type": "timeseries",
|
|
468
|
+
"display": "LINE",
|
|
469
|
+
"title": "Workflows by status",
|
|
470
|
+
"description": "Workflow runs per class and status, from each dispatcher sweep.",
|
|
471
|
+
"line_label": "%workflow_class% - %status%",
|
|
472
|
+
"format": "number",
|
|
473
|
+
"draw_null_as_zero": true,
|
|
474
|
+
"metrics": [
|
|
475
|
+
{
|
|
476
|
+
"name": "pgbus_workflows",
|
|
477
|
+
"fields": [
|
|
478
|
+
{
|
|
479
|
+
"field": "GAUGE"
|
|
480
|
+
}
|
|
481
|
+
],
|
|
482
|
+
"tags": [
|
|
483
|
+
{
|
|
484
|
+
"key": "workflow_class",
|
|
485
|
+
"value": "*"
|
|
486
|
+
},
|
|
487
|
+
{
|
|
488
|
+
"key": "status",
|
|
489
|
+
"value": "*"
|
|
490
|
+
}
|
|
491
|
+
]
|
|
492
|
+
}
|
|
493
|
+
]
|
|
494
|
+
},
|
|
495
|
+
{
|
|
496
|
+
"type": "timeseries",
|
|
497
|
+
"display": "LINE",
|
|
498
|
+
"title": "Stuck workflows",
|
|
499
|
+
"description": "Ongoing workflow runs that need a person, by class and reason (paused_error, stalled, paused_manual, overdue).",
|
|
500
|
+
"line_label": "%workflow_class% - %reason%",
|
|
501
|
+
"format": "number",
|
|
502
|
+
"draw_null_as_zero": true,
|
|
503
|
+
"metrics": [
|
|
504
|
+
{
|
|
505
|
+
"name": "pgbus_workflows_stuck",
|
|
506
|
+
"fields": [
|
|
507
|
+
{
|
|
508
|
+
"field": "GAUGE"
|
|
509
|
+
}
|
|
510
|
+
],
|
|
511
|
+
"tags": [
|
|
512
|
+
{
|
|
513
|
+
"key": "workflow_class",
|
|
514
|
+
"value": "*"
|
|
515
|
+
},
|
|
516
|
+
{
|
|
517
|
+
"key": "reason",
|
|
518
|
+
"value": "*"
|
|
519
|
+
}
|
|
520
|
+
]
|
|
521
|
+
}
|
|
522
|
+
]
|
|
523
|
+
},
|
|
524
|
+
{
|
|
525
|
+
"type": "timeseries",
|
|
526
|
+
"display": "LINE",
|
|
527
|
+
"title": "Late workflow waits",
|
|
528
|
+
"description": "Deadlines an on_timeout: :notify wait missed, by class and step: the run keeps waiting for its signal.",
|
|
529
|
+
"line_label": "%workflow_class% - %step%",
|
|
530
|
+
"format": "number",
|
|
531
|
+
"draw_null_as_zero": true,
|
|
532
|
+
"metrics": [
|
|
533
|
+
{
|
|
534
|
+
"name": "pgbus_workflow_wait_overdue_total",
|
|
535
|
+
"fields": [
|
|
536
|
+
{
|
|
537
|
+
"field": "COUNTER"
|
|
538
|
+
}
|
|
539
|
+
],
|
|
540
|
+
"tags": [
|
|
541
|
+
{
|
|
542
|
+
"key": "workflow_class",
|
|
543
|
+
"value": "*"
|
|
544
|
+
},
|
|
545
|
+
{
|
|
546
|
+
"key": "step",
|
|
547
|
+
"value": "*"
|
|
548
|
+
}
|
|
549
|
+
]
|
|
550
|
+
}
|
|
551
|
+
]
|
|
396
552
|
}
|
|
397
553
|
]
|
|
398
554
|
}
|
|
@@ -76,7 +76,8 @@ module Pgbus
|
|
|
76
76
|
subscribe("pgbus.outbox.publish") { |event| on_outbox_publish(event) },
|
|
77
77
|
subscribe("pgbus.recurring.enqueue") { |event| on_recurring_enqueue(event) },
|
|
78
78
|
subscribe("pgbus.worker.recycle") { |event| on_worker_recycle(event) },
|
|
79
|
-
subscribe("pgbus.consumer.recycle") { |event| on_consumer_recycle(event) }
|
|
79
|
+
subscribe("pgbus.consumer.recycle") { |event| on_consumer_recycle(event) },
|
|
80
|
+
*Pgbus::Metrics::WorkflowMetrics::EVENTS.map { |name| subscribe(name) { |event| on_workflow(event) } }
|
|
80
81
|
# pgbus.client.pool is deliberately NOT subscribed here — the
|
|
81
82
|
# minutely Probe#track_pool already reports pgbus_pool_size /
|
|
82
83
|
# pgbus_pool_available gauges for AppSignal (see probe.rb).
|
|
@@ -273,6 +274,20 @@ module Pgbus
|
|
|
273
274
|
{ reason: payload[:reason], kind: "consumer" }
|
|
274
275
|
)
|
|
275
276
|
end
|
|
277
|
+
|
|
278
|
+
# ── Workflows (issue #540) ──────────────────────────────────────
|
|
279
|
+
|
|
280
|
+
# The same names and labels as Metrics::Subscriber
|
|
281
|
+
# (Pgbus::Metrics::WorkflowMetrics), as AppSignal custom metrics.
|
|
282
|
+
def on_workflow(event)
|
|
283
|
+
Pgbus::Metrics::WorkflowMetrics.samples(event.name, event.payload).each do |kind, name, value, tags|
|
|
284
|
+
case kind
|
|
285
|
+
when :histogram then ::Appsignal.add_distribution_value(name, value, tags)
|
|
286
|
+
when :counter then ::Appsignal.increment_counter(name, value, tags)
|
|
287
|
+
when :gauge then ::Appsignal.set_gauge(name, value, tags)
|
|
288
|
+
end
|
|
289
|
+
end
|
|
290
|
+
end
|
|
276
291
|
end
|
|
277
292
|
end
|
|
278
293
|
end
|
data/lib/pgbus/mcp/rack_app.rb
CHANGED
|
@@ -19,7 +19,8 @@ module Pgbus
|
|
|
19
19
|
#
|
|
20
20
|
# Security: requests are rejected with 401 unless they carry the configured
|
|
21
21
|
# token (or pass the supplied auth callable). Run it on an internal network
|
|
22
|
-
# / behind your VPN, never internet-exposed.
|
|
22
|
+
# / behind your VPN, never internet-exposed. With config.mcp_write_tools on
|
|
23
|
+
# it also serves the workflow write tools (issue #550): gate it.
|
|
23
24
|
#
|
|
24
25
|
# DNS-rebinding protection: since mcp 0.23 the transport validates the Host
|
|
25
26
|
# header (loopback only, by default) and the Origin header (same-origin
|
|
@@ -123,9 +124,13 @@ module Pgbus
|
|
|
123
124
|
@token.nil? && @auth.nil?
|
|
124
125
|
end
|
|
125
126
|
|
|
127
|
+
# With config.mcp_write_tools on, an unauthenticated mount can change
|
|
128
|
+
# workflow runs. Warned, not refused: a mount inside the app's own
|
|
129
|
+
# authenticated routes is gated where this class cannot see it.
|
|
126
130
|
def warn_unauthenticated!
|
|
127
131
|
Pgbus.logger.warn do
|
|
128
|
-
"
|
|
132
|
+
writes = Server.write_tools? ? " config.mcp_write_tools is on: anyone who reaches it can change workflow runs." : ""
|
|
133
|
+
"[Pgbus::MCP] HTTP diagnostic server mounted without authentication.#{writes} " \
|
|
129
134
|
"Pass token: or auth: to Pgbus::MCP.rack_app, and keep it on an internal network."
|
|
130
135
|
end
|
|
131
136
|
end
|
data/lib/pgbus/mcp/runner.rb
CHANGED
|
@@ -66,7 +66,8 @@ module Pgbus
|
|
|
66
66
|
|
|
67
67
|
def log_start(allow_payloads)
|
|
68
68
|
mode = allow_payloads ? "payloads ALLOWED" : "payloads redacted"
|
|
69
|
-
|
|
69
|
+
kind = Server.write_tools? ? "diagnostic server with the workflow write tools ON (mcp_write_tools)" : "read-only diagnostic server"
|
|
70
|
+
Pgbus.logger.info { "[Pgbus::MCP] starting #{kind} over stdio (#{mode})" }
|
|
70
71
|
end
|
|
71
72
|
end
|
|
72
73
|
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "date"
|
|
4
|
+
require "time"
|
|
5
|
+
require "active_support/core_ext/time"
|
|
6
|
+
|
|
7
|
+
module Pgbus
|
|
8
|
+
module MCP
|
|
9
|
+
# How MCP tools write a data-source row as JSON: times as ISO 8601 UTC,
|
|
10
|
+
# symbols as strings, keys as strings, recursively.
|
|
11
|
+
module Serialization
|
|
12
|
+
module_function
|
|
13
|
+
|
|
14
|
+
def serialize(value)
|
|
15
|
+
case value
|
|
16
|
+
when Hash then value.to_h { |key, nested| [key.to_s, serialize(nested)] }
|
|
17
|
+
when Array then value.map { |nested| serialize(nested) }
|
|
18
|
+
when Symbol then value.to_s
|
|
19
|
+
when Time, DateTime, ActiveSupport::TimeWithZone then value.utc.iso8601
|
|
20
|
+
else value
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|