nats_pubsub 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/exe/nats_pubsub +44 -0
- data/lib/generators/nats_pubsub/config/config_generator.rb +174 -0
- data/lib/generators/nats_pubsub/config/templates/env.example.tt +46 -0
- data/lib/generators/nats_pubsub/config/templates/nats_pubsub.rb.tt +105 -0
- data/lib/generators/nats_pubsub/initializer/initializer_generator.rb +36 -0
- data/lib/generators/nats_pubsub/initializer/templates/nats_pubsub.rb +27 -0
- data/lib/generators/nats_pubsub/install/install_generator.rb +75 -0
- data/lib/generators/nats_pubsub/migrations/migrations_generator.rb +74 -0
- data/lib/generators/nats_pubsub/migrations/templates/create_nats_pubsub_inbox.rb.erb +88 -0
- data/lib/generators/nats_pubsub/migrations/templates/create_nats_pubsub_outbox.rb.erb +81 -0
- data/lib/generators/nats_pubsub/subscriber/subscriber_generator.rb +139 -0
- data/lib/generators/nats_pubsub/subscriber/templates/subscriber.rb.tt +117 -0
- data/lib/generators/nats_pubsub/subscriber/templates/subscriber_spec.rb.tt +116 -0
- data/lib/generators/nats_pubsub/subscriber/templates/subscriber_test.rb.tt +117 -0
- data/lib/nats_pubsub/active_record/publishable.rb +192 -0
- data/lib/nats_pubsub/cli.rb +105 -0
- data/lib/nats_pubsub/core/base_repository.rb +73 -0
- data/lib/nats_pubsub/core/config.rb +152 -0
- data/lib/nats_pubsub/core/config_presets.rb +139 -0
- data/lib/nats_pubsub/core/connection.rb +103 -0
- data/lib/nats_pubsub/core/constants.rb +190 -0
- data/lib/nats_pubsub/core/duration.rb +113 -0
- data/lib/nats_pubsub/core/error_action.rb +288 -0
- data/lib/nats_pubsub/core/event.rb +275 -0
- data/lib/nats_pubsub/core/health_check.rb +470 -0
- data/lib/nats_pubsub/core/logging.rb +72 -0
- data/lib/nats_pubsub/core/message_context.rb +193 -0
- data/lib/nats_pubsub/core/presets.rb +222 -0
- data/lib/nats_pubsub/core/retry_strategy.rb +71 -0
- data/lib/nats_pubsub/core/structured_logger.rb +141 -0
- data/lib/nats_pubsub/core/subject.rb +185 -0
- data/lib/nats_pubsub/instrumentation.rb +327 -0
- data/lib/nats_pubsub/middleware/active_record.rb +18 -0
- data/lib/nats_pubsub/middleware/chain.rb +92 -0
- data/lib/nats_pubsub/middleware/logging.rb +48 -0
- data/lib/nats_pubsub/middleware/retry_logger.rb +24 -0
- data/lib/nats_pubsub/middleware/structured_logging.rb +57 -0
- data/lib/nats_pubsub/models/event_model.rb +73 -0
- data/lib/nats_pubsub/models/inbox_event.rb +109 -0
- data/lib/nats_pubsub/models/model_codec_setup.rb +61 -0
- data/lib/nats_pubsub/models/model_utils.rb +57 -0
- data/lib/nats_pubsub/models/outbox_event.rb +113 -0
- data/lib/nats_pubsub/publisher/envelope_builder.rb +99 -0
- data/lib/nats_pubsub/publisher/fluent_batch.rb +262 -0
- data/lib/nats_pubsub/publisher/outbox_publisher.rb +97 -0
- data/lib/nats_pubsub/publisher/outbox_repository.rb +117 -0
- data/lib/nats_pubsub/publisher/publish_argument_parser.rb +108 -0
- data/lib/nats_pubsub/publisher/publish_result.rb +149 -0
- data/lib/nats_pubsub/publisher/publisher.rb +156 -0
- data/lib/nats_pubsub/rails/health_endpoint.rb +239 -0
- data/lib/nats_pubsub/railtie.rb +52 -0
- data/lib/nats_pubsub/subscribers/dlq_handler.rb +69 -0
- data/lib/nats_pubsub/subscribers/error_context.rb +137 -0
- data/lib/nats_pubsub/subscribers/error_handler.rb +110 -0
- data/lib/nats_pubsub/subscribers/graceful_shutdown.rb +128 -0
- data/lib/nats_pubsub/subscribers/inbox/inbox_message.rb +79 -0
- data/lib/nats_pubsub/subscribers/inbox/inbox_processor.rb +53 -0
- data/lib/nats_pubsub/subscribers/inbox/inbox_repository.rb +74 -0
- data/lib/nats_pubsub/subscribers/message_context.rb +86 -0
- data/lib/nats_pubsub/subscribers/message_processor.rb +225 -0
- data/lib/nats_pubsub/subscribers/message_router.rb +77 -0
- data/lib/nats_pubsub/subscribers/pool.rb +166 -0
- data/lib/nats_pubsub/subscribers/registry.rb +114 -0
- data/lib/nats_pubsub/subscribers/subscriber.rb +186 -0
- data/lib/nats_pubsub/subscribers/subscription_manager.rb +206 -0
- data/lib/nats_pubsub/subscribers/worker.rb +152 -0
- data/lib/nats_pubsub/tasks/install.rake +10 -0
- data/lib/nats_pubsub/testing/helpers.rb +199 -0
- data/lib/nats_pubsub/testing/matchers.rb +208 -0
- data/lib/nats_pubsub/testing/test_harness.rb +250 -0
- data/lib/nats_pubsub/testing.rb +157 -0
- data/lib/nats_pubsub/topology/overlap_guard.rb +88 -0
- data/lib/nats_pubsub/topology/stream.rb +102 -0
- data/lib/nats_pubsub/topology/stream_support.rb +170 -0
- data/lib/nats_pubsub/topology/subject_matcher.rb +77 -0
- data/lib/nats_pubsub/topology/topology.rb +24 -0
- data/lib/nats_pubsub/version.rb +8 -0
- data/lib/nats_pubsub/web/views/dashboard.erb +55 -0
- data/lib/nats_pubsub/web/views/inbox_detail.erb +91 -0
- data/lib/nats_pubsub/web/views/inbox_list.erb +62 -0
- data/lib/nats_pubsub/web/views/layout.erb +68 -0
- data/lib/nats_pubsub/web/views/outbox_detail.erb +77 -0
- data/lib/nats_pubsub/web/views/outbox_list.erb +62 -0
- data/lib/nats_pubsub/web.rb +181 -0
- data/lib/nats_pubsub.rb +290 -0
- metadata +225 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: a8b2a1a0ff0af4b6b51d7014811eccb2aa24b6481d4e8a297402c2d9efa088f5
|
|
4
|
+
data.tar.gz: a738e8679fe09033660b70936de917a875d24543815acd0e1035dd3b068741b6
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: dd2c9890448f3d1fd84e812d9210cc801d0f3226a2d69ce2f29ac2811bca0de1426effb90943ec9e2ebd60563c61c6f645624d42b40ef4606e2dc7e272c00427
|
|
7
|
+
data.tar.gz: 3aba338bb2e5db35d118f994901489f746f9ca7743ccc6f8afd615a925ae2895db9e78235e9f010557e37e37803815768f25486333d79bd9c94cedecbacea8b7
|
data/exe/nats_pubsub
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require 'optparse'
|
|
5
|
+
|
|
6
|
+
options = {
|
|
7
|
+
environment: ENV['RAILS_ENV'] || 'development',
|
|
8
|
+
require: nil,
|
|
9
|
+
concurrency: ENV.fetch('CONCURRENCY', 5).to_i,
|
|
10
|
+
verbose: false
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
OptionParser.new do |opts|
|
|
14
|
+
opts.banner = 'Usage: nats_pubsub [options]'
|
|
15
|
+
|
|
16
|
+
opts.on('-r', '--require PATH', 'File to require (e.g., ./config/environment.rb)') do |path|
|
|
17
|
+
options[:require] = path
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
opts.on('-e', '--environment ENV', "Environment (default: #{options[:environment]})") do |env|
|
|
21
|
+
options[:environment] = env
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
opts.on('-c', '--concurrency NUM', Integer, 'Number of consumer threads (default: 5)') do |n|
|
|
25
|
+
options[:concurrency] = n
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
opts.on('-v', '--verbose', 'Verbose logging') do
|
|
29
|
+
options[:verbose] = true
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
opts.on('-h', '--help', 'Show this help') do
|
|
33
|
+
puts opts
|
|
34
|
+
puts "\nExamples:"
|
|
35
|
+
puts ' nats_pubsub -e production -c 10'
|
|
36
|
+
puts ' nats_pubsub -r ./config/environment.rb -c 5'
|
|
37
|
+
exit
|
|
38
|
+
end
|
|
39
|
+
end.parse!
|
|
40
|
+
|
|
41
|
+
require 'nats_pubsub'
|
|
42
|
+
require 'nats_pubsub/cli'
|
|
43
|
+
|
|
44
|
+
NatsPubsub::CLI.new(options).run
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'rails/generators'
|
|
4
|
+
|
|
5
|
+
module NatsPubsub
|
|
6
|
+
module Generators
|
|
7
|
+
# Config generator that updates NatsPubsub configuration
|
|
8
|
+
#
|
|
9
|
+
# Usage:
|
|
10
|
+
# rails generate nats_pubsub:config [options]
|
|
11
|
+
#
|
|
12
|
+
# Examples:
|
|
13
|
+
# rails generate nats_pubsub:config --outbox
|
|
14
|
+
# rails generate nats_pubsub:config --inbox
|
|
15
|
+
# rails generate nats_pubsub:config --outbox --inbox
|
|
16
|
+
# rails generate nats_pubsub:config --concurrency=10
|
|
17
|
+
# rails generate nats_pubsub:config --env-file
|
|
18
|
+
#
|
|
19
|
+
# Options:
|
|
20
|
+
# --outbox Enable outbox pattern
|
|
21
|
+
# --inbox Enable inbox pattern
|
|
22
|
+
# --concurrency=N Set concurrency level (default: 5)
|
|
23
|
+
# --max-deliver=N Set max delivery attempts (default: 5)
|
|
24
|
+
# --ack-wait=DURATION Set ack wait timeout (default: 30s)
|
|
25
|
+
# --env-file Generate .env.example file
|
|
26
|
+
# --force Overwrite existing configuration
|
|
27
|
+
#
|
|
28
|
+
# This will:
|
|
29
|
+
# - Update config/initializers/nats_pubsub.rb
|
|
30
|
+
# - Optionally generate .env.example
|
|
31
|
+
class ConfigGenerator < Rails::Generators::Base
|
|
32
|
+
source_root File.expand_path('templates', __dir__)
|
|
33
|
+
desc 'Updates NatsPubsub configuration'
|
|
34
|
+
|
|
35
|
+
class_option :outbox, type: :boolean, default: false,
|
|
36
|
+
desc: 'Enable outbox pattern'
|
|
37
|
+
class_option :inbox, type: :boolean, default: false,
|
|
38
|
+
desc: 'Enable inbox pattern'
|
|
39
|
+
class_option :concurrency, type: :numeric, default: nil,
|
|
40
|
+
desc: 'Set concurrency level'
|
|
41
|
+
class_option :max_deliver, type: :numeric, default: nil,
|
|
42
|
+
desc: 'Set max delivery attempts'
|
|
43
|
+
class_option :ack_wait, type: :string, default: nil,
|
|
44
|
+
desc: 'Set ack wait timeout (e.g., 30s, 1m)'
|
|
45
|
+
class_option :env_file, type: :boolean, default: false,
|
|
46
|
+
desc: 'Generate .env.example file'
|
|
47
|
+
class_option :force, type: :boolean, default: false,
|
|
48
|
+
desc: 'Overwrite existing configuration'
|
|
49
|
+
|
|
50
|
+
def check_initializer_exists
|
|
51
|
+
@initializer_path = 'config/initializers/nats_pubsub.rb'
|
|
52
|
+
@initializer_exists = File.exist?(File.join(destination_root, @initializer_path))
|
|
53
|
+
|
|
54
|
+
unless @initializer_exists
|
|
55
|
+
say_status :error, 'NatsPubsub initializer not found. Run: rails generate nats_pubsub:initializer', :red
|
|
56
|
+
exit(1) unless options[:force]
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def update_or_create_initializer
|
|
61
|
+
if @initializer_exists && !options[:force]
|
|
62
|
+
update_existing_initializer
|
|
63
|
+
else
|
|
64
|
+
create_new_initializer
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def generate_env_file
|
|
69
|
+
return unless options[:env_file]
|
|
70
|
+
|
|
71
|
+
template 'env.example.tt', '.env.example'
|
|
72
|
+
say_status :created, '.env.example', :green
|
|
73
|
+
rescue StandardError => e
|
|
74
|
+
say_status :error, "Failed to create .env.example: #{e.message}", :red
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def show_instructions
|
|
78
|
+
say "\n"
|
|
79
|
+
say_status :info, 'Configuration updated successfully!', :green
|
|
80
|
+
say "\n"
|
|
81
|
+
|
|
82
|
+
if options[:outbox] || options[:inbox]
|
|
83
|
+
say 'Next steps:', :yellow
|
|
84
|
+
say ' 1. Run migrations if you enabled outbox/inbox:', :yellow
|
|
85
|
+
say ' rails db:migrate', :white
|
|
86
|
+
say "\n"
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
if options[:env_file]
|
|
90
|
+
say ' 2. Copy .env.example to .env and configure:', :yellow
|
|
91
|
+
say ' cp .env.example .env', :white
|
|
92
|
+
say "\n"
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
say ' 3. Restart your Rails server', :yellow
|
|
96
|
+
say "\n"
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
private
|
|
100
|
+
|
|
101
|
+
def update_existing_initializer
|
|
102
|
+
content = File.read(File.join(destination_root, @initializer_path))
|
|
103
|
+
|
|
104
|
+
content = update_outbox_setting(content) if options[:outbox]
|
|
105
|
+
content = update_inbox_setting(content) if options[:inbox]
|
|
106
|
+
content = update_concurrency_setting(content) if options[:concurrency]
|
|
107
|
+
content = update_max_deliver_setting(content) if options[:max_deliver]
|
|
108
|
+
content = update_ack_wait_setting(content) if options[:ack_wait]
|
|
109
|
+
|
|
110
|
+
File.write(File.join(destination_root, @initializer_path), content)
|
|
111
|
+
say_status :updated, @initializer_path, :green
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def create_new_initializer
|
|
115
|
+
template 'nats_pubsub.rb.tt', @initializer_path
|
|
116
|
+
say_status :created, @initializer_path, :green
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def update_outbox_setting(content)
|
|
120
|
+
content.gsub(/config\.use_outbox\s*=\s*\w+/, 'config.use_outbox = true')
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def update_inbox_setting(content)
|
|
124
|
+
content.gsub(/config\.use_inbox\s*=\s*\w+/, 'config.use_inbox = true')
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def update_concurrency_setting(content)
|
|
128
|
+
if content.match?(/config\.concurrency\s*=/)
|
|
129
|
+
content.gsub(/config\.concurrency\s*=\s*\d+/, "config.concurrency = #{options[:concurrency]}")
|
|
130
|
+
else
|
|
131
|
+
# Add concurrency setting after consumer tuning section
|
|
132
|
+
content.gsub(/(# Consumer Tuning\n)/, "\\1 config.concurrency = #{options[:concurrency]}\n")
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def update_max_deliver_setting(content)
|
|
137
|
+
content.gsub(/config\.max_deliver\s*=\s*\d+/, "config.max_deliver = #{options[:max_deliver]}")
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def update_ack_wait_setting(content)
|
|
141
|
+
content.gsub(/config\.ack_wait\s*=\s*['"][^'"]+['"]/, "config.ack_wait = '#{options[:ack_wait]}'")
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# Template helper methods
|
|
145
|
+
def use_outbox?
|
|
146
|
+
options[:outbox]
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def use_inbox?
|
|
150
|
+
options[:inbox]
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def concurrency_value
|
|
154
|
+
options[:concurrency] || 5
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
def max_deliver_value
|
|
158
|
+
options[:max_deliver] || 5
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def ack_wait_value
|
|
162
|
+
options[:ack_wait] || '30s'
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def app_name
|
|
166
|
+
Rails.application.class.module_parent_name.underscore rescue 'app'
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def rails_env
|
|
170
|
+
Rails.env rescue 'development'
|
|
171
|
+
end
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
end
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# NatsPubsub Configuration
|
|
2
|
+
# Copy this file to .env and configure for your environment
|
|
3
|
+
|
|
4
|
+
# ===== NATS Connection =====
|
|
5
|
+
|
|
6
|
+
# NATS server URLs (comma-separated for multiple servers)
|
|
7
|
+
# Examples:
|
|
8
|
+
# Single server: nats://localhost:4222
|
|
9
|
+
# Multiple servers: nats://server1:4222,nats://server2:4222
|
|
10
|
+
# With auth: nats://user:pass@localhost:4222
|
|
11
|
+
NATS_URLS=nats://localhost:4222
|
|
12
|
+
|
|
13
|
+
# Environment prefix for subjects
|
|
14
|
+
# Used to namespace events by environment (development, staging, production)
|
|
15
|
+
NATS_ENV=<%= rails_env %>
|
|
16
|
+
|
|
17
|
+
# Application name
|
|
18
|
+
# Used in subject naming: {env}.{app_name}.{domain}.{resource}.{action}
|
|
19
|
+
APP_NAME=<%= app_name %>
|
|
20
|
+
|
|
21
|
+
# Destination app for cross-app data sync (optional)
|
|
22
|
+
# Required when syncing data between different applications
|
|
23
|
+
# DESTINATION_APP=other_app_name
|
|
24
|
+
|
|
25
|
+
# ===== Connection Pool =====
|
|
26
|
+
|
|
27
|
+
# Number of connections in the pool
|
|
28
|
+
# Increase for high-concurrency workloads
|
|
29
|
+
NATS_POOL_SIZE=5
|
|
30
|
+
|
|
31
|
+
# Timeout for acquiring a connection (seconds)
|
|
32
|
+
NATS_POOL_TIMEOUT=5
|
|
33
|
+
|
|
34
|
+
# ===== Production Settings =====
|
|
35
|
+
|
|
36
|
+
# For production, use these settings:
|
|
37
|
+
|
|
38
|
+
# Multiple NATS servers for high availability
|
|
39
|
+
# NATS_URLS=nats://nats1.example.com:4222,nats://nats2.example.com:4222,nats://nats3.example.com:4222
|
|
40
|
+
|
|
41
|
+
# Production environment
|
|
42
|
+
# NATS_ENV=production
|
|
43
|
+
|
|
44
|
+
# Increase pool size for high traffic
|
|
45
|
+
# NATS_POOL_SIZE=20
|
|
46
|
+
# NATS_POOL_TIMEOUT=10
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# NatsPubsub configuration
|
|
4
|
+
NatsPubsub.configure do |config|
|
|
5
|
+
# ===== Connection Settings =====
|
|
6
|
+
|
|
7
|
+
# NATS server URLs (comma-separated for multiple servers)
|
|
8
|
+
config.nats_urls = ENV.fetch('NATS_URLS', 'nats://localhost:4222')
|
|
9
|
+
|
|
10
|
+
# Environment prefix for subjects (e.g., development, staging, production)
|
|
11
|
+
config.env = ENV.fetch('NATS_ENV', '<%= rails_env %>')
|
|
12
|
+
|
|
13
|
+
# Application name used in subject naming
|
|
14
|
+
config.app_name = ENV.fetch('APP_NAME', '<%= app_name %>')
|
|
15
|
+
|
|
16
|
+
# Destination app for cross-app data sync (optional)
|
|
17
|
+
config.destination_app = ENV.fetch('DESTINATION_APP', nil)
|
|
18
|
+
|
|
19
|
+
# ===== Connection Pool Settings =====
|
|
20
|
+
|
|
21
|
+
# Number of connections in the pool
|
|
22
|
+
config.connection_pool_size = ENV.fetch('NATS_POOL_SIZE', 5).to_i
|
|
23
|
+
|
|
24
|
+
# Timeout for acquiring a connection from the pool (seconds)
|
|
25
|
+
config.connection_pool_timeout = ENV.fetch('NATS_POOL_TIMEOUT', 5).to_i
|
|
26
|
+
|
|
27
|
+
# ===== Consumer Tuning =====
|
|
28
|
+
|
|
29
|
+
# Maximum number of delivery attempts before giving up
|
|
30
|
+
config.max_deliver = <%= max_deliver_value %>
|
|
31
|
+
|
|
32
|
+
# How long to wait for acknowledgment before redelivery
|
|
33
|
+
config.ack_wait = '<%= ack_wait_value %>'
|
|
34
|
+
|
|
35
|
+
# Backoff delays between retry attempts
|
|
36
|
+
config.backoff = %w[1s 5s 15s 30s 60s]
|
|
37
|
+
|
|
38
|
+
# Number of concurrent message processors
|
|
39
|
+
config.concurrency = <%= concurrency_value %>
|
|
40
|
+
|
|
41
|
+
# ===== Reliability Patterns =====
|
|
42
|
+
|
|
43
|
+
# Outbox Pattern: Store events in database before publishing
|
|
44
|
+
# Ensures at-least-once delivery and prevents event loss
|
|
45
|
+
config.use_outbox = <%= use_outbox? %>
|
|
46
|
+
|
|
47
|
+
# Inbox Pattern: Store received events for idempotency
|
|
48
|
+
# Prevents duplicate processing of the same event
|
|
49
|
+
config.use_inbox = <%= use_inbox? %>
|
|
50
|
+
|
|
51
|
+
# Dead Letter Queue: Send failed messages to DLQ after max attempts
|
|
52
|
+
config.use_dlq = true
|
|
53
|
+
config.dlq_max_attempts = 5
|
|
54
|
+
config.dlq_stream_suffix = '-dlq'
|
|
55
|
+
|
|
56
|
+
# ===== Model Configuration =====
|
|
57
|
+
|
|
58
|
+
# ActiveRecord models for outbox and inbox tables
|
|
59
|
+
# Override these if you use custom model names
|
|
60
|
+
config.outbox_model = 'NatsPubsub::OutboxEvent'
|
|
61
|
+
config.inbox_model = 'NatsPubsub::InboxEvent'
|
|
62
|
+
|
|
63
|
+
# ===== Logging =====
|
|
64
|
+
|
|
65
|
+
# Logger instance (defaults to Rails.logger)
|
|
66
|
+
config.logger = Rails.logger
|
|
67
|
+
|
|
68
|
+
# ===== Middleware Configuration =====
|
|
69
|
+
|
|
70
|
+
# Configure server-side middleware for message processing
|
|
71
|
+
# Middleware runs in the order they are added
|
|
72
|
+
#
|
|
73
|
+
# Example:
|
|
74
|
+
# config.server_middleware do |chain|
|
|
75
|
+
# chain.add MyCustomMiddleware, arg1: 'value1'
|
|
76
|
+
# chain.add AnotherMiddleware
|
|
77
|
+
# end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# ===== Important Notes =====
|
|
81
|
+
#
|
|
82
|
+
# 1. Outbox Pattern:
|
|
83
|
+
# - Requires running migrations: rails generate nats_pubsub:migrations
|
|
84
|
+
# - Ensures reliable event publishing even if NATS is down
|
|
85
|
+
# - Background worker publishes events from the outbox table
|
|
86
|
+
#
|
|
87
|
+
# 2. Inbox Pattern:
|
|
88
|
+
# - Requires running migrations: rails generate nats_pubsub:migrations
|
|
89
|
+
# - Prevents duplicate event processing (idempotency)
|
|
90
|
+
# - Useful for critical business operations
|
|
91
|
+
#
|
|
92
|
+
# 3. Environment Variables:
|
|
93
|
+
# - Use .env files or system environment variables
|
|
94
|
+
# - Generate example: rails generate nats_pubsub:config --env-file
|
|
95
|
+
#
|
|
96
|
+
# 4. Production Recommendations:
|
|
97
|
+
# - Enable outbox pattern for critical events
|
|
98
|
+
# - Enable inbox pattern for operations that must be idempotent
|
|
99
|
+
# - Use multiple NATS servers for high availability
|
|
100
|
+
# - Tune concurrency based on your workload
|
|
101
|
+
# - Monitor DLQ for persistent failures
|
|
102
|
+
#
|
|
103
|
+
# 5. Testing:
|
|
104
|
+
# - Use NatsPubsub::Testing.fake! in tests
|
|
105
|
+
# - See: packages/ruby/TESTING_GUIDE.md
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'rails/generators'
|
|
4
|
+
|
|
5
|
+
module NatsPubsub
|
|
6
|
+
module Generators
|
|
7
|
+
# Initializer generator that creates NatsPubsub configuration file
|
|
8
|
+
#
|
|
9
|
+
# Usage:
|
|
10
|
+
# rails generate nats_pubsub:initializer
|
|
11
|
+
#
|
|
12
|
+
# This will create:
|
|
13
|
+
# config/initializers/nats_pubsub.rb
|
|
14
|
+
#
|
|
15
|
+
# The initializer contains configuration for:
|
|
16
|
+
# - NATS server connection
|
|
17
|
+
# - Application settings
|
|
18
|
+
# - JetStream options
|
|
19
|
+
# - Logging configuration
|
|
20
|
+
#
|
|
21
|
+
# Example:
|
|
22
|
+
# rails generate nats_pubsub:initializer
|
|
23
|
+
class InitializerGenerator < Rails::Generators::Base
|
|
24
|
+
source_root File.expand_path('templates', __dir__)
|
|
25
|
+
desc 'Creates NatsPubsub initializer at config/initializers/nats_pubsub.rb'
|
|
26
|
+
|
|
27
|
+
def create_initializer
|
|
28
|
+
template 'nats_pubsub.rb', 'config/initializers/nats_pubsub.rb'
|
|
29
|
+
say_status :created, 'config/initializers/nats_pubsub.rb', :green
|
|
30
|
+
rescue StandardError => e
|
|
31
|
+
say_status :error, "Failed to create initializer: #{e.message}", :red
|
|
32
|
+
raise
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# NatsPubsub configuration
|
|
4
|
+
NatsPubsub.configure do |config|
|
|
5
|
+
# NATS Connection
|
|
6
|
+
config.nats_urls = ENV.fetch('NATS_URLS', 'nats://localhost:4222')
|
|
7
|
+
config.env = ENV.fetch('NATS_ENV', Rails.env)
|
|
8
|
+
config.app_name = ENV.fetch('APP_NAME', Rails.application.class.module_parent_name.underscore)
|
|
9
|
+
config.destination_app = ENV.fetch('DESTINATION_APP', nil) # required for cross-app data sync
|
|
10
|
+
|
|
11
|
+
# Consumer Tuning
|
|
12
|
+
config.max_deliver = 5
|
|
13
|
+
config.ack_wait = '30s'
|
|
14
|
+
config.backoff = %w[1s 5s 15s 30s 60s]
|
|
15
|
+
|
|
16
|
+
# Reliability Features
|
|
17
|
+
config.use_outbox = false
|
|
18
|
+
config.use_inbox = false
|
|
19
|
+
config.use_dlq = true
|
|
20
|
+
|
|
21
|
+
# Models (override if you keep custom AR classes)
|
|
22
|
+
config.outbox_model = 'NatsPubsub::OutboxEvent'
|
|
23
|
+
config.inbox_model = 'NatsPubsub::InboxEvent'
|
|
24
|
+
|
|
25
|
+
# Logging
|
|
26
|
+
# config.logger = Rails.logger
|
|
27
|
+
end
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'rails/generators'
|
|
4
|
+
|
|
5
|
+
module NatsPubsub
|
|
6
|
+
module Generators
|
|
7
|
+
# Install generator that creates NatsPubsub initializer and migrations
|
|
8
|
+
#
|
|
9
|
+
# Usage:
|
|
10
|
+
# rails generate nats_pubsub:install
|
|
11
|
+
#
|
|
12
|
+
# Options:
|
|
13
|
+
# --skip-initializer Skip initializer generation
|
|
14
|
+
# --skip-migrations Skip migration generation
|
|
15
|
+
#
|
|
16
|
+
# Example:
|
|
17
|
+
# rails generate nats_pubsub:install --skip-migrations
|
|
18
|
+
class InstallGenerator < Rails::Generators::Base
|
|
19
|
+
desc 'Creates NatsPubsub initializer and migrations'
|
|
20
|
+
|
|
21
|
+
class_option :skip_initializer, type: :boolean, default: false,
|
|
22
|
+
desc: 'Skip initializer generation'
|
|
23
|
+
class_option :skip_migrations, type: :boolean, default: false,
|
|
24
|
+
desc: 'Skip migration generation'
|
|
25
|
+
|
|
26
|
+
def create_initializer
|
|
27
|
+
return if options[:skip_initializer]
|
|
28
|
+
|
|
29
|
+
say_status :invoke, 'nats_pubsub:initializer', :green
|
|
30
|
+
Rails::Generators.invoke('nats_pubsub:initializer', [],
|
|
31
|
+
behavior: behavior,
|
|
32
|
+
destination_root: destination_root)
|
|
33
|
+
rescue StandardError => e
|
|
34
|
+
say_status :error, "Failed to create initializer: #{e.message}", :red
|
|
35
|
+
raise unless behavior == :revoke
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def create_migrations
|
|
39
|
+
return if options[:skip_migrations]
|
|
40
|
+
|
|
41
|
+
say_status :invoke, 'nats_pubsub:migrations', :green
|
|
42
|
+
Rails::Generators.invoke('nats_pubsub:migrations', [],
|
|
43
|
+
behavior: behavior,
|
|
44
|
+
destination_root: destination_root)
|
|
45
|
+
rescue StandardError => e
|
|
46
|
+
say_status :error, "Failed to create migrations: #{e.message}", :red
|
|
47
|
+
raise unless behavior == :revoke
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def show_next_steps
|
|
51
|
+
say "\n"
|
|
52
|
+
say_status :info, 'NatsPubsub installed successfully!', :green
|
|
53
|
+
say "\n"
|
|
54
|
+
say 'Next steps:', :yellow
|
|
55
|
+
say ' 1. Review and configure:', :white
|
|
56
|
+
say ' config/initializers/nats_pubsub.rb', :white
|
|
57
|
+
say "\n"
|
|
58
|
+
say ' 2. Run migrations (if using outbox/inbox):', :white
|
|
59
|
+
say ' rails db:migrate', :white
|
|
60
|
+
say "\n"
|
|
61
|
+
say ' 3. Create your first subscriber:', :white
|
|
62
|
+
say ' rails generate nats_pubsub:subscriber UserNotification users.user', :white
|
|
63
|
+
say "\n"
|
|
64
|
+
say 'Additional generators:', :yellow
|
|
65
|
+
say ' • rails generate nats_pubsub:subscriber NAME [topics...]', :white
|
|
66
|
+
say ' • rails generate nats_pubsub:config [--outbox] [--inbox]', :white
|
|
67
|
+
say "\n"
|
|
68
|
+
say 'Documentation:', :yellow
|
|
69
|
+
say ' • Testing Guide: packages/ruby/TESTING_GUIDE.md', :white
|
|
70
|
+
say ' • Main README: README.md', :white
|
|
71
|
+
say "\n"
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'rails/generators'
|
|
4
|
+
require 'rails/generators/active_record'
|
|
5
|
+
|
|
6
|
+
module NatsPubsub
|
|
7
|
+
module Generators
|
|
8
|
+
# Migrations generator that creates Inbox and Outbox event tables
|
|
9
|
+
#
|
|
10
|
+
# Usage:
|
|
11
|
+
# rails generate nats_pubsub:migrations
|
|
12
|
+
#
|
|
13
|
+
# This will create:
|
|
14
|
+
# db/migrate/[timestamp]_create_nats_pubsub_outbox.rb
|
|
15
|
+
# db/migrate/[timestamp]_create_nats_pubsub_inbox.rb
|
|
16
|
+
#
|
|
17
|
+
# The outbox table stores events to be published to NATS:
|
|
18
|
+
# - Implements transactional outbox pattern
|
|
19
|
+
# - Tracks publishing status and attempts
|
|
20
|
+
# - Provides at-least-once delivery guarantee
|
|
21
|
+
#
|
|
22
|
+
# The inbox table stores received events from NATS:
|
|
23
|
+
# - Prevents duplicate processing
|
|
24
|
+
# - Tracks processing status
|
|
25
|
+
# - Supports idempotency via event_id
|
|
26
|
+
#
|
|
27
|
+
# Example:
|
|
28
|
+
# rails generate nats_pubsub:migrations
|
|
29
|
+
# rake db:migrate
|
|
30
|
+
class MigrationsGenerator < Rails::Generators::Base
|
|
31
|
+
include Rails::Generators::Migration
|
|
32
|
+
|
|
33
|
+
source_root File.expand_path('templates', __dir__)
|
|
34
|
+
desc 'Creates Inbox/Outbox migrations for NatsPubsub'
|
|
35
|
+
|
|
36
|
+
def create_outbox_migration
|
|
37
|
+
name = 'create_nats_pubsub_outbox'
|
|
38
|
+
return say_status :skip, "migration #{name} already exists", :yellow if migration_exists?('db/migrate', name)
|
|
39
|
+
|
|
40
|
+
migration_template 'create_nats_pubsub_outbox.rb.erb', "db/migrate/#{name}.rb"
|
|
41
|
+
say_status :created, "db/migrate/#{name}.rb", :green
|
|
42
|
+
rescue StandardError => e
|
|
43
|
+
say_status :error, "Failed to create outbox migration: #{e.message}", :red
|
|
44
|
+
raise
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def create_inbox_migration
|
|
48
|
+
name = 'create_nats_pubsub_inbox'
|
|
49
|
+
return say_status :skip, "migration #{name} already exists", :yellow if migration_exists?('db/migrate', name)
|
|
50
|
+
|
|
51
|
+
migration_template 'create_nats_pubsub_inbox.rb.erb', "db/migrate/#{name}.rb"
|
|
52
|
+
say_status :created, "db/migrate/#{name}.rb", :green
|
|
53
|
+
rescue StandardError => e
|
|
54
|
+
say_status :error, "Failed to create inbox migration: #{e.message}", :red
|
|
55
|
+
raise
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# -- Rails::Generators::Migration plumbing --
|
|
59
|
+
def self.next_migration_number(dirname)
|
|
60
|
+
if ActiveRecord::Base.timestamped_migrations
|
|
61
|
+
Time.now.utc.strftime('%Y%m%d%H%M%S')
|
|
62
|
+
else
|
|
63
|
+
format('%.3d', current_migration_number(dirname) + 1)
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
private
|
|
68
|
+
|
|
69
|
+
def migration_exists?(dirname, file_name)
|
|
70
|
+
Dir.glob(File.join(dirname, '[0-9]*_*.rb')).grep(/\d+_#{file_name}\.rb$/).any?
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class CreateNatsPubsubInbox < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
|
4
|
+
# Disable DDL transaction for concurrent index creation
|
|
5
|
+
disable_ddl_transaction!
|
|
6
|
+
|
|
7
|
+
def up
|
|
8
|
+
# Idempotency check - safe to run multiple times
|
|
9
|
+
return if table_exists?(:nats_pubsub_inbox)
|
|
10
|
+
|
|
11
|
+
create_table :nats_pubsub_inbox do |t|
|
|
12
|
+
t.string :event_id # preferred dedupe key
|
|
13
|
+
t.string :subject, null: false
|
|
14
|
+
t.jsonb :payload, null: false, default: {}
|
|
15
|
+
t.jsonb :headers, null: false, default: {}
|
|
16
|
+
t.string :stream
|
|
17
|
+
t.bigint :stream_seq
|
|
18
|
+
t.integer :deliveries
|
|
19
|
+
t.string :status, null: false, default: 'received' # received|processing|processed|failed
|
|
20
|
+
t.text :last_error
|
|
21
|
+
t.datetime :received_at
|
|
22
|
+
t.datetime :processed_at
|
|
23
|
+
t.timestamps
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# Add indexes concurrently to avoid table locks
|
|
27
|
+
# Partial unique index on event_id (when not null)
|
|
28
|
+
add_index :nats_pubsub_inbox, :event_id,
|
|
29
|
+
unique: true,
|
|
30
|
+
where: 'event_id IS NOT NULL',
|
|
31
|
+
algorithm: :concurrently,
|
|
32
|
+
if_not_exists: true
|
|
33
|
+
|
|
34
|
+
# Partial unique index on stream + stream_seq (when not null)
|
|
35
|
+
add_index :nats_pubsub_inbox, [:stream, :stream_seq],
|
|
36
|
+
unique: true,
|
|
37
|
+
where: 'stream IS NOT NULL AND stream_seq IS NOT NULL',
|
|
38
|
+
algorithm: :concurrently,
|
|
39
|
+
if_not_exists: true,
|
|
40
|
+
name: 'index_inbox_on_stream_and_seq'
|
|
41
|
+
|
|
42
|
+
# Index on status for filtering
|
|
43
|
+
add_index :nats_pubsub_inbox, :status,
|
|
44
|
+
algorithm: :concurrently,
|
|
45
|
+
if_not_exists: true
|
|
46
|
+
|
|
47
|
+
# Composite index for common queries (status + received_at)
|
|
48
|
+
add_index :nats_pubsub_inbox, [:status, :received_at],
|
|
49
|
+
algorithm: :concurrently,
|
|
50
|
+
if_not_exists: true,
|
|
51
|
+
name: 'index_inbox_on_status_and_received'
|
|
52
|
+
|
|
53
|
+
# Partial index for failed events
|
|
54
|
+
add_index :nats_pubsub_inbox, [:deliveries, :last_error],
|
|
55
|
+
where: "status = 'failed'",
|
|
56
|
+
algorithm: :concurrently,
|
|
57
|
+
if_not_exists: true,
|
|
58
|
+
name: 'index_inbox_failed_deliveries'
|
|
59
|
+
|
|
60
|
+
# Partial index for processed events (for cleanup)
|
|
61
|
+
add_index :nats_pubsub_inbox, :processed_at,
|
|
62
|
+
where: "status = 'processed'",
|
|
63
|
+
algorithm: :concurrently,
|
|
64
|
+
if_not_exists: true,
|
|
65
|
+
name: 'index_inbox_processed_at'
|
|
66
|
+
|
|
67
|
+
# GIN index for JSONB payload queries (PostgreSQL only)
|
|
68
|
+
if ActiveRecord::Base.connection.adapter_name.downcase.include?('postgres')
|
|
69
|
+
add_index :nats_pubsub_inbox, :payload,
|
|
70
|
+
using: :gin,
|
|
71
|
+
algorithm: :concurrently,
|
|
72
|
+
if_not_exists: true,
|
|
73
|
+
name: 'index_inbox_payload_gin'
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# Database-level constraint for status values
|
|
77
|
+
execute <<-SQL
|
|
78
|
+
ALTER TABLE nats_pubsub_inbox
|
|
79
|
+
ADD CONSTRAINT check_inbox_status_values
|
|
80
|
+
CHECK (status IN ('received', 'processing', 'processed', 'failed'))
|
|
81
|
+
SQL
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def down
|
|
85
|
+
# Safe rollback with checks
|
|
86
|
+
drop_table :nats_pubsub_inbox if table_exists?(:nats_pubsub_inbox)
|
|
87
|
+
end
|
|
88
|
+
end
|