jetstream_bridge 2.7.0 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/jetstream_bridge/consumer/consumer.rb +1 -1
- data/lib/jetstream_bridge/consumer/dlq_publisher.rb +2 -2
- data/lib/jetstream_bridge/consumer/inbox/inbox_message.rb +28 -5
- data/lib/jetstream_bridge/consumer/message_processor.rb +3 -7
- data/lib/jetstream_bridge/consumer/subscription_manager.rb +8 -13
- data/lib/jetstream_bridge/core/connection.rb +1 -1
- data/lib/jetstream_bridge/core/duration.rb +10 -5
- data/lib/jetstream_bridge/core/model_codec_setup.rb +3 -1
- data/lib/jetstream_bridge/core/model_utils.rb +9 -4
- data/lib/jetstream_bridge/inbox_event.rb +4 -2
- data/lib/jetstream_bridge/outbox_event.rb +3 -3
- data/lib/jetstream_bridge/publisher/publisher.rb +16 -7
- data/lib/jetstream_bridge/topology/overlap_guard.rb +3 -3
- data/lib/jetstream_bridge/version.rb +1 -1
- data/lib/jetstream_bridge.rb +12 -2
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0fd7228628d4d5308efceec8d1bf60d9dcff0caf311ab6d459c6c9fc32fbea11
|
4
|
+
data.tar.gz: 4fe44a58012328837b727807d5d07aa3f265ff8c23dc852a906d84714eaebbf8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d9343bc42829b93c1279e4bd2a22fafdd77f69a1e18330b9771f37caeb04fac83fab7687b099484e670a8027d139ee5354601485a06d83ac79b1e3bd265b3a29
|
7
|
+
data.tar.gz: aab3b5ae4b601ece51560a46fca18f459f01de29b3731fd545ad08e1a00607e54b03167b484a5aa59e414be448c877d8f909c17640fa00ab2c4301618953c2fe
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require '
|
3
|
+
require 'oj'
|
4
4
|
require 'time'
|
5
5
|
require_relative '../core/logging'
|
6
6
|
|
@@ -46,7 +46,7 @@ module JetstreamBridge
|
|
46
46
|
headers['x-dead-letter'] = 'true'
|
47
47
|
headers['x-dlq-reason'] = reason
|
48
48
|
headers['x-deliveries'] = deliveries.to_s
|
49
|
-
headers['x-dlq-context'] =
|
49
|
+
headers['x-dlq-context'] = Oj.dump(envelope, mode: :compat)
|
50
50
|
headers
|
51
51
|
end
|
52
52
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require '
|
3
|
+
require 'oj'
|
4
4
|
|
5
5
|
module JetstreamBridge
|
6
6
|
# Immutable value object for a single NATS message.
|
@@ -12,6 +12,7 @@ module JetstreamBridge
|
|
12
12
|
seq = meta.respond_to?(:stream_sequence) ? meta.stream_sequence : nil
|
13
13
|
deliveries = meta.respond_to?(:num_delivered) ? meta.num_delivered : nil
|
14
14
|
stream = meta.respond_to?(:stream) ? meta.stream : nil
|
15
|
+
consumer = meta.respond_to?(:consumer) ? meta.consumer : nil
|
15
16
|
subject = msg.subject.to_s
|
16
17
|
|
17
18
|
headers = {}
|
@@ -19,18 +20,18 @@ module JetstreamBridge
|
|
19
20
|
|
20
21
|
raw = msg.data
|
21
22
|
body = begin
|
22
|
-
|
23
|
-
rescue
|
23
|
+
Oj.load(raw, mode: :strict)
|
24
|
+
rescue Oj::Error
|
24
25
|
{}
|
25
26
|
end
|
26
27
|
|
27
28
|
id = (headers['nats-msg-id'] || body['event_id']).to_s.strip
|
28
29
|
id = "seq:#{seq}" if id.empty?
|
29
30
|
|
30
|
-
new(msg, seq, deliveries, stream, subject, headers, body, raw, id, Time.now.utc)
|
31
|
+
new(msg, seq, deliveries, stream, subject, headers, body, raw, id, Time.now.utc, consumer)
|
31
32
|
end
|
32
33
|
|
33
|
-
def initialize(msg, seq, deliveries, stream, subject, headers, body, raw, event_id, now)
|
34
|
+
def initialize(msg, seq, deliveries, stream, subject, headers, body, raw, event_id, now, consumer = nil)
|
34
35
|
@msg = msg
|
35
36
|
@seq = seq
|
36
37
|
@deliveries = deliveries
|
@@ -41,10 +42,32 @@ module JetstreamBridge
|
|
41
42
|
@raw = raw
|
42
43
|
@event_id = event_id
|
43
44
|
@now = now
|
45
|
+
@consumer = consumer
|
44
46
|
end
|
45
47
|
|
46
48
|
def body_for_store
|
47
49
|
body.empty? ? raw : body
|
48
50
|
end
|
51
|
+
|
52
|
+
def data
|
53
|
+
raw
|
54
|
+
end
|
55
|
+
|
56
|
+
def header
|
57
|
+
headers
|
58
|
+
end
|
59
|
+
|
60
|
+
def metadata
|
61
|
+
@metadata ||= Struct.new(:num_delivered, :sequence, :consumer, :stream)
|
62
|
+
.new(deliveries, seq, @consumer, stream)
|
63
|
+
end
|
64
|
+
|
65
|
+
def ack(*args, **kwargs)
|
66
|
+
msg.ack(*args, **kwargs) if msg.respond_to?(:ack)
|
67
|
+
end
|
68
|
+
|
69
|
+
def nak(*args, **kwargs)
|
70
|
+
msg.nak(*args, **kwargs) if msg.respond_to?(:nak)
|
71
|
+
end
|
49
72
|
end
|
50
73
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require '
|
3
|
+
require 'oj'
|
4
4
|
require_relative '../core/logging'
|
5
5
|
require_relative 'message_context'
|
6
6
|
require_relative 'dlq_publisher'
|
@@ -37,12 +37,8 @@ module JetstreamBridge
|
|
37
37
|
|
38
38
|
def parse_message(msg, ctx)
|
39
39
|
data = msg.data
|
40
|
-
|
41
|
-
|
42
|
-
else
|
43
|
-
JSON.parse(data)
|
44
|
-
end
|
45
|
-
rescue JSON::ParserError => e
|
40
|
+
Oj.load(data, mode: :strict)
|
41
|
+
rescue Oj::ParseError => e
|
46
42
|
@dlq.publish(msg, ctx,
|
47
43
|
reason: 'malformed_json', error_class: e.class.name, error_message: e.message)
|
48
44
|
msg.ack
|
@@ -11,6 +11,8 @@ module JetstreamBridge
|
|
11
11
|
@jts = jts
|
12
12
|
@durable = durable
|
13
13
|
@cfg = cfg
|
14
|
+
@desired_cfg = ConsumerConfig.consumer_config(@durable, filter_subject)
|
15
|
+
@desired_cfg_norm = normalize_consumer_config(@desired_cfg)
|
14
16
|
end
|
15
17
|
|
16
18
|
def stream_name
|
@@ -22,18 +24,18 @@ module JetstreamBridge
|
|
22
24
|
end
|
23
25
|
|
24
26
|
def desired_consumer_cfg
|
25
|
-
|
27
|
+
@desired_cfg
|
26
28
|
end
|
27
29
|
|
28
30
|
def ensure_consumer!
|
29
31
|
info = consumer_info_or_nil
|
30
32
|
return create_consumer! unless info
|
31
33
|
|
32
|
-
|
33
|
-
if
|
34
|
+
have_norm = normalize_consumer_config(info.config)
|
35
|
+
if have_norm == @desired_cfg_norm
|
34
36
|
log_consumer_ok
|
35
37
|
else
|
36
|
-
log_consumer_diff(
|
38
|
+
log_consumer_diff(have_norm)
|
37
39
|
recreate_consumer!
|
38
40
|
end
|
39
41
|
end
|
@@ -58,15 +60,8 @@ module JetstreamBridge
|
|
58
60
|
|
59
61
|
# ---- comparison ----
|
60
62
|
|
61
|
-
def
|
62
|
-
|
63
|
-
want_norm = normalize_consumer_config(want)
|
64
|
-
have_norm == want_norm
|
65
|
-
end
|
66
|
-
|
67
|
-
def log_consumer_diff(info, want)
|
68
|
-
have_norm = normalize_consumer_config(info.config)
|
69
|
-
want_norm = normalize_consumer_config(want)
|
63
|
+
def log_consumer_diff(have_norm)
|
64
|
+
want_norm = @desired_cfg_norm
|
70
65
|
|
71
66
|
diffs = {}
|
72
67
|
(have_norm.keys | want_norm.keys).each do |k|
|
@@ -14,6 +14,7 @@ module JetstreamBridge
|
|
14
14
|
# Examples:
|
15
15
|
# Duration.to_millis(30) #=> 30000 (auto)
|
16
16
|
# Duration.to_millis(1500) #=> 1500 (auto)
|
17
|
+
# Duration.to_millis("1500") #=> 1500 (auto)
|
17
18
|
# Duration.to_millis(1500, default_unit: :s) #=> 1_500_000
|
18
19
|
# Duration.to_millis("30s") #=> 30000
|
19
20
|
# Duration.to_millis("500ms") #=> 500
|
@@ -48,7 +49,7 @@ module JetstreamBridge
|
|
48
49
|
case val
|
49
50
|
when Integer then int_to_ms(val, default_unit: default_unit)
|
50
51
|
when Float then float_to_ms(val, default_unit: default_unit)
|
51
|
-
when String then string_to_ms(val, default_unit: default_unit
|
52
|
+
when String then string_to_ms(val, default_unit: default_unit)
|
52
53
|
else
|
53
54
|
raise ArgumentError, "invalid duration type: #{val.class}" unless val.respond_to?(:to_f)
|
54
55
|
|
@@ -59,7 +60,10 @@ module JetstreamBridge
|
|
59
60
|
|
60
61
|
# Normalize an array of durations into integer milliseconds.
|
61
62
|
def normalize_list_to_millis(values, default_unit: :auto)
|
62
|
-
Array(values)
|
63
|
+
vals = Array(values)
|
64
|
+
return [] if vals.empty?
|
65
|
+
|
66
|
+
vals.map { |v| to_millis(v, default_unit: default_unit) }
|
63
67
|
end
|
64
68
|
|
65
69
|
# --- internal helpers ---
|
@@ -70,7 +74,7 @@ module JetstreamBridge
|
|
70
74
|
# Preserve existing heuristic for compatibility
|
71
75
|
num >= 1_000 ? num : num * 1_000
|
72
76
|
else
|
73
|
-
coerce_numeric_to_ms(
|
77
|
+
coerce_numeric_to_ms(num.to_f, default_unit)
|
74
78
|
end
|
75
79
|
end
|
76
80
|
|
@@ -80,8 +84,9 @@ module JetstreamBridge
|
|
80
84
|
|
81
85
|
def string_to_ms(str, default_unit:)
|
82
86
|
s = str.strip
|
83
|
-
# Plain number
|
84
|
-
|
87
|
+
# Plain number strings are treated like integers so the :auto
|
88
|
+
# heuristic still applies (<1000 => seconds, >=1000 => ms).
|
89
|
+
return int_to_ms(s.delete('_').to_i, default_unit: default_unit) if NUMBER_RE.match?(s)
|
85
90
|
|
86
91
|
m = TOKEN_RE.match(s)
|
87
92
|
raise ArgumentError, "invalid duration: #{str.inspect}" unless m
|
@@ -1,5 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'oj'
|
4
|
+
|
3
5
|
module JetstreamBridge
|
4
6
|
module ModelCodecSetup
|
5
7
|
module_function
|
@@ -17,7 +19,7 @@ module JetstreamBridge
|
|
17
19
|
next unless column?(klass, attr)
|
18
20
|
next if json_column?(klass, attr) || already_serialized?(klass, attr)
|
19
21
|
|
20
|
-
klass.serialize attr.to_sym, coder:
|
22
|
+
klass.serialize attr.to_sym, coder: Oj
|
21
23
|
end
|
22
24
|
rescue ActiveRecord::StatementInvalid, ActiveRecord::ConnectionNotEstablished, ActiveRecord::NoDatabaseError
|
23
25
|
# ignore when schema isn’t available yet
|
@@ -1,5 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'oj'
|
4
|
+
|
3
5
|
module JetstreamBridge
|
4
6
|
module ModelUtils
|
5
7
|
module_function
|
@@ -36,15 +38,18 @@ module JetstreamBridge
|
|
36
38
|
end
|
37
39
|
|
38
40
|
def json_dump(obj)
|
39
|
-
obj.is_a?(String)
|
40
|
-
|
41
|
+
return obj if obj.is_a?(String)
|
42
|
+
|
43
|
+
Oj.dump(obj, mode: :compat)
|
44
|
+
rescue Oj::Error, TypeError
|
41
45
|
obj.to_s
|
42
46
|
end
|
43
47
|
|
44
48
|
def json_load(str)
|
45
49
|
return str if str.is_a?(Hash)
|
46
|
-
|
47
|
-
|
50
|
+
|
51
|
+
Oj.load(str.to_s, mode: :strict)
|
52
|
+
rescue Oj::Error
|
48
53
|
{}
|
49
54
|
end
|
50
55
|
end
|
@@ -1,5 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'oj'
|
4
|
+
|
3
5
|
begin
|
4
6
|
require 'active_record'
|
5
7
|
rescue LoadError
|
@@ -84,8 +86,8 @@ module JetstreamBridge
|
|
84
86
|
v = self[:payload]
|
85
87
|
case v
|
86
88
|
when String then begin
|
87
|
-
|
88
|
-
rescue
|
89
|
+
Oj.load(v, mode: :strict)
|
90
|
+
rescue Oj::Error
|
89
91
|
{}
|
90
92
|
end
|
91
93
|
when Hash then v
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require '
|
3
|
+
require 'oj'
|
4
4
|
|
5
5
|
begin
|
6
6
|
require 'active_record'
|
@@ -93,8 +93,8 @@ module JetstreamBridge
|
|
93
93
|
v = self[:payload]
|
94
94
|
case v
|
95
95
|
when String then begin
|
96
|
-
|
97
|
-
rescue
|
96
|
+
Oj.load(v, mode: :strict)
|
97
|
+
rescue Oj::Error
|
98
98
|
{}
|
99
99
|
end
|
100
100
|
when Hash then v
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require '
|
3
|
+
require 'oj'
|
4
4
|
require 'securerandom'
|
5
5
|
require_relative '../core/connection'
|
6
6
|
require_relative '../core/logging'
|
@@ -50,12 +50,21 @@ module JetstreamBridge
|
|
50
50
|
def do_publish?(subject, envelope)
|
51
51
|
headers = { 'nats-msg-id' => envelope['event_id'] }
|
52
52
|
|
53
|
-
@jts.publish(subject,
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
53
|
+
ack = @jts.publish(subject, Oj.dump(envelope, mode: :compat), header: headers)
|
54
|
+
duplicate = ack.respond_to?(:duplicate?) && ack.duplicate?
|
55
|
+
msg = "Published #{subject} event_id=#{envelope['event_id']}"
|
56
|
+
msg += ' (duplicate)' if duplicate
|
57
|
+
|
58
|
+
Logging.info(msg, tag: 'JetstreamBridge::Publisher')
|
59
|
+
|
60
|
+
if ack.respond_to?(:error) && ack.error
|
61
|
+
Logging.error(
|
62
|
+
"Publish ack error: #{ack.error}",
|
63
|
+
tag: 'JetstreamBridge::Publisher'
|
64
|
+
)
|
65
|
+
end
|
66
|
+
|
67
|
+
!ack.respond_to?(:error) || ack.error.nil?
|
59
68
|
end
|
60
69
|
|
61
70
|
# ---- Outbox path ----
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require '
|
3
|
+
require 'oj'
|
4
4
|
require_relative 'subject_matcher'
|
5
5
|
require_relative '../core/logging'
|
6
6
|
|
@@ -68,8 +68,8 @@ module JetstreamBridge
|
|
68
68
|
|
69
69
|
def js_api_request(jts, subject, payload = {})
|
70
70
|
# JetStream client should expose the underlying NATS client as `nc`
|
71
|
-
msg = jts.nc.request(subject,
|
72
|
-
|
71
|
+
msg = jts.nc.request(subject, Oj.dump(payload, mode: :compat))
|
72
|
+
Oj.load(msg.data, mode: :strict)
|
73
73
|
end
|
74
74
|
|
75
75
|
def conflict_message(target, conflicts)
|
data/lib/jetstream_bridge.rb
CHANGED
@@ -46,9 +46,19 @@ module JetstreamBridge
|
|
46
46
|
config.use_dlq
|
47
47
|
end
|
48
48
|
|
49
|
-
|
49
|
+
# Establishes a connection and ensures stream topology.
|
50
|
+
#
|
51
|
+
# @return [Object] JetStream context
|
52
|
+
def ensure_topology!
|
50
53
|
Connection.connect!
|
51
|
-
|
54
|
+
Connection.jetstream
|
55
|
+
end
|
56
|
+
|
57
|
+
# @deprecated Use {ensure_topology!} instead. This method will be removed
|
58
|
+
# in a future version.
|
59
|
+
def ensure_topology?
|
60
|
+
Logging.warn('ensure_topology? is deprecated; use ensure_topology! instead', tag: 'JetstreamBridge')
|
61
|
+
!!ensure_topology!
|
52
62
|
end
|
53
63
|
|
54
64
|
private
|