evt-consumer 2.3.1.3 → 2.4.0.0.dev.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e4020fbe8e266cf3f1d73aca2b434808eca913b81a4fba0b6f3d589107272521
4
- data.tar.gz: c6b343c8f96151a7d81e5adf4fb10e2fef8ef85b3b58906cfde8f36558740ae4
3
+ metadata.gz: 50a848468126bd23736a624d0746d224389adc1ec14f19d8b61657c59b907595
4
+ data.tar.gz: c45ab4bcaaaa9e065e39c9fa95a713c2a0b65d5598d0c8abb6d9f38c0f9b4326
5
5
  SHA512:
6
- metadata.gz: 8c53bc549066aef8ac4302622e8d7de78fd48e08817b88e55dcf30bc6e05f009a99a1a41cd2af4a0a845dc240acf851d5f28df56d6f0b840a8f2c3fb8fce0de6
7
- data.tar.gz: 25de424a409939c05dc8805a6cb197e07c00fa24db2f3bf2eef41bb68b5df8b2ab7c052ec62478ea1f9a21552b9d1a0877e88c1535c4aa3360b335543da541ce
6
+ metadata.gz: a40e03f7c32d10b668644ad66a5107baded5b9fb5f1e6b52bcdb021c147a569a9345dd468c51c0222c0969afa59bd7b6950959c3baf6eec1b51bc3fe5f0a2e00
7
+ data.tar.gz: 85cf95b31370ca71fcbf71759686b4577a7445f821e685e29828b038dac3cc041f433005a0fd6707a2340800faf96ade571865600704398e8e30a4bb98c69908
@@ -23,6 +23,11 @@ module Consumer
23
23
  @identifier ||= self.class.identifier
24
24
  end
25
25
 
26
+ def starting_position
27
+ @starting_position ||= position_store.get
28
+ end
29
+ attr_writer :starting_position
30
+
26
31
  attr_writer :position_update_interval
27
32
  def position_update_interval
28
33
  @position_update_interval ||= Defaults.position_update_interval
@@ -52,7 +57,7 @@ module Consumer
52
57
  end
53
58
 
54
59
  def start(&probe)
55
- logger.info(tags: [:consumer, :start]) { "Starting consumer: #{self.class.name} (Category: #{category}, Identifier: #{identifier || '(none)'}, Position: #{subscription.position})" }
60
+ logger.info(tags: [:consumer, :start]) { "Starting consumer: #{self.class.name} (Category: #{category}, Identifier: #{identifier || '(none)'}, Position: #{starting_position})" }
56
61
 
57
62
  if Defaults.startup_info?
58
63
  print_info
@@ -67,15 +72,12 @@ module Consumer
67
72
 
68
73
  _, subscription_thread = ::Actor::Start.(subscription)
69
74
 
70
- actor_address, actor_thread = Actor.start(self, subscription, include: :thread)
71
-
72
75
  if probe
73
76
  subscription_address = subscription.address
74
-
75
- probe.(self, [actor_thread, subscription_thread], [actor_address, subscription_address])
77
+ probe.(self, subscription_address, subscription_thread)
76
78
  end
77
79
 
78
- logger.info(tags: [:consumer, :start]) { "Started consumer: #{self.class.name} (Category: #{category}, Identifier: #{identifier || '(none)'}, Position: #{subscription.position})" }
80
+ logger.info(tags: [:consumer, :start]) { "Started consumer: #{self.class.name} (Category: #{category}, Identifier: #{identifier || '(none)'}, Position: #{starting_position})" }
79
81
 
80
82
  AsyncInvocation::Incorrect
81
83
  end
@@ -84,7 +86,7 @@ module Consumer
84
86
  STDOUT.puts
85
87
  STDOUT.puts " Consumer: #{self.class.name}"
86
88
  STDOUT.puts " Category: #{category}"
87
- STDOUT.puts " Position: #{subscription.position}"
89
+ STDOUT.puts " Position: #{starting_position}"
88
90
  STDOUT.puts " Identifier: #{identifier || '(none)'}"
89
91
 
90
92
  print_startup_info if respond_to?(:print_startup_info)
@@ -102,7 +104,7 @@ module Consumer
102
104
 
103
105
  def log_info
104
106
  logger.info(tags: [:consumer, :start]) { "Category: #{category} (Consumer: #{self.class.name})" }
105
- logger.info(tags: [:consumer, :start]) { "Position: #{subscription.position} (Consumer: #{self.class.name})" }
107
+ logger.info(tags: [:consumer, :start]) { "Position: #{starting_position} (Consumer: #{self.class.name})" }
106
108
  logger.info(tags: [:consumer, :start]) { "Identifier: #{identifier || 'nil'} (Consumer: #{self.class.name})" }
107
109
 
108
110
  log_startup_info if respond_to?(:log_startup_info)
@@ -160,14 +162,9 @@ module Consumer
160
162
 
161
163
  super(**kwargs)
162
164
 
163
- starting_position = self.position_store.get
164
-
165
- Subscription.configure(
166
- self,
167
- get,
168
- position: starting_position,
169
- poll_interval_milliseconds: poll_interval_milliseconds
170
- )
165
+ get = self.get
166
+ position = self.starting_position
167
+ Subscription.configure(self, get, position: position, poll_interval_milliseconds: poll_interval_milliseconds)
171
168
 
172
169
  logger.debug(tag: :consumer) { "Done configuring (Category: #{category}, Starting Position: #{starting_position})" }
173
170
  end
@@ -1,5 +1,19 @@
1
1
  module Consumer
2
2
  module Controls
3
- Get = MessageStore::Controls::Get
3
+ module Get
4
+ def self.example(batch_size: nil)
5
+ batch_size ||= self.batch_size
6
+
7
+ MessageStore::Controls::Get.example(stream_name: stream_name, batch_size: batch_size)
8
+ end
9
+
10
+ def self.stream_name
11
+ Category.example
12
+ end
13
+
14
+ def self.batch_size
15
+ MessageData::Batch.size
16
+ end
17
+ end
4
18
  end
5
19
  end
@@ -1,6 +1,10 @@
1
1
  module Consumer
2
2
  module Controls
3
3
  module Position
4
+ def self.example
5
+ Global.example
6
+ end
7
+
4
8
  module Stream
5
9
  def self.example
6
10
  MessageStore::Controls::MessageData.position
@@ -1,18 +1,18 @@
1
1
  module Consumer
2
2
  module Controls
3
3
  module Subscription
4
- def self.example(category: nil, next_batch: nil, position: nil, batch_size: nil, count: nil)
5
- get = Get.example(stream_name: category, batch_size: batch_size, count: count)
4
+ def self.example(get: nil, batch_size: nil)
5
+ get ||= Get.example(batch_size: batch_size)
6
6
 
7
- subscription = ::Consumer::Subscription.new(get)
8
-
9
- subscription.position = position if position
7
+ ::Consumer::Subscription.new(get, position)
8
+ end
10
9
 
11
- unless next_batch.nil?
12
- subscription.next_batch = next_batch
13
- end
10
+ def self.get
11
+ Get.example
12
+ end
14
13
 
15
- subscription
14
+ def self.position
15
+ Position::Global.example
16
16
  end
17
17
  end
18
18
  end
@@ -16,13 +16,12 @@ require 'consumer/controls/settings'
16
16
 
17
17
  require 'consumer/controls/position_store'
18
18
  require 'consumer/controls/position_store/file'
19
- require 'consumer/controls/subscription'
20
19
 
21
20
  require 'consumer/controls/handle'
22
21
  require 'consumer/controls/handle/raise_error'
23
22
  require 'consumer/controls/handle/settings'
24
23
 
25
- require 'consumer/controls/actor'
24
+ require 'consumer/controls/subscription'
26
25
 
27
26
  require 'consumer/controls/consumer'
28
27
  require 'consumer/controls/consumer/incrementing'
@@ -1,35 +1,39 @@
1
1
  module Consumer
2
2
  class Subscription
3
3
  include ::Actor
4
- include ::Configure
5
4
  include Dependency
6
5
  include Initializer
7
6
  include Log::Dependency
8
7
 
9
- configure :subscription
8
+ initializer :get, a(:position)
10
9
 
11
- initializer :get
12
-
13
- attr_accessor :next_batch
10
+ dependency :consumer, Consumer
11
+ dependency :poll, Poll
14
12
 
15
- attr_writer :position
16
- def position
17
- @position ||= 0
13
+ attr_writer :prefetch_queue
14
+ def prefetch_queue
15
+ @prefetch_queue ||= []
18
16
  end
19
17
 
20
18
  def batch_size
21
19
  get.batch_size
22
20
  end
23
21
 
24
- dependency :poll, Poll
22
+ def category
23
+ get.category
24
+ end
25
25
 
26
- def self.build(get, position: nil, poll_interval_milliseconds: nil)
26
+ def position
27
+ @position ||= 0
28
+ end
29
+
30
+ def self.build(consumer, get, position: nil, poll_interval_milliseconds: nil)
27
31
  poll_interval_milliseconds ||= Defaults.poll_interval_milliseconds
28
32
  poll_timeout_milliseconds = Defaults.poll_timeout_milliseconds
29
33
 
30
- instance = new(get)
34
+ instance = new(get, position)
31
35
 
32
- instance.position = position
36
+ instance.consumer = consumer
33
37
 
34
38
  Poll.configure(
35
39
  instance,
@@ -41,56 +45,48 @@ module Consumer
41
45
  instance
42
46
  end
43
47
 
48
+ def self.configure(receiver, get, consumer: nil, position: nil, poll_interval_milliseconds: nil, attr_name: nil)
49
+ attr_name ||= :subscription
50
+
51
+ consumer ||= receiver
52
+
53
+ instance = build(consumer, get, position: position, poll_interval_milliseconds: poll_interval_milliseconds)
54
+ receiver.public_send(:"#{attr_name}=", instance)
55
+ instance
56
+ end
57
+
44
58
  handle :start do
45
- :resupply
59
+ :next_batch
46
60
  end
47
61
 
48
- handle :resupply do
49
- logger.trace { "Resupplying (Category: #{get.category}, Position: #{position})" }
62
+ handle :next_batch do
63
+ logger.trace(tag: :actor) { "Getting batch (Position: #{position}, Category: #{category}, Batch Size: #{batch_size})" }
50
64
 
51
65
  batch = poll.() do
52
66
  get.(position)
53
67
  end
54
68
 
55
69
  if batch.nil? || batch.empty?
56
- logger.debug { "Did not resupply; no events available (Stream: #{get.stream_name}, Position: #{position})" }
57
-
58
- :resupply
70
+ logger.debug { "No batch retrieved (Position: #{position}, Category: #{category}, Batch Size: #{batch_size})" }
59
71
  else
60
- self.next_batch = batch
61
-
62
- logger.debug { "Resupplied (Category: #{get.category}, Position: #{position}, Batch Size: #{batch.count})" }
63
- end
64
- end
72
+ logger.debug(tag: :actor) { "Batch retrieved (Position: #{position}, Category: #{category}, Batch Size: #{batch_size})" }
65
73
 
66
- handle :get_batch do |get_batch|
67
- logger.trace { "Batch request received" }
68
-
69
- if next_batch.nil?
70
- logger.debug { "Could not fulfill batch request; deferring" }
71
-
72
- return get_batch
74
+ batch.each do |message|
75
+ dispatch(message)
76
+ end
73
77
  end
74
78
 
75
- batch = reset_next_batch
76
-
77
- reply_message = get_batch.reply_message(batch)
78
-
79
- send.(reply_message, get_batch.reply_address)
80
-
81
- logger.debug { "Batch request fulfilled; resupplying (Batch Size: #{batch.count})" }
82
-
83
- :resupply
79
+ :next_batch
84
80
  end
85
81
 
86
- def reset_next_batch
87
- batch = next_batch
82
+ def dispatch(message)
83
+ logger.trace(tag: :actor) { "Dispatching message (Position: #{position}, Category: #{category})" }
88
84
 
89
- self.next_batch = nil
85
+ consumer.dispatch(message)
90
86
 
91
- self.position = batch.last.global_position + 1
87
+ self.position = message.global_position + 1
92
88
 
93
- batch
89
+ logger.debug(tag: :actor) { "Dispatched message (Position: #{position}, Category: #{category})" }
94
90
  end
95
91
  end
96
92
  end
data/lib/consumer.rb CHANGED
@@ -17,11 +17,8 @@ require 'consumer/position_store'
17
17
  require 'consumer/position_store/substitute'
18
18
  require 'consumer/position_store/telemetry'
19
19
 
20
- require 'consumer/subscription'
21
- require 'consumer/subscription/defaults'
22
- require 'consumer/subscription/get_batch'
23
-
24
20
  require 'consumer/consumer'
25
21
  require 'consumer/substitute'
26
22
 
27
- require 'consumer/actor'
23
+ require 'consumer/subscription'
24
+ require 'consumer/subscription/defaults'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: evt-consumer
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.3.1.3
4
+ version: 2.4.0.0.dev.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - The Eventide Project
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-01-10 00:00:00.000000000 Z
11
+ date: 2024-04-16 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: ntl-actor
@@ -101,10 +101,8 @@ extensions: []
101
101
  extra_rdoc_files: []
102
102
  files:
103
103
  - lib/consumer.rb
104
- - lib/consumer/actor.rb
105
104
  - lib/consumer/consumer.rb
106
105
  - lib/consumer/controls.rb
107
- - lib/consumer/controls/actor.rb
108
106
  - lib/consumer/controls/category.rb
109
107
  - lib/consumer/controls/consumer.rb
110
108
  - lib/consumer/controls/consumer/error_handler.rb
@@ -135,7 +133,6 @@ files:
135
133
  - lib/consumer/position_store/telemetry.rb
136
134
  - lib/consumer/subscription.rb
137
135
  - lib/consumer/subscription/defaults.rb
138
- - lib/consumer/subscription/get_batch.rb
139
136
  - lib/consumer/substitute.rb
140
137
  homepage: https://github.com/eventide-project/consumer
141
138
  licenses:
@@ -152,11 +149,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
152
149
  version: 2.4.0
153
150
  required_rubygems_version: !ruby/object:Gem::Requirement
154
151
  requirements:
155
- - - ">="
152
+ - - ">"
156
153
  - !ruby/object:Gem::Version
157
- version: '0'
154
+ version: 1.3.1
158
155
  requirements: []
159
- rubygems_version: 3.3.3
156
+ rubygems_version: 3.4.10
160
157
  signing_key:
161
158
  specification_version: 4
162
159
  summary: Continuous subscription to a category and message dispatching to handlers
@@ -1,86 +0,0 @@
1
- module Consumer
2
- class Actor
3
- include ::Actor
4
- include Dependency
5
- include Initializer
6
- include Log::Dependency
7
-
8
- initializer :subscription_address, a(:delay_threshold)
9
-
10
- dependency :consumer, Consumer
11
-
12
- attr_writer :prefetch_queue
13
- def prefetch_queue
14
- @prefetch_queue ||= []
15
- end
16
-
17
- def self.build(consumer, subscription)
18
- subscription_address = subscription.address
19
-
20
- delay_threshold = subscription.batch_size
21
-
22
- instance = new(subscription_address, delay_threshold)
23
- instance.consumer = consumer
24
- instance.configure
25
- instance
26
- end
27
-
28
- handle :start do
29
- request_batch
30
- end
31
-
32
- handle Subscription::GetBatch::Reply do |get_batch_reply|
33
- received_batch = get_batch_reply.batch
34
-
35
- logger.trace(tag: :actor) { "Received batch (Received Batch Size: #{received_batch.size}, Prefetch Queue Depth: #{prefetch_queue.size}, Delay Threshold: #{delay_threshold})" }
36
-
37
- prefetch_queue_empty = prefetch_queue.empty?
38
-
39
- self.prefetch_queue += received_batch
40
-
41
- unless prefetch_queue.size > delay_threshold
42
- request_batch
43
- end
44
-
45
- if prefetch_queue_empty
46
- next_message = :dispatch
47
- end
48
-
49
- logger.debug(tag: :actor) { "Batch received (Received Batch Size: #{received_batch.size}, Prefetch Queue Depth: #{prefetch_queue.size}, Delay Threshold: #{delay_threshold}, Next Actor Message: #{next_message || '(none)'})" }
50
-
51
- next_message
52
- end
53
-
54
- handle :dispatch do
55
- logger.trace(tag: :actor) { "Dispatching Message (Prefetch Queue Depth: #{prefetch_queue.size}, Delay Threshold: #{delay_threshold})" }
56
-
57
- dispatch_message = prefetch_queue.shift
58
-
59
- if prefetch_queue.size == delay_threshold
60
- request_batch
61
- end
62
-
63
- consumer.dispatch(dispatch_message)
64
-
65
- unless prefetch_queue.empty?
66
- next_message = :dispatch
67
- end
68
-
69
- logger.debug(tag: :actor) { "Dispatched Message (Prefetch Queue Depth: #{prefetch_queue.size}, Delay Threshold: #{delay_threshold}, Global Position: #{dispatch_message.global_position}, Next Actor Message: #{next_message || '(none)'})" }
70
-
71
- next_message
72
- end
73
-
74
- def request_batch
75
- logger.trace(tag: :actor) { "Requesting batch (Prefetch Queue Depth: #{prefetch_queue.size}, Delay Threshold: #{delay_threshold})" }
76
-
77
- get_batch = Subscription::GetBatch.new(address)
78
-
79
- send.(get_batch, subscription_address)
80
-
81
- logger.debug(tag: :actor) { "Sent batch request (Prefetch Queue Depth: #{prefetch_queue.size}, Delay Threshold: #{delay_threshold})" }
82
-
83
- nil
84
- end
85
- end
86
- end
@@ -1,24 +0,0 @@
1
- module Consumer
2
- module Controls
3
- module Actor
4
- def self.example(subscription_address: nil, delay_threshold: nil)
5
- delay_threshold ||= DelayThreshold.example
6
- subscription_address ||= Address.example
7
-
8
- ::Consumer::Actor.new(subscription_address, delay_threshold)
9
- end
10
-
11
- module DelayThreshold
12
- def self.example
13
- 11
14
- end
15
- end
16
-
17
- module Address
18
- def self.example
19
- ::Actor::Messaging::Address.build
20
- end
21
- end
22
- end
23
- end
24
- end
@@ -1,17 +0,0 @@
1
- module Consumer
2
- class Subscription
3
- GetBatch = Struct.new(:reply_address)
4
-
5
- class GetBatch
6
- include ::Actor::Messaging::Message
7
-
8
- def reply_message(batch)
9
- Reply.new batch
10
- end
11
-
12
- Reply = Struct.new :batch do
13
- include ::Actor::Messaging::Message
14
- end
15
- end
16
- end
17
- end