evt-consumer 2.3.1.5 → 2.4.0.0.dev.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e29553138798ba95289cf828a829e0e896a32eee0cce2f8472aefaf42a31c9ce
4
- data.tar.gz: 0c4224924aa28a4ac399261f0798fcea52a8d89208d2c22b3cca6ee4ae44554b
3
+ metadata.gz: 50a848468126bd23736a624d0746d224389adc1ec14f19d8b61657c59b907595
4
+ data.tar.gz: c45ab4bcaaaa9e065e39c9fa95a713c2a0b65d5598d0c8abb6d9f38c0f9b4326
5
5
  SHA512:
6
- metadata.gz: bb8806c27379b594d45685ecb0e7a8247f0be73f1d6b37efd746bcbb06907d6de35f559239a5508f074e5295bd9785673fc0b1384283cfc156d6243c40f668ca
7
- data.tar.gz: 8b7014058b97994de853451068a54544f0228667c8637d48ce7618d9ab172ce0b3518b12f336a0ce9a127be179926312db506389a7509a4eed53a9f1ec794d0d
6
+ metadata.gz: a40e03f7c32d10b668644ad66a5107baded5b9fb5f1e6b52bcdb021c147a569a9345dd468c51c0222c0969afa59bd7b6950959c3baf6eec1b51bc3fe5f0a2e00
7
+ data.tar.gz: 85cf95b31370ca71fcbf71759686b4577a7445f821e685e29828b038dac3cc041f433005a0fd6707a2340800faf96ade571865600704398e8e30a4bb98c69908
@@ -5,7 +5,7 @@ module Consumer
5
5
  cls.class_exec do
6
6
  include Dependency
7
7
  include Initializer
8
- include TemplateMethod
8
+ include Virtual
9
9
  include Log::Dependency
10
10
 
11
11
  extend Build
@@ -23,6 +23,11 @@ module Consumer
23
23
  @identifier ||= self.class.identifier
24
24
  end
25
25
 
26
+ def starting_position
27
+ @starting_position ||= position_store.get
28
+ end
29
+ attr_writer :starting_position
30
+
26
31
  attr_writer :position_update_interval
27
32
  def position_update_interval
28
33
  @position_update_interval ||= Defaults.position_update_interval
@@ -43,7 +48,7 @@ module Consumer
43
48
  dependency :position_store, PositionStore
44
49
  dependency :subscription, Subscription
45
50
 
46
- template_method :error_raised do |error, message_data|
51
+ virtual :error_raised do |error, message_data|
47
52
  raise error
48
53
  end
49
54
 
@@ -52,7 +57,7 @@ module Consumer
52
57
  end
53
58
 
54
59
  def start(&probe)
55
- logger.info(tags: [:consumer, :start]) { "Starting consumer: #{self.class.name} (Category: #{category}, Identifier: #{identifier || '(none)'}, Position: #{subscription.position})" }
60
+ logger.info(tags: [:consumer, :start]) { "Starting consumer: #{self.class.name} (Category: #{category}, Identifier: #{identifier || '(none)'}, Position: #{starting_position})" }
56
61
 
57
62
  if Defaults.startup_info?
58
63
  print_info
@@ -67,15 +72,12 @@ module Consumer
67
72
 
68
73
  _, subscription_thread = ::Actor::Start.(subscription)
69
74
 
70
- actor_address, actor_thread = Actor.start(self, subscription, include: :thread)
71
-
72
75
  if probe
73
76
  subscription_address = subscription.address
74
-
75
- probe.(self, [actor_thread, subscription_thread], [actor_address, subscription_address])
77
+ probe.(self, subscription_address, subscription_thread)
76
78
  end
77
79
 
78
- logger.info(tags: [:consumer, :start]) { "Started consumer: #{self.class.name} (Category: #{category}, Identifier: #{identifier || '(none)'}, Position: #{subscription.position})" }
80
+ logger.info(tags: [:consumer, :start]) { "Started consumer: #{self.class.name} (Category: #{category}, Identifier: #{identifier || '(none)'}, Position: #{starting_position})" }
79
81
 
80
82
  AsyncInvocation::Incorrect
81
83
  end
@@ -84,7 +86,7 @@ module Consumer
84
86
  STDOUT.puts
85
87
  STDOUT.puts " Consumer: #{self.class.name}"
86
88
  STDOUT.puts " Category: #{category}"
87
- STDOUT.puts " Position: #{subscription.position}"
89
+ STDOUT.puts " Position: #{starting_position}"
88
90
  STDOUT.puts " Identifier: #{identifier || '(none)'}"
89
91
 
90
92
  print_startup_info if respond_to?(:print_startup_info)
@@ -102,7 +104,7 @@ module Consumer
102
104
 
103
105
  def log_info
104
106
  logger.info(tags: [:consumer, :start]) { "Category: #{category} (Consumer: #{self.class.name})" }
105
- logger.info(tags: [:consumer, :start]) { "Position: #{subscription.position} (Consumer: #{self.class.name})" }
107
+ logger.info(tags: [:consumer, :start]) { "Position: #{starting_position} (Consumer: #{self.class.name})" }
106
108
  logger.info(tags: [:consumer, :start]) { "Identifier: #{identifier || 'nil'} (Consumer: #{self.class.name})" }
107
109
 
108
110
  log_startup_info if respond_to?(:log_startup_info)
@@ -160,14 +162,9 @@ module Consumer
160
162
 
161
163
  super(**kwargs)
162
164
 
163
- starting_position = self.position_store.get
164
-
165
- Subscription.configure(
166
- self,
167
- get,
168
- position: starting_position,
169
- poll_interval_milliseconds: poll_interval_milliseconds
170
- )
165
+ get = self.get
166
+ position = self.starting_position
167
+ Subscription.configure(self, get, position: position, poll_interval_milliseconds: poll_interval_milliseconds)
171
168
 
172
169
  logger.debug(tag: :consumer) { "Done configuring (Category: #{category}, Starting Position: #{starting_position})" }
173
170
  end
@@ -1,5 +1,19 @@
1
1
  module Consumer
2
2
  module Controls
3
- Get = MessageStore::Controls::Get
3
+ module Get
4
+ def self.example(batch_size: nil)
5
+ batch_size ||= self.batch_size
6
+
7
+ MessageStore::Controls::Get.example(stream_name: stream_name, batch_size: batch_size)
8
+ end
9
+
10
+ def self.stream_name
11
+ Category.example
12
+ end
13
+
14
+ def self.batch_size
15
+ MessageData::Batch.size
16
+ end
17
+ end
4
18
  end
5
19
  end
@@ -1,6 +1,10 @@
1
1
  module Consumer
2
2
  module Controls
3
3
  module Position
4
+ def self.example
5
+ Global.example
6
+ end
7
+
4
8
  module Stream
5
9
  def self.example
6
10
  MessageStore::Controls::MessageData.position
@@ -1,18 +1,18 @@
1
1
  module Consumer
2
2
  module Controls
3
3
  module Subscription
4
- def self.example(category: nil, next_batch: nil, position: nil, batch_size: nil, count: nil)
5
- get = Get.example(stream_name: category, batch_size: batch_size, count: count)
4
+ def self.example(get: nil, batch_size: nil)
5
+ get ||= Get.example(batch_size: batch_size)
6
6
 
7
- subscription = ::Consumer::Subscription.new(get)
8
-
9
- subscription.position = position if position
7
+ ::Consumer::Subscription.new(get, position)
8
+ end
10
9
 
11
- unless next_batch.nil?
12
- subscription.next_batch = next_batch
13
- end
10
+ def self.get
11
+ Get.example
12
+ end
14
13
 
15
- subscription
14
+ def self.position
15
+ Position::Global.example
16
16
  end
17
17
  end
18
18
  end
@@ -16,13 +16,12 @@ require 'consumer/controls/settings'
16
16
 
17
17
  require 'consumer/controls/position_store'
18
18
  require 'consumer/controls/position_store/file'
19
- require 'consumer/controls/subscription'
20
19
 
21
20
  require 'consumer/controls/handle'
22
21
  require 'consumer/controls/handle/raise_error'
23
22
  require 'consumer/controls/handle/settings'
24
23
 
25
- require 'consumer/controls/actor'
24
+ require 'consumer/controls/subscription'
26
25
 
27
26
  require 'consumer/controls/consumer'
28
27
  require 'consumer/controls/consumer/incrementing'
@@ -3,8 +3,7 @@ module Consumer
3
3
  def self.included(cls)
4
4
  cls.class_exec do
5
5
  include Dependency
6
- include TemplateMethod
7
-
6
+ include Virtual
8
7
  include Log::Dependency
9
8
 
10
9
  extend Build
@@ -16,17 +15,16 @@ module Consumer
16
15
 
17
16
  dependency :telemetry, ::Telemetry
18
17
 
19
- template_method :configure
20
- template_method :location
18
+ virtual :location
21
19
  end
22
20
  end
23
21
 
22
+ Virtual::VirtualMethod.define(self, :configure)
23
+
24
24
  module Build
25
25
  def self.extended(cls)
26
26
  cls.singleton_class.class_exec do
27
- include TemplateMethod
28
-
29
- template_method! :build
27
+ Virtual::PureMethod.define(self, :build)
30
28
  end
31
29
  end
32
30
  end
@@ -57,9 +55,7 @@ module Consumer
57
55
 
58
56
  module Get
59
57
  def self.prepended(cls)
60
- cls.class_exec do
61
- template_method! :get
62
- end
58
+ Virtual::PureMethod.define(cls, :get)
63
59
  end
64
60
 
65
61
  def get
@@ -77,9 +73,7 @@ module Consumer
77
73
 
78
74
  module Put
79
75
  def self.prepended(cls)
80
- cls.class_exec do
81
- template_method :put
82
- end
76
+ Virtual::VirtualMethod.define(cls, :put)
83
77
  end
84
78
 
85
79
  def put(position)
@@ -1,35 +1,39 @@
1
1
  module Consumer
2
2
  class Subscription
3
3
  include ::Actor
4
- include ::Configure
5
4
  include Dependency
6
5
  include Initializer
7
6
  include Log::Dependency
8
7
 
9
- configure :subscription
8
+ initializer :get, a(:position)
10
9
 
11
- initializer :get
12
-
13
- attr_accessor :next_batch
10
+ dependency :consumer, Consumer
11
+ dependency :poll, Poll
14
12
 
15
- attr_writer :position
16
- def position
17
- @position ||= 0
13
+ attr_writer :prefetch_queue
14
+ def prefetch_queue
15
+ @prefetch_queue ||= []
18
16
  end
19
17
 
20
18
  def batch_size
21
19
  get.batch_size
22
20
  end
23
21
 
24
- dependency :poll, Poll
22
+ def category
23
+ get.category
24
+ end
25
25
 
26
- def self.build(get, position: nil, poll_interval_milliseconds: nil)
26
+ def position
27
+ @position ||= 0
28
+ end
29
+
30
+ def self.build(consumer, get, position: nil, poll_interval_milliseconds: nil)
27
31
  poll_interval_milliseconds ||= Defaults.poll_interval_milliseconds
28
32
  poll_timeout_milliseconds = Defaults.poll_timeout_milliseconds
29
33
 
30
- instance = new(get)
34
+ instance = new(get, position)
31
35
 
32
- instance.position = position
36
+ instance.consumer = consumer
33
37
 
34
38
  Poll.configure(
35
39
  instance,
@@ -41,56 +45,48 @@ module Consumer
41
45
  instance
42
46
  end
43
47
 
48
+ def self.configure(receiver, get, consumer: nil, position: nil, poll_interval_milliseconds: nil, attr_name: nil)
49
+ attr_name ||= :subscription
50
+
51
+ consumer ||= receiver
52
+
53
+ instance = build(consumer, get, position: position, poll_interval_milliseconds: poll_interval_milliseconds)
54
+ receiver.public_send(:"#{attr_name}=", instance)
55
+ instance
56
+ end
57
+
44
58
  handle :start do
45
- :resupply
59
+ :next_batch
46
60
  end
47
61
 
48
- handle :resupply do
49
- logger.trace { "Resupplying (Category: #{get.category}, Position: #{position})" }
62
+ handle :next_batch do
63
+ logger.trace(tag: :actor) { "Getting batch (Position: #{position}, Category: #{category}, Batch Size: #{batch_size})" }
50
64
 
51
65
  batch = poll.() do
52
66
  get.(position)
53
67
  end
54
68
 
55
69
  if batch.nil? || batch.empty?
56
- logger.debug { "Did not resupply; no events available (Stream: #{get.stream_name}, Position: #{position})" }
57
-
58
- :resupply
70
+ logger.debug { "No batch retrieved (Position: #{position}, Category: #{category}, Batch Size: #{batch_size})" }
59
71
  else
60
- self.next_batch = batch
61
-
62
- logger.debug { "Resupplied (Category: #{get.category}, Position: #{position}, Batch Size: #{batch.count})" }
63
- end
64
- end
72
+ logger.debug(tag: :actor) { "Batch retrieved (Position: #{position}, Category: #{category}, Batch Size: #{batch_size})" }
65
73
 
66
- handle :get_batch do |get_batch|
67
- logger.trace { "Batch request received" }
68
-
69
- if next_batch.nil?
70
- logger.debug { "Could not fulfill batch request; deferring" }
71
-
72
- return get_batch
74
+ batch.each do |message|
75
+ dispatch(message)
76
+ end
73
77
  end
74
78
 
75
- batch = reset_next_batch
76
-
77
- reply_message = get_batch.reply_message(batch)
78
-
79
- send.(reply_message, get_batch.reply_address)
80
-
81
- logger.debug { "Batch request fulfilled; resupplying (Batch Size: #{batch.count})" }
82
-
83
- :resupply
79
+ :next_batch
84
80
  end
85
81
 
86
- def reset_next_batch
87
- batch = next_batch
82
+ def dispatch(message)
83
+ logger.trace(tag: :actor) { "Dispatching message (Position: #{position}, Category: #{category})" }
88
84
 
89
- self.next_batch = nil
85
+ consumer.dispatch(message)
90
86
 
91
- self.position = batch.last.global_position + 1
87
+ self.position = message.global_position + 1
92
88
 
93
- batch
89
+ logger.debug(tag: :actor) { "Dispatched message (Position: #{position}, Category: #{category})" }
94
90
  end
95
91
  end
96
92
  end
data/lib/consumer.rb CHANGED
@@ -17,11 +17,8 @@ require 'consumer/position_store'
17
17
  require 'consumer/position_store/substitute'
18
18
  require 'consumer/position_store/telemetry'
19
19
 
20
- require 'consumer/subscription'
21
- require 'consumer/subscription/defaults'
22
- require 'consumer/subscription/get_batch'
23
-
24
20
  require 'consumer/consumer'
25
21
  require 'consumer/substitute'
26
22
 
27
- require 'consumer/actor'
23
+ require 'consumer/subscription'
24
+ require 'consumer/subscription/defaults'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: evt-consumer
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.3.1.5
4
+ version: 2.4.0.0.dev.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - The Eventide Project
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2024-04-21 00:00:00.000000000 Z
11
+ date: 2024-04-16 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: ntl-actor
@@ -101,10 +101,8 @@ extensions: []
101
101
  extra_rdoc_files: []
102
102
  files:
103
103
  - lib/consumer.rb
104
- - lib/consumer/actor.rb
105
104
  - lib/consumer/consumer.rb
106
105
  - lib/consumer/controls.rb
107
- - lib/consumer/controls/actor.rb
108
106
  - lib/consumer/controls/category.rb
109
107
  - lib/consumer/controls/consumer.rb
110
108
  - lib/consumer/controls/consumer/error_handler.rb
@@ -135,7 +133,6 @@ files:
135
133
  - lib/consumer/position_store/telemetry.rb
136
134
  - lib/consumer/subscription.rb
137
135
  - lib/consumer/subscription/defaults.rb
138
- - lib/consumer/subscription/get_batch.rb
139
136
  - lib/consumer/substitute.rb
140
137
  homepage: https://github.com/eventide-project/consumer
141
138
  licenses:
@@ -152,9 +149,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
152
149
  version: 2.4.0
153
150
  required_rubygems_version: !ruby/object:Gem::Requirement
154
151
  requirements:
155
- - - ">="
152
+ - - ">"
156
153
  - !ruby/object:Gem::Version
157
- version: '0'
154
+ version: 1.3.1
158
155
  requirements: []
159
156
  rubygems_version: 3.4.10
160
157
  signing_key:
@@ -1,86 +0,0 @@
1
- module Consumer
2
- class Actor
3
- include ::Actor
4
- include Dependency
5
- include Initializer
6
- include Log::Dependency
7
-
8
- initializer :subscription_address, a(:delay_threshold)
9
-
10
- dependency :consumer, Consumer
11
-
12
- attr_writer :prefetch_queue
13
- def prefetch_queue
14
- @prefetch_queue ||= []
15
- end
16
-
17
- def self.build(consumer, subscription)
18
- subscription_address = subscription.address
19
-
20
- delay_threshold = subscription.batch_size
21
-
22
- instance = new(subscription_address, delay_threshold)
23
- instance.consumer = consumer
24
- instance.configure
25
- instance
26
- end
27
-
28
- handle :start do
29
- request_batch
30
- end
31
-
32
- handle Subscription::GetBatch::Reply do |get_batch_reply|
33
- received_batch = get_batch_reply.batch
34
-
35
- logger.trace(tag: :actor) { "Received batch (Received Batch Size: #{received_batch.size}, Prefetch Queue Depth: #{prefetch_queue.size}, Delay Threshold: #{delay_threshold})" }
36
-
37
- prefetch_queue_empty = prefetch_queue.empty?
38
-
39
- self.prefetch_queue += received_batch
40
-
41
- unless prefetch_queue.size > delay_threshold
42
- request_batch
43
- end
44
-
45
- if prefetch_queue_empty
46
- next_message = :dispatch
47
- end
48
-
49
- logger.debug(tag: :actor) { "Batch received (Received Batch Size: #{received_batch.size}, Prefetch Queue Depth: #{prefetch_queue.size}, Delay Threshold: #{delay_threshold}, Next Actor Message: #{next_message || '(none)'})" }
50
-
51
- next_message
52
- end
53
-
54
- handle :dispatch do
55
- logger.trace(tag: :actor) { "Dispatching Message (Prefetch Queue Depth: #{prefetch_queue.size}, Delay Threshold: #{delay_threshold})" }
56
-
57
- dispatch_message = prefetch_queue.shift
58
-
59
- if prefetch_queue.size == delay_threshold
60
- request_batch
61
- end
62
-
63
- consumer.dispatch(dispatch_message)
64
-
65
- unless prefetch_queue.empty?
66
- next_message = :dispatch
67
- end
68
-
69
- logger.debug(tag: :actor) { "Dispatched Message (Prefetch Queue Depth: #{prefetch_queue.size}, Delay Threshold: #{delay_threshold}, Global Position: #{dispatch_message.global_position}, Next Actor Message: #{next_message || '(none)'})" }
70
-
71
- next_message
72
- end
73
-
74
- def request_batch
75
- logger.trace(tag: :actor) { "Requesting batch (Prefetch Queue Depth: #{prefetch_queue.size}, Delay Threshold: #{delay_threshold})" }
76
-
77
- get_batch = Subscription::GetBatch.new(address)
78
-
79
- send.(get_batch, subscription_address)
80
-
81
- logger.debug(tag: :actor) { "Sent batch request (Prefetch Queue Depth: #{prefetch_queue.size}, Delay Threshold: #{delay_threshold})" }
82
-
83
- nil
84
- end
85
- end
86
- end
@@ -1,24 +0,0 @@
1
- module Consumer
2
- module Controls
3
- module Actor
4
- def self.example(subscription_address: nil, delay_threshold: nil)
5
- delay_threshold ||= DelayThreshold.example
6
- subscription_address ||= Address.example
7
-
8
- ::Consumer::Actor.new(subscription_address, delay_threshold)
9
- end
10
-
11
- module DelayThreshold
12
- def self.example
13
- 11
14
- end
15
- end
16
-
17
- module Address
18
- def self.example
19
- ::Actor::Messaging::Address.build
20
- end
21
- end
22
- end
23
- end
24
- end
@@ -1,17 +0,0 @@
1
- module Consumer
2
- class Subscription
3
- GetBatch = Struct.new(:reply_address)
4
-
5
- class GetBatch
6
- include ::Actor::Messaging::Message
7
-
8
- def reply_message(batch)
9
- Reply.new batch
10
- end
11
-
12
- Reply = Struct.new :batch do
13
- include ::Actor::Messaging::Message
14
- end
15
- end
16
- end
17
- end