cdc-concurrent 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +5 -0
- data/LICENSE.txt +21 -0
- data/README.md +119 -0
- data/lib/cdc/concurrent/configuration.rb +19 -0
- data/lib/cdc/concurrent/errors.rb +20 -0
- data/lib/cdc/concurrent/processor_extensions.rb +37 -0
- data/lib/cdc/concurrent/processor_pool.rb +93 -0
- data/lib/cdc/concurrent/result_collector.rb +24 -0
- data/lib/cdc/concurrent/router.rb +30 -0
- data/lib/cdc/concurrent/runtime.rb +48 -0
- data/lib/cdc/concurrent/transaction_pool.rb +32 -0
- data/lib/cdc/concurrent/version.rb +8 -0
- data/lib/cdc/concurrent.rb +21 -0
- data/lib/cdc_concurrent.rb +3 -0
- data/sig/cdc/concurrent/configuration.rbs +6 -0
- data/sig/cdc/concurrent/errors.rbs +23 -0
- data/sig/cdc/concurrent/processor_extensions.rbs +26 -0
- data/sig/cdc/concurrent/processor_pool.rbs +41 -0
- data/sig/cdc/concurrent/result_collector.rbs +14 -0
- data/sig/cdc/concurrent/router.rbs +18 -0
- data/sig/cdc/concurrent/runtime.rbs +35 -0
- data/sig/cdc/concurrent/transaction_pool.rbs +21 -0
- data/sig/cdc/concurrent/version.rbs +6 -0
- data/sig/cdc/concurrent.rbs +5 -0
- data/sig/cdc_concurrent.rbs +0 -0
- metadata +101 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: a8f1b6796df6daf32144dc7df7206777863c4d693997e3b83e500d0c360ff699
|
|
4
|
+
data.tar.gz: eabe19f74600193950154f4921203123b78bd1c2db8a2ca576bf74c83ab1eb5b
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: eabccfa38225330187c12f4c681c68f973126e60910c00402cc274a400e776753e46b1a5605692b363f8d064971cb26253ab72d305b69d9e2106e49733c964d7
|
|
7
|
+
data.tar.gz: 2210868f800f9ae0cddf5cf24bf6104be5dfd9f74d2b74694f66571cdd87031ba714ba2b12356c4401b71e4dcad901187848ad1f8a8746f7e0b2f236122dec9c
|
data/CHANGELOG.md
ADDED
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Kenneth C. Demanawa
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
# cdc-concurrent
|
|
2
|
+
|
|
3
|
+
Optional I/O-concurrent runtime adapter for `cdc-core`.
|
|
4
|
+
|
|
5
|
+
`cdc-concurrent` executes `CDC::Core::Processor` objects with Fiber-scheduler-based I/O concurrency using `async`. It is the I/O-bound twin of `cdc-parallel`.
|
|
6
|
+
|
|
7
|
+
## Requirements
|
|
8
|
+
|
|
9
|
+
- Ruby 3.4+
|
|
10
|
+
- `cdc-core`
|
|
11
|
+
- `async`
|
|
12
|
+
|
|
13
|
+
## Purpose
|
|
14
|
+
|
|
15
|
+
```text
|
|
16
|
+
cdc-core
|
|
17
|
+
|
|
|
18
|
+
|-- cdc-parallel
|
|
19
|
+
| CPU-bound parallelism
|
|
20
|
+
|
|
|
21
|
+
`-- cdc-concurrent
|
|
22
|
+
I/O-bound concurrency
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Use `cdc-concurrent` for processors that spend most of their time waiting on fiber-scheduler-compatible I/O:
|
|
26
|
+
|
|
27
|
+
- HTTP webhooks
|
|
28
|
+
- external API enrichment
|
|
29
|
+
- Redis publishing
|
|
30
|
+
- OpenSearch or Elasticsearch indexing
|
|
31
|
+
- S3 or object-storage writes
|
|
32
|
+
- async sink fanout
|
|
33
|
+
- database writes through compatible drivers
|
|
34
|
+
|
|
35
|
+
Use `cdc-parallel` for CPU-bound work such as pgoutput parsing, OID decoding, JSON parsing, diff computation, compression, and analytics calculations.
|
|
36
|
+
|
|
37
|
+
## Installation
|
|
38
|
+
|
|
39
|
+
```ruby
|
|
40
|
+
gem "cdc-concurrent"
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## Usage
|
|
44
|
+
|
|
45
|
+
```ruby
|
|
46
|
+
require "cdc/core"
|
|
47
|
+
require "cdc/concurrent"
|
|
48
|
+
|
|
49
|
+
class WebhookProcessor < CDC::Core::Processor
|
|
50
|
+
concurrent_safe!
|
|
51
|
+
|
|
52
|
+
def process(event)
|
|
53
|
+
# Perform fiber-scheduler-compatible I/O here.
|
|
54
|
+
CDC::Core::ProcessorResult.success(event)
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
runtime = CDC::Concurrent::Runtime.new(
|
|
59
|
+
processor: WebhookProcessor.new,
|
|
60
|
+
concurrency: 100,
|
|
61
|
+
timeout: 5.0
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
result = runtime.process(event)
|
|
65
|
+
runtime.shutdown
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## Batch Processing
|
|
69
|
+
|
|
70
|
+
```ruby
|
|
71
|
+
results = runtime.process_many(events)
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Results preserve input order by default. Set `preserve_order: false` when completion order is acceptable.
|
|
75
|
+
|
|
76
|
+
## Transaction Processing
|
|
77
|
+
|
|
78
|
+
```ruby
|
|
79
|
+
result = runtime.process_transaction(transaction)
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Transactions are processed event-by-event. The returned `ProcessorResult#event` contains the per-event results. If any event fails, the transaction result fails and carries the first error.
|
|
83
|
+
|
|
84
|
+
## Processor Safety
|
|
85
|
+
|
|
86
|
+
Only processors that declare `concurrent_safe!` can run in this runtime.
|
|
87
|
+
|
|
88
|
+
```ruby
|
|
89
|
+
class SinkProcessor < CDC::Core::Processor
|
|
90
|
+
concurrent_safe!
|
|
91
|
+
end
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
Unsafe processors raise:
|
|
95
|
+
|
|
96
|
+
```ruby
|
|
97
|
+
CDC::Concurrent::UnsafeProcessorError
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
A concurrent-safe processor should avoid unsafe shared mutable instance state. This runtime runs tasks concurrently in one Ruby process; it does not isolate mutable objects like Ractors do.
|
|
101
|
+
|
|
102
|
+
## Important Limits
|
|
103
|
+
|
|
104
|
+
`cdc-concurrent` improves throughput only for I/O that cooperates with Ruby's Fiber scheduler. Blocking libraries that do not yield to the scheduler will still block the process.
|
|
105
|
+
|
|
106
|
+
For CPU-bound processing, use `cdc-parallel`.
|
|
107
|
+
|
|
108
|
+
## Roadmap
|
|
109
|
+
|
|
110
|
+
- Move `concurrent_safe!` into `cdc-core`
|
|
111
|
+
- Retry and backoff policies
|
|
112
|
+
- Dead-letter handling
|
|
113
|
+
- Async HTTP webhook helpers
|
|
114
|
+
- Sink abstractions
|
|
115
|
+
- Async Redis/OpenSearch integrations
|
|
116
|
+
|
|
117
|
+
## License
|
|
118
|
+
|
|
119
|
+
MIT.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CDC
|
|
4
|
+
module Concurrent
|
|
5
|
+
# Immutable configuration for concurrent runtimes.
|
|
6
|
+
Configuration = Data.define(:concurrency, :timeout, :preserve_order) do
|
|
7
|
+
# @param concurrency [Integer] maximum concurrent tasks.
|
|
8
|
+
# @param timeout [Float, nil] optional timeout.
|
|
9
|
+
# @param preserve_order [Boolean] whether batch results preserve input order.
|
|
10
|
+
def initialize(concurrency: 100, timeout: nil, preserve_order: true)
|
|
11
|
+
raise ArgumentError, "concurrency must be an Integer" unless concurrency.is_a?(Integer)
|
|
12
|
+
raise ArgumentError, "concurrency must be greater than zero" unless concurrency.positive?
|
|
13
|
+
|
|
14
|
+
super
|
|
15
|
+
freeze
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CDC
|
|
4
|
+
module Concurrent
|
|
5
|
+
# Base cdc-concurrent error.
|
|
6
|
+
class Error < StandardError; end
|
|
7
|
+
|
|
8
|
+
# Raised when a processor has not declared itself concurrent-safe.
|
|
9
|
+
class UnsafeProcessorError < Error; end
|
|
10
|
+
|
|
11
|
+
# Raised when work is submitted after shutdown.
|
|
12
|
+
class ShutdownError < Error; end
|
|
13
|
+
|
|
14
|
+
# Raised when the runtime receives an unsupported work item.
|
|
15
|
+
class UnsupportedWorkItemError < Error; end
|
|
16
|
+
|
|
17
|
+
# Raised when processing exceeds the configured timeout.
|
|
18
|
+
class TimeoutError < Error; end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CDC
|
|
4
|
+
# Optional concurrent runtime adapter for cdc-core processors.
|
|
5
|
+
module Concurrent
|
|
6
|
+
# Adds concurrent-safe declarations to CDC::Core::Processor subclasses.
|
|
7
|
+
module ProcessorExtensions
|
|
8
|
+
# Class methods added to CDC::Core::Processor.
|
|
9
|
+
module ClassMethods
|
|
10
|
+
# Declare this processor safe for concurrent execution.
|
|
11
|
+
def concurrent_safe!
|
|
12
|
+
@concurrent_safe = true
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# @return [Boolean] whether instances are concurrent-safe.
|
|
16
|
+
def concurrent_safe?
|
|
17
|
+
@concurrent_safe == true
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# @return [Boolean] whether this processor instance is concurrent-safe.
|
|
22
|
+
def concurrent_safe?
|
|
23
|
+
self.class.respond_to?(:concurrent_safe?) && self.class.concurrent_safe?
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# Installs concurrent-safe declarations on CDC::Core::Processor.
|
|
28
|
+
#
|
|
29
|
+
# @return [void]
|
|
30
|
+
def self.install_processor_extensions!
|
|
31
|
+
CDC::Core::Processor.extend(ProcessorExtensions::ClassMethods)
|
|
32
|
+
CDC::Core::Processor.include(ProcessorExtensions)
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
CDC::Concurrent.install_processor_extensions!
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CDC
|
|
4
|
+
module Concurrent
|
|
5
|
+
# Executes one concurrent-safe processor using Async tasks.
|
|
6
|
+
class ProcessorPool
|
|
7
|
+
# @param processor [CDC::Core::Processor]
|
|
8
|
+
# @param concurrency [Integer]
|
|
9
|
+
# @param timeout [Float, nil]
|
|
10
|
+
# @param preserve_order [Boolean]
|
|
11
|
+
def initialize(processor:, concurrency: 100, timeout: nil, preserve_order: true)
|
|
12
|
+
validate_processor!(processor)
|
|
13
|
+
|
|
14
|
+
@processor = processor
|
|
15
|
+
@configuration = Configuration.new(concurrency:, timeout:, preserve_order:)
|
|
16
|
+
@shutdown = false
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# @param event [CDC::Core::ChangeEvent]
|
|
20
|
+
# @return [CDC::Core::ProcessorResult]
|
|
21
|
+
def process(event)
|
|
22
|
+
raise ShutdownError, "processor pool has been shut down" if @shutdown
|
|
23
|
+
|
|
24
|
+
process_one(event)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# @param events [Array<CDC::Core::ChangeEvent>]
|
|
28
|
+
# @return [Array<CDC::Core::ProcessorResult>]
|
|
29
|
+
def process_many(events)
|
|
30
|
+
raise ShutdownError, "processor pool has been shut down" if @shutdown
|
|
31
|
+
return [].freeze if events.empty?
|
|
32
|
+
|
|
33
|
+
indexed_results = []
|
|
34
|
+
|
|
35
|
+
process_batch(events, indexed_results)
|
|
36
|
+
|
|
37
|
+
indexed_results.sort_by!(&:first) if @configuration.preserve_order
|
|
38
|
+
indexed_results.map(&:last).freeze
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# @return [void]
|
|
42
|
+
def shutdown
|
|
43
|
+
@shutdown = true
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
private
|
|
47
|
+
|
|
48
|
+
def validate_processor!(processor)
|
|
49
|
+
return if processor.respond_to?(:concurrent_safe?) && processor.concurrent_safe?
|
|
50
|
+
|
|
51
|
+
raise UnsafeProcessorError, "#{processor.class} must declare concurrent_safe!"
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def process_batch(events, indexed_results)
|
|
55
|
+
Async do |task|
|
|
56
|
+
semaphore = Async::Semaphore.new(@configuration.concurrency, parent: task)
|
|
57
|
+
|
|
58
|
+
events.each_with_index do |event, index|
|
|
59
|
+
semaphore.async do |subtask|
|
|
60
|
+
indexed_results << [index, process_with_task(subtask, event)]
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end.wait
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def process_one(event)
|
|
67
|
+
result = nil
|
|
68
|
+
|
|
69
|
+
Async do |task|
|
|
70
|
+
result = process_with_task(task, event)
|
|
71
|
+
end.wait
|
|
72
|
+
|
|
73
|
+
result
|
|
74
|
+
rescue StandardError => e
|
|
75
|
+
ResultCollector.failure(e)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def process_with_task(task, event)
|
|
79
|
+
ResultCollector.normalize(call_processor(task, event))
|
|
80
|
+
rescue Async::TimeoutError => e
|
|
81
|
+
ResultCollector.failure(TimeoutError.new(e.message))
|
|
82
|
+
rescue StandardError => e
|
|
83
|
+
ResultCollector.failure(e)
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def call_processor(task, event)
|
|
87
|
+
return @processor.process(event) unless @configuration.timeout
|
|
88
|
+
|
|
89
|
+
task.with_timeout(@configuration.timeout) { @processor.process(event) }
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CDC
|
|
4
|
+
module Concurrent
|
|
5
|
+
# Normalizes values returned by concurrent workers.
|
|
6
|
+
class ResultCollector
|
|
7
|
+
# @param value [Object]
|
|
8
|
+
# @return [CDC::Core::ProcessorResult]
|
|
9
|
+
def self.normalize(value)
|
|
10
|
+
return value if value.is_a?(CDC::Core::ProcessorResult)
|
|
11
|
+
|
|
12
|
+
CDC::Core::ProcessorResult.success(value)
|
|
13
|
+
rescue StandardError => e
|
|
14
|
+
failure(e)
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# @param error [Exception]
|
|
18
|
+
# @return [CDC::Core::ProcessorResult]
|
|
19
|
+
def self.failure(error)
|
|
20
|
+
CDC::Core::ProcessorResult.failure(error)
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CDC
|
|
4
|
+
module Concurrent
|
|
5
|
+
# Routes CDC work items to the correct concurrent pool.
|
|
6
|
+
class Router
|
|
7
|
+
# @param processor_pool [ProcessorPool]
|
|
8
|
+
# @param transaction_pool [TransactionPool]
|
|
9
|
+
def initialize(processor_pool:, transaction_pool:)
|
|
10
|
+
@processor_pool = processor_pool
|
|
11
|
+
@transaction_pool = transaction_pool
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# @param item [Object]
|
|
15
|
+
# @return [Object]
|
|
16
|
+
def process(item)
|
|
17
|
+
case item
|
|
18
|
+
when CDC::Core::ChangeEvent
|
|
19
|
+
@processor_pool.process(item)
|
|
20
|
+
when CDC::Core::TransactionEnvelope
|
|
21
|
+
@transaction_pool.process(item)
|
|
22
|
+
when Array
|
|
23
|
+
@processor_pool.process_many(item)
|
|
24
|
+
else
|
|
25
|
+
raise UnsupportedWorkItemError, "unsupported CDC work item: #{item.class}"
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CDC
|
|
4
|
+
module Concurrent
|
|
5
|
+
# High-level concurrent runtime facade for cdc-core processors.
|
|
6
|
+
class Runtime
|
|
7
|
+
# @param processor [CDC::Core::Processor]
|
|
8
|
+
# @param concurrency [Integer]
|
|
9
|
+
# @param timeout [Float, nil]
|
|
10
|
+
# @param preserve_order [Boolean]
|
|
11
|
+
def initialize(processor:, concurrency: 100, timeout: nil, preserve_order: true)
|
|
12
|
+
@processor_pool = ProcessorPool.new(processor:, concurrency:, timeout:, preserve_order:)
|
|
13
|
+
@transaction_pool = TransactionPool.new(processor:, concurrency:, timeout:, preserve_order:)
|
|
14
|
+
@router = Router.new(processor_pool: @processor_pool, transaction_pool: @transaction_pool)
|
|
15
|
+
@shutdown = false
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# @param item [Object]
|
|
19
|
+
# @return [Object]
|
|
20
|
+
def process(item)
|
|
21
|
+
raise ShutdownError, "runtime has been shut down" if @shutdown
|
|
22
|
+
|
|
23
|
+
@router.process(item)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# @param events [Array<CDC::Core::ChangeEvent>]
|
|
27
|
+
# @return [Array<CDC::Core::ProcessorResult>]
|
|
28
|
+
def process_many(events)
|
|
29
|
+
process(events)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# @param transaction [CDC::Core::TransactionEnvelope]
|
|
33
|
+
# @return [CDC::Core::ProcessorResult]
|
|
34
|
+
def process_transaction(transaction)
|
|
35
|
+
process(transaction)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# @return [void]
|
|
39
|
+
def shutdown
|
|
40
|
+
return if @shutdown
|
|
41
|
+
|
|
42
|
+
@shutdown = true
|
|
43
|
+
@processor_pool.shutdown
|
|
44
|
+
@transaction_pool.shutdown
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CDC
|
|
4
|
+
module Concurrent
|
|
5
|
+
# Processes TransactionEnvelope events as a single ordering-preserving unit.
|
|
6
|
+
class TransactionPool
|
|
7
|
+
# @param processor [CDC::Core::Processor]
|
|
8
|
+
# @param concurrency [Integer]
|
|
9
|
+
# @param timeout [Float, nil]
|
|
10
|
+
# @param preserve_order [Boolean]
|
|
11
|
+
def initialize(processor:, concurrency: 100, timeout: nil, preserve_order: true)
|
|
12
|
+
@processor_pool = ProcessorPool.new(processor:, concurrency:, timeout:, preserve_order:)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# @param transaction [CDC::Core::TransactionEnvelope]
|
|
16
|
+
# @return [CDC::Core::ProcessorResult]
|
|
17
|
+
def process(transaction)
|
|
18
|
+
results = @processor_pool.process_many(transaction.events).freeze
|
|
19
|
+
failure = results.find(&:failure?)
|
|
20
|
+
|
|
21
|
+
return CDC::Core::ProcessorResult.failure(failure.error, event: results) if failure
|
|
22
|
+
|
|
23
|
+
CDC::Core::ProcessorResult.success(results)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# @return [void]
|
|
27
|
+
def shutdown
|
|
28
|
+
@processor_pool.shutdown
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "async"
|
|
4
|
+
require "async/semaphore"
|
|
5
|
+
require "cdc/core"
|
|
6
|
+
|
|
7
|
+
require_relative "concurrent/version"
|
|
8
|
+
require_relative "concurrent/errors"
|
|
9
|
+
require_relative "concurrent/configuration"
|
|
10
|
+
require_relative "concurrent/processor_extensions"
|
|
11
|
+
require_relative "concurrent/result_collector"
|
|
12
|
+
require_relative "concurrent/processor_pool"
|
|
13
|
+
require_relative "concurrent/transaction_pool"
|
|
14
|
+
require_relative "concurrent/router"
|
|
15
|
+
require_relative "concurrent/runtime"
|
|
16
|
+
|
|
17
|
+
module CDC
|
|
18
|
+
# Optional concurrent runtime adapter for cdc-core processors.
|
|
19
|
+
module Concurrent
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
module CDC
|
|
2
|
+
module Concurrent
|
|
3
|
+
# Base cdc-concurrent error.
|
|
4
|
+
class Error < StandardError
|
|
5
|
+
end
|
|
6
|
+
|
|
7
|
+
# Raised when a processor has not declared itself concurrent-safe.
|
|
8
|
+
class UnsafeProcessorError < Error
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
# Raised when work is submitted after shutdown.
|
|
12
|
+
class ShutdownError < Error
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# Raised when the runtime receives an unsupported work item.
|
|
16
|
+
class UnsupportedWorkItemError < Error
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# Raised when processing exceeds the configured timeout.
|
|
20
|
+
class TimeoutError < Error
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
module CDC
|
|
2
|
+
# Optional concurrent runtime adapter for cdc-core processors.
|
|
3
|
+
module Concurrent
|
|
4
|
+
# Adds concurrent-safe declarations to CDC::Core::Processor subclasses.
|
|
5
|
+
module ProcessorExtensions
|
|
6
|
+
# Class methods added to CDC::Core::Processor.
|
|
7
|
+
module ClassMethods
|
|
8
|
+
@concurrent_safe: untyped
|
|
9
|
+
|
|
10
|
+
# Declare this processor safe for concurrent execution.
|
|
11
|
+
def concurrent_safe!: () -> untyped
|
|
12
|
+
|
|
13
|
+
# @return [Boolean] whether instances are concurrent-safe.
|
|
14
|
+
def concurrent_safe?: () -> untyped
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# @return [Boolean] whether this processor instance is concurrent-safe.
|
|
18
|
+
def concurrent_safe?: () -> untyped
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Installs concurrent-safe declarations on CDC::Core::Processor.
|
|
22
|
+
#
|
|
23
|
+
# @return [void]
|
|
24
|
+
def self.install_processor_extensions!: () -> untyped
|
|
25
|
+
end
|
|
26
|
+
end
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
module CDC
|
|
2
|
+
module Concurrent
|
|
3
|
+
# Executes one concurrent-safe processor using Async tasks.
|
|
4
|
+
class ProcessorPool
|
|
5
|
+
@processor: untyped
|
|
6
|
+
|
|
7
|
+
@configuration: untyped
|
|
8
|
+
|
|
9
|
+
@shutdown: untyped
|
|
10
|
+
|
|
11
|
+
# @param processor [CDC::Core::Processor]
|
|
12
|
+
# @param concurrency [Integer]
|
|
13
|
+
# @param timeout [Float, nil]
|
|
14
|
+
# @param preserve_order [Boolean]
|
|
15
|
+
def initialize: (processor: untyped, ?concurrency: ::Integer, ?timeout: untyped?, ?preserve_order: bool) -> void
|
|
16
|
+
|
|
17
|
+
# @param event [CDC::Core::ChangeEvent]
|
|
18
|
+
# @return [CDC::Core::ProcessorResult]
|
|
19
|
+
def process: (untyped event) -> untyped
|
|
20
|
+
|
|
21
|
+
# @param events [Array<CDC::Core::ChangeEvent>]
|
|
22
|
+
# @return [Array<CDC::Core::ProcessorResult>]
|
|
23
|
+
def process_many: (untyped events) -> (::Array[untyped] | untyped)
|
|
24
|
+
|
|
25
|
+
# @return [void]
|
|
26
|
+
def shutdown: () -> untyped
|
|
27
|
+
|
|
28
|
+
private
|
|
29
|
+
|
|
30
|
+
def validate_processor!: (untyped processor) -> (nil | untyped)
|
|
31
|
+
|
|
32
|
+
def process_batch: (untyped events, untyped indexed_results) -> untyped
|
|
33
|
+
|
|
34
|
+
def process_one: (untyped event) -> untyped
|
|
35
|
+
|
|
36
|
+
def process_with_task: (untyped task, untyped event) -> untyped
|
|
37
|
+
|
|
38
|
+
def call_processor: (untyped task, untyped event) -> untyped
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
module CDC
|
|
2
|
+
module Concurrent
|
|
3
|
+
# Normalizes values returned by concurrent workers.
|
|
4
|
+
class ResultCollector
|
|
5
|
+
# @param value [Object]
|
|
6
|
+
# @return [CDC::Core::ProcessorResult]
|
|
7
|
+
def self.normalize: (untyped value) -> untyped
|
|
8
|
+
|
|
9
|
+
# @param error [Exception]
|
|
10
|
+
# @return [CDC::Core::ProcessorResult]
|
|
11
|
+
def self.failure: (untyped error) -> untyped
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
module CDC
|
|
2
|
+
module Concurrent
|
|
3
|
+
# Routes CDC work items to the correct concurrent pool.
|
|
4
|
+
class Router
|
|
5
|
+
@processor_pool: untyped
|
|
6
|
+
|
|
7
|
+
@transaction_pool: untyped
|
|
8
|
+
|
|
9
|
+
# @param processor_pool [ProcessorPool]
|
|
10
|
+
# @param transaction_pool [TransactionPool]
|
|
11
|
+
def initialize: (processor_pool: untyped, transaction_pool: untyped) -> void
|
|
12
|
+
|
|
13
|
+
# @param item [Object]
|
|
14
|
+
# @return [Object]
|
|
15
|
+
def process: (untyped item) -> untyped
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
module CDC
|
|
2
|
+
module Concurrent
|
|
3
|
+
# High-level concurrent runtime facade for cdc-core processors.
|
|
4
|
+
class Runtime
|
|
5
|
+
@processor_pool: untyped
|
|
6
|
+
|
|
7
|
+
@transaction_pool: untyped
|
|
8
|
+
|
|
9
|
+
@router: untyped
|
|
10
|
+
|
|
11
|
+
@shutdown: untyped
|
|
12
|
+
|
|
13
|
+
# @param processor [CDC::Core::Processor]
|
|
14
|
+
# @param concurrency [Integer]
|
|
15
|
+
# @param timeout [Float, nil]
|
|
16
|
+
# @param preserve_order [Boolean]
|
|
17
|
+
def initialize: (processor: untyped, ?concurrency: ::Integer, ?timeout: untyped?, ?preserve_order: bool) -> void
|
|
18
|
+
|
|
19
|
+
# @param item [Object]
|
|
20
|
+
# @return [Object]
|
|
21
|
+
def process: (untyped item) -> untyped
|
|
22
|
+
|
|
23
|
+
# @param events [Array<CDC::Core::ChangeEvent>]
|
|
24
|
+
# @return [Array<CDC::Core::ProcessorResult>]
|
|
25
|
+
def process_many: (untyped events) -> untyped
|
|
26
|
+
|
|
27
|
+
# @param transaction [CDC::Core::TransactionEnvelope]
|
|
28
|
+
# @return [CDC::Core::ProcessorResult]
|
|
29
|
+
def process_transaction: (untyped transaction) -> untyped
|
|
30
|
+
|
|
31
|
+
# @return [void]
|
|
32
|
+
def shutdown: () -> (nil | untyped)
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
module CDC
|
|
2
|
+
module Concurrent
|
|
3
|
+
# Processes TransactionEnvelope events as a single ordering-preserving unit.
|
|
4
|
+
class TransactionPool
|
|
5
|
+
@processor_pool: untyped
|
|
6
|
+
|
|
7
|
+
# @param processor [CDC::Core::Processor]
|
|
8
|
+
# @param concurrency [Integer]
|
|
9
|
+
# @param timeout [Float, nil]
|
|
10
|
+
# @param preserve_order [Boolean]
|
|
11
|
+
def initialize: (processor: untyped, ?concurrency: ::Integer, ?timeout: untyped?, ?preserve_order: bool) -> void
|
|
12
|
+
|
|
13
|
+
# @param transaction [CDC::Core::TransactionEnvelope]
|
|
14
|
+
# @return [CDC::Core::ProcessorResult]
|
|
15
|
+
def process: (untyped transaction) -> untyped
|
|
16
|
+
|
|
17
|
+
# @return [void]
|
|
18
|
+
def shutdown: () -> untyped
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
File without changes
|
metadata
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: cdc-concurrent
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.0.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Ken C. Demanawa
|
|
8
|
+
bindir: bin
|
|
9
|
+
cert_chain: []
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
|
+
dependencies:
|
|
12
|
+
- !ruby/object:Gem::Dependency
|
|
13
|
+
name: async
|
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
|
15
|
+
requirements:
|
|
16
|
+
- - "~>"
|
|
17
|
+
- !ruby/object:Gem::Version
|
|
18
|
+
version: '2.0'
|
|
19
|
+
type: :runtime
|
|
20
|
+
prerelease: false
|
|
21
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
22
|
+
requirements:
|
|
23
|
+
- - "~>"
|
|
24
|
+
- !ruby/object:Gem::Version
|
|
25
|
+
version: '2.0'
|
|
26
|
+
- !ruby/object:Gem::Dependency
|
|
27
|
+
name: cdc-core
|
|
28
|
+
requirement: !ruby/object:Gem::Requirement
|
|
29
|
+
requirements:
|
|
30
|
+
- - "~>"
|
|
31
|
+
- !ruby/object:Gem::Version
|
|
32
|
+
version: '0.1'
|
|
33
|
+
type: :runtime
|
|
34
|
+
prerelease: false
|
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
36
|
+
requirements:
|
|
37
|
+
- - "~>"
|
|
38
|
+
- !ruby/object:Gem::Version
|
|
39
|
+
version: '0.1'
|
|
40
|
+
description: |
|
|
41
|
+
cdc-concurrent provides optional Async-backed I/O-concurrent execution for
|
|
42
|
+
cdc-core. It accelerates I/O-bound PostgreSQL Change Data Capture (CDC)
|
|
43
|
+
event processing while preserving the cdc-core programming model.
|
|
44
|
+
email:
|
|
45
|
+
- kenneth.c.demanawa@gmail.com
|
|
46
|
+
executables: []
|
|
47
|
+
extensions: []
|
|
48
|
+
extra_rdoc_files: []
|
|
49
|
+
files:
|
|
50
|
+
- CHANGELOG.md
|
|
51
|
+
- LICENSE.txt
|
|
52
|
+
- README.md
|
|
53
|
+
- lib/cdc/concurrent.rb
|
|
54
|
+
- lib/cdc/concurrent/configuration.rb
|
|
55
|
+
- lib/cdc/concurrent/errors.rb
|
|
56
|
+
- lib/cdc/concurrent/processor_extensions.rb
|
|
57
|
+
- lib/cdc/concurrent/processor_pool.rb
|
|
58
|
+
- lib/cdc/concurrent/result_collector.rb
|
|
59
|
+
- lib/cdc/concurrent/router.rb
|
|
60
|
+
- lib/cdc/concurrent/runtime.rb
|
|
61
|
+
- lib/cdc/concurrent/transaction_pool.rb
|
|
62
|
+
- lib/cdc/concurrent/version.rb
|
|
63
|
+
- lib/cdc_concurrent.rb
|
|
64
|
+
- sig/cdc/concurrent.rbs
|
|
65
|
+
- sig/cdc/concurrent/configuration.rbs
|
|
66
|
+
- sig/cdc/concurrent/errors.rbs
|
|
67
|
+
- sig/cdc/concurrent/processor_extensions.rbs
|
|
68
|
+
- sig/cdc/concurrent/processor_pool.rbs
|
|
69
|
+
- sig/cdc/concurrent/result_collector.rbs
|
|
70
|
+
- sig/cdc/concurrent/router.rbs
|
|
71
|
+
- sig/cdc/concurrent/runtime.rbs
|
|
72
|
+
- sig/cdc/concurrent/transaction_pool.rbs
|
|
73
|
+
- sig/cdc/concurrent/version.rbs
|
|
74
|
+
- sig/cdc_concurrent.rbs
|
|
75
|
+
homepage: https://kanutocd.github.io/cdc-concurrent
|
|
76
|
+
licenses:
|
|
77
|
+
- MIT
|
|
78
|
+
metadata:
|
|
79
|
+
homepage_uri: https://kanutocd.github.io/cdc-concurrent
|
|
80
|
+
documentation_uri: https://kanutocd.github.io/cdc-concurrent
|
|
81
|
+
source_code_uri: https://github.com/kanutocd/cdc-concurrent
|
|
82
|
+
changelog_uri: https://github.com/kanutocd/cdc-concurrent/blob/main/CHANGELOG.md
|
|
83
|
+
rubygems_mfa_required: 'true'
|
|
84
|
+
rdoc_options: []
|
|
85
|
+
require_paths:
|
|
86
|
+
- lib
|
|
87
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
88
|
+
requirements:
|
|
89
|
+
- - ">="
|
|
90
|
+
- !ruby/object:Gem::Version
|
|
91
|
+
version: 3.4.0
|
|
92
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
93
|
+
requirements:
|
|
94
|
+
- - ">="
|
|
95
|
+
- !ruby/object:Gem::Version
|
|
96
|
+
version: '0'
|
|
97
|
+
requirements: []
|
|
98
|
+
rubygems_version: 4.0.10
|
|
99
|
+
specification_version: 4
|
|
100
|
+
summary: Optional I/O-concurrent Change Data Capture (CDC) runtime for cdc-core.
|
|
101
|
+
test_files: []
|