cdc-parallel 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +30 -0
- data/LICENSE.txt +21 -0
- data/README.md +163 -0
- data/lib/cdc/parallel/configuration.rb +21 -0
- data/lib/cdc/parallel/errors.rb +34 -0
- data/lib/cdc/parallel/processor_pool.rb +65 -0
- data/lib/cdc/parallel/result_collector.rb +50 -0
- data/lib/cdc/parallel/router.rb +30 -0
- data/lib/cdc/parallel/runtime.rb +48 -0
- data/lib/cdc/parallel/transaction_pool.rb +31 -0
- data/lib/cdc/parallel/version.rb +8 -0
- data/lib/cdc/parallel.rb +19 -0
- data/lib/cdc_parallel.rb +3 -0
- data/sig/cdc/parallel/configuration.rbs +8 -0
- data/sig/cdc/parallel/errors.rbs +40 -0
- data/sig/cdc/parallel/processor_pool.rbs +40 -0
- data/sig/cdc/parallel/result_collector.rbs +22 -0
- data/sig/cdc/parallel/router.rbs +20 -0
- data/sig/cdc/parallel/runtime.rbs +37 -0
- data/sig/cdc/parallel/transaction_pool.rbs +24 -0
- data/sig/cdc/parallel/version.rbs +6 -0
- data/sig/cdc/parallel.rbs +5 -0
- data/sig/cdc_parallel.rbs +0 -0
- data/sig/shims/cdc_core.rbs +14 -0
- data/sig/shims/data_define.rbs +0 -0
- data/sig/shims/etc.rbs +3 -0
- metadata +102 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 7b66764665f18a0f84cc6b45c5138d932fb000d01be2bef02a26504e7013c5cb
|
|
4
|
+
data.tar.gz: 12daffb757cae8c4e9c5330cce505cc964834ac205fc89d3febc29944772f815
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: e7e3bd0dcd37187e7f31a9930fdea00680f898ce52c31cf33b232004bd9da4ec07cb132fc1bd71a337c7a8db99b7888618d96fc435132e16eb3038e1ddb8588f
|
|
7
|
+
data.tar.gz: f9af8ecfb077ef1898d312f45169b6b0b81d3ea641af1e9f00374999cd06542bcda7561fc8878b147f61ebcd4c0f5503f88d3f332f1657be19bc45504d77de49
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on Keep a Changelog, and this project adheres to Semantic Versioning.
|
|
6
|
+
|
|
7
|
+
## [Unreleased]
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- Placeholder for future development.
|
|
12
|
+
|
|
13
|
+
## [0.1.0] - 2026-05-31
|
|
14
|
+
|
|
15
|
+
### Added
|
|
16
|
+
|
|
17
|
+
- Initial `CDC::Parallel` namespace.
|
|
18
|
+
- Added `CDC::Parallel::Runtime`.
|
|
19
|
+
- Added `CDC::Parallel::ProcessorPool`.
|
|
20
|
+
- Added `CDC::Parallel::TransactionPool`.
|
|
21
|
+
- Added `CDC::Parallel::Router`.
|
|
22
|
+
- Added `CDC::Parallel::ResultCollector`.
|
|
23
|
+
- Added Parallel-safe processor validation.
|
|
24
|
+
- Added support for `CDC::Core::ChangeEvent` processing.
|
|
25
|
+
- Added support for `CDC::Core::TransactionEnvelope` processing.
|
|
26
|
+
- Added graceful shutdown behavior.
|
|
27
|
+
- Added RBS signatures.
|
|
28
|
+
- Added Minitest suite.
|
|
29
|
+
- Added README and example.
|
|
30
|
+
- Added CI and release workflows.
|
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Kenneth C. Demanawa
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
# cdc-parallel
|
|
2
|
+
|
|
3
|
+
[](https://badge.fury.io/rb/cdc-parallel)
|
|
4
|
+
[](https://github.com/kanutocd/cdc-parallel/actions)
|
|
5
|
+
[](https://codecov.io/gh/kanutocd/cdc-parallel)
|
|
6
|
+
[](https://www.ruby-lang.org/en/)
|
|
7
|
+
[](https://opensource.org/licenses/MIT)
|
|
8
|
+
|
|
9
|
+
Optional high-throughput Ractor runtime for `cdc-core`.
|
|
10
|
+
|
|
11
|
+
`cdc-parallel` executes `CDC::Core::Processor` objects in Ractors when those processors explicitly declare themselves Ractor-safe.
|
|
12
|
+
|
|
13
|
+
## Requirements
|
|
14
|
+
|
|
15
|
+
- Ruby 4.0+
|
|
16
|
+
- `cdc-core`
|
|
17
|
+
- `parallel-pool`
|
|
18
|
+
|
|
19
|
+
Ruby 4.0+ is required because this gem targets the stabilized Ruby Ractor API.
|
|
20
|
+
|
|
21
|
+
## Purpose
|
|
22
|
+
|
|
23
|
+
```text
|
|
24
|
+
cdc-core
|
|
25
|
+
│
|
|
26
|
+
▼
|
|
27
|
+
cdc-parallel
|
|
28
|
+
│
|
|
29
|
+
▼
|
|
30
|
+
parallel Parallel-aware processing
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
`cdc-parallel` is a runtime adapter. It does not define CDC events and does not parse database streams.
|
|
34
|
+
|
|
35
|
+
## Installation
|
|
36
|
+
|
|
37
|
+
```ruby
|
|
38
|
+
gem "cdc-parallel"
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
## Usage
|
|
42
|
+
|
|
43
|
+
```ruby
|
|
44
|
+
require "cdc/core"
|
|
45
|
+
require "cdc/parallel"
|
|
46
|
+
|
|
47
|
+
class MetricsProcessor < CDC::Core::Processor
|
|
48
|
+
ractor_safe!
|
|
49
|
+
|
|
50
|
+
def process(event)
|
|
51
|
+
CDC::Core::ProcessorResult.success(
|
|
52
|
+
table: event.table,
|
|
53
|
+
operation: event.operation
|
|
54
|
+
)
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
runtime =
|
|
59
|
+
CDC::Parallel::Runtime.new(
|
|
60
|
+
processor: MetricsProcessor.new,
|
|
61
|
+
size: 4
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
result = runtime.process(event)
|
|
65
|
+
|
|
66
|
+
runtime.shutdown
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
## Processor Safety
|
|
70
|
+
|
|
71
|
+
Only processors that declare `ractor_safe!` can run in this runtime.
|
|
72
|
+
|
|
73
|
+
```ruby
|
|
74
|
+
class AnalyticsProcessor < CDC::Core::Processor
|
|
75
|
+
ractor_safe!
|
|
76
|
+
end
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Unsafe processors raise:
|
|
80
|
+
|
|
81
|
+
```ruby
|
|
82
|
+
CDC::Parallel::UnsafeProcessorError
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
## What Belongs Here
|
|
86
|
+
|
|
87
|
+
- Ractor processor execution
|
|
88
|
+
- Transaction envelope processing
|
|
89
|
+
- Processor safety validation
|
|
90
|
+
- Graceful shutdown
|
|
91
|
+
- Result normalization
|
|
92
|
+
|
|
93
|
+
## What Does Not Belong Here
|
|
94
|
+
|
|
95
|
+
- PostgreSQL connection handling
|
|
96
|
+
- pgoutput parsing
|
|
97
|
+
- pgoutput decoding
|
|
98
|
+
- Rails integration
|
|
99
|
+
- Audit persistence
|
|
100
|
+
- Kafka/Redis/S3 publishing
|
|
101
|
+
|
|
102
|
+
## Ecosystem Position
|
|
103
|
+
|
|
104
|
+
```text
|
|
105
|
+
cdc-parallel
|
|
106
|
+
│
|
|
107
|
+
▼
|
|
108
|
+
pgoutput-parser
|
|
109
|
+
│
|
|
110
|
+
▼
|
|
111
|
+
pgoutput-decoder
|
|
112
|
+
│
|
|
113
|
+
▼
|
|
114
|
+
cdc-core
|
|
115
|
+
│
|
|
116
|
+
▼
|
|
117
|
+
cdc-parallel
|
|
118
|
+
│
|
|
119
|
+
▼
|
|
120
|
+
whodunit-chronicles
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
## Roadmap
|
|
124
|
+
|
|
125
|
+
- Persistent worker pools using `parallel-pool`
|
|
126
|
+
- Mixed `CompositeProcessor` routing
|
|
127
|
+
- Ratomic-backed queues
|
|
128
|
+
- Ratomic-backed metrics
|
|
129
|
+
- Backpressure policies
|
|
130
|
+
- Transaction ordering strategies
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
## Test Organization
|
|
134
|
+
|
|
135
|
+
The test suite is grouped by intent so the same structure can be reused across CDC ecosystem gems.
|
|
136
|
+
|
|
137
|
+
```text
|
|
138
|
+
test/unit/ focused class and branch coverage
|
|
139
|
+
test/integration/ component interaction and runtime integration
|
|
140
|
+
test/behavior/ ecosystem contracts and guardrails
|
|
141
|
+
test/performance/ opt-in smoke benchmarks
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
Run the default quality suite:
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
bundle exec rake test
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
Run a specific group:
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
bundle exec rake test:unit
|
|
154
|
+
bundle exec rake test:integration
|
|
155
|
+
bundle exec rake test:behavior
|
|
156
|
+
bundle exec rake test:performance
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
The default `test` task runs unit, integration, and behavior tests. Performance tests are intentionally separate because they are environment-sensitive.
|
|
160
|
+
|
|
161
|
+
## License
|
|
162
|
+
|
|
163
|
+
MIT.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CDC
|
|
4
|
+
module Parallel
|
|
5
|
+
# Immutable configuration for Ractor runtimes.
|
|
6
|
+
#
|
|
7
|
+
# @!attribute size
|
|
8
|
+
# @return [Integer] worker count.
|
|
9
|
+
# @!attribute timeout
|
|
10
|
+
# @return [Float, nil] optional wait timeout in seconds.
|
|
11
|
+
class Configuration < Data.define(:size, :timeout)
|
|
12
|
+
def initialize(size: Etc.nprocessors, timeout: nil)
|
|
13
|
+
raise ArgumentError, "size must be an Integer" unless size.is_a?(Integer)
|
|
14
|
+
raise ArgumentError, "size must be greater than zero" unless size.positive?
|
|
15
|
+
|
|
16
|
+
super
|
|
17
|
+
::Ractor.make_shareable(self)
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CDC
|
|
4
|
+
module Parallel
|
|
5
|
+
# Base cdc-parallel error.
|
|
6
|
+
class Error < StandardError; end
|
|
7
|
+
|
|
8
|
+
# Raised when a processor has not declared itself Ractor-safe.
|
|
9
|
+
class UnsafeProcessorError < Error; end
|
|
10
|
+
|
|
11
|
+
# Raised when work is submitted after shutdown.
|
|
12
|
+
class ShutdownError < Error; end
|
|
13
|
+
|
|
14
|
+
# Raised when the runtime receives an unsupported work item.
|
|
15
|
+
class UnsupportedWorkItemError < Error; end
|
|
16
|
+
|
|
17
|
+
# Raised when processor execution fails inside a worker Ractor.
|
|
18
|
+
class ProcessorExecutionError < Error
|
|
19
|
+
attr_reader :original_class, :original_message, :original_backtrace
|
|
20
|
+
|
|
21
|
+
def initialize(original_class:, original_message:, original_backtrace: [])
|
|
22
|
+
@original_class = original_class
|
|
23
|
+
@original_message = original_message
|
|
24
|
+
@original_backtrace = original_backtrace.freeze
|
|
25
|
+
|
|
26
|
+
super("#{original_class}: #{original_message}")
|
|
27
|
+
set_backtrace(@original_backtrace) unless @original_backtrace.empty?
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# Raised when a worker does not return a result before timeout.
|
|
32
|
+
class TimeoutError < Error; end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CDC
|
|
4
|
+
module Parallel
|
|
5
|
+
# Executes one Ractor-safe processor in isolated Ractor workers.
|
|
6
|
+
#
|
|
7
|
+
# This v0.1 implementation intentionally uses one-shot worker Ractors for
|
|
8
|
+
# deterministic synchronous semantics while preserving the public pool API.
|
|
9
|
+
# The parallel-pool dependency is kept as the runtime foundation for later
|
|
10
|
+
# async/throughput-focused versions.
|
|
11
|
+
class ProcessorPool
|
|
12
|
+
# @param processor [CDC::Core::Processor]
|
|
13
|
+
# @param size [Integer]
|
|
14
|
+
# @param timeout [Float, nil]
|
|
15
|
+
# @return [void]
|
|
16
|
+
def initialize(processor:, size: Etc.nprocessors, timeout: nil)
|
|
17
|
+
validate_processor!(processor)
|
|
18
|
+
|
|
19
|
+
@processor = ::Ractor.make_shareable(processor)
|
|
20
|
+
@configuration = Configuration.new(size:, timeout:)
|
|
21
|
+
@shutdown = false
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Process one ChangeEvent.
|
|
25
|
+
#
|
|
26
|
+
# @param event [CDC::Core::ChangeEvent]
|
|
27
|
+
# @return [CDC::Core::ProcessorResult]
|
|
28
|
+
def process(event)
|
|
29
|
+
raise ShutdownError, "processor pool has been shut down" if @shutdown
|
|
30
|
+
|
|
31
|
+
work = ::Ractor.make_shareable(event)
|
|
32
|
+
worker = ::Ractor.new(@processor, work) do |processor, item|
|
|
33
|
+
CDC::Parallel::ResultCollector.normalize(processor.process(item))
|
|
34
|
+
rescue StandardError => e
|
|
35
|
+
CDC::Parallel::ResultCollector.worker_failure(e)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
ResultCollector.normalize(take(worker))
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Shut down the pool.
|
|
42
|
+
#
|
|
43
|
+
# @return [void]
|
|
44
|
+
def shutdown
|
|
45
|
+
@shutdown = true
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
private
|
|
49
|
+
|
|
50
|
+
def validate_processor!(processor)
|
|
51
|
+
return if processor.class.respond_to?(:ractor_safe?) && processor.class.ractor_safe?
|
|
52
|
+
|
|
53
|
+
raise UnsafeProcessorError, "#{processor.class} must declare ractor_safe!"
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def take(worker)
|
|
57
|
+
if worker.respond_to?(:value)
|
|
58
|
+
worker.value
|
|
59
|
+
else
|
|
60
|
+
worker.take
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CDC
|
|
4
|
+
module Parallel
|
|
5
|
+
# Normalizes values returned by workers.
|
|
6
|
+
class ResultCollector
|
|
7
|
+
FAILURE_MARKER = :__cdc_parallel_failure__
|
|
8
|
+
|
|
9
|
+
# Build a shareable failure payload that can safely cross a Ractor boundary.
|
|
10
|
+
#
|
|
11
|
+
# @param error [Exception]
|
|
12
|
+
# @return [Hash]
|
|
13
|
+
def self.worker_failure(error)
|
|
14
|
+
::Ractor.make_shareable(
|
|
15
|
+
{
|
|
16
|
+
type: FAILURE_MARKER,
|
|
17
|
+
class: error.class.name,
|
|
18
|
+
message: error.message,
|
|
19
|
+
backtrace: (error.backtrace || []).map { |line| String(line) }.freeze
|
|
20
|
+
}
|
|
21
|
+
)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Normalize a worker return value into a ProcessorResult.
|
|
25
|
+
#
|
|
26
|
+
# @param value [Object]
|
|
27
|
+
# @return [CDC::Core::ProcessorResult]
|
|
28
|
+
def self.normalize(value)
|
|
29
|
+
if worker_failure?(value)
|
|
30
|
+
CDC::Core::ProcessorResult.failure(
|
|
31
|
+
ProcessorExecutionError.new(
|
|
32
|
+
original_class: value[:class],
|
|
33
|
+
original_message: value[:message],
|
|
34
|
+
original_backtrace: value[:backtrace]
|
|
35
|
+
)
|
|
36
|
+
)
|
|
37
|
+
elsif value.is_a?(CDC::Core::ProcessorResult)
|
|
38
|
+
value
|
|
39
|
+
else
|
|
40
|
+
CDC::Core::ProcessorResult.success(value)
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def self.worker_failure?(value)
|
|
45
|
+
value.is_a?(Hash) && value[:type] == FAILURE_MARKER
|
|
46
|
+
end
|
|
47
|
+
private_class_method :worker_failure?
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CDC
|
|
4
|
+
module Parallel
|
|
5
|
+
# Routes supported CDC objects to the correct runtime pool.
|
|
6
|
+
class Router
|
|
7
|
+
# @param processor_pool [ProcessorPool]
|
|
8
|
+
# @param transaction_pool [TransactionPool]
|
|
9
|
+
def initialize(processor_pool:, transaction_pool:)
|
|
10
|
+
@processor_pool = processor_pool
|
|
11
|
+
@transaction_pool = transaction_pool
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# Process a supported CDC work item.
|
|
15
|
+
#
|
|
16
|
+
# @param item [CDC::Core::ChangeEvent, CDC::Core::TransactionEnvelope]
|
|
17
|
+
# @return [CDC::Core::ProcessorResult]
|
|
18
|
+
def process(item)
|
|
19
|
+
case item
|
|
20
|
+
when CDC::Core::ChangeEvent
|
|
21
|
+
@processor_pool.process(item)
|
|
22
|
+
when CDC::Core::TransactionEnvelope
|
|
23
|
+
@transaction_pool.process(item)
|
|
24
|
+
else
|
|
25
|
+
raise UnsupportedWorkItemError, "unsupported CDC work item: #{item.class}"
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CDC
|
|
4
|
+
module Parallel
|
|
5
|
+
# High-level Ractor runtime facade for cdc-core processors.
|
|
6
|
+
class Runtime
|
|
7
|
+
# @param processor [CDC::Core::Processor]
|
|
8
|
+
# @param size [Integer]
|
|
9
|
+
# @param timeout [Float, nil]
|
|
10
|
+
# @return [void]
|
|
11
|
+
def initialize(processor:, size: Etc.nprocessors, timeout: nil)
|
|
12
|
+
@processor_pool = ProcessorPool.new(processor:, size:, timeout:)
|
|
13
|
+
@transaction_pool = TransactionPool.new(processor:, size:, timeout:)
|
|
14
|
+
@router = Router.new(processor_pool: @processor_pool, transaction_pool: @transaction_pool)
|
|
15
|
+
@shutdown = false
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# Process a ChangeEvent or TransactionEnvelope.
|
|
19
|
+
#
|
|
20
|
+
# @param item [CDC::Core::ChangeEvent, CDC::Core::TransactionEnvelope]
|
|
21
|
+
# @return [CDC::Core::ProcessorResult]
|
|
22
|
+
def process(item)
|
|
23
|
+
raise ShutdownError, "runtime has been shut down" if @shutdown
|
|
24
|
+
|
|
25
|
+
@router.process(item)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Alias for transaction-oriented processing.
|
|
29
|
+
#
|
|
30
|
+
# @param transaction [CDC::Core::TransactionEnvelope]
|
|
31
|
+
# @return [CDC::Core::ProcessorResult]
|
|
32
|
+
def process_transaction(transaction)
|
|
33
|
+
process(transaction)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Shut down all runtime resources.
|
|
37
|
+
#
|
|
38
|
+
# @return [void]
|
|
39
|
+
def shutdown
|
|
40
|
+
return if @shutdown
|
|
41
|
+
|
|
42
|
+
@shutdown = true
|
|
43
|
+
@processor_pool.shutdown
|
|
44
|
+
@transaction_pool.shutdown
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CDC
|
|
4
|
+
module Parallel
|
|
5
|
+
# Processes a TransactionEnvelope as a single ordering-preserving unit.
|
|
6
|
+
class TransactionPool
|
|
7
|
+
# @param processor [CDC::Core::Processor]
|
|
8
|
+
# @param size [Integer]
|
|
9
|
+
# @param timeout [Float, nil]
|
|
10
|
+
def initialize(processor:, size: Etc.nprocessors, timeout: nil)
|
|
11
|
+
@processor_pool = ProcessorPool.new(processor:, size:, timeout:)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# Process all events inside a transaction envelope.
|
|
15
|
+
#
|
|
16
|
+
# @param transaction [CDC::Core::TransactionEnvelope]
|
|
17
|
+
# @return [CDC::Core::ProcessorResult]
|
|
18
|
+
def process(transaction)
|
|
19
|
+
results = transaction.events.map { |event| @processor_pool.process(event) }.freeze
|
|
20
|
+
ResultCollector.normalize(results)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# Shut down worker resources.
|
|
24
|
+
#
|
|
25
|
+
# @return [void]
|
|
26
|
+
def shutdown
|
|
27
|
+
@processor_pool.shutdown
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
data/lib/cdc/parallel.rb
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "etc"
|
|
4
|
+
require "ractor-pool"
|
|
5
|
+
|
|
6
|
+
require_relative "parallel/version"
|
|
7
|
+
require_relative "parallel/errors"
|
|
8
|
+
require_relative "parallel/configuration"
|
|
9
|
+
require_relative "parallel/result_collector"
|
|
10
|
+
require_relative "parallel/processor_pool"
|
|
11
|
+
require_relative "parallel/transaction_pool"
|
|
12
|
+
require_relative "parallel/router"
|
|
13
|
+
require_relative "parallel/runtime"
|
|
14
|
+
|
|
15
|
+
module CDC
|
|
16
|
+
# Optional high-throughput Ractor runtime for cdc-core processors.
|
|
17
|
+
module Parallel
|
|
18
|
+
end
|
|
19
|
+
end
|
data/lib/cdc_parallel.rb
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
module CDC
|
|
2
|
+
module Parallel
|
|
3
|
+
# Base cdc-parallel error.
|
|
4
|
+
class Error < StandardError
|
|
5
|
+
end
|
|
6
|
+
|
|
7
|
+
# Raised when a processor has not declared itself Ractor-safe.
|
|
8
|
+
class UnsafeProcessorError < Error
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
# Raised when work is submitted after shutdown.
|
|
12
|
+
class ShutdownError < Error
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# Raised when the runtime receives an unsupported work item.
|
|
16
|
+
class UnsupportedWorkItemError < Error
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# Raised when processor execution fails inside a worker Ractor.
|
|
20
|
+
class ProcessorExecutionError < Error
|
|
21
|
+
@original_class: untyped
|
|
22
|
+
|
|
23
|
+
@original_message: untyped
|
|
24
|
+
|
|
25
|
+
@original_backtrace: untyped
|
|
26
|
+
|
|
27
|
+
attr_reader original_class: untyped
|
|
28
|
+
|
|
29
|
+
attr_reader original_message: untyped
|
|
30
|
+
|
|
31
|
+
attr_reader original_backtrace: untyped
|
|
32
|
+
|
|
33
|
+
def initialize: (original_class: untyped, original_message: untyped, ?original_backtrace: untyped) -> void
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Raised when a worker does not return a result before timeout.
|
|
37
|
+
class TimeoutError < Error
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
module CDC
|
|
2
|
+
module Parallel
|
|
3
|
+
# Executes one Ractor-safe processor in isolated Ractor workers.
|
|
4
|
+
#
|
|
5
|
+
# This v0.1 implementation intentionally uses one-shot worker Ractors for
|
|
6
|
+
# deterministic synchronous semantics while preserving the public pool API.
|
|
7
|
+
# The parallel-pool dependency is kept as the runtime foundation for later
|
|
8
|
+
# async/throughput-focused versions.
|
|
9
|
+
class ProcessorPool
|
|
10
|
+
@processor: untyped
|
|
11
|
+
|
|
12
|
+
@configuration: untyped
|
|
13
|
+
|
|
14
|
+
@shutdown: untyped
|
|
15
|
+
|
|
16
|
+
# @param processor [CDC::Core::Processor]
|
|
17
|
+
# @param size [Integer]
|
|
18
|
+
# @param timeout [Float, nil]
|
|
19
|
+
# @return [void]
|
|
20
|
+
def initialize: (processor: untyped, ?size: untyped, ?timeout: untyped?) -> void
|
|
21
|
+
|
|
22
|
+
# Process one ChangeEvent.
|
|
23
|
+
#
|
|
24
|
+
# @param event [CDC::Core::ChangeEvent]
|
|
25
|
+
# @return [CDC::Core::ProcessorResult]
|
|
26
|
+
def process: (untyped event) -> untyped
|
|
27
|
+
|
|
28
|
+
# Shut down the pool.
|
|
29
|
+
#
|
|
30
|
+
# @return [void]
|
|
31
|
+
def shutdown: () -> untyped
|
|
32
|
+
|
|
33
|
+
private
|
|
34
|
+
|
|
35
|
+
def validate_processor!: (untyped processor) -> (nil | untyped)
|
|
36
|
+
|
|
37
|
+
def take: (untyped worker) -> untyped
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
module CDC
|
|
2
|
+
module Parallel
|
|
3
|
+
# Normalizes values returned by workers.
|
|
4
|
+
class ResultCollector
|
|
5
|
+
FAILURE_MARKER: :__cdc_parallel_failure__
|
|
6
|
+
|
|
7
|
+
# Build a shareable failure payload that can safely cross a Ractor boundary.
|
|
8
|
+
#
|
|
9
|
+
# @param error [Exception]
|
|
10
|
+
# @return [Hash]
|
|
11
|
+
def self.worker_failure: (untyped error) -> untyped
|
|
12
|
+
|
|
13
|
+
# Normalize a worker return value into a ProcessorResult.
|
|
14
|
+
#
|
|
15
|
+
# @param value [Object]
|
|
16
|
+
# @return [CDC::Core::ProcessorResult]
|
|
17
|
+
def self.normalize: (untyped value) -> untyped
|
|
18
|
+
|
|
19
|
+
def self.worker_failure?: (untyped value) -> untyped
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
module CDC
|
|
2
|
+
module Parallel
|
|
3
|
+
# Routes supported CDC objects to the correct runtime pool.
|
|
4
|
+
class Router
|
|
5
|
+
@processor_pool: untyped
|
|
6
|
+
|
|
7
|
+
@transaction_pool: untyped
|
|
8
|
+
|
|
9
|
+
# @param processor_pool [ProcessorPool]
|
|
10
|
+
# @param transaction_pool [TransactionPool]
|
|
11
|
+
def initialize: (processor_pool: untyped, transaction_pool: untyped) -> void
|
|
12
|
+
|
|
13
|
+
# Process a supported CDC work item.
|
|
14
|
+
#
|
|
15
|
+
# @param item [CDC::Core::ChangeEvent, CDC::Core::TransactionEnvelope]
|
|
16
|
+
# @return [CDC::Core::ProcessorResult]
|
|
17
|
+
def process: (untyped item) -> untyped
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
module CDC
|
|
2
|
+
module Parallel
|
|
3
|
+
# High-level Ractor runtime facade for cdc-core processors.
|
|
4
|
+
class Runtime
|
|
5
|
+
@processor_pool: untyped
|
|
6
|
+
|
|
7
|
+
@transaction_pool: untyped
|
|
8
|
+
|
|
9
|
+
@router: untyped
|
|
10
|
+
|
|
11
|
+
@shutdown: untyped
|
|
12
|
+
|
|
13
|
+
# @param processor [CDC::Core::Processor]
|
|
14
|
+
# @param size [Integer]
|
|
15
|
+
# @param timeout [Float, nil]
|
|
16
|
+
# @return [void]
|
|
17
|
+
def initialize: (processor: untyped, ?size: untyped, ?timeout: untyped?) -> void
|
|
18
|
+
|
|
19
|
+
# Process a ChangeEvent or TransactionEnvelope.
|
|
20
|
+
#
|
|
21
|
+
# @param item [CDC::Core::ChangeEvent, CDC::Core::TransactionEnvelope]
|
|
22
|
+
# @return [CDC::Core::ProcessorResult]
|
|
23
|
+
def process: (untyped item) -> untyped
|
|
24
|
+
|
|
25
|
+
# Alias for transaction-oriented processing.
|
|
26
|
+
#
|
|
27
|
+
# @param transaction [CDC::Core::TransactionEnvelope]
|
|
28
|
+
# @return [CDC::Core::ProcessorResult]
|
|
29
|
+
def process_transaction: (untyped transaction) -> untyped
|
|
30
|
+
|
|
31
|
+
# Shut down all runtime resources.
|
|
32
|
+
#
|
|
33
|
+
# @return [void]
|
|
34
|
+
def shutdown: () -> (nil | untyped)
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
module CDC
|
|
2
|
+
module Parallel
|
|
3
|
+
# Processes a TransactionEnvelope as a single ordering-preserving unit.
|
|
4
|
+
class TransactionPool
|
|
5
|
+
@processor_pool: untyped
|
|
6
|
+
|
|
7
|
+
# @param processor [CDC::Core::Processor]
|
|
8
|
+
# @param size [Integer]
|
|
9
|
+
# @param timeout [Float, nil]
|
|
10
|
+
def initialize: (processor: untyped, ?size: untyped, ?timeout: untyped?) -> void
|
|
11
|
+
|
|
12
|
+
# Process all events inside a transaction envelope.
|
|
13
|
+
#
|
|
14
|
+
# @param transaction [CDC::Core::TransactionEnvelope]
|
|
15
|
+
# @return [CDC::Core::ProcessorResult]
|
|
16
|
+
def process: (untyped transaction) -> untyped
|
|
17
|
+
|
|
18
|
+
# Shut down worker resources.
|
|
19
|
+
#
|
|
20
|
+
# @return [void]
|
|
21
|
+
def shutdown: () -> untyped
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
File without changes
|
|
File without changes
|
data/sig/shims/etc.rbs
ADDED
metadata
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: cdc-parallel
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Ken C. Demanawa
|
|
8
|
+
bindir: bin
|
|
9
|
+
cert_chain: []
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
|
+
dependencies:
|
|
12
|
+
- !ruby/object:Gem::Dependency
|
|
13
|
+
name: cdc-core
|
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
|
15
|
+
requirements:
|
|
16
|
+
- - "~>"
|
|
17
|
+
- !ruby/object:Gem::Version
|
|
18
|
+
version: '0.1'
|
|
19
|
+
type: :runtime
|
|
20
|
+
prerelease: false
|
|
21
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
22
|
+
requirements:
|
|
23
|
+
- - "~>"
|
|
24
|
+
- !ruby/object:Gem::Version
|
|
25
|
+
version: '0.1'
|
|
26
|
+
- !ruby/object:Gem::Dependency
|
|
27
|
+
name: ractor-pool
|
|
28
|
+
requirement: !ruby/object:Gem::Requirement
|
|
29
|
+
requirements:
|
|
30
|
+
- - "~>"
|
|
31
|
+
- !ruby/object:Gem::Version
|
|
32
|
+
version: 0.4.0
|
|
33
|
+
type: :runtime
|
|
34
|
+
prerelease: false
|
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
36
|
+
requirements:
|
|
37
|
+
- - "~>"
|
|
38
|
+
- !ruby/object:Gem::Version
|
|
39
|
+
version: 0.4.0
|
|
40
|
+
description: |
|
|
41
|
+
cdc-parallel provides optional Ractor-backed parallel execution for
|
|
42
|
+
cdc-core. It accelerates Change Data Capture (CDC) pipelines while
|
|
43
|
+
preserving the simplicity and composability of the CDC Ecosystem.
|
|
44
|
+
email:
|
|
45
|
+
- kenneth.c.demanawa@gmail.com
|
|
46
|
+
executables: []
|
|
47
|
+
extensions: []
|
|
48
|
+
extra_rdoc_files: []
|
|
49
|
+
files:
|
|
50
|
+
- CHANGELOG.md
|
|
51
|
+
- LICENSE.txt
|
|
52
|
+
- README.md
|
|
53
|
+
- lib/cdc/parallel.rb
|
|
54
|
+
- lib/cdc/parallel/configuration.rb
|
|
55
|
+
- lib/cdc/parallel/errors.rb
|
|
56
|
+
- lib/cdc/parallel/processor_pool.rb
|
|
57
|
+
- lib/cdc/parallel/result_collector.rb
|
|
58
|
+
- lib/cdc/parallel/router.rb
|
|
59
|
+
- lib/cdc/parallel/runtime.rb
|
|
60
|
+
- lib/cdc/parallel/transaction_pool.rb
|
|
61
|
+
- lib/cdc/parallel/version.rb
|
|
62
|
+
- lib/cdc_parallel.rb
|
|
63
|
+
- sig/cdc/parallel.rbs
|
|
64
|
+
- sig/cdc/parallel/configuration.rbs
|
|
65
|
+
- sig/cdc/parallel/errors.rbs
|
|
66
|
+
- sig/cdc/parallel/processor_pool.rbs
|
|
67
|
+
- sig/cdc/parallel/result_collector.rbs
|
|
68
|
+
- sig/cdc/parallel/router.rbs
|
|
69
|
+
- sig/cdc/parallel/runtime.rbs
|
|
70
|
+
- sig/cdc/parallel/transaction_pool.rbs
|
|
71
|
+
- sig/cdc/parallel/version.rbs
|
|
72
|
+
- sig/cdc_parallel.rbs
|
|
73
|
+
- sig/shims/cdc_core.rbs
|
|
74
|
+
- sig/shims/data_define.rbs
|
|
75
|
+
- sig/shims/etc.rbs
|
|
76
|
+
homepage: https://kanutocd.github.io/cdc-parallel/
|
|
77
|
+
licenses:
|
|
78
|
+
- MIT
|
|
79
|
+
metadata:
|
|
80
|
+
homepage_uri: https://kanutocd.github.io/cdc-parallel/
|
|
81
|
+
source_code_uri: https://github.com/kanutocd/cdc-parallel
|
|
82
|
+
changelog_uri: https://github.com/kanutocd/cdc-parallel/blob/main/CHANGELOG.md
|
|
83
|
+
documentation_uri: https://kanutocd.github.io/cdc-parallel/
|
|
84
|
+
rubygems_mfa_required: 'true'
|
|
85
|
+
rdoc_options: []
|
|
86
|
+
require_paths:
|
|
87
|
+
- lib
|
|
88
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
89
|
+
requirements:
|
|
90
|
+
- - ">="
|
|
91
|
+
- !ruby/object:Gem::Version
|
|
92
|
+
version: '4.0'
|
|
93
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
94
|
+
requirements:
|
|
95
|
+
- - ">="
|
|
96
|
+
- !ruby/object:Gem::Version
|
|
97
|
+
version: '0'
|
|
98
|
+
requirements: []
|
|
99
|
+
rubygems_version: 4.0.10
|
|
100
|
+
specification_version: 4
|
|
101
|
+
summary: Optional parallel runtime for the CDC Ecosystem.
|
|
102
|
+
test_files: []
|