offset_machines 0.5.1-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +73 -0
- data/LICENSE.txt +21 -0
- data/README.md +270 -0
- data/ext/offset_machines_native/core/CHANGELOG.md +52 -0
- data/ext/offset_machines_native/core/README.md +81 -0
- data/ext/offset_machines_native/extconf.rb +2 -0
- data/lib/offset_machines/version.rb +5 -0
- data/lib/offset_machines.rb +39 -0
- data/lib/offset_machines_native/offset_machines_native.so +0 -0
- data/sig/offset_machines.rbs +52 -0
- metadata +61 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: d8bc0de2146d7c36df2588db124d552780401e7a24197b5bd9e96b1a834fde5f
|
|
4
|
+
data.tar.gz: 6ff753da34c575ce153b31720825b0149b0f4745bb96f06e2248da6e0df15216
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 857fe577d6a59dfc9aaf37f3f16f6f5b205f777ff8b512d0b7125ba7c74183c8f6ead190e8468ccfe1ad97a3d735c9bf8b1a4bc0b54d6bf74288e3e2094396c9
|
|
7
|
+
data.tar.gz: e730733c793989668357d351cb54f425058e9dcffd7b278eedd06ea2d2e2822c3dd08bc40665a8631172524d3c4804b186a300e6acae3fa7837ded3a16a75485
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## [0.5.1](https://github.com/seuros/offset_machines/compare/offset_machines/v0.5.0...offset_machines/v0.5.1) (2026-11-16)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Bug Fixes
|
|
7
|
+
|
|
8
|
+
* fix wrapped classes at load so building a value runs no Ruby ([2c922c7](https://github.com/seuros/offset_machines/commit/2c922c777837a19e8d10535d473616c7b95217b7))
|
|
9
|
+
|
|
10
|
+
## [0.5.0](https://github.com/seuros/offset_machines/compare/offset_machines/v0.4.2...offset_machines/v0.5.0) (2026-11-16)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
### ⚠ BREAKING CHANGES
|
|
14
|
+
|
|
15
|
+
* seal Tracker, add Tracker.assigned and word-skipping gaps
|
|
16
|
+
|
|
17
|
+
### Features
|
|
18
|
+
|
|
19
|
+
* seal Tracker, add Tracker.assigned and word-skipping gaps ([e1e46e8](https://github.com/seuros/offset_machines/commit/e1e46e845103b9b356fa480938dd49df29056ce4))
|
|
20
|
+
|
|
21
|
+
## [0.4.2](https://github.com/seuros/offset_machines/compare/offset_machines/v0.4.1...offset_machines/v0.4.2) (2026-11-14)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
### Bug Fixes
|
|
25
|
+
|
|
26
|
+
* **deps:** require matryona-machines >= 0.4 ([2eb009b](https://github.com/seuros/offset_machines/commit/2eb009bca4484ce5c18051152640f02c8c75ab2b))
|
|
27
|
+
|
|
28
|
+
## [0.4.1](https://github.com/seuros/offset_machines/compare/offset_machines/v0.4.0...offset_machines/v0.4.1) (2026-11-14)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
### Bug Fixes
|
|
32
|
+
|
|
33
|
+
* **deps:** require matryona-machines >= 0.3 ([444c58e](https://github.com/seuros/offset_machines/commit/444c58e13ba7858a1a08c50ff9ff07690ac0136e))
|
|
34
|
+
|
|
35
|
+
## [0.4.0](https://github.com/seuros/offset_machines/compare/offset_machines/v0.3.0...offset_machines/v0.4.0) (2026-11-14)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
### ⚠ BREAKING CHANGES
|
|
39
|
+
|
|
40
|
+
* build the binding on matryona-machines 0.2; optional marks, compiler-checked symbols; ship no test files
|
|
41
|
+
|
|
42
|
+
### Features
|
|
43
|
+
|
|
44
|
+
* build the binding on matryona-machines 0.2; optional marks, compiler-checked symbols; ship no test files ([57a0356](https://github.com/seuros/offset_machines/commit/57a03561720799184005b4ff9b9debe857f80fb7))
|
|
45
|
+
|
|
46
|
+
## [0.3.0](https://github.com/seuros/offset_machines/compare/offset_machines/v0.2.0...offset_machines/v0.3.0) (2026-11-14)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
### ⚠ BREAKING CHANGES
|
|
50
|
+
|
|
51
|
+
* validate Status through the core, seal it, and consolidate tracker internals
|
|
52
|
+
|
|
53
|
+
### Features
|
|
54
|
+
|
|
55
|
+
* validate Status through the core, seal it, and consolidate tracker internals ([3cc7dd3](https://github.com/seuros/offset_machines/commit/3cc7dd3e160c178f1d368464b1dd4029a60be7bd))
|
|
56
|
+
|
|
57
|
+
## [0.2.0](https://github.com/seuros/offset_machines/compare/offset_machines/v0.1.1...offset_machines/v0.2.0) (2026-11-13)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
### ⚠ BREAKING CHANGES
|
|
61
|
+
|
|
62
|
+
* see CONVENTIONS: one exhaustive Refusal per crate, Result from every mutator, honoured idempotent repeats, Ruby mirrors the core. No compatibility aliases. Ruby ack answers :recorded; reassign answers :exhausted instead of nil.
|
|
63
|
+
|
|
64
|
+
### Features
|
|
65
|
+
|
|
66
|
+
* report refusals through Result<_, Refusal>, keep duplicate acks and commits honoured, add is_acked ([d1ac915](https://github.com/seuros/offset_machines/commit/d1ac9152bfc994abcc5f6bab20220a3961b5043b))
|
|
67
|
+
|
|
68
|
+
## [0.1.1](https://github.com/seuros/offset_machines/compare/offset_machines-v0.1.0...offset_machines/v0.1.1) (2026-11-12)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
### Features
|
|
72
|
+
|
|
73
|
+
* initial extraction ([e061fd5](https://github.com/seuros/offset_machines/commit/e061fd59c8326783c70701e249d48f55805996e8))
|
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Abdelkader Boudih
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
# OffsetMachines
|
|
2
|
+
|
|
3
|
+
> Offset 102 finishing does not mean offset 101 did.
|
|
4
|
+
|
|
5
|
+
An acknowledgement tracker for one partition, for Ruby. Your workers finish offsets in whatever order they like. You tell the tracker which ones are done. It tells you the one number that is safe to commit, and which offsets you are still waiting for. You persist that number yourself and confirm it back. Only then does the window move.
|
|
6
|
+
|
|
7
|
+
This is an initial release and the API is unstable. It is a policy core, not a consumer runtime: it does not fetch, run handlers, talk to a broker, or store anything.
|
|
8
|
+
|
|
9
|
+
## Quick Start
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
gem 'offset_machines'
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
```ruby
|
|
16
|
+
tracker = OffsetMachines::Tracker.new
|
|
17
|
+
tracker.assign(1, 100) # generation 1, the store says 100 is next
|
|
18
|
+
|
|
19
|
+
def finished(tracker, generation, offset, store)
|
|
20
|
+
case tracker.ack(generation, offset) # :recorded and :duplicate both mean "it is acked"
|
|
21
|
+
when :stale
|
|
22
|
+
return # the partition was reassigned. This worker's result is not ours.
|
|
23
|
+
when :out_of_window
|
|
24
|
+
raise "offset #{offset} is past the window, stop fetching ahead"
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
candidate = tracker.commit_candidate or return
|
|
28
|
+
store.commit(candidate)
|
|
29
|
+
tracker.mark_committed(generation, candidate)
|
|
30
|
+
end
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
One tracker per partition. Generations and offsets are Integers. There is no clock in here at all.
|
|
34
|
+
|
|
35
|
+
## A Message to the Resistance
|
|
36
|
+
|
|
37
|
+
So you made your consumer concurrent. Ten workers on one partition, and throughput looks great. Offset 105 finishes first, so you commit 106. Then the process dies with 101 still on a worker's desk. The broker hands you back everything from 106 onward, and 101 is never seen again.
|
|
38
|
+
|
|
39
|
+
You know what the docs told you to do. "Commit only what you have processed." So you keep a `highest_done` variable. That is the bug you just had, with a name. So you keep a sorted set of finished offsets and walk it under a mutex. Then a rebalance takes the partition away and gives it back, and a worker from before the rebalance acks an offset into the new assignment's set.
|
|
40
|
+
|
|
41
|
+
The piece you keep rebuilding is a bitmap with a floor: everything below the floor is committed, bits above it are finished work, and the floor only moves across an unbroken run of bits. Plus a generation number, so a worker from a previous assignment gets told no. That bitmap is this gem.
|
|
42
|
+
|
|
43
|
+
## Chapter 1: Stardate 2026.316, the AMS Kreps
|
|
44
|
+
|
|
45
|
+
> **AMS Kreps**, NCC-4220, *Kreps*-class log tender.
|
|
46
|
+
> The log is only as committed as its first hole.
|
|
47
|
+
|
|
48
|
+
The tender doesn't carry the log. It carries the tally sheet. Crews report which entries they have finished, in any order, and the tender keeps one mark: the entry up to which nothing is missing. The ship is named for Jay Kreps, who wrote down why the log is the thing everything else hangs off.
|
|
49
|
+
|
|
50
|
+
On a bridge somewhere, a cadet volunteers:
|
|
51
|
+
|
|
52
|
+
```ruby
|
|
53
|
+
def on_done(offset)
|
|
54
|
+
@highest = [@highest, offset].max
|
|
55
|
+
consumer.commit(@highest + 1)
|
|
56
|
+
end
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
"That," says the tender's tallymaster, "commits every entry you haven't finished yet, provided one after it did."
|
|
60
|
+
|
|
61
|
+
## The Flight Plan
|
|
62
|
+
|
|
63
|
+
Each offset in the window is in one of these states:
|
|
64
|
+
|
|
65
|
+
```mermaid
|
|
66
|
+
stateDiagram-v2
|
|
67
|
+
[*] --> outstanding: assign(generation, base)
|
|
68
|
+
outstanding --> acked: ack, returns :recorded
|
|
69
|
+
acked --> acked: ack again, returns :duplicate
|
|
70
|
+
acked --> committed: mark_committed past it
|
|
71
|
+
committed --> committed: ack again, returns :duplicate
|
|
72
|
+
outstanding --> [*]: newer assign discards the window
|
|
73
|
+
acked --> [*]: newer assign discards the window
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
And the afternoon three workers finished out of order:
|
|
77
|
+
|
|
78
|
+
```mermaid
|
|
79
|
+
sequenceDiagram
|
|
80
|
+
participant W as Workers
|
|
81
|
+
participant T as Tracker
|
|
82
|
+
participant S as Your store
|
|
83
|
+
|
|
84
|
+
W->>T: ack(1, 102)
|
|
85
|
+
T-->>W: :recorded
|
|
86
|
+
Note over T: candidate nil. 100 is missing.
|
|
87
|
+
|
|
88
|
+
W->>T: ack(1, 100)
|
|
89
|
+
T-->>W: :recorded
|
|
90
|
+
Note over T: candidate 101. gap at 101.
|
|
91
|
+
|
|
92
|
+
S->>T: commit_candidate
|
|
93
|
+
T-->>S: 101
|
|
94
|
+
S->>S: persist 101
|
|
95
|
+
S->>T: mark_committed(1, 101)
|
|
96
|
+
T-->>S: :advanced
|
|
97
|
+
|
|
98
|
+
W->>T: ack(1, 101)
|
|
99
|
+
T-->>W: :recorded
|
|
100
|
+
Note over T: candidate 103. No gaps.
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
## How the Tracker Works
|
|
104
|
+
|
|
105
|
+
### Assign
|
|
106
|
+
|
|
107
|
+
```ruby
|
|
108
|
+
tracker = OffsetMachines::Tracker.new
|
|
109
|
+
|
|
110
|
+
tracker.ack(1, 100) # => :stale, nothing is assigned yet
|
|
111
|
+
tracker.assign(1, 100) # => :assigned, generation 1, window starts at 100
|
|
112
|
+
tracker.assign(1, 500) # => :stale, not newer than generation 1
|
|
113
|
+
tracker.generation # => 1
|
|
114
|
+
tracker.base # => 100
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
A new tracker is unassigned and refuses everything. `assign` takes a generation strictly greater than the current one and the offset your store says is next. Generation 0 is never accepted.
|
|
118
|
+
|
|
119
|
+
`assign` answers `:assigned`, or refuses with `:stale`.
|
|
120
|
+
|
|
121
|
+
```ruby
|
|
122
|
+
tracker = OffsetMachines::Tracker.assigned(1, 100) # generation 1, window at 100
|
|
123
|
+
tracker.ack(1, 100) # => :recorded
|
|
124
|
+
OffsetMachines::Tracker.assigned(0, 100) # raises RangeError
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
`Tracker.assigned(generation, base)` is `new` followed by an honoured `assign`, in one step with no refusal to handle: on a new tracker `assign` refuses only generation 0, and `assigned` raises `RangeError` for it instead, because the core's constructor takes a generation that is never zero.
|
|
128
|
+
|
|
129
|
+
### Ack
|
|
130
|
+
|
|
131
|
+
```ruby
|
|
132
|
+
tracker.ack(1, 102) # => :recorded, finished out of order
|
|
133
|
+
tracker.ack(1, 100) # => :recorded
|
|
134
|
+
tracker.ack(1, 100) # => :duplicate, already acked
|
|
135
|
+
tracker.ack(1, 99) # => :duplicate, below the window, already committed
|
|
136
|
+
tracker.ack(2, 101) # => :stale, not the current generation
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
An ack records that an offset finished processing. It does not commit anything.
|
|
140
|
+
|
|
141
|
+
`ack` answers `:recorded` or `:duplicate`, or refuses with `:stale` or `:out_of_window`. `:duplicate` is not a refusal: the offset is acked (or already committed), which is what you asked for. It was just true before you asked.
|
|
142
|
+
|
|
143
|
+
### Commit candidate
|
|
144
|
+
|
|
145
|
+
```ruby
|
|
146
|
+
tracker.commit_candidate # => 101
|
|
147
|
+
tracker.frontier # => 103
|
|
148
|
+
tracker.acked # => 2
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
The candidate is one past the unbroken run of acked offsets starting at `base`. It is the number you would hand a broker as "next offset to read". It is `nil` while `base` itself is unacked. `frontier` is one past the highest acked offset, and `acked` counts the bits.
|
|
152
|
+
|
|
153
|
+
### Acked?
|
|
154
|
+
|
|
155
|
+
```ruby
|
|
156
|
+
tracker.acked?(99) # => true, below the window: committed
|
|
157
|
+
tracker.acked?(100) # => true, recorded
|
|
158
|
+
tracker.acked?(101) # => false, still outstanding
|
|
159
|
+
tracker.acked?(900) # => false, beyond the window nothing is recorded
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
`acked?(offset)` asks whether an offset is already acknowledged under the current generation: it is below `base`, or its bit is set in the window. It is true exactly when `ack(generation, offset)` would answer `:duplicate`, and it changes nothing. It takes no generation because it reads only the current one. A new tracker answers `false` for everything, and so does a fresh assignment for every offset at or above its base.
|
|
163
|
+
|
|
164
|
+
### Gaps
|
|
165
|
+
|
|
166
|
+
```ruby
|
|
167
|
+
tracker.gaps(0, 10) # => [101]
|
|
168
|
+
tracker.ack(1, 105) # => :recorded
|
|
169
|
+
tracker.gaps(0, 10) # => [101, 103, 104]
|
|
170
|
+
tracker.gaps(0, 2) # => [101, 103]
|
|
171
|
+
tracker.gaps(104, 2) # => [104]
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
`gaps(from, limit)` lists the unacked offsets below the frontier, ascending, at most `limit` of them. To page through, ask again from the last one plus one. These are the offsets holding your commit back.
|
|
175
|
+
|
|
176
|
+
### Mark committed
|
|
177
|
+
|
|
178
|
+
```ruby
|
|
179
|
+
tracker.mark_committed(1, 102) # => :unacked, 101 isn't done
|
|
180
|
+
tracker.mark_committed(1, 101) # => :advanced
|
|
181
|
+
tracker.base # => 101
|
|
182
|
+
tracker.mark_committed(1, 101) # => :duplicate
|
|
183
|
+
tracker.commit_candidate # => nil, 101 is the new floor and it's unacked
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
`commit_candidate` only proposes. The window slides when you call `mark_committed` after your store has the offset. You may confirm less than the candidate. You may not confirm more.
|
|
187
|
+
|
|
188
|
+
`mark_committed` answers `:advanced` or `:duplicate`, or refuses with `:stale` or `:unacked`. `:duplicate` means the offset is at or below the floor, so it is already committed. Confirming twice is harmless.
|
|
189
|
+
|
|
190
|
+
```ruby
|
|
191
|
+
tracker.ack(1, 101) # => :recorded
|
|
192
|
+
tracker.commit_candidate # => 103
|
|
193
|
+
tracker.status
|
|
194
|
+
# => #<data OffsetMachines::Status generation=1, base=101, commit_candidate=103, frontier=106, acked=3>
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
### Reassign
|
|
198
|
+
|
|
199
|
+
```ruby
|
|
200
|
+
tracker.reassign(101) # => 2, the new generation
|
|
201
|
+
tracker.acked # => 0, the old window is gone
|
|
202
|
+
tracker.ack(1, 101) # => :stale
|
|
203
|
+
tracker.mark_committed(1, 102) # => :stale
|
|
204
|
+
tracker.ack(2, 101) # => :recorded
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
`reassign(base)` is `assign` with the next generation. It answers the new generation as an Integer, or refuses with `:exhausted` when the generation is already `2**64-1`. Every ack from the previous generation is thrown away, including the ones that had not been committed. Those offsets will be delivered again. That is what a rebalance means.
|
|
208
|
+
|
|
209
|
+
### Capacity
|
|
210
|
+
|
|
211
|
+
The window holds 256 offsets starting at `base`. Past that, `ack` refuses with `:out_of_window` and records nothing:
|
|
212
|
+
|
|
213
|
+
```ruby
|
|
214
|
+
tracker = OffsetMachines::Tracker.new
|
|
215
|
+
tracker.assign(1, 0)
|
|
216
|
+
tracker.capacity # => 256
|
|
217
|
+
OffsetMachines::CAPACITY # => 256
|
|
218
|
+
|
|
219
|
+
tracker.ack(1, 255) # => :recorded
|
|
220
|
+
tracker.ack(1, 256) # => :out_of_window
|
|
221
|
+
tracker.gaps(0, 1000).size # => 255
|
|
222
|
+
|
|
223
|
+
256.times { |i| tracker.ack(1, i) }
|
|
224
|
+
tracker.mark_committed(1, 256) # => :advanced
|
|
225
|
+
tracker.ack(1, 256) # => :recorded
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
The window never grows. An offset that falls outside it is a signal to stop fetching until the slow one finishes, or to shrink your concurrency.
|
|
229
|
+
|
|
230
|
+
## The Rules of the Tender
|
|
231
|
+
|
|
232
|
+
- **Generations and offsets are Integers** in `0..2**64-1`. Floats, nil, and strings raise `TypeError`. Out of range raises `RangeError`.
|
|
233
|
+
- **Refusals don't raise.** `:stale`, `:out_of_window`, `:unacked`, and `:exhausted` are answers. Only malformed input raises. A refused call changes nothing, not one bit.
|
|
234
|
+
- **A repeat is not a refusal.** `:duplicate` from `ack` or `mark_committed` means what you asked for already holds. It is honoured, and it also changes nothing.
|
|
235
|
+
- **Every answer, by method.**
|
|
236
|
+
|
|
237
|
+
| Method | Honoured | Refused | Records the clock |
|
|
238
|
+
| --- | --- | --- | --- |
|
|
239
|
+
| `Tracker.assigned(generation, base)` | a `Tracker` | raises `RangeError` / `TypeError` | never |
|
|
240
|
+
| `assign(generation, base)` | `:assigned` | `:stale` | never |
|
|
241
|
+
| `reassign(base)` | the new generation (Integer) | `:exhausted` | never |
|
|
242
|
+
| `ack(generation, offset)` | `:recorded`, `:duplicate` | `:stale`, `:out_of_window` | never |
|
|
243
|
+
| `mark_committed(generation, offset)` | `:advanced`, `:duplicate` | `:stale`, `:unacked` | never |
|
|
244
|
+
| `Status.new(generation, base, commit_candidate, frontier, acked)` | a `Status` | raises `RangeError` / `TypeError` | never |
|
|
245
|
+
|
|
246
|
+
Each refusal has its own symbol, the snake_case of the core's `Refusal` variant. There is no catch-all. The tracker takes no time at all, so no answer depends on an instant and none records one.
|
|
247
|
+
|
|
248
|
+
- **A `Status` is valid or does not exist.** `Status.new` (positional, in the order above, or by keyword) checks its five values through the core's `Status::new`: they must be a view some tracker of this window could report. A non-Integer raises `TypeError` (the two marks may be `nil`); anything else no tracker reports, such as a frontier past the window, a candidate above the frontier, or an `acked` count that doesn't fit the marks, raises `RangeError`. `Status` is final: subclassing it raises `TypeError` and `allocate` is gone. That stops accidents, not reflection (`Marshal.load`, `Data#initialize` bound by hand); no method of the gem reads a `Status` back, so a forged one never reaches the tracker. `Tracker` is sealed the same way: it wraps native state, so subclassing raises `TypeError` and `allocate` is gone.
|
|
249
|
+
|
|
250
|
+
- **The last offset is reserved.** Offset `2**64-1` is never ackable, so every commit candidate fits in an Integer the tracker accepts back.
|
|
251
|
+
- **One tracker, one partition.** Offsets from two partitions in one tracker is a bug the tracker can't see.
|
|
252
|
+
- **Frozen means frozen.** `freeze` the tracker and `assign`, `reassign`, `ack`, and `mark_committed` all raise `FrozenError`. Reads still work.
|
|
253
|
+
- **Native is required.** There is no pure-Ruby fallback.
|
|
254
|
+
|
|
255
|
+
## What You Think You're Doing vs Reality
|
|
256
|
+
|
|
257
|
+
### You Think: "I'll commit the highest offset I've finished."
|
|
258
|
+
### Reality: You commit every unfinished offset below it.
|
|
259
|
+
|
|
260
|
+
A commit is a claim about everything before a number, not about the number. The only number you can honestly commit is the end of the unbroken run. `commit_candidate` is that number, and it stays put until the slow offset finishes, no matter how far ahead the fast ones get.
|
|
261
|
+
|
|
262
|
+
## What's Yours
|
|
263
|
+
|
|
264
|
+
The tracker lives in your process and forgets everything when the process does. After a restart or a rebalance, `assign` a newer generation at the offset your store holds. Where generations come from is yours too: a consumer-group epoch, a lease token, a counter you persist. Fetching, running handlers, retrying a failed offset, the commit itself and its durability are all outside the gem. An offset is acked when you say so. If you ack before the side effect is durable, the tracker will cheerfully propose committing past work you can still lose.
|
|
265
|
+
|
|
266
|
+
## License
|
|
267
|
+
|
|
268
|
+
MIT License. See [LICENSE.txt](LICENSE.txt) for details.
|
|
269
|
+
|
|
270
|
+
**Remember: the fast offsets will finish first. The question is whether you commit past the slow one.**
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## [0.6.0](https://github.com/seuros/offset_machines/compare/offset-machines-v0.5.0...offset-machines-v0.6.0) (2026-11-16)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### ⚠ BREAKING CHANGES
|
|
7
|
+
|
|
8
|
+
* seal Tracker, add Tracker.assigned and word-skipping gaps
|
|
9
|
+
|
|
10
|
+
### Features
|
|
11
|
+
|
|
12
|
+
* seal Tracker, add Tracker.assigned and word-skipping gaps ([e1e46e8](https://github.com/seuros/offset_machines/commit/e1e46e845103b9b356fa480938dd49df29056ce4))
|
|
13
|
+
|
|
14
|
+
## [0.5.0](https://github.com/seuros/offset_machines/compare/offset-machines-v0.4.0...offset-machines-v0.5.0) (2026-11-14)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
### ⚠ BREAKING CHANGES
|
|
18
|
+
|
|
19
|
+
* build the binding on matryona-machines 0.2; optional marks, compiler-checked symbols; ship no test files
|
|
20
|
+
|
|
21
|
+
### Features
|
|
22
|
+
|
|
23
|
+
* build the binding on matryona-machines 0.2; optional marks, compiler-checked symbols; ship no test files ([57a0356](https://github.com/seuros/offset_machines/commit/57a03561720799184005b4ff9b9debe857f80fb7))
|
|
24
|
+
|
|
25
|
+
## [0.4.0](https://github.com/seuros/offset_machines/compare/offset-machines-v0.3.0...offset-machines-v0.4.0) (2026-11-14)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
### ⚠ BREAKING CHANGES
|
|
29
|
+
|
|
30
|
+
* validate Status through the core, seal it, and consolidate tracker internals
|
|
31
|
+
|
|
32
|
+
### Features
|
|
33
|
+
|
|
34
|
+
* validate Status through the core, seal it, and consolidate tracker internals ([3cc7dd3](https://github.com/seuros/offset_machines/commit/3cc7dd3e160c178f1d368464b1dd4029a60be7bd))
|
|
35
|
+
|
|
36
|
+
## [0.3.0](https://github.com/seuros/offset_machines/compare/offset-machines-v0.2.0...offset-machines-v0.3.0) (2026-11-13)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
### ⚠ BREAKING CHANGES
|
|
40
|
+
|
|
41
|
+
* see CONVENTIONS: one exhaustive Refusal per crate, Result from every mutator, honoured idempotent repeats, Ruby mirrors the core. No compatibility aliases. Ruby ack answers :recorded; reassign answers :exhausted instead of nil.
|
|
42
|
+
|
|
43
|
+
### Features
|
|
44
|
+
|
|
45
|
+
* report refusals through Result<_, Refusal>, keep duplicate acks and commits honoured, add is_acked ([d1ac915](https://github.com/seuros/offset_machines/commit/d1ac9152bfc994abcc5f6bab20220a3961b5043b))
|
|
46
|
+
|
|
47
|
+
## [0.2.0](https://github.com/seuros/offset_machines/compare/offset-machines-v0.1.0...offset-machines-v0.2.0) (2026-11-12)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
### Features
|
|
51
|
+
|
|
52
|
+
* initial extraction ([e061fd5](https://github.com/seuros/offset_machines/commit/e061fd59c8326783c70701e249d48f55805996e8))
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# offset-machines
|
|
2
|
+
|
|
3
|
+
> **AMS Kreps**, NCC-4220, *Kreps*-class log tender.
|
|
4
|
+
|
|
5
|
+
Engine-room flight manual: tally finished offsets and find the first hole in the log.
|
|
6
|
+
|
|
7
|
+
Pre-1.0. Allocation-free `no_std` per-partition acknowledgement
|
|
8
|
+
bitmap: record which offsets finished processing, learn the contiguous commit
|
|
9
|
+
candidate, and list the gaps behind the highest acknowledged offset.
|
|
10
|
+
|
|
11
|
+
This is not a commit log, consumer runtime or offset store. The caller owns
|
|
12
|
+
execution, storage and the commit itself. The tracker takes explicit inputs,
|
|
13
|
+
has a compile-time window (`Tracker<WORDS>` covers `WORDS * 64` offsets) and
|
|
14
|
+
refuses rather than grows.
|
|
15
|
+
|
|
16
|
+
- `assign(generation, base) -> Result<(), Refusal>` installs a strictly newer
|
|
17
|
+
assignment generation and restarts the window; older or equal generations
|
|
18
|
+
(and zero) are refused as `Stale`.
|
|
19
|
+
- `Tracker::assigned(generation, base) -> Tracker` builds a tracker already
|
|
20
|
+
assigned: what an honoured `assign` makes of a fresh one. The generation is
|
|
21
|
+
a `NonZeroU64`, the one value `assign` refuses on a fresh tracker being
|
|
22
|
+
zero, so it cannot fail.
|
|
23
|
+
- `reassign(base) -> Result<u64, Refusal>` bumps the generation by one and
|
|
24
|
+
returns it; `Exhausted` when the generation is already `u64::MAX`.
|
|
25
|
+
- `ack(generation, offset) -> Result<Ack, Refusal>` answers `Ack::Recorded`, or
|
|
26
|
+
`Ack::Duplicate` for an idempotent repeat (already acked, or below the
|
|
27
|
+
window). Refused as `Stale` (wrong generation, or unassigned) or
|
|
28
|
+
`OutOfWindow`. Offset `u64::MAX` is never ackable.
|
|
29
|
+
- `is_acked(offset) -> bool` is the read-only form of that answer: `true` when
|
|
30
|
+
`offset` is below the base (committed) or recorded in the window, which is
|
|
31
|
+
exactly when `ack` under the current generation answers `Ack::Duplicate`.
|
|
32
|
+
`false` beyond the window, for `u64::MAX` and while unassigned.
|
|
33
|
+
- `commit_candidate()` is one past the contiguous acknowledged prefix, or
|
|
34
|
+
`None`. The adapter persists it however it likes.
|
|
35
|
+
- `mark_committed(generation, offset) -> Result<Commit, Refusal>` is the
|
|
36
|
+
adapter's confirmation of persistence. Only then does the window slide:
|
|
37
|
+
`Commit::Advanced`. Partial confirmation up to the candidate is allowed. An
|
|
38
|
+
offset at or below the base is an idempotent repeat, `Commit::Duplicate`.
|
|
39
|
+
Refused as `Stale` (wrong generation, or unassigned) or `Unacked` (beyond
|
|
40
|
+
the candidate).
|
|
41
|
+
- `status() -> Status` is the five readers as one value. `Status::new::<WORDS>(..)
|
|
42
|
+
-> Option<Status>` is how a `Status` is built from loose numbers: `None` for
|
|
43
|
+
any combination no `Tracker<WORDS>` reports (a frontier past the window, a
|
|
44
|
+
candidate above the frontier, a count that doesn't fit the marks, a base or
|
|
45
|
+
count while unassigned). The fields are private; a `Status` is valid by
|
|
46
|
+
construction.
|
|
47
|
+
- `gaps(from)` iterates unacknowledged offsets below the frontier; resume by
|
|
48
|
+
passing the last yielded offset plus one. Output is bounded by the window.
|
|
49
|
+
|
|
50
|
+
Every call that is not honoured returns `Err(Refusal)` and leaves the tracker
|
|
51
|
+
bit-for-bit unchanged. `Refusal` is the crate's only refusal type: `Stale`,
|
|
52
|
+
`OutOfWindow`, `Unacked`, `Exhausted`. No enum is `#[non_exhaustive]`: a new
|
|
53
|
+
variant is a breaking change every caller matches on. Idempotent repeats are honoured (`Ok`)
|
|
54
|
+
and also change nothing.
|
|
55
|
+
|
|
56
|
+
There is no clock: no operation takes or records an instant.
|
|
57
|
+
|
|
58
|
+
Crash recovery, persistence, rebalancing protocol and retry policy are adapter
|
|
59
|
+
concerns; after a restart, `assign` a newer generation at the stored offset.
|
|
60
|
+
|
|
61
|
+
```rust
|
|
62
|
+
use offset_machines::{Ack, Commit, Refusal, Tracker};
|
|
63
|
+
let mut tracker = Tracker::<4>::new(); // 256-offset window
|
|
64
|
+
assert_eq!(tracker.ack(1, 100), Err(Refusal::Stale)); // nothing assigned yet
|
|
65
|
+
assert_eq!(tracker.assign(1, 100), Ok(()));
|
|
66
|
+
assert_eq!(tracker.ack(1, 100), Ok(Ack::Recorded));
|
|
67
|
+
assert_eq!(tracker.ack(1, 102), Ok(Ack::Recorded));
|
|
68
|
+
assert_eq!(tracker.ack(1, 102), Ok(Ack::Duplicate));
|
|
69
|
+
assert!(tracker.is_acked(99) && tracker.is_acked(102) && !tracker.is_acked(101));
|
|
70
|
+
assert_eq!(tracker.commit_candidate(), Some(101));
|
|
71
|
+
assert_eq!(tracker.gaps(0).next(), Some(101));
|
|
72
|
+
assert_eq!(tracker.mark_committed(1, 102), Err(Refusal::Unacked));
|
|
73
|
+
assert_eq!(tracker.mark_committed(1, 101), Ok(Commit::Advanced));
|
|
74
|
+
assert_eq!(tracker.mark_committed(1, 101), Ok(Commit::Duplicate));
|
|
75
|
+
assert_eq!(tracker.base(), 101);
|
|
76
|
+
assert_eq!(tracker.reassign(101), Ok(2));
|
|
77
|
+
|
|
78
|
+
// Already assigned, with no refusal to handle: the generation is never zero.
|
|
79
|
+
let fresh = Tracker::<4>::assigned(core::num::NonZeroU64::MIN, 500);
|
|
80
|
+
assert_eq!((fresh.generation(), fresh.base()), (1, 500));
|
|
81
|
+
```
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'offset_machines/version'
|
|
4
|
+
|
|
5
|
+
module OffsetMachines
|
|
6
|
+
# The core's `Status`: an immutable point-in-time view of a tracker, the
|
|
7
|
+
# answers of its five readers together. Like `Status::new`, it holds only
|
|
8
|
+
# what a tracker of this window could report: anything else raises
|
|
9
|
+
# RangeError (TypeError for a non-Integer; the two marks may be nil).
|
|
10
|
+
Status = Data.define(:generation, :base, :commit_candidate, :frontier, :acked) do
|
|
11
|
+
def initialize(generation:, base:, commit_candidate:, frontier:, acked:)
|
|
12
|
+
OffsetMachines.__send__(:__status, generation, base, commit_candidate, frontier, acked)
|
|
13
|
+
super
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# Native is required: no algorithm fallback. It validates the Data value above
|
|
19
|
+
# and builds `Tracker#status` from the core's `Tracker::status`, the five
|
|
20
|
+
# readers taken at once rather than dispatched one by one.
|
|
21
|
+
require 'offset_machines_native/offset_machines_native'
|
|
22
|
+
|
|
23
|
+
module OffsetMachines
|
|
24
|
+
private_class_method :__status
|
|
25
|
+
|
|
26
|
+
# The value is final. `initialize` is where it validates, and a subclass
|
|
27
|
+
# could override it, or `allocate` build an instance that never ran it:
|
|
28
|
+
# subclassing raises TypeError and `allocate` is gone. This stops accidents,
|
|
29
|
+
# not reflection (Marshal, `Data#initialize` bound by hand); no method of
|
|
30
|
+
# this gem reads a Status back, so a forged one never reaches the core.
|
|
31
|
+
#
|
|
32
|
+
# `Tracker` wraps native data and is sealed the same way: a subclass could
|
|
33
|
+
# override what callers rely on, and `allocate` would build a shell with no
|
|
34
|
+
# core behind it.
|
|
35
|
+
[Status, Tracker].each do |klass|
|
|
36
|
+
klass.singleton_class.undef_method(:allocate)
|
|
37
|
+
klass.define_singleton_method(:inherited) { |_| raise TypeError, "#{klass} cannot be subclassed" }
|
|
38
|
+
end
|
|
39
|
+
end
|
|
Binary file
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
module OffsetMachines
|
|
2
|
+
VERSION: String
|
|
3
|
+
CAPACITY: Integer
|
|
4
|
+
# Refusals: the call was not honoured and changed nothing. One symbol per
|
|
5
|
+
# core `Refusal` variant; there is no catch-all.
|
|
6
|
+
type refusal = :stale | :out_of_window | :unacked | :exhausted
|
|
7
|
+
# Honoured: :assigned. Refused: :stale.
|
|
8
|
+
type assign_result = :assigned | :stale
|
|
9
|
+
# Honoured: the new generation. Refused: :exhausted.
|
|
10
|
+
type reassign_result = Integer | :exhausted
|
|
11
|
+
# Honoured: :recorded, :duplicate (idempotent repeat). Refused: :stale, :out_of_window.
|
|
12
|
+
type ack_result = :recorded | :duplicate | :stale | :out_of_window
|
|
13
|
+
# Honoured: :advanced, :duplicate (idempotent repeat). Refused: :stale, :unacked.
|
|
14
|
+
type commit_result = :advanced | :duplicate | :stale | :unacked
|
|
15
|
+
# Native check behind `Status.new`: nil when a tracker of this window could
|
|
16
|
+
# report the five values together, raises (TypeError/RangeError) otherwise.
|
|
17
|
+
private def self.__status: (Integer generation, Integer base, Integer? commit_candidate, Integer? frontier, Integer acked) -> nil
|
|
18
|
+
# The core's `Status`. `new` mirrors `Status::new`: it raises RangeError
|
|
19
|
+
# unless a tracker of this window could report the five values together,
|
|
20
|
+
# TypeError for a non-Integer (the two marks may be nil).
|
|
21
|
+
class Status < Data
|
|
22
|
+
attr_reader generation: Integer
|
|
23
|
+
attr_reader base: Integer
|
|
24
|
+
attr_reader commit_candidate: Integer?
|
|
25
|
+
attr_reader frontier: Integer?
|
|
26
|
+
attr_reader acked: Integer
|
|
27
|
+
def self.new: (Integer generation, Integer base, Integer? commit_candidate, Integer? frontier, Integer acked) -> Status
|
|
28
|
+
| (generation: Integer, base: Integer, commit_candidate: Integer?, frontier: Integer?, acked: Integer) -> Status
|
|
29
|
+
end
|
|
30
|
+
class Tracker
|
|
31
|
+
def self.new: () -> Tracker
|
|
32
|
+
# Already assigned: what an honoured `assign` makes of a new tracker.
|
|
33
|
+
# Generation 0 raises RangeError, the one value `assign` refuses there.
|
|
34
|
+
def self.assigned: (Integer generation, Integer base_offset) -> Tracker
|
|
35
|
+
def capacity: () -> Integer
|
|
36
|
+
def generation: () -> Integer
|
|
37
|
+
def base: () -> Integer
|
|
38
|
+
def acked: () -> Integer
|
|
39
|
+
def assign: (Integer generation, Integer base_offset) -> assign_result
|
|
40
|
+
def reassign: (Integer base_offset) -> reassign_result
|
|
41
|
+
def ack: (Integer generation, Integer offset) -> ack_result
|
|
42
|
+
def mark_committed: (Integer generation, Integer offset) -> commit_result
|
|
43
|
+
# Already acknowledged under the current generation: below `base`, or
|
|
44
|
+
# recorded in the window. Exactly when `ack(generation, offset)` would
|
|
45
|
+
# answer :duplicate. Always false while unassigned.
|
|
46
|
+
def acked?: (Integer offset) -> bool
|
|
47
|
+
def commit_candidate: () -> Integer?
|
|
48
|
+
def frontier: () -> Integer?
|
|
49
|
+
def gaps: (Integer from, Integer limit) -> Array[Integer]
|
|
50
|
+
def status: () -> Status
|
|
51
|
+
end
|
|
52
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: offset_machines
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.5.1
|
|
5
|
+
platform: aarch64-linux
|
|
6
|
+
authors:
|
|
7
|
+
- Abdelkader Boudih
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: bin
|
|
10
|
+
cert_chain: []
|
|
11
|
+
date: 2026-11-16 00:00:00.000000000 Z
|
|
12
|
+
dependencies: []
|
|
13
|
+
description: 'Offset acknowledgement tracker: contiguous commit candidate, gaps, generation
|
|
14
|
+
checks. Not a commit log or runtime.'
|
|
15
|
+
email:
|
|
16
|
+
- oss@seuros.com
|
|
17
|
+
executables: []
|
|
18
|
+
extensions: []
|
|
19
|
+
extra_rdoc_files: []
|
|
20
|
+
files:
|
|
21
|
+
- CHANGELOG.md
|
|
22
|
+
- LICENSE.txt
|
|
23
|
+
- README.md
|
|
24
|
+
- ext/offset_machines_native/core/CHANGELOG.md
|
|
25
|
+
- ext/offset_machines_native/core/README.md
|
|
26
|
+
- ext/offset_machines_native/extconf.rb
|
|
27
|
+
- lib/offset_machines.rb
|
|
28
|
+
- lib/offset_machines/version.rb
|
|
29
|
+
- lib/offset_machines_native/offset_machines_native.so
|
|
30
|
+
- sig/offset_machines.rbs
|
|
31
|
+
homepage: https://github.com/seuros/offset_machines
|
|
32
|
+
licenses:
|
|
33
|
+
- MIT
|
|
34
|
+
metadata:
|
|
35
|
+
source_code_uri: https://github.com/seuros/offset_machines
|
|
36
|
+
rubygems_mfa_required: 'true'
|
|
37
|
+
cargo_crate_name: offset_machines_native
|
|
38
|
+
cargo_manifest_path: ext/offset_machines_native/ffi/Cargo.toml
|
|
39
|
+
post_install_message:
|
|
40
|
+
rdoc_options: []
|
|
41
|
+
require_paths:
|
|
42
|
+
- lib
|
|
43
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
44
|
+
requirements:
|
|
45
|
+
- - ">="
|
|
46
|
+
- !ruby/object:Gem::Version
|
|
47
|
+
version: '4.0'
|
|
48
|
+
- - "<"
|
|
49
|
+
- !ruby/object:Gem::Version
|
|
50
|
+
version: 4.1.dev
|
|
51
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
52
|
+
requirements:
|
|
53
|
+
- - ">="
|
|
54
|
+
- !ruby/object:Gem::Version
|
|
55
|
+
version: '0'
|
|
56
|
+
requirements: []
|
|
57
|
+
rubygems_version: 3.5.23
|
|
58
|
+
signing_key:
|
|
59
|
+
specification_version: 4
|
|
60
|
+
summary: Bounded per-partition acknowledgement bitmap with a native Rust core
|
|
61
|
+
test_files: []
|