dedup_machines 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +68 -0
- data/Cargo.toml +26 -0
- data/LICENSE.txt +21 -0
- data/README.md +208 -0
- data/ext/dedup_machines_native/core/CHANGELOG.md +54 -0
- data/ext/dedup_machines_native/core/Cargo.toml +21 -0
- data/ext/dedup_machines_native/core/README.md +70 -0
- data/ext/dedup_machines_native/core/src/hashed.rs +127 -0
- data/ext/dedup_machines_native/core/src/lib.rs +470 -0
- data/ext/dedup_machines_native/core/src/test_support.rs +16 -0
- data/ext/dedup_machines_native/extconf.rb +2 -0
- data/ext/dedup_machines_native/ffi/Cargo.toml +20 -0
- data/ext/dedup_machines_native/ffi/extconf.rb +14 -0
- data/ext/dedup_machines_native/ffi/src/init.rs +18 -0
- data/ext/dedup_machines_native/ffi/src/lib.rs +96 -0
- data/lib/dedup_machines/version.rb +5 -0
- data/lib/dedup_machines.rb +5 -0
- data/sig/dedup_machines.rbs +12 -0
- metadata +84 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: e71d79c60c485879816ca73b9d2a57d6bb7beedb88b6783c3633ebc9efa9cb1e
|
|
4
|
+
data.tar.gz: 4d398af5165da22c1009cb91b3bca1de9288651606de6053ddda1a6803994e9e
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: dbfa660abb94f87711ddde8b8ca52c8ec6f034a2acafb46bb739b71eb558153bb1721bbbaa455c822890b00970cd74639b289f9d8a58df21b94a502703a0171e
|
|
7
|
+
data.tar.gz: 65916915e33cbc7a5ba15b2932b1920cf4e542d49ae3e73c36662873c24df7303cad74abb26e10fd0cc223636de069f8440ca4e8f028c5597e5b6a3405143665
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## [0.2.2](https://github.com/seuros/dedup_machines/compare/dedup_machines/v0.2.1...dedup_machines/v0.2.2) (2026-11-06)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Bug Fixes
|
|
7
|
+
|
|
8
|
+
* satisfy the nursery option_if_let_else lint in Entry::new ([88cf1eb](https://github.com/seuros/dedup_machines/commit/88cf1ebadcbfac752ac6e20fb0d2ce07dae765e9))
|
|
9
|
+
* share core test helpers via test_support ([0c94d57](https://github.com/seuros/dedup_machines/commit/0c94d57d888c41b77d4a11c9db346fc0587ee78c))
|
|
10
|
+
|
|
11
|
+
## [0.2.1](https://github.com/seuros/dedup_machines/compare/dedup_machines/v0.2.0...dedup_machines/v0.2.1) (2026-11-06)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
### Features
|
|
15
|
+
|
|
16
|
+
* unseal Storage so a ledger can run over an external backend ([61c8c77](https://github.com/seuros/dedup_machines/commit/61c8c77d45923109919a5dda5b1315ce64427aba))
|
|
17
|
+
|
|
18
|
+
## [0.2.0](https://github.com/seuros/dedup_machines/compare/dedup_machines/v0.1.5...dedup_machines/v0.2.0) (2026-11-06)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
### ⚠ BREAKING CHANGES
|
|
22
|
+
|
|
23
|
+
* Claim carries a lifetime and value type, and Completed holds a reference to the stored value.
|
|
24
|
+
|
|
25
|
+
### Features
|
|
26
|
+
|
|
27
|
+
* heap-backed ledger, fingerprints, stored values, partition caps ([a9fee64](https://github.com/seuros/dedup_machines/commit/a9fee64ce94d8a32e62c9d97e014f3664fa8b20e))
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
### Bug Fixes
|
|
31
|
+
|
|
32
|
+
* document the error path of complete_with ([0c9dc20](https://github.com/seuros/dedup_machines/commit/0c9dc206c4c8ec55be2d6a37420eb7c4d101fb8b))
|
|
33
|
+
* satisfy pedantic clippy on the test helpers and hashed constructor ([001a626](https://github.com/seuros/dedup_machines/commit/001a6265621f8f46a7b1d38e881ce04ead09c40f))
|
|
34
|
+
|
|
35
|
+
## [0.1.5](https://github.com/seuros/dedup_machines/compare/dedup_machines/v0.1.4...dedup_machines/v0.1.5) (2026-11-06)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
### Bug Fixes
|
|
39
|
+
|
|
40
|
+
* crates.io keywords and categories for the core ([2c5c7ee](https://github.com/seuros/dedup_machines/commit/2c5c7eebc94e34f772bdd7508d12f7d79c334c81))
|
|
41
|
+
|
|
42
|
+
## [0.1.4](https://github.com/seuros/dedup_machines/compare/dedup_machines/v0.1.3...dedup_machines/v0.1.4) (2026-11-06)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
### Bug Fixes
|
|
46
|
+
|
|
47
|
+
* keep test sources out of the packaged gem and crate ([dc18303](https://github.com/seuros/dedup_machines/commit/dc1830331de56e948e2ce082686be122a8a537aa))
|
|
48
|
+
|
|
49
|
+
## [0.1.3](https://github.com/seuros/dedup_machines/compare/dedup_machines/v0.1.2...dedup_machines/v0.1.3) (2026-11-06)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
### Bug Fixes
|
|
53
|
+
|
|
54
|
+
* name the gemspec block spec so the release workflow detects the extension ([a4584eb](https://github.com/seuros/dedup_machines/commit/a4584ebb0b40e77e8bdfb1e31ee72c7862cfa95e))
|
|
55
|
+
|
|
56
|
+
## [0.1.2](https://github.com/seuros/dedup_machines/compare/dedup_machines/v0.1.1...dedup_machines/v0.1.2) (2026-11-06)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
### Bug Fixes
|
|
60
|
+
|
|
61
|
+
* cross builds no longer compile for the host ([86e820b](https://github.com/seuros/dedup_machines/commit/86e820b3b73fdf971817d5415f3ef941d6f82125))
|
|
62
|
+
|
|
63
|
+
## [0.1.1](https://github.com/seuros/dedup_machines/compare/dedup_machines-v0.1.0...dedup_machines/v0.1.1) (2026-11-06)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
### Features
|
|
67
|
+
|
|
68
|
+
* initial extraction ([126339b](https://github.com/seuros/dedup_machines/commit/126339bf2ec50261af08c5e710436bafa2530365))
|
data/Cargo.toml
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
[workspace]
|
|
2
|
+
members = ["ext/dedup_machines_native/core", "ext/dedup_machines_native/ffi"]
|
|
3
|
+
default-members = ["ext/dedup_machines_native/core"]
|
|
4
|
+
resolver = "3"
|
|
5
|
+
|
|
6
|
+
[workspace.package]
|
|
7
|
+
edition = "2024"
|
|
8
|
+
rust-version = "1.99"
|
|
9
|
+
license = "MIT"
|
|
10
|
+
|
|
11
|
+
[workspace.dependencies]
|
|
12
|
+
dedup-machines = { path = "ext/dedup_machines_native/core" }
|
|
13
|
+
magnus = { version = "0.9.3", default-features = false }
|
|
14
|
+
rb-sys = { version = "0.9.130", default-features = false, features = ["stable-api-compiled-fallback"] }
|
|
15
|
+
|
|
16
|
+
[workspace.lints.rust]
|
|
17
|
+
unsafe_code = "deny"
|
|
18
|
+
unused_must_use = "warn"
|
|
19
|
+
missing_docs = "warn"
|
|
20
|
+
|
|
21
|
+
[workspace.lints.clippy]
|
|
22
|
+
all = { level = "warn", priority = -1 }
|
|
23
|
+
pedantic = { level = "warn", priority = -1 }
|
|
24
|
+
nursery = { level = "warn", priority = -1 }
|
|
25
|
+
unwrap_used = "deny"
|
|
26
|
+
expect_used = "deny"
|
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Abdelkader Boudih
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
# DedupMachines
|
|
2
|
+
|
|
3
|
+
> The second copy of a message is not a second order. 📨
|
|
4
|
+
|
|
5
|
+
A claim ledger for Ruby. Before you act on a message, you claim its key. The ledger tells you whether the key is fresh, already being worked on, or already done. Finish the work and mark it complete. Lose the worker and abandon the claim. Either way, the next copy of the same message gets the right answer instead of a second side effect.
|
|
6
|
+
|
|
7
|
+
## Quick Start
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
gem 'dedup_machines'
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
```ruby
|
|
14
|
+
ledger = DedupMachines::Ledger.new
|
|
15
|
+
|
|
16
|
+
def handle(ledger, event, clock)
|
|
17
|
+
case (claim = ledger.claim(event.id, clock.now_ns, 24 * 3_600_000_000_000))
|
|
18
|
+
when :in_progress, :completed
|
|
19
|
+
return # someone has it, or had it. Nothing to do.
|
|
20
|
+
when :full, :rejected
|
|
21
|
+
raise "ledger refused #{event.id}: #{claim}"
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
charge_customer(event)
|
|
25
|
+
ledger.complete(claim, clock.now_ns)
|
|
26
|
+
rescue ChargeFailed
|
|
27
|
+
ledger.abandon(claim)
|
|
28
|
+
raise
|
|
29
|
+
end
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
One ledger, three methods. Keys and times are Integers. Times are nanoseconds from a clock that only goes forward.
|
|
33
|
+
|
|
34
|
+
## A Message to the Resistance
|
|
35
|
+
|
|
36
|
+
So Stripe sent you `payment_intent.succeeded`. Your webhook handler fulfilled the order and sent the confirmation email. Then Stripe, as the docs say it might, sent the same event again. Your handler fulfilled the order and sent the confirmation email.
|
|
37
|
+
|
|
38
|
+
You know what the docs told you to do. "Make your handler idempotent." Four words, and the entire problem is yours again. So you add a `processed_events` table. Then two workers pick up the same event at the same instant, both find nothing in the table, both proceed. So you add a unique index and a transaction. Then a worker dies after the insert and before the charge, and that event is now marked done forever, and nobody gets charged.
|
|
39
|
+
|
|
40
|
+
The piece you keep rebuilding is a tiny state machine per key: nobody has it, someone is working on it, someone finished it. With a timeout, so a dead worker's claim doesn't outlive the heat death of the universe. That machine is this gem.
|
|
41
|
+
|
|
42
|
+
## Chapter 1: Stardate 2025.186, the AMS Birrell
|
|
43
|
+
|
|
44
|
+
> **AMS Birrell**, NCC-4205, *Birrell*-class courier.
|
|
45
|
+
> An echo from the relay is not a second delivery order.
|
|
46
|
+
|
|
47
|
+
The courier doesn't deliver anything. It keeps the manifest. Every dispatch has an exact identity, and the manifest says whether that identity is unclaimed, out with a crew, or signed for. A relay that bounces the same order back an hour later finds it already signed for and files it under echoes.
|
|
48
|
+
|
|
49
|
+
On a bridge somewhere, a cadet volunteers:
|
|
50
|
+
|
|
51
|
+
```ruby
|
|
52
|
+
def on_webhook(event)
|
|
53
|
+
return if Redis.current.get("seen:#{event.id}")
|
|
54
|
+
fulfil(event)
|
|
55
|
+
Redis.current.set("seen:#{event.id}", 1)
|
|
56
|
+
end
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
"That," says the courier's quartermaster, "is a race with a two-line head start."
|
|
60
|
+
|
|
61
|
+
## The Flight Plan
|
|
62
|
+
|
|
63
|
+
Each key the ledger knows about is in one of these states:
|
|
64
|
+
|
|
65
|
+
```mermaid
|
|
66
|
+
stateDiagram-v2
|
|
67
|
+
[*] --> unclaimed
|
|
68
|
+
unclaimed --> in_progress: claim, returns a Claim
|
|
69
|
+
in_progress --> in_progress: claim again, returns :in_progress
|
|
70
|
+
in_progress --> completed: complete(claim)
|
|
71
|
+
in_progress --> unclaimed: abandon(claim)
|
|
72
|
+
completed --> completed: claim again, returns :completed
|
|
73
|
+
completed --> unclaimed: ttl elapsed since the claim
|
|
74
|
+
in_progress --> unclaimed: ttl elapsed since the claim
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
And the afternoon Stripe sent event 42 twice, into two workers:
|
|
78
|
+
|
|
79
|
+
```mermaid
|
|
80
|
+
sequenceDiagram
|
|
81
|
+
participant S as Stripe
|
|
82
|
+
participant W1 as Worker 1
|
|
83
|
+
participant L as Ledger
|
|
84
|
+
participant W2 as Worker 2
|
|
85
|
+
|
|
86
|
+
S->>W1: event 42
|
|
87
|
+
W1->>L: claim(42)
|
|
88
|
+
L-->>W1: Claim
|
|
89
|
+
W1->>W1: charge customer
|
|
90
|
+
|
|
91
|
+
S->>W2: event 42 (retry)
|
|
92
|
+
W2->>L: claim(42)
|
|
93
|
+
L-->>W2: :in_progress
|
|
94
|
+
Note over W2: drops it
|
|
95
|
+
|
|
96
|
+
W1->>L: complete(claim)
|
|
97
|
+
L-->>W1: true
|
|
98
|
+
|
|
99
|
+
S->>W2: event 42 (another retry)
|
|
100
|
+
W2->>L: claim(42)
|
|
101
|
+
L-->>W2: :completed
|
|
102
|
+
Note over W2: drops it. One charge.
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
## How the Ledger Works
|
|
106
|
+
|
|
107
|
+
### Claim
|
|
108
|
+
|
|
109
|
+
```ruby
|
|
110
|
+
ledger = DedupMachines::Ledger.new
|
|
111
|
+
|
|
112
|
+
claim = ledger.claim(42, 0, 10) # key 42, now 0 ns, ttl 10 ns
|
|
113
|
+
claim.class # => DedupMachines::Claim
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
A `Claim` means the key was unclaimed and is now yours. Ask again while you're still holding it and you get a symbol instead:
|
|
117
|
+
|
|
118
|
+
```ruby
|
|
119
|
+
ledger.claim(42, 1, 10) # => :in_progress
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
### Complete
|
|
123
|
+
|
|
124
|
+
```ruby
|
|
125
|
+
ledger.complete(claim, 2) # => true
|
|
126
|
+
ledger.complete(claim, 2) # => false, already spent
|
|
127
|
+
ledger.claim(42, 9, 10) # => :completed
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
A completed key keeps answering `:completed` until its ttl runs out. The ttl is measured from the moment of the claim, not from completion.
|
|
131
|
+
|
|
132
|
+
### Expiry
|
|
133
|
+
|
|
134
|
+
```ruby
|
|
135
|
+
fresh = ledger.claim(42, 10, 10) # ttl elapsed: a new Claim
|
|
136
|
+
ledger.complete(claim, 10) # => false, the old handle is dead
|
|
137
|
+
ledger.abandon(fresh) # => true
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
Ten nanoseconds after a claim with a ten-nanosecond ttl, the key is unclaimed again. The old `Claim` can't be completed or abandoned any more. Only the new one counts.
|
|
141
|
+
|
|
142
|
+
### Abandon
|
|
143
|
+
|
|
144
|
+
A worker that knows it failed gives the key back so the next copy can try:
|
|
145
|
+
|
|
146
|
+
```ruby
|
|
147
|
+
ledger = DedupMachines::Ledger.new
|
|
148
|
+
claim = ledger.claim(7, 0, 100)
|
|
149
|
+
|
|
150
|
+
ledger.abandon(claim) # => true
|
|
151
|
+
ledger.claim(7, 1, 100).class # => DedupMachines::Claim
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
Abandon only an attempt you know did nothing. If the charge might have gone through, that's a reconciliation problem, not an abandon.
|
|
155
|
+
|
|
156
|
+
### Capacity
|
|
157
|
+
|
|
158
|
+
The ledger holds 64 keys at a time. Past that, `claim` says `:full` until something completes and expires, or is abandoned:
|
|
159
|
+
|
|
160
|
+
```ruby
|
|
161
|
+
ledger = DedupMachines::Ledger.new
|
|
162
|
+
ledger.capacity # => 64
|
|
163
|
+
claims = 64.times.map { |i| ledger.claim(i, 0, 100) }
|
|
164
|
+
ledger.claim(65, 0, 100) # => :full
|
|
165
|
+
|
|
166
|
+
ledger.abandon(claims.first) # => true
|
|
167
|
+
ledger.claim(65, 0, 100).class # => DedupMachines::Claim
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
An unexpired key is never evicted to make room. Size your ttl to your traffic, or shard across ledgers.
|
|
171
|
+
|
|
172
|
+
### Time only moves forward
|
|
173
|
+
|
|
174
|
+
The ledger remembers the latest `now_ns` it has seen. Hand it an earlier one and it refuses:
|
|
175
|
+
|
|
176
|
+
```ruby
|
|
177
|
+
ledger = DedupMachines::Ledger.new
|
|
178
|
+
claim = ledger.claim(1, 10, 10)
|
|
179
|
+
ledger.complete(claim, 9) # => false
|
|
180
|
+
ledger.complete(claim, 10) # => true
|
|
181
|
+
ledger.claim(2, 9, 10) # => :rejected
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
A zero ttl is `:rejected` too. Feed it a monotonic clock in nanoseconds.
|
|
185
|
+
|
|
186
|
+
## The Rules of the Courier
|
|
187
|
+
|
|
188
|
+
- **Keys and times are Integers** in `0..2**64-1`. Times are nanoseconds. Floats, nil, and strings raise `TypeError`. Out of range raises `RangeError`.
|
|
189
|
+
- **Claims belong to one ledger.** A `Claim` from one `Ledger` is refused by another, with `false`, even for the same key. Claims can't be built or copied by hand.
|
|
190
|
+
- **Refusals don't raise.** `:in_progress`, `:completed`, `:full`, `:rejected`, and `false` are answers. Only malformed input raises.
|
|
191
|
+
- **Frozen means frozen.** `freeze` the ledger and `claim`, `complete`, and `abandon` all raise `FrozenError`.
|
|
192
|
+
|
|
193
|
+
## What You Think You're Doing vs Reality
|
|
194
|
+
|
|
195
|
+
### You Think: "I'll check if I've seen it, then process it."
|
|
196
|
+
### Reality: Two workers both haven't seen it.
|
|
197
|
+
|
|
198
|
+
The check and the processing are two steps, and a second copy of the message fits neatly between them. The only check that works is one that *takes* the key as it reads it. `claim` is that check. There's no window, because there's no second step.
|
|
199
|
+
|
|
200
|
+
## What's Yours
|
|
201
|
+
|
|
202
|
+
The ledger lives in your process and forgets everything when the process does. If duplicates can arrive after a restart, write completed keys somewhere durable and seed a fresh ledger's decisions from that. A claim that times out may belong to a worker that's still running. Fence or reconcile its side effects before letting the next copy through.
|
|
203
|
+
|
|
204
|
+
## License
|
|
205
|
+
|
|
206
|
+
MIT License. See [LICENSE.txt](LICENSE.txt) for details.
|
|
207
|
+
|
|
208
|
+
**Remember: Stripe will send it twice. The question is whether you charge twice.**
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## [0.4.1](https://github.com/seuros/dedup_machines/compare/dedup-machines-v0.4.0...dedup-machines-v0.4.1) (2026-11-06)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Bug Fixes
|
|
7
|
+
|
|
8
|
+
* satisfy the nursery option_if_let_else lint in Entry::new ([88cf1eb](https://github.com/seuros/dedup_machines/commit/88cf1ebadcbfac752ac6e20fb0d2ce07dae765e9))
|
|
9
|
+
* share core test helpers via test_support ([0c94d57](https://github.com/seuros/dedup_machines/commit/0c94d57d888c41b77d4a11c9db346fc0587ee78c))
|
|
10
|
+
|
|
11
|
+
## [0.4.0](https://github.com/seuros/dedup_machines/compare/dedup-machines-v0.3.0...dedup-machines-v0.4.0) (2026-11-06)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
### Features
|
|
15
|
+
|
|
16
|
+
* unseal Storage so a ledger can run over an external backend ([61c8c77](https://github.com/seuros/dedup_machines/commit/61c8c77d45923109919a5dda5b1315ce64427aba))
|
|
17
|
+
|
|
18
|
+
## [0.3.0](https://github.com/seuros/dedup_machines/compare/dedup-machines-v0.2.2...dedup-machines-v0.3.0) (2026-11-06)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
### ⚠ BREAKING CHANGES
|
|
22
|
+
|
|
23
|
+
* Claim carries a lifetime and value type, and Completed holds a reference to the stored value.
|
|
24
|
+
|
|
25
|
+
### Features
|
|
26
|
+
|
|
27
|
+
* heap-backed ledger, fingerprints, stored values, partition caps ([a9fee64](https://github.com/seuros/dedup_machines/commit/a9fee64ce94d8a32e62c9d97e014f3664fa8b20e))
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
### Bug Fixes
|
|
31
|
+
|
|
32
|
+
* document the error path of complete_with ([0c9dc20](https://github.com/seuros/dedup_machines/commit/0c9dc206c4c8ec55be2d6a37420eb7c4d101fb8b))
|
|
33
|
+
* satisfy pedantic clippy on the test helpers and hashed constructor ([001a626](https://github.com/seuros/dedup_machines/commit/001a6265621f8f46a7b1d38e881ce04ead09c40f))
|
|
34
|
+
|
|
35
|
+
## [0.2.2](https://github.com/seuros/dedup_machines/compare/dedup-machines-v0.2.1...dedup-machines-v0.2.2) (2026-11-06)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
### Bug Fixes
|
|
39
|
+
|
|
40
|
+
* crates.io keywords and categories for the core ([2c5c7ee](https://github.com/seuros/dedup_machines/commit/2c5c7eebc94e34f772bdd7508d12f7d79c334c81))
|
|
41
|
+
|
|
42
|
+
## [0.2.1](https://github.com/seuros/dedup_machines/compare/dedup-machines-v0.2.0...dedup-machines-v0.2.1) (2026-11-06)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
### Bug Fixes
|
|
46
|
+
|
|
47
|
+
* keep test sources out of the packaged gem and crate ([dc18303](https://github.com/seuros/dedup_machines/commit/dc1830331de56e948e2ce082686be122a8a537aa))
|
|
48
|
+
|
|
49
|
+
## [0.2.0](https://github.com/seuros/dedup_machines/compare/dedup-machines-v0.1.0...dedup-machines-v0.2.0) (2026-11-06)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
### Features
|
|
53
|
+
|
|
54
|
+
* initial extraction ([126339b](https://github.com/seuros/dedup_machines/commit/126339bf2ec50261af08c5e710436bafa2530365))
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
[package]
|
|
2
|
+
name = "dedup-machines"
|
|
3
|
+
version = "0.4.1"
|
|
4
|
+
edition.workspace = true
|
|
5
|
+
rust-version.workspace = true
|
|
6
|
+
license.workspace = true
|
|
7
|
+
description = "Bounded exact duplicate claims with generation-checked completion"
|
|
8
|
+
readme = "README.md"
|
|
9
|
+
exclude = ["src/**/tests.rs", "tests/"]
|
|
10
|
+
keywords = ["deduplication", "idempotency", "ttl", "exactly-once", "no-std"]
|
|
11
|
+
categories = ["no-std", "algorithms"]
|
|
12
|
+
[features]
|
|
13
|
+
default = []
|
|
14
|
+
alloc = ["dep:hashbrown"]
|
|
15
|
+
std = ["alloc"]
|
|
16
|
+
|
|
17
|
+
[dependencies]
|
|
18
|
+
hashbrown = { version = "0.17", default-features = false, features = ["default-hasher"], optional = true }
|
|
19
|
+
|
|
20
|
+
[lints]
|
|
21
|
+
workspace = true
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# dedup-machines
|
|
2
|
+
|
|
3
|
+
> **AMS Birrell**, NCC-4205, *Birrell*-class courier.
|
|
4
|
+
|
|
5
|
+
Engine-room flight manual: track repeated dispatches in a bounded delivery ledger.
|
|
6
|
+
|
|
7
|
+
Allocation-free `no_std` exact-key claim ledger. Live claims and completed keys
|
|
8
|
+
are retained until their original claim TTL; full tables reject new keys instead
|
|
9
|
+
of evicting live records. Completion is generation-checked, so an expired worker
|
|
10
|
+
cannot complete a replacement claim for the same slot/key.
|
|
11
|
+
|
|
12
|
+
Time is explicit and monotonic. Zero/overflowing TTLs and backwards time are
|
|
13
|
+
rejected. Pending claims may expire: their workers can still be executing, so
|
|
14
|
+
fencing or transactional storage is required for side effects. Completion does
|
|
15
|
+
not extend TTL. `abandon` frees a matching pending claim after a known failure.
|
|
16
|
+
Do not abandon an uncertain external effect and assume a retry is safe.
|
|
17
|
+
|
|
18
|
+
Completion may carry a value: `complete_with` stores it and every later claim
|
|
19
|
+
of that key reads it back as `Completed(&value)`. Claims may carry a
|
|
20
|
+
fingerprint of their payload: a record under the same key with a different
|
|
21
|
+
fingerprint (or none) answers `Conflict`, so an id reused for another payload is
|
|
22
|
+
refused instead of replayed. Claims may name a partition; with a partition cap
|
|
23
|
+
set, a bucket at its quota answers `PartitionFull` while the table still has
|
|
24
|
+
room. `expire_where` drops records whose owner is known to be gone before their
|
|
25
|
+
TTL would.
|
|
26
|
+
|
|
27
|
+
This in-memory ledger has no persistence, cross-process arbitration, or wire
|
|
28
|
+
checkpoint format. Handles are scoped to one ledger lifetime; replacing the
|
|
29
|
+
ledger resets its namespace.
|
|
30
|
+
|
|
31
|
+
```rust
|
|
32
|
+
use core::time::Duration;
|
|
33
|
+
use dedup_machines::{Claim, Dedup};
|
|
34
|
+
let mut ledger = Dedup::<u64, 8>::new();
|
|
35
|
+
let Claim::Acquired(id) = ledger.claim(42, Duration::ZERO, Duration::from_secs(30))
|
|
36
|
+
else { panic!("fresh key must fit"); };
|
|
37
|
+
assert!(ledger.complete(id, Duration::ZERO));
|
|
38
|
+
assert_eq!(ledger.claim(42, Duration::ZERO, Duration::from_secs(30)), Claim::Completed(&()));
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
The `alloc` feature (`std` implies it) adds `HashDedup`, the same ledger over a
|
|
42
|
+
heap table with a runtime capacity and hashed lookup, for keys that are not
|
|
43
|
+
`Copy`:
|
|
44
|
+
|
|
45
|
+
```rust
|
|
46
|
+
# #[cfg(feature = "alloc")] {
|
|
47
|
+
use core::time::Duration;
|
|
48
|
+
use dedup_machines::{Attempt, Claim, HashDedup};
|
|
49
|
+
let mut ledger = HashDedup::<String, String>::with_capacity(1024).with_partition_cap(128);
|
|
50
|
+
let relay = |session: u64, source: &str| Attempt::new(format!("{session}/{source}")).partition(session);
|
|
51
|
+
let Claim::Acquired(id) = ledger.admit(relay(7, "tool"), Duration::ZERO, Duration::from_secs(3600))
|
|
52
|
+
else { panic!("fresh key must fit"); };
|
|
53
|
+
assert_eq!(ledger.complete_with(id, "elicit-1".into(), Duration::ZERO), Ok(()));
|
|
54
|
+
assert_eq!(
|
|
55
|
+
ledger.admit(relay(7, "tool"), Duration::from_secs(1), Duration::from_secs(3600)),
|
|
56
|
+
Claim::Completed(&"elicit-1".to_string())
|
|
57
|
+
);
|
|
58
|
+
assert_eq!(ledger.expire_where(|key| key.starts_with("7/")), 1);
|
|
59
|
+
# }
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
## Bring Your Own Storage
|
|
63
|
+
|
|
64
|
+
`Fixed` and `Hashed` keep records in memory. `Storage` is a public trait, so
|
|
65
|
+
the same ledger can run over a database table or a shared-memory arena: a
|
|
66
|
+
backend stores the `Entry` values the ledger hands it and finds them again by
|
|
67
|
+
key or slot. `Entry::new` and its accessors round-trip every field, and
|
|
68
|
+
`Ledger::restore` continues from a persisted generation and clock watermark so
|
|
69
|
+
no stored claim handle can be reissued.
|
|
70
|
+
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
use alloc::vec::Vec;
|
|
2
|
+
use core::hash::{BuildHasher, Hash};
|
|
3
|
+
|
|
4
|
+
use hashbrown::{DefaultHashBuilder, HashTable};
|
|
5
|
+
|
|
6
|
+
use crate::{Entry, Ledger, Storage};
|
|
7
|
+
|
|
8
|
+
/// Heap-backed storage with a runtime capacity and hashed key lookup.
|
|
9
|
+
#[derive(Debug)]
|
|
10
|
+
pub struct Hashed<K, V, H = DefaultHashBuilder> {
|
|
11
|
+
slots: Vec<Option<Entry<K, V>>>,
|
|
12
|
+
free: Vec<usize>,
|
|
13
|
+
index: HashTable<usize>,
|
|
14
|
+
hasher: H,
|
|
15
|
+
capacity: usize,
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/// Heap-backed exact-key ledger.
|
|
19
|
+
pub type HashDedup<K, V = (), H = DefaultHashBuilder> = Ledger<Hashed<K, V, H>>;
|
|
20
|
+
|
|
21
|
+
impl<K: Hash + Eq, V> HashDedup<K, V> {
|
|
22
|
+
/// Start an empty ledger holding at most `capacity` unexpired records.
|
|
23
|
+
#[must_use]
|
|
24
|
+
pub fn with_capacity(capacity: usize) -> Self {
|
|
25
|
+
Self::with_capacity_and_hasher(capacity, DefaultHashBuilder::default())
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
impl<K: Hash + Eq, V, H: BuildHasher> HashDedup<K, V, H> {
|
|
30
|
+
/// Start an empty ledger with an explicit hasher.
|
|
31
|
+
#[must_use]
|
|
32
|
+
pub const fn with_capacity_and_hasher(capacity: usize, hasher: H) -> Self {
|
|
33
|
+
Self::over(Hashed {
|
|
34
|
+
slots: Vec::new(),
|
|
35
|
+
free: Vec::new(),
|
|
36
|
+
index: HashTable::new(),
|
|
37
|
+
hasher,
|
|
38
|
+
capacity,
|
|
39
|
+
})
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
impl<K: Hash + Eq, V, H: BuildHasher> Hashed<K, V, H> {
|
|
44
|
+
fn key_at(&self, slot: usize) -> Option<&K> {
|
|
45
|
+
self.slots.get(slot)?.as_ref().map(|entry| &entry.key)
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
fn unlink(&mut self, slot: usize) -> bool {
|
|
49
|
+
let Some(key) = self.key_at(slot) else {
|
|
50
|
+
return false;
|
|
51
|
+
};
|
|
52
|
+
let hash = self.hasher.hash_one(key);
|
|
53
|
+
let Ok(occupied) = self.index.find_entry(hash, |&candidate| candidate == slot) else {
|
|
54
|
+
return false;
|
|
55
|
+
};
|
|
56
|
+
occupied.remove();
|
|
57
|
+
self.slots.get_mut(slot).and_then(Option::take);
|
|
58
|
+
self.free.push(slot);
|
|
59
|
+
true
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
impl<K: Hash + Eq, V, H: BuildHasher> Storage for Hashed<K, V, H> {
|
|
64
|
+
type Key = K;
|
|
65
|
+
type Value = V;
|
|
66
|
+
|
|
67
|
+
fn get(&self, slot: usize) -> Option<&Entry<K, V>> {
|
|
68
|
+
self.slots.get(slot)?.as_ref()
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
fn get_mut(&mut self, slot: usize) -> Option<&mut Entry<K, V>> {
|
|
72
|
+
self.slots.get_mut(slot)?.as_mut()
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
fn find(&self, key: &K) -> Option<usize> {
|
|
76
|
+
let hash = self.hasher.hash_one(key);
|
|
77
|
+
self.index
|
|
78
|
+
.find(hash, |&slot| self.key_at(slot) == Some(key))
|
|
79
|
+
.copied()
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
fn insert(&mut self, entry: Entry<K, V>) -> Option<usize> {
|
|
83
|
+
if self.index.len() >= self.capacity {
|
|
84
|
+
return None;
|
|
85
|
+
}
|
|
86
|
+
let hash = self.hasher.hash_one(&entry.key);
|
|
87
|
+
let slot = match self.free.pop() {
|
|
88
|
+
Some(slot) if slot < self.slots.len() => {
|
|
89
|
+
self.slots[slot] = Some(entry);
|
|
90
|
+
slot
|
|
91
|
+
}
|
|
92
|
+
_ => {
|
|
93
|
+
self.slots.push(Some(entry));
|
|
94
|
+
self.slots.len() - 1
|
|
95
|
+
}
|
|
96
|
+
};
|
|
97
|
+
let (slots, hasher) = (&self.slots, &self.hasher);
|
|
98
|
+
self.index.insert_unique(hash, slot, |&other| {
|
|
99
|
+
slots
|
|
100
|
+
.get(other)
|
|
101
|
+
.and_then(Option::as_ref)
|
|
102
|
+
.map_or(0, |entry| hasher.hash_one(&entry.key))
|
|
103
|
+
});
|
|
104
|
+
Some(slot)
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
fn remove(&mut self, slot: usize) -> bool {
|
|
108
|
+
self.unlink(slot)
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
fn retain(&mut self, mut keep: impl FnMut(&Entry<K, V>) -> bool) -> usize {
|
|
112
|
+
let mut removed = 0;
|
|
113
|
+
for slot in 0..self.slots.len() {
|
|
114
|
+
if self.get(slot).is_some_and(|entry| !keep(entry)) && self.unlink(slot) {
|
|
115
|
+
removed += 1;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
removed
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
fn iter(&self) -> impl Iterator<Item = &Entry<K, V>> {
|
|
122
|
+
self.slots.iter().flatten()
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
#[cfg(test)]
|
|
127
|
+
mod tests;
|