dedupe_requests 1.0.0.pre1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +18 -0
- data/LICENSE.txt +21 -0
- data/README.md +193 -0
- data/examples/README.md +156 -0
- data/examples/config.ru +249 -0
- data/examples/end_to_end_test.rb +315 -0
- data/lib/dedupe_requests/configuration.rb +80 -0
- data/lib/dedupe_requests/controller.rb +145 -0
- data/lib/dedupe_requests/fingerprint.rb +80 -0
- data/lib/dedupe_requests/guard.rb +37 -0
- data/lib/dedupe_requests/redis_store.rb +68 -0
- data/lib/dedupe_requests/version.rb +5 -0
- data/lib/dedupe_requests.rb +33 -0
- metadata +103 -0
|
@@ -0,0 +1,315 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
#
|
|
3
|
+
# End-to-end test of dedupe_requests against a REAL HTTP server.
|
|
4
|
+
#
|
|
5
|
+
# Boots examples/config.ru under Puma on a real socket and fires real HTTP at it
|
|
6
|
+
# with Net::HTTP. The test speaks ONLY HTTP — it never touches Redis. The gem
|
|
7
|
+
# writes the claims, Redis expires them on its own TTL; we just send requests in
|
|
8
|
+
# a realistic order, with realistic JSON payloads, from a few simulated callers,
|
|
9
|
+
# and assert on the status codes that come back.
|
|
10
|
+
#
|
|
11
|
+
# Covers, on one enforce-mode server:
|
|
12
|
+
#
|
|
13
|
+
# (1) baseline at the application-controller level (WidgetsController declares nothing)
|
|
14
|
+
# (2) skipping a baseline action in a subclass (DraftsController skip: [:create])
|
|
15
|
+
# (3) adding an action in a subclass (OrdersController on: [:approve])
|
|
16
|
+
# (4) changing the TTL in a subclass (PaymentsController on: [:create], ttl)
|
|
17
|
+
# (5) per-caller scoping (same payload, different caller -> not a dup)
|
|
18
|
+
# (6) different payload, same caller (-> not a dup)
|
|
19
|
+
# (7) concurrent in-flight duplicate (two at once -> exactly one wins)
|
|
20
|
+
# (8) release on failure (4xx response and a raised 500 -> retry allowed)
|
|
21
|
+
# (9) GET / DELETE are never deduplicated (even when the action is guarded by name)
|
|
22
|
+
# (10) 3xx redirect keeps the claim (Post/Redirect/Get -> duplicate still blocked)
|
|
23
|
+
# (12) duplicate-notification hooks fire (on_duplicate_detected AND on_duplicate_rejected)
|
|
24
|
+
#
|
|
25
|
+
# On a second server booted in observe mode:
|
|
26
|
+
#
|
|
27
|
+
# (11) observe mode lets duplicates through (detected fires, rejected does NOT)
|
|
28
|
+
#
|
|
29
|
+
# On a third server booted with a custom fingerprint:
|
|
30
|
+
#
|
|
31
|
+
# (13) the fingerprint override callable is used (custom fingerprint replaces the default)
|
|
32
|
+
#
|
|
33
|
+
# On a fourth server booted with a custom caller_id:
|
|
34
|
+
#
|
|
35
|
+
# (14) the caller_id override callable is used (identity from X-Api-Key, not Authorization)
|
|
36
|
+
#
|
|
37
|
+
# Callers are simulated with an `Authorization: Bearer <token>` header, which is
|
|
38
|
+
# what the gem's default caller_id reads. Two requests are a duplicate only when
|
|
39
|
+
# caller + path + payload all match.
|
|
40
|
+
#
|
|
41
|
+
# Usage (needs a running Redis — the gem talks to it, not this test):
|
|
42
|
+
#
|
|
43
|
+
# redis-server &
|
|
44
|
+
# bundle exec ruby examples/end_to_end_test.rb # or: bundle exec rake integration
|
|
45
|
+
#
|
|
46
|
+
# Exits 0 if every check passes, 1 otherwise.
|
|
47
|
+
|
|
48
|
+
require "net/http"
|
|
49
|
+
require "json"
|
|
50
|
+
require "securerandom"
|
|
51
|
+
require "tmpdir"
|
|
52
|
+
|
|
53
|
+
ENFORCE_PORT = Integer(ENV.fetch("PORT", "9377"))
|
|
54
|
+
OBSERVE_PORT = ENFORCE_PORT + 1
|
|
55
|
+
FINGERPRINT_PORT = ENFORCE_PORT + 2
|
|
56
|
+
CALLER_ID_PORT = ENFORCE_PORT + 3
|
|
57
|
+
HOST = "127.0.0.1"
|
|
58
|
+
ROOT = File.expand_path("..", __dir__)
|
|
59
|
+
CONFIG_RU = File.join(__dir__, "config.ru")
|
|
60
|
+
REDIS_URL = ENV.fetch("REDIS_URL", "redis://localhost:6379/15")
|
|
61
|
+
RUN = SecureRandom.uuid # unique per run, so a rerun's payloads don't collide with still-live claims
|
|
62
|
+
|
|
63
|
+
GLOBAL_TTL = 2
|
|
64
|
+
PAYMENT_TTL = 5
|
|
65
|
+
SLOW_SECONDS = 1
|
|
66
|
+
|
|
67
|
+
$port = ENFORCE_PORT # which booted server the request helpers talk to
|
|
68
|
+
|
|
69
|
+
# Simulated callers -> the Authorization header the gem's default caller_id reads.
|
|
70
|
+
CALLERS = {
|
|
71
|
+
alice: "Bearer token-alice",
|
|
72
|
+
bob: "Bearer token-bob",
|
|
73
|
+
carol: "Bearer token-carol"
|
|
74
|
+
}.freeze
|
|
75
|
+
|
|
76
|
+
# Realistic per-endpoint payloads. Each distinct logical request gets its own
|
|
77
|
+
# body so that, within a run, only an intentional repeat (same caller + path +
|
|
78
|
+
# payload) looks like a duplicate.
|
|
79
|
+
WIDGET_CREATE = { name: "Blue Widget", color: "blue", quantity: 3 }.freeze
|
|
80
|
+
WIDGET_UPDATE = { color: "green", quantity: 5 }.freeze
|
|
81
|
+
WIDGET_RED = { name: "Red Widget", color: "red", quantity: 1 }.freeze
|
|
82
|
+
WIDGET_GREEN = { name: "Green Widget", color: "green", quantity: 7 }.freeze
|
|
83
|
+
DRAFT_CREATE = { title: "Q3 Plan", body: "First outline of the Q3 roadmap" }.freeze
|
|
84
|
+
DRAFT_UPDATE = { body: "Revised outline with a budget section" }.freeze
|
|
85
|
+
ORDER_APPROVE = { approved_by: "manager-7", note: "cleared for fulfillment" }.freeze
|
|
86
|
+
ORDER_CREATE = { customer_id: 42, items: [{ sku: "ABC-123", qty: 2 }], total_cents: 4990 }.freeze
|
|
87
|
+
ORDER_CREATE_TTL = { customer_id: 77, items: [{ sku: "XYZ-9", qty: 1 }], total_cents: 1500 }.freeze
|
|
88
|
+
PAYMENT_CREATE = { order_id: 1001, amount_cents: 1500, currency: "USD" }.freeze
|
|
89
|
+
PAYMENT_MULTI = { order_id: 2002, amount_cents: 9900, currency: "EUR" }.freeze
|
|
90
|
+
SLOW_JOB = { job: "reindex", shard: 4 }.freeze
|
|
91
|
+
FAIL_PAYLOAD = { trigger: "boom" }.freeze
|
|
92
|
+
READ_PAYLOAD = { page: 1 }.freeze
|
|
93
|
+
REDIRECT_PAYLOAD = { ticket: "T-555" }.freeze
|
|
94
|
+
OBSERVE_PAYLOAD = { note: "observe-mode duplicate" }.freeze
|
|
95
|
+
HOOK_PAYLOAD = { event: "hook-check" }.freeze
|
|
96
|
+
|
|
97
|
+
# ---------------------------------------------------------------------------
|
|
98
|
+
# tiny assertion harness
|
|
99
|
+
# ---------------------------------------------------------------------------
|
|
100
|
+
FAILURES = []
|
|
101
|
+
def check(label, got, expected)
|
|
102
|
+
ok = got == expected
|
|
103
|
+
puts format(" %-64s %-26s %s", label, "got #{got.inspect}", ok ? "OK" : "FAIL (want #{expected.inspect})")
|
|
104
|
+
FAILURES << label unless ok
|
|
105
|
+
ok
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# ---------------------------------------------------------------------------
|
|
109
|
+
# real HTTP, real socket — the only thing this test talks to
|
|
110
|
+
# ---------------------------------------------------------------------------
|
|
111
|
+
# A request from caller `as`, carrying a realistic JSON `payload`. The ?run=
|
|
112
|
+
# query only isolates one test run from the next (it is NOT meant as a payload);
|
|
113
|
+
# the gem fingerprints caller + verb + path + query + body.
|
|
114
|
+
def request(method, path, as:, payload: nil, api_key: nil)
|
|
115
|
+
klass = { post: Net::HTTP::Post, patch: Net::HTTP::Patch, get: Net::HTTP::Get, delete: Net::HTTP::Delete }.fetch(method)
|
|
116
|
+
req = klass.new("#{path}?run=#{RUN}", "content-type" => "application/json")
|
|
117
|
+
req["Authorization"] = CALLERS.fetch(as)
|
|
118
|
+
req["X-Api-Key"] = api_key if api_key
|
|
119
|
+
req.body = JSON.generate(payload) if payload
|
|
120
|
+
Net::HTTP.start(HOST, $port) { |http| http.request(req) }
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def post(path, payload:, as: :alice, api_key: nil)
|
|
124
|
+
request(:post, path, payload: payload, as: as, api_key: api_key)
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def patch(path, payload:, as: :alice, api_key: nil)
|
|
128
|
+
request(:patch, path, payload: payload, as: as, api_key: api_key)
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def status(response)
|
|
132
|
+
response.code.to_i
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# Read the hook invocations the server recorded, over HTTP (GET is never deduped).
|
|
136
|
+
def hook_events
|
|
137
|
+
JSON.parse(request(:get, "/_hooks", as: :alice).body).fetch("events")
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def wait_for_server(port)
|
|
141
|
+
40.times do
|
|
142
|
+
Net::HTTP.start(HOST, port) { |http| http.get("/") }
|
|
143
|
+
return true
|
|
144
|
+
rescue StandardError
|
|
145
|
+
sleep 0.25
|
|
146
|
+
end
|
|
147
|
+
false
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
# ---------------------------------------------------------------------------
|
|
151
|
+
# boot Puma serving config.ru on a real port, run a block against it, tear down
|
|
152
|
+
# ---------------------------------------------------------------------------
|
|
153
|
+
def with_server(port, extra_env = {})
|
|
154
|
+
env = {
|
|
155
|
+
"REDIS_URL" => REDIS_URL,
|
|
156
|
+
"DEDUPE_TTL" => GLOBAL_TTL.to_s,
|
|
157
|
+
"DEDUPE_PAYMENT_TTL" => PAYMENT_TTL.to_s,
|
|
158
|
+
"DEDUPE_SLOW_SECONDS" => SLOW_SECONDS.to_s
|
|
159
|
+
}.merge(extra_env)
|
|
160
|
+
log = File.join(Dir.tmpdir, "dedupe_puma_#{port}.log")
|
|
161
|
+
pid = spawn(
|
|
162
|
+
env, "bundle", "exec", "puma", CONFIG_RU, "-b", "tcp://#{HOST}:#{port}", "-t", "5:5", "--silent",
|
|
163
|
+
chdir: ROOT, out: log, err: log
|
|
164
|
+
)
|
|
165
|
+
begin
|
|
166
|
+
unless wait_for_server(port)
|
|
167
|
+
warn "server did not come up on #{HOST}:#{port} — puma log:"
|
|
168
|
+
warn File.read(log) if File.exist?(log)
|
|
169
|
+
FAILURES << "server boot on port #{port}"
|
|
170
|
+
return
|
|
171
|
+
end
|
|
172
|
+
$port = port
|
|
173
|
+
yield
|
|
174
|
+
ensure
|
|
175
|
+
begin
|
|
176
|
+
Process.kill("TERM", pid)
|
|
177
|
+
Process.wait(pid)
|
|
178
|
+
rescue Errno::ESRCH
|
|
179
|
+
# server already gone
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# ===========================================================================
|
|
185
|
+
# enforce-mode server: scenarios (1)..(10)
|
|
186
|
+
# ===========================================================================
|
|
187
|
+
with_server(ENFORCE_PORT) do
|
|
188
|
+
puts "\n(1) BASELINE at the application-controller level (WidgetsController declares nothing)"
|
|
189
|
+
check("POST /widgets alice (first)", status(post("/widgets", payload: WIDGET_CREATE)), 201)
|
|
190
|
+
check("POST /widgets alice (duplicate)", status(post("/widgets", payload: WIDGET_CREATE)), 409)
|
|
191
|
+
check("PATCH /widgets/1 alice (first)", status(patch("/widgets/1", payload: WIDGET_UPDATE)), 201)
|
|
192
|
+
check("PATCH /widgets/1 alice (duplicate)", status(patch("/widgets/1", payload: WIDGET_UPDATE)), 409)
|
|
193
|
+
|
|
194
|
+
puts "\n(2) SKIP override in a subclass (DraftsController skip: [:create]; update still guarded)"
|
|
195
|
+
check("POST /drafts alice (first)", status(post("/drafts", payload: DRAFT_CREATE)), 201)
|
|
196
|
+
check("POST /drafts alice (same again)", status(post("/drafts", payload: DRAFT_CREATE)), 201) # NOT deduped
|
|
197
|
+
check("PATCH /drafts/1 alice (first)", status(patch("/drafts/1", payload: DRAFT_UPDATE)), 201)
|
|
198
|
+
check("PATCH /drafts/1 alice (duplicate)", status(patch("/drafts/1", payload: DRAFT_UPDATE)), 409) # still guarded
|
|
199
|
+
|
|
200
|
+
puts "\n(3) ADD an action in a subclass (OrdersController on: [:approve]; baseline still inherited)"
|
|
201
|
+
check("POST /orders/1/approve alice (first)", status(post("/orders/1/approve", payload: ORDER_APPROVE)), 201)
|
|
202
|
+
check("POST /orders/1/approve alice (duplicate)", status(post("/orders/1/approve", payload: ORDER_APPROVE)), 409)
|
|
203
|
+
check("POST /orders alice (inherited create)", status(post("/orders", payload: ORDER_CREATE)), 201)
|
|
204
|
+
check("POST /orders alice (inherited duplicate)", status(post("/orders", payload: ORDER_CREATE)), 409)
|
|
205
|
+
|
|
206
|
+
puts "\n(4) CHANGE TTL in a subclass — proven by real expiry (orders #{GLOBAL_TTL}s vs payments #{PAYMENT_TTL}s)"
|
|
207
|
+
check("POST /orders alice (opens a #{GLOBAL_TTL}s claim)", status(post("/orders", payload: ORDER_CREATE_TTL)), 201)
|
|
208
|
+
check("POST /payments alice (opens a #{PAYMENT_TTL}s claim)", status(post("/payments", payload: PAYMENT_CREATE)), 201)
|
|
209
|
+
check("POST /orders alice (duplicate, inside #{GLOBAL_TTL}s)", status(post("/orders", payload: ORDER_CREATE_TTL)), 409)
|
|
210
|
+
check("POST /payments alice (duplicate, inside #{PAYMENT_TTL}s)", status(post("/payments", payload: PAYMENT_CREATE)), 409)
|
|
211
|
+
first_wait = GLOBAL_TTL + 1
|
|
212
|
+
puts " ...waiting #{first_wait}s for the #{GLOBAL_TTL}s claim to expire (the #{PAYMENT_TTL}s one should not)..."
|
|
213
|
+
sleep first_wait
|
|
214
|
+
check("POST /orders alice (its #{GLOBAL_TTL}s window expired -> allowed)", status(post("/orders", payload: ORDER_CREATE_TTL)), 201)
|
|
215
|
+
check("POST /payments alice (its #{PAYMENT_TTL}s window still open -> blocked)", status(post("/payments", payload: PAYMENT_CREATE)), 409)
|
|
216
|
+
second_wait = PAYMENT_TTL - first_wait + 1
|
|
217
|
+
puts " ...waiting #{second_wait}s more for the #{PAYMENT_TTL}s claim to expire..."
|
|
218
|
+
sleep second_wait
|
|
219
|
+
check("POST /payments alice (its #{PAYMENT_TTL}s window expired -> allowed)", status(post("/payments", payload: PAYMENT_CREATE)), 201)
|
|
220
|
+
|
|
221
|
+
puts "\n(5) PER-CALLER scoping (same payload from a different caller is NOT a duplicate)"
|
|
222
|
+
check("POST /payments alice (first)", status(post("/payments", payload: PAYMENT_MULTI, as: :alice)), 201)
|
|
223
|
+
check("POST /payments alice (duplicate for alice)", status(post("/payments", payload: PAYMENT_MULTI, as: :alice)), 409)
|
|
224
|
+
check("POST /payments bob (same payload, new caller)", status(post("/payments", payload: PAYMENT_MULTI, as: :bob)), 201)
|
|
225
|
+
check("POST /payments carol (same payload, new caller)", status(post("/payments", payload: PAYMENT_MULTI, as: :carol)), 201)
|
|
226
|
+
check("POST /payments bob (duplicate for bob)", status(post("/payments", payload: PAYMENT_MULTI, as: :bob)), 409)
|
|
227
|
+
|
|
228
|
+
puts "\n(6) DIFFERENT payload, same caller (a different body is a different request, not a dup)"
|
|
229
|
+
check("POST /widgets alice (red widget)", status(post("/widgets", payload: WIDGET_RED)), 201)
|
|
230
|
+
check("POST /widgets alice (green widget)", status(post("/widgets", payload: WIDGET_GREEN)), 201) # different body
|
|
231
|
+
check("POST /widgets alice (red widget again)", status(post("/widgets", payload: WIDGET_RED)), 409) # same body -> dup
|
|
232
|
+
|
|
233
|
+
puts "\n(7) CONCURRENT in-flight duplicate (two #{SLOW_SECONDS}s requests at once -> exactly one wins)"
|
|
234
|
+
outcomes = [:alice, :alice].map do |as|
|
|
235
|
+
Thread.new { status(post("/slow", payload: SLOW_JOB, as: as)) }
|
|
236
|
+
end.map(&:value).sort
|
|
237
|
+
check("one request claimed, the other was rejected", outcomes, [201, 409])
|
|
238
|
+
|
|
239
|
+
puts "\n(8) RELEASE ON FAILURE (a failed request frees the claim so a retry is allowed)"
|
|
240
|
+
check("POST /failures alice (422, claim released)", status(post("/failures", payload: FAIL_PAYLOAD)), 422)
|
|
241
|
+
check("POST /failures alice (retry not blocked)", status(post("/failures", payload: FAIL_PAYLOAD)), 422) # not 409
|
|
242
|
+
check("PATCH /failures/1 alice (raises -> 500)", status(patch("/failures/1", payload: FAIL_PAYLOAD)), 500)
|
|
243
|
+
check("PATCH /failures/1 alice (retry not blocked)", status(patch("/failures/1", payload: FAIL_PAYLOAD)), 500) # not 409
|
|
244
|
+
|
|
245
|
+
puts "\n(9) GET / DELETE are never deduplicated (even though :index/:destroy are guarded by name)"
|
|
246
|
+
check("GET /reads alice (first)", status(request(:get, "/reads", as: :alice)), 200)
|
|
247
|
+
check("GET /reads alice (again)", status(request(:get, "/reads", as: :alice)), 200) # not 409
|
|
248
|
+
check("DELETE /reads/1 alice (first)", status(request(:delete, "/reads/1", as: :alice)), 200)
|
|
249
|
+
check("DELETE /reads/1 alice (again)", status(request(:delete, "/reads/1", as: :alice)), 200) # not 409
|
|
250
|
+
|
|
251
|
+
puts "\n(10) 3xx REDIRECT keeps the claim (Post/Redirect/Get is a successful create)"
|
|
252
|
+
check("POST /redirects alice (first -> 303)", status(post("/redirects", payload: REDIRECT_PAYLOAD)), 303)
|
|
253
|
+
check("POST /redirects alice (duplicate blocked)", status(post("/redirects", payload: REDIRECT_PAYLOAD)), 409)
|
|
254
|
+
|
|
255
|
+
puts "\n(12) HOOKS fire as expected (enforce: on_duplicate_detected AND on_duplicate_rejected)"
|
|
256
|
+
post("/hooked", payload: HOOK_PAYLOAD) # first claims; no hook fires
|
|
257
|
+
check("POST /hooked alice (duplicate -> 409)", status(post("/hooked", payload: HOOK_PAYLOAD)), 409)
|
|
258
|
+
hooked = hook_events.select { |e| e["path"] == "/hooked" }
|
|
259
|
+
check("on_duplicate_detected fired once for /hooked", hooked.count { |e| e["hook"] == "detected" }, 1)
|
|
260
|
+
check("on_duplicate_rejected fired once for /hooked", hooked.count { |e| e["hook"] == "rejected" }, 1)
|
|
261
|
+
detected = hooked.find { |e| e["hook"] == "detected" } || {}
|
|
262
|
+
check("detected hook carries action=create", detected["action"], "create")
|
|
263
|
+
check("detected hook carries verb=POST", detected["verb"], "POST")
|
|
264
|
+
check("detected hook carries a fingerprint", !detected["fingerprint"].to_s.empty?, true)
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
# ===========================================================================
|
|
268
|
+
# observe-mode server: scenario (11)
|
|
269
|
+
# ===========================================================================
|
|
270
|
+
with_server(OBSERVE_PORT, "DEDUPE_MODE" => "observe") do
|
|
271
|
+
puts "\n(11) OBSERVE mode lets duplicates through (detected, but NOT rejected)"
|
|
272
|
+
check("POST /payments alice (first)", status(post("/payments", payload: OBSERVE_PAYLOAD)), 201)
|
|
273
|
+
check("POST /payments alice (duplicate -> allowed)", status(post("/payments", payload: OBSERVE_PAYLOAD)), 201) # 201, not 409
|
|
274
|
+
observed = hook_events.select { |e| e["path"] == "/payments" }
|
|
275
|
+
check("observe: on_duplicate_detected DID fire", observed.count { |e| e["hook"] == "detected" } >= 1, true)
|
|
276
|
+
check("observe: on_duplicate_rejected did NOT fire", observed.any? { |e| e["hook"] == "rejected" }, false)
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
# ===========================================================================
|
|
280
|
+
# custom-fingerprint server: scenario (13)
|
|
281
|
+
# ===========================================================================
|
|
282
|
+
with_server(FINGERPRINT_PORT, "DEDUPE_CUSTOM_FINGERPRINT" => "1") do
|
|
283
|
+
puts "\n(13) FINGERPRINT override hook (custom fingerprint keys on verb+path only)"
|
|
284
|
+
check("POST /widgets alice (body A)", status(post("/widgets", payload: WIDGET_RED)), 201)
|
|
285
|
+
check("POST /widgets alice (different body B -> still dup)", status(post("/widgets", payload: WIDGET_GREEN)), 409) # body ignored
|
|
286
|
+
check("POST /widgets bob (different caller -> still dup)", status(post("/widgets", payload: WIDGET_CREATE, as: :bob)), 409) # caller ignored
|
|
287
|
+
fp = hook_events.select { |e| e["hook"] == "fingerprint" && e["path"] == "/widgets" }
|
|
288
|
+
check("custom fingerprint hook was invoked per request", fp.size, 3)
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
# ===========================================================================
|
|
292
|
+
# custom-caller_id server: scenario (14)
|
|
293
|
+
# ===========================================================================
|
|
294
|
+
with_server(CALLER_ID_PORT, "DEDUPE_CUSTOM_CALLER_ID" => "1") do
|
|
295
|
+
puts "\n(14) CALLER_ID override hook (identity comes from X-Api-Key, not Authorization)"
|
|
296
|
+
# Same X-Api-Key + same body, but DIFFERENT Authorization: a duplicate, which only
|
|
297
|
+
# happens if caller_id reads X-Api-Key (the default would have used Authorization).
|
|
298
|
+
check("POST /widgets api_key=k1 alice (first)", status(post("/widgets", payload: WIDGET_RED, as: :alice, api_key: "k1")), 201)
|
|
299
|
+
check("POST /widgets api_key=k1 bob (diff auth -> dup)", status(post("/widgets", payload: WIDGET_RED, as: :bob, api_key: "k1")), 409)
|
|
300
|
+
check("POST /widgets api_key=k2 alice (diff key -> new caller)", status(post("/widgets", payload: WIDGET_RED, as: :alice, api_key: "k2")), 201)
|
|
301
|
+
ci = hook_events.select { |e| e["hook"] == "caller_id" && e["path"] == "/widgets" }
|
|
302
|
+
check("custom caller_id hook was invoked per request", ci.size, 3)
|
|
303
|
+
check("custom caller_id hook saw the X-Api-Key value", ci.map { |e| e["key"] }.uniq.sort, %w[k1 k2])
|
|
304
|
+
end
|
|
305
|
+
|
|
306
|
+
# ===========================================================================
|
|
307
|
+
puts "\n#{'-' * 78}"
|
|
308
|
+
if FAILURES.empty?
|
|
309
|
+
puts "PASS - all checks green"
|
|
310
|
+
exit 0
|
|
311
|
+
else
|
|
312
|
+
puts "FAIL - #{FAILURES.size} check(s) failed:"
|
|
313
|
+
FAILURES.each { |f| puts " - #{f}" }
|
|
314
|
+
exit 1
|
|
315
|
+
end
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module DedupeRequests
|
|
4
|
+
class Configuration
|
|
5
|
+
MODES = %i[off observe enforce].freeze
|
|
6
|
+
|
|
7
|
+
DEFAULT_CONFLICT_BODY = {
|
|
8
|
+
"errors" => [{
|
|
9
|
+
"error_key" => "base",
|
|
10
|
+
"category" => "duplicate_operation",
|
|
11
|
+
"message" => "Duplicate request detected. A matching request is in-flight or recently completed."
|
|
12
|
+
}]
|
|
13
|
+
}.freeze
|
|
14
|
+
|
|
15
|
+
# Per-caller identity. The callable is given the CONTROLLER, so it can read
|
|
16
|
+
# anything the controller exposes — `current_user`, a helper method, or a
|
|
17
|
+
# header via `controller.request`. Examples:
|
|
18
|
+
# c.caller_id = ->(controller) { controller.current_user&.id }
|
|
19
|
+
# c.caller_id = ->(controller) { controller.request.get_header("HTTP_X_API_KEY") }
|
|
20
|
+
#
|
|
21
|
+
# The default derives identity from the request's Authorization header,
|
|
22
|
+
# falling back to a Rails-style session cookie (so token- and cookie-auth
|
|
23
|
+
# apps work with no configuration). It accepts either a controller or a bare
|
|
24
|
+
# request.
|
|
25
|
+
DEFAULT_CALLER_ID = lambda do |context|
|
|
26
|
+
request = context.respond_to?(:request) ? context.request : context
|
|
27
|
+
if request.respond_to?(:get_header)
|
|
28
|
+
auth = request.get_header("HTTP_AUTHORIZATION")
|
|
29
|
+
return auth if auth && !auth.to_s.empty?
|
|
30
|
+
end
|
|
31
|
+
if request.respond_to?(:cookies)
|
|
32
|
+
request.cookies.each { |name, value| return value if name.to_s =~ /\A_.*_session\z/i }
|
|
33
|
+
end
|
|
34
|
+
nil
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
attr_accessor :redis, :ttl, :digest, :namespace, :caller_id, :fingerprint,
|
|
38
|
+
:conflict_status, :logger,
|
|
39
|
+
:on_duplicate_detected, :on_duplicate_rejected
|
|
40
|
+
attr_writer :store, :conflict_body
|
|
41
|
+
attr_reader :mode
|
|
42
|
+
|
|
43
|
+
def initialize
|
|
44
|
+
@redis = nil
|
|
45
|
+
@store = nil
|
|
46
|
+
@mode = :enforce
|
|
47
|
+
@ttl = 90
|
|
48
|
+
@digest = :sha256
|
|
49
|
+
@namespace = "dedupe_requests"
|
|
50
|
+
@caller_id = DEFAULT_CALLER_ID
|
|
51
|
+
@fingerprint = nil
|
|
52
|
+
@conflict_status = 409
|
|
53
|
+
@logger = nil
|
|
54
|
+
@on_duplicate_detected = nil
|
|
55
|
+
@on_duplicate_rejected = nil
|
|
56
|
+
@conflict_body = nil
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def mode=(value)
|
|
60
|
+
sym = value.to_sym
|
|
61
|
+
unless MODES.include?(sym)
|
|
62
|
+
raise ArgumentError, "unknown mode #{value.inspect} (expected one of #{MODES.join(', ')})"
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
@mode = sym
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def enabled?
|
|
69
|
+
@mode != :off
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def store
|
|
73
|
+
@store ||= (RedisStore.new(@redis, namespace: @namespace, logger: @logger) if @redis)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def conflict_body
|
|
77
|
+
@conflict_body || DEFAULT_CONFLICT_BODY
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "active_support/concern"
|
|
4
|
+
require "active_support/core_ext/class/attribute"
|
|
5
|
+
|
|
6
|
+
module DedupeRequests
|
|
7
|
+
# Rails controller integration.
|
|
8
|
+
#
|
|
9
|
+
# class ApplicationController < ActionController::Base
|
|
10
|
+
# include DedupeRequests::Controller
|
|
11
|
+
# dedupe_requests on: %i[create update]
|
|
12
|
+
# end
|
|
13
|
+
#
|
|
14
|
+
# Registers a SINGLE around_action that, for each guarded action, claims before
|
|
15
|
+
# the action runs and releases on a 4xx/5xx response or a raised exception
|
|
16
|
+
# (2xx/3xx keep the claim). The guarded actions and their per-action TTLs live
|
|
17
|
+
# in an inherited class_attribute map, so subclasses extend or trim it.
|
|
18
|
+
module Controller
|
|
19
|
+
extend ActiveSupport::Concern
|
|
20
|
+
|
|
21
|
+
included do
|
|
22
|
+
# Map of guarded action (Symbol) => TTL (Integer, or nil meaning "use the
|
|
23
|
+
# global config TTL"). Inherited and copy-on-write, so a subclass can add
|
|
24
|
+
# to or trim it without touching the parent.
|
|
25
|
+
class_attribute :dedupe_requests_action_ttls, instance_accessor: false, default: {}
|
|
26
|
+
around_action :dedupe_requests_around
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
class_methods do
|
|
30
|
+
# Guard the named actions. A `ttl:`, if given, applies to exactly the
|
|
31
|
+
# actions named in THIS call. Calls accumulate, so per-action TTLs are
|
|
32
|
+
# expressed by repeating the line:
|
|
33
|
+
#
|
|
34
|
+
# dedupe_requests on: %i[create update] # both, global TTL
|
|
35
|
+
# dedupe_requests on: [:create], ttl: 120 # create → 120
|
|
36
|
+
# dedupe_requests on: [:update], ttl: 180 # update → 180
|
|
37
|
+
#
|
|
38
|
+
# Re-naming an action overrides its TTL. Subclasses inherit the map and can
|
|
39
|
+
# add to it or remove from it (`skip:` / `skip_dedupe_requests`).
|
|
40
|
+
def dedupe_requests(on: nil, skip: nil, ttl: nil)
|
|
41
|
+
map = dedupe_requests_action_ttls.dup
|
|
42
|
+
|
|
43
|
+
Array(on).each { |action| map[action.to_sym] = ttl }
|
|
44
|
+
Array(skip).each { |action| map.delete(action.to_sym) }
|
|
45
|
+
|
|
46
|
+
self.dedupe_requests_action_ttls = map
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def skip_dedupe_requests(on: nil)
|
|
50
|
+
map = dedupe_requests_action_ttls.dup
|
|
51
|
+
Array(on).each { |action| map.delete(action.to_sym) }
|
|
52
|
+
self.dedupe_requests_action_ttls = map
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# The set of guarded actions (the keys of the TTL map).
|
|
56
|
+
def dedupe_requests_actions
|
|
57
|
+
dedupe_requests_action_ttls.keys
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
private
|
|
62
|
+
|
|
63
|
+
def dedupe_requests_around
|
|
64
|
+
unless dedupe_requests_applies?
|
|
65
|
+
yield
|
|
66
|
+
return
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
result = dedupe_requests_guard.claim(
|
|
70
|
+
request,
|
|
71
|
+
ttl: dedupe_requests_ttl_for(action_name),
|
|
72
|
+
caller_id: dedupe_requests_caller_id
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
case result.outcome
|
|
76
|
+
when :duplicate
|
|
77
|
+
dedupe_requests_notify(:on_duplicate_detected, result)
|
|
78
|
+
if DedupeRequests.config.mode == :enforce
|
|
79
|
+
dedupe_requests_notify(:on_duplicate_rejected, result)
|
|
80
|
+
dedupe_requests_render_conflict
|
|
81
|
+
else
|
|
82
|
+
yield # observe mode: detected but allowed through
|
|
83
|
+
end
|
|
84
|
+
when :claimed
|
|
85
|
+
begin
|
|
86
|
+
yield
|
|
87
|
+
rescue Exception # rubocop:disable Lint/RescueException
|
|
88
|
+
dedupe_requests_guard.release(result)
|
|
89
|
+
raise
|
|
90
|
+
else
|
|
91
|
+
dedupe_requests_guard.release(result) if dedupe_requests_release?(response.status)
|
|
92
|
+
end
|
|
93
|
+
else # :skip
|
|
94
|
+
yield
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def dedupe_requests_applies?
|
|
99
|
+
return false unless DedupeRequests.config.enabled?
|
|
100
|
+
|
|
101
|
+
self.class.dedupe_requests_action_ttls.key?(action_name.to_sym)
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# Per-action TTL, falling back to the global config TTL.
|
|
105
|
+
def dedupe_requests_ttl_for(action)
|
|
106
|
+
self.class.dedupe_requests_action_ttls[action.to_sym] || DedupeRequests.config.ttl
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# Resolve the caller identity by handing the whole controller to the
|
|
110
|
+
# configured `caller_id` callable (so it can use current_user, a header, etc.).
|
|
111
|
+
def dedupe_requests_caller_id
|
|
112
|
+
DedupeRequests.config.caller_id&.call(self)
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def dedupe_requests_guard
|
|
116
|
+
@dedupe_requests_guard ||= DedupeRequests::Guard.new(DedupeRequests.config)
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# Keep the fingerprint when the request was handled — a 2xx, or a 3xx
|
|
120
|
+
# redirect (the Post/Redirect/Get pattern is a *successful* create) — so a
|
|
121
|
+
# later duplicate is still blocked for the full TTL. Only a 4xx/5xx (or a
|
|
122
|
+
# raised exception) releases it, so a genuinely failed request can be retried.
|
|
123
|
+
def dedupe_requests_release?(status)
|
|
124
|
+
status.to_i >= 400
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def dedupe_requests_render_conflict
|
|
128
|
+
response.set_header("X-Dedupe-Request", "true")
|
|
129
|
+
render json: DedupeRequests.config.conflict_body, status: DedupeRequests.config.conflict_status
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def dedupe_requests_notify(hook, result)
|
|
133
|
+
callback = DedupeRequests.config.public_send(hook)
|
|
134
|
+
return unless callback
|
|
135
|
+
|
|
136
|
+
callback.call(
|
|
137
|
+
fingerprint: result.fingerprint,
|
|
138
|
+
controller: controller_name,
|
|
139
|
+
action: action_name,
|
|
140
|
+
verb: request.request_method,
|
|
141
|
+
path: request.path
|
|
142
|
+
)
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
end
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
|
|
5
|
+
begin
|
|
6
|
+
require "openssl"
|
|
7
|
+
rescue LoadError
|
|
8
|
+
# Ruby built without OpenSSL — the digests fall back to the stdlib Digest below.
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
module DedupeRequests
|
|
12
|
+
# Computes a stable fingerprint for a request.
|
|
13
|
+
#
|
|
14
|
+
# The fingerprint covers: caller_id + verb + path + query string + body.
|
|
15
|
+
# Time is NOT part of the fingerprint — the dedup window is the Redis TTL.
|
|
16
|
+
module Fingerprint
|
|
17
|
+
# Prefer OpenSSL (uses the CPU's SHA instructions — several times faster on
|
|
18
|
+
# large bodies), falling back to the stdlib Digest when OpenSSL is unavailable
|
|
19
|
+
# or has the algorithm disabled (e.g. MD5 under FIPS). Output is identical.
|
|
20
|
+
HASHERS = {
|
|
21
|
+
sha256: ["SHA256", Digest::SHA256],
|
|
22
|
+
sha1: ["SHA1", Digest::SHA1],
|
|
23
|
+
sha512: ["SHA512", Digest::SHA512],
|
|
24
|
+
md5: ["MD5", Digest::MD5]
|
|
25
|
+
}.freeze
|
|
26
|
+
|
|
27
|
+
ALGORITHMS = HASHERS.transform_values do |(openssl_name, digest_class)|
|
|
28
|
+
lambda do |data|
|
|
29
|
+
OpenSSL::Digest.hexdigest(openssl_name, data)
|
|
30
|
+
rescue StandardError
|
|
31
|
+
digest_class.hexdigest(data)
|
|
32
|
+
end
|
|
33
|
+
end.freeze
|
|
34
|
+
|
|
35
|
+
module_function
|
|
36
|
+
|
|
37
|
+
def for_request(request, config, caller_id: nil)
|
|
38
|
+
return config.fingerprint.call(request) if config.fingerprint
|
|
39
|
+
|
|
40
|
+
parts = [
|
|
41
|
+
caller_id.to_s,
|
|
42
|
+
request.request_method.to_s,
|
|
43
|
+
request.path.to_s,
|
|
44
|
+
request.query_string.to_s,
|
|
45
|
+
body(request)
|
|
46
|
+
]
|
|
47
|
+
digest(parts, config.digest)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Length-prefixes each field so a value cannot "shift" across a field
|
|
51
|
+
# boundary and collide with a different set of fields.
|
|
52
|
+
def digest(parts, algorithm = :sha256)
|
|
53
|
+
data = Array(parts).map do |part|
|
|
54
|
+
s = part.to_s
|
|
55
|
+
"#{s.bytesize}:#{s}"
|
|
56
|
+
end.join
|
|
57
|
+
resolve(algorithm).call(data)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def resolve(algorithm)
|
|
61
|
+
return algorithm if algorithm.respond_to?(:call)
|
|
62
|
+
|
|
63
|
+
ALGORITHMS.fetch(algorithm.to_sym) do
|
|
64
|
+
raise ArgumentError, "unknown digest #{algorithm.inspect} (known: #{ALGORITHMS.keys.join(', ')} or a callable)"
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def body(request)
|
|
69
|
+
request.respond_to?(:raw_post) ? request.raw_post.to_s : read_and_rewind(request).to_s
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def read_and_rewind(request)
|
|
73
|
+
return "" unless request.respond_to?(:body) && request.body
|
|
74
|
+
|
|
75
|
+
data = request.body.read
|
|
76
|
+
request.body.rewind if request.body.respond_to?(:rewind)
|
|
77
|
+
data
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module DedupeRequests
|
|
4
|
+
# Framework-agnostic core: turns a request into a claim decision, and releases
|
|
5
|
+
# a claim. Knows nothing about Rails rendering — that lives in the concern.
|
|
6
|
+
class Guard
|
|
7
|
+
# outcome: :claimed | :duplicate | :skip
|
|
8
|
+
Result = Struct.new(:outcome, :fingerprint, :token)
|
|
9
|
+
|
|
10
|
+
def initialize(config)
|
|
11
|
+
@config = config
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def claim(request, ttl: @config.ttl, caller_id: nil)
|
|
15
|
+
return Result.new(:skip) unless @config.enabled?
|
|
16
|
+
return Result.new(:skip) unless DedupeRequests::MUTATING_VERBS.include?(request.request_method.to_s)
|
|
17
|
+
|
|
18
|
+
store = @config.store
|
|
19
|
+
return Result.new(:skip) unless store
|
|
20
|
+
|
|
21
|
+
fingerprint = Fingerprint.for_request(request, @config, caller_id: caller_id)
|
|
22
|
+
token = store.claim(fingerprint, ttl: ttl)
|
|
23
|
+
|
|
24
|
+
case token
|
|
25
|
+
when :error then Result.new(:skip) # Redis down → fail open
|
|
26
|
+
when false then Result.new(:duplicate, fingerprint)
|
|
27
|
+
else Result.new(:claimed, fingerprint, token)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def release(result)
|
|
32
|
+
return false unless result && result.outcome == :claimed
|
|
33
|
+
|
|
34
|
+
@config.store.release(result.fingerprint, result.token)
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|