honker 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ext/honker/honker-core/Cargo.toml +2 -1
- data/ext/honker/honker-core/src/honker_ops.rs +340 -169
- data/ext/honker/honker-core/src/lib.rs +151 -9
- data/ext/honker/honker-extension/Cargo.toml +2 -2
- data/lib/honker/lock.rb +3 -3
- data/lib/honker/scheduler.rb +26 -18
- data/lib/honker/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 8dae25a4f10b6fe9125f3f5f56a0a5e482c17a9c716c58e3ccb785d7bccd2a6b
|
|
4
|
+
data.tar.gz: 996f23893693f8c8a5ca3c47ac1e391ca858efc835e8a0c04229721ecafc50fe
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: ced93ac94928611a6060ae308bf4637d6ff97c33ce8716281f64fa680f128329db7fb22f1a35d9d955f373061e7e705bdfeca670f397335a5771002cb3ab324e
|
|
7
|
+
data.tar.gz: 5efad42d288ad8dffe9bfed6aed633ad2f96dd00dbc5df9dbf310282e9e103eb8d2c8f2746d87b5fcc537e6fbd737b2ec719e60264d4ddaeb57c117c97f834ef
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "honker-core"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.4.0"
|
|
4
4
|
edition = "2024"
|
|
5
5
|
description = "Shared Rust foundation for Honker bindings (SQLite loadable extension, PyO3, napi-rs, and friends). Not intended for direct use."
|
|
6
6
|
license = "MIT OR Apache-2.0"
|
|
@@ -28,6 +28,7 @@ name = "honker_core"
|
|
|
28
28
|
chrono = { version = "0.4", default-features = false, features = ["clock"] }
|
|
29
29
|
parking_lot = "0.12.5"
|
|
30
30
|
rusqlite = { version = "0.39.0", features = ["functions", "hooks"] }
|
|
31
|
+
serde_json = "1"
|
|
31
32
|
thiserror = "2.0.18"
|
|
32
33
|
# Optional: kernel-watch backend.
|
|
33
34
|
# macos_kqueue: on macOS, use kqueue (RecommendedWatcher = KqueueWatcher) instead of
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
|
|
21
21
|
use rusqlite::Connection;
|
|
22
22
|
use rusqlite::functions::FunctionFlags;
|
|
23
|
+
use serde_json::{Value, json};
|
|
23
24
|
|
|
24
25
|
/// Wrap a Displayable error for SQLite scalar-function returns.
|
|
25
26
|
fn to_sql_err<E: std::fmt::Display>(e: E) -> rusqlite::Error {
|
|
@@ -102,6 +103,18 @@ pub fn attach_honker_functions(conn: &Connection) -> rusqlite::Result<()> {
|
|
|
102
103
|
},
|
|
103
104
|
)?;
|
|
104
105
|
|
|
106
|
+
// honker_lock_renew(name, owner, ttl_s) -> 1 if this owner still
|
|
107
|
+
// holds the lock and expires_at was extended, 0 otherwise.
|
|
108
|
+
// Distinct from honker_lock_acquire: INSERT OR IGNORE does not
|
|
109
|
+
// refresh expires_at for an existing (name, owner) row.
|
|
110
|
+
conn.create_scalar_function("honker_lock_renew", 3, FunctionFlags::SQLITE_UTF8, |ctx| {
|
|
111
|
+
let name: String = ctx.get(0)?;
|
|
112
|
+
let owner: String = ctx.get(1)?;
|
|
113
|
+
let ttl: i64 = ctx.get(2)?;
|
|
114
|
+
let db = unsafe { ctx.get_connection() }?;
|
|
115
|
+
lock_renew(&db, &name, &owner, ttl).map_err(to_sql_err)
|
|
116
|
+
})?;
|
|
117
|
+
|
|
105
118
|
conn.create_scalar_function(
|
|
106
119
|
"honker_rate_limit_try",
|
|
107
120
|
3,
|
|
@@ -128,6 +141,8 @@ pub fn attach_honker_functions(conn: &Connection) -> rusqlite::Result<()> {
|
|
|
128
141
|
|
|
129
142
|
// honker_scheduler_register(name, queue, cron_expr, payload_json,
|
|
130
143
|
// priority, expires_s_or_null) -> 1.
|
|
144
|
+
// Optional 7th arg max_attempts (default 3) pins the attempt budget
|
|
145
|
+
// on every job the scheduler enqueues for this task.
|
|
131
146
|
// Upserts the task row. `next_fire_at` is recomputed as the next
|
|
132
147
|
// cron boundary strictly after `unixepoch()`. Calling twice with
|
|
133
148
|
// the same name replaces the first registration entirely.
|
|
@@ -144,7 +159,33 @@ pub fn attach_honker_functions(conn: &Connection) -> rusqlite::Result<()> {
|
|
|
144
159
|
let expires_s: Option<i64> = ctx.get(5)?;
|
|
145
160
|
let db = unsafe { ctx.get_connection() }?;
|
|
146
161
|
scheduler_register(
|
|
147
|
-
&db, &name, &queue, &cron_expr, &payload, priority, expires_s,
|
|
162
|
+
&db, &name, &queue, &cron_expr, &payload, priority, expires_s, 3,
|
|
163
|
+
)
|
|
164
|
+
.map_err(to_sql_err)
|
|
165
|
+
},
|
|
166
|
+
)?;
|
|
167
|
+
conn.create_scalar_function(
|
|
168
|
+
"honker_scheduler_register",
|
|
169
|
+
7,
|
|
170
|
+
FunctionFlags::SQLITE_UTF8,
|
|
171
|
+
|ctx| {
|
|
172
|
+
let name: String = ctx.get(0)?;
|
|
173
|
+
let queue: String = ctx.get(1)?;
|
|
174
|
+
let cron_expr: String = ctx.get(2)?;
|
|
175
|
+
let payload: String = ctx.get(3)?;
|
|
176
|
+
let priority: i64 = ctx.get(4)?;
|
|
177
|
+
let expires_s: Option<i64> = ctx.get(5)?;
|
|
178
|
+
let max_attempts: i64 = ctx.get(6)?;
|
|
179
|
+
let db = unsafe { ctx.get_connection() }?;
|
|
180
|
+
scheduler_register(
|
|
181
|
+
&db,
|
|
182
|
+
&name,
|
|
183
|
+
&queue,
|
|
184
|
+
&cron_expr,
|
|
185
|
+
&payload,
|
|
186
|
+
priority,
|
|
187
|
+
expires_s,
|
|
188
|
+
max_attempts,
|
|
148
189
|
)
|
|
149
190
|
.map_err(to_sql_err)
|
|
150
191
|
},
|
|
@@ -229,10 +270,13 @@ pub fn attach_honker_functions(conn: &Connection) -> rusqlite::Result<()> {
|
|
|
229
270
|
// honker_scheduler_update(name, cron_expr_or_null, payload_or_null,
|
|
230
271
|
// priority_or_null, expires_s_or_null,
|
|
231
272
|
// touch_expires) -> 1 if updated, 0 if missing.
|
|
273
|
+
// Optional 8-arg form adds max_attempts_or_null, touch_max_attempts.
|
|
232
274
|
// `touch_expires` is a 0/1 flag: when 1 we treat the expires_s arg
|
|
233
275
|
// as the desired value (which may be NULL = "clear"); when 0 we
|
|
234
276
|
// leave expires_s untouched. SQL has no good way to distinguish
|
|
235
|
-
// "user passed NULL" from "user did not specify" otherwise.
|
|
277
|
+
// "user passed NULL" from "user did not specify" otherwise. Same
|
|
278
|
+
// pattern for max_attempts so old 6-arg raw callers stay compatible;
|
|
279
|
+
// explicit NULL resets max_attempts to the scheduler default (3).
|
|
236
280
|
conn.create_scalar_function(
|
|
237
281
|
"honker_scheduler_update",
|
|
238
282
|
6,
|
|
@@ -257,6 +301,43 @@ pub fn attach_honker_functions(conn: &Connection) -> rusqlite::Result<()> {
|
|
|
257
301
|
payload.as_deref(),
|
|
258
302
|
priority,
|
|
259
303
|
expires_s,
|
|
304
|
+
None,
|
|
305
|
+
)
|
|
306
|
+
.map_err(to_sql_err)
|
|
307
|
+
},
|
|
308
|
+
)?;
|
|
309
|
+
conn.create_scalar_function(
|
|
310
|
+
"honker_scheduler_update",
|
|
311
|
+
8,
|
|
312
|
+
FunctionFlags::SQLITE_UTF8,
|
|
313
|
+
|ctx| {
|
|
314
|
+
let name: String = ctx.get(0)?;
|
|
315
|
+
let cron_expr: Option<String> = ctx.get(1)?;
|
|
316
|
+
let payload: Option<String> = ctx.get(2)?;
|
|
317
|
+
let priority: Option<i64> = ctx.get(3)?;
|
|
318
|
+
let expires_s_arg: Option<i64> = ctx.get(4)?;
|
|
319
|
+
let touch_expires: i64 = ctx.get(5)?;
|
|
320
|
+
let max_attempts_arg: Option<i64> = ctx.get(6)?;
|
|
321
|
+
let touch_max_attempts: i64 = ctx.get(7)?;
|
|
322
|
+
let db = unsafe { ctx.get_connection() }?;
|
|
323
|
+
let expires_s = if touch_expires != 0 {
|
|
324
|
+
Some(expires_s_arg)
|
|
325
|
+
} else {
|
|
326
|
+
None
|
|
327
|
+
};
|
|
328
|
+
let max_attempts = if touch_max_attempts != 0 {
|
|
329
|
+
Some(max_attempts_arg)
|
|
330
|
+
} else {
|
|
331
|
+
None
|
|
332
|
+
};
|
|
333
|
+
scheduler_update(
|
|
334
|
+
&db,
|
|
335
|
+
&name,
|
|
336
|
+
cron_expr.as_deref(),
|
|
337
|
+
payload.as_deref(),
|
|
338
|
+
priority,
|
|
339
|
+
expires_s,
|
|
340
|
+
max_attempts,
|
|
260
341
|
)
|
|
261
342
|
.map_err(to_sql_err)
|
|
262
343
|
},
|
|
@@ -457,6 +538,60 @@ pub fn attach_honker_functions(conn: &Connection) -> rusqlite::Result<()> {
|
|
|
457
538
|
// Claim / ack
|
|
458
539
|
// ---------------------------------------------------------------------
|
|
459
540
|
|
|
541
|
+
/// Move claimable rows that have already exhausted `max_attempts` into
|
|
542
|
+
/// `_honker_dead`. Without this, a worker that dies after the last
|
|
543
|
+
/// allowed claim leaves the row reclaimable forever — every reclaim
|
|
544
|
+
/// would bump `attempts` past `max_attempts` with no dead-letter path
|
|
545
|
+
/// (dead-letter previously only ran inside `retry()`).
|
|
546
|
+
///
|
|
547
|
+
/// "Claimable" here matches the reclaim predicate: pending+due or
|
|
548
|
+
/// processing with an expired visibility timeout. In-flight claims
|
|
549
|
+
/// that still hold a valid timeout are left alone so the holder can
|
|
550
|
+
/// still ack / retry / fail.
|
|
551
|
+
fn dead_letter_exhausted_claimable(conn: &Connection, queue: &str) -> rusqlite::Result<i64> {
|
|
552
|
+
let mut select = conn.prepare_cached(
|
|
553
|
+
"DELETE FROM _honker_live
|
|
554
|
+
WHERE queue = ?1
|
|
555
|
+
AND attempts >= max_attempts
|
|
556
|
+
AND (expires_at IS NULL OR expires_at > unixepoch())
|
|
557
|
+
AND (
|
|
558
|
+
(state = 'pending' AND run_at <= unixepoch())
|
|
559
|
+
OR (state = 'processing' AND claim_expires_at < unixepoch())
|
|
560
|
+
)
|
|
561
|
+
RETURNING id, queue, payload, priority, run_at, max_attempts,
|
|
562
|
+
attempts, created_at",
|
|
563
|
+
)?;
|
|
564
|
+
#[allow(clippy::type_complexity)]
|
|
565
|
+
let rows: Vec<(i64, String, String, i64, i64, i64, i64, i64)> = select
|
|
566
|
+
.query_map(rusqlite::params![queue], |r| {
|
|
567
|
+
Ok((
|
|
568
|
+
r.get(0)?,
|
|
569
|
+
r.get(1)?,
|
|
570
|
+
r.get(2)?,
|
|
571
|
+
r.get(3)?,
|
|
572
|
+
r.get(4)?,
|
|
573
|
+
r.get(5)?,
|
|
574
|
+
r.get(6)?,
|
|
575
|
+
r.get(7)?,
|
|
576
|
+
))
|
|
577
|
+
})?
|
|
578
|
+
.collect::<Result<Vec<_>, _>>()?;
|
|
579
|
+
if rows.is_empty() {
|
|
580
|
+
return Ok(0);
|
|
581
|
+
}
|
|
582
|
+
let mut insert = conn.prepare_cached(
|
|
583
|
+
"INSERT INTO _honker_dead
|
|
584
|
+
(id, queue, payload, priority, run_at, max_attempts,
|
|
585
|
+
attempts, last_error, created_at)
|
|
586
|
+
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, 'max attempts exceeded', ?8)",
|
|
587
|
+
)?;
|
|
588
|
+
let count = rows.len() as i64;
|
|
589
|
+
for r in rows {
|
|
590
|
+
insert.execute(rusqlite::params![r.0, r.1, r.2, r.3, r.4, r.5, r.6, r.7])?;
|
|
591
|
+
}
|
|
592
|
+
Ok(count)
|
|
593
|
+
}
|
|
594
|
+
|
|
460
595
|
/// Returns JSON text: `[{"id":1,"queue":"...","payload":"...","worker_id":"...","attempts":N,"claim_expires_at":T}, ...]`
|
|
461
596
|
pub fn claim_batch(
|
|
462
597
|
conn: &Connection,
|
|
@@ -465,6 +600,12 @@ pub fn claim_batch(
|
|
|
465
600
|
n: i64,
|
|
466
601
|
timeout_s: i64,
|
|
467
602
|
) -> rusqlite::Result<String> {
|
|
603
|
+
// Drop reclaimable rows that already used their attempt budget so
|
|
604
|
+
// they cannot be claimed again (and so they don't clog the claim
|
|
605
|
+
// index forever). Same outer SQL statement / connection, so this
|
|
606
|
+
// shares the caller's transaction with the claim UPDATE below.
|
|
607
|
+
dead_letter_exhausted_claimable(conn, queue)?;
|
|
608
|
+
|
|
468
609
|
let mut stmt = conn.prepare_cached(
|
|
469
610
|
"UPDATE _honker_live
|
|
470
611
|
SET state = 'processing',
|
|
@@ -475,6 +616,7 @@ pub fn claim_batch(
|
|
|
475
616
|
SELECT id FROM _honker_live
|
|
476
617
|
WHERE queue = ?2
|
|
477
618
|
AND state IN ('pending', 'processing')
|
|
619
|
+
AND attempts < max_attempts
|
|
478
620
|
AND (expires_at IS NULL OR expires_at > unixepoch())
|
|
479
621
|
AND ((state = 'pending' AND run_at <= unixepoch())
|
|
480
622
|
OR (state = 'processing' AND claim_expires_at < unixepoch()))
|
|
@@ -493,22 +635,21 @@ pub fn claim_batch(
|
|
|
493
635
|
row.get::<_, i64>(5)?,
|
|
494
636
|
))
|
|
495
637
|
})?;
|
|
496
|
-
let mut out =
|
|
497
|
-
let mut first = true;
|
|
638
|
+
let mut out = Vec::new();
|
|
498
639
|
for row in rows {
|
|
499
640
|
let (id, q, payload, w, attempts, claim_expires_at) = row?;
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
"
|
|
506
|
-
|
|
507
|
-
attempts
|
|
508
|
-
|
|
641
|
+
// payload stays a JSON string (double-encoded on the wire) so
|
|
642
|
+
// every binding's existing parse path keeps working.
|
|
643
|
+
out.push(json!({
|
|
644
|
+
"id": id,
|
|
645
|
+
"queue": q,
|
|
646
|
+
"payload": payload,
|
|
647
|
+
"worker_id": w,
|
|
648
|
+
"attempts": attempts,
|
|
649
|
+
"claim_expires_at": claim_expires_at,
|
|
650
|
+
}));
|
|
509
651
|
}
|
|
510
|
-
out.
|
|
511
|
-
Ok(out)
|
|
652
|
+
Ok(Value::Array(out).to_string())
|
|
512
653
|
}
|
|
513
654
|
|
|
514
655
|
pub fn ack_batch(conn: &Connection, ids_json: &str, worker_id: &str) -> rusqlite::Result<i64> {
|
|
@@ -532,6 +673,9 @@ pub fn ack_batch(conn: &Connection, ids_json: &str, worker_id: &str) -> rusqlite
|
|
|
532
673
|
/// * a pending row's `run_at`
|
|
533
674
|
/// * one second after a processing row's `claim_expires_at`
|
|
534
675
|
///
|
|
676
|
+
/// Rows that have already exhausted `max_attempts` are ignored — they
|
|
677
|
+
/// are dead-lettered on the next claim path, not reclaimable.
|
|
678
|
+
///
|
|
535
679
|
/// Returns 0 if no such future deadline exists.
|
|
536
680
|
pub fn queue_next_claim_at(conn: &Connection, queue: &str) -> rusqlite::Result<i64> {
|
|
537
681
|
Ok(conn
|
|
@@ -542,6 +686,7 @@ pub fn queue_next_claim_at(conn: &Connection, queue: &str) -> rusqlite::Result<i
|
|
|
542
686
|
FROM _honker_live
|
|
543
687
|
WHERE queue = ?1
|
|
544
688
|
AND state = 'pending'
|
|
689
|
+
AND attempts < max_attempts
|
|
545
690
|
AND (expires_at IS NULL OR expires_at > unixepoch())
|
|
546
691
|
AND run_at > unixepoch()
|
|
547
692
|
UNION ALL
|
|
@@ -549,6 +694,7 @@ pub fn queue_next_claim_at(conn: &Connection, queue: &str) -> rusqlite::Result<i
|
|
|
549
694
|
FROM _honker_live
|
|
550
695
|
WHERE queue = ?1
|
|
551
696
|
AND state = 'processing'
|
|
697
|
+
AND attempts < max_attempts
|
|
552
698
|
AND (expires_at IS NULL OR expires_at > unixepoch())
|
|
553
699
|
AND claim_expires_at >= unixepoch()
|
|
554
700
|
)",
|
|
@@ -587,8 +733,12 @@ pub fn enqueue(
|
|
|
587
733
|
(None, None) => now,
|
|
588
734
|
};
|
|
589
735
|
let expires_at: Option<i64> = expires.map(|e| now + e);
|
|
590
|
-
let channel = format!("honker:{}", queue);
|
|
591
736
|
|
|
737
|
+
// No synthetic `_honker_notifications` row. The live-table INSERT
|
|
738
|
+
// already advances PRAGMA data_version on commit, which is what
|
|
739
|
+
// SharedUpdateWatcher / every binding's update_events path observes.
|
|
740
|
+
// Writing a wake row per enqueue used to grow the notifications
|
|
741
|
+
// table without bound on high-throughput queues.
|
|
592
742
|
let id: i64 = conn.query_row(
|
|
593
743
|
"INSERT INTO _honker_live
|
|
594
744
|
(queue, payload, run_at, priority, max_attempts, expires_at)
|
|
@@ -604,12 +754,6 @@ pub fn enqueue(
|
|
|
604
754
|
],
|
|
605
755
|
|r| r.get(0),
|
|
606
756
|
)?;
|
|
607
|
-
// Fire a wake so workers parked on this queue's channel re-poll.
|
|
608
|
-
conn.execute(
|
|
609
|
-
"INSERT INTO _honker_notifications (channel, payload)
|
|
610
|
-
VALUES (?1, 'new')",
|
|
611
|
-
rusqlite::params![channel],
|
|
612
|
-
)?;
|
|
613
757
|
Ok(id)
|
|
614
758
|
}
|
|
615
759
|
|
|
@@ -700,14 +844,8 @@ pub fn retry(
|
|
|
700
844
|
WHERE id = ?1",
|
|
701
845
|
rusqlite::params![id, delay_s],
|
|
702
846
|
)?;
|
|
703
|
-
//
|
|
704
|
-
//
|
|
705
|
-
let channel = format!("honker:{}", queue);
|
|
706
|
-
conn.execute(
|
|
707
|
-
"INSERT INTO _honker_notifications (channel, payload)
|
|
708
|
-
VALUES (?1, 'new')",
|
|
709
|
-
rusqlite::params![channel],
|
|
710
|
-
)?;
|
|
847
|
+
// Wake comes from the live-table UPDATE + commit (data_version).
|
|
848
|
+
// No synthetic notification row — see enqueue() for rationale.
|
|
711
849
|
}
|
|
712
850
|
Ok(1)
|
|
713
851
|
}
|
|
@@ -837,32 +975,21 @@ pub fn get_job(conn: &Connection, job_id: i64) -> rusqlite::Result<String> {
|
|
|
837
975
|
else {
|
|
838
976
|
return Ok(String::new());
|
|
839
977
|
};
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
"
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
json_str(&payload),
|
|
856
|
-
json_str(&state),
|
|
857
|
-
priority,
|
|
858
|
-
run_at,
|
|
859
|
-
opt_str(&worker_id),
|
|
860
|
-
opt_i(claim_expires_at),
|
|
861
|
-
attempts,
|
|
862
|
-
max_attempts,
|
|
863
|
-
created_at,
|
|
864
|
-
opt_i(expires_at),
|
|
865
|
-
))
|
|
978
|
+
Ok(json!({
|
|
979
|
+
"id": id,
|
|
980
|
+
"queue": queue,
|
|
981
|
+
"payload": payload,
|
|
982
|
+
"state": state,
|
|
983
|
+
"priority": priority,
|
|
984
|
+
"run_at": run_at,
|
|
985
|
+
"worker_id": worker_id,
|
|
986
|
+
"claim_expires_at": claim_expires_at,
|
|
987
|
+
"attempts": attempts,
|
|
988
|
+
"max_attempts": max_attempts,
|
|
989
|
+
"created_at": created_at,
|
|
990
|
+
"expires_at": expires_at,
|
|
991
|
+
})
|
|
992
|
+
.to_string())
|
|
866
993
|
}
|
|
867
994
|
|
|
868
995
|
/// Extend the current claim by `extend_s` seconds. Returns 1 if the
|
|
@@ -874,10 +1001,14 @@ pub fn heartbeat(
|
|
|
874
1001
|
worker_id: &str,
|
|
875
1002
|
extend_s: i64,
|
|
876
1003
|
) -> rusqlite::Result<i64> {
|
|
1004
|
+
// Require a still-valid claim. Without `claim_expires_at >= now`,
|
|
1005
|
+
// a late heartbeat after visibility timeout can steal the job
|
|
1006
|
+
// back from a reclaimer (dual execution).
|
|
877
1007
|
let updated = conn.execute(
|
|
878
1008
|
"UPDATE _honker_live
|
|
879
1009
|
SET claim_expires_at = unixepoch() + ?3
|
|
880
|
-
WHERE id = ?1 AND worker_id = ?2 AND state = 'processing'
|
|
1010
|
+
WHERE id = ?1 AND worker_id = ?2 AND state = 'processing'
|
|
1011
|
+
AND claim_expires_at >= unixepoch()",
|
|
881
1012
|
rusqlite::params![job_id, worker_id, extend_s],
|
|
882
1013
|
)?;
|
|
883
1014
|
Ok(updated as i64)
|
|
@@ -972,6 +1103,25 @@ pub fn lock_release(conn: &Connection, name: &str, owner: &str) -> rusqlite::Res
|
|
|
972
1103
|
Ok(deleted as i64)
|
|
973
1104
|
}
|
|
974
1105
|
|
|
1106
|
+
/// Extend `expires_at` for a lock held by `owner`. Returns 1 if the
|
|
1107
|
+
/// row was updated, 0 if the lock is missing or held by someone else.
|
|
1108
|
+
///
|
|
1109
|
+
/// `honker_lock_acquire` uses `INSERT OR IGNORE` and does **not**
|
|
1110
|
+
/// refresh TTL on same-owner re-acquire — callers that need renewal
|
|
1111
|
+
/// (scheduler leaders, long critical sections) must use this.
|
|
1112
|
+
pub fn lock_renew(conn: &Connection, name: &str, owner: &str, ttl_s: i64) -> rusqlite::Result<i64> {
|
|
1113
|
+
if ttl_s <= 0 {
|
|
1114
|
+
return Err(to_sql_err("ttl_s must be positive"));
|
|
1115
|
+
}
|
|
1116
|
+
let updated = conn.execute(
|
|
1117
|
+
"UPDATE _honker_locks
|
|
1118
|
+
SET expires_at = unixepoch() + ?3
|
|
1119
|
+
WHERE name = ?1 AND owner = ?2",
|
|
1120
|
+
rusqlite::params![name, owner, ttl_s],
|
|
1121
|
+
)?;
|
|
1122
|
+
Ok(if updated > 0 { 1 } else { 0 })
|
|
1123
|
+
}
|
|
1124
|
+
|
|
975
1125
|
// ---------------------------------------------------------------------
|
|
976
1126
|
// Rate limiting
|
|
977
1127
|
// ---------------------------------------------------------------------
|
|
@@ -1016,7 +1166,8 @@ pub fn rate_limit_sweep(conn: &Connection, older_than_s: i64) -> rusqlite::Resul
|
|
|
1016
1166
|
/// Register (or re-register) a periodic task. `next_fire_at` is
|
|
1017
1167
|
/// computed as the next cron boundary strictly after
|
|
1018
1168
|
/// `unixepoch()`. Calling twice with the same name replaces the
|
|
1019
|
-
/// first registration entirely.
|
|
1169
|
+
/// first registration entirely. `max_attempts` is stored on the task
|
|
1170
|
+
/// row and applied to every job `scheduler_tick` enqueues for it.
|
|
1020
1171
|
pub fn scheduler_register(
|
|
1021
1172
|
conn: &Connection,
|
|
1022
1173
|
name: &str,
|
|
@@ -1025,20 +1176,23 @@ pub fn scheduler_register(
|
|
|
1025
1176
|
payload: &str,
|
|
1026
1177
|
priority: i64,
|
|
1027
1178
|
expires_s: Option<i64>,
|
|
1179
|
+
max_attempts: i64,
|
|
1028
1180
|
) -> rusqlite::Result<i64> {
|
|
1181
|
+
let max_attempts = if max_attempts < 1 { 1 } else { max_attempts };
|
|
1029
1182
|
let now = now_unix(conn)?;
|
|
1030
1183
|
let next_fire_at = super::cron::next_after_unix(cron_expr, now).map_err(to_sql_err)?;
|
|
1031
1184
|
conn.execute(
|
|
1032
1185
|
"INSERT INTO _honker_scheduler_tasks
|
|
1033
|
-
(name, queue, cron_expr, payload, priority, expires_s, next_fire_at)
|
|
1034
|
-
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)
|
|
1186
|
+
(name, queue, cron_expr, payload, priority, expires_s, next_fire_at, max_attempts)
|
|
1187
|
+
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)
|
|
1035
1188
|
ON CONFLICT(name) DO UPDATE SET
|
|
1036
1189
|
queue = excluded.queue,
|
|
1037
1190
|
cron_expr = excluded.cron_expr,
|
|
1038
1191
|
payload = excluded.payload,
|
|
1039
1192
|
priority = excluded.priority,
|
|
1040
1193
|
expires_s = excluded.expires_s,
|
|
1041
|
-
next_fire_at = excluded.next_fire_at
|
|
1194
|
+
next_fire_at = excluded.next_fire_at,
|
|
1195
|
+
max_attempts = excluded.max_attempts",
|
|
1042
1196
|
rusqlite::params![
|
|
1043
1197
|
name,
|
|
1044
1198
|
queue,
|
|
@@ -1046,7 +1200,8 @@ pub fn scheduler_register(
|
|
|
1046
1200
|
payload,
|
|
1047
1201
|
priority,
|
|
1048
1202
|
expires_s,
|
|
1049
|
-
next_fire_at
|
|
1203
|
+
next_fire_at,
|
|
1204
|
+
max_attempts
|
|
1050
1205
|
],
|
|
1051
1206
|
)?;
|
|
1052
1207
|
// Wake any sleeping scheduler leader so it re-computes
|
|
@@ -1054,6 +1209,9 @@ pub fn scheduler_register(
|
|
|
1054
1209
|
// this, a leader that went to sleep for an hour before a newly-
|
|
1055
1210
|
// registered 1-minute-from-now task existed would oversleep past
|
|
1056
1211
|
// its first fire.
|
|
1212
|
+
//
|
|
1213
|
+
// Wake is the register/update write itself advancing data_version
|
|
1214
|
+
// on commit — see scheduler_wake.
|
|
1057
1215
|
scheduler_wake(conn)?;
|
|
1058
1216
|
Ok(1)
|
|
1059
1217
|
}
|
|
@@ -1073,29 +1231,42 @@ pub fn scheduler_unregister(conn: &Connection, name: &str) -> rusqlite::Result<i
|
|
|
1073
1231
|
Ok(n as i64)
|
|
1074
1232
|
}
|
|
1075
1233
|
|
|
1076
|
-
///
|
|
1077
|
-
///
|
|
1078
|
-
///
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
)?;
|
|
1234
|
+
/// Ensure a sleeping scheduler leader sitting on `update_events()`
|
|
1235
|
+
/// re-evaluates after a register/unregister/pause/resume/update.
|
|
1236
|
+
///
|
|
1237
|
+
/// The register/unregister/pause/resume/update statements already
|
|
1238
|
+
/// mutate `_honker_scheduler_tasks`, which advances data_version on
|
|
1239
|
+
/// commit. A synthetic notification row used to be written here and
|
|
1240
|
+
/// grew without bound under frequent schedule edits — no longer needed.
|
|
1241
|
+
fn scheduler_wake(_conn: &Connection) -> rusqlite::Result<()> {
|
|
1085
1242
|
Ok(())
|
|
1086
1243
|
}
|
|
1087
1244
|
|
|
1245
|
+
/// Max fires enqueued for a single schedule row in one
|
|
1246
|
+
/// `scheduler_tick` call. After a long outage an `@every 1s` task
|
|
1247
|
+
/// would otherwise enqueue tens of thousands of jobs in one writer
|
|
1248
|
+
/// transaction.
|
|
1249
|
+
///
|
|
1250
|
+
/// **Semantics (intentional):** once the cap is hit, remaining missed
|
|
1251
|
+
/// boundaries for that task are **skipped** — `next_fire_at` jumps to
|
|
1252
|
+
/// the next boundary strictly after `now_unix`. Those intermediate
|
|
1253
|
+
/// fires are never enqueued. Run the scheduler continuously, use
|
|
1254
|
+
/// coarser schedules, or raise this constant if every missed fire
|
|
1255
|
+
/// must be delivered.
|
|
1256
|
+
pub const SCHEDULER_MAX_CATCHUP_FIRES: i64 = 64;
|
|
1257
|
+
|
|
1088
1258
|
/// For each registered task whose `next_fire_at <= now_unix`,
|
|
1089
1259
|
/// enqueue the payload into its queue and advance `next_fire_at`
|
|
1090
1260
|
/// to the next boundary. Keeps advancing within one tick while
|
|
1091
1261
|
/// boundaries remain in the past (catches up after a scheduler
|
|
1092
|
-
/// outage)
|
|
1262
|
+
/// outage), up to [`SCHEDULER_MAX_CATCHUP_FIRES`] per task.
|
|
1093
1263
|
/// Returns a JSON array of `{name, queue, fire_at, job_id}` fires.
|
|
1094
1264
|
pub fn scheduler_tick(conn: &Connection, now_unix: i64) -> rusqlite::Result<String> {
|
|
1095
1265
|
#[allow(clippy::type_complexity)]
|
|
1096
|
-
let tasks: Vec<(String, String, String, String, i64, Option<i64>, i64)> = {
|
|
1266
|
+
let tasks: Vec<(String, String, String, String, i64, Option<i64>, i64, i64)> = {
|
|
1097
1267
|
let mut stmt = conn.prepare_cached(
|
|
1098
|
-
"SELECT name, queue, cron_expr, payload, priority, expires_s,
|
|
1268
|
+
"SELECT name, queue, cron_expr, payload, priority, expires_s,
|
|
1269
|
+
next_fire_at, COALESCE(max_attempts, 3)
|
|
1099
1270
|
FROM _honker_scheduler_tasks
|
|
1100
1271
|
WHERE next_fire_at <= ?1 AND enabled = 1",
|
|
1101
1272
|
)?;
|
|
@@ -1108,31 +1279,47 @@ pub fn scheduler_tick(conn: &Connection, now_unix: i64) -> rusqlite::Result<Stri
|
|
|
1108
1279
|
r.get::<_, i64>(4)?,
|
|
1109
1280
|
r.get::<_, Option<i64>>(5)?,
|
|
1110
1281
|
r.get::<_, i64>(6)?,
|
|
1282
|
+
r.get::<_, i64>(7)?,
|
|
1111
1283
|
))
|
|
1112
1284
|
})?
|
|
1113
1285
|
.collect::<Result<Vec<_>, _>>()?
|
|
1114
1286
|
};
|
|
1115
|
-
let mut out =
|
|
1116
|
-
|
|
1117
|
-
|
|
1287
|
+
let mut out = Vec::new();
|
|
1288
|
+
for (name, queue, cron_expr, payload, priority, expires_s, mut next_fire_at, max_attempts) in
|
|
1289
|
+
tasks
|
|
1290
|
+
{
|
|
1291
|
+
let mut fires_this_task: i64 = 0;
|
|
1118
1292
|
while next_fire_at <= now_unix {
|
|
1293
|
+
if fires_this_task >= SCHEDULER_MAX_CATCHUP_FIRES {
|
|
1294
|
+
// Skip the remaining backlog. Resume from the next
|
|
1295
|
+
// boundary strictly after now so we don't immediately
|
|
1296
|
+
// re-enter the catch-up loop on the next tick.
|
|
1297
|
+
// Intermediate boundaries are intentionally never
|
|
1298
|
+
// enqueued (see SCHEDULER_MAX_CATCHUP_FIRES docs).
|
|
1299
|
+
next_fire_at =
|
|
1300
|
+
super::cron::next_after_unix(&cron_expr, now_unix).map_err(to_sql_err)?;
|
|
1301
|
+
break;
|
|
1302
|
+
}
|
|
1119
1303
|
// Enqueue at this boundary. `run_at` is NULL (claimable
|
|
1120
1304
|
// immediately); `expires` is the task's expires_s if set.
|
|
1305
|
+
// max_attempts comes from the schedule row, not a constant.
|
|
1121
1306
|
let job_id = enqueue(
|
|
1122
|
-
conn,
|
|
1307
|
+
conn,
|
|
1308
|
+
&queue,
|
|
1309
|
+
&payload,
|
|
1310
|
+
None,
|
|
1311
|
+
None,
|
|
1312
|
+
priority,
|
|
1313
|
+
max_attempts,
|
|
1123
1314
|
expires_s,
|
|
1124
1315
|
)?;
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
json_str(&queue),
|
|
1133
|
-
next_fire_at,
|
|
1134
|
-
job_id,
|
|
1135
|
-
));
|
|
1316
|
+
out.push(json!({
|
|
1317
|
+
"name": name,
|
|
1318
|
+
"queue": queue,
|
|
1319
|
+
"fire_at": next_fire_at,
|
|
1320
|
+
"job_id": job_id,
|
|
1321
|
+
}));
|
|
1322
|
+
fires_this_task += 1;
|
|
1136
1323
|
// Advance to the next boundary strictly after this one.
|
|
1137
1324
|
next_fire_at =
|
|
1138
1325
|
super::cron::next_after_unix(&cron_expr, next_fire_at).map_err(to_sql_err)?;
|
|
@@ -1144,8 +1331,7 @@ pub fn scheduler_tick(conn: &Connection, now_unix: i64) -> rusqlite::Result<Stri
|
|
|
1144
1331
|
rusqlite::params![name, next_fire_at],
|
|
1145
1332
|
)?;
|
|
1146
1333
|
}
|
|
1147
|
-
out.
|
|
1148
|
-
Ok(out)
|
|
1334
|
+
Ok(Value::Array(out).to_string())
|
|
1149
1335
|
}
|
|
1150
1336
|
|
|
1151
1337
|
pub fn scheduler_soonest(conn: &Connection) -> rusqlite::Result<i64> {
|
|
@@ -1185,15 +1371,25 @@ pub fn scheduler_resume(conn: &Connection, name: &str) -> rusqlite::Result<i64>
|
|
|
1185
1371
|
|
|
1186
1372
|
/// Return all registered schedules as a JSON array. Each row:
|
|
1187
1373
|
/// `{name, queue, cron_expr, payload, priority, expires_s,
|
|
1188
|
-
/// next_fire_at, enabled}`.
|
|
1374
|
+
/// next_fire_at, enabled, max_attempts}`.
|
|
1189
1375
|
pub fn scheduler_list(conn: &Connection) -> rusqlite::Result<String> {
|
|
1190
1376
|
let mut stmt = conn.prepare(
|
|
1191
1377
|
"SELECT name, queue, cron_expr, payload, priority, expires_s,
|
|
1192
|
-
next_fire_at, enabled
|
|
1378
|
+
next_fire_at, enabled, COALESCE(max_attempts, 3)
|
|
1193
1379
|
FROM _honker_scheduler_tasks
|
|
1194
1380
|
ORDER BY name",
|
|
1195
1381
|
)?;
|
|
1196
|
-
let rows: Vec<(
|
|
1382
|
+
let rows: Vec<(
|
|
1383
|
+
String,
|
|
1384
|
+
String,
|
|
1385
|
+
String,
|
|
1386
|
+
String,
|
|
1387
|
+
i64,
|
|
1388
|
+
Option<i64>,
|
|
1389
|
+
i64,
|
|
1390
|
+
i64,
|
|
1391
|
+
i64,
|
|
1392
|
+
)> = stmt
|
|
1197
1393
|
.query_map([], |r| {
|
|
1198
1394
|
Ok((
|
|
1199
1395
|
r.get(0)?,
|
|
@@ -1204,35 +1400,36 @@ pub fn scheduler_list(conn: &Connection) -> rusqlite::Result<String> {
|
|
|
1204
1400
|
r.get(5)?,
|
|
1205
1401
|
r.get(6)?,
|
|
1206
1402
|
r.get(7)?,
|
|
1403
|
+
r.get(8)?,
|
|
1207
1404
|
))
|
|
1208
1405
|
})?
|
|
1209
1406
|
.collect::<Result<Vec<_>, _>>()?;
|
|
1210
|
-
let mut out =
|
|
1211
|
-
for (
|
|
1212
|
-
|
|
1407
|
+
let mut out = Vec::new();
|
|
1408
|
+
for (
|
|
1409
|
+
name,
|
|
1410
|
+
queue,
|
|
1411
|
+
cron_expr,
|
|
1412
|
+
payload,
|
|
1413
|
+
priority,
|
|
1414
|
+
expires_s,
|
|
1415
|
+
next_fire_at,
|
|
1416
|
+
enabled,
|
|
1417
|
+
max_attempts,
|
|
1418
|
+
) in rows
|
|
1213
1419
|
{
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
"
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
json_str(queue),
|
|
1226
|
-
json_str(cron_expr),
|
|
1227
|
-
json_str(payload),
|
|
1228
|
-
priority,
|
|
1229
|
-
expires_repr,
|
|
1230
|
-
next_fire_at,
|
|
1231
|
-
if *enabled != 0 { "true" } else { "false" },
|
|
1232
|
-
));
|
|
1420
|
+
out.push(json!({
|
|
1421
|
+
"name": name,
|
|
1422
|
+
"queue": queue,
|
|
1423
|
+
"cron_expr": cron_expr,
|
|
1424
|
+
"payload": payload,
|
|
1425
|
+
"priority": priority,
|
|
1426
|
+
"expires_s": expires_s,
|
|
1427
|
+
"next_fire_at": next_fire_at,
|
|
1428
|
+
"enabled": enabled != 0,
|
|
1429
|
+
"max_attempts": max_attempts,
|
|
1430
|
+
}));
|
|
1233
1431
|
}
|
|
1234
|
-
out.
|
|
1235
|
-
Ok(out)
|
|
1432
|
+
Ok(Value::Array(out).to_string())
|
|
1236
1433
|
}
|
|
1237
1434
|
|
|
1238
1435
|
/// Mutate one or more fields of a registered schedule. Pass `None` for
|
|
@@ -1247,6 +1444,7 @@ pub fn scheduler_update(
|
|
|
1247
1444
|
payload: Option<&str>,
|
|
1248
1445
|
priority: Option<i64>,
|
|
1249
1446
|
expires_s: Option<Option<i64>>,
|
|
1447
|
+
max_attempts: Option<Option<i64>>,
|
|
1250
1448
|
) -> rusqlite::Result<i64> {
|
|
1251
1449
|
// Verify exists first so we can return 0 cleanly without dynamic SQL gymnastics.
|
|
1252
1450
|
let exists: bool = conn
|
|
@@ -1259,8 +1457,11 @@ pub fn scheduler_update(
|
|
|
1259
1457
|
if !exists {
|
|
1260
1458
|
return Ok(0);
|
|
1261
1459
|
}
|
|
1262
|
-
let any_field =
|
|
1263
|
-
|
|
1460
|
+
let any_field = cron_expr.is_some()
|
|
1461
|
+
|| payload.is_some()
|
|
1462
|
+
|| priority.is_some()
|
|
1463
|
+
|| expires_s.is_some()
|
|
1464
|
+
|| max_attempts.is_some();
|
|
1264
1465
|
if !any_field {
|
|
1265
1466
|
// No fields to change. Don't wake the leader for a no-op.
|
|
1266
1467
|
return Ok(0);
|
|
@@ -1294,6 +1495,14 @@ pub fn scheduler_update(
|
|
|
1294
1495
|
rusqlite::params![name, e],
|
|
1295
1496
|
)?;
|
|
1296
1497
|
}
|
|
1498
|
+
if let Some(m) = max_attempts {
|
|
1499
|
+
let m = m.unwrap_or(3);
|
|
1500
|
+
let m = if m < 1 { 1 } else { m };
|
|
1501
|
+
conn.execute(
|
|
1502
|
+
"UPDATE _honker_scheduler_tasks SET max_attempts = ?2 WHERE name = ?1",
|
|
1503
|
+
rusqlite::params![name, m],
|
|
1504
|
+
)?;
|
|
1505
|
+
}
|
|
1297
1506
|
if let Some(expr) = cron_expr {
|
|
1298
1507
|
conn.execute(
|
|
1299
1508
|
"UPDATE _honker_scheduler_tasks
|
|
@@ -1381,6 +1590,8 @@ pub fn stream_publish(
|
|
|
1381
1590
|
key: Option<&str>,
|
|
1382
1591
|
payload: &str,
|
|
1383
1592
|
) -> rusqlite::Result<i64> {
|
|
1593
|
+
// Stream row INSERT advances data_version on commit — same wake
|
|
1594
|
+
// path as enqueue. No synthetic notification row (see enqueue).
|
|
1384
1595
|
let offset: i64 = conn.query_row(
|
|
1385
1596
|
"INSERT INTO _honker_stream (topic, key, payload)
|
|
1386
1597
|
VALUES (?1, ?2, ?3)
|
|
@@ -1388,12 +1599,6 @@ pub fn stream_publish(
|
|
|
1388
1599
|
rusqlite::params![topic, key, payload],
|
|
1389
1600
|
|r| r.get(0),
|
|
1390
1601
|
)?;
|
|
1391
|
-
let channel = format!("honker:stream:{}", topic);
|
|
1392
|
-
conn.execute(
|
|
1393
|
-
"INSERT INTO _honker_notifications (channel, payload)
|
|
1394
|
-
VALUES (?1, 'new')",
|
|
1395
|
-
rusqlite::params![channel],
|
|
1396
|
-
)?;
|
|
1397
1602
|
Ok(offset)
|
|
1398
1603
|
}
|
|
1399
1604
|
|
|
@@ -1422,29 +1627,18 @@ pub fn stream_read_since(
|
|
|
1422
1627
|
r.get::<_, i64>(4)?,
|
|
1423
1628
|
))
|
|
1424
1629
|
})?;
|
|
1425
|
-
let mut out =
|
|
1426
|
-
let mut first = true;
|
|
1630
|
+
let mut out = Vec::new();
|
|
1427
1631
|
for row in rows {
|
|
1428
1632
|
let (off, top, key, payload, created_at) = row?;
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
};
|
|
1437
|
-
out.push_str(&format!(
|
|
1438
|
-
"{{\"offset\":{},\"topic\":{},\"key\":{},\"payload\":{},\"created_at\":{}}}",
|
|
1439
|
-
off,
|
|
1440
|
-
json_str(&top),
|
|
1441
|
-
key_tok,
|
|
1442
|
-
json_str(&payload),
|
|
1443
|
-
created_at,
|
|
1444
|
-
));
|
|
1633
|
+
out.push(json!({
|
|
1634
|
+
"offset": off,
|
|
1635
|
+
"topic": top,
|
|
1636
|
+
"key": key,
|
|
1637
|
+
"payload": payload,
|
|
1638
|
+
"created_at": created_at,
|
|
1639
|
+
}));
|
|
1445
1640
|
}
|
|
1446
|
-
out.
|
|
1447
|
-
Ok(out)
|
|
1641
|
+
Ok(Value::Array(out).to_string())
|
|
1448
1642
|
}
|
|
1449
1643
|
|
|
1450
1644
|
pub fn stream_save_offset(
|
|
@@ -1483,26 +1677,3 @@ pub fn stream_get_offset(conn: &Connection, consumer: &str, topic: &str) -> rusq
|
|
|
1483
1677
|
fn now_unix(conn: &Connection) -> rusqlite::Result<i64> {
|
|
1484
1678
|
conn.query_row("SELECT unixepoch()", [], |r| r.get(0))
|
|
1485
1679
|
}
|
|
1486
|
-
|
|
1487
|
-
/// Escape a string for inclusion as a JSON string literal. Used by
|
|
1488
|
-
/// `claim_batch` to build its JSON array return value without
|
|
1489
|
-
/// pulling in serde_json just for one site.
|
|
1490
|
-
fn json_str(s: &str) -> String {
|
|
1491
|
-
let mut out = String::with_capacity(s.len() + 2);
|
|
1492
|
-
out.push('"');
|
|
1493
|
-
for c in s.chars() {
|
|
1494
|
-
match c {
|
|
1495
|
-
'"' => out.push_str("\\\""),
|
|
1496
|
-
'\\' => out.push_str("\\\\"),
|
|
1497
|
-
'\n' => out.push_str("\\n"),
|
|
1498
|
-
'\r' => out.push_str("\\r"),
|
|
1499
|
-
'\t' => out.push_str("\\t"),
|
|
1500
|
-
c if (c as u32) < 0x20 => {
|
|
1501
|
-
out.push_str(&format!("\\u{:04x}", c as u32));
|
|
1502
|
-
}
|
|
1503
|
-
c => out.push(c),
|
|
1504
|
-
}
|
|
1505
|
-
}
|
|
1506
|
-
out.push('"');
|
|
1507
|
-
out
|
|
1508
|
-
}
|
|
@@ -347,7 +347,8 @@ pub const BOOTSTRAP_HONKER_SQL: &str = "
|
|
|
347
347
|
priority INTEGER NOT NULL DEFAULT 0,
|
|
348
348
|
expires_s INTEGER,
|
|
349
349
|
next_fire_at INTEGER NOT NULL,
|
|
350
|
-
enabled INTEGER NOT NULL DEFAULT 1
|
|
350
|
+
enabled INTEGER NOT NULL DEFAULT 1,
|
|
351
|
+
max_attempts INTEGER NOT NULL DEFAULT 3
|
|
351
352
|
);
|
|
352
353
|
CREATE TABLE IF NOT EXISTS _honker_results (
|
|
353
354
|
job_id INTEGER PRIMARY KEY,
|
|
@@ -407,6 +408,23 @@ pub fn bootstrap_honker_schema(conn: &Connection) -> Result<(), Error> {
|
|
|
407
408
|
Err(e) => return Err(e.into()),
|
|
408
409
|
}
|
|
409
410
|
}
|
|
411
|
+
// Migration: max_attempts on scheduler tasks (per-fire job budget).
|
|
412
|
+
let has_max_attempts: bool = {
|
|
413
|
+
let mut stmt = conn.prepare(
|
|
414
|
+
"SELECT 1 FROM pragma_table_info('_honker_scheduler_tasks') WHERE name='max_attempts'",
|
|
415
|
+
)?;
|
|
416
|
+
stmt.query_row([], |_| Ok(true)).unwrap_or(false)
|
|
417
|
+
};
|
|
418
|
+
if !has_max_attempts {
|
|
419
|
+
match conn.execute(
|
|
420
|
+
"ALTER TABLE _honker_scheduler_tasks ADD COLUMN max_attempts INTEGER NOT NULL DEFAULT 3",
|
|
421
|
+
[],
|
|
422
|
+
) {
|
|
423
|
+
Ok(_) => {}
|
|
424
|
+
Err(e) if e.to_string().to_lowercase().contains("duplicate column") => {}
|
|
425
|
+
Err(e) => return Err(e.into()),
|
|
426
|
+
}
|
|
427
|
+
}
|
|
410
428
|
Ok(())
|
|
411
429
|
}
|
|
412
430
|
|
|
@@ -567,14 +585,28 @@ impl Readers {
|
|
|
567
585
|
*out += 1;
|
|
568
586
|
drop(out);
|
|
569
587
|
drop(pool);
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
588
|
+
match open_conn(&self.path, false) {
|
|
589
|
+
Ok(conn) => {
|
|
590
|
+
// Re-check: if close() raced us, drop the
|
|
591
|
+
// brand-new connection instead of handing it
|
|
592
|
+
// out. Release the capacity slot so a later
|
|
593
|
+
// open after a failed race doesn't think the
|
|
594
|
+
// pool is full forever.
|
|
595
|
+
if self.closed.load(Ordering::Acquire) {
|
|
596
|
+
*self.outstanding.lock() -= 1;
|
|
597
|
+
drop(conn);
|
|
598
|
+
return Err(closed_err());
|
|
599
|
+
}
|
|
600
|
+
return Ok(conn);
|
|
601
|
+
}
|
|
602
|
+
Err(e) => {
|
|
603
|
+
// Open failed — free the slot we reserved or
|
|
604
|
+
// every transient open failure permanently
|
|
605
|
+
// shrinks max_readers until the pool is dead.
|
|
606
|
+
*self.outstanding.lock() -= 1;
|
|
607
|
+
return Err(e);
|
|
608
|
+
}
|
|
576
609
|
}
|
|
577
|
-
return Ok(conn);
|
|
578
610
|
}
|
|
579
611
|
drop(out);
|
|
580
612
|
self.available.wait(&mut pool);
|
|
@@ -1275,6 +1307,7 @@ mod tests {
|
|
|
1275
1307
|
|r| r.get(0),
|
|
1276
1308
|
)
|
|
1277
1309
|
.unwrap();
|
|
1310
|
+
// Enqueue / stream_publish must NOT write synthetic wake rows.
|
|
1278
1311
|
let enqueue_wakes: i64 = conn
|
|
1279
1312
|
.query_row(
|
|
1280
1313
|
"SELECT COUNT(*) FROM _honker_notifications WHERE channel='honker:pressure'",
|
|
@@ -1282,6 +1315,13 @@ mod tests {
|
|
|
1282
1315
|
|r| r.get(0),
|
|
1283
1316
|
)
|
|
1284
1317
|
.unwrap();
|
|
1318
|
+
let stream_wakes: i64 = conn
|
|
1319
|
+
.query_row(
|
|
1320
|
+
"SELECT COUNT(*) FROM _honker_notifications WHERE channel='honker:stream:pressure-events'",
|
|
1321
|
+
[],
|
|
1322
|
+
|r| r.get(0),
|
|
1323
|
+
)
|
|
1324
|
+
.unwrap();
|
|
1285
1325
|
let integrity: String = conn
|
|
1286
1326
|
.query_row("PRAGMA integrity_check", [], |r| r.get(0))
|
|
1287
1327
|
.unwrap();
|
|
@@ -1289,7 +1329,14 @@ mod tests {
|
|
|
1289
1329
|
assert_eq!(dead, 0);
|
|
1290
1330
|
assert_eq!(stream_rows as usize, total_jobs);
|
|
1291
1331
|
assert_eq!(notes as usize, total_jobs);
|
|
1292
|
-
assert_eq!(
|
|
1332
|
+
assert_eq!(
|
|
1333
|
+
enqueue_wakes, 0,
|
|
1334
|
+
"enqueue must not write wake notifications"
|
|
1335
|
+
);
|
|
1336
|
+
assert_eq!(
|
|
1337
|
+
stream_wakes, 0,
|
|
1338
|
+
"stream_publish must not write wake notifications"
|
|
1339
|
+
);
|
|
1293
1340
|
assert_eq!(integrity, "ok");
|
|
1294
1341
|
|
|
1295
1342
|
drop(conn);
|
|
@@ -2285,6 +2332,7 @@ while True:
|
|
|
2285
2332
|
"expires_s",
|
|
2286
2333
|
"next_fire_at",
|
|
2287
2334
|
"enabled",
|
|
2335
|
+
"max_attempts",
|
|
2288
2336
|
],
|
|
2289
2337
|
);
|
|
2290
2338
|
let res_cols: Vec<String> = conn
|
|
@@ -3162,4 +3210,98 @@ while True:
|
|
|
3162
3210
|
"expected probe to fail for inaccessible dir, got Ok"
|
|
3163
3211
|
);
|
|
3164
3212
|
}
|
|
3213
|
+
|
|
3214
|
+
#[test]
|
|
3215
|
+
fn readers_open_failure_does_not_leak_capacity() {
|
|
3216
|
+
// Path under a missing directory — open_conn fails every time.
|
|
3217
|
+
// Without the outstanding decrement on open failure, two
|
|
3218
|
+
// failures with max=2 permanently fill the counter and the
|
|
3219
|
+
// third acquire blocks forever on the condvar.
|
|
3220
|
+
let r = Arc::new(Readers::new(
|
|
3221
|
+
"/this/parent/does/not/exist/honker-readers-leak.db".into(),
|
|
3222
|
+
2,
|
|
3223
|
+
));
|
|
3224
|
+
for i in 0..5 {
|
|
3225
|
+
let r = r.clone();
|
|
3226
|
+
let handle = std::thread::spawn(move || r.acquire());
|
|
3227
|
+
let result = handle.join().expect("acquire thread panicked");
|
|
3228
|
+
assert!(
|
|
3229
|
+
result.is_err(),
|
|
3230
|
+
"attempt {i}: expected open failure, got Ok"
|
|
3231
|
+
);
|
|
3232
|
+
}
|
|
3233
|
+
}
|
|
3234
|
+
|
|
3235
|
+
#[test]
|
|
3236
|
+
fn claim_batch_dead_letters_reclaim_past_max_attempts() {
|
|
3237
|
+
// Worker dies without retry/ack after the last allowed claim.
|
|
3238
|
+
// The next claim must dead-letter the row, not reclaim it.
|
|
3239
|
+
let path = temp_db("claim-max-attempts");
|
|
3240
|
+
let conn = open_core_test_conn(&path);
|
|
3241
|
+
|
|
3242
|
+
let job_id: i64 = conn
|
|
3243
|
+
.query_row(
|
|
3244
|
+
"SELECT honker_enqueue('q', '{\"n\":1}', NULL, NULL, 0, 2, NULL)",
|
|
3245
|
+
[],
|
|
3246
|
+
|r| r.get(0),
|
|
3247
|
+
)
|
|
3248
|
+
.unwrap();
|
|
3249
|
+
|
|
3250
|
+
// Claim 1 → attempts=1. Expire visibility.
|
|
3251
|
+
let claimed1: String = conn
|
|
3252
|
+
.query_row("SELECT honker_claim_batch('q', 'w1', 1, 30)", [], |r| {
|
|
3253
|
+
r.get(0)
|
|
3254
|
+
})
|
|
3255
|
+
.unwrap();
|
|
3256
|
+
assert!(claimed1.contains(&format!("\"id\":{job_id}")));
|
|
3257
|
+
conn.execute(
|
|
3258
|
+
"UPDATE _honker_live SET claim_expires_at = unixepoch() - 1 WHERE id=?1",
|
|
3259
|
+
rusqlite::params![job_id],
|
|
3260
|
+
)
|
|
3261
|
+
.unwrap();
|
|
3262
|
+
|
|
3263
|
+
// Claim 2 (reclaim) → attempts=2. Expire again.
|
|
3264
|
+
let claimed2: String = conn
|
|
3265
|
+
.query_row("SELECT honker_claim_batch('q', 'w2', 1, 30)", [], |r| {
|
|
3266
|
+
r.get(0)
|
|
3267
|
+
})
|
|
3268
|
+
.unwrap();
|
|
3269
|
+
assert!(claimed2.contains(&format!("\"id\":{job_id}")));
|
|
3270
|
+
conn.execute(
|
|
3271
|
+
"UPDATE _honker_live SET claim_expires_at = unixepoch() - 1 WHERE id=?1",
|
|
3272
|
+
rusqlite::params![job_id],
|
|
3273
|
+
)
|
|
3274
|
+
.unwrap();
|
|
3275
|
+
|
|
3276
|
+
// Claim 3: exhausted → empty claim, row in dead.
|
|
3277
|
+
let claimed3: String = conn
|
|
3278
|
+
.query_row("SELECT honker_claim_batch('q', 'w3', 1, 30)", [], |r| {
|
|
3279
|
+
r.get(0)
|
|
3280
|
+
})
|
|
3281
|
+
.unwrap();
|
|
3282
|
+
assert_eq!(claimed3, "[]");
|
|
3283
|
+
|
|
3284
|
+
let live: i64 = conn
|
|
3285
|
+
.query_row(
|
|
3286
|
+
"SELECT COUNT(*) FROM _honker_live WHERE id=?1",
|
|
3287
|
+
rusqlite::params![job_id],
|
|
3288
|
+
|r| r.get(0),
|
|
3289
|
+
)
|
|
3290
|
+
.unwrap();
|
|
3291
|
+
let (dead_attempts, last_error): (i64, String) = conn
|
|
3292
|
+
.query_row(
|
|
3293
|
+
"SELECT attempts, last_error FROM _honker_dead WHERE id=?1",
|
|
3294
|
+
rusqlite::params![job_id],
|
|
3295
|
+
|r| Ok((r.get(0)?, r.get(1)?)),
|
|
3296
|
+
)
|
|
3297
|
+
.unwrap();
|
|
3298
|
+
assert_eq!(live, 0);
|
|
3299
|
+
assert_eq!(dead_attempts, 2);
|
|
3300
|
+
assert_eq!(last_error, "max attempts exceeded");
|
|
3301
|
+
|
|
3302
|
+
drop(conn);
|
|
3303
|
+
let _ = std::fs::remove_file(&path);
|
|
3304
|
+
let _ = std::fs::remove_file(format!("{}-wal", path.display()));
|
|
3305
|
+
let _ = std::fs::remove_file(format!("{}-shm", path.display()));
|
|
3306
|
+
}
|
|
3165
3307
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "honker-extension"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.4.0"
|
|
4
4
|
edition = "2024"
|
|
5
5
|
description = "SQLite loadable extension for Honker. Adds honker_* SQL functions (queues, streams, scheduler, pub/sub) to any SQLite client."
|
|
6
6
|
license = "MIT OR Apache-2.0"
|
|
@@ -23,7 +23,7 @@ crate-type = ["cdylib"]
|
|
|
23
23
|
# Using both `path` and `version` lets Cargo publish this crate to
|
|
24
24
|
# crates.io referencing the real honker-core = "0.2" while still
|
|
25
25
|
# using the in-tree source for local builds.
|
|
26
|
-
honker-core = { path = "../honker-core", version = "0.
|
|
26
|
+
honker-core = { path = "../honker-core", version = "0.4.0", default-features = false }
|
|
27
27
|
# "loadable_extension" feature makes rusqlite usable from inside a
|
|
28
28
|
# sqlite3_extension_init entry point.
|
|
29
29
|
rusqlite = { version = "0.39.0", features = ["functions", "hooks", "loadable_extension"] }
|
data/lib/honker/lock.rb
CHANGED
|
@@ -56,11 +56,11 @@ module Honker
|
|
|
56
56
|
|
|
57
57
|
# Extend the TTL. Returns true if we still hold the lock; false if
|
|
58
58
|
# it was stolen (the TTL elapsed and another owner acquired it).
|
|
59
|
-
#
|
|
60
|
-
# existing
|
|
59
|
+
# Uses honker_lock_renew — honker_lock_acquire does not refresh
|
|
60
|
+
# expires_at for an existing (name, owner) row.
|
|
61
61
|
def heartbeat(ttl_s:)
|
|
62
62
|
@db.db.get_first_row(
|
|
63
|
-
"SELECT
|
|
63
|
+
"SELECT honker_lock_renew(?, ?, ?)",
|
|
64
64
|
[@name, @owner, ttl_s],
|
|
65
65
|
)[0] == 1
|
|
66
66
|
end
|
data/lib/honker/scheduler.rb
CHANGED
|
@@ -43,13 +43,13 @@ module Honker
|
|
|
43
43
|
#
|
|
44
44
|
# Idempotent by `name`; registering the same name twice replaces
|
|
45
45
|
# the previous row.
|
|
46
|
-
def add(name:, queue:, cron: nil, schedule: nil, payload:, priority: 0, expires_s: nil)
|
|
46
|
+
def add(name:, queue:, cron: nil, schedule: nil, payload:, priority: 0, expires_s: nil, max_attempts: 3)
|
|
47
47
|
expr = schedule || cron
|
|
48
48
|
raise ArgumentError, "must provide cron: or schedule:" if expr.nil? || expr.empty?
|
|
49
49
|
|
|
50
50
|
@db.db.get_first_row(
|
|
51
|
-
"SELECT honker_scheduler_register(?, ?, ?, ?, ?, ?)",
|
|
52
|
-
[name, queue, expr, JSON.dump(payload), priority, expires_s],
|
|
51
|
+
"SELECT honker_scheduler_register(?, ?, ?, ?, ?, ?, ?)",
|
|
52
|
+
[name, queue, expr, JSON.dump(payload), priority, expires_s, max_attempts],
|
|
53
53
|
)
|
|
54
54
|
@db.mark_updated
|
|
55
55
|
nil
|
|
@@ -105,7 +105,7 @@ module Honker
|
|
|
105
105
|
|
|
106
106
|
# Return every registered schedule with current state. Each entry
|
|
107
107
|
# is a Hash with: name, queue, cron_expr, payload (JSON string),
|
|
108
|
-
# priority, expires_s, next_fire_at, enabled.
|
|
108
|
+
# priority, expires_s, next_fire_at, enabled, max_attempts.
|
|
109
109
|
def list
|
|
110
110
|
raw = @db.db.get_first_row("SELECT honker_scheduler_list()")[0]
|
|
111
111
|
return [] if raw.nil? || raw.empty?
|
|
@@ -115,9 +115,10 @@ module Honker
|
|
|
115
115
|
|
|
116
116
|
# Mutate fields in place. Pass only the kwargs you want changed
|
|
117
117
|
# (omitting a kwarg leaves the field alone). `payload: nil`
|
|
118
|
-
# writes JSON null
|
|
118
|
+
# writes JSON null; `max_attempts: nil` resets to default 3.
|
|
119
|
+
# Cron change recomputes next_fire_at from now.
|
|
119
120
|
# Returns true iff a row was updated.
|
|
120
|
-
def update(name, schedule: UNSET, cron: UNSET, payload: UNSET, priority: UNSET, expires_s: UNSET)
|
|
121
|
+
def update(name, schedule: UNSET, cron: UNSET, payload: UNSET, priority: UNSET, expires_s: UNSET, max_attempts: UNSET)
|
|
121
122
|
expr = nil
|
|
122
123
|
expr = schedule if schedule != UNSET
|
|
123
124
|
expr = cron if expr.nil? && cron != UNSET
|
|
@@ -126,13 +127,16 @@ module Honker
|
|
|
126
127
|
priority_arg = (priority == UNSET) ? nil : priority
|
|
127
128
|
touch_expires = (expires_s == UNSET) ? 0 : 1
|
|
128
129
|
expires_arg = (expires_s == UNSET) ? nil : expires_s
|
|
130
|
+
touch_max_attempts = (max_attempts == UNSET) ? 0 : 1
|
|
131
|
+
max_attempts_arg = (max_attempts == UNSET) ? nil : max_attempts
|
|
129
132
|
|
|
130
|
-
any_field = !expr.nil? || payload != UNSET || priority != UNSET || expires_s != UNSET
|
|
133
|
+
any_field = !expr.nil? || payload != UNSET || priority != UNSET || expires_s != UNSET || max_attempts != UNSET
|
|
131
134
|
return false unless any_field
|
|
132
135
|
|
|
133
136
|
n = @db.db.get_first_row(
|
|
134
|
-
"SELECT honker_scheduler_update(?, ?, ?, ?, ?, ?)",
|
|
135
|
-
[name, expr, payload_arg, priority_arg, expires_arg, touch_expires
|
|
137
|
+
"SELECT honker_scheduler_update(?, ?, ?, ?, ?, ?, ?, ?)",
|
|
138
|
+
[name, expr, payload_arg, priority_arg, expires_arg, touch_expires,
|
|
139
|
+
max_attempts_arg, touch_max_attempts],
|
|
136
140
|
)[0]
|
|
137
141
|
@db.mark_updated if n.positive?
|
|
138
142
|
n.positive?
|
|
@@ -187,18 +191,15 @@ module Honker
|
|
|
187
191
|
def leader_loop(owner, stop_fn)
|
|
188
192
|
last_heartbeat = monotonic_now
|
|
189
193
|
until stop_fn.call
|
|
194
|
+
still_ours = lock_renew(LEADER_LOCK, owner, LOCK_TTL_S)
|
|
195
|
+
# IMPORTANT: if refresh failed, a new leader has the lock.
|
|
196
|
+
# Break out before ticking so we don't double-fire.
|
|
197
|
+
return unless still_ours
|
|
198
|
+
|
|
199
|
+
last_heartbeat = monotonic_now
|
|
190
200
|
# tick errors escape up to `run`, which releases the lock in
|
|
191
201
|
# its `ensure` before re-raising.
|
|
192
202
|
tick
|
|
193
|
-
if monotonic_now - last_heartbeat >= HEARTBEAT_S
|
|
194
|
-
still_ours = lock_try_acquire(LEADER_LOCK, owner, LOCK_TTL_S)
|
|
195
|
-
# IMPORTANT: if refresh failed, a new leader has the lock.
|
|
196
|
-
# Break out of the leader loop so we don't double-fire. This
|
|
197
|
-
# is the bug the Rust binding fixed; don't reintroduce it.
|
|
198
|
-
return unless still_ours
|
|
199
|
-
|
|
200
|
-
last_heartbeat = monotonic_now
|
|
201
|
-
end
|
|
202
203
|
|
|
203
204
|
wait_s = HEARTBEAT_S - (monotonic_now - last_heartbeat)
|
|
204
205
|
wait_s = 0 if wait_s.negative?
|
|
@@ -239,6 +240,13 @@ module Honker
|
|
|
239
240
|
)[0] == 1
|
|
240
241
|
end
|
|
241
242
|
|
|
243
|
+
def lock_renew(name, owner, ttl_s)
|
|
244
|
+
@db.db.get_first_row(
|
|
245
|
+
"SELECT honker_lock_renew(?, ?, ?)",
|
|
246
|
+
[name, owner, ttl_s],
|
|
247
|
+
)[0] == 1
|
|
248
|
+
end
|
|
249
|
+
|
|
242
250
|
def lock_release(name, owner)
|
|
243
251
|
@db.db.get_first_row(
|
|
244
252
|
"SELECT honker_lock_release(?, ?)",
|
data/lib/honker/version.rb
CHANGED