questdb 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.cargo/audit.toml +17 -9
- data/CHANGELOG.md +7 -0
- data/README.md +1 -1
- data/THIRD_PARTY_NOTICES.md +2 -2
- data/docs/configuration.md +3 -2
- data/docs/design.md +1 -1
- data/docs/python-parity.md +1 -1
- data/docs/releasing.md +1 -1
- data/ext/questdb_ingress/extconf.rb +2 -0
- data/ext/questdb_ingress/questdb_ingress.c +2 -1
- data/ext/questdb_ingress/unlink_build_ruby.rb +58 -0
- data/lib/questdb/arrow.rb +2 -0
- data/lib/questdb/client.rb +6 -0
- data/lib/questdb/config.rb +19 -0
- data/lib/questdb/pooled_sender.rb +4 -1
- data/lib/questdb/sender.rb +10 -1
- data/lib/questdb/version.rb +2 -2
- data/script/check-native-linkage +115 -0
- data/test/client_test.rb +30 -0
- data/test/extconf_link_flags_test.rb +160 -0
- data/vendor/c-questdb-client/.bumpversion.toml +1 -1
- data/vendor/c-questdb-client/CMakeLists.txt +1 -1
- data/vendor/c-questdb-client/cpp_test/qwp_mock_server.hpp +10 -1
- data/vendor/c-questdb-client/cpp_test/test_arrow_c.c +39 -0
- data/vendor/c-questdb-client/cpp_test/test_reader_mock.cpp +428 -0
- data/vendor/c-questdb-client/doc/COMPATIBILITY.md +1 -1
- data/vendor/c-questdb-client/doc/CONSIDERATIONS.md +6 -3
- data/vendor/c-questdb-client/doc/RELEASE_RUNBOOK.md +7 -7
- data/vendor/c-questdb-client/doc/RELEASING.md +6 -6
- data/vendor/c-questdb-client/include/questdb/egress/qwp_reader.h +48 -1
- data/vendor/c-questdb-client/include/questdb/egress/qwp_reader.hpp +52 -0
- data/vendor/c-questdb-client/include/questdb/ingress/line_sender.h +46 -4
- data/vendor/c-questdb-client/include/questdb/ingress/line_sender.hpp +7 -4
- data/vendor/c-questdb-client/include/questdb/ingress/line_sender_core.hpp +1 -0
- data/vendor/c-questdb-client/questdb-rs/Cargo.toml +1 -1
- data/vendor/c-questdb-client/questdb-rs/README.md +13 -13
- data/vendor/c-questdb-client/questdb-rs/src/egress/config.rs +78 -0
- data/vendor/c-questdb-client/questdb-rs/src/egress/mod.rs +2 -2
- data/vendor/c-questdb-client/questdb-rs/src/egress/query_request.rs +174 -1
- data/vendor/c-questdb-client/questdb-rs/src/egress/reader.rs +264 -14
- data/vendor/c-questdb-client/questdb-rs/src/egress/wire/capabilities.rs +69 -0
- data/vendor/c-questdb-client/questdb-rs/src/egress/wire/msg_kind.rs +33 -0
- data/vendor/c-questdb-client/questdb-rs/src/error.rs +33 -5
- data/vendor/c-questdb-client/questdb-rs/src/ingress/buffer/qwp.rs +96 -11
- data/vendor/c-questdb-client/questdb-rs/src/ingress/buffer.rs +6 -3
- data/vendor/c-questdb-client/questdb-rs/src/ingress/column_sender/sender.rs +1 -16
- data/vendor/c-questdb-client/questdb-rs/src/ingress/sender/qwp_ws.rs +19 -13
- data/vendor/c-questdb-client/questdb-rs/src/ingress/sender/qwp_ws_driver.rs +2 -9
- data/vendor/c-questdb-client/questdb-rs/src/ingress/sender/qwp_ws_publisher.rs +3 -3
- data/vendor/c-questdb-client/questdb-rs/src/ingress/sender.rs +101 -17
- data/vendor/c-questdb-client/questdb-rs/src/ingress/tests.rs +32 -0
- data/vendor/c-questdb-client/questdb-rs/src/ingress.rs +3 -2
- data/vendor/c-questdb-client/questdb-rs/src/tests/qwp_ws.rs +96 -16
- data/vendor/c-questdb-client/questdb-rs/tests/egress_failover.rs +219 -1
- data/vendor/c-questdb-client/questdb-rs/tests/egress_live_server.rs +34 -0
- data/vendor/c-questdb-client/questdb-rs-ffi/Cargo.lock +2 -2
- data/vendor/c-questdb-client/questdb-rs-ffi/Cargo.toml +1 -1
- data/vendor/c-questdb-client/questdb-rs-ffi/src/egress.rs +31 -0
- data/vendor/c-questdb-client/questdb-rs-ffi/src/lib.rs +70 -3
- data/vendor/c-questdb-client/system_test/arrow_ffi.py +1 -0
- data/vendor/c-questdb-client/system_test/enterprise_e2e/tests/test_failover.py +12 -6
- data/vendor/c-questdb-client/system_test/enterprise_e2e/tests/test_orphan_drainer_bindings.py +12 -4
- data/vendor/c-questdb-client/system_test/failover_clients/Cargo.lock +1 -1
- data/vendor/c-questdb-client/system_test/soak/workload_rs/Cargo.lock +1 -1
- data/vendor/c-questdb-client.version +1 -1
- metadata +5 -2
|
@@ -44,7 +44,7 @@ use crate::ingress::SenderBuilder;
|
|
|
44
44
|
use crate::ingress::{Buffer, Protocol, ProtocolVersion};
|
|
45
45
|
use std::fmt::{Debug, Formatter};
|
|
46
46
|
#[cfg(feature = "sync-sender-qwp-ws")]
|
|
47
|
-
use std::sync::atomic::
|
|
47
|
+
use std::sync::atomic::Ordering;
|
|
48
48
|
#[cfg(feature = "sync-sender-qwp-ws")]
|
|
49
49
|
use std::time::{Duration, Instant};
|
|
50
50
|
|
|
@@ -134,13 +134,44 @@ mod http;
|
|
|
134
134
|
#[cfg(feature = "sync-sender-http")]
|
|
135
135
|
pub(crate) use http::*;
|
|
136
136
|
|
|
137
|
+
/// The binding per-frame cap of a store-and-forward QWP/WebSocket publication:
|
|
138
|
+
/// the configured `max_buf_size`, clamped to the server's advertised
|
|
139
|
+
/// `X-QWP-Max-Batch-Size` (0 = not advertised) and to the queue's segment
|
|
140
|
+
/// payload capacity, which rejects any larger frame. Also reports whether the
|
|
141
|
+
/// server advertised a cap, whether or not it is the binding one.
|
|
137
142
|
#[cfg(feature = "sync-sender-qwp-ws")]
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
143
|
+
pub(crate) fn effective_hard_frame_cap(
|
|
144
|
+
max_buf_size: usize,
|
|
145
|
+
server_max_batch_size: usize,
|
|
146
|
+
sfa_frame_payload_cap: usize,
|
|
147
|
+
) -> (usize, bool) {
|
|
148
|
+
let configured_cap = if server_max_batch_size == 0 {
|
|
149
|
+
max_buf_size
|
|
150
|
+
} else {
|
|
151
|
+
max_buf_size.min(server_max_batch_size)
|
|
152
|
+
};
|
|
153
|
+
(
|
|
154
|
+
configured_cap.min(sfa_frame_payload_cap),
|
|
155
|
+
server_max_batch_size != 0,
|
|
156
|
+
)
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/// Name the limit that [`effective_hard_frame_cap`] returns, so a size error
|
|
160
|
+
/// points at the setting that actually binds.
|
|
161
|
+
#[cfg(feature = "sync-sender-qwp-ws")]
|
|
162
|
+
fn effective_hard_frame_cap_source(
|
|
163
|
+
max_buf_size: usize,
|
|
164
|
+
server_max_batch_size: usize,
|
|
165
|
+
sfa_frame_payload_cap: usize,
|
|
166
|
+
) -> &'static str {
|
|
167
|
+
let (cap, _) =
|
|
168
|
+
effective_hard_frame_cap(max_buf_size, server_max_batch_size, sfa_frame_payload_cap);
|
|
169
|
+
if server_max_batch_size != 0 && cap == server_max_batch_size {
|
|
170
|
+
"the server's advertised X-QWP-Max-Batch-Size"
|
|
171
|
+
} else if cap == max_buf_size {
|
|
172
|
+
"max_buf_size"
|
|
142
173
|
} else {
|
|
143
|
-
|
|
174
|
+
"the sf_max_segment_bytes segment payload capacity"
|
|
144
175
|
}
|
|
145
176
|
}
|
|
146
177
|
|
|
@@ -367,17 +398,14 @@ impl Sender {
|
|
|
367
398
|
));
|
|
368
399
|
}
|
|
369
400
|
|
|
401
|
+
let (server_max, sfa_frame_payload_cap) = self
|
|
402
|
+
.qwp_ws_frame_cap_inputs()
|
|
403
|
+
.expect("QWP/WebSocket handler was checked above");
|
|
404
|
+
let (max, _) =
|
|
405
|
+
effective_hard_frame_cap(self.max_buf_size, server_max, sfa_frame_payload_cap);
|
|
370
406
|
let result = match &mut self.handler {
|
|
371
|
-
SyncProtocolHandler::SyncQwpWs(state) =>
|
|
372
|
-
|
|
373
|
-
effective_qwp_ws_max_buf_size(self.max_buf_size, &state.server_max_batch_size);
|
|
374
|
-
flush_qwp_ws(state, qwp, max)
|
|
375
|
-
}
|
|
376
|
-
SyncProtocolHandler::ManualQwpWs(state) => {
|
|
377
|
-
let max =
|
|
378
|
-
effective_qwp_ws_max_buf_size(self.max_buf_size, &state.server_max_batch_size);
|
|
379
|
-
flush_qwp_ws_manual(state, qwp, max)
|
|
380
|
-
}
|
|
407
|
+
SyncProtocolHandler::SyncQwpWs(state) => flush_qwp_ws(state, qwp, max),
|
|
408
|
+
SyncProtocolHandler::ManualQwpWs(state) => flush_qwp_ws_manual(state, qwp, max),
|
|
381
409
|
_ => unreachable!("QWP/WebSocket handler was checked above"),
|
|
382
410
|
};
|
|
383
411
|
if result
|
|
@@ -386,7 +414,38 @@ impl Sender {
|
|
|
386
414
|
{
|
|
387
415
|
self.connected = false;
|
|
388
416
|
}
|
|
389
|
-
result
|
|
417
|
+
result.map_err(|err| {
|
|
418
|
+
if err.code() == crate::ErrorCode::BatchTooLarge {
|
|
419
|
+
let source = effective_hard_frame_cap_source(
|
|
420
|
+
self.max_buf_size,
|
|
421
|
+
server_max,
|
|
422
|
+
sfa_frame_payload_cap,
|
|
423
|
+
);
|
|
424
|
+
crate::Error::new(
|
|
425
|
+
crate::ErrorCode::BatchTooLarge,
|
|
426
|
+
format!("{} The cap is set by {}.", err.msg(), source),
|
|
427
|
+
)
|
|
428
|
+
} else {
|
|
429
|
+
err
|
|
430
|
+
}
|
|
431
|
+
})
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
/// The server's advertised cap (0 = not advertised) and the queue's segment
|
|
435
|
+
/// payload capacity, or `None` for a sender that is not QWP/WebSocket.
|
|
436
|
+
#[cfg(feature = "sync-sender-qwp-ws")]
|
|
437
|
+
fn qwp_ws_frame_cap_inputs(&self) -> Option<(usize, usize)> {
|
|
438
|
+
match &self.handler {
|
|
439
|
+
SyncProtocolHandler::SyncQwpWs(state) => Some((
|
|
440
|
+
state.server_max_batch_size.load(Ordering::Acquire),
|
|
441
|
+
state.sfa_frame_payload_cap,
|
|
442
|
+
)),
|
|
443
|
+
SyncProtocolHandler::ManualQwpWs(state) => Some((
|
|
444
|
+
state.server_max_batch_size.load(Ordering::Acquire),
|
|
445
|
+
state.sfa_frame_payload_cap,
|
|
446
|
+
)),
|
|
447
|
+
_ => None,
|
|
448
|
+
}
|
|
390
449
|
}
|
|
391
450
|
|
|
392
451
|
#[allow(unused_variables)]
|
|
@@ -633,6 +692,31 @@ impl Sender {
|
|
|
633
692
|
self.flush_qwp_ws_buffer(buf, false)
|
|
634
693
|
}
|
|
635
694
|
|
|
695
|
+
/// Return the binding per-frame cap of this QWP/WebSocket sender and whether
|
|
696
|
+
/// the current connection advertised `X-QWP-Max-Batch-Size`, whether or not
|
|
697
|
+
/// that value is the binding limit.
|
|
698
|
+
///
|
|
699
|
+
/// The cap is the smallest of `max_buf_size`, the server's advertised cap
|
|
700
|
+
/// and the store-and-forward segment payload capacity. A flush whose
|
|
701
|
+
/// encoded frame exceeds it fails with
|
|
702
|
+
/// [`ErrorCode::BatchTooLarge`](crate::ErrorCode::BatchTooLarge) and
|
|
703
|
+
/// publishes nothing. The value can change after a reconnect.
|
|
704
|
+
#[cfg(feature = "sync-sender-qwp-ws")]
|
|
705
|
+
pub fn effective_frame_cap(&self) -> Result<(usize, bool)> {
|
|
706
|
+
let (server_max, sfa_frame_payload_cap) =
|
|
707
|
+
self.qwp_ws_frame_cap_inputs().ok_or_else(|| {
|
|
708
|
+
error::fmt!(
|
|
709
|
+
InvalidApiCall,
|
|
710
|
+
"effective_frame_cap is only supported for QWP/WebSocket senders."
|
|
711
|
+
)
|
|
712
|
+
})?;
|
|
713
|
+
Ok(effective_hard_frame_cap(
|
|
714
|
+
self.max_buf_size,
|
|
715
|
+
server_max,
|
|
716
|
+
sfa_frame_payload_cap,
|
|
717
|
+
))
|
|
718
|
+
}
|
|
719
|
+
|
|
636
720
|
/// Return the highest frame sequence number published locally by this
|
|
637
721
|
/// QWP/WebSocket sender, or `None` if no frame has been published.
|
|
638
722
|
///
|
|
@@ -363,6 +363,8 @@ const EGRESS_ONLY_CONFIG_KEYS: &[&str] = &[
|
|
|
363
363
|
"failover_backoff_initial_ms",
|
|
364
364
|
"failover_backoff_max_ms",
|
|
365
365
|
"failover_max_duration_ms",
|
|
366
|
+
// Egress-only per-query timeout default
|
|
367
|
+
"query_timeout_ms",
|
|
366
368
|
// Java-egress-only decoded-batch pool size (Rust egress is sync/pull,
|
|
367
369
|
// see comment in `egress/config.rs`); still ignored on ingress
|
|
368
370
|
// because that's an egress-side concern either way.
|
|
@@ -429,6 +431,36 @@ fn ingress_accepts_full_egress_connect_string_unchanged() {
|
|
|
429
431
|
assert_specified_eq(&builder.password, Some("p".to_string()));
|
|
430
432
|
}
|
|
431
433
|
|
|
434
|
+
#[cfg(all(feature = "sync-sender-http", feature = "sync-sender-qwp-ws"))]
|
|
435
|
+
#[test]
|
|
436
|
+
fn qwpws_config_knows_every_egress_only_key() {
|
|
437
|
+
// `ingress_silently_accepts_every_egress_only_key` only covers
|
|
438
|
+
// `http::`, where the parser's catch-all ignores anything it does not
|
|
439
|
+
// recognise. On `ws::` the parser REJECTS unknown keys, so an
|
|
440
|
+
// egress-only key missing from `QWP_WS_PORTABLE_CONFIG_KEYS` breaks
|
|
441
|
+
// every connect string shared between a sender and a reader — which is
|
|
442
|
+
// the normal shape for `questdb.connect()`, where one string configures
|
|
443
|
+
// both pools.
|
|
444
|
+
//
|
|
445
|
+
// The assertion is on the failure MODE, not on success: a key may still
|
|
446
|
+
// be rejected for its value (`compression=1` is not a valid codec), and
|
|
447
|
+
// that is fine. What must never happen is "Unknown config key".
|
|
448
|
+
for key in EGRESS_ONLY_CONFIG_KEYS {
|
|
449
|
+
for val in ["1", "primary", "halt", ""] {
|
|
450
|
+
let conf = format!("ws::addr=localhost:9000;{key}={val};");
|
|
451
|
+
if let Err(e) = SenderBuilder::from_conf(&conf) {
|
|
452
|
+
assert!(
|
|
453
|
+
!e.msg().contains("Unknown config key"),
|
|
454
|
+
"egress-only key {key:?} is missing from \
|
|
455
|
+
QWP_WS_PORTABLE_CONFIG_KEYS, so a shared ws:: connect \
|
|
456
|
+
string carrying it fails to parse: {}",
|
|
457
|
+
e.msg()
|
|
458
|
+
);
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
|
|
432
464
|
#[cfg(feature = "sync-sender-qwp-ws")]
|
|
433
465
|
#[test]
|
|
434
466
|
fn qwpws_config_silently_accepts_reserved_on_error_policy_keys() {
|
|
@@ -602,8 +602,8 @@ impl QwpWsConnector {
|
|
|
602
602
|
)?;
|
|
603
603
|
// The per-frame cap is the negotiated one: the configured
|
|
604
604
|
// max_buf_size clamped to the server's advertised
|
|
605
|
-
// X-QWP-Max-Batch-Size (0 = not advertised)
|
|
606
|
-
//
|
|
605
|
+
// X-QWP-Max-Batch-Size (0 = not advertised). A direct connection
|
|
606
|
+
// has no store-and-forward queue, so no segment cap applies.
|
|
607
607
|
let max_buf_size = if connected.server_max_batch_size > 0 {
|
|
608
608
|
self.max_buf_size.min(connected.server_max_batch_size)
|
|
609
609
|
} else {
|
|
@@ -1128,6 +1128,7 @@ impl SenderBuilder {
|
|
|
1128
1128
|
"pool_reap",
|
|
1129
1129
|
"query_pool_max",
|
|
1130
1130
|
"query_pool_min",
|
|
1131
|
+
"query_timeout_ms",
|
|
1131
1132
|
"sender_pool_max",
|
|
1132
1133
|
"sender_pool_min",
|
|
1133
1134
|
"target",
|
|
@@ -2078,14 +2078,14 @@ fn qwp_ws_max_buf_size_rejects_oversized_replay_frame_in_all_progress_modes() {
|
|
|
2078
2078
|
let err = sender.flush(&mut buf).unwrap_err();
|
|
2079
2079
|
assert_eq!(
|
|
2080
2080
|
err.code(),
|
|
2081
|
-
crate::ErrorCode::
|
|
2081
|
+
crate::ErrorCode::BatchTooLarge,
|
|
2082
2082
|
"mode={}",
|
|
2083
2083
|
progress.name()
|
|
2084
2084
|
);
|
|
2085
2085
|
assert_eq!(
|
|
2086
2086
|
err.msg(),
|
|
2087
2087
|
format!(
|
|
2088
|
-
"Could not flush buffer: QWP/WebSocket encoded message size of {encoded_len} exceeds
|
|
2088
|
+
"Could not flush buffer: QWP/WebSocket encoded message size of {encoded_len} bytes exceeds the per-frame cap of {max} bytes. The cap is set by max_buf_size."
|
|
2089
2089
|
),
|
|
2090
2090
|
"mode={}",
|
|
2091
2091
|
progress.name()
|
|
@@ -5144,10 +5144,9 @@ fn qwp_ws_rewind_after_a_failed_publication_republishes_only_surviving_rows() {
|
|
|
5144
5144
|
let bookmark = buf.bookmark().unwrap();
|
|
5145
5145
|
write_doomed_rows(&mut buf);
|
|
5146
5146
|
let err = sender.flush_and_keep(&buf).unwrap_err();
|
|
5147
|
-
assert_eq!(err.code(), ErrorCode::
|
|
5147
|
+
assert_eq!(err.code(), ErrorCode::BatchTooLarge, "{err}");
|
|
5148
5148
|
assert!(
|
|
5149
|
-
err.msg()
|
|
5150
|
-
.contains("exceeds maximum configured allowed size"),
|
|
5149
|
+
err.msg().contains("exceeds the per-frame cap"),
|
|
5151
5150
|
"the doomed flush must fail on the size cap, not another guard: {err}"
|
|
5152
5151
|
);
|
|
5153
5152
|
|
|
@@ -7775,13 +7774,15 @@ fn server_max_batch_size_clamps_flush_below_configured_max_in_all_progress_modes
|
|
|
7775
7774
|
let err = sender.flush(&mut buf).unwrap_err();
|
|
7776
7775
|
assert_eq!(
|
|
7777
7776
|
err.code(),
|
|
7778
|
-
crate::ErrorCode::
|
|
7777
|
+
crate::ErrorCode::BatchTooLarge,
|
|
7779
7778
|
"mode={}",
|
|
7780
7779
|
progress.name()
|
|
7781
7780
|
);
|
|
7782
7781
|
assert!(
|
|
7783
|
-
err.msg()
|
|
7784
|
-
|
|
7782
|
+
err.msg().contains(
|
|
7783
|
+
"exceeds the per-frame cap of 512 bytes. \
|
|
7784
|
+
The cap is set by the server's advertised X-QWP-Max-Batch-Size."
|
|
7785
|
+
),
|
|
7785
7786
|
"mode={}: got: {}",
|
|
7786
7787
|
progress.name(),
|
|
7787
7788
|
err.msg()
|
|
@@ -7795,6 +7796,76 @@ fn server_max_batch_size_clamps_flush_below_configured_max_in_all_progress_modes
|
|
|
7795
7796
|
}
|
|
7796
7797
|
}
|
|
7797
7798
|
|
|
7799
|
+
#[test]
|
|
7800
|
+
fn segment_payload_capacity_clamps_flush_below_server_cap_in_all_progress_modes() {
|
|
7801
|
+
for progress in [ProgressCase::Background, ProgressCase::Manual] {
|
|
7802
|
+
let server_cap: usize = 100 * 1024;
|
|
7803
|
+
let port = spawn_max_batch_size_upgrade_only_server(Some(server_cap));
|
|
7804
|
+
let conf = format!(
|
|
7805
|
+
"ws::addr=127.0.0.1:{port};qwp_ws_progress={};sf_max_segment_bytes=4096;",
|
|
7806
|
+
progress.name()
|
|
7807
|
+
);
|
|
7808
|
+
let mut sender = SenderBuilder::from_conf(conf).unwrap().build().unwrap();
|
|
7809
|
+
|
|
7810
|
+
let (cap, server_cap_known) = sender.effective_frame_cap().unwrap();
|
|
7811
|
+
assert!(server_cap_known, "mode={}", progress.name());
|
|
7812
|
+
assert!(
|
|
7813
|
+
cap < 4096,
|
|
7814
|
+
"mode={}: the segment payload capacity must bind, got {cap}",
|
|
7815
|
+
progress.name()
|
|
7816
|
+
);
|
|
7817
|
+
|
|
7818
|
+
let mut buf = build_buffer_with_payload_bytes(cap + 200);
|
|
7819
|
+
let err = sender.flush(&mut buf).unwrap_err();
|
|
7820
|
+
assert_eq!(
|
|
7821
|
+
err.code(),
|
|
7822
|
+
crate::ErrorCode::BatchTooLarge,
|
|
7823
|
+
"mode={}",
|
|
7824
|
+
progress.name()
|
|
7825
|
+
);
|
|
7826
|
+
assert!(
|
|
7827
|
+
err.msg().contains(&format!(
|
|
7828
|
+
"exceeds the per-frame cap of {cap} bytes. \
|
|
7829
|
+
The cap is set by the sf_max_segment_bytes segment payload capacity."
|
|
7830
|
+
)),
|
|
7831
|
+
"mode={}: got: {}",
|
|
7832
|
+
progress.name(),
|
|
7833
|
+
err.msg()
|
|
7834
|
+
);
|
|
7835
|
+
assert!(!buf.is_empty(), "mode={}", progress.name());
|
|
7836
|
+
}
|
|
7837
|
+
}
|
|
7838
|
+
|
|
7839
|
+
#[test]
|
|
7840
|
+
fn effective_frame_cap_reports_the_binding_limit_in_all_progress_modes() {
|
|
7841
|
+
for progress in [ProgressCase::Background, ProgressCase::Manual] {
|
|
7842
|
+
let server_cap: usize = 512;
|
|
7843
|
+
let port = spawn_max_batch_size_upgrade_only_server(Some(server_cap));
|
|
7844
|
+
let builder = SenderBuilder::new(Protocol::Ws, "127.0.0.1", port)
|
|
7845
|
+
.max_buf_size(100 * 1024 * 1024)
|
|
7846
|
+
.unwrap();
|
|
7847
|
+
let sender = build_qwp_ws_sender_from_builder(progress, builder);
|
|
7848
|
+
assert_eq!(
|
|
7849
|
+
sender.effective_frame_cap().unwrap(),
|
|
7850
|
+
(server_cap, true),
|
|
7851
|
+
"mode={}",
|
|
7852
|
+
progress.name()
|
|
7853
|
+
);
|
|
7854
|
+
|
|
7855
|
+
let port = spawn_max_batch_size_upgrade_only_server(None);
|
|
7856
|
+
let builder = SenderBuilder::new(Protocol::Ws, "127.0.0.1", port)
|
|
7857
|
+
.max_buf_size(2048)
|
|
7858
|
+
.unwrap();
|
|
7859
|
+
let sender = build_qwp_ws_sender_from_builder(progress, builder);
|
|
7860
|
+
assert_eq!(
|
|
7861
|
+
sender.effective_frame_cap().unwrap(),
|
|
7862
|
+
(2048, false),
|
|
7863
|
+
"mode={}",
|
|
7864
|
+
progress.name()
|
|
7865
|
+
);
|
|
7866
|
+
}
|
|
7867
|
+
}
|
|
7868
|
+
|
|
7798
7869
|
#[test]
|
|
7799
7870
|
fn server_max_batch_size_allows_flush_when_encoded_fits_in_all_progress_modes() {
|
|
7800
7871
|
for progress in [ProgressCase::Background, ProgressCase::Manual] {
|
|
@@ -7849,12 +7920,15 @@ fn absent_server_max_batch_size_falls_back_to_configured_max_in_all_progress_mod
|
|
|
7849
7920
|
let err = sender.flush(&mut buf).unwrap_err();
|
|
7850
7921
|
assert_eq!(
|
|
7851
7922
|
err.code(),
|
|
7852
|
-
crate::ErrorCode::
|
|
7923
|
+
crate::ErrorCode::BatchTooLarge,
|
|
7853
7924
|
"mode={}",
|
|
7854
7925
|
progress.name()
|
|
7855
7926
|
);
|
|
7856
7927
|
assert!(
|
|
7857
|
-
err.msg().contains(&
|
|
7928
|
+
err.msg().contains(&format!(
|
|
7929
|
+
"exceeds the per-frame cap of {configured_max} bytes. \
|
|
7930
|
+
The cap is set by max_buf_size."
|
|
7931
|
+
)),
|
|
7858
7932
|
"mode={}: error should reference configured max {configured_max}, got: {}",
|
|
7859
7933
|
progress.name(),
|
|
7860
7934
|
err.msg()
|
|
@@ -7889,12 +7963,15 @@ fn configured_max_wins_when_smaller_than_server_cap_in_all_progress_modes() {
|
|
|
7889
7963
|
let err = sender.flush(&mut buf).unwrap_err();
|
|
7890
7964
|
assert_eq!(
|
|
7891
7965
|
err.code(),
|
|
7892
|
-
crate::ErrorCode::
|
|
7966
|
+
crate::ErrorCode::BatchTooLarge,
|
|
7893
7967
|
"mode={}",
|
|
7894
7968
|
progress.name()
|
|
7895
7969
|
);
|
|
7896
7970
|
assert!(
|
|
7897
|
-
err.msg().contains(&
|
|
7971
|
+
err.msg().contains(&format!(
|
|
7972
|
+
"exceeds the per-frame cap of {configured_max} bytes. \
|
|
7973
|
+
The cap is set by max_buf_size."
|
|
7974
|
+
)),
|
|
7898
7975
|
"mode={}: error should reference configured max {configured_max}, got: {}",
|
|
7899
7976
|
progress.name(),
|
|
7900
7977
|
err.msg()
|
|
@@ -7936,8 +8013,8 @@ fn server_cap_prevents_message_too_big_by_rejecting_locally_in_all_progress_mode
|
|
|
7936
8013
|
let err = sender.flush(&mut big_buf).unwrap_err();
|
|
7937
8014
|
assert_eq!(
|
|
7938
8015
|
err.code(),
|
|
7939
|
-
crate::ErrorCode::
|
|
7940
|
-
"mode={}: must get local
|
|
8016
|
+
crate::ErrorCode::BatchTooLarge,
|
|
8017
|
+
"mode={}: must get local BatchTooLarge, not a server error",
|
|
7941
8018
|
progress.name()
|
|
7942
8019
|
);
|
|
7943
8020
|
}
|
|
@@ -7999,12 +8076,15 @@ fn server_cap_one_byte_below_encoded_len_rejects_flush_in_all_progress_modes() {
|
|
|
7999
8076
|
let err = sender.flush(&mut buf).unwrap_err();
|
|
8000
8077
|
assert_eq!(
|
|
8001
8078
|
err.code(),
|
|
8002
|
-
crate::ErrorCode::
|
|
8079
|
+
crate::ErrorCode::BatchTooLarge,
|
|
8003
8080
|
"mode={}",
|
|
8004
8081
|
progress.name()
|
|
8005
8082
|
);
|
|
8006
8083
|
assert!(
|
|
8007
|
-
err.msg().contains(&
|
|
8084
|
+
err.msg().contains(&format!(
|
|
8085
|
+
"exceeds the per-frame cap of {server_cap} bytes. \
|
|
8086
|
+
The cap is set by the server's advertised X-QWP-Max-Batch-Size."
|
|
8087
|
+
)),
|
|
8008
8088
|
"mode={}: error should reference server cap {server_cap}, got: {}",
|
|
8009
8089
|
progress.name(),
|
|
8010
8090
|
err.msg()
|
|
@@ -80,6 +80,15 @@ fn encode_varint_u64(mut v: u64, out: &mut Vec<u8>) {
|
|
|
80
80
|
}
|
|
81
81
|
|
|
82
82
|
fn server_info_frame(role: ServerRole, node_id: &str, cluster_id: &str) -> Vec<u8> {
|
|
83
|
+
server_info_frame_with_caps(role, node_id, cluster_id, 0)
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
fn server_info_frame_with_caps(
|
|
87
|
+
role: ServerRole,
|
|
88
|
+
node_id: &str,
|
|
89
|
+
cluster_id: &str,
|
|
90
|
+
capabilities: u32,
|
|
91
|
+
) -> Vec<u8> {
|
|
83
92
|
let role_byte = match role {
|
|
84
93
|
ServerRole::Standalone => 0x00,
|
|
85
94
|
ServerRole::Primary => 0x01,
|
|
@@ -90,7 +99,7 @@ fn server_info_frame(role: ServerRole, node_id: &str, cluster_id: &str) -> Vec<u
|
|
|
90
99
|
};
|
|
91
100
|
let mut payload = vec![MSG_SERVER_INFO, role_byte];
|
|
92
101
|
payload.extend_from_slice(&0u64.to_le_bytes()); // epoch
|
|
93
|
-
payload.extend_from_slice(&
|
|
102
|
+
payload.extend_from_slice(&capabilities.to_le_bytes());
|
|
94
103
|
payload.extend_from_slice(&0i64.to_le_bytes()); // server_wall_ns
|
|
95
104
|
payload.extend_from_slice(&(cluster_id.len() as u16).to_le_bytes());
|
|
96
105
|
payload.extend_from_slice(cluster_id.as_bytes());
|
|
@@ -121,6 +130,9 @@ enum BatchColumn {
|
|
|
121
130
|
/// its dict from scratch); `codes` are the per-row ids that index it.
|
|
122
131
|
#[cfg_attr(not(feature = "arrow"), allow(dead_code))]
|
|
123
132
|
Symbol { dict: Vec<String>, codes: Vec<u32> },
|
|
133
|
+
/// DOUBLE_ARRAY column named `a`, one zero-filled array per shape.
|
|
134
|
+
#[cfg_attr(not(feature = "arrow-egress"), allow(dead_code))]
|
|
135
|
+
DoubleArray(Vec<Vec<u32>>),
|
|
124
136
|
}
|
|
125
137
|
|
|
126
138
|
/// Single-table `RESULT_BATCH` frame carrying one non-null column. The
|
|
@@ -132,6 +144,7 @@ fn result_batch_frame(request_id: i64, batch_seq: u64, column: &BatchColumn) ->
|
|
|
132
144
|
const KIND_LONG: u8 = 0x05;
|
|
133
145
|
const KIND_DOUBLE: u8 = 0x07;
|
|
134
146
|
const KIND_SYMBOL: u8 = 0x09;
|
|
147
|
+
const KIND_DOUBLE_ARRAY: u8 = 0x11;
|
|
135
148
|
const NULL_FLAG_NONE: u8 = 0x00;
|
|
136
149
|
// Frame header flag: SYMBOL columns ride the connection-scoped dict,
|
|
137
150
|
// so the batch carries the delta-dict section (`flags::DELTA_SYMBOL_DICT`).
|
|
@@ -140,6 +153,7 @@ fn result_batch_frame(request_id: i64, batch_seq: u64, column: &BatchColumn) ->
|
|
|
140
153
|
BatchColumn::Long(v) => ("v", KIND_LONG, v.len()),
|
|
141
154
|
BatchColumn::Double(v) => ("d", KIND_DOUBLE, v.len()),
|
|
142
155
|
BatchColumn::Symbol { codes, .. } => ("s", KIND_SYMBOL, codes.len()),
|
|
156
|
+
BatchColumn::DoubleArray(shapes) => ("a", KIND_DOUBLE_ARRAY, shapes.len()),
|
|
143
157
|
};
|
|
144
158
|
let flags = match column {
|
|
145
159
|
BatchColumn::Symbol { .. } => FLAG_DELTA_SYMBOL_DICT,
|
|
@@ -185,6 +199,16 @@ fn result_batch_frame(request_id: i64, batch_seq: u64, column: &BatchColumn) ->
|
|
|
185
199
|
encode_varint_u64(*code as u64, &mut payload);
|
|
186
200
|
}
|
|
187
201
|
}
|
|
202
|
+
BatchColumn::DoubleArray(shapes) => {
|
|
203
|
+
for shape in shapes {
|
|
204
|
+
payload.push(shape.len() as u8);
|
|
205
|
+
for dim in shape {
|
|
206
|
+
payload.extend_from_slice(&dim.to_le_bytes());
|
|
207
|
+
}
|
|
208
|
+
let elements: u32 = shape.iter().product();
|
|
209
|
+
payload.extend(std::iter::repeat_n(0u8, elements as usize * 8));
|
|
210
|
+
}
|
|
211
|
+
}
|
|
188
212
|
}
|
|
189
213
|
framed(1, flags, 1, &payload)
|
|
190
214
|
}
|
|
@@ -1034,6 +1058,50 @@ fn non_stale_internal_error_still_surfaces_to_caller() {
|
|
|
1034
1058
|
assert_eq!(srv.accepts(), 1, "no retry, so no second connection");
|
|
1035
1059
|
}
|
|
1036
1060
|
|
|
1061
|
+
#[test]
|
|
1062
|
+
#[cfg_attr(
|
|
1063
|
+
windows,
|
|
1064
|
+
ignore = "WinSock send() to a peer that has RST'd can block for the full WRITE_TIMEOUT"
|
|
1065
|
+
)]
|
|
1066
|
+
fn stale_cached_plan_replay_onto_dead_peer_closes_the_connection() {
|
|
1067
|
+
let srv = MockServer::start(vec![vec![
|
|
1068
|
+
Action::SendServerInfo {
|
|
1069
|
+
role: ServerRole::Standalone,
|
|
1070
|
+
node_id: "n1".into(),
|
|
1071
|
+
},
|
|
1072
|
+
Action::AwaitQueryRequest,
|
|
1073
|
+
Action::SendQueryError {
|
|
1074
|
+
status: STATUS_INTERNAL_ERROR,
|
|
1075
|
+
message: stale_plan_message(),
|
|
1076
|
+
},
|
|
1077
|
+
Action::AbortiveRst,
|
|
1078
|
+
]]);
|
|
1079
|
+
let conf = format!("ws::addr={};failover=off", srv.url());
|
|
1080
|
+
let mut reader = Reader::from_conf(&conf).expect("connect");
|
|
1081
|
+
let mut cursor = reader.prepare("select 1").execute().expect("execute");
|
|
1082
|
+
|
|
1083
|
+
let err = cursor
|
|
1084
|
+
.next_batch()
|
|
1085
|
+
.err()
|
|
1086
|
+
.expect("a dead peer must surface as an error, not as rows");
|
|
1087
|
+
assert_ne!(
|
|
1088
|
+
err.code(),
|
|
1089
|
+
ErrorCode::ServerInternalError,
|
|
1090
|
+
"the stale-plan fault itself must never reach the caller: {}",
|
|
1091
|
+
err.msg()
|
|
1092
|
+
);
|
|
1093
|
+
assert!(
|
|
1094
|
+
!cursor.connection_reusable(),
|
|
1095
|
+
"a connection whose replay hit a dead peer must not be recycled"
|
|
1096
|
+
);
|
|
1097
|
+
drop(cursor);
|
|
1098
|
+
assert!(
|
|
1099
|
+
reader.transport_torn_down(),
|
|
1100
|
+
"the transport must be closed, not left half-written for the next query"
|
|
1101
|
+
);
|
|
1102
|
+
assert_eq!(srv.accepts(), 1, "failover is off, so no reconnect");
|
|
1103
|
+
}
|
|
1104
|
+
|
|
1037
1105
|
#[test]
|
|
1038
1106
|
fn stale_cached_plan_after_rows_delivered_surfaces() {
|
|
1039
1107
|
// The load-bearing `!data_delivered` guard: once a batch has been handed
|
|
@@ -1269,6 +1337,127 @@ fn pre_batch_failover_without_callback_still_replays() {
|
|
|
1269
1337
|
);
|
|
1270
1338
|
}
|
|
1271
1339
|
|
|
1340
|
+
const CAP_QUERY_FLAGS: u32 = 0x02;
|
|
1341
|
+
const CAP_QUERY_TIMEOUT: u32 = 0x08;
|
|
1342
|
+
|
|
1343
|
+
fn server_info_with_caps(node_id: &str, capabilities: u32) -> Action {
|
|
1344
|
+
Action::SendRaw(server_info_frame_with_caps(
|
|
1345
|
+
ServerRole::Standalone,
|
|
1346
|
+
node_id,
|
|
1347
|
+
"test-cluster",
|
|
1348
|
+
capabilities,
|
|
1349
|
+
))
|
|
1350
|
+
}
|
|
1351
|
+
|
|
1352
|
+
#[test]
|
|
1353
|
+
fn sub_millisecond_timeout_rounds_up_on_the_wire() {
|
|
1354
|
+
let srv = MockServer::start(vec![vec![
|
|
1355
|
+
server_info_with_caps("n1", CAP_QUERY_FLAGS | CAP_QUERY_TIMEOUT),
|
|
1356
|
+
Action::AwaitQueryRequest,
|
|
1357
|
+
Action::SendResultEnd,
|
|
1358
|
+
]]);
|
|
1359
|
+
let mut reader = Reader::from_conf(format!("ws::addr={}", srv.url())).expect("connect");
|
|
1360
|
+
let mut cursor = reader
|
|
1361
|
+
.prepare("select 1")
|
|
1362
|
+
.timeout(Duration::from_micros(500))
|
|
1363
|
+
.execute()
|
|
1364
|
+
.expect("execute");
|
|
1365
|
+
assert!(matches!(cursor.next_batch(), Ok(None)));
|
|
1366
|
+
drop(cursor);
|
|
1367
|
+
|
|
1368
|
+
let requests = srv.captured_requests();
|
|
1369
|
+
assert_eq!(requests.len(), 1);
|
|
1370
|
+
// bind_count = 0, query_flags = QUERY_FLAG_TIMEOUT, timeout_ms = 1.
|
|
1371
|
+
assert!(
|
|
1372
|
+
requests[0].ends_with(&[0x00, 0x02, 0x01]),
|
|
1373
|
+
"500 us must round up to timeout_ms = 1, got {:02X?}",
|
|
1374
|
+
requests[0]
|
|
1375
|
+
);
|
|
1376
|
+
}
|
|
1377
|
+
|
|
1378
|
+
#[test]
|
|
1379
|
+
fn timeout_failover_onto_server_without_timeout_cap_refuses_replay() {
|
|
1380
|
+
let srv_a = MockServer::start(vec![vec![
|
|
1381
|
+
server_info_with_caps("a", CAP_QUERY_FLAGS | CAP_QUERY_TIMEOUT),
|
|
1382
|
+
Action::AwaitQueryRequest,
|
|
1383
|
+
Action::HardDrop,
|
|
1384
|
+
]]);
|
|
1385
|
+
let srv_b = MockServer::start(vec![vec![
|
|
1386
|
+
server_info_with_caps("b", CAP_QUERY_FLAGS),
|
|
1387
|
+
Action::AwaitQueryRequest,
|
|
1388
|
+
Action::SendResultEnd,
|
|
1389
|
+
]]);
|
|
1390
|
+
let conf = format!(
|
|
1391
|
+
"ws::addr={};failover_backoff_initial_ms=1;failover_backoff_max_ms=10",
|
|
1392
|
+
build_addr_list(&[&srv_a, &srv_b])
|
|
1393
|
+
);
|
|
1394
|
+
let mut reader = Reader::from_conf(&conf).expect("connect to A");
|
|
1395
|
+
let mut cursor = reader
|
|
1396
|
+
.prepare("select 1")
|
|
1397
|
+
.timeout(Duration::from_millis(10))
|
|
1398
|
+
.execute()
|
|
1399
|
+
.expect("execute");
|
|
1400
|
+
|
|
1401
|
+
let err = cursor.next_batch().err().expect("replay must be refused");
|
|
1402
|
+
assert_eq!(err.code(), ErrorCode::UnsupportedServer, "{}", err.msg());
|
|
1403
|
+
assert_eq!(cursor.failover_resets(), 0);
|
|
1404
|
+
drop(cursor);
|
|
1405
|
+
assert!(
|
|
1406
|
+
srv_b.captured_requests().is_empty(),
|
|
1407
|
+
"the timed query must never reach a server that would ignore the timeout"
|
|
1408
|
+
);
|
|
1409
|
+
|
|
1410
|
+
let mut cursor = reader.prepare("select 1").execute().expect("reuse B");
|
|
1411
|
+
assert!(matches!(cursor.next_batch(), Ok(None)));
|
|
1412
|
+
drop(cursor);
|
|
1413
|
+
assert_eq!(reader.current_addr().port, srv_b.addr.port());
|
|
1414
|
+
assert_eq!(srv_b.captured_requests().len(), 1);
|
|
1415
|
+
}
|
|
1416
|
+
|
|
1417
|
+
#[test]
|
|
1418
|
+
#[cfg_attr(
|
|
1419
|
+
windows,
|
|
1420
|
+
ignore = "WinSock send() to a peer that has RST'd can block for the full WRITE_TIMEOUT"
|
|
1421
|
+
)]
|
|
1422
|
+
fn add_credit_refused_timeout_replay_stays_terminal() {
|
|
1423
|
+
let srv_a = MockServer::start(vec![vec![
|
|
1424
|
+
server_info_with_caps("a", CAP_QUERY_FLAGS | CAP_QUERY_TIMEOUT),
|
|
1425
|
+
Action::AwaitQueryRequest,
|
|
1426
|
+
Action::AbortiveRst,
|
|
1427
|
+
]]);
|
|
1428
|
+
let srv_b = MockServer::start(vec![vec![
|
|
1429
|
+
server_info_with_caps("b", CAP_QUERY_FLAGS),
|
|
1430
|
+
Action::AwaitQueryRequest,
|
|
1431
|
+
Action::SendResultEnd,
|
|
1432
|
+
]]);
|
|
1433
|
+
let conf = format!(
|
|
1434
|
+
"ws::addr={};failover_backoff_initial_ms=1;failover_backoff_max_ms=10",
|
|
1435
|
+
build_addr_list(&[&srv_a, &srv_b])
|
|
1436
|
+
);
|
|
1437
|
+
let mut reader = Reader::from_conf(&conf).expect("connect to A");
|
|
1438
|
+
let mut cursor = reader
|
|
1439
|
+
.prepare("select 1")
|
|
1440
|
+
.timeout(Duration::from_millis(10))
|
|
1441
|
+
.execute()
|
|
1442
|
+
.expect("execute");
|
|
1443
|
+
|
|
1444
|
+
std::thread::sleep(Duration::from_millis(100));
|
|
1445
|
+
// A write can land in the send buffer before the RST is observed;
|
|
1446
|
+
// keep granting until one fails and drives the failover.
|
|
1447
|
+
let err = (0..50)
|
|
1448
|
+
.find_map(|_| cursor.add_credit(64).err())
|
|
1449
|
+
.expect("add_credit must eventually hit A's RST");
|
|
1450
|
+
assert_eq!(err.code(), ErrorCode::UnsupportedServer, "{}", err.msg());
|
|
1451
|
+
|
|
1452
|
+
let next = cursor
|
|
1453
|
+
.next_batch()
|
|
1454
|
+
.err()
|
|
1455
|
+
.expect("must not read as clean EOF");
|
|
1456
|
+
assert_eq!(next.code(), ErrorCode::UnsupportedServer);
|
|
1457
|
+
let again = cursor.add_credit(64).expect_err("cursor stays terminal");
|
|
1458
|
+
assert_eq!(again.code(), ErrorCode::UnsupportedServer);
|
|
1459
|
+
}
|
|
1460
|
+
|
|
1272
1461
|
/// Mid-query failover *after* a batch was already delivered: with a
|
|
1273
1462
|
/// reset callback installed (the replay opt-in), the replayed query
|
|
1274
1463
|
/// must re-read its schema from the new node's `batch_seq == 0` frame
|
|
@@ -1508,6 +1697,35 @@ fn failover_arrow_reader_schema_drift_poisons() {
|
|
|
1508
1697
|
assert_eq!(cursor.failover_resets(), 1);
|
|
1509
1698
|
}
|
|
1510
1699
|
|
|
1700
|
+
#[cfg(feature = "arrow-egress")]
|
|
1701
|
+
#[test]
|
|
1702
|
+
fn arrow_conversion_error_mid_stream_is_not_reusable() {
|
|
1703
|
+
let srv = MockServer::start(vec![vec![
|
|
1704
|
+
Action::SendServerInfo {
|
|
1705
|
+
role: ServerRole::Standalone,
|
|
1706
|
+
node_id: "n1".into(),
|
|
1707
|
+
},
|
|
1708
|
+
Action::AwaitQueryRequest,
|
|
1709
|
+
Action::SendBatch {
|
|
1710
|
+
batch_seq: 0,
|
|
1711
|
+
column: BatchColumn::DoubleArray(vec![vec![1], vec![1, 1]]),
|
|
1712
|
+
},
|
|
1713
|
+
Action::SendResultEnd,
|
|
1714
|
+
]]);
|
|
1715
|
+
let mut reader = Reader::from_conf(format!("ws::addr={}", srv.url())).expect("connect");
|
|
1716
|
+
let mut cursor = reader
|
|
1717
|
+
.prepare("select a from t")
|
|
1718
|
+
.execute()
|
|
1719
|
+
.expect("execute");
|
|
1720
|
+
|
|
1721
|
+
cursor
|
|
1722
|
+
.next_arrow_batch()
|
|
1723
|
+
.expect_err("mixed-rank rows cannot become one Arrow list column");
|
|
1724
|
+
assert!(!cursor.connection_reusable());
|
|
1725
|
+
drop(cursor);
|
|
1726
|
+
assert!(reader.transport_torn_down());
|
|
1727
|
+
}
|
|
1728
|
+
|
|
1511
1729
|
/// Regression pin for the reconnect-path schema clear
|
|
1512
1730
|
/// (`query_schema = None` in `Reader::reconnect_with_failover`): when
|
|
1513
1731
|
/// the replayed query's *first* frame from the new node is a
|