questdb 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.cargo/audit.toml +17 -9
- data/CHANGELOG.md +7 -0
- data/README.md +1 -1
- data/THIRD_PARTY_NOTICES.md +2 -2
- data/docs/configuration.md +3 -2
- data/docs/design.md +1 -1
- data/docs/python-parity.md +1 -1
- data/docs/releasing.md +1 -1
- data/ext/questdb_ingress/extconf.rb +2 -0
- data/ext/questdb_ingress/questdb_ingress.c +2 -1
- data/ext/questdb_ingress/unlink_build_ruby.rb +58 -0
- data/lib/questdb/arrow.rb +2 -0
- data/lib/questdb/client.rb +6 -0
- data/lib/questdb/config.rb +19 -0
- data/lib/questdb/pooled_sender.rb +4 -1
- data/lib/questdb/sender.rb +10 -1
- data/lib/questdb/version.rb +2 -2
- data/script/check-native-linkage +115 -0
- data/test/client_test.rb +30 -0
- data/test/extconf_link_flags_test.rb +160 -0
- data/vendor/c-questdb-client/.bumpversion.toml +1 -1
- data/vendor/c-questdb-client/CMakeLists.txt +1 -1
- data/vendor/c-questdb-client/cpp_test/qwp_mock_server.hpp +10 -1
- data/vendor/c-questdb-client/cpp_test/test_arrow_c.c +39 -0
- data/vendor/c-questdb-client/cpp_test/test_reader_mock.cpp +428 -0
- data/vendor/c-questdb-client/doc/COMPATIBILITY.md +1 -1
- data/vendor/c-questdb-client/doc/CONSIDERATIONS.md +6 -3
- data/vendor/c-questdb-client/doc/RELEASE_RUNBOOK.md +7 -7
- data/vendor/c-questdb-client/doc/RELEASING.md +6 -6
- data/vendor/c-questdb-client/include/questdb/egress/qwp_reader.h +48 -1
- data/vendor/c-questdb-client/include/questdb/egress/qwp_reader.hpp +52 -0
- data/vendor/c-questdb-client/include/questdb/ingress/line_sender.h +46 -4
- data/vendor/c-questdb-client/include/questdb/ingress/line_sender.hpp +7 -4
- data/vendor/c-questdb-client/include/questdb/ingress/line_sender_core.hpp +1 -0
- data/vendor/c-questdb-client/questdb-rs/Cargo.toml +1 -1
- data/vendor/c-questdb-client/questdb-rs/README.md +13 -13
- data/vendor/c-questdb-client/questdb-rs/src/egress/config.rs +78 -0
- data/vendor/c-questdb-client/questdb-rs/src/egress/mod.rs +2 -2
- data/vendor/c-questdb-client/questdb-rs/src/egress/query_request.rs +174 -1
- data/vendor/c-questdb-client/questdb-rs/src/egress/reader.rs +264 -14
- data/vendor/c-questdb-client/questdb-rs/src/egress/wire/capabilities.rs +69 -0
- data/vendor/c-questdb-client/questdb-rs/src/egress/wire/msg_kind.rs +33 -0
- data/vendor/c-questdb-client/questdb-rs/src/error.rs +33 -5
- data/vendor/c-questdb-client/questdb-rs/src/ingress/buffer/qwp.rs +96 -11
- data/vendor/c-questdb-client/questdb-rs/src/ingress/buffer.rs +6 -3
- data/vendor/c-questdb-client/questdb-rs/src/ingress/column_sender/sender.rs +1 -16
- data/vendor/c-questdb-client/questdb-rs/src/ingress/sender/qwp_ws.rs +19 -13
- data/vendor/c-questdb-client/questdb-rs/src/ingress/sender/qwp_ws_driver.rs +2 -9
- data/vendor/c-questdb-client/questdb-rs/src/ingress/sender/qwp_ws_publisher.rs +3 -3
- data/vendor/c-questdb-client/questdb-rs/src/ingress/sender.rs +101 -17
- data/vendor/c-questdb-client/questdb-rs/src/ingress/tests.rs +32 -0
- data/vendor/c-questdb-client/questdb-rs/src/ingress.rs +3 -2
- data/vendor/c-questdb-client/questdb-rs/src/tests/qwp_ws.rs +96 -16
- data/vendor/c-questdb-client/questdb-rs/tests/egress_failover.rs +219 -1
- data/vendor/c-questdb-client/questdb-rs/tests/egress_live_server.rs +34 -0
- data/vendor/c-questdb-client/questdb-rs-ffi/Cargo.lock +2 -2
- data/vendor/c-questdb-client/questdb-rs-ffi/Cargo.toml +1 -1
- data/vendor/c-questdb-client/questdb-rs-ffi/src/egress.rs +31 -0
- data/vendor/c-questdb-client/questdb-rs-ffi/src/lib.rs +70 -3
- data/vendor/c-questdb-client/system_test/arrow_ffi.py +1 -0
- data/vendor/c-questdb-client/system_test/enterprise_e2e/tests/test_failover.py +12 -6
- data/vendor/c-questdb-client/system_test/enterprise_e2e/tests/test_orphan_drainer_bindings.py +12 -4
- data/vendor/c-questdb-client/system_test/failover_clients/Cargo.lock +1 -1
- data/vendor/c-questdb-client/system_test/soak/workload_rs/Cargo.lock +1 -1
- data/vendor/c-questdb-client.version +1 -1
- metadata +5 -2
|
@@ -51,6 +51,28 @@ pub fn has_query_flags(capabilities: u32) -> bool {
|
|
|
51
51
|
capabilities & CAP_QUERY_FLAGS != 0
|
|
52
52
|
}
|
|
53
53
|
|
|
54
|
+
/// Bit `0x04` is `CAP_COMPRESSION` on the server (zstd frame compression,
|
|
55
|
+
/// negotiated through the `X-QWP-Accept-Encoding` upgrade header rather than
|
|
56
|
+
/// through this word on the client side). Named here so the next capability
|
|
57
|
+
/// does not reuse the value.
|
|
58
|
+
pub const CAP_COMPRESSION: u32 = 0x0000_0004;
|
|
59
|
+
|
|
60
|
+
/// Server honours `QUERY_REQUEST.timeout_ms`: it runs the query under the
|
|
61
|
+
/// client-supplied timeout instead of the server-wide `query.timeout`, and
|
|
62
|
+
/// reports expiry as [`StatusCode::QueryTimeout`].
|
|
63
|
+
///
|
|
64
|
+
/// Clients append `timeout_ms` only when this bit is set, so a server
|
|
65
|
+
/// without it never receives the field. The server sets no ceiling, so a
|
|
66
|
+
/// client timeout may legitimately exceed `query.timeout`.
|
|
67
|
+
///
|
|
68
|
+
/// [`StatusCode::QueryTimeout`]: super::msg_kind::StatusCode::QueryTimeout
|
|
69
|
+
pub const CAP_QUERY_TIMEOUT: u32 = 0x0000_0008;
|
|
70
|
+
|
|
71
|
+
/// True if the server honours the `QUERY_REQUEST` `timeout_ms` field.
|
|
72
|
+
pub fn has_query_timeout(capabilities: u32) -> bool {
|
|
73
|
+
capabilities & CAP_QUERY_TIMEOUT != 0
|
|
74
|
+
}
|
|
75
|
+
|
|
54
76
|
#[cfg(test)]
|
|
55
77
|
mod tests {
|
|
56
78
|
use super::*;
|
|
@@ -64,4 +86,51 @@ mod tests {
|
|
|
64
86
|
// Future bits with CAP_ZONE clear must not trip it.
|
|
65
87
|
assert!(!has_zone(0xFFFF_FFFE));
|
|
66
88
|
}
|
|
89
|
+
|
|
90
|
+
#[test]
|
|
91
|
+
fn has_query_flags_predicate() {
|
|
92
|
+
assert!(!has_query_flags(0));
|
|
93
|
+
assert!(has_query_flags(CAP_QUERY_FLAGS));
|
|
94
|
+
assert!(has_query_flags(CAP_QUERY_FLAGS | 0x8000_0000));
|
|
95
|
+
assert!(!has_query_flags(!CAP_QUERY_FLAGS));
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
#[test]
|
|
99
|
+
fn has_query_timeout_predicate() {
|
|
100
|
+
assert!(!has_query_timeout(0));
|
|
101
|
+
assert!(has_query_timeout(CAP_QUERY_TIMEOUT));
|
|
102
|
+
// The timeout bit rides alongside the flags bit on a current
|
|
103
|
+
// server; neither predicate may depend on the other's absence.
|
|
104
|
+
assert!(has_query_timeout(CAP_QUERY_TIMEOUT | CAP_QUERY_FLAGS));
|
|
105
|
+
assert!(has_query_flags(CAP_QUERY_TIMEOUT | CAP_QUERY_FLAGS));
|
|
106
|
+
assert!(!has_query_timeout(!CAP_QUERY_TIMEOUT));
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
#[test]
|
|
110
|
+
fn capability_bits_are_distinct_powers_of_two() {
|
|
111
|
+
// The server assigns these; a duplicated or non-power-of-two value
|
|
112
|
+
// would make one predicate answer for another capability.
|
|
113
|
+
let bits = [
|
|
114
|
+
CAP_ZONE,
|
|
115
|
+
CAP_QUERY_FLAGS,
|
|
116
|
+
CAP_COMPRESSION,
|
|
117
|
+
CAP_QUERY_TIMEOUT,
|
|
118
|
+
];
|
|
119
|
+
let mut seen = 0u32;
|
|
120
|
+
for b in bits {
|
|
121
|
+
assert_eq!(b.count_ones(), 1, "0x{b:08X} is not a single bit");
|
|
122
|
+
assert_eq!(seen & b, 0, "0x{b:08X} duplicates an earlier bit");
|
|
123
|
+
seen |= b;
|
|
124
|
+
}
|
|
125
|
+
// Pin the wire values: the server's QwpConstants owns them.
|
|
126
|
+
assert_eq!(
|
|
127
|
+
(
|
|
128
|
+
CAP_ZONE,
|
|
129
|
+
CAP_QUERY_FLAGS,
|
|
130
|
+
CAP_COMPRESSION,
|
|
131
|
+
CAP_QUERY_TIMEOUT
|
|
132
|
+
),
|
|
133
|
+
(0x01, 0x02, 0x04, 0x08)
|
|
134
|
+
);
|
|
135
|
+
}
|
|
67
136
|
}
|
|
@@ -93,6 +93,12 @@ pub enum StatusCode {
|
|
|
93
93
|
SecurityError = 0x08,
|
|
94
94
|
Cancelled = 0x0A,
|
|
95
95
|
LimitExceeded = 0x0B,
|
|
96
|
+
/// The query ran past the `timeout_ms` it carried. Per-query, not
|
|
97
|
+
/// per-connection: the connection stays open and authenticated, so a
|
|
98
|
+
/// pooled client runs its next query on it instead of reconnecting.
|
|
99
|
+
/// Only sent to clients that supplied a timeout; a request without one
|
|
100
|
+
/// gets [`Self::LimitExceeded`] for the same condition.
|
|
101
|
+
QueryTimeout = 0x0E,
|
|
96
102
|
}
|
|
97
103
|
|
|
98
104
|
impl StatusCode {
|
|
@@ -104,6 +110,7 @@ impl StatusCode {
|
|
|
104
110
|
0x08 => StatusCode::SecurityError,
|
|
105
111
|
0x0A => StatusCode::Cancelled,
|
|
106
112
|
0x0B => StatusCode::LimitExceeded,
|
|
113
|
+
0x0E => StatusCode::QueryTimeout,
|
|
107
114
|
other => {
|
|
108
115
|
return Err(fmt!(
|
|
109
116
|
ProtocolError,
|
|
@@ -157,8 +164,34 @@ mod tests {
|
|
|
157
164
|
StatusCode::SecurityError,
|
|
158
165
|
StatusCode::Cancelled,
|
|
159
166
|
StatusCode::LimitExceeded,
|
|
167
|
+
StatusCode::QueryTimeout,
|
|
160
168
|
] {
|
|
161
169
|
assert_eq!(StatusCode::from_u8(s.as_u8()).unwrap(), s);
|
|
162
170
|
}
|
|
163
171
|
}
|
|
172
|
+
|
|
173
|
+
#[test]
|
|
174
|
+
fn status_code_wire_values_pinned() {
|
|
175
|
+
// The server's QwpConstants owns these numbers; a drifted
|
|
176
|
+
// discriminant would mis-report one server error as another.
|
|
177
|
+
assert_eq!(StatusCode::SchemaMismatch.as_u8(), 0x03);
|
|
178
|
+
assert_eq!(StatusCode::ParseError.as_u8(), 0x05);
|
|
179
|
+
assert_eq!(StatusCode::InternalError.as_u8(), 0x06);
|
|
180
|
+
assert_eq!(StatusCode::SecurityError.as_u8(), 0x08);
|
|
181
|
+
assert_eq!(StatusCode::Cancelled.as_u8(), 0x0A);
|
|
182
|
+
assert_eq!(StatusCode::LimitExceeded.as_u8(), 0x0B);
|
|
183
|
+
assert_eq!(StatusCode::QueryTimeout.as_u8(), 0x0E);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
#[test]
|
|
187
|
+
fn unknown_status_code_rejected() {
|
|
188
|
+
// 0x0C / 0x0D sit between LimitExceeded and QueryTimeout and are
|
|
189
|
+
// unassigned: they must not silently decode as a neighbour.
|
|
190
|
+
for byte in [0x00, 0x04, 0x0C, 0x0D, 0x0F, 0xFF] {
|
|
191
|
+
assert!(
|
|
192
|
+
StatusCode::from_u8(byte).is_err(),
|
|
193
|
+
"0x{byte:02X} must not decode"
|
|
194
|
+
);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
164
197
|
}
|
|
@@ -140,6 +140,12 @@ pub enum ErrorCode {
|
|
|
140
140
|
HandshakeError,
|
|
141
141
|
|
|
142
142
|
/// Server returned an unsupported QWP version, encoding, or capability.
|
|
143
|
+
///
|
|
144
|
+
/// Also raised client-side, before anything is written, when a query
|
|
145
|
+
/// carries a per-query timeout and the server does not advertise
|
|
146
|
+
/// `CAP_QUERY_TIMEOUT` — see
|
|
147
|
+
/// [`ReaderQuery::timeout`](crate::egress::ReaderQuery::timeout). The
|
|
148
|
+
/// connection is untouched by that refusal.
|
|
143
149
|
UnsupportedServer,
|
|
144
150
|
|
|
145
151
|
/// Wire-format violation: bad magic, truncated frame, unknown discriminant,
|
|
@@ -174,6 +180,24 @@ pub enum ErrorCode {
|
|
|
174
180
|
/// Query was cancelled (locally or via server `CANCELLED` status `0x0A`).
|
|
175
181
|
Cancelled,
|
|
176
182
|
|
|
183
|
+
/// The query ran past the timeout it carried: server-reported QWP
|
|
184
|
+
/// `QUERY_TIMEOUT` (status `0x0E`), produced only for a request that
|
|
185
|
+
/// supplied `timeout_ms`. Per-query, so the connection stays usable and
|
|
186
|
+
/// a pooled client runs its next query on it.
|
|
187
|
+
///
|
|
188
|
+
/// A timeout requested against a server that does not advertise
|
|
189
|
+
/// `CAP_QUERY_TIMEOUT` is refused with [`Self::UnsupportedServer`]
|
|
190
|
+
/// instead, so a caller that retries a timeout with a larger budget
|
|
191
|
+
/// does not loop against a server that ignores every budget — see
|
|
192
|
+
/// [`ReaderQuery::timeout`](crate::egress::ReaderQuery::timeout).
|
|
193
|
+
///
|
|
194
|
+
/// **Do not retry a write on this error.** A DDL / `INSERT` / `UPDATE`
|
|
195
|
+
/// that timed out waiting for the table writer may still be applied
|
|
196
|
+
/// afterwards, and a statement that completes past its timeout is
|
|
197
|
+
/// reported as `EXEC_DONE` rather than a timeout precisely so a
|
|
198
|
+
/// retrying client cannot apply it twice.
|
|
199
|
+
QueryTimeout,
|
|
200
|
+
|
|
177
201
|
/// Mid-query failover was eligible but at least one batch had already
|
|
178
202
|
/// been delivered to the caller, and the cursor's `on_failover_reset`
|
|
179
203
|
/// callback was not installed. Failover would replay the query from the
|
|
@@ -202,11 +226,14 @@ pub enum ErrorCode {
|
|
|
202
226
|
/// bug; not user-recoverable. Only emitted on the `arrow` feature.
|
|
203
227
|
ArrowExport,
|
|
204
228
|
|
|
205
|
-
///
|
|
206
|
-
///
|
|
207
|
-
///
|
|
208
|
-
/// splits oversize inputs into smaller frames automatically, so this
|
|
209
|
-
/// surfaces
|
|
229
|
+
/// A QWP/WebSocket frame exceeds the negotiated per-frame cap
|
|
230
|
+
/// (`min(max_buf_size, server X-QWP-Max-Batch-Size)`, further bounded by
|
|
231
|
+
/// the store-and-forward segment payload capacity). Chunk publication
|
|
232
|
+
/// splits oversize inputs into smaller frames automatically, so there this
|
|
233
|
+
/// only surfaces for an irreducible unit (the table schema plus a single
|
|
234
|
+
/// row block). A row [`Buffer`](crate::ingress::Buffer) is published as one
|
|
235
|
+
/// frame and is never split, so flushing one whose encoded frame exceeds
|
|
236
|
+
/// the cap fails with this code and publishes nothing. Distinct from
|
|
210
237
|
/// [`InvalidApiCall`](Self::InvalidApiCall) so callers can recognise it
|
|
211
238
|
/// without matching on the error message text.
|
|
212
239
|
BatchTooLarge,
|
|
@@ -608,6 +635,7 @@ mod tests {
|
|
|
608
635
|
ErrorCode::LimitExceeded => {}
|
|
609
636
|
ErrorCode::ServerLimitExceeded => {}
|
|
610
637
|
ErrorCode::Cancelled => {}
|
|
638
|
+
ErrorCode::QueryTimeout => {}
|
|
611
639
|
ErrorCode::FailoverWouldDuplicate => {}
|
|
612
640
|
ErrorCode::SchemaDrift => {}
|
|
613
641
|
ErrorCode::NoSchema => {}
|
|
@@ -2559,9 +2559,8 @@ impl QwpWsSizeHint {
|
|
|
2559
2559
|
}
|
|
2560
2560
|
|
|
2561
2561
|
QWP_MESSAGE_HEADER_SIZE
|
|
2562
|
-
+ 2
|
|
2563
2562
|
+ self.encoded_tables_len
|
|
2564
|
-
+
|
|
2563
|
+
+ QWP_WS_SYMBOL_ID_MAX_LEN
|
|
2565
2564
|
+ qwp_varint_size(self.symbol_dict_count as u64)
|
|
2566
2565
|
+ self.symbol_dict_bytes
|
|
2567
2566
|
}
|
|
@@ -2831,7 +2830,7 @@ impl QwpWsColumnarBuffer {
|
|
|
2831
2830
|
|
|
2832
2831
|
#[cfg(test)]
|
|
2833
2832
|
fn recompute_len_slow(&self) -> usize {
|
|
2834
|
-
let mut total = QWP_MESSAGE_HEADER_SIZE
|
|
2833
|
+
let mut total = QWP_MESSAGE_HEADER_SIZE;
|
|
2835
2834
|
let mut symbol_dict_count = 0usize;
|
|
2836
2835
|
let mut symbol_dict_bytes = 0usize;
|
|
2837
2836
|
for table in self.tables.iter().filter(|table| table.row_count > 0) {
|
|
@@ -2854,7 +2853,10 @@ impl QwpWsColumnarBuffer {
|
|
|
2854
2853
|
}
|
|
2855
2854
|
}
|
|
2856
2855
|
}
|
|
2857
|
-
total
|
|
2856
|
+
total
|
|
2857
|
+
+ QWP_WS_SYMBOL_ID_MAX_LEN
|
|
2858
|
+
+ qwp_varint_size(symbol_dict_count as u64)
|
|
2859
|
+
+ symbol_dict_bytes
|
|
2858
2860
|
}
|
|
2859
2861
|
|
|
2860
2862
|
#[cfg(test)]
|
|
@@ -4127,7 +4129,7 @@ impl QwpWsColumnBuffer {
|
|
|
4127
4129
|
fn rollback_rows_from(&mut self, from: u32) {
|
|
4128
4130
|
if let QwpWsColumnValues::Symbol { cells, dict, .. } = &self.values {
|
|
4129
4131
|
for cell in cells.iter().rev().take_while(|cell| cell.row_idx >= from) {
|
|
4130
|
-
self.symbol_cells_encoded_len -=
|
|
4132
|
+
self.symbol_cells_encoded_len -= QWP_WS_SYMBOL_ID_MAX_LEN;
|
|
4131
4133
|
if cell.is_new {
|
|
4132
4134
|
let entry = &dict[cell.local_id as usize];
|
|
4133
4135
|
self.symbol_dict_encoded_len -= qwp_string_byte_len(entry.len as usize);
|
|
@@ -4317,7 +4319,7 @@ impl QwpWsColumnBuffer {
|
|
|
4317
4319
|
is_new,
|
|
4318
4320
|
});
|
|
4319
4321
|
self.increment_non_null()?;
|
|
4320
|
-
self.symbol_cells_encoded_len +=
|
|
4322
|
+
self.symbol_cells_encoded_len += QWP_WS_SYMBOL_ID_MAX_LEN;
|
|
4321
4323
|
if is_new {
|
|
4322
4324
|
self.symbol_dict_encoded_len += qwp_string_byte_len(bytes.len());
|
|
4323
4325
|
}
|
|
@@ -4881,10 +4883,7 @@ impl QwpWsColumnValues {
|
|
|
4881
4883
|
cells.len().saturating_mul(8)
|
|
4882
4884
|
}
|
|
4883
4885
|
Self::String { cells, data } => (cells.len() + 1).saturating_mul(4) + data.len(),
|
|
4884
|
-
Self::Symbol { cells, .. } => cells
|
|
4885
|
-
.iter()
|
|
4886
|
-
.map(|cell| qwp_varint_size(cell.local_id as u64))
|
|
4887
|
-
.sum(),
|
|
4886
|
+
Self::Symbol { cells, .. } => cells.len().saturating_mul(QWP_WS_SYMBOL_ID_MAX_LEN),
|
|
4888
4887
|
Self::Decimal { cells, .. } => {
|
|
4889
4888
|
1 + cells
|
|
4890
4889
|
.iter()
|
|
@@ -5502,6 +5501,15 @@ impl Default for SymbolGlobalDict {
|
|
|
5502
5501
|
#[cfg(feature = "_sender-qwp-ws")]
|
|
5503
5502
|
pub(crate) const MAX_CONN_SYMBOL_DICT_SIZE: usize = 2_000_000;
|
|
5504
5503
|
|
|
5504
|
+
/// Bytes the QWP/WS size hint charges for each connection-wide symbol id a
|
|
5505
|
+
/// frame carries: every symbol cell, and the delta dictionary's start id.
|
|
5506
|
+
/// The buffer cannot see those ids until the flush interns its symbols, so
|
|
5507
|
+
/// it charges the widest varint an id can take. Ids stay below
|
|
5508
|
+
/// [`MAX_CONN_SYMBOL_DICT_SIZE`] and the start id reaches it at most, which
|
|
5509
|
+
/// keeps the hint at or above the encoded frame.
|
|
5510
|
+
#[cfg(feature = "_sender-qwp-ws")]
|
|
5511
|
+
const QWP_WS_SYMBOL_ID_MAX_LEN: usize = qwp_varint_size(MAX_CONN_SYMBOL_DICT_SIZE as u64);
|
|
5512
|
+
|
|
5505
5513
|
/// Per-connection cap on the cumulative UTF-8 heap (in bytes) the QWP/WS global
|
|
5506
5514
|
/// symbol dictionary holds. Matches `MAX_CONN_DICT_HEAP_BYTES` in the egress
|
|
5507
5515
|
/// reader (`egress/symbol_dict.rs`) and the Java reference client. The entry
|
|
@@ -7421,7 +7429,7 @@ fn qwp_string_byte_len(byte_len: usize) -> usize {
|
|
|
7421
7429
|
qwp_varint_size(byte_len as u64) + byte_len
|
|
7422
7430
|
}
|
|
7423
7431
|
|
|
7424
|
-
fn qwp_varint_size(mut value: u64) -> usize {
|
|
7432
|
+
const fn qwp_varint_size(mut value: u64) -> usize {
|
|
7425
7433
|
let mut size = 1usize;
|
|
7426
7434
|
while value > 0x7F {
|
|
7427
7435
|
value >>= 7;
|
|
@@ -10562,6 +10570,83 @@ mod tests {
|
|
|
10562
10570
|
assert_eq!(buf.len(), buf.recompute_len_slow());
|
|
10563
10571
|
}
|
|
10564
10572
|
|
|
10573
|
+
#[cfg(feature = "_sender-qwp-ws")]
|
|
10574
|
+
#[test]
|
|
10575
|
+
fn qwp_ws_size_hint_covers_encoded_frame_when_symbol_ids_widen() {
|
|
10576
|
+
// A frame carries each symbol cell as its connection-wide id. The
|
|
10577
|
+
// 500-value column comes first, so the other columns' ids start past
|
|
10578
|
+
// its own, and the connection may have interned symbols before.
|
|
10579
|
+
fn add_rows(buf: &mut QwpWsColumnarBuffer, rows: std::ops::Range<i64>) {
|
|
10580
|
+
for i in rows {
|
|
10581
|
+
let row = buf.table("trades").unwrap();
|
|
10582
|
+
row.symbol("ticker", format!("T{:03}", (i * 7919) % 500).as_str())
|
|
10583
|
+
.unwrap();
|
|
10584
|
+
row.symbol(
|
|
10585
|
+
"exchange",
|
|
10586
|
+
["NYSE", "NASDAQ", "ARCA", "BATS"][(i % 4) as usize],
|
|
10587
|
+
)
|
|
10588
|
+
.unwrap();
|
|
10589
|
+
row.symbol("side", ["buy", "sell"][((i / 3) % 2) as usize])
|
|
10590
|
+
.unwrap();
|
|
10591
|
+
row.symbol("ccy", ["USD", "EUR", "GBP"][((i / 5) % 3) as usize])
|
|
10592
|
+
.unwrap();
|
|
10593
|
+
if i % 3 != 0 {
|
|
10594
|
+
row.symbol("venue", format!("V{}", (i / 7) % 8).as_str())
|
|
10595
|
+
.unwrap();
|
|
10596
|
+
}
|
|
10597
|
+
row.column_f64("price", i as f64 * 0.5).unwrap();
|
|
10598
|
+
row.at_now().unwrap();
|
|
10599
|
+
}
|
|
10600
|
+
}
|
|
10601
|
+
|
|
10602
|
+
fn assert_covers(
|
|
10603
|
+
buf: &QwpWsColumnarBuffer,
|
|
10604
|
+
global_dict: &mut SymbolGlobalDict,
|
|
10605
|
+
warm: usize,
|
|
10606
|
+
what: &str,
|
|
10607
|
+
) {
|
|
10608
|
+
let mut scratch = QwpWsEncodeScratch::new();
|
|
10609
|
+
let mut out = Vec::new();
|
|
10610
|
+
buf.encode_ws_replay_message_with_defer(
|
|
10611
|
+
&mut out,
|
|
10612
|
+
&mut scratch,
|
|
10613
|
+
global_dict,
|
|
10614
|
+
QWP_VERSION_1,
|
|
10615
|
+
false,
|
|
10616
|
+
true,
|
|
10617
|
+
)
|
|
10618
|
+
.unwrap();
|
|
10619
|
+
assert!(
|
|
10620
|
+
out.len() <= buf.len(),
|
|
10621
|
+
"{what} after {warm} interned symbols: encoded {} bytes, size hint {}",
|
|
10622
|
+
out.len(),
|
|
10623
|
+
buf.len()
|
|
10624
|
+
);
|
|
10625
|
+
}
|
|
10626
|
+
|
|
10627
|
+
for warm in [0usize, 127, 128, 16_383, 16_384] {
|
|
10628
|
+
let mut global_dict = SymbolGlobalDict::new();
|
|
10629
|
+
for i in 0..warm {
|
|
10630
|
+
global_dict.intern(format!("w{i:05}").as_bytes()).unwrap();
|
|
10631
|
+
}
|
|
10632
|
+
let mut buf = QwpWsColumnarBuffer::new(127);
|
|
10633
|
+
|
|
10634
|
+
add_rows(&mut buf, 0..600);
|
|
10635
|
+
assert_covers(&buf, &mut global_dict, warm, "new symbols");
|
|
10636
|
+
|
|
10637
|
+
buf.clear();
|
|
10638
|
+
add_rows(&mut buf, 600..1200);
|
|
10639
|
+
assert_covers(&buf, &mut global_dict, warm, "known symbols");
|
|
10640
|
+
|
|
10641
|
+
let before_marker = buf.len();
|
|
10642
|
+
buf.set_marker().unwrap();
|
|
10643
|
+
add_rows(&mut buf, 1200..1201);
|
|
10644
|
+
buf.rewind_to_marker().unwrap();
|
|
10645
|
+
assert_eq!(buf.len(), before_marker);
|
|
10646
|
+
assert_covers(&buf, &mut global_dict, warm, "after a rewind");
|
|
10647
|
+
}
|
|
10648
|
+
}
|
|
10649
|
+
|
|
10565
10650
|
#[cfg(feature = "_sender-qwp-ws")]
|
|
10566
10651
|
#[test]
|
|
10567
10652
|
fn qwp_ws_cached_size_hint_recomputes_only_dirty_tables() {
|
|
@@ -549,9 +549,12 @@ impl Buffer {
|
|
|
549
549
|
///
|
|
550
550
|
/// For ILP buffers this is the exact serialized byte count. For QWP/UDP
|
|
551
551
|
/// buffers this is the size hint used for flush planning. For QWP/WebSocket
|
|
552
|
-
/// buffers this is
|
|
553
|
-
///
|
|
554
|
-
///
|
|
552
|
+
/// buffers this is a size hint that is at least the size of the frame a
|
|
553
|
+
/// flush encodes: it charges each symbol value the widest connection-wide
|
|
554
|
+
/// id. It falls short only once store-and-forward sends full symbol
|
|
555
|
+
/// dictionaries, which it does when its symbol side-file cannot be opened
|
|
556
|
+
/// or rolled back. The sender enforces `max_buf_size` against the encoded
|
|
557
|
+
/// replay message.
|
|
555
558
|
pub fn len(&self) -> usize {
|
|
556
559
|
match &self.inner {
|
|
557
560
|
BufferInner::Ilp(inner) => inner.len(),
|
|
@@ -33,6 +33,7 @@ use crate::ErrorCode;
|
|
|
33
33
|
use crate::ingress::AckLevel;
|
|
34
34
|
use crate::ingress::QwpWsSenderError;
|
|
35
35
|
use crate::ingress::buffer::{Buffer, QwpWsColumnarBuffer, QwpWsEncodeScratch, SymbolGlobalDict};
|
|
36
|
+
use crate::ingress::sender::effective_hard_frame_cap;
|
|
36
37
|
use crate::ingress::sender::qwp_ws::{
|
|
37
38
|
SyncQwpWsHandlerState, publish_qwp_ws_payload_background, qwp_ws_acked_fsn_background,
|
|
38
39
|
qwp_ws_begin_close_background, qwp_ws_check_error_background,
|
|
@@ -2250,22 +2251,6 @@ impl SfaBackend {
|
|
|
2250
2251
|
}
|
|
2251
2252
|
}
|
|
2252
2253
|
|
|
2253
|
-
fn effective_hard_frame_cap(
|
|
2254
|
-
max_buf_size: usize,
|
|
2255
|
-
server_max_batch_size: usize,
|
|
2256
|
-
sfa_frame_payload_cap: usize,
|
|
2257
|
-
) -> (usize, bool) {
|
|
2258
|
-
let configured_cap = if server_max_batch_size == 0 {
|
|
2259
|
-
max_buf_size
|
|
2260
|
-
} else {
|
|
2261
|
-
max_buf_size.min(server_max_batch_size)
|
|
2262
|
-
};
|
|
2263
|
-
(
|
|
2264
|
-
configured_cap.min(sfa_frame_payload_cap),
|
|
2265
|
-
server_max_batch_size != 0,
|
|
2266
|
-
)
|
|
2267
|
-
}
|
|
2268
|
-
|
|
2269
2254
|
/// The store-and-forward `wait` poll loop made no progress toward `boundary`
|
|
2270
2255
|
/// for `sync_timeout`. The connection is alive but the server is not advancing
|
|
2271
2256
|
/// the ack/durable watermark (e.g. back-pressured WAL or a stuck commit).
|
|
@@ -108,14 +108,12 @@ pub(crate) struct TrafficGate {
|
|
|
108
108
|
#[derive(Default)]
|
|
109
109
|
struct TrafficGateState {
|
|
110
110
|
current: Option<TcpStream>,
|
|
111
|
-
/// Raw handle of the ORIGINAL socket, not the `current` dup
|
|
112
|
-
/// `
|
|
113
|
-
///
|
|
114
|
-
///
|
|
115
|
-
/// `
|
|
116
|
-
///
|
|
117
|
-
/// of the original is mutex-ordered behind `clear()`/re-register, so
|
|
118
|
-
/// the handle cannot be recycled while the gate still holds it.
|
|
111
|
+
/// Raw handle of the ORIGINAL socket, not the `current` dup: a shutdown
|
|
112
|
+
/// through the `try_clone` (WSADuplicateSocket) dup can leave later
|
|
113
|
+
/// `recv()`s on the original blocking. Only valid while `current` is
|
|
114
|
+
/// `Some`: every close of the original is mutex-ordered behind
|
|
115
|
+
/// `clear()`/re-register, so the handle cannot be recycled while the
|
|
116
|
+
/// gate still holds it.
|
|
119
117
|
#[cfg(windows)]
|
|
120
118
|
original: Option<std::os::windows::io::RawSocket>,
|
|
121
119
|
shut: bool,
|
|
@@ -222,15 +220,19 @@ impl TrafficGate {
|
|
|
222
220
|
state.shut = true;
|
|
223
221
|
#[cfg(windows)]
|
|
224
222
|
if let Some(original) = state.original.take() {
|
|
225
|
-
|
|
226
|
-
// another thread. Cancel outstanding I/O on the worker's own
|
|
227
|
-
// handle first, mirroring the Java client's windows/net.c
|
|
228
|
-
// shutdown(). A failure with ERROR_NOT_FOUND (nothing pending)
|
|
229
|
-
// is expected and deliberately ignored.
|
|
223
|
+
use std::os::windows::io::FromRawSocket;
|
|
230
224
|
use windows_sys::Win32::Foundation::HANDLE;
|
|
225
|
+
// Shutdown does not wake an in-flight recv(), so CancelIoEx
|
|
226
|
+
// follows it; the reverse order lets a recv() slip in between.
|
|
227
|
+
// SAFETY: `original` is open while `current` is `Some`.
|
|
228
|
+
let original_stream =
|
|
229
|
+
std::mem::ManuallyDrop::new(unsafe { TcpStream::from_raw_socket(original) });
|
|
230
|
+
let shut = shutdown_socket(&original_stream);
|
|
231
231
|
unsafe {
|
|
232
232
|
windows_sys::Win32::System::IO::CancelIoEx(original as HANDLE, std::ptr::null());
|
|
233
233
|
}
|
|
234
|
+
state.current = None;
|
|
235
|
+
return shut;
|
|
234
236
|
}
|
|
235
237
|
if let Some(stream) = state.current.take() {
|
|
236
238
|
shutdown_socket(&stream)?;
|
|
@@ -511,6 +513,9 @@ pub(crate) struct ManualQwpWsHandlerState {
|
|
|
511
513
|
send_core: QwpWsSendCore<BlockingQwpWsTransport>,
|
|
512
514
|
pub(crate) server_max_batch_size: Arc<AtomicUsize>,
|
|
513
515
|
pub(crate) request_durable_ack: bool,
|
|
516
|
+
/// Hard cap on a single queued frame's payload, as on
|
|
517
|
+
/// [`SyncQwpWsHandlerState::sfa_frame_payload_cap`].
|
|
518
|
+
pub(crate) sfa_frame_payload_cap: usize,
|
|
514
519
|
orphan_drainers: Option<ManualOrphanDrainers>,
|
|
515
520
|
append_deadline: Duration,
|
|
516
521
|
close_drain_timeout: Duration,
|
|
@@ -3698,6 +3703,7 @@ pub(crate) fn open_manual_qwp_ws(
|
|
|
3698
3703
|
send_core: parts.send_core,
|
|
3699
3704
|
server_max_batch_size,
|
|
3700
3705
|
request_durable_ack: *qwp_ws.request_durable_ack,
|
|
3706
|
+
sfa_frame_payload_cap: segment_payload_capacity(*qwp_ws.sf_max_segment_bytes),
|
|
3701
3707
|
orphan_drainers,
|
|
3702
3708
|
append_deadline: *qwp_ws.sf_append_deadline,
|
|
3703
3709
|
close_drain_timeout: *qwp_ws.close_flush_timeout,
|
|
@@ -5774,15 +5774,8 @@ mod tests {
|
|
|
5774
5774
|
.unwrap();
|
|
5775
5775
|
let _server_streams = server.join().unwrap();
|
|
5776
5776
|
|
|
5777
|
-
//
|
|
5778
|
-
//
|
|
5779
|
-
// already in flight, and Winsock shutdown() does not wake one entered
|
|
5780
|
-
// afterwards, so a single read can straddle the shutdown and miss
|
|
5781
|
-
// both wake-ups. Retrying sidesteps the race: once the gate has shut
|
|
5782
|
-
// the replacement socket down, the next read attempt fails
|
|
5783
|
-
// immediately. A missing reconnect registration still fails the test,
|
|
5784
|
-
// because the gate never touches this socket and every attempt times
|
|
5785
|
-
// out until the deadline.
|
|
5777
|
+
// Retry short reads so a missed wake-up fails at the deadline
|
|
5778
|
+
// instead of hanging the test.
|
|
5786
5779
|
transport
|
|
5787
5780
|
.stream
|
|
5788
5781
|
.set_timeouts(Some(Duration::from_millis(100)), None)
|
|
@@ -275,8 +275,8 @@ pub(crate) fn qwp_ws_encoded_message_size_error(
|
|
|
275
275
|
max_buf_size: usize,
|
|
276
276
|
) -> crate::Error {
|
|
277
277
|
error::fmt!(
|
|
278
|
-
|
|
279
|
-
"Could not flush buffer: QWP/WebSocket encoded message size of {} exceeds
|
|
278
|
+
BatchTooLarge,
|
|
279
|
+
"Could not flush buffer: QWP/WebSocket encoded message size of {} bytes exceeds the per-frame cap of {} bytes.",
|
|
280
280
|
encoded_len,
|
|
281
281
|
max_buf_size
|
|
282
282
|
)
|
|
@@ -405,7 +405,7 @@ mod tests {
|
|
|
405
405
|
let err = encoder
|
|
406
406
|
.encode_with_max_size(&buffer, local_hint)
|
|
407
407
|
.unwrap_err();
|
|
408
|
-
assert_eq!(err.code(), ErrorCode::
|
|
408
|
+
assert_eq!(err.code(), ErrorCode::BatchTooLarge);
|
|
409
409
|
assert!(err.msg().contains("QWP/WebSocket encoded message size"));
|
|
410
410
|
assert_eq!(encoder.global_dict.len(), seeded_symbols);
|
|
411
411
|
|