questdb 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. data/.cargo/audit.toml +17 -9
  3. data/CHANGELOG.md +7 -0
  4. data/README.md +1 -1
  5. data/THIRD_PARTY_NOTICES.md +2 -2
  6. data/docs/configuration.md +3 -2
  7. data/docs/design.md +1 -1
  8. data/docs/python-parity.md +1 -1
  9. data/docs/releasing.md +1 -1
  10. data/ext/questdb_ingress/extconf.rb +2 -0
  11. data/ext/questdb_ingress/questdb_ingress.c +2 -1
  12. data/ext/questdb_ingress/unlink_build_ruby.rb +58 -0
  13. data/lib/questdb/arrow.rb +2 -0
  14. data/lib/questdb/client.rb +6 -0
  15. data/lib/questdb/config.rb +19 -0
  16. data/lib/questdb/pooled_sender.rb +4 -1
  17. data/lib/questdb/sender.rb +10 -1
  18. data/lib/questdb/version.rb +2 -2
  19. data/script/check-native-linkage +115 -0
  20. data/test/client_test.rb +30 -0
  21. data/test/extconf_link_flags_test.rb +160 -0
  22. data/vendor/c-questdb-client/.bumpversion.toml +1 -1
  23. data/vendor/c-questdb-client/CMakeLists.txt +1 -1
  24. data/vendor/c-questdb-client/cpp_test/qwp_mock_server.hpp +10 -1
  25. data/vendor/c-questdb-client/cpp_test/test_arrow_c.c +39 -0
  26. data/vendor/c-questdb-client/cpp_test/test_reader_mock.cpp +428 -0
  27. data/vendor/c-questdb-client/doc/COMPATIBILITY.md +1 -1
  28. data/vendor/c-questdb-client/doc/CONSIDERATIONS.md +6 -3
  29. data/vendor/c-questdb-client/doc/RELEASE_RUNBOOK.md +7 -7
  30. data/vendor/c-questdb-client/doc/RELEASING.md +6 -6
  31. data/vendor/c-questdb-client/include/questdb/egress/qwp_reader.h +48 -1
  32. data/vendor/c-questdb-client/include/questdb/egress/qwp_reader.hpp +52 -0
  33. data/vendor/c-questdb-client/include/questdb/ingress/line_sender.h +46 -4
  34. data/vendor/c-questdb-client/include/questdb/ingress/line_sender.hpp +7 -4
  35. data/vendor/c-questdb-client/include/questdb/ingress/line_sender_core.hpp +1 -0
  36. data/vendor/c-questdb-client/questdb-rs/Cargo.toml +1 -1
  37. data/vendor/c-questdb-client/questdb-rs/README.md +13 -13
  38. data/vendor/c-questdb-client/questdb-rs/src/egress/config.rs +78 -0
  39. data/vendor/c-questdb-client/questdb-rs/src/egress/mod.rs +2 -2
  40. data/vendor/c-questdb-client/questdb-rs/src/egress/query_request.rs +174 -1
  41. data/vendor/c-questdb-client/questdb-rs/src/egress/reader.rs +264 -14
  42. data/vendor/c-questdb-client/questdb-rs/src/egress/wire/capabilities.rs +69 -0
  43. data/vendor/c-questdb-client/questdb-rs/src/egress/wire/msg_kind.rs +33 -0
  44. data/vendor/c-questdb-client/questdb-rs/src/error.rs +33 -5
  45. data/vendor/c-questdb-client/questdb-rs/src/ingress/buffer/qwp.rs +96 -11
  46. data/vendor/c-questdb-client/questdb-rs/src/ingress/buffer.rs +6 -3
  47. data/vendor/c-questdb-client/questdb-rs/src/ingress/column_sender/sender.rs +1 -16
  48. data/vendor/c-questdb-client/questdb-rs/src/ingress/sender/qwp_ws.rs +19 -13
  49. data/vendor/c-questdb-client/questdb-rs/src/ingress/sender/qwp_ws_driver.rs +2 -9
  50. data/vendor/c-questdb-client/questdb-rs/src/ingress/sender/qwp_ws_publisher.rs +3 -3
  51. data/vendor/c-questdb-client/questdb-rs/src/ingress/sender.rs +101 -17
  52. data/vendor/c-questdb-client/questdb-rs/src/ingress/tests.rs +32 -0
  53. data/vendor/c-questdb-client/questdb-rs/src/ingress.rs +3 -2
  54. data/vendor/c-questdb-client/questdb-rs/src/tests/qwp_ws.rs +96 -16
  55. data/vendor/c-questdb-client/questdb-rs/tests/egress_failover.rs +219 -1
  56. data/vendor/c-questdb-client/questdb-rs/tests/egress_live_server.rs +34 -0
  57. data/vendor/c-questdb-client/questdb-rs-ffi/Cargo.lock +2 -2
  58. data/vendor/c-questdb-client/questdb-rs-ffi/Cargo.toml +1 -1
  59. data/vendor/c-questdb-client/questdb-rs-ffi/src/egress.rs +31 -0
  60. data/vendor/c-questdb-client/questdb-rs-ffi/src/lib.rs +70 -3
  61. data/vendor/c-questdb-client/system_test/arrow_ffi.py +1 -0
  62. data/vendor/c-questdb-client/system_test/enterprise_e2e/tests/test_failover.py +12 -6
  63. data/vendor/c-questdb-client/system_test/enterprise_e2e/tests/test_orphan_drainer_bindings.py +12 -4
  64. data/vendor/c-questdb-client/system_test/failover_clients/Cargo.lock +1 -1
  65. data/vendor/c-questdb-client/system_test/soak/workload_rs/Cargo.lock +1 -1
  66. data/vendor/c-questdb-client.version +1 -1
  67. metadata +5 -2
@@ -51,6 +51,28 @@ pub fn has_query_flags(capabilities: u32) -> bool {
51
51
  capabilities & CAP_QUERY_FLAGS != 0
52
52
  }
53
53
 
54
+ /// Bit `0x04` is `CAP_COMPRESSION` on the server (zstd frame compression,
55
+ /// negotiated through the `X-QWP-Accept-Encoding` upgrade header rather than
56
+ /// through this word on the client side). Named here so the next capability
57
+ /// does not reuse the value.
58
+ pub const CAP_COMPRESSION: u32 = 0x0000_0004;
59
+
60
+ /// Server honours `QUERY_REQUEST.timeout_ms`: it runs the query under the
61
+ /// client-supplied timeout instead of the server-wide `query.timeout`, and
62
+ /// reports expiry as [`StatusCode::QueryTimeout`].
63
+ ///
64
+ /// Clients append `timeout_ms` only when this bit is set, so a server
65
+ /// without it never receives the field. The server sets no ceiling, so a
66
+ /// client timeout may legitimately exceed `query.timeout`.
67
+ ///
68
+ /// [`StatusCode::QueryTimeout`]: super::msg_kind::StatusCode::QueryTimeout
69
+ pub const CAP_QUERY_TIMEOUT: u32 = 0x0000_0008;
70
+
71
+ /// True if the server honours the `QUERY_REQUEST` `timeout_ms` field.
72
+ pub fn has_query_timeout(capabilities: u32) -> bool {
73
+ capabilities & CAP_QUERY_TIMEOUT != 0
74
+ }
75
+
54
76
  #[cfg(test)]
55
77
  mod tests {
56
78
  use super::*;
@@ -64,4 +86,51 @@ mod tests {
64
86
  // Future bits with CAP_ZONE clear must not trip it.
65
87
  assert!(!has_zone(0xFFFF_FFFE));
66
88
  }
89
+
90
+ #[test]
91
+ fn has_query_flags_predicate() {
92
+ assert!(!has_query_flags(0));
93
+ assert!(has_query_flags(CAP_QUERY_FLAGS));
94
+ assert!(has_query_flags(CAP_QUERY_FLAGS | 0x8000_0000));
95
+ assert!(!has_query_flags(!CAP_QUERY_FLAGS));
96
+ }
97
+
98
+ #[test]
99
+ fn has_query_timeout_predicate() {
100
+ assert!(!has_query_timeout(0));
101
+ assert!(has_query_timeout(CAP_QUERY_TIMEOUT));
102
+ // The timeout bit rides alongside the flags bit on a current
103
+ // server; neither predicate may depend on the other's absence.
104
+ assert!(has_query_timeout(CAP_QUERY_TIMEOUT | CAP_QUERY_FLAGS));
105
+ assert!(has_query_flags(CAP_QUERY_TIMEOUT | CAP_QUERY_FLAGS));
106
+ assert!(!has_query_timeout(!CAP_QUERY_TIMEOUT));
107
+ }
108
+
109
+ #[test]
110
+ fn capability_bits_are_distinct_powers_of_two() {
111
+ // The server assigns these; a duplicated or non-power-of-two value
112
+ // would make one predicate answer for another capability.
113
+ let bits = [
114
+ CAP_ZONE,
115
+ CAP_QUERY_FLAGS,
116
+ CAP_COMPRESSION,
117
+ CAP_QUERY_TIMEOUT,
118
+ ];
119
+ let mut seen = 0u32;
120
+ for b in bits {
121
+ assert_eq!(b.count_ones(), 1, "0x{b:08X} is not a single bit");
122
+ assert_eq!(seen & b, 0, "0x{b:08X} duplicates an earlier bit");
123
+ seen |= b;
124
+ }
125
+ // Pin the wire values: the server's QwpConstants owns them.
126
+ assert_eq!(
127
+ (
128
+ CAP_ZONE,
129
+ CAP_QUERY_FLAGS,
130
+ CAP_COMPRESSION,
131
+ CAP_QUERY_TIMEOUT
132
+ ),
133
+ (0x01, 0x02, 0x04, 0x08)
134
+ );
135
+ }
67
136
  }
@@ -93,6 +93,12 @@ pub enum StatusCode {
93
93
  SecurityError = 0x08,
94
94
  Cancelled = 0x0A,
95
95
  LimitExceeded = 0x0B,
96
+ /// The query ran past the `timeout_ms` it carried. Per-query, not
97
+ /// per-connection: the connection stays open and authenticated, so a
98
+ /// pooled client runs its next query on it instead of reconnecting.
99
+ /// Only sent to clients that supplied a timeout; a request without one
100
+ /// gets [`Self::LimitExceeded`] for the same condition.
101
+ QueryTimeout = 0x0E,
96
102
  }
97
103
 
98
104
  impl StatusCode {
@@ -104,6 +110,7 @@ impl StatusCode {
104
110
  0x08 => StatusCode::SecurityError,
105
111
  0x0A => StatusCode::Cancelled,
106
112
  0x0B => StatusCode::LimitExceeded,
113
+ 0x0E => StatusCode::QueryTimeout,
107
114
  other => {
108
115
  return Err(fmt!(
109
116
  ProtocolError,
@@ -157,8 +164,34 @@ mod tests {
157
164
  StatusCode::SecurityError,
158
165
  StatusCode::Cancelled,
159
166
  StatusCode::LimitExceeded,
167
+ StatusCode::QueryTimeout,
160
168
  ] {
161
169
  assert_eq!(StatusCode::from_u8(s.as_u8()).unwrap(), s);
162
170
  }
163
171
  }
172
+
173
+ #[test]
174
+ fn status_code_wire_values_pinned() {
175
+ // The server's QwpConstants owns these numbers; a drifted
176
+ // discriminant would mis-report one server error as another.
177
+ assert_eq!(StatusCode::SchemaMismatch.as_u8(), 0x03);
178
+ assert_eq!(StatusCode::ParseError.as_u8(), 0x05);
179
+ assert_eq!(StatusCode::InternalError.as_u8(), 0x06);
180
+ assert_eq!(StatusCode::SecurityError.as_u8(), 0x08);
181
+ assert_eq!(StatusCode::Cancelled.as_u8(), 0x0A);
182
+ assert_eq!(StatusCode::LimitExceeded.as_u8(), 0x0B);
183
+ assert_eq!(StatusCode::QueryTimeout.as_u8(), 0x0E);
184
+ }
185
+
186
+ #[test]
187
+ fn unknown_status_code_rejected() {
188
+ // 0x0C / 0x0D sit between LimitExceeded and QueryTimeout and are
189
+ // unassigned: they must not silently decode as a neighbour.
190
+ for byte in [0x00, 0x04, 0x0C, 0x0D, 0x0F, 0xFF] {
191
+ assert!(
192
+ StatusCode::from_u8(byte).is_err(),
193
+ "0x{byte:02X} must not decode"
194
+ );
195
+ }
196
+ }
164
197
  }
@@ -140,6 +140,12 @@ pub enum ErrorCode {
140
140
  HandshakeError,
141
141
 
142
142
  /// Server returned an unsupported QWP version, encoding, or capability.
143
+ ///
144
+ /// Also raised client-side, before anything is written, when a query
145
+ /// carries a per-query timeout and the server does not advertise
146
+ /// `CAP_QUERY_TIMEOUT` — see
147
+ /// [`ReaderQuery::timeout`](crate::egress::ReaderQuery::timeout). The
148
+ /// connection is untouched by that refusal.
143
149
  UnsupportedServer,
144
150
 
145
151
  /// Wire-format violation: bad magic, truncated frame, unknown discriminant,
@@ -174,6 +180,24 @@ pub enum ErrorCode {
174
180
  /// Query was cancelled (locally or via server `CANCELLED` status `0x0A`).
175
181
  Cancelled,
176
182
 
183
+ /// The query ran past the timeout it carried: server-reported QWP
184
+ /// `QUERY_TIMEOUT` (status `0x0E`), produced only for a request that
185
+ /// supplied `timeout_ms`. Per-query, so the connection stays usable and
186
+ /// a pooled client runs its next query on it.
187
+ ///
188
+ /// A timeout requested against a server that does not advertise
189
+ /// `CAP_QUERY_TIMEOUT` is refused with [`Self::UnsupportedServer`]
190
+ /// instead, so a caller that retries a timeout with a larger budget
191
+ /// does not loop against a server that ignores every budget — see
192
+ /// [`ReaderQuery::timeout`](crate::egress::ReaderQuery::timeout).
193
+ ///
194
+ /// **Do not retry a write on this error.** A DDL / `INSERT` / `UPDATE`
195
+ /// that timed out waiting for the table writer may still be applied
196
+ /// afterwards, and a statement that completes past its timeout is
197
+ /// reported as `EXEC_DONE` rather than a timeout precisely so a
198
+ /// retrying client cannot apply it twice.
199
+ QueryTimeout,
200
+
177
201
  /// Mid-query failover was eligible but at least one batch had already
178
202
  /// been delivered to the caller, and the cursor's `on_failover_reset`
179
203
  /// callback was not installed. Failover would replay the query from the
@@ -202,11 +226,14 @@ pub enum ErrorCode {
202
226
  /// bug; not user-recoverable. Only emitted on the `arrow` feature.
203
227
  ArrowExport,
204
228
 
205
- /// An irreducible QWP/WebSocket unit (the table schema plus a single
206
- /// row block) exceeds the negotiated per-batch cap
207
- /// (`min(max_buf_size, server X-QWP-Max-Batch-Size)`). Chunk publication
208
- /// splits oversize inputs into smaller frames automatically, so this only
209
- /// surfaces when splitting cannot make a frame fit. Distinct from
229
+ /// A QWP/WebSocket frame exceeds the negotiated per-frame cap
230
+ /// (`min(max_buf_size, server X-QWP-Max-Batch-Size)`, further bounded by
231
+ /// the store-and-forward segment payload capacity). Chunk publication
232
+ /// splits oversize inputs into smaller frames automatically, so there this
233
+ /// only surfaces for an irreducible unit (the table schema plus a single
234
+ /// row block). A row [`Buffer`](crate::ingress::Buffer) is published as one
235
+ /// frame and is never split, so flushing one whose encoded frame exceeds
236
+ /// the cap fails with this code and publishes nothing. Distinct from
210
237
  /// [`InvalidApiCall`](Self::InvalidApiCall) so callers can recognise it
211
238
  /// without matching on the error message text.
212
239
  BatchTooLarge,
@@ -608,6 +635,7 @@ mod tests {
608
635
  ErrorCode::LimitExceeded => {}
609
636
  ErrorCode::ServerLimitExceeded => {}
610
637
  ErrorCode::Cancelled => {}
638
+ ErrorCode::QueryTimeout => {}
611
639
  ErrorCode::FailoverWouldDuplicate => {}
612
640
  ErrorCode::SchemaDrift => {}
613
641
  ErrorCode::NoSchema => {}
@@ -2559,9 +2559,8 @@ impl QwpWsSizeHint {
2559
2559
  }
2560
2560
 
2561
2561
  QWP_MESSAGE_HEADER_SIZE
2562
- + 2
2563
2562
  + self.encoded_tables_len
2564
- + qwp_varint_size(0)
2563
+ + QWP_WS_SYMBOL_ID_MAX_LEN
2565
2564
  + qwp_varint_size(self.symbol_dict_count as u64)
2566
2565
  + self.symbol_dict_bytes
2567
2566
  }
@@ -2831,7 +2830,7 @@ impl QwpWsColumnarBuffer {
2831
2830
 
2832
2831
  #[cfg(test)]
2833
2832
  fn recompute_len_slow(&self) -> usize {
2834
- let mut total = QWP_MESSAGE_HEADER_SIZE + 2;
2833
+ let mut total = QWP_MESSAGE_HEADER_SIZE;
2835
2834
  let mut symbol_dict_count = 0usize;
2836
2835
  let mut symbol_dict_bytes = 0usize;
2837
2836
  for table in self.tables.iter().filter(|table| table.row_count > 0) {
@@ -2854,7 +2853,10 @@ impl QwpWsColumnarBuffer {
2854
2853
  }
2855
2854
  }
2856
2855
  }
2857
- total + qwp_varint_size(0) + qwp_varint_size(symbol_dict_count as u64) + symbol_dict_bytes
2856
+ total
2857
+ + QWP_WS_SYMBOL_ID_MAX_LEN
2858
+ + qwp_varint_size(symbol_dict_count as u64)
2859
+ + symbol_dict_bytes
2858
2860
  }
2859
2861
 
2860
2862
  #[cfg(test)]
@@ -4127,7 +4129,7 @@ impl QwpWsColumnBuffer {
4127
4129
  fn rollback_rows_from(&mut self, from: u32) {
4128
4130
  if let QwpWsColumnValues::Symbol { cells, dict, .. } = &self.values {
4129
4131
  for cell in cells.iter().rev().take_while(|cell| cell.row_idx >= from) {
4130
- self.symbol_cells_encoded_len -= qwp_varint_size(cell.local_id as u64);
4132
+ self.symbol_cells_encoded_len -= QWP_WS_SYMBOL_ID_MAX_LEN;
4131
4133
  if cell.is_new {
4132
4134
  let entry = &dict[cell.local_id as usize];
4133
4135
  self.symbol_dict_encoded_len -= qwp_string_byte_len(entry.len as usize);
@@ -4317,7 +4319,7 @@ impl QwpWsColumnBuffer {
4317
4319
  is_new,
4318
4320
  });
4319
4321
  self.increment_non_null()?;
4320
- self.symbol_cells_encoded_len += qwp_varint_size(local_id as u64);
4322
+ self.symbol_cells_encoded_len += QWP_WS_SYMBOL_ID_MAX_LEN;
4321
4323
  if is_new {
4322
4324
  self.symbol_dict_encoded_len += qwp_string_byte_len(bytes.len());
4323
4325
  }
@@ -4881,10 +4883,7 @@ impl QwpWsColumnValues {
4881
4883
  cells.len().saturating_mul(8)
4882
4884
  }
4883
4885
  Self::String { cells, data } => (cells.len() + 1).saturating_mul(4) + data.len(),
4884
- Self::Symbol { cells, .. } => cells
4885
- .iter()
4886
- .map(|cell| qwp_varint_size(cell.local_id as u64))
4887
- .sum(),
4886
+ Self::Symbol { cells, .. } => cells.len().saturating_mul(QWP_WS_SYMBOL_ID_MAX_LEN),
4888
4887
  Self::Decimal { cells, .. } => {
4889
4888
  1 + cells
4890
4889
  .iter()
@@ -5502,6 +5501,15 @@ impl Default for SymbolGlobalDict {
5502
5501
  #[cfg(feature = "_sender-qwp-ws")]
5503
5502
  pub(crate) const MAX_CONN_SYMBOL_DICT_SIZE: usize = 2_000_000;
5504
5503
 
5504
+ /// Bytes the QWP/WS size hint charges for each connection-wide symbol id a
5505
+ /// frame carries: every symbol cell, and the delta dictionary's start id.
5506
+ /// The buffer cannot see those ids until the flush interns its symbols, so
5507
+ /// it charges the widest varint an id can take. Ids stay below
5508
+ /// [`MAX_CONN_SYMBOL_DICT_SIZE`] and the start id reaches it at most, which
5509
+ /// keeps the hint at or above the encoded frame.
5510
+ #[cfg(feature = "_sender-qwp-ws")]
5511
+ const QWP_WS_SYMBOL_ID_MAX_LEN: usize = qwp_varint_size(MAX_CONN_SYMBOL_DICT_SIZE as u64);
5512
+
5505
5513
  /// Per-connection cap on the cumulative UTF-8 heap (in bytes) the QWP/WS global
5506
5514
  /// symbol dictionary holds. Matches `MAX_CONN_DICT_HEAP_BYTES` in the egress
5507
5515
  /// reader (`egress/symbol_dict.rs`) and the Java reference client. The entry
@@ -7421,7 +7429,7 @@ fn qwp_string_byte_len(byte_len: usize) -> usize {
7421
7429
  qwp_varint_size(byte_len as u64) + byte_len
7422
7430
  }
7423
7431
 
7424
- fn qwp_varint_size(mut value: u64) -> usize {
7432
+ const fn qwp_varint_size(mut value: u64) -> usize {
7425
7433
  let mut size = 1usize;
7426
7434
  while value > 0x7F {
7427
7435
  value >>= 7;
@@ -10562,6 +10570,83 @@ mod tests {
10562
10570
  assert_eq!(buf.len(), buf.recompute_len_slow());
10563
10571
  }
10564
10572
 
10573
+ #[cfg(feature = "_sender-qwp-ws")]
10574
+ #[test]
10575
+ fn qwp_ws_size_hint_covers_encoded_frame_when_symbol_ids_widen() {
10576
+ // A frame carries each symbol cell as its connection-wide id. The
10577
+ // 500-value column comes first, so the other columns' ids start past
10578
+ // its own, and the connection may have interned symbols before.
10579
+ fn add_rows(buf: &mut QwpWsColumnarBuffer, rows: std::ops::Range<i64>) {
10580
+ for i in rows {
10581
+ let row = buf.table("trades").unwrap();
10582
+ row.symbol("ticker", format!("T{:03}", (i * 7919) % 500).as_str())
10583
+ .unwrap();
10584
+ row.symbol(
10585
+ "exchange",
10586
+ ["NYSE", "NASDAQ", "ARCA", "BATS"][(i % 4) as usize],
10587
+ )
10588
+ .unwrap();
10589
+ row.symbol("side", ["buy", "sell"][((i / 3) % 2) as usize])
10590
+ .unwrap();
10591
+ row.symbol("ccy", ["USD", "EUR", "GBP"][((i / 5) % 3) as usize])
10592
+ .unwrap();
10593
+ if i % 3 != 0 {
10594
+ row.symbol("venue", format!("V{}", (i / 7) % 8).as_str())
10595
+ .unwrap();
10596
+ }
10597
+ row.column_f64("price", i as f64 * 0.5).unwrap();
10598
+ row.at_now().unwrap();
10599
+ }
10600
+ }
10601
+
10602
+ fn assert_covers(
10603
+ buf: &QwpWsColumnarBuffer,
10604
+ global_dict: &mut SymbolGlobalDict,
10605
+ warm: usize,
10606
+ what: &str,
10607
+ ) {
10608
+ let mut scratch = QwpWsEncodeScratch::new();
10609
+ let mut out = Vec::new();
10610
+ buf.encode_ws_replay_message_with_defer(
10611
+ &mut out,
10612
+ &mut scratch,
10613
+ global_dict,
10614
+ QWP_VERSION_1,
10615
+ false,
10616
+ true,
10617
+ )
10618
+ .unwrap();
10619
+ assert!(
10620
+ out.len() <= buf.len(),
10621
+ "{what} after {warm} interned symbols: encoded {} bytes, size hint {}",
10622
+ out.len(),
10623
+ buf.len()
10624
+ );
10625
+ }
10626
+
10627
+ for warm in [0usize, 127, 128, 16_383, 16_384] {
10628
+ let mut global_dict = SymbolGlobalDict::new();
10629
+ for i in 0..warm {
10630
+ global_dict.intern(format!("w{i:05}").as_bytes()).unwrap();
10631
+ }
10632
+ let mut buf = QwpWsColumnarBuffer::new(127);
10633
+
10634
+ add_rows(&mut buf, 0..600);
10635
+ assert_covers(&buf, &mut global_dict, warm, "new symbols");
10636
+
10637
+ buf.clear();
10638
+ add_rows(&mut buf, 600..1200);
10639
+ assert_covers(&buf, &mut global_dict, warm, "known symbols");
10640
+
10641
+ let before_marker = buf.len();
10642
+ buf.set_marker().unwrap();
10643
+ add_rows(&mut buf, 1200..1201);
10644
+ buf.rewind_to_marker().unwrap();
10645
+ assert_eq!(buf.len(), before_marker);
10646
+ assert_covers(&buf, &mut global_dict, warm, "after a rewind");
10647
+ }
10648
+ }
10649
+
10565
10650
  #[cfg(feature = "_sender-qwp-ws")]
10566
10651
  #[test]
10567
10652
  fn qwp_ws_cached_size_hint_recomputes_only_dirty_tables() {
@@ -549,9 +549,12 @@ impl Buffer {
549
549
  ///
550
550
  /// For ILP buffers this is the exact serialized byte count. For QWP/UDP
551
551
  /// buffers this is the size hint used for flush planning. For QWP/WebSocket
552
- /// buffers this is only a local size hint; symbol-ID remapping and replay
553
- /// dictionary state can change the eventual frame size. The sender
554
- /// enforces `max_buf_size` against the encoded replay message.
552
+ /// buffers this is a size hint that is at least the size of the frame a
553
+ /// flush encodes: it charges each symbol value the widest connection-wide
554
+ /// id. It falls short only once store-and-forward sends full symbol
555
+ /// dictionaries, which it does when its symbol side-file cannot be opened
556
+ /// or rolled back. The sender enforces `max_buf_size` against the encoded
557
+ /// replay message.
555
558
  pub fn len(&self) -> usize {
556
559
  match &self.inner {
557
560
  BufferInner::Ilp(inner) => inner.len(),
@@ -33,6 +33,7 @@ use crate::ErrorCode;
33
33
  use crate::ingress::AckLevel;
34
34
  use crate::ingress::QwpWsSenderError;
35
35
  use crate::ingress::buffer::{Buffer, QwpWsColumnarBuffer, QwpWsEncodeScratch, SymbolGlobalDict};
36
+ use crate::ingress::sender::effective_hard_frame_cap;
36
37
  use crate::ingress::sender::qwp_ws::{
37
38
  SyncQwpWsHandlerState, publish_qwp_ws_payload_background, qwp_ws_acked_fsn_background,
38
39
  qwp_ws_begin_close_background, qwp_ws_check_error_background,
@@ -2250,22 +2251,6 @@ impl SfaBackend {
2250
2251
  }
2251
2252
  }
2252
2253
 
2253
- fn effective_hard_frame_cap(
2254
- max_buf_size: usize,
2255
- server_max_batch_size: usize,
2256
- sfa_frame_payload_cap: usize,
2257
- ) -> (usize, bool) {
2258
- let configured_cap = if server_max_batch_size == 0 {
2259
- max_buf_size
2260
- } else {
2261
- max_buf_size.min(server_max_batch_size)
2262
- };
2263
- (
2264
- configured_cap.min(sfa_frame_payload_cap),
2265
- server_max_batch_size != 0,
2266
- )
2267
- }
2268
-
2269
2254
  /// The store-and-forward `wait` poll loop made no progress toward `boundary`
2270
2255
  /// for `sync_timeout`. The connection is alive but the server is not advancing
2271
2256
  /// the ack/durable watermark (e.g. back-pressured WAL or a stuck commit).
@@ -108,14 +108,12 @@ pub(crate) struct TrafficGate {
108
108
  #[derive(Default)]
109
109
  struct TrafficGateState {
110
110
  current: Option<TcpStream>,
111
- /// Raw handle of the ORIGINAL socket, not the `current` dup. Winsock
112
- /// `shutdown()` does not unblock a `recv()` already in progress on
113
- /// another thread, so `shutdown()` first runs `CancelIoEx` — and that
114
- /// must target the handle the blocked thread issued its I/O on; a
115
- /// `try_clone` (WSADuplicateSocket) handle is not guaranteed to share
116
- /// the file object. Only valid while `current` is `Some`: every close
117
- /// of the original is mutex-ordered behind `clear()`/re-register, so
118
- /// the handle cannot be recycled while the gate still holds it.
111
+ /// Raw handle of the ORIGINAL socket, not the `current` dup: a shutdown
112
+ /// through the `try_clone` (WSADuplicateSocket) dup can leave later
113
+ /// `recv()`s on the original blocking. Only valid while `current` is
114
+ /// `Some`: every close of the original is mutex-ordered behind
115
+ /// `clear()`/re-register, so the handle cannot be recycled while the
116
+ /// gate still holds it.
119
117
  #[cfg(windows)]
120
118
  original: Option<std::os::windows::io::RawSocket>,
121
119
  shut: bool,
@@ -222,15 +220,19 @@ impl TrafficGate {
222
220
  state.shut = true;
223
221
  #[cfg(windows)]
224
222
  if let Some(original) = state.original.take() {
225
- // Winsock shutdown() does not unblock an in-progress recv() on
226
- // another thread. Cancel outstanding I/O on the worker's own
227
- // handle first, mirroring the Java client's windows/net.c
228
- // shutdown(). A failure with ERROR_NOT_FOUND (nothing pending)
229
- // is expected and deliberately ignored.
223
+ use std::os::windows::io::FromRawSocket;
230
224
  use windows_sys::Win32::Foundation::HANDLE;
225
+ // Shutdown does not wake an in-flight recv(), so CancelIoEx
226
+ // follows it; the reverse order lets a recv() slip in between.
227
+ // SAFETY: `original` is open while `current` is `Some`.
228
+ let original_stream =
229
+ std::mem::ManuallyDrop::new(unsafe { TcpStream::from_raw_socket(original) });
230
+ let shut = shutdown_socket(&original_stream);
231
231
  unsafe {
232
232
  windows_sys::Win32::System::IO::CancelIoEx(original as HANDLE, std::ptr::null());
233
233
  }
234
+ state.current = None;
235
+ return shut;
234
236
  }
235
237
  if let Some(stream) = state.current.take() {
236
238
  shutdown_socket(&stream)?;
@@ -511,6 +513,9 @@ pub(crate) struct ManualQwpWsHandlerState {
511
513
  send_core: QwpWsSendCore<BlockingQwpWsTransport>,
512
514
  pub(crate) server_max_batch_size: Arc<AtomicUsize>,
513
515
  pub(crate) request_durable_ack: bool,
516
+ /// Hard cap on a single queued frame's payload, as on
517
+ /// [`SyncQwpWsHandlerState::sfa_frame_payload_cap`].
518
+ pub(crate) sfa_frame_payload_cap: usize,
514
519
  orphan_drainers: Option<ManualOrphanDrainers>,
515
520
  append_deadline: Duration,
516
521
  close_drain_timeout: Duration,
@@ -3698,6 +3703,7 @@ pub(crate) fn open_manual_qwp_ws(
3698
3703
  send_core: parts.send_core,
3699
3704
  server_max_batch_size,
3700
3705
  request_durable_ack: *qwp_ws.request_durable_ack,
3706
+ sfa_frame_payload_cap: segment_payload_capacity(*qwp_ws.sf_max_segment_bytes),
3701
3707
  orphan_drainers,
3702
3708
  append_deadline: *qwp_ws.sf_append_deadline,
3703
3709
  close_drain_timeout: *qwp_ws.close_flush_timeout,
@@ -5774,15 +5774,8 @@ mod tests {
5774
5774
  .unwrap();
5775
5775
  let _server_streams = server.join().unwrap();
5776
5776
 
5777
- // A short read timeout plus a retry loop instead of one indefinitely
5778
- // blocking read: on Windows the gate's CancelIoEx only cancels a recv
5779
- // already in flight, and Winsock shutdown() does not wake one entered
5780
- // afterwards, so a single read can straddle the shutdown and miss
5781
- // both wake-ups. Retrying sidesteps the race: once the gate has shut
5782
- // the replacement socket down, the next read attempt fails
5783
- // immediately. A missing reconnect registration still fails the test,
5784
- // because the gate never touches this socket and every attempt times
5785
- // out until the deadline.
5777
+ // Retry short reads so a missed wake-up fails at the deadline
5778
+ // instead of hanging the test.
5786
5779
  transport
5787
5780
  .stream
5788
5781
  .set_timeouts(Some(Duration::from_millis(100)), None)
@@ -275,8 +275,8 @@ pub(crate) fn qwp_ws_encoded_message_size_error(
275
275
  max_buf_size: usize,
276
276
  ) -> crate::Error {
277
277
  error::fmt!(
278
- InvalidApiCall,
279
- "Could not flush buffer: QWP/WebSocket encoded message size of {} exceeds maximum configured allowed size of {} bytes.",
278
+ BatchTooLarge,
279
+ "Could not flush buffer: QWP/WebSocket encoded message size of {} bytes exceeds the per-frame cap of {} bytes.",
280
280
  encoded_len,
281
281
  max_buf_size
282
282
  )
@@ -405,7 +405,7 @@ mod tests {
405
405
  let err = encoder
406
406
  .encode_with_max_size(&buffer, local_hint)
407
407
  .unwrap_err();
408
- assert_eq!(err.code(), ErrorCode::InvalidApiCall);
408
+ assert_eq!(err.code(), ErrorCode::BatchTooLarge);
409
409
  assert!(err.msg().contains("QWP/WebSocket encoded message size"));
410
410
  assert_eq!(encoder.global_dict.len(), seeded_symbols);
411
411