prosody 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.release-please-manifest.json +1 -1
- data/ARCHITECTURE.md +12 -2
- data/CHANGELOG.md +13 -0
- data/Cargo.lock +571 -435
- data/Cargo.toml +6 -6
- data/README.md +211 -20
- data/Rakefile +11 -1
- data/examples/keyed_state.rb +58 -0
- data/examples/keyed_state.rbs +18 -0
- data/examples/keyed_state_windowing.rb +47 -0
- data/examples/keyed_state_windowing.rbs +16 -0
- data/ext/prosody/Cargo.toml +1 -1
- data/ext/prosody/src/client/config.rs +387 -19
- data/ext/prosody/src/handler/context.rs +146 -5
- data/ext/prosody/src/handler/message.rs +17 -0
- data/ext/prosody/src/handler/mod.rs +4 -2
- data/ext/prosody/src/handler/state.rs +1035 -0
- data/lib/prosody/configuration.rb +26 -0
- data/lib/prosody/handler.rb +6 -2
- data/lib/prosody/native_stubs.rb +381 -6
- data/lib/prosody/state.rb +693 -0
- data/lib/prosody/version.rb +1 -1
- data/lib/prosody.rb +5 -0
- data/release-please-config.json +4 -0
- data/sig/configuration.rbs +24 -1
- data/sig/handler.rbs +7 -3
- data/sig/processor.rbs +28 -12
- data/sig/prosody.rbs +11 -6
- data/sig/sentry.rbs +6 -0
- data/sig/state.rbs +272 -0
- data/steep_expectations.yml +47 -0
- data/typecheck/payload_types.rb +43 -0
- data/typecheck/payload_types.rbs +20 -0
- data/typecheck_negative/payload_types.rb +16 -0
- data/typecheck_negative/payload_types.rbs +8 -0
- metadata +22 -9
|
@@ -7,9 +7,12 @@
|
|
|
7
7
|
//! builders.
|
|
8
8
|
|
|
9
9
|
use magnus::{Error, Ruby, Value};
|
|
10
|
+
use prosody::JsonCodec;
|
|
10
11
|
use prosody::cassandra::config::CassandraConfigurationBuilder;
|
|
11
12
|
use prosody::consumer::ConsumerConfigurationBuilder;
|
|
13
|
+
use prosody::consumer::KeyedStateConfiguration;
|
|
12
14
|
use prosody::consumer::SpanRelation;
|
|
15
|
+
use prosody::consumer::kafka_state::{message_deque_state, message_map_state, message_state};
|
|
13
16
|
use prosody::consumer::middleware::deduplication::DeduplicationConfigurationBuilder;
|
|
14
17
|
use prosody::consumer::middleware::defer::DeferConfigurationBuilder;
|
|
15
18
|
use prosody::consumer::middleware::monopolization::MonopolizationConfigurationBuilder;
|
|
@@ -19,11 +22,21 @@ use prosody::consumer::middleware::timeout::TimeoutConfigurationBuilder;
|
|
|
19
22
|
use prosody::consumer::middleware::topic::FailureTopicConfigurationBuilder;
|
|
20
23
|
use prosody::high_level::ConsumerBuilders;
|
|
21
24
|
use prosody::high_level::mode::Mode;
|
|
25
|
+
use prosody::loader::KafkaLoader;
|
|
26
|
+
use prosody::loader::KafkaLoaderConfiguration;
|
|
22
27
|
use prosody::producer::ProducerConfigurationBuilder;
|
|
28
|
+
use prosody::state::descriptor::{
|
|
29
|
+
DequeDescriptor, MapDescriptor, StateDescriptor, deque_state, map_state, value_state,
|
|
30
|
+
};
|
|
31
|
+
use prosody::state::order_codec::Utf8KeyCodec;
|
|
23
32
|
use prosody::telemetry::emitter::TelemetryEmitterConfiguration;
|
|
33
|
+
use prosody::timers::duration::CompactDuration;
|
|
24
34
|
use serde::{Deserialize, Deserializer};
|
|
25
35
|
use serde_magnus::deserialize;
|
|
26
36
|
use serde_untagged::UntaggedEnumVisitor;
|
|
37
|
+
use std::collections::HashSet;
|
|
38
|
+
use std::num::{NonZeroU64, NonZeroUsize};
|
|
39
|
+
use std::path::PathBuf;
|
|
27
40
|
use std::time::Duration;
|
|
28
41
|
|
|
29
42
|
/// Configuration structure for the Prosody client that maps Ruby configuration
|
|
@@ -202,6 +215,51 @@ pub struct NativeConfiguration {
|
|
|
202
215
|
|
|
203
216
|
/// Span linking for timer execution spans (`child` or `follows_from`).
|
|
204
217
|
timer_spans: Option<String>,
|
|
218
|
+
|
|
219
|
+
// Keyed-state configuration
|
|
220
|
+
/// Keyed-state collections to register before subscribe.
|
|
221
|
+
state_collections: Option<Vec<StateCollectionConfig>>,
|
|
222
|
+
|
|
223
|
+
/// Root directory for the local keyed-state cache. Must not be empty.
|
|
224
|
+
state_cache_dir: Option<String>,
|
|
225
|
+
|
|
226
|
+
/// Capacity of the in-memory keyed-state cache, in bytes.
|
|
227
|
+
state_cache_size_bytes: Option<u64>,
|
|
228
|
+
|
|
229
|
+
/// Delay in whole seconds before the keyed-state recovery sweep.
|
|
230
|
+
///
|
|
231
|
+
/// Crosses as an `f64` so fractional/negative/non-finite values reach the
|
|
232
|
+
/// whole-number guard rather than being silently truncated.
|
|
233
|
+
state_recovery_delay: Option<f64>,
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/// Declares one keyed-state collection to register before subscribe.
|
|
237
|
+
#[derive(Clone, Debug, Deserialize)]
|
|
238
|
+
struct StateCollectionConfig {
|
|
239
|
+
/// The collection name (non-empty, unique within the client).
|
|
240
|
+
name: String,
|
|
241
|
+
|
|
242
|
+
/// The collection kind: `"value"`, `"map"`, or `"deque"`.
|
|
243
|
+
kind: String,
|
|
244
|
+
|
|
245
|
+
/// The item payload: `"json"` or `"message"`.
|
|
246
|
+
payload: String,
|
|
247
|
+
|
|
248
|
+
/// Optional per-write TTL in whole seconds. Crosses as `f64` so
|
|
249
|
+
/// fractional/negative/non-finite values reach the whole-number guard.
|
|
250
|
+
ttl_seconds: Option<f64>,
|
|
251
|
+
|
|
252
|
+
/// Optional opt-out of transactional staging.
|
|
253
|
+
read_uncommitted: Option<bool>,
|
|
254
|
+
|
|
255
|
+
/// Optional map-only keyset bound (`0..=4096`). Crosses as `f64` so
|
|
256
|
+
/// fractional/negative/non-finite values reach the whole-number guard.
|
|
257
|
+
keyset_limit: Option<f64>,
|
|
258
|
+
|
|
259
|
+
/// Optional deque-only window capacity (`>= 1`). Runtime-only and not
|
|
260
|
+
/// persisted. Crosses as `f64` so fractional/negative/non-finite values
|
|
261
|
+
/// reach the whole-number guard.
|
|
262
|
+
capacity: Option<f64>,
|
|
205
263
|
}
|
|
206
264
|
|
|
207
265
|
/// Configuration for the health probe port.
|
|
@@ -651,22 +709,10 @@ impl<'a> From<&'a NativeConfiguration> for DeferConfigurationBuilder {
|
|
|
651
709
|
builder.failure_window(Duration::from_secs_f32(*failure_window));
|
|
652
710
|
}
|
|
653
711
|
|
|
654
|
-
if let Some(cache_size) = &config.defer_cache_size {
|
|
655
|
-
builder.cache_size(*cache_size as usize);
|
|
656
|
-
}
|
|
657
|
-
|
|
658
712
|
if let Some(store_cache_size) = &config.defer_store_cache_size {
|
|
659
713
|
builder.store_cache_size(*store_cache_size as usize);
|
|
660
714
|
}
|
|
661
715
|
|
|
662
|
-
if let Some(seek_timeout) = &config.defer_seek_timeout {
|
|
663
|
-
builder.seek_timeout(Duration::from_secs_f32(*seek_timeout));
|
|
664
|
-
}
|
|
665
|
-
|
|
666
|
-
if let Some(discard_threshold) = &config.defer_discard_threshold {
|
|
667
|
-
builder.discard_threshold(*discard_threshold);
|
|
668
|
-
}
|
|
669
|
-
|
|
670
716
|
builder
|
|
671
717
|
}
|
|
672
718
|
}
|
|
@@ -696,25 +742,41 @@ impl<'a> From<&'a NativeConfiguration> for TimeoutConfigurationBuilder {
|
|
|
696
742
|
}
|
|
697
743
|
}
|
|
698
744
|
|
|
699
|
-
impl<'a>
|
|
700
|
-
|
|
745
|
+
impl<'a> TryFrom<&'a NativeConfiguration> for DeduplicationConfigurationBuilder {
|
|
746
|
+
type Error = String;
|
|
747
|
+
|
|
748
|
+
/// Attempts to convert a `NativeConfiguration` reference into a
|
|
701
749
|
/// `DeduplicationConfigurationBuilder`.
|
|
702
750
|
///
|
|
703
751
|
/// This takes the relevant deduplication settings from the configuration
|
|
704
752
|
/// and sets them on a new `DeduplicationConfigurationBuilder` instance.
|
|
705
753
|
///
|
|
754
|
+
/// Consumer deduplication is mandatory in the core (it is the keyed-state
|
|
755
|
+
/// commit oracle), so `cache_capacity` is `NonZeroUsize` and a zero
|
|
756
|
+
/// capacity is unrepresentable rather than a silent "disable". An explicit
|
|
757
|
+
/// `idempotence_cache_size` of `0` is therefore rejected here rather than
|
|
758
|
+
/// silently defaulting; this mirrors the sibling `prosody-js` binding and
|
|
759
|
+
/// the core's own rejection of `PROSODY_IDEMPOTENCE_CACHE_SIZE=0`.
|
|
760
|
+
///
|
|
706
761
|
/// # Arguments
|
|
707
762
|
///
|
|
708
763
|
/// * `config` - The configuration to convert
|
|
709
764
|
///
|
|
710
765
|
/// # Returns
|
|
711
766
|
///
|
|
712
|
-
/// A configured `DeduplicationConfigurationBuilder`
|
|
713
|
-
|
|
767
|
+
/// A configured `DeduplicationConfigurationBuilder` if successful
|
|
768
|
+
///
|
|
769
|
+
/// # Errors
|
|
770
|
+
///
|
|
771
|
+
/// Returns a `String` error if `idempotence_cache_size` is explicitly set
|
|
772
|
+
/// to `0`.
|
|
773
|
+
fn try_from(config: &'a NativeConfiguration) -> Result<Self, Self::Error> {
|
|
714
774
|
let mut builder = Self::default();
|
|
715
775
|
|
|
716
776
|
if let Some(cache_capacity) = &config.idempotence_cache_size {
|
|
717
|
-
|
|
777
|
+
let cache_capacity = NonZeroUsize::new(*cache_capacity as usize)
|
|
778
|
+
.ok_or_else(|| "idempotence_cache_size must be greater than 0".to_owned())?;
|
|
779
|
+
builder.cache_capacity(cache_capacity);
|
|
718
780
|
}
|
|
719
781
|
|
|
720
782
|
if let Some(version) = &config.idempotence_version {
|
|
@@ -728,7 +790,7 @@ impl<'a> From<&'a NativeConfiguration> for DeduplicationConfigurationBuilder {
|
|
|
728
790
|
builder.ttl(Duration::from_secs_f64(*ttl));
|
|
729
791
|
}
|
|
730
792
|
|
|
731
|
-
builder
|
|
793
|
+
Ok(builder)
|
|
732
794
|
}
|
|
733
795
|
}
|
|
734
796
|
|
|
@@ -770,6 +832,285 @@ impl<'a> TryFrom<&'a NativeConfiguration> for TelemetryEmitterConfiguration {
|
|
|
770
832
|
}
|
|
771
833
|
}
|
|
772
834
|
|
|
835
|
+
/// The kind of a keyed-state collection.
|
|
836
|
+
enum CollectionKind {
|
|
837
|
+
/// A single-value collection.
|
|
838
|
+
Value,
|
|
839
|
+
/// A `String`-keyed ordered map.
|
|
840
|
+
Map,
|
|
841
|
+
/// A deque.
|
|
842
|
+
Deque,
|
|
843
|
+
}
|
|
844
|
+
|
|
845
|
+
/// The item payload of a keyed-state collection.
|
|
846
|
+
enum CollectionPayload {
|
|
847
|
+
/// JSON values.
|
|
848
|
+
Json,
|
|
849
|
+
/// The full Kafka message the handler received.
|
|
850
|
+
Message,
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
/// Parses a collection-kind token.
|
|
854
|
+
///
|
|
855
|
+
/// # Errors
|
|
856
|
+
///
|
|
857
|
+
/// Returns a permanent-category error naming the field if the token is not
|
|
858
|
+
/// `"value"`, `"map"`, or `"deque"`.
|
|
859
|
+
fn parse_kind(index: usize, kind: &str) -> Result<CollectionKind, String> {
|
|
860
|
+
match kind {
|
|
861
|
+
"value" => Ok(CollectionKind::Value),
|
|
862
|
+
"map" => Ok(CollectionKind::Map),
|
|
863
|
+
"deque" => Ok(CollectionKind::Deque),
|
|
864
|
+
other => Err(format!(
|
|
865
|
+
"state_collections[{index}].kind: expected \"value\", \"map\", or \"deque\", got \
|
|
866
|
+
{other:?}"
|
|
867
|
+
)),
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
/// Parses a collection-payload token.
|
|
872
|
+
///
|
|
873
|
+
/// # Errors
|
|
874
|
+
///
|
|
875
|
+
/// Returns a permanent-category error naming the field if the token is not
|
|
876
|
+
/// `"json"` or `"message"`.
|
|
877
|
+
fn parse_payload(index: usize, payload: &str) -> Result<CollectionPayload, String> {
|
|
878
|
+
match payload {
|
|
879
|
+
"json" => Ok(CollectionPayload::Json),
|
|
880
|
+
"message" => Ok(CollectionPayload::Message),
|
|
881
|
+
other => Err(format!(
|
|
882
|
+
"state_collections[{index}].payload: expected \"json\" or \"message\", got {other:?}"
|
|
883
|
+
)),
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
/// Validates a numeric field as a whole number within `min..=max`.
|
|
888
|
+
///
|
|
889
|
+
/// The field arrives as an `f64` (the raw Ruby number, un-coerced) so that
|
|
890
|
+
/// fractional, negative, and non-finite values reach this guard instead of
|
|
891
|
+
/// being silently truncated or wrapped by an earlier integer conversion.
|
|
892
|
+
///
|
|
893
|
+
/// # Errors
|
|
894
|
+
///
|
|
895
|
+
/// Returns a permanent-category error naming the field if the value is not a
|
|
896
|
+
/// whole number in the inclusive range.
|
|
897
|
+
fn whole_number_field(value: f64, field: &str, min: u32, max: u32) -> Result<u32, String> {
|
|
898
|
+
if value.is_finite()
|
|
899
|
+
&& value.fract() == 0.0
|
|
900
|
+
&& value >= f64::from(min)
|
|
901
|
+
&& value <= f64::from(max)
|
|
902
|
+
{
|
|
903
|
+
Ok(value as u32)
|
|
904
|
+
} else {
|
|
905
|
+
Err(format!("{field}: must be a whole number in {min}..={max}"))
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
/// Applies the shared descriptor options (TTL, commit mode) fluently.
|
|
910
|
+
fn with_def<D: StateDescriptor>(
|
|
911
|
+
descriptor: D,
|
|
912
|
+
ttl_seconds: Option<u32>,
|
|
913
|
+
read_uncommitted: Option<bool>,
|
|
914
|
+
) -> D {
|
|
915
|
+
let mut descriptor = descriptor;
|
|
916
|
+
if let Some(ttl) = ttl_seconds {
|
|
917
|
+
descriptor = descriptor.ttl(CompactDuration::new(ttl));
|
|
918
|
+
}
|
|
919
|
+
if read_uncommitted == Some(true) {
|
|
920
|
+
descriptor = descriptor.read_uncommitted();
|
|
921
|
+
}
|
|
922
|
+
descriptor
|
|
923
|
+
}
|
|
924
|
+
|
|
925
|
+
/// Applies the map-only keyset bound when configured.
|
|
926
|
+
fn with_keyset<KC, V>(
|
|
927
|
+
descriptor: MapDescriptor<KC, V>,
|
|
928
|
+
keyset_limit: Option<u32>,
|
|
929
|
+
) -> MapDescriptor<KC, V> {
|
|
930
|
+
match keyset_limit {
|
|
931
|
+
Some(limit) => descriptor.keyset_limit(limit as usize),
|
|
932
|
+
None => descriptor,
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
|
|
936
|
+
/// Applies the deque-only window capacity when configured.
|
|
937
|
+
fn with_capacity<T>(
|
|
938
|
+
descriptor: DequeDescriptor<T>,
|
|
939
|
+
capacity: Option<NonZeroUsize>,
|
|
940
|
+
) -> DequeDescriptor<T> {
|
|
941
|
+
match capacity {
|
|
942
|
+
Some(cap) => descriptor.capacity(cap),
|
|
943
|
+
None => descriptor,
|
|
944
|
+
}
|
|
945
|
+
}
|
|
946
|
+
|
|
947
|
+
/// Validates one collection and registers its descriptor over the closed 3×2
|
|
948
|
+
/// (kind × payload) matrix.
|
|
949
|
+
///
|
|
950
|
+
/// # Errors
|
|
951
|
+
///
|
|
952
|
+
/// Returns a permanent-category error naming the offending field if a field is
|
|
953
|
+
/// invalid.
|
|
954
|
+
fn register_state_collection(
|
|
955
|
+
keyed: &mut KeyedStateConfiguration,
|
|
956
|
+
index: usize,
|
|
957
|
+
collection: &StateCollectionConfig,
|
|
958
|
+
) -> Result<(), String> {
|
|
959
|
+
if collection.name.is_empty() {
|
|
960
|
+
return Err(format!(
|
|
961
|
+
"state_collections[{index}].name: must not be empty"
|
|
962
|
+
));
|
|
963
|
+
}
|
|
964
|
+
|
|
965
|
+
let kind = parse_kind(index, &collection.kind)?;
|
|
966
|
+
let payload = parse_payload(index, &collection.payload)?;
|
|
967
|
+
|
|
968
|
+
let ttl_seconds = match collection.ttl_seconds {
|
|
969
|
+
Some(value) => Some(whole_number_field(
|
|
970
|
+
value,
|
|
971
|
+
&format!("state_collections[{index}].ttl_seconds"),
|
|
972
|
+
1,
|
|
973
|
+
u32::MAX,
|
|
974
|
+
)?),
|
|
975
|
+
None => None,
|
|
976
|
+
};
|
|
977
|
+
|
|
978
|
+
let keyset_limit = match collection.keyset_limit {
|
|
979
|
+
Some(value) => {
|
|
980
|
+
if !matches!(kind, CollectionKind::Map) {
|
|
981
|
+
return Err(format!(
|
|
982
|
+
"state_collections[{index}].keyset_limit: only valid for map collections"
|
|
983
|
+
));
|
|
984
|
+
}
|
|
985
|
+
Some(whole_number_field(
|
|
986
|
+
value,
|
|
987
|
+
&format!("state_collections[{index}].keyset_limit"),
|
|
988
|
+
0,
|
|
989
|
+
4096,
|
|
990
|
+
)?)
|
|
991
|
+
}
|
|
992
|
+
None => None,
|
|
993
|
+
};
|
|
994
|
+
|
|
995
|
+
let capacity = match collection.capacity {
|
|
996
|
+
Some(value) => {
|
|
997
|
+
if !matches!(kind, CollectionKind::Deque) {
|
|
998
|
+
return Err(format!(
|
|
999
|
+
"state_collections[{index}].capacity: only valid for deque collections"
|
|
1000
|
+
));
|
|
1001
|
+
}
|
|
1002
|
+
let n = whole_number_field(
|
|
1003
|
+
value,
|
|
1004
|
+
&format!("state_collections[{index}].capacity"),
|
|
1005
|
+
1,
|
|
1006
|
+
u32::MAX,
|
|
1007
|
+
)?;
|
|
1008
|
+
NonZeroUsize::new(n as usize)
|
|
1009
|
+
}
|
|
1010
|
+
None => None,
|
|
1011
|
+
};
|
|
1012
|
+
|
|
1013
|
+
let read_uncommitted = collection.read_uncommitted;
|
|
1014
|
+
let name = collection.name.as_str();
|
|
1015
|
+
match (kind, payload) {
|
|
1016
|
+
(CollectionKind::Value, CollectionPayload::Json) => {
|
|
1017
|
+
let _ = keyed.register(with_def(
|
|
1018
|
+
value_state::<JsonCodec>(name),
|
|
1019
|
+
ttl_seconds,
|
|
1020
|
+
read_uncommitted,
|
|
1021
|
+
));
|
|
1022
|
+
}
|
|
1023
|
+
(CollectionKind::Map, CollectionPayload::Json) => {
|
|
1024
|
+
let descriptor = with_def(
|
|
1025
|
+
map_state::<Utf8KeyCodec, JsonCodec>(name),
|
|
1026
|
+
ttl_seconds,
|
|
1027
|
+
read_uncommitted,
|
|
1028
|
+
);
|
|
1029
|
+
let _ = keyed.register(with_keyset(descriptor, keyset_limit));
|
|
1030
|
+
}
|
|
1031
|
+
(CollectionKind::Deque, CollectionPayload::Json) => {
|
|
1032
|
+
let descriptor = with_def(
|
|
1033
|
+
deque_state::<JsonCodec>(name),
|
|
1034
|
+
ttl_seconds,
|
|
1035
|
+
read_uncommitted,
|
|
1036
|
+
);
|
|
1037
|
+
let _ = keyed.register(with_capacity(descriptor, capacity));
|
|
1038
|
+
}
|
|
1039
|
+
(CollectionKind::Value, CollectionPayload::Message) => {
|
|
1040
|
+
let _ = keyed.register(with_def(
|
|
1041
|
+
message_state::<KafkaLoader<JsonCodec>>(name),
|
|
1042
|
+
ttl_seconds,
|
|
1043
|
+
read_uncommitted,
|
|
1044
|
+
));
|
|
1045
|
+
}
|
|
1046
|
+
(CollectionKind::Map, CollectionPayload::Message) => {
|
|
1047
|
+
let descriptor = with_def(
|
|
1048
|
+
message_map_state::<Utf8KeyCodec, KafkaLoader<JsonCodec>>(name),
|
|
1049
|
+
ttl_seconds,
|
|
1050
|
+
read_uncommitted,
|
|
1051
|
+
);
|
|
1052
|
+
let _ = keyed.register(with_keyset(descriptor, keyset_limit));
|
|
1053
|
+
}
|
|
1054
|
+
(CollectionKind::Deque, CollectionPayload::Message) => {
|
|
1055
|
+
let descriptor = with_def(
|
|
1056
|
+
message_deque_state::<KafkaLoader<JsonCodec>>(name),
|
|
1057
|
+
ttl_seconds,
|
|
1058
|
+
read_uncommitted,
|
|
1059
|
+
);
|
|
1060
|
+
let _ = keyed.register(with_capacity(descriptor, capacity));
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1063
|
+
|
|
1064
|
+
Ok(())
|
|
1065
|
+
}
|
|
1066
|
+
|
|
1067
|
+
/// Builds the `KeyedStateConfiguration`, registering each declared collection
|
|
1068
|
+
/// synchronously (before subscribe) and rejecting duplicate names.
|
|
1069
|
+
///
|
|
1070
|
+
/// # Errors
|
|
1071
|
+
///
|
|
1072
|
+
/// Returns an error if a keyed-state field is invalid or a name is duplicated.
|
|
1073
|
+
fn build_keyed_state_config(
|
|
1074
|
+
config: &NativeConfiguration,
|
|
1075
|
+
) -> Result<KeyedStateConfiguration, String> {
|
|
1076
|
+
let mut builder = KeyedStateConfiguration::builder();
|
|
1077
|
+
|
|
1078
|
+
if let Some(dir) = &config.state_cache_dir {
|
|
1079
|
+
if dir.is_empty() {
|
|
1080
|
+
return Err("state_cache_dir: must not be an empty string".to_owned());
|
|
1081
|
+
}
|
|
1082
|
+
builder.cache_dir(PathBuf::from(dir));
|
|
1083
|
+
}
|
|
1084
|
+
|
|
1085
|
+
if let Some(seconds) = config.state_recovery_delay {
|
|
1086
|
+
let seconds = whole_number_field(seconds, "state_recovery_delay", 1, u32::MAX)?;
|
|
1087
|
+
builder.recovery_delay(CompactDuration::new(seconds));
|
|
1088
|
+
}
|
|
1089
|
+
|
|
1090
|
+
if let Some(bytes) = config.state_cache_size_bytes {
|
|
1091
|
+
let bytes = NonZeroU64::new(bytes)
|
|
1092
|
+
.ok_or_else(|| "state_cache_size_bytes: must be greater than 0".to_owned())?;
|
|
1093
|
+
builder.cache_size_bytes(Some(bytes));
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1096
|
+
let mut keyed = builder.build().map_err(|error| error.to_string())?;
|
|
1097
|
+
|
|
1098
|
+
if let Some(collections) = &config.state_collections {
|
|
1099
|
+
let mut seen = HashSet::with_capacity(collections.len());
|
|
1100
|
+
for (index, collection) in collections.iter().enumerate() {
|
|
1101
|
+
if !seen.insert(collection.name.as_str()) {
|
|
1102
|
+
return Err(format!(
|
|
1103
|
+
"state_collections[{index}].name: duplicate collection name {:?}",
|
|
1104
|
+
collection.name
|
|
1105
|
+
));
|
|
1106
|
+
}
|
|
1107
|
+
register_state_collection(&mut keyed, index, collection)?;
|
|
1108
|
+
}
|
|
1109
|
+
}
|
|
1110
|
+
|
|
1111
|
+
Ok(keyed)
|
|
1112
|
+
}
|
|
1113
|
+
|
|
773
1114
|
impl<'a> TryFrom<&'a NativeConfiguration> for ConsumerBuilders {
|
|
774
1115
|
type Error = String;
|
|
775
1116
|
|
|
@@ -795,6 +1136,8 @@ impl<'a> TryFrom<&'a NativeConfiguration> for ConsumerBuilders {
|
|
|
795
1136
|
/// environment variable contains an unparseable value).
|
|
796
1137
|
/// - `message_spans` or `timer_spans` contains an unrecognized value
|
|
797
1138
|
/// (expected `"child"` or `"follows_from"`).
|
|
1139
|
+
/// - The Kafka loader configuration cannot be built (e.g. a tuning value
|
|
1140
|
+
/// fails validation).
|
|
798
1141
|
fn try_from(config: &'a NativeConfiguration) -> Result<Self, Self::Error> {
|
|
799
1142
|
let mut consumer: ConsumerConfigurationBuilder = config.into();
|
|
800
1143
|
|
|
@@ -812,6 +1155,30 @@ impl<'a> TryFrom<&'a NativeConfiguration> for ConsumerBuilders {
|
|
|
812
1155
|
consumer.timer_spans(relation);
|
|
813
1156
|
}
|
|
814
1157
|
|
|
1158
|
+
// The Kafka message loader that the defer middleware uses to reload
|
|
1159
|
+
// failed messages is now consumer-wide configuration. Route the
|
|
1160
|
+
// defer-loader tuning knobs onto the consumer builder's loader.
|
|
1161
|
+
if config.defer_cache_size.is_some()
|
|
1162
|
+
|| config.defer_seek_timeout.is_some()
|
|
1163
|
+
|| config.defer_discard_threshold.is_some()
|
|
1164
|
+
{
|
|
1165
|
+
let mut loader = KafkaLoaderConfiguration::builder();
|
|
1166
|
+
|
|
1167
|
+
if let Some(cache_size) = &config.defer_cache_size {
|
|
1168
|
+
loader.cache_size(*cache_size as usize);
|
|
1169
|
+
}
|
|
1170
|
+
|
|
1171
|
+
if let Some(seek_timeout) = &config.defer_seek_timeout {
|
|
1172
|
+
loader.seek_timeout(Duration::from_secs_f32(*seek_timeout));
|
|
1173
|
+
}
|
|
1174
|
+
|
|
1175
|
+
if let Some(discard_threshold) = &config.defer_discard_threshold {
|
|
1176
|
+
loader.discard_threshold(*discard_threshold);
|
|
1177
|
+
}
|
|
1178
|
+
|
|
1179
|
+
consumer.loader(loader.build().map_err(|e| e.to_string())?);
|
|
1180
|
+
}
|
|
1181
|
+
|
|
815
1182
|
Ok(Self {
|
|
816
1183
|
consumer,
|
|
817
1184
|
retry: config.into(),
|
|
@@ -820,8 +1187,9 @@ impl<'a> TryFrom<&'a NativeConfiguration> for ConsumerBuilders {
|
|
|
820
1187
|
monopolization: config.into(),
|
|
821
1188
|
defer: config.into(),
|
|
822
1189
|
timeout: config.into(),
|
|
823
|
-
dedup: config.
|
|
1190
|
+
dedup: config.try_into()?,
|
|
824
1191
|
emitter: config.try_into()?,
|
|
1192
|
+
keyed_state: build_keyed_state_config(config)?,
|
|
825
1193
|
})
|
|
826
1194
|
}
|
|
827
1195
|
}
|
|
@@ -5,6 +5,10 @@
|
|
|
5
5
|
//! information from Kafka messages and schedule timer events.
|
|
6
6
|
|
|
7
7
|
use crate::bridge::Bridge;
|
|
8
|
+
use crate::handler::state::{
|
|
9
|
+
DequeStateVariant, MapStateVariant, NativeDequeState, NativeMapState, NativeValueState,
|
|
10
|
+
ValueStateVariant, state_error,
|
|
11
|
+
};
|
|
8
12
|
use crate::tracing_util::extract_opentelemetry_context;
|
|
9
13
|
use crate::{ROOT_MOD, id};
|
|
10
14
|
use educe::Educe;
|
|
@@ -35,9 +39,8 @@ pub struct Context {
|
|
|
35
39
|
///
|
|
36
40
|
/// This field is marked as hidden in debug output to prevent logging large
|
|
37
41
|
/// data.
|
|
38
|
-
#[allow(dead_code)]
|
|
39
42
|
#[educe(Debug(ignore))]
|
|
40
|
-
inner: BoxEventContext
|
|
43
|
+
inner: BoxEventContext<serde_json::Value>,
|
|
41
44
|
|
|
42
45
|
/// Bridge for handling async operations
|
|
43
46
|
#[educe(Debug(ignore))]
|
|
@@ -57,7 +60,7 @@ impl Context {
|
|
|
57
60
|
/// * `bridge` - The bridge for handling async operations
|
|
58
61
|
/// * `propagator` - Shared OpenTelemetry propagator for distributed tracing
|
|
59
62
|
pub fn new(
|
|
60
|
-
inner: BoxEventContext
|
|
63
|
+
inner: BoxEventContext<serde_json::Value>,
|
|
61
64
|
bridge: Bridge,
|
|
62
65
|
propagator: Arc<TextMapCompositePropagator>,
|
|
63
66
|
) -> Self {
|
|
@@ -302,10 +305,10 @@ impl Context {
|
|
|
302
305
|
async move { inner.scheduled(TimerType::Application).await },
|
|
303
306
|
span,
|
|
304
307
|
)?
|
|
305
|
-
.map_err(|
|
|
308
|
+
.map_err(|error| {
|
|
306
309
|
Error::new(
|
|
307
310
|
ruby.exception_runtime_error(),
|
|
308
|
-
format!("Failed to get scheduled times: {
|
|
311
|
+
format!("Failed to get scheduled times: {error:#}"),
|
|
309
312
|
)
|
|
310
313
|
})?;
|
|
311
314
|
|
|
@@ -319,6 +322,127 @@ impl Context {
|
|
|
319
322
|
|
|
320
323
|
Ok(ruby_array.as_value())
|
|
321
324
|
}
|
|
325
|
+
|
|
326
|
+
/// Vends the handle for the named JSON value collection.
|
|
327
|
+
///
|
|
328
|
+
/// Vending verifies the collection's registration (core-side); no span is
|
|
329
|
+
/// opened here — vended handles outlive the call, and every operation opens
|
|
330
|
+
/// its own span.
|
|
331
|
+
///
|
|
332
|
+
/// # Errors
|
|
333
|
+
///
|
|
334
|
+
/// Returns a permanent state error if the name is unregistered or its
|
|
335
|
+
/// registered identity mismatches.
|
|
336
|
+
#[allow(clippy::needless_pass_by_value, reason = "Magnus method argument type")]
|
|
337
|
+
fn value_state(ruby: &Ruby, this: &Self, name: String) -> Result<NativeValueState, Error> {
|
|
338
|
+
let handle = this
|
|
339
|
+
.inner
|
|
340
|
+
.value_state(&name)
|
|
341
|
+
.map_err(|error| state_error(ruby, &error))?;
|
|
342
|
+
Ok(NativeValueState::new(
|
|
343
|
+
ValueStateVariant::Json(Arc::from(handle)),
|
|
344
|
+
this.bridge.clone(),
|
|
345
|
+
Arc::clone(&this.propagator),
|
|
346
|
+
))
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/// Vends the handle for the named JSON map collection.
|
|
350
|
+
///
|
|
351
|
+
/// # Errors
|
|
352
|
+
///
|
|
353
|
+
/// See [`value_state`](Self::value_state).
|
|
354
|
+
#[allow(clippy::needless_pass_by_value, reason = "Magnus method argument type")]
|
|
355
|
+
fn map_state(ruby: &Ruby, this: &Self, name: String) -> Result<NativeMapState, Error> {
|
|
356
|
+
let handle = this
|
|
357
|
+
.inner
|
|
358
|
+
.map_state(&name)
|
|
359
|
+
.map_err(|error| state_error(ruby, &error))?;
|
|
360
|
+
Ok(NativeMapState::new(
|
|
361
|
+
MapStateVariant::Json(Arc::from(handle)),
|
|
362
|
+
this.bridge.clone(),
|
|
363
|
+
Arc::clone(&this.propagator),
|
|
364
|
+
))
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
/// Vends the handle for the named JSON deque collection.
|
|
368
|
+
///
|
|
369
|
+
/// # Errors
|
|
370
|
+
///
|
|
371
|
+
/// See [`value_state`](Self::value_state).
|
|
372
|
+
#[allow(clippy::needless_pass_by_value, reason = "Magnus method argument type")]
|
|
373
|
+
fn deque_state(ruby: &Ruby, this: &Self, name: String) -> Result<NativeDequeState, Error> {
|
|
374
|
+
let handle = this
|
|
375
|
+
.inner
|
|
376
|
+
.deque_state(&name)
|
|
377
|
+
.map_err(|error| state_error(ruby, &error))?;
|
|
378
|
+
Ok(NativeDequeState::new(
|
|
379
|
+
DequeStateVariant::Json(Arc::from(handle)),
|
|
380
|
+
this.bridge.clone(),
|
|
381
|
+
Arc::clone(&this.propagator),
|
|
382
|
+
))
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
/// Vends the handle for the named Kafka-message value collection.
|
|
386
|
+
///
|
|
387
|
+
/// # Errors
|
|
388
|
+
///
|
|
389
|
+
/// See [`value_state`](Self::value_state).
|
|
390
|
+
#[allow(clippy::needless_pass_by_value, reason = "Magnus method argument type")]
|
|
391
|
+
fn message_value_state(
|
|
392
|
+
ruby: &Ruby,
|
|
393
|
+
this: &Self,
|
|
394
|
+
name: String,
|
|
395
|
+
) -> Result<NativeValueState, Error> {
|
|
396
|
+
let handle = this
|
|
397
|
+
.inner
|
|
398
|
+
.message_value_state(&name)
|
|
399
|
+
.map_err(|error| state_error(ruby, &error))?;
|
|
400
|
+
Ok(NativeValueState::new(
|
|
401
|
+
ValueStateVariant::Message(Arc::from(handle)),
|
|
402
|
+
this.bridge.clone(),
|
|
403
|
+
Arc::clone(&this.propagator),
|
|
404
|
+
))
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
/// Vends the handle for the named Kafka-message map collection.
|
|
408
|
+
///
|
|
409
|
+
/// # Errors
|
|
410
|
+
///
|
|
411
|
+
/// See [`value_state`](Self::value_state).
|
|
412
|
+
#[allow(clippy::needless_pass_by_value, reason = "Magnus method argument type")]
|
|
413
|
+
fn message_map_state(ruby: &Ruby, this: &Self, name: String) -> Result<NativeMapState, Error> {
|
|
414
|
+
let handle = this
|
|
415
|
+
.inner
|
|
416
|
+
.message_map_state(&name)
|
|
417
|
+
.map_err(|error| state_error(ruby, &error))?;
|
|
418
|
+
Ok(NativeMapState::new(
|
|
419
|
+
MapStateVariant::Message(Arc::from(handle)),
|
|
420
|
+
this.bridge.clone(),
|
|
421
|
+
Arc::clone(&this.propagator),
|
|
422
|
+
))
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
/// Vends the handle for the named Kafka-message deque collection.
|
|
426
|
+
///
|
|
427
|
+
/// # Errors
|
|
428
|
+
///
|
|
429
|
+
/// See [`value_state`](Self::value_state).
|
|
430
|
+
#[allow(clippy::needless_pass_by_value, reason = "Magnus method argument type")]
|
|
431
|
+
fn message_deque_state(
|
|
432
|
+
ruby: &Ruby,
|
|
433
|
+
this: &Self,
|
|
434
|
+
name: String,
|
|
435
|
+
) -> Result<NativeDequeState, Error> {
|
|
436
|
+
let handle = this
|
|
437
|
+
.inner
|
|
438
|
+
.message_deque_state(&name)
|
|
439
|
+
.map_err(|error| state_error(ruby, &error))?;
|
|
440
|
+
Ok(NativeDequeState::new(
|
|
441
|
+
DequeStateVariant::Message(Arc::from(handle)),
|
|
442
|
+
this.bridge.clone(),
|
|
443
|
+
Arc::clone(&this.propagator),
|
|
444
|
+
))
|
|
445
|
+
}
|
|
322
446
|
}
|
|
323
447
|
|
|
324
448
|
/// Initializes the Context class in Ruby.
|
|
@@ -357,6 +481,23 @@ pub fn init(ruby: &Ruby) -> Result<(), Error> {
|
|
|
357
481
|
)?;
|
|
358
482
|
class.define_method(id!(ruby, "scheduled"), method!(Context::scheduled, 0))?;
|
|
359
483
|
|
|
484
|
+
// Keyed-state vend methods
|
|
485
|
+
class.define_method(id!(ruby, "value_state"), method!(Context::value_state, 1))?;
|
|
486
|
+
class.define_method(id!(ruby, "map_state"), method!(Context::map_state, 1))?;
|
|
487
|
+
class.define_method(id!(ruby, "deque_state"), method!(Context::deque_state, 1))?;
|
|
488
|
+
class.define_method(
|
|
489
|
+
id!(ruby, "message_value_state"),
|
|
490
|
+
method!(Context::message_value_state, 1),
|
|
491
|
+
)?;
|
|
492
|
+
class.define_method(
|
|
493
|
+
id!(ruby, "message_map_state"),
|
|
494
|
+
method!(Context::message_map_state, 1),
|
|
495
|
+
)?;
|
|
496
|
+
class.define_method(
|
|
497
|
+
id!(ruby, "message_deque_state"),
|
|
498
|
+
method!(Context::message_deque_state, 1),
|
|
499
|
+
)?;
|
|
500
|
+
|
|
360
501
|
Ok(())
|
|
361
502
|
}
|
|
362
503
|
|