prosody 0.5.1 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. checksums.yaml +4 -4
  2. data/.config/rail.toml +26 -0
  3. data/.release-please-manifest.json +1 -1
  4. data/.ruby-version +1 -1
  5. data/.taplo.toml +1 -1
  6. data/AGENTS.md +29 -15
  7. data/CHANGELOG.md +7 -0
  8. data/CONFIGURATION.md +19 -16
  9. data/Cargo.lock +254 -235
  10. data/Cargo.toml +11 -8
  11. data/README.md +100 -23
  12. data/examples/keyed_state.rb +10 -2
  13. data/examples/keyed_state.rbs +1 -0
  14. data/ext/prosody/Cargo.toml +1 -1
  15. data/ext/prosody/src/admin.rs +36 -36
  16. data/ext/prosody/src/bridge/mod.rs +5 -10
  17. data/ext/prosody/src/client/config/connections.rs +170 -0
  18. data/ext/prosody/src/client/config/middleware.rs +184 -0
  19. data/ext/prosody/src/client/config/mod.rs +396 -0
  20. data/ext/prosody/src/client/config/state.rs +323 -0
  21. data/ext/prosody/src/client/mod.rs +31 -117
  22. data/ext/prosody/src/client/readers.rs +106 -0
  23. data/ext/prosody/src/client/request.rs +2 -2
  24. data/ext/prosody/src/client/support.rs +13 -32
  25. data/ext/prosody/src/gvl.rs +8 -6
  26. data/ext/prosody/src/handler/{context.rs → context/mod.rs} +36 -150
  27. data/ext/prosody/src/handler/context/vending.rs +138 -0
  28. data/ext/prosody/src/handler/message.rs +36 -0
  29. data/ext/prosody/src/handler/mod.rs +18 -8
  30. data/ext/prosody/src/handler/state/deque.rs +144 -0
  31. data/ext/prosody/src/handler/state/mod.rs +163 -268
  32. data/ext/prosody/src/handler/state/query.rs +264 -0
  33. data/ext/prosody/src/handler/state/registration.rs +15 -86
  34. data/ext/prosody/src/handler/state/scan.rs +33 -141
  35. data/ext/prosody/src/handler/state/set.rs +98 -0
  36. data/ext/prosody/src/lib.rs +54 -36
  37. data/ext/prosody/src/logging.rs +6 -6
  38. data/ext/prosody/src/published.rs +171 -152
  39. data/ext/prosody/src/scheduler/result.rs +2 -1
  40. data/ext/prosody/src/util.rs +65 -3
  41. data/lib/prosody/client.rb +32 -0
  42. data/lib/prosody/configuration.rb +20 -12
  43. data/lib/prosody/demand.rb +27 -0
  44. data/lib/prosody/native_stubs/client.rb +157 -0
  45. data/lib/prosody/native_stubs/context.rb +178 -0
  46. data/lib/prosody/native_stubs/message.rb +133 -0
  47. data/lib/prosody/native_stubs.rb +11 -956
  48. data/lib/prosody/state/deque.rb +253 -0
  49. data/lib/prosody/state/map.rb +313 -0
  50. data/lib/prosody/state/set.rb +121 -0
  51. data/lib/prosody/state/value.rb +58 -0
  52. data/lib/prosody/state.rb +157 -677
  53. data/lib/prosody/version.rb +1 -1
  54. data/lib/prosody.rb +2 -1
  55. data/sig/configuration.rbs +19 -10
  56. data/sig/prosody.rbs +37 -4
  57. data/sig/published.rbs +102 -0
  58. data/sig/state.rbs +109 -122
  59. data/typecheck/payload_types.rb +12 -0
  60. data/typecheck/payload_types.rbs +1 -0
  61. metadata +30 -11
  62. data/ext/prosody/src/client/config.rs +0 -1300
data/Cargo.toml CHANGED
@@ -10,23 +10,22 @@ resolver = "2"
10
10
  atomic-take = "1.1"
11
11
  bumpalo = "3.20"
12
12
  crossbeam-channel = "0.5"
13
- educe = "0.7"
13
+ educe = "0.8"
14
14
  futures = "0.3"
15
- magnus = { version = "0.8", default-features = false }
16
- # TODO: Use a crates.io release after it includes the fix for https://github.com/microsoft/mimalloc/issues/1287.
17
- mimalloc = { git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11" }
18
- opentelemetry = "0.32"
19
- prosody = { version = "0.6.0", features = ["libz-static"] }
15
+ magnus = { version = "0.9", default-features = false }
16
+ opentelemetry = "0.33"
17
+ prosody = { version = "0.7.0", features = ["libz-static"] }
20
18
  rb-sys = "0.9"
19
+ rustfs-mimalloc = "0.5.6"
21
20
  serde = "1.0"
22
21
  serde-untagged = "0.1"
23
22
  serde_json = "1.0"
24
23
  serde_magnus = "0.11"
25
24
  thiserror = "2.0"
26
- tokio = "1.52"
25
+ tokio = "1.53"
27
26
  tokio-stream = "0.1"
28
27
  tracing = "0.1"
29
- tracing-opentelemetry = "0.33"
28
+ tracing-opentelemetry = "0.34"
30
29
  tracing-subscriber = "0.3"
31
30
 
32
31
  [profile.release]
@@ -78,3 +77,7 @@ unneeded_field_pattern = "warn"
78
77
  unreachable = "warn"
79
78
  unwrap_used = "deny"
80
79
  verbose_file_reads = "warn"
80
+
81
+ [patch.crates-io]
82
+ # TODO: Use a crates.io release after serde_magnus supports magnus 0.9.
83
+ serde_magnus = { git = "https://github.com/hadronzoo/serde-magnus", rev = "27aee737829a6bca399289b6e64ce4bcc08bfd94" }
data/README.md CHANGED
@@ -277,10 +277,10 @@ You can monitor the stall state programmatically using the client's methods:
277
277
 
278
278
  ```ruby
279
279
  # Get the number of partitions currently assigned to this consumer
280
- partition_count = client.assigned_partitions
280
+ partition_count = client.assigned_partition_count
281
281
 
282
282
  # Check if the consumer has stalled partitions
283
- if client.is_stalled?
283
+ if client.stalled?
284
284
  warn 'Consumer has stalled partitions'
285
285
  end
286
286
  ```
@@ -299,7 +299,7 @@ Send a request from a handler or other application code. The Prosody client does
299
299
 
300
300
  Do not rely on hash order. The hash contains one entry for each selected subsystem. A missing response becomes a timeout `Failure`; Prosody does not omit the subsystem. The request raises an error for request-level failures, such as invalid input, a Kafka send failure, or shutdown. Do not wait for a request if the current consumer group must process it for the same key. That group cannot process it until the handler returns.
301
301
 
302
- Message and excise handler return values become successful outcomes. Each return value must have a JSON representation.
302
+ Message and excise handler return values become successful outcomes. Each return value must have a JSON representation. A return value without one is a transient handler error, so Prosody retries the message.
303
303
 
304
304
  Set `subsystem` to `inventory` on the client that subscribes this handler.
305
305
 
@@ -626,12 +626,53 @@ State operations look synchronous. They yield the current fiber while Prosody pe
626
626
  | Collection | JSON payload | Kafka message | Main operations |
627
627
  | --- | --- | --- | --- |
628
628
  | Value | `Prosody.value` | `Prosody.message_value` | `get`, `set`, `clear` |
629
- | Ordered string map | `Prosody.map` | `Prosody.message_map` | `get`, `get_many`, `key?`, `set`, `delete`, `each_pair`, `each_key`, `clear` |
629
+ | Ordered string map | `Prosody.map` | `Prosody.message_map` | `get`, `get_many`, `key?`, `contains_many`, `empty?`, `set`, `delete`, `each_pair`, `each_key`, `clear` |
630
+ | Ordered string set | `Prosody.set` | - | `add` / `<<`, `delete`, `include?`, `contains_many`, `empty?`, `each`, `clear` |
630
631
  | Deque | `Prosody.deque` | `Prosody.message_deque` | `push`, `unshift`, `pop`, `shift`, `get`, `length`, `each`, `clear` |
631
632
 
632
- Map and deque scans return enumerators when called without a block. Map keys are strings.
633
+ Map, set, and deque scans return enumerators when called without a block. Map keys and set members are strings. A set stores membership only, so it has no payload type.
633
634
 
634
- `nil` means absence. Do not store this value. Use `clear` or `delete`.
635
+ A value, map, or deque read returns `nil` when no value is present. Do not store `nil` as a value. A `nil` write fails with a `PermanentStateError`. Use `clear` (value, deque) or `delete` (map) to remove a value.
636
+
637
+ A set handle mirrors Ruby's `Set`:
638
+
639
+ ```ruby
640
+ SEEN_ORDERS = Prosody.set("seen-orders", ttl: 7 * 24 * 60 * 60)
641
+
642
+ def on_message(context, message)
643
+ seen = context.state(SEEN_ORDERS)
644
+ return if seen.include?(message.payload["order_id"])
645
+
646
+ seen << message.payload["order_id"]
647
+ fulfill(message)
648
+ end
649
+ ```
650
+
651
+ ### Query keywords
652
+
653
+ Every traversal method accepts optional query keywords. Prosody applies them in storage, so a query reads only the selected entries. The `each` and `each_*` methods iterate forward. The `reverse_each` and `reverse_each_*` methods iterate backward.
654
+
655
+ | Keyword | Selects |
656
+ | --- | --- |
657
+ | `from:` / `after:` | Starts at the key or position, or just after it, in iteration order |
658
+ | `to:` / `before:` | Stops at the key or position, or just before it, in iteration order |
659
+ | `range:` | Keeps keys or positions in a Ruby `Range`: `"a".."m"`, `"a"..."m"`, `.."m"`, or `"a"..`. Write the range in ascending order; it applies in both directions. A descending range is empty |
660
+ | `prefix:` | Keeps map keys or set members that start with the string |
661
+ | `limit:` | Stops after this many items; a positive `Integer` |
662
+
663
+ A reverse traversal starts at the high end. Keywords narrow the selection and never widen it. Pass at most one of `from:` and `after:`, and at most one of `to:` and `before:`. Set traversals select members. Deque positions count from the front and must be non-negative. To read the last N items, use `reverse_each(limit: N)`. Deques have no `prefix:`. A bad keyword raises `ArgumentError` or `TypeError`.
664
+
665
+ To read a map in pages, pass the last key of the previous page as `after:`:
666
+
667
+ ```ruby
668
+ page = map.each_pair(prefix: "order:", limit: 100).to_a
669
+ until page.empty?
670
+ page.each { |key, order| archive(key, order) }
671
+ page = map.each_pair(prefix: "order:", after: page.last.first, limit: 100).to_a
672
+ end
673
+ ```
674
+
675
+ The same pattern works backward with `reverse_each_pair`.
635
676
 
636
677
  ### When keyed-state changes become visible
637
678
 
@@ -643,6 +684,8 @@ This transaction applies only to keyed state. Some workflows need state changes
643
684
  - `commit` commits the collection's pending changes before the handler ends. A later handler failure does not remove them.
644
685
  - `rollback` discards pending changes since the last `commit`. It cannot undo committed changes.
645
686
 
687
+ `commit` and `rollback` return `:applied` when they wrote or discarded pending changes. They return `:no_op` when the collection had no pending changes.
688
+
646
689
  ### Published state
647
690
 
648
691
  Some callers need only the current value for a key. They can accept a stale value or a race with a concurrent update.
@@ -665,7 +708,7 @@ current_order = context.state(CURRENT_ORDER)
665
708
  current_order.set({"sku" => "book"})
666
709
  ```
667
710
 
668
- Read published state from a handler or other application code. The Prosody client does not need an active subscription.
711
+ Read published state from a handler or other application code. The Prosody client does not need an active subscription. A client that only reads published state needs no `subscribed_topics`.
669
712
 
670
713
  Use the subsystem and the same definition to open a reader:
671
714
 
@@ -674,9 +717,9 @@ order_reader = client.state("checkout", CURRENT_ORDER)
674
717
  current_order = order_reader.get("customer-123")
675
718
  ```
676
719
 
677
- The reader cannot see pending changes that exist only in a handler. It cannot change the collection. Each read takes an explicit key because no handler supplies one.
720
+ The reader cannot see pending changes that exist only in a handler. It cannot change the collection. Each read takes an explicit String key because no handler supplies one.
678
721
 
679
- Map and deque readers fetch data in chunks. They do not load the complete collection before iteration starts. Readers return an `Enumerator` without a block.
722
+ Map, set, and deque readers fetch data in chunks. They do not load the complete collection before iteration starts. Readers return an `Enumerator` without a block. Reader traversals accept the same [query keywords](#query-keywords) after the key. A failed read raises `Prosody::TransientStateError` or `Prosody::PermanentStateError`, as an owned handle does, for point reads and traversals alike. A reader that Prosody cannot open, such as one with a zero `read_cache`, raises the same classes.
680
723
 
681
724
  Use `reverse_each_pair`, `reverse_each_key`, `reverse_each_value`, or `reverse_each` for reverse traversal.
682
725
 
@@ -783,9 +826,9 @@ all traces to Ruby.
783
826
  To use OpenTelemetry tracing with Prosody, you need to install the following gems:
784
827
 
785
828
  ```ruby
786
- gem 'opentelemetry-sdk', '~> 1.10'
787
- gem 'opentelemetry-api', '~> 1.7'
788
- gem 'opentelemetry-exporter-otlp', '~> 0.31'
829
+ gem 'opentelemetry-sdk', '~> 1.13'
830
+ gem 'opentelemetry-api', '~> 1.10'
831
+ gem 'opentelemetry-exporter-otlp', '~> 0.36'
789
832
  ```
790
833
 
791
834
  ### Initializing Tracing
@@ -907,6 +950,16 @@ Call `shutdown` when the application terminates. It stops all client services an
907
950
  client.shutdown
908
951
  ```
909
952
 
953
+ To scope a client to a block, use `Prosody::Client.open`. It yields the client and calls `shutdown` when the block exits, also when the block raises. It returns the value of the block.
954
+
955
+ ```ruby
956
+ Prosody::Client.open(bootstrap_servers: "localhost:9092") do |client|
957
+ client.send_message("my-topic", "key", {"hello" => "world"})
958
+ end
959
+ ```
960
+
961
+ Repeated `shutdown` calls wait for the same operation, so the block can also call `shutdown`.
962
+
910
963
  Handle application shutdown with signal handlers:
911
964
 
912
965
  ```ruby
@@ -972,6 +1025,18 @@ Best practices:
972
1025
  - Be cautious with permanent errors as they prevent retries and can result in data loss.
973
1026
  - Consider system reliability and data consistency when classifying errors.
974
1027
 
1028
+ A handler can tell a retry from a normal delivery through `context.demand`:
1029
+
1030
+ ```ruby
1031
+ def on_message(context, message)
1032
+ demand = context.demand
1033
+ logger.warn("retry #{demand.retries} for #{message.key}") if demand.failure?
1034
+ process(message)
1035
+ end
1036
+ ```
1037
+
1038
+ `demand.retries` is the number of retries: 0 for a normal delivery and 1 on the first retry. After Prosody defers an event, the count starts again at 1. The count is an estimate. Keep an exact attempt count in keyed state if you need one.
1039
+
975
1040
  ### Handling Task Cancellation
976
1041
 
977
1042
  Prosody cancels tasks during partition rebalancing, timeout, or shutdown. During shutdown, handlers run freely for most of the `shutdown_timeout` before the cancellation signal fires—giving in-flight work time to complete. When cancelled, your handler receives `Async::Stop` at the next yield point (I/O operation, sleep, etc.).
@@ -1064,23 +1129,24 @@ Ensure you have thoroughly tested your changes before merging to `main`.
1064
1129
  ### Prosody::Client
1065
1130
 
1066
1131
  - `new(config)` or `new(**options)`: Create a client from a `Configuration`, hash, or keyword options.
1132
+ - `open(config) { |client| ... }`: Create a client, yield it, and shut it down when the block exits. It returns the value of the block.
1067
1133
  - `send_message(String topic, String key, Prosody::json_value payload)`: Send a JSON-serializable message.
1068
1134
  - `excise(String topic, String key)`: Send an excise record for a key.
1069
1135
  - `request(topic:, key:, payload:, subsystems:, timeout:)`: Return one outcome for each subsystem.
1070
1136
  - `request_excise(topic:, key:, subsystems:, timeout:)`: Return one excise outcome for each subsystem.
1071
1137
  - `consumer_state`: Get the client state (`:shut_down`, `:unconfigured`, `:configured`, or `:running`).
1072
1138
  - `source_system`: Get the source system identifier configured for the client.
1073
- - `state(subsystem, definition)`: Open a typed, read-only published value, map, or deque.
1139
+ - `state(subsystem, definition)`: Open a typed, read-only published value, map, set, or deque.
1074
1140
  - `subscribe(handler)`: Start event processing with the specified handler.
1075
1141
  - `unsubscribe`: Stop the consumer. You can subscribe again later.
1076
1142
  - `shutdown`: Stop all client services. Concurrent and repeated calls wait for the same operation.
1077
- - `assigned_partitions`: Get the number of partitions currently assigned to this consumer.
1078
- - `is_stalled?`: Check if the consumer has stalled partitions.
1143
+ - `assigned_partition_count`: Get the number of partitions currently assigned to this consumer.
1144
+ - `stalled?`: Check if the consumer has stalled partitions.
1079
1145
 
1080
1146
  ### Prosody::AdminClient
1081
1147
 
1082
1148
  - `new(bootstrap_servers)`: Create an admin client for the specified Kafka servers.
1083
- - `create_topic(name, partitions, replication_factor)`: Create a Kafka topic.
1149
+ - `create_topic(name, partitions, replication_factor, cleanup_policy: nil, retention: nil)`: Create a Kafka topic. `cleanup_policy` is a Kafka cleanup policy such as `"delete"`, `"compact"`, or `"delete,compact"`. `retention` is the message retention in seconds. A `nil` keyword uses the cluster default.
1084
1150
  - `delete_topic(name)`: Delete a Kafka topic.
1085
1151
 
1086
1152
  ### Prosody::EventHandler
@@ -1144,10 +1210,12 @@ Messages have the following attributes:
1144
1210
  - `timestamp` (Time): The timestamp when the message was created or sent.
1145
1211
  - `key` (String): The message key.
1146
1212
  - `payload` (`Payload`): The JSON-deserialized message payload.
1213
+ - `source_system` (String or nil): The source system of the producer that sent the message, or `nil` when the message has none.
1214
+ - `response_requested?` (Boolean): Whether the producer requested a response. When it is false, Prosody discards the handler result, so a handler can skip work that only builds the response.
1147
1215
 
1148
1216
  ### Prosody::ExciseMessage
1149
1217
 
1150
- An `ExciseMessage` has `topic`, `partition`, `offset`, `timestamp`, and `key` attributes. It has no `payload` attribute.
1218
+ An `ExciseMessage` has `topic`, `partition`, `offset`, `timestamp`, `key`, `source_system`, and `response_requested?` attributes. It has no `payload` attribute.
1151
1219
 
1152
1220
  ### Prosody::Context
1153
1221
 
@@ -1155,7 +1223,8 @@ Represents the current event context:
1155
1223
 
1156
1224
  - `should_cancel?`: Check if cancellation has been requested (includes timeout and shutdown).
1157
1225
  - `on_cancel`: Wait until cancellation occurs.
1158
- - `state(definition)`: Bind a registered collection for the current attempt. An unregistered or mismatched definition raises `PermanentStateError`. See [Keyed State](#keyed-state-2).
1226
+ - `demand`: A `Prosody::Demand` that tells whether this call is a normal delivery or a retry. `kind` is `:normal` or `:failure`, and `normal?` and `failure?` test it. `retries` is the number of retries: 0 for a normal delivery and 1 on the first retry. The count is an estimate. Keep an exact attempt count in keyed state if you need one.
1227
+ - `state(definition)`: Bind a registered collection for the current attempt. An unregistered or mismatched definition raises `PermanentStateError`. See [Keyed State](#keyed-state).
1159
1228
 
1160
1229
  Timer scheduling methods:
1161
1230
 
@@ -1186,6 +1255,7 @@ Definition constructors (each returns a frozen definition object used both in `C
1186
1255
 
1187
1256
  - `Prosody.value(name, ttl: nil, read_uncommitted: nil, published: nil, read_cache: nil)`
1188
1257
  - `Prosody.map(name, ttl: nil, keyset_limit: nil, read_uncommitted: nil, published: nil, read_cache: nil)`
1258
+ - `Prosody.set(name, ttl: nil, keyset_limit: nil, read_uncommitted: nil, published: nil, read_cache: nil)`
1189
1259
  - `Prosody.deque(name, ttl: nil, capacity: nil, read_uncommitted: nil, published: nil, read_cache: nil)`
1190
1260
  - `Prosody.message_value(name, ttl: nil, read_uncommitted: nil)`
1191
1261
  - `Prosody.message_map(name, ttl: nil, keyset_limit: nil, read_uncommitted: nil)`
@@ -1195,13 +1265,15 @@ Each constructor returns a `StateDefinition`. It exposes `name`, `kind`, `payloa
1195
1265
 
1196
1266
  Published readers take the user key as their first argument. `Prosody::PublishedValue` provides `get`.
1197
1267
 
1198
- `Prosody::PublishedMap` provides `get`, `get_many`, `key?`, `has_key?`, `include?`, and `member?`.
1268
+ `Prosody::PublishedMap` provides `get`, `get_many`, `key?`, `has_key?`, `include?`, `member?`, `contains_many`, and `empty?`.
1199
1269
 
1200
1270
  It provides `each` or `each_pair`, `each_key`, and `each_value`. The reverse methods are `reverse_each_pair`, `reverse_each_key`, and `reverse_each_value`.
1201
1271
 
1272
+ `Prosody::PublishedSet` provides `include?` or `member?`, `contains_many`, `empty?`, `each`, and `reverse_each`.
1273
+
1202
1274
  `Prosody::PublishedDeque` provides `get`, `length` or `size`, `empty?`, `first`, `last`, `each`, and `reverse_each`.
1203
1275
 
1204
- Traversal methods return an `Enumerator` without a block.
1276
+ Traversal methods return an `Enumerator` without a block. Every traversal accepts the optional [query keywords](#query-keywords) `from:`, `after:`, `to:`, `before:`, `range:`, and `limit:`. Map and set traversals also accept `prefix:`.
1205
1277
 
1206
1278
  `Prosody::ValueState`:
1207
1279
 
@@ -1210,10 +1282,16 @@ Traversal methods return an `Enumerator` without a block.
1210
1282
  `Prosody::MapState` (keys are `String`):
1211
1283
 
1212
1284
  - `get` / `[]`, `get_many`, `set` / `[]=`, `store`, `delete`, and `clear`
1213
- - `key?`, `has_key?`, `include?`, `member?`, `dig`, `slice`, `values_at`, `fetch`, and `fetch_values`
1285
+ - `key?`, `has_key?`, `include?`, `member?`, `contains_many`, `empty?`, `dig`, `slice`, `values_at`, `fetch`, and `fetch_values`
1214
1286
  - `each` / `each_pair`, `each_key`, and `each_value`, including each reverse form
1215
1287
  - `commit` and `rollback`
1216
1288
 
1289
+ `Prosody::SetState` (members are `String`):
1290
+
1291
+ - `add` / `<<`, `delete`, and `clear`, which return the set
1292
+ - `include?` / `member?`, `contains_many`, and `empty?`
1293
+ - `each` / `reverse_each`, `commit`, and `rollback`
1294
+
1217
1295
  `Prosody::DequeState`:
1218
1296
 
1219
1297
  - `push`, `append`, `<<`, `unshift`, `prepend`, `pop`, and `shift`
@@ -1223,8 +1301,7 @@ Traversal methods return an `Enumerator` without a block.
1223
1301
  Errors:
1224
1302
 
1225
1303
  - `Prosody::TransientStateError < Prosody::TransientError`: Reports a keyed-state error that Prosody can retry.
1226
- - `Prosody::PermanentStateError < Prosody::PermanentError`: Reports a keyed-state error that another attempt cannot resolve.
1227
- - `Prosody::NullValueError < Prosody::TransientStateError`: raised when a `nil` is written; use `clear`/`delete` instead.
1304
+ - `Prosody::PermanentStateError < Prosody::PermanentError`: Reports a keyed-state error that another attempt cannot resolve, such as a `nil` write.
1228
1305
 
1229
1306
  Handler error types:
1230
1307
 
@@ -3,7 +3,7 @@
3
3
  require "prosody"
4
4
  require "logger"
5
5
 
6
- # Keyed-state example: per-key value/map/deque collections that survive across
6
+ # Keyed-state example: per-key value/map/set/deque collections that survive across
7
7
  # events. Definitions are declared once and reused for both registration (on the
8
8
  # client) and binding (inside the handler). Every state op yields the fiber,
9
9
  # never the thread.
@@ -12,6 +12,7 @@ require "logger"
12
12
  # keyed_state.rbs for payload and state types checked by Steep.
13
13
  CART = Prosody.value("cart", ttl: 30 * 24 * 3600) # ValueState
14
14
  TOTALS = Prosody.map("totals") # keys are always String
15
+ SEEN = Prosody.set("seen", ttl: 7 * 24 * 3600) # String members, no payload
15
16
  BACKLOG = Prosody.message_deque("backlog", capacity: 100) # bounded window of messages
16
17
 
17
18
  class KeyedStateHandler < Prosody::EventHandler
@@ -19,6 +20,7 @@ class KeyedStateHandler < Prosody::EventHandler
19
20
  puts "Excise #{message.key}"
20
21
  context.state(CART).clear
21
22
  context.state(TOTALS).clear
23
+ context.state(SEEN).clear
22
24
  context.state(BACKLOG).clear
23
25
  nil
24
26
  end
@@ -29,6 +31,10 @@ class KeyedStateHandler < Prosody::EventHandler
29
31
 
30
32
  def on_message(context, message)
31
33
  payload = message.payload
34
+ seen = context.state(SEEN)
35
+ return if seen.include?(payload["order_id"]) # skip an order seen before
36
+ seen << payload["order_id"]
37
+
32
38
  cart = context.state(CART) # bound for this attempt only
33
39
  current = cart.get || {"items" => []} # Hash, or nil when absent
34
40
  cart.set(current.merge("items" => current["items"] + [payload["order_id"]]))
@@ -37,6 +43,8 @@ class KeyedStateHandler < Prosody::EventHandler
37
43
  totals.set(message.key, payload["total"])
38
44
  # Steep infers key as String and total as Integer from TOTALS's RBS type.
39
45
  totals.each_pair { |key, total| @logger.info(format_total(key, total)) }
46
+ # Query keywords narrow a traversal: here, the first ten keys after this one.
47
+ totals.each_key(after: message.key, limit: 10) { |key| @logger.info("next key: #{key}") }
40
48
 
41
49
  backlog = context.state(BACKLOG)
42
50
  backlog.push(message) # stores the full Prosody::Message
@@ -63,7 +71,7 @@ if __FILE__ == $PROGRAM_NAME
63
71
  mock: true,
64
72
  group_id: "keyed-state-example",
65
73
  subscribed_topics: "orders",
66
- state_collections: [CART, TOTALS, BACKLOG]
74
+ state_collections: [CART, TOTALS, SEEN, BACKLOG]
67
75
  )
68
76
  client.subscribe(KeyedStateHandler.new(logger: Logger.new($stdout)))
69
77
  client.shutdown
@@ -3,6 +3,7 @@ type keyed_state_order_event = { "order_id" => String, "total" => Integer }
3
3
 
4
4
  CART: Prosody::_ValueDefinition[keyed_state_cart]
5
5
  TOTALS: Prosody::_MapDefinition[Integer]
6
+ SEEN: Prosody::_SetDefinition
6
7
  BACKLOG: Prosody::_MessageDequeDefinition[keyed_state_order_event]
7
8
 
8
9
  class KeyedStateHandler < Prosody::EventHandler[keyed_state_order_event]
@@ -15,10 +15,10 @@ crossbeam-channel.workspace = true
15
15
  educe.workspace = true
16
16
  futures.workspace = true
17
17
  magnus = { workspace = true, default-features = false }
18
- mimalloc = { workspace = true, features = ["local_dynamic_tls"] }
19
18
  opentelemetry.workspace = true
20
19
  prosody = { workspace = true }
21
20
  rb-sys.workspace = true
21
+ rustfs-mimalloc = { workspace = true, features = ["local_dynamic_tls"] }
22
22
  serde = { workspace = true, features = ["derive"] }
23
23
  serde-untagged.workspace = true
24
24
  serde_json.workspace = true
@@ -4,9 +4,10 @@
4
4
  //! This module implements Ruby bindings for creating and deleting Kafka topics.
5
5
 
6
6
  use crate::bridge::Bridge;
7
- use crate::util::ensure_runtime_context;
7
+ use crate::util::{ForkGuard, ensure_runtime_context, seconds};
8
8
  use crate::{ROOT_MOD, id};
9
- use magnus::{Error, Module, Object, Ruby, function, method};
9
+ use magnus::scan_args::{get_kwargs, scan_args};
10
+ use magnus::{Error, Module, Object, RHash, Ruby, Value, function, method};
10
11
  use prosody::admin::{AdminConfiguration, ProsodyAdminClient, TopicConfiguration};
11
12
  use std::sync::Arc;
12
13
  use tracing::Span;
@@ -22,8 +23,8 @@ pub struct AdminClient {
22
23
  client: Arc<ProsodyAdminClient>,
23
24
  /// Bridge for executing asynchronous operations from Ruby
24
25
  bridge: Bridge,
25
- /// PID at construction time, used to detect post-fork usage
26
- pid: u32,
26
+ /// Refuses use in a forked child process
27
+ fork: ForkGuard,
27
28
  }
28
29
 
29
30
  impl AdminClient {
@@ -39,7 +40,6 @@ impl AdminClient {
39
40
  /// Returns a `Magnus::Error` if:
40
41
  /// - The client cannot be created with the provided bootstrap servers
41
42
  /// - The bridge is not initialized
42
- #[allow(clippy::needless_pass_by_value)]
43
43
  pub fn new(ruby: &Ruby, bootstrap_servers: Vec<String>) -> Result<Self, Error> {
44
44
  let _guard = ensure_runtime_context(ruby);
45
45
  let admin_config = AdminConfiguration::new(bootstrap_servers)
@@ -61,48 +61,48 @@ impl AdminClient {
61
61
  Ok(Self {
62
62
  client,
63
63
  bridge,
64
- pid: std::process::id(),
64
+ fork: ForkGuard::new("Prosody::AdminClient"),
65
65
  })
66
66
  }
67
67
 
68
- fn check_fork(ruby: &Ruby, this: &Self) -> Result<(), Error> {
69
- if std::process::id() != this.pid {
70
- return Err(Error::new(
71
- ruby.exception_runtime_error(),
72
- "Prosody::AdminClient cannot be used after fork. Create a new client in the child \
73
- process.",
74
- ));
75
- }
76
- Ok(())
77
- }
78
-
79
68
  /// Creates a new Kafka topic.
80
69
  ///
81
- /// # Arguments
82
- ///
83
- /// * `ruby` - Reference to the Ruby VM
84
- /// * `this` - The admin client instance
85
- /// * `name` - Name of the topic to create
86
- /// * `partition_count` - Number of partitions for the topic
87
- /// * `replication_factor` - Replication factor for the topic
70
+ /// Ruby calls it as `create_topic(name, partition_count,
71
+ /// replication_factor, cleanup_policy: nil, retention: nil)`. A `nil`
72
+ /// keyword uses the cluster default. `retention` is in seconds.
88
73
  ///
89
74
  /// # Errors
90
75
  ///
91
76
  /// Returns a `Magnus::Error` if:
77
+ /// - An argument has the wrong type, or `retention` has no `Duration` form
92
78
  /// - The topic creation fails
93
79
  /// - There's an issue with the asynchronous execution
94
- pub fn create_topic(
95
- ruby: &Ruby,
96
- this: &Self,
97
- name: String,
98
- partition_count: u16,
99
- replication_factor: u16,
100
- ) -> Result<(), Error> {
101
- Self::check_fork(ruby, this)?;
102
- let topic_config = TopicConfiguration::builder()
80
+ pub fn create_topic(ruby: &Ruby, this: &Self, args: &[Value]) -> Result<(), Error> {
81
+ this.fork.check(ruby)?;
82
+ let args = scan_args::<(String, u16, u16), (), (), (), RHash, ()>(args)?;
83
+ let (name, partition_count, replication_factor) = args.required;
84
+ let keywords = get_kwargs::<_, (), (Option<Option<String>>, Option<Option<f64>>), ()>(
85
+ args.keywords,
86
+ &[],
87
+ &["cleanup_policy", "retention"],
88
+ )?;
89
+ let (cleanup_policy, retention) = keywords.optional;
90
+
91
+ let mut builder = TopicConfiguration::builder();
92
+ builder
103
93
  .name(name)
104
94
  .partition_count(partition_count)
105
- .replication_factor(replication_factor)
95
+ .replication_factor(replication_factor);
96
+ if let Some(cleanup_policy) = cleanup_policy.flatten() {
97
+ builder.cleanup_policy(cleanup_policy);
98
+ }
99
+ if let Some(retention) = retention.flatten() {
100
+ builder.retention(
101
+ seconds("retention", retention)
102
+ .map_err(|error| Error::new(ruby.exception_arg_error(), error))?,
103
+ );
104
+ }
105
+ let topic_config = builder
106
106
  .build()
107
107
  .map_err(|error| Error::new(ruby.exception_runtime_error(), error.to_string()))?;
108
108
 
@@ -128,7 +128,7 @@ impl AdminClient {
128
128
  /// - The topic deletion fails
129
129
  /// - There's an issue with the asynchronous execution
130
130
  pub fn delete_topic(ruby: &Ruby, this: &Self, name: String) -> Result<(), Error> {
131
- Self::check_fork(ruby, this)?;
131
+ this.fork.check(ruby)?;
132
132
  let client = this.client.clone();
133
133
  let future = async move { client.delete_topic(&name).await };
134
134
 
@@ -156,7 +156,7 @@ pub fn init(ruby: &Ruby) -> Result<(), Error> {
156
156
  class.define_singleton_method("new", function!(AdminClient::new, 1))?;
157
157
  class.define_method(
158
158
  id!(ruby, "create_topic"),
159
- method!(AdminClient::create_topic, 3),
159
+ method!(AdminClient::create_topic, -1),
160
160
  )?;
161
161
  class.define_method(
162
162
  id!(ruby, "delete_topic"),
@@ -24,18 +24,13 @@ mod callback;
24
24
  /// Maximum number of commands to process in a single poll operation.
25
25
  const POLL_BATCH_SIZE: usize = 16;
26
26
 
27
- /// Lazily initialized reference to Ruby's Thread class.
28
- #[allow(clippy::expect_used)]
29
- pub static THREAD_CLASS: Lazy<RClass> = Lazy::new(|ruby| {
30
- ruby.class_object()
31
- .const_get(id!(ruby, "Thread"))
32
- .expect("Failed to load Thread class")
33
- });
34
-
35
27
  /// Lazily initialized reference to Ruby's `Thread::Queue` class.
36
- #[allow(clippy::expect_used)]
28
+ #[expect(
29
+ clippy::expect_used,
30
+ reason = "magnus Lazy takes an infallible initializer"
31
+ )]
37
32
  pub static QUEUE_CLASS: Lazy<RClass> = Lazy::new(|ruby| {
38
- ruby.get_inner(&THREAD_CLASS)
33
+ ruby.class_thread()
39
34
  .const_get(id!(ruby, "Queue"))
40
35
  .expect("Failed to load Queue class")
41
36
  });