prosody 0.5.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.config/rail.toml +26 -0
- data/.release-please-manifest.json +1 -1
- data/.ruby-version +1 -1
- data/.taplo.toml +1 -1
- data/AGENTS.md +29 -15
- data/CHANGELOG.md +7 -0
- data/CONFIGURATION.md +19 -16
- data/Cargo.lock +254 -235
- data/Cargo.toml +11 -8
- data/README.md +100 -23
- data/examples/keyed_state.rb +10 -2
- data/examples/keyed_state.rbs +1 -0
- data/ext/prosody/Cargo.toml +1 -1
- data/ext/prosody/src/admin.rs +36 -36
- data/ext/prosody/src/bridge/mod.rs +5 -10
- data/ext/prosody/src/client/config/connections.rs +170 -0
- data/ext/prosody/src/client/config/middleware.rs +184 -0
- data/ext/prosody/src/client/config/mod.rs +396 -0
- data/ext/prosody/src/client/config/state.rs +323 -0
- data/ext/prosody/src/client/mod.rs +31 -117
- data/ext/prosody/src/client/readers.rs +106 -0
- data/ext/prosody/src/client/request.rs +2 -2
- data/ext/prosody/src/client/support.rs +13 -32
- data/ext/prosody/src/gvl.rs +8 -6
- data/ext/prosody/src/handler/{context.rs → context/mod.rs} +36 -150
- data/ext/prosody/src/handler/context/vending.rs +138 -0
- data/ext/prosody/src/handler/message.rs +36 -0
- data/ext/prosody/src/handler/mod.rs +18 -8
- data/ext/prosody/src/handler/state/deque.rs +144 -0
- data/ext/prosody/src/handler/state/mod.rs +163 -268
- data/ext/prosody/src/handler/state/query.rs +264 -0
- data/ext/prosody/src/handler/state/registration.rs +15 -86
- data/ext/prosody/src/handler/state/scan.rs +33 -141
- data/ext/prosody/src/handler/state/set.rs +98 -0
- data/ext/prosody/src/lib.rs +54 -36
- data/ext/prosody/src/logging.rs +6 -6
- data/ext/prosody/src/published.rs +171 -152
- data/ext/prosody/src/scheduler/result.rs +2 -1
- data/ext/prosody/src/util.rs +65 -3
- data/lib/prosody/client.rb +32 -0
- data/lib/prosody/configuration.rb +20 -12
- data/lib/prosody/demand.rb +27 -0
- data/lib/prosody/native_stubs/client.rb +157 -0
- data/lib/prosody/native_stubs/context.rb +178 -0
- data/lib/prosody/native_stubs/message.rb +133 -0
- data/lib/prosody/native_stubs.rb +11 -956
- data/lib/prosody/state/deque.rb +253 -0
- data/lib/prosody/state/map.rb +313 -0
- data/lib/prosody/state/set.rb +121 -0
- data/lib/prosody/state/value.rb +58 -0
- data/lib/prosody/state.rb +157 -677
- data/lib/prosody/version.rb +1 -1
- data/lib/prosody.rb +2 -1
- data/sig/configuration.rbs +19 -10
- data/sig/prosody.rbs +37 -4
- data/sig/published.rbs +102 -0
- data/sig/state.rbs +109 -122
- data/typecheck/payload_types.rb +12 -0
- data/typecheck/payload_types.rbs +1 -0
- metadata +30 -11
- data/ext/prosody/src/client/config.rs +0 -1300
data/Cargo.toml
CHANGED
|
@@ -10,23 +10,22 @@ resolver = "2"
|
|
|
10
10
|
atomic-take = "1.1"
|
|
11
11
|
bumpalo = "3.20"
|
|
12
12
|
crossbeam-channel = "0.5"
|
|
13
|
-
educe = "0.
|
|
13
|
+
educe = "0.8"
|
|
14
14
|
futures = "0.3"
|
|
15
|
-
magnus = { version = "0.
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
opentelemetry = "0.32"
|
|
19
|
-
prosody = { version = "0.6.0", features = ["libz-static"] }
|
|
15
|
+
magnus = { version = "0.9", default-features = false }
|
|
16
|
+
opentelemetry = "0.33"
|
|
17
|
+
prosody = { version = "0.7.0", features = ["libz-static"] }
|
|
20
18
|
rb-sys = "0.9"
|
|
19
|
+
rustfs-mimalloc = "0.5.6"
|
|
21
20
|
serde = "1.0"
|
|
22
21
|
serde-untagged = "0.1"
|
|
23
22
|
serde_json = "1.0"
|
|
24
23
|
serde_magnus = "0.11"
|
|
25
24
|
thiserror = "2.0"
|
|
26
|
-
tokio = "1.
|
|
25
|
+
tokio = "1.53"
|
|
27
26
|
tokio-stream = "0.1"
|
|
28
27
|
tracing = "0.1"
|
|
29
|
-
tracing-opentelemetry = "0.
|
|
28
|
+
tracing-opentelemetry = "0.34"
|
|
30
29
|
tracing-subscriber = "0.3"
|
|
31
30
|
|
|
32
31
|
[profile.release]
|
|
@@ -78,3 +77,7 @@ unneeded_field_pattern = "warn"
|
|
|
78
77
|
unreachable = "warn"
|
|
79
78
|
unwrap_used = "deny"
|
|
80
79
|
verbose_file_reads = "warn"
|
|
80
|
+
|
|
81
|
+
[patch.crates-io]
|
|
82
|
+
# TODO: Use a crates.io release after serde_magnus supports magnus 0.9.
|
|
83
|
+
serde_magnus = { git = "https://github.com/hadronzoo/serde-magnus", rev = "27aee737829a6bca399289b6e64ce4bcc08bfd94" }
|
data/README.md
CHANGED
|
@@ -277,10 +277,10 @@ You can monitor the stall state programmatically using the client's methods:
|
|
|
277
277
|
|
|
278
278
|
```ruby
|
|
279
279
|
# Get the number of partitions currently assigned to this consumer
|
|
280
|
-
partition_count = client.
|
|
280
|
+
partition_count = client.assigned_partition_count
|
|
281
281
|
|
|
282
282
|
# Check if the consumer has stalled partitions
|
|
283
|
-
if client.
|
|
283
|
+
if client.stalled?
|
|
284
284
|
warn 'Consumer has stalled partitions'
|
|
285
285
|
end
|
|
286
286
|
```
|
|
@@ -299,7 +299,7 @@ Send a request from a handler or other application code. The Prosody client does
|
|
|
299
299
|
|
|
300
300
|
Do not rely on hash order. The hash contains one entry for each selected subsystem. A missing response becomes a timeout `Failure`; Prosody does not omit the subsystem. The request raises an error for request-level failures, such as invalid input, a Kafka send failure, or shutdown. Do not wait for a request if the current consumer group must process it for the same key. That group cannot process it until the handler returns.
|
|
301
301
|
|
|
302
|
-
Message and excise handler return values become successful outcomes. Each return value must have a JSON representation.
|
|
302
|
+
Message and excise handler return values become successful outcomes. Each return value must have a JSON representation. A return value without one is a transient handler error, so Prosody retries the message.
|
|
303
303
|
|
|
304
304
|
Set `subsystem` to `inventory` on the client that subscribes this handler.
|
|
305
305
|
|
|
@@ -626,12 +626,53 @@ State operations look synchronous. They yield the current fiber while Prosody pe
|
|
|
626
626
|
| Collection | JSON payload | Kafka message | Main operations |
|
|
627
627
|
| --- | --- | --- | --- |
|
|
628
628
|
| Value | `Prosody.value` | `Prosody.message_value` | `get`, `set`, `clear` |
|
|
629
|
-
| Ordered string map | `Prosody.map` | `Prosody.message_map` | `get`, `get_many`, `key?`, `set`, `delete`, `each_pair`, `each_key`, `clear` |
|
|
629
|
+
| Ordered string map | `Prosody.map` | `Prosody.message_map` | `get`, `get_many`, `key?`, `contains_many`, `empty?`, `set`, `delete`, `each_pair`, `each_key`, `clear` |
|
|
630
|
+
| Ordered string set | `Prosody.set` | - | `add` / `<<`, `delete`, `include?`, `contains_many`, `empty?`, `each`, `clear` |
|
|
630
631
|
| Deque | `Prosody.deque` | `Prosody.message_deque` | `push`, `unshift`, `pop`, `shift`, `get`, `length`, `each`, `clear` |
|
|
631
632
|
|
|
632
|
-
Map and deque scans return enumerators when called without a block. Map keys are strings.
|
|
633
|
+
Map, set, and deque scans return enumerators when called without a block. Map keys and set members are strings. A set stores membership only, so it has no payload type.
|
|
633
634
|
|
|
634
|
-
`nil`
|
|
635
|
+
A value, map, or deque read returns `nil` when no value is present. Do not store `nil` as a value. A `nil` write fails with a `PermanentStateError`. Use `clear` (value, deque) or `delete` (map) to remove a value.
|
|
636
|
+
|
|
637
|
+
A set handle mirrors Ruby's `Set`:
|
|
638
|
+
|
|
639
|
+
```ruby
|
|
640
|
+
SEEN_ORDERS = Prosody.set("seen-orders", ttl: 7 * 24 * 60 * 60)
|
|
641
|
+
|
|
642
|
+
def on_message(context, message)
|
|
643
|
+
seen = context.state(SEEN_ORDERS)
|
|
644
|
+
return if seen.include?(message.payload["order_id"])
|
|
645
|
+
|
|
646
|
+
seen << message.payload["order_id"]
|
|
647
|
+
fulfill(message)
|
|
648
|
+
end
|
|
649
|
+
```
|
|
650
|
+
|
|
651
|
+
### Query keywords
|
|
652
|
+
|
|
653
|
+
Every traversal method accepts optional query keywords. Prosody applies them in storage, so a query reads only the selected entries. The `each` and `each_*` methods iterate forward. The `reverse_each` and `reverse_each_*` methods iterate backward.
|
|
654
|
+
|
|
655
|
+
| Keyword | Selects |
|
|
656
|
+
| --- | --- |
|
|
657
|
+
| `from:` / `after:` | Starts at the key or position, or just after it, in iteration order |
|
|
658
|
+
| `to:` / `before:` | Stops at the key or position, or just before it, in iteration order |
|
|
659
|
+
| `range:` | Keeps keys or positions in a Ruby `Range`: `"a".."m"`, `"a"..."m"`, `.."m"`, or `"a"..`. Write the range in ascending order; it applies in both directions. A descending range is empty |
|
|
660
|
+
| `prefix:` | Keeps map keys or set members that start with the string |
|
|
661
|
+
| `limit:` | Stops after this many items; a positive `Integer` |
|
|
662
|
+
|
|
663
|
+
A reverse traversal starts at the high end. Keywords narrow the selection and never widen it. Pass at most one of `from:` and `after:`, and at most one of `to:` and `before:`. Set traversals select members. Deque positions count from the front and must be non-negative. To read the last N items, use `reverse_each(limit: N)`. Deques have no `prefix:`. A bad keyword raises `ArgumentError` or `TypeError`.
|
|
664
|
+
|
|
665
|
+
To read a map in pages, pass the last key of the previous page as `after:`:
|
|
666
|
+
|
|
667
|
+
```ruby
|
|
668
|
+
page = map.each_pair(prefix: "order:", limit: 100).to_a
|
|
669
|
+
until page.empty?
|
|
670
|
+
page.each { |key, order| archive(key, order) }
|
|
671
|
+
page = map.each_pair(prefix: "order:", after: page.last.first, limit: 100).to_a
|
|
672
|
+
end
|
|
673
|
+
```
|
|
674
|
+
|
|
675
|
+
The same pattern works backward with `reverse_each_pair`.
|
|
635
676
|
|
|
636
677
|
### When keyed-state changes become visible
|
|
637
678
|
|
|
@@ -643,6 +684,8 @@ This transaction applies only to keyed state. Some workflows need state changes
|
|
|
643
684
|
- `commit` commits the collection's pending changes before the handler ends. A later handler failure does not remove them.
|
|
644
685
|
- `rollback` discards pending changes since the last `commit`. It cannot undo committed changes.
|
|
645
686
|
|
|
687
|
+
`commit` and `rollback` return `:applied` when they wrote or discarded pending changes. They return `:no_op` when the collection had no pending changes.
|
|
688
|
+
|
|
646
689
|
### Published state
|
|
647
690
|
|
|
648
691
|
Some callers need only the current value for a key. They can accept a stale value or a race with a concurrent update.
|
|
@@ -665,7 +708,7 @@ current_order = context.state(CURRENT_ORDER)
|
|
|
665
708
|
current_order.set({"sku" => "book"})
|
|
666
709
|
```
|
|
667
710
|
|
|
668
|
-
Read published state from a handler or other application code. The Prosody client does not need an active subscription.
|
|
711
|
+
Read published state from a handler or other application code. The Prosody client does not need an active subscription. A client that only reads published state needs no `subscribed_topics`.
|
|
669
712
|
|
|
670
713
|
Use the subsystem and the same definition to open a reader:
|
|
671
714
|
|
|
@@ -674,9 +717,9 @@ order_reader = client.state("checkout", CURRENT_ORDER)
|
|
|
674
717
|
current_order = order_reader.get("customer-123")
|
|
675
718
|
```
|
|
676
719
|
|
|
677
|
-
The reader cannot see pending changes that exist only in a handler. It cannot change the collection. Each read takes an explicit key because no handler supplies one.
|
|
720
|
+
The reader cannot see pending changes that exist only in a handler. It cannot change the collection. Each read takes an explicit String key because no handler supplies one.
|
|
678
721
|
|
|
679
|
-
Map and deque readers fetch data in chunks. They do not load the complete collection before iteration starts. Readers return an `Enumerator` without a block.
|
|
722
|
+
Map, set, and deque readers fetch data in chunks. They do not load the complete collection before iteration starts. Readers return an `Enumerator` without a block. Reader traversals accept the same [query keywords](#query-keywords) after the key. A failed read raises `Prosody::TransientStateError` or `Prosody::PermanentStateError`, as an owned handle does, for point reads and traversals alike. A reader that Prosody cannot open, such as one with a zero `read_cache`, raises the same classes.
|
|
680
723
|
|
|
681
724
|
Use `reverse_each_pair`, `reverse_each_key`, `reverse_each_value`, or `reverse_each` for reverse traversal.
|
|
682
725
|
|
|
@@ -783,9 +826,9 @@ all traces to Ruby.
|
|
|
783
826
|
To use OpenTelemetry tracing with Prosody, you need to install the following gems:
|
|
784
827
|
|
|
785
828
|
```ruby
|
|
786
|
-
gem 'opentelemetry-sdk', '~> 1.
|
|
787
|
-
gem 'opentelemetry-api', '~> 1.
|
|
788
|
-
gem 'opentelemetry-exporter-otlp', '~> 0.
|
|
829
|
+
gem 'opentelemetry-sdk', '~> 1.13'
|
|
830
|
+
gem 'opentelemetry-api', '~> 1.10'
|
|
831
|
+
gem 'opentelemetry-exporter-otlp', '~> 0.36'
|
|
789
832
|
```
|
|
790
833
|
|
|
791
834
|
### Initializing Tracing
|
|
@@ -907,6 +950,16 @@ Call `shutdown` when the application terminates. It stops all client services an
|
|
|
907
950
|
client.shutdown
|
|
908
951
|
```
|
|
909
952
|
|
|
953
|
+
To scope a client to a block, use `Prosody::Client.open`. It yields the client and calls `shutdown` when the block exits, also when the block raises. It returns the value of the block.
|
|
954
|
+
|
|
955
|
+
```ruby
|
|
956
|
+
Prosody::Client.open(bootstrap_servers: "localhost:9092") do |client|
|
|
957
|
+
client.send_message("my-topic", "key", {"hello" => "world"})
|
|
958
|
+
end
|
|
959
|
+
```
|
|
960
|
+
|
|
961
|
+
Repeated `shutdown` calls wait for the same operation, so the block can also call `shutdown`.
|
|
962
|
+
|
|
910
963
|
Handle application shutdown with signal handlers:
|
|
911
964
|
|
|
912
965
|
```ruby
|
|
@@ -972,6 +1025,18 @@ Best practices:
|
|
|
972
1025
|
- Be cautious with permanent errors as they prevent retries and can result in data loss.
|
|
973
1026
|
- Consider system reliability and data consistency when classifying errors.
|
|
974
1027
|
|
|
1028
|
+
A handler can tell a retry from a normal delivery through `context.demand`:
|
|
1029
|
+
|
|
1030
|
+
```ruby
|
|
1031
|
+
def on_message(context, message)
|
|
1032
|
+
demand = context.demand
|
|
1033
|
+
logger.warn("retry #{demand.retries} for #{message.key}") if demand.failure?
|
|
1034
|
+
process(message)
|
|
1035
|
+
end
|
|
1036
|
+
```
|
|
1037
|
+
|
|
1038
|
+
`demand.retries` is the number of retries: 0 for a normal delivery and 1 on the first retry. After Prosody defers an event, the count starts again at 1. The count is an estimate. Keep an exact attempt count in keyed state if you need one.
|
|
1039
|
+
|
|
975
1040
|
### Handling Task Cancellation
|
|
976
1041
|
|
|
977
1042
|
Prosody cancels tasks during partition rebalancing, timeout, or shutdown. During shutdown, handlers run freely for most of the `shutdown_timeout` before the cancellation signal fires—giving in-flight work time to complete. When cancelled, your handler receives `Async::Stop` at the next yield point (I/O operation, sleep, etc.).
|
|
@@ -1064,23 +1129,24 @@ Ensure you have thoroughly tested your changes before merging to `main`.
|
|
|
1064
1129
|
### Prosody::Client
|
|
1065
1130
|
|
|
1066
1131
|
- `new(config)` or `new(**options)`: Create a client from a `Configuration`, hash, or keyword options.
|
|
1132
|
+
- `open(config) { |client| ... }`: Create a client, yield it, and shut it down when the block exits. It returns the value of the block.
|
|
1067
1133
|
- `send_message(String topic, String key, Prosody::json_value payload)`: Send a JSON-serializable message.
|
|
1068
1134
|
- `excise(String topic, String key)`: Send an excise record for a key.
|
|
1069
1135
|
- `request(topic:, key:, payload:, subsystems:, timeout:)`: Return one outcome for each subsystem.
|
|
1070
1136
|
- `request_excise(topic:, key:, subsystems:, timeout:)`: Return one excise outcome for each subsystem.
|
|
1071
1137
|
- `consumer_state`: Get the client state (`:shut_down`, `:unconfigured`, `:configured`, or `:running`).
|
|
1072
1138
|
- `source_system`: Get the source system identifier configured for the client.
|
|
1073
|
-
- `state(subsystem, definition)`: Open a typed, read-only published value, map, or deque.
|
|
1139
|
+
- `state(subsystem, definition)`: Open a typed, read-only published value, map, set, or deque.
|
|
1074
1140
|
- `subscribe(handler)`: Start event processing with the specified handler.
|
|
1075
1141
|
- `unsubscribe`: Stop the consumer. You can subscribe again later.
|
|
1076
1142
|
- `shutdown`: Stop all client services. Concurrent and repeated calls wait for the same operation.
|
|
1077
|
-
- `
|
|
1078
|
-
- `
|
|
1143
|
+
- `assigned_partition_count`: Get the number of partitions currently assigned to this consumer.
|
|
1144
|
+
- `stalled?`: Check if the consumer has stalled partitions.
|
|
1079
1145
|
|
|
1080
1146
|
### Prosody::AdminClient
|
|
1081
1147
|
|
|
1082
1148
|
- `new(bootstrap_servers)`: Create an admin client for the specified Kafka servers.
|
|
1083
|
-
- `create_topic(name, partitions, replication_factor)`: Create a Kafka topic.
|
|
1149
|
+
- `create_topic(name, partitions, replication_factor, cleanup_policy: nil, retention: nil)`: Create a Kafka topic. `cleanup_policy` is a Kafka cleanup policy such as `"delete"`, `"compact"`, or `"delete,compact"`. `retention` is the message retention in seconds. A `nil` keyword uses the cluster default.
|
|
1084
1150
|
- `delete_topic(name)`: Delete a Kafka topic.
|
|
1085
1151
|
|
|
1086
1152
|
### Prosody::EventHandler
|
|
@@ -1144,10 +1210,12 @@ Messages have the following attributes:
|
|
|
1144
1210
|
- `timestamp` (Time): The timestamp when the message was created or sent.
|
|
1145
1211
|
- `key` (String): The message key.
|
|
1146
1212
|
- `payload` (`Payload`): The JSON-deserialized message payload.
|
|
1213
|
+
- `source_system` (String or nil): The source system of the producer that sent the message, or `nil` when the message has none.
|
|
1214
|
+
- `response_requested?` (Boolean): Whether the producer requested a response. When it is false, Prosody discards the handler result, so a handler can skip work that only builds the response.
|
|
1147
1215
|
|
|
1148
1216
|
### Prosody::ExciseMessage
|
|
1149
1217
|
|
|
1150
|
-
An `ExciseMessage` has `topic`, `partition`, `offset`, `timestamp`,
|
|
1218
|
+
An `ExciseMessage` has `topic`, `partition`, `offset`, `timestamp`, `key`, `source_system`, and `response_requested?` attributes. It has no `payload` attribute.
|
|
1151
1219
|
|
|
1152
1220
|
### Prosody::Context
|
|
1153
1221
|
|
|
@@ -1155,7 +1223,8 @@ Represents the current event context:
|
|
|
1155
1223
|
|
|
1156
1224
|
- `should_cancel?`: Check if cancellation has been requested (includes timeout and shutdown).
|
|
1157
1225
|
- `on_cancel`: Wait until cancellation occurs.
|
|
1158
|
-
- `
|
|
1226
|
+
- `demand`: A `Prosody::Demand` that tells whether this call is a normal delivery or a retry. `kind` is `:normal` or `:failure`, and `normal?` and `failure?` test it. `retries` is the number of retries: 0 for a normal delivery and 1 on the first retry. The count is an estimate. Keep an exact attempt count in keyed state if you need one.
|
|
1227
|
+
- `state(definition)`: Bind a registered collection for the current attempt. An unregistered or mismatched definition raises `PermanentStateError`. See [Keyed State](#keyed-state).
|
|
1159
1228
|
|
|
1160
1229
|
Timer scheduling methods:
|
|
1161
1230
|
|
|
@@ -1186,6 +1255,7 @@ Definition constructors (each returns a frozen definition object used both in `C
|
|
|
1186
1255
|
|
|
1187
1256
|
- `Prosody.value(name, ttl: nil, read_uncommitted: nil, published: nil, read_cache: nil)`
|
|
1188
1257
|
- `Prosody.map(name, ttl: nil, keyset_limit: nil, read_uncommitted: nil, published: nil, read_cache: nil)`
|
|
1258
|
+
- `Prosody.set(name, ttl: nil, keyset_limit: nil, read_uncommitted: nil, published: nil, read_cache: nil)`
|
|
1189
1259
|
- `Prosody.deque(name, ttl: nil, capacity: nil, read_uncommitted: nil, published: nil, read_cache: nil)`
|
|
1190
1260
|
- `Prosody.message_value(name, ttl: nil, read_uncommitted: nil)`
|
|
1191
1261
|
- `Prosody.message_map(name, ttl: nil, keyset_limit: nil, read_uncommitted: nil)`
|
|
@@ -1195,13 +1265,15 @@ Each constructor returns a `StateDefinition`. It exposes `name`, `kind`, `payloa
|
|
|
1195
1265
|
|
|
1196
1266
|
Published readers take the user key as their first argument. `Prosody::PublishedValue` provides `get`.
|
|
1197
1267
|
|
|
1198
|
-
`Prosody::PublishedMap` provides `get`, `get_many`, `key?`, `has_key?`, `include?`, and `
|
|
1268
|
+
`Prosody::PublishedMap` provides `get`, `get_many`, `key?`, `has_key?`, `include?`, `member?`, `contains_many`, and `empty?`.
|
|
1199
1269
|
|
|
1200
1270
|
It provides `each` or `each_pair`, `each_key`, and `each_value`. The reverse methods are `reverse_each_pair`, `reverse_each_key`, and `reverse_each_value`.
|
|
1201
1271
|
|
|
1272
|
+
`Prosody::PublishedSet` provides `include?` or `member?`, `contains_many`, `empty?`, `each`, and `reverse_each`.
|
|
1273
|
+
|
|
1202
1274
|
`Prosody::PublishedDeque` provides `get`, `length` or `size`, `empty?`, `first`, `last`, `each`, and `reverse_each`.
|
|
1203
1275
|
|
|
1204
|
-
Traversal methods return an `Enumerator` without a block.
|
|
1276
|
+
Traversal methods return an `Enumerator` without a block. Every traversal accepts the optional [query keywords](#query-keywords) `from:`, `after:`, `to:`, `before:`, `range:`, and `limit:`. Map and set traversals also accept `prefix:`.
|
|
1205
1277
|
|
|
1206
1278
|
`Prosody::ValueState`:
|
|
1207
1279
|
|
|
@@ -1210,10 +1282,16 @@ Traversal methods return an `Enumerator` without a block.
|
|
|
1210
1282
|
`Prosody::MapState` (keys are `String`):
|
|
1211
1283
|
|
|
1212
1284
|
- `get` / `[]`, `get_many`, `set` / `[]=`, `store`, `delete`, and `clear`
|
|
1213
|
-
- `key?`, `has_key?`, `include?`, `member?`, `dig`, `slice`, `values_at`, `fetch`, and `fetch_values`
|
|
1285
|
+
- `key?`, `has_key?`, `include?`, `member?`, `contains_many`, `empty?`, `dig`, `slice`, `values_at`, `fetch`, and `fetch_values`
|
|
1214
1286
|
- `each` / `each_pair`, `each_key`, and `each_value`, including each reverse form
|
|
1215
1287
|
- `commit` and `rollback`
|
|
1216
1288
|
|
|
1289
|
+
`Prosody::SetState` (members are `String`):
|
|
1290
|
+
|
|
1291
|
+
- `add` / `<<`, `delete`, and `clear`, which return the set
|
|
1292
|
+
- `include?` / `member?`, `contains_many`, and `empty?`
|
|
1293
|
+
- `each` / `reverse_each`, `commit`, and `rollback`
|
|
1294
|
+
|
|
1217
1295
|
`Prosody::DequeState`:
|
|
1218
1296
|
|
|
1219
1297
|
- `push`, `append`, `<<`, `unshift`, `prepend`, `pop`, and `shift`
|
|
@@ -1223,8 +1301,7 @@ Traversal methods return an `Enumerator` without a block.
|
|
|
1223
1301
|
Errors:
|
|
1224
1302
|
|
|
1225
1303
|
- `Prosody::TransientStateError < Prosody::TransientError`: Reports a keyed-state error that Prosody can retry.
|
|
1226
|
-
- `Prosody::PermanentStateError < Prosody::PermanentError`: Reports a keyed-state error that another attempt cannot resolve.
|
|
1227
|
-
- `Prosody::NullValueError < Prosody::TransientStateError`: raised when a `nil` is written; use `clear`/`delete` instead.
|
|
1304
|
+
- `Prosody::PermanentStateError < Prosody::PermanentError`: Reports a keyed-state error that another attempt cannot resolve, such as a `nil` write.
|
|
1228
1305
|
|
|
1229
1306
|
Handler error types:
|
|
1230
1307
|
|
data/examples/keyed_state.rb
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
require "prosody"
|
|
4
4
|
require "logger"
|
|
5
5
|
|
|
6
|
-
# Keyed-state example: per-key value/map/deque collections that survive across
|
|
6
|
+
# Keyed-state example: per-key value/map/set/deque collections that survive across
|
|
7
7
|
# events. Definitions are declared once and reused for both registration (on the
|
|
8
8
|
# client) and binding (inside the handler). Every state op yields the fiber,
|
|
9
9
|
# never the thread.
|
|
@@ -12,6 +12,7 @@ require "logger"
|
|
|
12
12
|
# keyed_state.rbs for payload and state types checked by Steep.
|
|
13
13
|
CART = Prosody.value("cart", ttl: 30 * 24 * 3600) # ValueState
|
|
14
14
|
TOTALS = Prosody.map("totals") # keys are always String
|
|
15
|
+
SEEN = Prosody.set("seen", ttl: 7 * 24 * 3600) # String members, no payload
|
|
15
16
|
BACKLOG = Prosody.message_deque("backlog", capacity: 100) # bounded window of messages
|
|
16
17
|
|
|
17
18
|
class KeyedStateHandler < Prosody::EventHandler
|
|
@@ -19,6 +20,7 @@ class KeyedStateHandler < Prosody::EventHandler
|
|
|
19
20
|
puts "Excise #{message.key}"
|
|
20
21
|
context.state(CART).clear
|
|
21
22
|
context.state(TOTALS).clear
|
|
23
|
+
context.state(SEEN).clear
|
|
22
24
|
context.state(BACKLOG).clear
|
|
23
25
|
nil
|
|
24
26
|
end
|
|
@@ -29,6 +31,10 @@ class KeyedStateHandler < Prosody::EventHandler
|
|
|
29
31
|
|
|
30
32
|
def on_message(context, message)
|
|
31
33
|
payload = message.payload
|
|
34
|
+
seen = context.state(SEEN)
|
|
35
|
+
return if seen.include?(payload["order_id"]) # skip an order seen before
|
|
36
|
+
seen << payload["order_id"]
|
|
37
|
+
|
|
32
38
|
cart = context.state(CART) # bound for this attempt only
|
|
33
39
|
current = cart.get || {"items" => []} # Hash, or nil when absent
|
|
34
40
|
cart.set(current.merge("items" => current["items"] + [payload["order_id"]]))
|
|
@@ -37,6 +43,8 @@ class KeyedStateHandler < Prosody::EventHandler
|
|
|
37
43
|
totals.set(message.key, payload["total"])
|
|
38
44
|
# Steep infers key as String and total as Integer from TOTALS's RBS type.
|
|
39
45
|
totals.each_pair { |key, total| @logger.info(format_total(key, total)) }
|
|
46
|
+
# Query keywords narrow a traversal: here, the first ten keys after this one.
|
|
47
|
+
totals.each_key(after: message.key, limit: 10) { |key| @logger.info("next key: #{key}") }
|
|
40
48
|
|
|
41
49
|
backlog = context.state(BACKLOG)
|
|
42
50
|
backlog.push(message) # stores the full Prosody::Message
|
|
@@ -63,7 +71,7 @@ if __FILE__ == $PROGRAM_NAME
|
|
|
63
71
|
mock: true,
|
|
64
72
|
group_id: "keyed-state-example",
|
|
65
73
|
subscribed_topics: "orders",
|
|
66
|
-
state_collections: [CART, TOTALS, BACKLOG]
|
|
74
|
+
state_collections: [CART, TOTALS, SEEN, BACKLOG]
|
|
67
75
|
)
|
|
68
76
|
client.subscribe(KeyedStateHandler.new(logger: Logger.new($stdout)))
|
|
69
77
|
client.shutdown
|
data/examples/keyed_state.rbs
CHANGED
|
@@ -3,6 +3,7 @@ type keyed_state_order_event = { "order_id" => String, "total" => Integer }
|
|
|
3
3
|
|
|
4
4
|
CART: Prosody::_ValueDefinition[keyed_state_cart]
|
|
5
5
|
TOTALS: Prosody::_MapDefinition[Integer]
|
|
6
|
+
SEEN: Prosody::_SetDefinition
|
|
6
7
|
BACKLOG: Prosody::_MessageDequeDefinition[keyed_state_order_event]
|
|
7
8
|
|
|
8
9
|
class KeyedStateHandler < Prosody::EventHandler[keyed_state_order_event]
|
data/ext/prosody/Cargo.toml
CHANGED
|
@@ -15,10 +15,10 @@ crossbeam-channel.workspace = true
|
|
|
15
15
|
educe.workspace = true
|
|
16
16
|
futures.workspace = true
|
|
17
17
|
magnus = { workspace = true, default-features = false }
|
|
18
|
-
mimalloc = { workspace = true, features = ["local_dynamic_tls"] }
|
|
19
18
|
opentelemetry.workspace = true
|
|
20
19
|
prosody = { workspace = true }
|
|
21
20
|
rb-sys.workspace = true
|
|
21
|
+
rustfs-mimalloc = { workspace = true, features = ["local_dynamic_tls"] }
|
|
22
22
|
serde = { workspace = true, features = ["derive"] }
|
|
23
23
|
serde-untagged.workspace = true
|
|
24
24
|
serde_json.workspace = true
|
data/ext/prosody/src/admin.rs
CHANGED
|
@@ -4,9 +4,10 @@
|
|
|
4
4
|
//! This module implements Ruby bindings for creating and deleting Kafka topics.
|
|
5
5
|
|
|
6
6
|
use crate::bridge::Bridge;
|
|
7
|
-
use crate::util::ensure_runtime_context;
|
|
7
|
+
use crate::util::{ForkGuard, ensure_runtime_context, seconds};
|
|
8
8
|
use crate::{ROOT_MOD, id};
|
|
9
|
-
use magnus::{
|
|
9
|
+
use magnus::scan_args::{get_kwargs, scan_args};
|
|
10
|
+
use magnus::{Error, Module, Object, RHash, Ruby, Value, function, method};
|
|
10
11
|
use prosody::admin::{AdminConfiguration, ProsodyAdminClient, TopicConfiguration};
|
|
11
12
|
use std::sync::Arc;
|
|
12
13
|
use tracing::Span;
|
|
@@ -22,8 +23,8 @@ pub struct AdminClient {
|
|
|
22
23
|
client: Arc<ProsodyAdminClient>,
|
|
23
24
|
/// Bridge for executing asynchronous operations from Ruby
|
|
24
25
|
bridge: Bridge,
|
|
25
|
-
///
|
|
26
|
-
|
|
26
|
+
/// Refuses use in a forked child process
|
|
27
|
+
fork: ForkGuard,
|
|
27
28
|
}
|
|
28
29
|
|
|
29
30
|
impl AdminClient {
|
|
@@ -39,7 +40,6 @@ impl AdminClient {
|
|
|
39
40
|
/// Returns a `Magnus::Error` if:
|
|
40
41
|
/// - The client cannot be created with the provided bootstrap servers
|
|
41
42
|
/// - The bridge is not initialized
|
|
42
|
-
#[allow(clippy::needless_pass_by_value)]
|
|
43
43
|
pub fn new(ruby: &Ruby, bootstrap_servers: Vec<String>) -> Result<Self, Error> {
|
|
44
44
|
let _guard = ensure_runtime_context(ruby);
|
|
45
45
|
let admin_config = AdminConfiguration::new(bootstrap_servers)
|
|
@@ -61,48 +61,48 @@ impl AdminClient {
|
|
|
61
61
|
Ok(Self {
|
|
62
62
|
client,
|
|
63
63
|
bridge,
|
|
64
|
-
|
|
64
|
+
fork: ForkGuard::new("Prosody::AdminClient"),
|
|
65
65
|
})
|
|
66
66
|
}
|
|
67
67
|
|
|
68
|
-
fn check_fork(ruby: &Ruby, this: &Self) -> Result<(), Error> {
|
|
69
|
-
if std::process::id() != this.pid {
|
|
70
|
-
return Err(Error::new(
|
|
71
|
-
ruby.exception_runtime_error(),
|
|
72
|
-
"Prosody::AdminClient cannot be used after fork. Create a new client in the child \
|
|
73
|
-
process.",
|
|
74
|
-
));
|
|
75
|
-
}
|
|
76
|
-
Ok(())
|
|
77
|
-
}
|
|
78
|
-
|
|
79
68
|
/// Creates a new Kafka topic.
|
|
80
69
|
///
|
|
81
|
-
///
|
|
82
|
-
///
|
|
83
|
-
///
|
|
84
|
-
/// * `this` - The admin client instance
|
|
85
|
-
/// * `name` - Name of the topic to create
|
|
86
|
-
/// * `partition_count` - Number of partitions for the topic
|
|
87
|
-
/// * `replication_factor` - Replication factor for the topic
|
|
70
|
+
/// Ruby calls it as `create_topic(name, partition_count,
|
|
71
|
+
/// replication_factor, cleanup_policy: nil, retention: nil)`. A `nil`
|
|
72
|
+
/// keyword uses the cluster default. `retention` is in seconds.
|
|
88
73
|
///
|
|
89
74
|
/// # Errors
|
|
90
75
|
///
|
|
91
76
|
/// Returns a `Magnus::Error` if:
|
|
77
|
+
/// - An argument has the wrong type, or `retention` has no `Duration` form
|
|
92
78
|
/// - The topic creation fails
|
|
93
79
|
/// - There's an issue with the asynchronous execution
|
|
94
|
-
pub fn create_topic(
|
|
95
|
-
ruby
|
|
96
|
-
|
|
97
|
-
name
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
80
|
+
pub fn create_topic(ruby: &Ruby, this: &Self, args: &[Value]) -> Result<(), Error> {
|
|
81
|
+
this.fork.check(ruby)?;
|
|
82
|
+
let args = scan_args::<(String, u16, u16), (), (), (), RHash, ()>(args)?;
|
|
83
|
+
let (name, partition_count, replication_factor) = args.required;
|
|
84
|
+
let keywords = get_kwargs::<_, (), (Option<Option<String>>, Option<Option<f64>>), ()>(
|
|
85
|
+
args.keywords,
|
|
86
|
+
&[],
|
|
87
|
+
&["cleanup_policy", "retention"],
|
|
88
|
+
)?;
|
|
89
|
+
let (cleanup_policy, retention) = keywords.optional;
|
|
90
|
+
|
|
91
|
+
let mut builder = TopicConfiguration::builder();
|
|
92
|
+
builder
|
|
103
93
|
.name(name)
|
|
104
94
|
.partition_count(partition_count)
|
|
105
|
-
.replication_factor(replication_factor)
|
|
95
|
+
.replication_factor(replication_factor);
|
|
96
|
+
if let Some(cleanup_policy) = cleanup_policy.flatten() {
|
|
97
|
+
builder.cleanup_policy(cleanup_policy);
|
|
98
|
+
}
|
|
99
|
+
if let Some(retention) = retention.flatten() {
|
|
100
|
+
builder.retention(
|
|
101
|
+
seconds("retention", retention)
|
|
102
|
+
.map_err(|error| Error::new(ruby.exception_arg_error(), error))?,
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
let topic_config = builder
|
|
106
106
|
.build()
|
|
107
107
|
.map_err(|error| Error::new(ruby.exception_runtime_error(), error.to_string()))?;
|
|
108
108
|
|
|
@@ -128,7 +128,7 @@ impl AdminClient {
|
|
|
128
128
|
/// - The topic deletion fails
|
|
129
129
|
/// - There's an issue with the asynchronous execution
|
|
130
130
|
pub fn delete_topic(ruby: &Ruby, this: &Self, name: String) -> Result<(), Error> {
|
|
131
|
-
|
|
131
|
+
this.fork.check(ruby)?;
|
|
132
132
|
let client = this.client.clone();
|
|
133
133
|
let future = async move { client.delete_topic(&name).await };
|
|
134
134
|
|
|
@@ -156,7 +156,7 @@ pub fn init(ruby: &Ruby) -> Result<(), Error> {
|
|
|
156
156
|
class.define_singleton_method("new", function!(AdminClient::new, 1))?;
|
|
157
157
|
class.define_method(
|
|
158
158
|
id!(ruby, "create_topic"),
|
|
159
|
-
method!(AdminClient::create_topic,
|
|
159
|
+
method!(AdminClient::create_topic, -1),
|
|
160
160
|
)?;
|
|
161
161
|
class.define_method(
|
|
162
162
|
id!(ruby, "delete_topic"),
|
|
@@ -24,18 +24,13 @@ mod callback;
|
|
|
24
24
|
/// Maximum number of commands to process in a single poll operation.
|
|
25
25
|
const POLL_BATCH_SIZE: usize = 16;
|
|
26
26
|
|
|
27
|
-
/// Lazily initialized reference to Ruby's Thread class.
|
|
28
|
-
#[allow(clippy::expect_used)]
|
|
29
|
-
pub static THREAD_CLASS: Lazy<RClass> = Lazy::new(|ruby| {
|
|
30
|
-
ruby.class_object()
|
|
31
|
-
.const_get(id!(ruby, "Thread"))
|
|
32
|
-
.expect("Failed to load Thread class")
|
|
33
|
-
});
|
|
34
|
-
|
|
35
27
|
/// Lazily initialized reference to Ruby's `Thread::Queue` class.
|
|
36
|
-
#[
|
|
28
|
+
#[expect(
|
|
29
|
+
clippy::expect_used,
|
|
30
|
+
reason = "magnus Lazy takes an infallible initializer"
|
|
31
|
+
)]
|
|
37
32
|
pub static QUEUE_CLASS: Lazy<RClass> = Lazy::new(|ruby| {
|
|
38
|
-
ruby.
|
|
33
|
+
ruby.class_thread()
|
|
39
34
|
.const_get(id!(ruby, "Queue"))
|
|
40
35
|
.expect("Failed to load Queue class")
|
|
41
36
|
});
|