solid_objects 0.16.1 → 0.17.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +100 -0
  3. data/README.md +9 -0
  4. data/Rakefile +5 -0
  5. data/app/models/solid_objects/message.rb +15 -0
  6. data/docs/agents.md +253 -0
  7. data/docs/architecture.md +16 -6
  8. data/docs/authorization.md +10 -3
  9. data/docs/observability.md +186 -0
  10. data/docs/operations.md +10 -8
  11. data/docs/realtime.md +4 -3
  12. data/docs/reminders.md +3 -1
  13. data/docs/roadmap.md +17 -1
  14. data/docs/security.md +2 -2
  15. data/docs/transmission.md +17 -6
  16. data/docs/virtual-actors.md +226 -0
  17. data/examples/quickstart/README.md +246 -0
  18. data/examples/quickstart/app/actors/ticket_sale.rb +22 -0
  19. data/examples/quickstart/smoke.rb +421 -0
  20. data/lib/solid_objects/activation.rb +6 -3
  21. data/lib/solid_objects/actor.rb +29 -2
  22. data/lib/solid_objects/actor_channel.rb +9 -1
  23. data/lib/solid_objects/actor_snapshot.rb +14 -8
  24. data/lib/solid_objects/broadcast_executor.rb +1 -0
  25. data/lib/solid_objects/client.rb +15 -1
  26. data/lib/solid_objects/configuration.rb +4 -0
  27. data/lib/solid_objects/database_adapters/sqlite.rb +7 -1
  28. data/lib/solid_objects/diagnostics.rb +83 -0
  29. data/lib/solid_objects/effect_executor.rb +4 -0
  30. data/lib/solid_objects/errors.rb +3 -0
  31. data/lib/solid_objects/executor.rb +34 -16
  32. data/lib/solid_objects/instrumentation.rb +22 -1
  33. data/lib/solid_objects/log_subscriber.rb +1 -1
  34. data/lib/solid_objects/mailbox.rb +5 -1
  35. data/lib/solid_objects/message_reference.rb +9 -9
  36. data/lib/solid_objects/observer_registry.rb +47 -0
  37. data/lib/solid_objects/payload_broadcast.rb +9 -8
  38. data/lib/solid_objects/reference.rb +19 -0
  39. data/lib/solid_objects/reminder_scheduler.rb +5 -0
  40. data/lib/solid_objects/state_snapshot.rb +1 -0
  41. data/lib/solid_objects/supervisor.rb +3 -6
  42. data/lib/solid_objects/synchronous_invocation.rb +1 -21
  43. data/lib/solid_objects/telemetry.rb +137 -0
  44. data/lib/solid_objects/version.rb +1 -1
  45. data/lib/solid_objects/wake_up_adapters/postgresql.rb +1 -2
  46. data/lib/solid_objects/wake_up_adapters/redis.rb +1 -2
  47. data/lib/solid_objects/worker.rb +1 -2
  48. data/lib/solid_objects.rb +10 -0
  49. data/sig/generated/lib/solid_objects/actor.rbs +9 -0
  50. data/sig/generated/lib/solid_objects/actor_channel.rbs +3 -0
  51. data/sig/generated/lib/solid_objects/actor_snapshot.rbs +7 -2
  52. data/sig/generated/lib/solid_objects/client.rbs +3 -0
  53. data/sig/generated/lib/solid_objects/configuration.rbs +7 -3
  54. data/sig/generated/lib/solid_objects/diagnostics.rbs +25 -0
  55. data/sig/generated/lib/solid_objects/errors.rbs +3 -0
  56. data/sig/generated/lib/solid_objects/executor.rbs +10 -2
  57. data/sig/generated/lib/solid_objects/message_reference.rbs +6 -6
  58. data/sig/generated/lib/solid_objects/observer_registry.rbs +29 -0
  59. data/sig/generated/lib/solid_objects/payload_broadcast.rbs +2 -4
  60. data/sig/generated/lib/solid_objects/reference.rbs +9 -0
  61. data/sig/generated/lib/solid_objects/synchronous_invocation.rbs +0 -3
  62. data/sig/generated/lib/solid_objects/telemetry.rbs +30 -0
  63. data/sig/generated/lib/solid_objects.rbs +3 -0
  64. data/sig/generated/models/solid_objects/message.rbs +3 -0
  65. data/sig/public/json_value.rbs +3 -0
  66. data/sig/public/telemetry.rbs +49 -0
  67. data/sig/support/framework.rbs +5 -0
  68. metadata +24 -9
@@ -24,15 +24,19 @@ module SolidObjects
24
24
  @actor = build_actor(instance)
25
25
  @last_used_at = monotonic_now
26
26
  @pass_exhausted = false
27
+ SolidObjects.instrument(:"activation.started", instance_id: instance.id, actor_type: instance.actor_type, actor_id: instance.actor_id, owner_id: lease.owner_id, generation: lease.generation)
27
28
  actor.activate
28
29
  SolidObjects.instrument(
29
- :"activation.started",
30
+ :"activation.completed",
30
31
  instance_id: instance.id,
31
32
  actor_type: instance.actor_type,
32
33
  actor_id: instance.actor_id,
33
34
  owner_id: lease.owner_id,
34
35
  generation: lease.generation
35
36
  )
37
+ rescue => error
38
+ SolidObjects.instrument(:"activation.failed", instance_id: lease.instance_id, actor_type: instance&.actor_type, actor_id: instance&.actor_id, owner_id: lease.owner_id, generation: lease.generation, error_class: error.class.name)
39
+ raise
36
40
  end
37
41
 
38
42
  # @rbs () -> Integer
@@ -104,8 +108,7 @@ module SolidObjects
104
108
  actor_id: actor.actor_id,
105
109
  owner_id: lease.owner_id,
106
110
  generation: lease.generation,
107
- error_class: error.class.name,
108
- error_message: error.message
111
+ error_class: error.class.name
109
112
  )
110
113
  SolidObjects.configuration.logger.error(
111
114
  "SolidObjects activation deactivation failed " \
@@ -459,7 +459,9 @@ module SolidObjects
459
459
 
460
460
  # @rbs () -> Hash[String, untyped]
461
461
  def observable_values
462
- guard_application_writes("observables") do
462
+ return {} if self.class.definition.observables.empty?
463
+
464
+ read_projection("observables") do
463
465
  self.class.definition.observables.each_with_object({}) do |(name, handler), values|
464
466
  values[name.to_s] = Serialization.dump(instance_exec(&handler.block))
465
467
  end
@@ -472,7 +474,7 @@ module SolidObjects
472
474
  handler = self.class.definition.observables[observable_name]
473
475
  raise UnknownMessage, "unknown observable #{name.inspect}" unless handler
474
476
 
475
- guard_application_writes("observable.#{observable_name}") do
477
+ read_projection("observable.#{observable_name}") do
476
478
  Serialization.dump(instance_exec(&handler.block))
477
479
  end
478
480
  end
@@ -537,6 +539,24 @@ module SolidObjects
537
539
  outbound_message_intents.clear
538
540
  end
539
541
 
542
+ # @rbs () -> Integer
543
+ def intent_count
544
+ effect_intents.length + effect_recovery_intents.length + commit_action_intents.length +
545
+ reminder_intents.length + outbound_message_intents.length
546
+ end
547
+
548
+ # @rbs [Result] (String) { () -> Result } -> Result
549
+ def read_projection(operation)
550
+ state_before = state.to_h
551
+ intents_before = intent_snapshot
552
+ result = guard_application_writes(operation) { yield }
553
+ unless state.to_h == state_before && intent_snapshot == intents_before
554
+ raise QueryMutatedState, "projections must not mutate actor state or stage durable work"
555
+ end
556
+
557
+ result
558
+ end
559
+
540
560
  private
541
561
 
542
562
  attr_reader :effect_intents,
@@ -545,6 +565,13 @@ module SolidObjects
545
565
  :reminder_intents,
546
566
  :outbound_message_intents
547
567
 
568
+ # @rbs () -> Array[Array[Hash[Symbol, Object]]]
569
+ def intent_snapshot
570
+ [ effect_intents, effect_recovery_intents, commit_action_intents, reminder_intents, outbound_message_intents ].map do |intents|
571
+ intents.map { |intent| intent.to_h.deep_dup }
572
+ end
573
+ end
574
+
548
575
  # @rbs (String) { () -> untyped } -> untyped
549
576
  def guard_application_writes(operation, &block)
550
577
  ApplicationWriteGuard.call(
@@ -49,6 +49,7 @@ module SolidObjects
49
49
  end
50
50
  refresh_outdated_components(snapshot)
51
51
  transmit_state_payloads(snapshot)
52
+ SolidObjects.instrument(:"realtime.connected", actor_type: reference.actor_type, actor_id: reference.actor_id)
52
53
  rescue KeyError,
53
54
  JSON::ParserError,
54
55
  InvalidStreamToken,
@@ -57,6 +58,13 @@ module SolidObjects
57
58
  reject_and_report(reject_reason(error), actor_type:, actor_id:, error:)
58
59
  end
59
60
 
61
+ # @rbs () -> void
62
+ def unsubscribed
63
+ return unless reference
64
+
65
+ SolidObjects.instrument(:"realtime.disconnected", actor_type: reference.actor_type, actor_id: reference.actor_id)
66
+ end
67
+
60
68
  private
61
69
 
62
70
  attr_reader :reference,
@@ -141,7 +149,7 @@ module SolidObjects
141
149
  true
142
150
  rescue => error
143
151
  SolidObjects.instrument(
144
- :payload_broadcast_failed,
152
+ :"payload_broadcast.failed",
145
153
  actor_type: reference.actor_type,
146
154
  actor_id: reference.actor_id,
147
155
  payload_name: name,
@@ -7,6 +7,7 @@ module SolidObjects
7
7
  # @rbs @actor: Actor
8
8
  # @rbs @instance_id: Integer
9
9
  # @rbs @revision: Integer
10
+ # @rbs @state_data: Hash[String, json_value]
10
11
  # @rbs @observable_values: Hash[String, untyped]?
11
12
  # @rbs @observable_value_cache: Hash[String, untyped]
12
13
 
@@ -24,6 +25,7 @@ module SolidObjects
24
25
  end
25
26
  @instance_id = @instance&.id || 0
26
27
  @revision = @instance&.state_revision || 0
28
+ @state_data = State.new(actor_class.definition.state_definition, migrated_state).to_h
27
29
  @actor = build_actor
28
30
  @observable_values = nil
29
31
  @observable_value_cache = {}
@@ -43,14 +45,23 @@ module SolidObjects
43
45
  Serialization.readonly_copy(actor.observable_value(observable_name))
44
46
  end
45
47
 
48
+ # @rbs () -> Actor
49
+ def build_actor
50
+ actor_class.new(
51
+ actor_id: reference.actor_id,
52
+ state: State.new(actor_class.definition.state_definition, @state_data),
53
+ instance_id: instance&.id
54
+ )
55
+ end
56
+
46
57
  private
47
58
 
48
59
  attr_reader :instance
49
60
 
50
- # @rbs () -> Actor
51
- def build_actor
61
+ # @rbs () -> Hash[String, json_value]
62
+ def migrated_state
52
63
  state_version = instance&.state_version || actor_class.state_version
53
- state_data = ApplicationWriteGuard.call(
64
+ ApplicationWriteGuard.call(
54
65
  actor_type: reference.actor_type,
55
66
  actor_id: reference.actor_id,
56
67
  operation: "state_migration"
@@ -60,11 +71,6 @@ module SolidObjects
60
71
  instance&.state || {}
61
72
  )
62
73
  end
63
- actor_class.new(
64
- actor_id: reference.actor_id,
65
- state: State.new(actor_class.definition.state_definition, state_data),
66
- instance_id: @instance&.id
67
- )
68
74
  end
69
75
  end
70
76
  end
@@ -41,6 +41,7 @@ module SolidObjects
41
41
  broadcast = claim_next
42
42
  return false unless broadcast
43
43
 
44
+ Telemetry.outbox(broadcast)
44
45
  broadcast_adapter.call(broadcast)
45
46
  complete(broadcast)
46
47
  true
@@ -94,6 +94,19 @@ module SolidObjects
94
94
  raise SyncDiagnostics.new.database_contention_for(message_reference, timeout:)
95
95
  end
96
96
 
97
+ # @rbs (MessageReference, ?authorization_context: untyped) -> Message
98
+ def read_message(message_reference, authorization_context: nil)
99
+ Message.uncached do
100
+ message = Message.includes(:ready_message, :claimed_message, :dead_letter).find(message_reference.id)
101
+ validate_message_reference!(message_reference, message)
102
+ raise Unauthorized, "message result is not authorized" unless authorized_to_read?(message, authorization_context:)
103
+
104
+ message
105
+ end
106
+ rescue ActiveRecord::RecordNotFound, ActorDestroyed
107
+ raise Unauthorized, "message result is not authorized"
108
+ end
109
+
97
110
  # @rbs (?reference: Reference?, ?request_id: String?, ?idempotency_key: String?, ?authorization_context: untyped) -> MessageReference?
98
111
  def find_by(reference: nil, request_id: nil, idempotency_key: nil, authorization_context: nil)
99
112
  unless [ request_id, idempotency_key ].compact.one?
@@ -193,7 +206,8 @@ module SolidObjects
193
206
  :"sync.enqueue_timeout",
194
207
  actor_type: reference.actor_type,
195
208
  actor_id: reference.actor_id,
196
- operation: operation.to_s
209
+ operation: operation.to_s,
210
+ timeout_milliseconds: (timeout * 1000).round
197
211
  )
198
212
  raise SyncEnqueueTimeout.new(
199
213
  timeout:,
@@ -40,6 +40,7 @@ module SolidObjects
40
40
  # @rbs @broadcast_worker_count: Integer
41
41
  # @rbs @reminder_scheduler_count: Integer
42
42
  # @rbs @connects_to: Hash[Symbol, untyped]?
43
+ # @rbs @instrumentation: event_observer?
43
44
  # @rbs @logger: untyped
44
45
  # @rbs @stream_signing_secret: String?
45
46
  # @rbs @broadcast_adapter: Proc?
@@ -110,6 +111,8 @@ module SolidObjects
110
111
  :administration_identity,
111
112
  :transmission_actor_type_resolver
112
113
 
114
+ attr_accessor :instrumentation #: event_observer?
115
+
113
116
  # @rbs @additional_components: Array[untyped]
114
117
  attr_reader :additional_components
115
118
 
@@ -159,6 +162,7 @@ module SolidObjects
159
162
  @component_path_resolver = nil
160
163
  @component_authorization_context = ->(controller:) { controller }
161
164
  @payload_authorization_context = ->(connection:) { connection }
165
+ @instrumentation = nil
162
166
  @logger = if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
163
167
  Rails.logger
164
168
  else
@@ -22,7 +22,13 @@ module SolidObjects
22
22
  def transaction(&block)
23
23
  return with_lock_retry { super } if SyncDeadline.active?
24
24
 
25
- with_busy_retry { super }
25
+ with_busy_retry do
26
+ with_connection do |connection|
27
+ timeout = configured_busy_handler_timeout(connection)
28
+ connection.raw_connection.busy_handler_timeout = timeout if timeout
29
+ super
30
+ end
31
+ end
26
32
  end
27
33
 
28
34
  # A write outside a synchronous deadline has no Ruby-level budget, so it
@@ -0,0 +1,83 @@
1
+ # rbs_inline: enabled
2
+
3
+ module SolidObjects
4
+ class Diagnostics
5
+ # @rbs @reference: Reference
6
+ # @rbs (Reference) -> void
7
+ def initialize(reference)
8
+ @reference = reference
9
+ end
10
+
11
+ # @rbs (?limit: Integer, ?authorization_context: untyped) -> actor_diagnostics
12
+ def summary(limit: 100, authorization_context: nil)
13
+ authorize!(:inspect, authorization_context:)
14
+ raise ArgumentError, "diagnostic limit must be an integer between 1 and 100" unless limit.is_a?(Integer) && limit.between?(1, 100)
15
+
16
+ instance = Instance.find_by(actor_type: reference.actor_type, actor_id: reference.actor_id)
17
+ now = SolidObjects.database_adapter.database_now
18
+ instance_id = instance&.id
19
+ mailbox = ReadyMessage.where(instance_id:).order(:available_at).limit(limit + 1).pluck(:available_at) +
20
+ ClaimedMessage.where(instance_id:).order(:claimed_at).limit(limit + 1).pluck(:claimed_at)
21
+ outbox = Effect.where(instance_id:, status: %w[pending processing]).order(:available_at).limit(limit + 1).pluck(:available_at) +
22
+ Broadcast.where(instance_id:, status: %w[pending processing]).order(:available_at).limit(limit + 1).pluck(:available_at)
23
+ reminders = Reminder.where(instance_id:, status: %w[scheduled paused]).order(:next_run_at).limit(limit + 1).pluck(:next_run_at)
24
+ retries = ReadyMessage.joins(:message).where(instance_id:).where.not(Message.table_name => { error: nil }).order(:available_at).limit(limit + 1).pluck(:available_at)
25
+ recovery_messages = Message.where(instance_id:, delivery_mode: "internal").where("idempotency_key LIKE ?", "effect:%:recovery").select(:id)
26
+ recovery_failures = DeadLetter.where(instance_id:, message_id: recovery_messages).order(:last_failed_at).limit(limit + 1).pluck(:last_failed_at)
27
+ result = {
28
+ "actorType" => reference.actor_type,
29
+ "actorId" => reference.actor_id,
30
+ "incarnation" => instance_id&.to_s,
31
+ "revision" => instance&.state_revision&.to_s,
32
+ "adapter" => DatabaseAdapter.family(Record.connection).to_s,
33
+ "occurredAt" => now.utc.iso8601(3),
34
+ "limit" => limit,
35
+ "mailbox" => summarize(mailbox, now:, limit:),
36
+ "outbox" => summarize(outbox, now:, limit:),
37
+ "reminders" => summarize(reminders, now:, limit:),
38
+ "retries" => summarize(retries, now:, limit:),
39
+ "recoveryFailures" => summarize(recovery_failures, now:, limit:)
40
+ }
41
+ SolidObjects.instrument(:"mailbox.depth", actor_type: reference.actor_type, actor_id: reference.actor_id, instance_id:, count: result.fetch("mailbox").fetch("sampled"), truncated: result.fetch("mailbox").fetch("truncated"), depth: result.fetch("mailbox").fetch("truncated") ? nil : result.fetch("mailbox").fetch("sampled"))
42
+ Serialization.readonly_copy(result)
43
+ end
44
+
45
+ # @rbs (?authorization_context: untyped) { (portable_event) -> void } -> Proc
46
+ def observe(authorization_context: nil, &block)
47
+ raise ArgumentError, "an actor observer requires a block" unless block
48
+
49
+ authorize!(:observe, authorization_context:)
50
+ SolidObjects.observer_registry.subscribe do |notification|
51
+ payload = notification.payload
52
+ next unless payload[:actor_type] == reference.actor_type && payload[:actor_id] == reference.actor_id
53
+
54
+ Telemetry.deliver(block, Telemetry.event(notification.name.delete_prefix("solid_objects.").to_sym, payload))
55
+ end
56
+ end
57
+
58
+ private
59
+
60
+ attr_reader :reference #: Reference
61
+
62
+ # @rbs (Symbol, ?authorization_context: untyped) -> void
63
+ def authorize!(action, authorization_context: nil)
64
+ allowed = SolidObjects.configuration.authorize_administration.call(
65
+ action:,
66
+ resource: "actor_diagnostics",
67
+ resource_id: [ reference.actor_type, reference.actor_id ].to_json,
68
+ authorization_context:
69
+ )
70
+ raise Unauthorized, "actor diagnostics are not authorized" unless allowed
71
+ end
72
+
73
+ # @rbs (Array[Time?], now: Time, limit: Integer) -> diagnostic_summary
74
+ def summarize(timestamps, now:, limit:)
75
+ oldest = timestamps.compact.min
76
+ {
77
+ "sampled" => [ timestamps.length, limit ].min,
78
+ "truncated" => timestamps.length > limit,
79
+ "oldestAgeMilliseconds" => oldest ? [ ((now - oldest) * 1000).round, 0 ].max : nil
80
+ }
81
+ end
82
+ end
83
+ end
@@ -137,6 +137,7 @@ module SolidObjects
137
137
 
138
138
  # @rbs (Effect) -> untyped
139
139
  def deliver(effect)
140
+ Telemetry.outbox(effect)
140
141
  return deliver_actor_message(effect) if effect.name == ACTOR_MESSAGE_EFFECT
141
142
 
142
143
  handler = SolidObjects.effect_registry.fetch(effect.name)
@@ -208,6 +209,9 @@ module SolidObjects
208
209
  Mailbox.new.announce(result_message) if result_message
209
210
  SolidObjects.instrument(
210
211
  :"effect.completed",
212
+ instance_id: effect.instance_id,
213
+ actor_type: effect.instance.actor_type,
214
+ actor_id: effect.instance.actor_id,
211
215
  effect_id: effect.effect_id,
212
216
  effect_name: effect.name,
213
217
  message_id: effect.message_id,
@@ -7,6 +7,9 @@ module SolidObjects
7
7
  class NonRetryableError < Error
8
8
  end
9
9
 
10
+ class QueryMutatedState < NonRetryableError
11
+ end
12
+
10
13
  class UnsupportedDatabase < Error
11
14
  end
12
15
 
@@ -4,12 +4,14 @@ module SolidObjects
4
4
  class Executor
5
5
  # @rbs @activation: Activation
6
6
  # @rbs @message: Message
7
+ # @rbs @started_at: Float
7
8
  # @rbs @completion_transaction: untyped
8
9
 
9
10
  # @rbs (activation: Activation, message: Message) -> void
10
11
  def initialize(activation:, message:)
11
12
  @activation = activation
12
13
  @message = message
14
+ @started_at = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
13
15
  end
14
16
 
15
17
  # @rbs () -> bool
@@ -24,11 +26,12 @@ module SolidObjects
24
26
  request_id: message.request_id
25
27
  )
26
28
 
29
+ SolidObjects.instrument(:"recovery.reclaimed", **instrumentation_payload) if message.attempt_count > 1 && message.error.nil?
27
30
  SolidObjects.instrument(:"message.started", **instrumentation_payload)
28
31
  result = invoke_actor(message_context)
29
32
  observable_changes = changed_observables(observables_before, actor.observable_values)
30
33
  state_after = actor.state.to_h_with_byte_size
31
- ensure_query_did_not_mutate_state!(state_before, state_after.value)
34
+ ensure_query_is_read_only!(state_before, state_after.value)
32
35
  complete(
33
36
  result,
34
37
  observable_changes,
@@ -66,13 +69,12 @@ module SolidObjects
66
69
  end
67
70
  end
68
71
 
69
- # @rbs (Hash[String, untyped], Hash[String, untyped]) -> void
70
- def ensure_query_did_not_mutate_state!(state_before, state_after)
71
- return unless message.delivery_mode == "sync"
72
+ # @rbs (Hash[String, json_value], Hash[String, json_value]) -> void
73
+ def ensure_query_is_read_only!(state_before, state_after)
72
74
  return unless actor.class.definition.queries.key?(message.operation.to_sym)
73
- return if state_after == state_before
74
75
 
75
- raise InvalidActor, "query #{message.operation.inspect} mutated actor state"
76
+ raise QueryMutatedState, "query #{message.operation.inspect} mutated actor state" if state_after != state_before
77
+ raise QueryMutatedState, "query #{message.operation.inspect} staged durable work" if actor.intent_count.positive?
76
78
  end
77
79
 
78
80
  # @rbs (Hash[String, untyped], Hash[String, untyped]) -> Hash[String, untyped]
@@ -86,7 +88,7 @@ module SolidObjects
86
88
  def complete(result, observable_changes, state_after:, state_changed:)
87
89
  ensure_state_fits!(state_after.byte_size)
88
90
  serialized_result = Serialization.dump(
89
- (message.delivery_mode == "sync") ? result : nil,
91
+ result,
90
92
  max_bytes: SolidObjects.configuration.max_result_bytes
91
93
  )
92
94
  effect_intents = actor.drain_effect_intents
@@ -156,8 +158,9 @@ module SolidObjects
156
158
  actor_id: message.actor_id
157
159
  )
158
160
  end
161
+ SolidObjects.instrument_after_commit(:"recovery.completed", **instrumentation_payload) if recovery_message?
159
162
  report_large_state(state_after.byte_size)
160
- SolidObjects.instrument_after_commit(:"message.completed", **instrumentation_payload)
163
+ SolidObjects.instrument_after_commit(:"message.completed", **instrumentation_payload, revision: message.sequence, duration_milliseconds: elapsed_milliseconds)
161
164
  SolidObjects.wake_up.signal
162
165
  rescue CommittedTransactionError
163
166
  raise
@@ -209,11 +212,8 @@ module SolidObjects
209
212
  # @rbs (intent: Actor::CommitActionIntent, handler: Proc, context: CommitActionContext) -> untyped
210
213
  def execute_commit_action(intent:, handler:, context:)
211
214
  payload = {
215
+ **instrumentation_payload,
212
216
  commit_action_name: intent.name,
213
- message_id: message.id,
214
- request_id: message.request_id,
215
- actor_type: message.actor_type,
216
- actor_id: message.actor_id,
217
217
  activation_generation: activation.lease.generation
218
218
  }
219
219
  SolidObjects.instrument(:"commit_action.started", **payload)
@@ -224,8 +224,7 @@ module SolidObjects
224
224
  SolidObjects.instrument(
225
225
  :"commit_action.failed",
226
226
  **payload,
227
- error_class: error.class.name,
228
- error_message: error.message
227
+ error_class: error.class.name
229
228
  )
230
229
  raise
231
230
  end
@@ -421,8 +420,13 @@ module SolidObjects
421
420
  :"message.failed",
422
421
  **instrumentation_payload,
423
422
  error_class: error.class.name,
423
+ duration_milliseconds: elapsed_milliseconds,
424
+ retryable: !error.is_a?(NonRetryableError),
425
+ outcome: dead ? "dead" : "retrying",
424
426
  dead:
425
427
  )
428
+ SolidObjects.instrument_after_commit(:"recovery.failed", **instrumentation_payload) if dead && recovery_message?
429
+ SolidObjects.instrument_after_commit(dead ? :"dead_letter.created" : :"message.retry", **instrumentation_payload)
426
430
  SolidObjects.wake_up.signal
427
431
  end
428
432
 
@@ -458,7 +462,8 @@ module SolidObjects
458
462
  SolidObjects.instrument_after_commit(
459
463
  :"message.rejected",
460
464
  **instrumentation_payload,
461
- code: rejection.code
465
+ code: rejection.code,
466
+ duration_milliseconds: elapsed_milliseconds
462
467
  )
463
468
  SolidObjects.wake_up.signal
464
469
  end
@@ -520,15 +525,28 @@ module SolidObjects
520
525
  )
521
526
  end
522
527
 
528
+ # @rbs () -> bool
529
+ def recovery_message?
530
+ message.delivery_mode == "internal" && message.idempotency_key.to_s.start_with?("effect:") && message.idempotency_key.to_s.end_with?(":recovery")
531
+ end
532
+
533
+ # @rbs () -> (Integer | Float)
534
+ def elapsed_milliseconds
535
+ ((::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @started_at) * 1000).round(3)
536
+ end
537
+
523
538
  # @rbs () -> Hash[Symbol, untyped]
524
539
  def instrumentation_payload
525
540
  {
541
+ instance_id: message.instance_id,
526
542
  message_id: message.id,
527
543
  actor_type: message.actor_type,
528
544
  actor_id: message.actor_id,
529
545
  sequence: message.sequence,
530
546
  attempt: message.attempt_count,
531
- request_id: message.request_id
547
+ request_id: message.request_id,
548
+ operation: message.operation,
549
+ delivery_mode: message.delivery_mode
532
550
  }
533
551
  end
534
552
  end
@@ -4,7 +4,28 @@ module SolidObjects
4
4
  module Instrumentation
5
5
  # @rbs (Symbol, **untyped) { (Hash[Symbol, untyped]) -> untyped } -> untyped
6
6
  def instrument(event, **payload, &block)
7
- ActiveSupport::Notifications.instrument("solid_objects.#{event}", payload, &block)
7
+ executed = false
8
+ application_error = nil
9
+ result = nil
10
+ begin
11
+ ActiveSupport::Notifications.instrument("solid_objects.#{event}", payload) do
12
+ executed = true
13
+ begin
14
+ result = block&.call(payload)
15
+ rescue => error
16
+ application_error = error
17
+ raise
18
+ end
19
+ end
20
+ rescue => error
21
+ raise application_error if application_error
22
+
23
+ report_instrumentation_failure(event, error) unless event == :"instrumentation.failed"
24
+ result = block&.call(payload) unless executed
25
+ ensure
26
+ Telemetry.emit(event, payload)
27
+ end
28
+ result
8
29
  end
9
30
 
10
31
  # @rbs (Symbol, **untyped) -> void
@@ -12,7 +12,7 @@ module SolidObjects
12
12
  {
13
13
  event: event.name,
14
14
  duration_ms: event.duration.round(2)
15
- }.merge(event.payload)
15
+ }.merge(event.payload.except(:error_message, :arguments, :state, :credentials, :response, :exception, :exception_object))
16
16
  )
17
17
  end
18
18
  end
@@ -79,11 +79,15 @@ module SolidObjects
79
79
  def announce(message)
80
80
  SolidObjects.instrument(
81
81
  :"message.enqueued",
82
+ instance_id: message.instance_id,
83
+ attempt: message.attempt_count,
82
84
  message_id: message.id,
83
85
  actor_type: message.actor_type,
84
86
  actor_id: message.actor_id,
85
87
  sequence: message.sequence,
86
- request_id: message.request_id
88
+ request_id: message.request_id,
89
+ operation: message.operation,
90
+ delivery_mode: message.delivery_mode
87
91
  )
88
92
  SolidObjects.wake_up.signal
89
93
  end
@@ -33,20 +33,20 @@ module SolidObjects
33
33
  freeze
34
34
  end
35
35
 
36
- # @rbs () -> String
37
- def status
38
- Message.uncached { status_of(Message.find(id)) }
36
+ # @rbs (?authorization_context: untyped) -> String
37
+ def status(authorization_context: nil)
38
+ status_of(SolidObjects.client.read_message(self, authorization_context:))
39
39
  end
40
40
 
41
- # @rbs () -> untyped
42
- def result
43
- Message.uncached { Message.find(id).result }
41
+ # @rbs (?authorization_context: untyped) -> json_value
42
+ def result(authorization_context: nil)
43
+ SolidObjects.client.read_message(self, authorization_context:).result!
44
44
  end
45
45
 
46
- # @rbs () -> Outcome
47
- def outcome
46
+ # @rbs (?authorization_context: untyped) -> Outcome
47
+ def outcome(authorization_context: nil)
48
48
  Message.uncached do
49
- message = Message.find(id)
49
+ message = SolidObjects.client.read_message(self, authorization_context:)
50
50
  Outcome.new(
51
51
  status: status_of(message),
52
52
  result: Serialization.readonly_copy(message.result),
@@ -0,0 +1,47 @@
1
+ # rbs_inline: enabled
2
+
3
+ module SolidObjects
4
+ class ObserverRegistry
5
+ MAXIMUM_OBSERVERS = 1000
6
+
7
+ # @rbs @subscriptions: Array[Object]
8
+ # @rbs @mutex: Mutex
9
+
10
+ # @rbs () -> void
11
+ def initialize
12
+ @subscriptions = []
13
+ @mutex = Mutex.new
14
+ end
15
+
16
+ # @rbs () { (ActiveSupport::Notifications::Event) -> void } -> Proc
17
+ def subscribe(&listener)
18
+ subscription = mutex.synchronize do
19
+ raise ArgumentError, "at most #{MAXIMUM_OBSERVERS} local observers may be registered" if subscriptions.length >= MAXIMUM_OBSERVERS
20
+
21
+ ActiveSupport::Notifications.subscribe(/\Asolid_objects\./, &listener).tap { |created| subscriptions << created }
22
+ end
23
+ -> { unsubscribe(subscription) }
24
+ end
25
+
26
+ # @rbs () -> void
27
+ def clear
28
+ mutex.synchronize do
29
+ subscriptions.each { |subscription| ActiveSupport::Notifications.unsubscribe(subscription) }
30
+ subscriptions.clear
31
+ end
32
+ end
33
+
34
+ private
35
+
36
+ attr_reader :subscriptions #: Array[Object]
37
+ attr_reader :mutex #: Mutex
38
+
39
+ # @rbs (Object) -> void
40
+ def unsubscribe(subscription)
41
+ mutex.synchronize do
42
+ ActiveSupport::Notifications.unsubscribe(subscription)
43
+ subscriptions.delete(subscription)
44
+ end
45
+ end
46
+ end
47
+ end