deimos-ruby 1.16.1 → 1.16.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (98) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/CHANGELOG.md +13 -0
  4. data/Gemfile +6 -0
  5. data/README.md +22 -0
  6. data/lib/deimos/active_record_consume/batch_consumption.rb +7 -2
  7. data/lib/deimos/active_record_consume/batch_slicer.rb +2 -0
  8. data/lib/deimos/active_record_consume/message_consumption.rb +8 -4
  9. data/lib/deimos/active_record_consumer.rb +7 -4
  10. data/lib/deimos/active_record_producer.rb +10 -0
  11. data/lib/deimos/backends/base.rb +4 -2
  12. data/lib/deimos/backends/kafka.rb +1 -0
  13. data/lib/deimos/backends/kafka_async.rb +1 -0
  14. data/lib/deimos/config/configuration.rb +4 -0
  15. data/lib/deimos/config/phobos_config.rb +2 -1
  16. data/lib/deimos/consume/batch_consumption.rb +8 -1
  17. data/lib/deimos/consume/message_consumption.rb +4 -1
  18. data/lib/deimos/instrumentation.rb +11 -4
  19. data/lib/deimos/kafka_message.rb +1 -0
  20. data/lib/deimos/kafka_source.rb +5 -0
  21. data/lib/deimos/kafka_topic_info.rb +4 -0
  22. data/lib/deimos/message.rb +19 -2
  23. data/lib/deimos/metrics/datadog.rb +2 -1
  24. data/lib/deimos/metrics/mock.rb +2 -2
  25. data/lib/deimos/metrics/provider.rb +6 -0
  26. data/lib/deimos/monkey_patches/phobos_cli.rb +1 -1
  27. data/lib/deimos/monkey_patches/phobos_producer.rb +1 -0
  28. data/lib/deimos/producer.rb +12 -6
  29. data/lib/deimos/schema_backends/base.rb +31 -17
  30. data/lib/deimos/schema_backends/mock.rb +2 -2
  31. data/lib/deimos/schema_class/base.rb +9 -5
  32. data/lib/deimos/schema_class/enum.rb +4 -2
  33. data/lib/deimos/schema_class/record.rb +12 -6
  34. data/lib/deimos/shared_config.rb +6 -2
  35. data/lib/deimos/test_helpers.rb +21 -4
  36. data/lib/deimos/tracing/datadog.rb +1 -1
  37. data/lib/deimos/tracing/mock.rb +4 -3
  38. data/lib/deimos/tracing/provider.rb +5 -0
  39. data/lib/deimos/utils/db_poller.rb +9 -1
  40. data/lib/deimos/utils/db_producer.rb +14 -2
  41. data/lib/deimos/utils/deadlock_retry.rb +3 -0
  42. data/lib/deimos/utils/inline_consumer.rb +14 -6
  43. data/lib/deimos/utils/lag_reporter.rb +11 -0
  44. data/lib/deimos/utils/schema_controller_mixin.rb +8 -0
  45. data/lib/deimos/version.rb +1 -1
  46. data/lib/deimos.rb +3 -2
  47. data/lib/generators/deimos/active_record_generator.rb +1 -1
  48. data/lib/generators/deimos/db_backend_generator.rb +1 -0
  49. data/lib/generators/deimos/db_poller_generator.rb +1 -0
  50. data/lib/generators/deimos/schema_class/templates/schema_record.rb.tt +13 -5
  51. data/lib/generators/deimos/schema_class_generator.rb +51 -24
  52. data/rbs_collection.lock.yaml +176 -0
  53. data/rbs_collection.yaml +15 -0
  54. data/regenerate_test_schema_classes.rb +68 -0
  55. data/sig/avro.rbs +14 -0
  56. data/sig/defs.rbs +1859 -0
  57. data/sig/fig_tree.rbs +2 -0
  58. data/spec/active_record_producer_spec.rb +24 -0
  59. data/spec/generators/schema_class/my_schema_spec.rb +16 -0
  60. data/spec/generators/schema_class/my_schema_with_circular_reference_spec.rb +1 -1
  61. data/spec/generators/schema_class/my_schema_with_complex_types_spec.rb +32 -24
  62. data/spec/producer_spec.rb +11 -11
  63. data/spec/schemas/{generated.rb → my_namespace/generated.rb} +28 -32
  64. data/spec/schemas/{my_nested_schema.rb → my_namespace/my_nested_schema.rb} +23 -16
  65. data/spec/schemas/{my_schema.rb → my_namespace/my_schema.rb} +12 -5
  66. data/spec/schemas/my_namespace/my_schema_compound_key.rb +41 -0
  67. data/spec/schemas/my_namespace/my_schema_id_key.rb +36 -0
  68. data/spec/schemas/{my_schema_key.rb → my_namespace/my_schema_key.rb} +3 -3
  69. data/spec/schemas/my_namespace/my_schema_with_boolean.rb +41 -0
  70. data/spec/schemas/{my_schema_with_circular_reference.rb → my_namespace/my_schema_with_circular_reference.rb} +15 -10
  71. data/spec/schemas/{my_schema_with_complex_type.rb → my_namespace/my_schema_with_complex_type.rb} +34 -49
  72. data/spec/schemas/my_namespace/my_schema_with_date_time.rb +56 -0
  73. data/spec/schemas/my_namespace/my_schema_with_id.rb +51 -0
  74. data/spec/schemas/my_namespace/my_schema_with_unique_id.rb +56 -0
  75. data/spec/schemas/my_namespace/wibble.rb +76 -0
  76. data/spec/schemas/my_namespace/widget.rb +56 -0
  77. data/spec/schemas/my_namespace/widget_the_second.rb +56 -0
  78. data/spec/schemas/request/create_topic.rb +36 -0
  79. data/spec/schemas/request/index.rb +36 -0
  80. data/spec/schemas/request/update_request.rb +36 -0
  81. data/spec/schemas/response/create_topic.rb +36 -0
  82. data/spec/schemas/response/index.rb +36 -0
  83. data/spec/schemas/response/update_response.rb +36 -0
  84. data/spec/snapshots/consumers-no-nest.snap +93 -86
  85. data/spec/snapshots/consumers.snap +94 -87
  86. data/spec/snapshots/consumers_and_producers-no-nest.snap +108 -87
  87. data/spec/snapshots/consumers_and_producers.snap +109 -88
  88. data/spec/snapshots/consumers_circular-no-nest.snap +93 -86
  89. data/spec/snapshots/consumers_circular.snap +94 -87
  90. data/spec/snapshots/consumers_complex_types-no-nest.snap +93 -86
  91. data/spec/snapshots/consumers_complex_types.snap +94 -87
  92. data/spec/snapshots/consumers_nested-no-nest.snap +93 -86
  93. data/spec/snapshots/consumers_nested.snap +94 -87
  94. data/spec/snapshots/namespace_folders.snap +111 -90
  95. data/spec/snapshots/producers_with_key-no-nest.snap +94 -87
  96. data/spec/snapshots/producers_with_key.snap +95 -88
  97. data/spec/spec_helper.rb +2 -1
  98. metadata +53 -15
@@ -8,7 +8,7 @@ module Schemas
8
8
  class CreateTopic < Deimos::SchemaClass::Record
9
9
 
10
10
  ### Attribute Accessors ###
11
- # @param value [String]
11
+ # @return [String]
12
12
  attr_accessor :response_id
13
13
 
14
14
  # @override
@@ -51,7 +51,7 @@ module Schemas
51
51
  class ARecord < Deimos::SchemaClass::Record
52
52
 
53
53
  ### Attribute Accessors ###
54
- # @param value [String]
54
+ # @return [String]
55
55
  attr_accessor :a_record_field
56
56
 
57
57
  # @override
@@ -97,34 +97,34 @@ module Schemas
97
97
  attr_reader :a_record
98
98
 
99
99
  ### Attribute Accessors ###
100
- # @param value [String]
100
+ # @return [String]
101
101
  attr_accessor :a_string
102
- # @param value [Integer]
102
+ # @return [Integer]
103
103
  attr_accessor :a_int
104
- # @param value [Integer]
104
+ # @return [Integer]
105
105
  attr_accessor :a_long
106
- # @param value [Float]
106
+ # @return [Float]
107
107
  attr_accessor :a_float
108
- # @param value [Float]
108
+ # @return [Float]
109
109
  attr_accessor :a_double
110
- # @param value [nil, Integer]
110
+ # @return [nil, Integer]
111
111
  attr_accessor :an_optional_int
112
- # @param values [Array<Integer>]
112
+ # @return [Array<Integer>]
113
113
  attr_accessor :an_array
114
- # @param values [Hash<String, String>]
114
+ # @return [Hash<String, String>]
115
115
  attr_accessor :a_map
116
- # @param value [String]
116
+ # @return [String]
117
117
  attr_accessor :timestamp
118
- # @param value [String]
118
+ # @return [String]
119
119
  attr_accessor :message_id
120
120
 
121
121
  ### Attribute Writers ###
122
- # @param value [AnEnum]
122
+ # @return [AnEnum]
123
123
  def an_enum=(value)
124
124
  @an_enum = AnEnum.initialize_from_value(value)
125
125
  end
126
126
 
127
- # @param value [ARecord]
127
+ # @return [ARecord]
128
128
  def a_record=(value)
129
129
  @a_record = ARecord.initialize_from_value(value)
130
130
  end
@@ -167,6 +167,13 @@ module Schemas
167
167
  'com.my-namespace'
168
168
  end
169
169
 
170
+ def self.tombstone(key)
171
+ record = self.allocate
172
+ record.tombstone_key = key
173
+ record.a_string = key
174
+ record
175
+ end
176
+
170
177
  # @override
171
178
  def as_json(_opts={})
172
179
  {
@@ -198,7 +205,7 @@ module Schemas
198
205
  class Index < Deimos::SchemaClass::Record
199
206
 
200
207
  ### Attribute Accessors ###
201
- # @param value [String]
208
+ # @return [String]
202
209
  attr_accessor :response_id
203
210
 
204
211
  # @override
@@ -241,13 +248,13 @@ module Schemas
241
248
  class MyNestedRecord < Deimos::SchemaClass::Record
242
249
 
243
250
  ### Attribute Accessors ###
244
- # @param value [Integer]
251
+ # @return [Integer]
245
252
  attr_accessor :some_int
246
- # @param value [Float]
253
+ # @return [Float]
247
254
  attr_accessor :some_float
248
- # @param value [String]
255
+ # @return [String]
249
256
  attr_accessor :some_string
250
- # @param value [nil, Integer]
257
+ # @return [nil, Integer]
251
258
  attr_accessor :some_optional_int
252
259
 
253
260
  # @override
@@ -291,20 +298,20 @@ module Schemas
291
298
  attr_reader :some_optional_record
292
299
 
293
300
  ### Attribute Accessors ###
294
- # @param value [String]
301
+ # @return [String]
295
302
  attr_accessor :test_id
296
- # @param value [Float]
303
+ # @return [Float]
297
304
  attr_accessor :test_float
298
- # @param values [Array<String>]
305
+ # @return [Array<String>]
299
306
  attr_accessor :test_array
300
307
 
301
308
  ### Attribute Writers ###
302
- # @param value [MyNestedRecord]
309
+ # @return [MyNestedRecord]
303
310
  def some_nested_record=(value)
304
311
  @some_nested_record = MyNestedRecord.initialize_from_value(value)
305
312
  end
306
313
 
307
- # @param value [nil, MyNestedRecord]
314
+ # @return [nil, MyNestedRecord]
308
315
  def some_optional_record=(value)
309
316
  @some_optional_record = MyNestedRecord.initialize_from_value(value)
310
317
  end
@@ -333,6 +340,13 @@ module Schemas
333
340
  'com.my-namespace'
334
341
  end
335
342
 
343
+ def self.tombstone(key)
344
+ record = self.allocate
345
+ record.tombstone_key = key
346
+ record.test_id = key
347
+ record
348
+ end
349
+
336
350
  # @override
337
351
  def as_json(_opts={})
338
352
  {
@@ -361,13 +375,13 @@ module Schemas
361
375
  attr_reader :payload_key
362
376
 
363
377
  ### Attribute Accessors ###
364
- # @param value [String]
378
+ # @return [String]
365
379
  attr_accessor :test_id
366
- # @param value [Integer]
380
+ # @return [Integer]
367
381
  attr_accessor :some_int
368
382
 
369
383
  ### Attribute Writers ###
370
- # @param value [MySchemaKey]
384
+ # @return [MySchemaKey]
371
385
  def payload_key=(value)
372
386
  @payload_key = MySchemaKey.initialize_from_value(value)
373
387
  end
@@ -392,6 +406,13 @@ module Schemas
392
406
  'com.my-namespace'
393
407
  end
394
408
 
409
+ def self.tombstone(key)
410
+ record = self.allocate
411
+ record.tombstone_key = MySchemaKey.initialize_from_value(key)
412
+ record.payload_key = key
413
+ record
414
+ end
415
+
395
416
  # @override
396
417
  def as_json(_opts={})
397
418
  {
@@ -414,9 +435,9 @@ module Schemas
414
435
  class MySchemaCompoundKey < Deimos::SchemaClass::Record
415
436
 
416
437
  ### Attribute Accessors ###
417
- # @param value [String]
438
+ # @return [String]
418
439
  attr_accessor :part_one
419
- # @param value [String]
440
+ # @return [String]
420
441
  attr_accessor :part_two
421
442
 
422
443
  # @override
@@ -458,7 +479,7 @@ module Schemas
458
479
  class MySchemaIdKey < Deimos::SchemaClass::Record
459
480
 
460
481
  ### Attribute Accessors ###
461
- # @param value [Integer]
482
+ # @return [Integer]
462
483
  attr_accessor :id
463
484
 
464
485
  # @override
@@ -497,7 +518,7 @@ module Schemas
497
518
  class MySchemaKey < Deimos::SchemaClass::Record
498
519
 
499
520
  ### Attribute Accessors ###
500
- # @param value [String]
521
+ # @return [String]
501
522
  attr_accessor :test_id
502
523
 
503
524
  # @override
@@ -536,9 +557,9 @@ module Schemas
536
557
  class MySchemaWithBoolean < Deimos::SchemaClass::Record
537
558
 
538
559
  ### Attribute Accessors ###
539
- # @param value [String]
560
+ # @return [String]
540
561
  attr_accessor :test_id
541
- # @param value [Boolean]
562
+ # @return [Boolean]
542
563
  attr_accessor :some_bool
543
564
 
544
565
  # @override
@@ -584,7 +605,7 @@ module Schemas
584
605
  class Property < Deimos::SchemaClass::Record
585
606
 
586
607
  ### Attribute Accessors ###
587
- # @param value [Boolean, Integer, Integer, Float, Float, String, Array<Property>, Hash<String, Property>]
608
+ # @return [Boolean, Integer, Integer, Float, Float, String, Array<Property>, Hash<String, Property>]
588
609
  attr_accessor :property
589
610
 
590
611
  # @override
@@ -617,9 +638,9 @@ module Schemas
617
638
  attr_reader :properties
618
639
 
619
640
  ### Attribute Writers ###
620
- # @param values [Hash<String, Property>]
641
+ # @return [Hash<String, Property>]
621
642
  def properties=(values)
622
- @properties = values.transform_values do |value|
643
+ @properties = values&.transform_values do |value|
623
644
  Property.initialize_from_value(value)
624
645
  end
625
646
  end
@@ -664,7 +685,7 @@ module Schemas
664
685
  class ARecord < Deimos::SchemaClass::Record
665
686
 
666
687
  ### Attribute Accessors ###
667
- # @param value [String]
688
+ # @return [String]
668
689
  attr_accessor :a_record_field
669
690
 
670
691
  # @override
@@ -742,57 +763,57 @@ module Schemas
742
763
  attr_reader :some_enum_with_default
743
764
 
744
765
  ### Attribute Accessors ###
745
- # @param value [String]
766
+ # @return [String]
746
767
  attr_accessor :test_id
747
- # @param value [Float]
768
+ # @return [Float]
748
769
  attr_accessor :test_float
749
- # @param values [Array<String>]
770
+ # @return [Array<String>]
750
771
  attr_accessor :test_string_array
751
- # @param values [Array<Integer>]
772
+ # @return [Array<Integer>]
752
773
  attr_accessor :test_int_array
753
- # @param value [Integer, nil]
774
+ # @return [Integer, nil]
754
775
  attr_accessor :test_optional_int
755
- # @param values [Hash<String, Integer>]
776
+ # @return [Hash<String, Integer>]
756
777
  attr_accessor :some_integer_map
757
778
 
758
779
  ### Attribute Writers ###
759
- # @param value [ARecord]
780
+ # @return [ARecord]
760
781
  def some_record=(value)
761
782
  @some_record = ARecord.initialize_from_value(value)
762
783
  end
763
784
 
764
- # @param value [nil, ARecord]
785
+ # @return [nil, ARecord]
765
786
  def some_optional_record=(value)
766
787
  @some_optional_record = ARecord.initialize_from_value(value)
767
788
  end
768
789
 
769
- # @param values [Array<ARecord>]
790
+ # @return [Array<ARecord>]
770
791
  def some_record_array=(values)
771
- @some_record_array = values.map do |value|
792
+ @some_record_array = values&.map do |value|
772
793
  ARecord.initialize_from_value(value)
773
794
  end
774
795
  end
775
796
 
776
- # @param values [Hash<String, ARecord>]
797
+ # @return [Hash<String, ARecord>]
777
798
  def some_record_map=(values)
778
- @some_record_map = values.transform_values do |value|
799
+ @some_record_map = values&.transform_values do |value|
779
800
  ARecord.initialize_from_value(value)
780
801
  end
781
802
  end
782
803
 
783
- # @param values [Array<AnEnum>]
804
+ # @return [Array<AnEnum>]
784
805
  def some_enum_array=(values)
785
- @some_enum_array = values.map do |value|
806
+ @some_enum_array = values&.map do |value|
786
807
  AnEnum.initialize_from_value(value)
787
808
  end
788
809
  end
789
810
 
790
- # @param value [nil, AnotherEnum]
811
+ # @return [nil, AnotherEnum]
791
812
  def some_optional_enum=(value)
792
813
  @some_optional_enum = AnotherEnum.initialize_from_value(value)
793
814
  end
794
815
 
795
- # @param value [YetAnotherEnum]
816
+ # @return [YetAnotherEnum]
796
817
  def some_enum_with_default=(value)
797
818
  @some_enum_with_default = YetAnotherEnum.initialize_from_value(value)
798
819
  end
@@ -869,15 +890,15 @@ module Schemas
869
890
  class MySchemaWithDateTime < Deimos::SchemaClass::Record
870
891
 
871
892
  ### Attribute Accessors ###
872
- # @param value [String]
893
+ # @return [String]
873
894
  attr_accessor :test_id
874
- # @param value [Integer, nil]
895
+ # @return [Integer, nil]
875
896
  attr_accessor :updated_at
876
- # @param value [nil, Integer]
897
+ # @return [nil, Integer]
877
898
  attr_accessor :some_int
878
- # @param value [nil, Integer]
899
+ # @return [nil, Integer]
879
900
  attr_accessor :some_datetime_int
880
- # @param value [String]
901
+ # @return [String]
881
902
  attr_accessor :timestamp
882
903
 
883
904
  # @override
@@ -928,13 +949,13 @@ module Schemas
928
949
  class MySchemaWithId < Deimos::SchemaClass::Record
929
950
 
930
951
  ### Attribute Accessors ###
931
- # @param value [String]
952
+ # @return [String]
932
953
  attr_accessor :test_id
933
- # @param value [Integer]
954
+ # @return [Integer]
934
955
  attr_accessor :some_int
935
- # @param value [String]
956
+ # @return [String]
936
957
  attr_accessor :message_id
937
- # @param value [String]
958
+ # @return [String]
938
959
  attr_accessor :timestamp
939
960
 
940
961
  # @override
@@ -982,15 +1003,15 @@ module Schemas
982
1003
  class MySchemaWithUniqueId < Deimos::SchemaClass::Record
983
1004
 
984
1005
  ### Attribute Accessors ###
985
- # @param value [Integer]
1006
+ # @return [Integer]
986
1007
  attr_accessor :id
987
- # @param value [String]
1008
+ # @return [String]
988
1009
  attr_accessor :test_id
989
- # @param value [Integer]
1010
+ # @return [Integer]
990
1011
  attr_accessor :some_int
991
- # @param value [String]
1012
+ # @return [String]
992
1013
  attr_accessor :message_id
993
- # @param value [String]
1014
+ # @return [String]
994
1015
  attr_accessor :timestamp
995
1016
 
996
1017
  # @override
@@ -1041,7 +1062,7 @@ module Schemas
1041
1062
  class UpdateRequest < Deimos::SchemaClass::Record
1042
1063
 
1043
1064
  ### Attribute Accessors ###
1044
- # @param value [String]
1065
+ # @return [String]
1045
1066
  attr_accessor :update_request_id
1046
1067
 
1047
1068
  # @override
@@ -1080,7 +1101,7 @@ module Schemas
1080
1101
  class UpdateResponse < Deimos::SchemaClass::Record
1081
1102
 
1082
1103
  ### Attribute Accessors ###
1083
- # @param value [String]
1104
+ # @return [String]
1084
1105
  attr_accessor :update_response_id
1085
1106
 
1086
1107
  # @override
@@ -1119,23 +1140,23 @@ module Schemas
1119
1140
  class Wibble < Deimos::SchemaClass::Record
1120
1141
 
1121
1142
  ### Attribute Accessors ###
1122
- # @param value [Integer]
1143
+ # @return [Integer]
1123
1144
  attr_accessor :id
1124
- # @param value [Integer]
1145
+ # @return [Integer]
1125
1146
  attr_accessor :wibble_id
1126
- # @param value [String]
1147
+ # @return [String]
1127
1148
  attr_accessor :name
1128
- # @param value [String]
1149
+ # @return [String]
1129
1150
  attr_accessor :floop
1130
- # @param value [Integer]
1151
+ # @return [Integer]
1131
1152
  attr_accessor :birthday_int
1132
- # @param value [Integer]
1153
+ # @return [Integer]
1133
1154
  attr_accessor :birthday_long
1134
- # @param value [nil, Integer]
1155
+ # @return [nil, Integer]
1135
1156
  attr_accessor :birthday_optional
1136
- # @param value [Integer]
1157
+ # @return [Integer]
1137
1158
  attr_accessor :updated_at
1138
- # @param value [Integer]
1159
+ # @return [Integer]
1139
1160
  attr_accessor :created_at
1140
1161
 
1141
1162
  # @override
@@ -1198,15 +1219,15 @@ module Schemas
1198
1219
  class Widget < Deimos::SchemaClass::Record
1199
1220
 
1200
1221
  ### Attribute Accessors ###
1201
- # @param value [Integer]
1222
+ # @return [Integer]
1202
1223
  attr_accessor :id
1203
- # @param value [Integer]
1224
+ # @return [Integer]
1204
1225
  attr_accessor :widget_id
1205
- # @param value [String]
1226
+ # @return [String]
1206
1227
  attr_accessor :name
1207
- # @param value [Integer]
1228
+ # @return [Integer]
1208
1229
  attr_accessor :updated_at
1209
- # @param value [Integer]
1230
+ # @return [Integer]
1210
1231
  attr_accessor :created_at
1211
1232
 
1212
1233
  # @override
@@ -1257,15 +1278,15 @@ module Schemas
1257
1278
  class WidgetTheSecond < Deimos::SchemaClass::Record
1258
1279
 
1259
1280
  ### Attribute Accessors ###
1260
- # @param value [Integer]
1281
+ # @return [Integer]
1261
1282
  attr_accessor :id
1262
- # @param value [Integer]
1283
+ # @return [Integer]
1263
1284
  attr_accessor :widget_id
1264
- # @param value [String]
1285
+ # @return [String]
1265
1286
  attr_accessor :model_id
1266
- # @param value [Integer]
1287
+ # @return [Integer]
1267
1288
  attr_accessor :updated_at
1268
- # @param value [Integer]
1289
+ # @return [Integer]
1269
1290
  attr_accessor :created_at
1270
1291
 
1271
1292
  # @override