deimos-ruby 1.16.3 → 1.16.5

Sign up to get free protection for your applications and to get access to all the features.
Files changed (84) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ci.yml +3 -3
  3. data/.gitignore +1 -0
  4. data/.rubocop.yml +20 -14
  5. data/.rubocop_todo.yml +364 -0
  6. data/.ruby-version +2 -1
  7. data/CHANGELOG.md +9 -0
  8. data/Gemfile +6 -0
  9. data/README.md +7 -1
  10. data/Steepfile +6 -0
  11. data/deimos-ruby.gemspec +3 -2
  12. data/lib/deimos/active_record_consume/batch_consumption.rb +7 -2
  13. data/lib/deimos/active_record_consume/batch_slicer.rb +2 -0
  14. data/lib/deimos/active_record_consume/message_consumption.rb +8 -4
  15. data/lib/deimos/active_record_consumer.rb +7 -4
  16. data/lib/deimos/active_record_producer.rb +3 -0
  17. data/lib/deimos/backends/base.rb +4 -2
  18. data/lib/deimos/backends/kafka.rb +1 -0
  19. data/lib/deimos/backends/kafka_async.rb +1 -0
  20. data/lib/deimos/config/configuration.rb +4 -0
  21. data/lib/deimos/config/phobos_config.rb +2 -1
  22. data/lib/deimos/consume/batch_consumption.rb +8 -1
  23. data/lib/deimos/consume/message_consumption.rb +4 -1
  24. data/lib/deimos/instrumentation.rb +11 -4
  25. data/lib/deimos/kafka_message.rb +1 -0
  26. data/lib/deimos/kafka_source.rb +5 -0
  27. data/lib/deimos/kafka_topic_info.rb +4 -0
  28. data/lib/deimos/message.rb +19 -2
  29. data/lib/deimos/metrics/datadog.rb +2 -1
  30. data/lib/deimos/metrics/mock.rb +2 -2
  31. data/lib/deimos/metrics/provider.rb +6 -0
  32. data/lib/deimos/monkey_patches/phobos_cli.rb +1 -1
  33. data/lib/deimos/monkey_patches/phobos_producer.rb +1 -0
  34. data/lib/deimos/producer.rb +12 -6
  35. data/lib/deimos/schema_backends/base.rb +31 -17
  36. data/lib/deimos/schema_backends/mock.rb +2 -2
  37. data/lib/deimos/schema_class/base.rb +9 -5
  38. data/lib/deimos/schema_class/enum.rb +4 -2
  39. data/lib/deimos/schema_class/record.rb +5 -5
  40. data/lib/deimos/shared_config.rb +6 -2
  41. data/lib/deimos/test_helpers.rb +21 -4
  42. data/lib/deimos/tracing/datadog.rb +1 -1
  43. data/lib/deimos/tracing/mock.rb +4 -3
  44. data/lib/deimos/tracing/provider.rb +5 -0
  45. data/lib/deimos/utils/db_poller.rb +9 -1
  46. data/lib/deimos/utils/db_producer.rb +14 -2
  47. data/lib/deimos/utils/deadlock_retry.rb +3 -0
  48. data/lib/deimos/utils/inline_consumer.rb +14 -6
  49. data/lib/deimos/utils/lag_reporter.rb +11 -0
  50. data/lib/deimos/utils/schema_controller_mixin.rb +8 -0
  51. data/lib/deimos/version.rb +1 -1
  52. data/lib/deimos.rb +3 -2
  53. data/lib/generators/deimos/active_record_generator.rb +1 -1
  54. data/lib/generators/deimos/db_backend_generator.rb +1 -0
  55. data/lib/generators/deimos/db_poller_generator.rb +1 -0
  56. data/lib/generators/deimos/schema_class/templates/schema_record.rb.tt +1 -1
  57. data/lib/generators/deimos/schema_class_generator.rb +13 -4
  58. data/rbs_collection.lock.yaml +176 -0
  59. data/rbs_collection.yaml +15 -0
  60. data/sig/avro.rbs +14 -0
  61. data/sig/defs.rbs +1867 -0
  62. data/sig/fig_tree.rbs +2 -0
  63. data/spec/consumer_spec.rb +14 -14
  64. data/spec/generators/schema_class/my_schema_spec.rb +3 -3
  65. data/spec/generators/schema_class/my_schema_with_complex_types_spec.rb +1 -1
  66. data/spec/producer_spec.rb +1 -1
  67. data/spec/schemas/my_namespace/my_schema_with_complex_type.rb +3 -3
  68. data/spec/snapshots/consumers-no-nest.snap +1 -1
  69. data/spec/snapshots/consumers.snap +1 -1
  70. data/spec/snapshots/consumers_and_producers-no-nest.snap +3 -3
  71. data/spec/snapshots/consumers_and_producers.snap +3 -3
  72. data/spec/snapshots/consumers_circular-no-nest.snap +1 -1
  73. data/spec/snapshots/consumers_circular.snap +1 -1
  74. data/spec/snapshots/consumers_complex_types-no-nest.snap +1 -1
  75. data/spec/snapshots/consumers_complex_types.snap +1 -1
  76. data/spec/snapshots/consumers_nested-no-nest.snap +1 -1
  77. data/spec/snapshots/consumers_nested.snap +1 -1
  78. data/spec/snapshots/namespace_folders.snap +3 -3
  79. data/spec/snapshots/producers_with_key-no-nest.snap +1 -1
  80. data/spec/snapshots/producers_with_key.snap +1 -1
  81. metadata +39 -21
  82. data/.gemfiles/avro_turf-0.gemfile +0 -3
  83. data/.gemfiles/avro_turf-1.gemfile +0 -3
  84. data/.ruby-gemset +0 -1
@@ -31,6 +31,7 @@ module Deimos
31
31
  # @param kwactions [String]
32
32
  # @param request [String]
33
33
  # @param response [String]
34
+ # @return [void]
34
35
  def schemas(*actions, request: nil, response: nil, **kwactions)
35
36
  actions.each do |action|
36
37
  request ||= action.to_s.titleize
@@ -49,6 +50,7 @@ module Deimos
49
50
 
50
51
  # Set the namespace for both requests and responses.
51
52
  # @param name [String]
53
+ # @return [void]
52
54
  def namespace(name)
53
55
  request_namespace(name)
54
56
  response_namespace(name)
@@ -56,12 +58,14 @@ module Deimos
56
58
 
57
59
  # Set the namespace for requests.
58
60
  # @param name [String]
61
+ # @return [void]
59
62
  def request_namespace(name)
60
63
  namespaces[:request] = name
61
64
  end
62
65
 
63
66
  # Set the namespace for repsonses.
64
67
  # @param name [String]
68
+ # @return [void]
65
69
  def response_namespace(name)
66
70
  namespaces[:response] = name
67
71
  end
@@ -94,6 +98,7 @@ module Deimos
94
98
  end
95
99
 
96
100
  # Decode the payload with the parameters.
101
+ # @return [void]
97
102
  def decode_schema
98
103
  namespace, schema = parse_namespace(:request)
99
104
  decoder = Deimos.schema_backend(schema: schema, namespace: namespace)
@@ -109,6 +114,9 @@ module Deimos
109
114
 
110
115
  # Render a hash into a payload as specified by the configured schema and namespace.
111
116
  # @param payload [Hash]
117
+ # @param schema [String]
118
+ # @param namespace [String]
119
+ # @return [void]
112
120
  def render_schema(payload, schema: nil, namespace: nil)
113
121
  namespace, schema = parse_namespace(:response) if !schema && !namespace
114
122
  encoder = Deimos.schema_backend(schema: schema, namespace: namespace)
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Deimos
4
- VERSION = '1.16.3'
4
+ VERSION = '1.16.5'
5
5
  end
data/lib/deimos.rb CHANGED
@@ -45,7 +45,7 @@ require 'erb'
45
45
  # Parent module.
46
46
  module Deimos
47
47
  class << self
48
- # @return [Class < Deimos::SchemaBackends::Base]
48
+ # @return [Class<Deimos::SchemaBackends::Base>]
49
49
  def schema_backend_class
50
50
  backend = Deimos.config.schema.backend.to_s
51
51
 
@@ -54,7 +54,7 @@ module Deimos
54
54
  "Deimos::SchemaBackends::#{backend.classify}".constantize
55
55
  end
56
56
 
57
- # @param schema [String|Symbol]
57
+ # @param schema [String, Symbol]
58
58
  # @param namespace [String]
59
59
  # @return [Deimos::SchemaBackends::Base]
60
60
  def schema_backend(schema:, namespace:)
@@ -81,6 +81,7 @@ module Deimos
81
81
 
82
82
  # Start the DB producers to send Kafka messages.
83
83
  # @param thread_count [Integer] the number of threads to start.
84
+ # @return [void]
84
85
  def start_db_backend!(thread_count: 1)
85
86
  Sigurd.exit_on_signal = true
86
87
  if self.config.producers.backend != :db
@@ -69,7 +69,7 @@ module Deimos
69
69
  end
70
70
 
71
71
  desc 'Generate migration for a table based on an existing schema.'
72
- # :nodoc:
72
+ # @return [void]
73
73
  def generate
74
74
  migration_template('migration.rb', "db/migrate/create_#{table_name.underscore}.rb")
75
75
  template('model.rb', "app/models/#{table_name.underscore.singularize}.rb")
@@ -34,6 +34,7 @@ module Deimos
34
34
  end
35
35
 
36
36
  # Main method to create all the necessary files
37
+ # @return [void]
37
38
  def generate
38
39
  if Rails.version < '4'
39
40
  migration_template('rails3_migration',
@@ -34,6 +34,7 @@ module Deimos
34
34
  end
35
35
 
36
36
  # Main method to create all the necessary files
37
+ # @return [void]
37
38
  def generate
38
39
  if Rails.version < '4'
39
40
  migration_template('rails3_migration',
@@ -64,7 +64,7 @@
64
64
  end
65
65
  <%- if @tombstone_assignment %>
66
66
  def self.tombstone(key)
67
- record = self.new
67
+ record = self.allocate
68
68
  <%- if @tombstone_assignment.present? -%>
69
69
  <%= @tombstone_assignment %>
70
70
  <%- end -%>
@@ -11,11 +11,17 @@ module Deimos
11
11
  # Generator for Schema Classes used for the IDE and consumer/producer interfaces
12
12
  class SchemaClassGenerator < Rails::Generators::Base
13
13
 
14
+ # @return [Array<Symbol>]
14
15
  SPECIAL_TYPES = %i(record enum).freeze
16
+ # @return [String]
15
17
  INITIALIZE_WHITESPACE = "\n#{' ' * 19}"
18
+ # @return [Array<String>]
16
19
  IGNORE_DEFAULTS = %w(message_id timestamp).freeze
20
+ # @return [String]
17
21
  SCHEMA_CLASS_FILE = 'schema_class.rb'
22
+ # @return [String]
18
23
  SCHEMA_RECORD_PATH = File.expand_path('schema_class/templates/schema_record.rb.tt', __dir__).freeze
24
+ # @return [String]
19
25
  SCHEMA_ENUM_PATH = File.expand_path('schema_class/templates/schema_enum.rb.tt', __dir__).freeze
20
26
 
21
27
  source_root File.expand_path('schema_class/templates', __dir__)
@@ -42,6 +48,7 @@ module Deimos
42
48
  # @param schema_name [String]
43
49
  # @param namespace [String]
44
50
  # @param key_config [Hash,nil]
51
+ # @return [void]
45
52
  def generate_classes(schema_name, namespace, key_config)
46
53
  schema_base = Deimos.schema_backend(schema: schema_name, namespace: namespace)
47
54
  schema_base.load_schema
@@ -84,6 +91,7 @@ module Deimos
84
91
 
85
92
  # @param schema_base [Deimos::SchemaBackends::Base]
86
93
  # @param key_config [Hash,nil]
94
+ # @return [void]
87
95
  def generate_class_from_schema_base(schema_base, key_config: nil)
88
96
  @discovered_schemas = Set.new
89
97
  @sub_schema_templates = []
@@ -105,7 +113,8 @@ module Deimos
105
113
  end
106
114
 
107
115
  # @param schema [Avro::Schema::NamedSchema]
108
- # @param key_schema_base [Hash]
116
+ # @param key_config [Hash,nil]
117
+ # @return [void]
109
118
  def write_file(schema, key_config)
110
119
  class_template = _generate_class_template_from_schema(schema, key_config)
111
120
  @modules = Utils::SchemaClass.modules_for(schema.namespace)
@@ -143,7 +152,7 @@ module Deimos
143
152
  end
144
153
 
145
154
  desc 'Generate a class based on configured consumer and producers.'
146
- # :nodoc:
155
+ # @return [void]
147
156
  def generate
148
157
  _validate
149
158
  Rails.logger.info("Generating schemas from Deimos.config to #{Deimos.config.schema.generated_class_path}")
@@ -175,7 +184,7 @@ module Deimos
175
184
  def generate_from_schema_files(found_schemas)
176
185
  schema_store = AvroTurf::MutableSchemaStore.new(path: Deimos.config.schema.path)
177
186
  schema_store.load_schemas!
178
- schema_store.schemas.values.each do |schema|
187
+ schema_store.schemas.values.sort_by { |s| "#{s.namespace}#{s.name}" }.each do |schema|
179
188
  name = "#{schema.namespace}.#{schema.name}"
180
189
  next if found_schemas.include?(name)
181
190
 
@@ -252,7 +261,7 @@ module Deimos
252
261
  "#{result})"
253
262
  end
254
263
 
255
- # @param [SchemaField]
264
+ # @param field [SchemaField]
256
265
  # @return [String]
257
266
  def _field_default(field)
258
267
  default = field.default
@@ -0,0 +1,176 @@
1
+ ---
2
+ sources:
3
+ - name: ruby/gem_rbs_collection
4
+ remote: https://github.com/ruby/gem_rbs_collection.git
5
+ revision: main
6
+ repo_dir: gems
7
+ path: ".gem_rbs_collection"
8
+ gems:
9
+ - name: actionpack
10
+ version: '6.0'
11
+ source:
12
+ type: git
13
+ name: ruby/gem_rbs_collection
14
+ revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
15
+ remote: https://github.com/ruby/gem_rbs_collection.git
16
+ repo_dir: gems
17
+ - name: actionview
18
+ version: '6.0'
19
+ source:
20
+ type: git
21
+ name: ruby/gem_rbs_collection
22
+ revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
23
+ remote: https://github.com/ruby/gem_rbs_collection.git
24
+ repo_dir: gems
25
+ - name: activejob
26
+ version: '6.0'
27
+ source:
28
+ type: git
29
+ name: ruby/gem_rbs_collection
30
+ revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
31
+ remote: https://github.com/ruby/gem_rbs_collection.git
32
+ repo_dir: gems
33
+ - name: activemodel
34
+ version: '6.0'
35
+ source:
36
+ type: git
37
+ name: ruby/gem_rbs_collection
38
+ revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
39
+ remote: https://github.com/ruby/gem_rbs_collection.git
40
+ repo_dir: gems
41
+ - name: activerecord
42
+ version: '6.1'
43
+ source:
44
+ type: git
45
+ name: ruby/gem_rbs_collection
46
+ revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
47
+ remote: https://github.com/ruby/gem_rbs_collection.git
48
+ repo_dir: gems
49
+ - name: activestorage
50
+ version: '6.1'
51
+ source:
52
+ type: git
53
+ name: ruby/gem_rbs_collection
54
+ revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
55
+ remote: https://github.com/ruby/gem_rbs_collection.git
56
+ repo_dir: gems
57
+ - name: activesupport
58
+ version: '6.0'
59
+ source:
60
+ type: git
61
+ name: ruby/gem_rbs_collection
62
+ revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
63
+ remote: https://github.com/ruby/gem_rbs_collection.git
64
+ repo_dir: gems
65
+ - name: ast
66
+ version: '2.4'
67
+ source:
68
+ type: git
69
+ name: ruby/gem_rbs_collection
70
+ revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
71
+ remote: https://github.com/ruby/gem_rbs_collection.git
72
+ repo_dir: gems
73
+ - name: deimos-ruby
74
+ version: 1.16.4
75
+ source:
76
+ type: rubygems
77
+ - name: i18n
78
+ version: '1.10'
79
+ source:
80
+ type: git
81
+ name: ruby/gem_rbs_collection
82
+ revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
83
+ remote: https://github.com/ruby/gem_rbs_collection.git
84
+ repo_dir: gems
85
+ - name: listen
86
+ version: '3.2'
87
+ source:
88
+ type: git
89
+ name: ruby/gem_rbs_collection
90
+ revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
91
+ remote: https://github.com/ruby/gem_rbs_collection.git
92
+ repo_dir: gems
93
+ - name: minitest
94
+ version: '0'
95
+ source:
96
+ type: stdlib
97
+ - name: nokogiri
98
+ version: '1.11'
99
+ source:
100
+ type: git
101
+ name: ruby/gem_rbs_collection
102
+ revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
103
+ remote: https://github.com/ruby/gem_rbs_collection.git
104
+ repo_dir: gems
105
+ - name: parallel
106
+ version: '1.20'
107
+ source:
108
+ type: git
109
+ name: ruby/gem_rbs_collection
110
+ revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
111
+ remote: https://github.com/ruby/gem_rbs_collection.git
112
+ repo_dir: gems
113
+ - name: phobos
114
+ version: 2.1.5
115
+ source:
116
+ type: rubygems
117
+ - name: rack
118
+ version: 2.2.2
119
+ source:
120
+ type: git
121
+ name: ruby/gem_rbs_collection
122
+ revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
123
+ remote: https://github.com/ruby/gem_rbs_collection.git
124
+ repo_dir: gems
125
+ - name: railties
126
+ version: '6.0'
127
+ source:
128
+ type: git
129
+ name: ruby/gem_rbs_collection
130
+ revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
131
+ remote: https://github.com/ruby/gem_rbs_collection.git
132
+ repo_dir: gems
133
+ - name: rainbow
134
+ version: '3.0'
135
+ source:
136
+ type: git
137
+ name: ruby/gem_rbs_collection
138
+ revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
139
+ remote: https://github.com/ruby/gem_rbs_collection.git
140
+ repo_dir: gems
141
+ - name: tempfile
142
+ version: '0'
143
+ source:
144
+ type: stdlib
145
+ - name: monitor
146
+ version: '0'
147
+ source:
148
+ type: stdlib
149
+ - name: date
150
+ version: '0'
151
+ source:
152
+ type: stdlib
153
+ - name: singleton
154
+ version: '0'
155
+ source:
156
+ type: stdlib
157
+ - name: logger
158
+ version: '0'
159
+ source:
160
+ type: stdlib
161
+ - name: mutex_m
162
+ version: '0'
163
+ source:
164
+ type: stdlib
165
+ - name: time
166
+ version: '0'
167
+ source:
168
+ type: stdlib
169
+ - name: pathname
170
+ version: '0'
171
+ source:
172
+ type: stdlib
173
+ - name: tsort
174
+ version: '0'
175
+ source:
176
+ type: stdlib
@@ -0,0 +1,15 @@
1
+ # Download sources
2
+ sources:
3
+ - name: ruby/gem_rbs_collection
4
+ remote: https://github.com/ruby/gem_rbs_collection.git
5
+ revision: main
6
+ repo_dir: gems
7
+
8
+ # A directory to install the downloaded RBSs
9
+ path: .gem_rbs_collection
10
+
11
+ gems:
12
+ # Skip loading rbs gem's RBS.
13
+ # It's unnecessary if you don't use rbs as a library.
14
+ - name: rbs
15
+ ignore: true
data/sig/avro.rbs ADDED
@@ -0,0 +1,14 @@
1
+ module Avro
2
+ class Schema
3
+ class NamedSchema
4
+ end
5
+
6
+ class UnionSchema
7
+ end
8
+
9
+ class RecordSchema
10
+ end
11
+
12
+
13
+ end
14
+ end