deimos-ruby 1.16.3 → 1.16.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +3 -3
- data/.gitignore +1 -0
- data/.rubocop.yml +20 -14
- data/.rubocop_todo.yml +364 -0
- data/.ruby-version +2 -1
- data/CHANGELOG.md +9 -0
- data/Gemfile +6 -0
- data/README.md +7 -1
- data/Steepfile +6 -0
- data/deimos-ruby.gemspec +3 -2
- data/lib/deimos/active_record_consume/batch_consumption.rb +7 -2
- data/lib/deimos/active_record_consume/batch_slicer.rb +2 -0
- data/lib/deimos/active_record_consume/message_consumption.rb +8 -4
- data/lib/deimos/active_record_consumer.rb +7 -4
- data/lib/deimos/active_record_producer.rb +3 -0
- data/lib/deimos/backends/base.rb +4 -2
- data/lib/deimos/backends/kafka.rb +1 -0
- data/lib/deimos/backends/kafka_async.rb +1 -0
- data/lib/deimos/config/configuration.rb +4 -0
- data/lib/deimos/config/phobos_config.rb +2 -1
- data/lib/deimos/consume/batch_consumption.rb +8 -1
- data/lib/deimos/consume/message_consumption.rb +4 -1
- data/lib/deimos/instrumentation.rb +11 -4
- data/lib/deimos/kafka_message.rb +1 -0
- data/lib/deimos/kafka_source.rb +5 -0
- data/lib/deimos/kafka_topic_info.rb +4 -0
- data/lib/deimos/message.rb +19 -2
- data/lib/deimos/metrics/datadog.rb +2 -1
- data/lib/deimos/metrics/mock.rb +2 -2
- data/lib/deimos/metrics/provider.rb +6 -0
- data/lib/deimos/monkey_patches/phobos_cli.rb +1 -1
- data/lib/deimos/monkey_patches/phobos_producer.rb +1 -0
- data/lib/deimos/producer.rb +12 -6
- data/lib/deimos/schema_backends/base.rb +31 -17
- data/lib/deimos/schema_backends/mock.rb +2 -2
- data/lib/deimos/schema_class/base.rb +9 -5
- data/lib/deimos/schema_class/enum.rb +4 -2
- data/lib/deimos/schema_class/record.rb +5 -5
- data/lib/deimos/shared_config.rb +6 -2
- data/lib/deimos/test_helpers.rb +21 -4
- data/lib/deimos/tracing/datadog.rb +1 -1
- data/lib/deimos/tracing/mock.rb +4 -3
- data/lib/deimos/tracing/provider.rb +5 -0
- data/lib/deimos/utils/db_poller.rb +9 -1
- data/lib/deimos/utils/db_producer.rb +14 -2
- data/lib/deimos/utils/deadlock_retry.rb +3 -0
- data/lib/deimos/utils/inline_consumer.rb +14 -6
- data/lib/deimos/utils/lag_reporter.rb +11 -0
- data/lib/deimos/utils/schema_controller_mixin.rb +8 -0
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +3 -2
- data/lib/generators/deimos/active_record_generator.rb +1 -1
- data/lib/generators/deimos/db_backend_generator.rb +1 -0
- data/lib/generators/deimos/db_poller_generator.rb +1 -0
- data/lib/generators/deimos/schema_class/templates/schema_record.rb.tt +1 -1
- data/lib/generators/deimos/schema_class_generator.rb +13 -4
- data/rbs_collection.lock.yaml +176 -0
- data/rbs_collection.yaml +15 -0
- data/sig/avro.rbs +14 -0
- data/sig/defs.rbs +1867 -0
- data/sig/fig_tree.rbs +2 -0
- data/spec/consumer_spec.rb +14 -14
- data/spec/generators/schema_class/my_schema_spec.rb +3 -3
- data/spec/generators/schema_class/my_schema_with_complex_types_spec.rb +1 -1
- data/spec/producer_spec.rb +1 -1
- data/spec/schemas/my_namespace/my_schema_with_complex_type.rb +3 -3
- data/spec/snapshots/consumers-no-nest.snap +1 -1
- data/spec/snapshots/consumers.snap +1 -1
- data/spec/snapshots/consumers_and_producers-no-nest.snap +3 -3
- data/spec/snapshots/consumers_and_producers.snap +3 -3
- data/spec/snapshots/consumers_circular-no-nest.snap +1 -1
- data/spec/snapshots/consumers_circular.snap +1 -1
- data/spec/snapshots/consumers_complex_types-no-nest.snap +1 -1
- data/spec/snapshots/consumers_complex_types.snap +1 -1
- data/spec/snapshots/consumers_nested-no-nest.snap +1 -1
- data/spec/snapshots/consumers_nested.snap +1 -1
- data/spec/snapshots/namespace_folders.snap +3 -3
- data/spec/snapshots/producers_with_key-no-nest.snap +1 -1
- data/spec/snapshots/producers_with_key.snap +1 -1
- metadata +39 -21
- data/.gemfiles/avro_turf-0.gemfile +0 -3
- data/.gemfiles/avro_turf-1.gemfile +0 -3
- data/.ruby-gemset +0 -1
@@ -31,6 +31,7 @@ module Deimos
|
|
31
31
|
# @param kwactions [String]
|
32
32
|
# @param request [String]
|
33
33
|
# @param response [String]
|
34
|
+
# @return [void]
|
34
35
|
def schemas(*actions, request: nil, response: nil, **kwactions)
|
35
36
|
actions.each do |action|
|
36
37
|
request ||= action.to_s.titleize
|
@@ -49,6 +50,7 @@ module Deimos
|
|
49
50
|
|
50
51
|
# Set the namespace for both requests and responses.
|
51
52
|
# @param name [String]
|
53
|
+
# @return [void]
|
52
54
|
def namespace(name)
|
53
55
|
request_namespace(name)
|
54
56
|
response_namespace(name)
|
@@ -56,12 +58,14 @@ module Deimos
|
|
56
58
|
|
57
59
|
# Set the namespace for requests.
|
58
60
|
# @param name [String]
|
61
|
+
# @return [void]
|
59
62
|
def request_namespace(name)
|
60
63
|
namespaces[:request] = name
|
61
64
|
end
|
62
65
|
|
63
66
|
# Set the namespace for repsonses.
|
64
67
|
# @param name [String]
|
68
|
+
# @return [void]
|
65
69
|
def response_namespace(name)
|
66
70
|
namespaces[:response] = name
|
67
71
|
end
|
@@ -94,6 +98,7 @@ module Deimos
|
|
94
98
|
end
|
95
99
|
|
96
100
|
# Decode the payload with the parameters.
|
101
|
+
# @return [void]
|
97
102
|
def decode_schema
|
98
103
|
namespace, schema = parse_namespace(:request)
|
99
104
|
decoder = Deimos.schema_backend(schema: schema, namespace: namespace)
|
@@ -109,6 +114,9 @@ module Deimos
|
|
109
114
|
|
110
115
|
# Render a hash into a payload as specified by the configured schema and namespace.
|
111
116
|
# @param payload [Hash]
|
117
|
+
# @param schema [String]
|
118
|
+
# @param namespace [String]
|
119
|
+
# @return [void]
|
112
120
|
def render_schema(payload, schema: nil, namespace: nil)
|
113
121
|
namespace, schema = parse_namespace(:response) if !schema && !namespace
|
114
122
|
encoder = Deimos.schema_backend(schema: schema, namespace: namespace)
|
data/lib/deimos/version.rb
CHANGED
data/lib/deimos.rb
CHANGED
@@ -45,7 +45,7 @@ require 'erb'
|
|
45
45
|
# Parent module.
|
46
46
|
module Deimos
|
47
47
|
class << self
|
48
|
-
# @return [Class
|
48
|
+
# @return [Class<Deimos::SchemaBackends::Base>]
|
49
49
|
def schema_backend_class
|
50
50
|
backend = Deimos.config.schema.backend.to_s
|
51
51
|
|
@@ -54,7 +54,7 @@ module Deimos
|
|
54
54
|
"Deimos::SchemaBackends::#{backend.classify}".constantize
|
55
55
|
end
|
56
56
|
|
57
|
-
# @param schema [String
|
57
|
+
# @param schema [String, Symbol]
|
58
58
|
# @param namespace [String]
|
59
59
|
# @return [Deimos::SchemaBackends::Base]
|
60
60
|
def schema_backend(schema:, namespace:)
|
@@ -81,6 +81,7 @@ module Deimos
|
|
81
81
|
|
82
82
|
# Start the DB producers to send Kafka messages.
|
83
83
|
# @param thread_count [Integer] the number of threads to start.
|
84
|
+
# @return [void]
|
84
85
|
def start_db_backend!(thread_count: 1)
|
85
86
|
Sigurd.exit_on_signal = true
|
86
87
|
if self.config.producers.backend != :db
|
@@ -69,7 +69,7 @@ module Deimos
|
|
69
69
|
end
|
70
70
|
|
71
71
|
desc 'Generate migration for a table based on an existing schema.'
|
72
|
-
#
|
72
|
+
# @return [void]
|
73
73
|
def generate
|
74
74
|
migration_template('migration.rb', "db/migrate/create_#{table_name.underscore}.rb")
|
75
75
|
template('model.rb', "app/models/#{table_name.underscore.singularize}.rb")
|
@@ -11,11 +11,17 @@ module Deimos
|
|
11
11
|
# Generator for Schema Classes used for the IDE and consumer/producer interfaces
|
12
12
|
class SchemaClassGenerator < Rails::Generators::Base
|
13
13
|
|
14
|
+
# @return [Array<Symbol>]
|
14
15
|
SPECIAL_TYPES = %i(record enum).freeze
|
16
|
+
# @return [String]
|
15
17
|
INITIALIZE_WHITESPACE = "\n#{' ' * 19}"
|
18
|
+
# @return [Array<String>]
|
16
19
|
IGNORE_DEFAULTS = %w(message_id timestamp).freeze
|
20
|
+
# @return [String]
|
17
21
|
SCHEMA_CLASS_FILE = 'schema_class.rb'
|
22
|
+
# @return [String]
|
18
23
|
SCHEMA_RECORD_PATH = File.expand_path('schema_class/templates/schema_record.rb.tt', __dir__).freeze
|
24
|
+
# @return [String]
|
19
25
|
SCHEMA_ENUM_PATH = File.expand_path('schema_class/templates/schema_enum.rb.tt', __dir__).freeze
|
20
26
|
|
21
27
|
source_root File.expand_path('schema_class/templates', __dir__)
|
@@ -42,6 +48,7 @@ module Deimos
|
|
42
48
|
# @param schema_name [String]
|
43
49
|
# @param namespace [String]
|
44
50
|
# @param key_config [Hash,nil]
|
51
|
+
# @return [void]
|
45
52
|
def generate_classes(schema_name, namespace, key_config)
|
46
53
|
schema_base = Deimos.schema_backend(schema: schema_name, namespace: namespace)
|
47
54
|
schema_base.load_schema
|
@@ -84,6 +91,7 @@ module Deimos
|
|
84
91
|
|
85
92
|
# @param schema_base [Deimos::SchemaBackends::Base]
|
86
93
|
# @param key_config [Hash,nil]
|
94
|
+
# @return [void]
|
87
95
|
def generate_class_from_schema_base(schema_base, key_config: nil)
|
88
96
|
@discovered_schemas = Set.new
|
89
97
|
@sub_schema_templates = []
|
@@ -105,7 +113,8 @@ module Deimos
|
|
105
113
|
end
|
106
114
|
|
107
115
|
# @param schema [Avro::Schema::NamedSchema]
|
108
|
-
# @param
|
116
|
+
# @param key_config [Hash,nil]
|
117
|
+
# @return [void]
|
109
118
|
def write_file(schema, key_config)
|
110
119
|
class_template = _generate_class_template_from_schema(schema, key_config)
|
111
120
|
@modules = Utils::SchemaClass.modules_for(schema.namespace)
|
@@ -143,7 +152,7 @@ module Deimos
|
|
143
152
|
end
|
144
153
|
|
145
154
|
desc 'Generate a class based on configured consumer and producers.'
|
146
|
-
#
|
155
|
+
# @return [void]
|
147
156
|
def generate
|
148
157
|
_validate
|
149
158
|
Rails.logger.info("Generating schemas from Deimos.config to #{Deimos.config.schema.generated_class_path}")
|
@@ -175,7 +184,7 @@ module Deimos
|
|
175
184
|
def generate_from_schema_files(found_schemas)
|
176
185
|
schema_store = AvroTurf::MutableSchemaStore.new(path: Deimos.config.schema.path)
|
177
186
|
schema_store.load_schemas!
|
178
|
-
schema_store.schemas.values.each do |schema|
|
187
|
+
schema_store.schemas.values.sort_by { |s| "#{s.namespace}#{s.name}" }.each do |schema|
|
179
188
|
name = "#{schema.namespace}.#{schema.name}"
|
180
189
|
next if found_schemas.include?(name)
|
181
190
|
|
@@ -252,7 +261,7 @@ module Deimos
|
|
252
261
|
"#{result})"
|
253
262
|
end
|
254
263
|
|
255
|
-
# @param [SchemaField]
|
264
|
+
# @param field [SchemaField]
|
256
265
|
# @return [String]
|
257
266
|
def _field_default(field)
|
258
267
|
default = field.default
|
@@ -0,0 +1,176 @@
|
|
1
|
+
---
|
2
|
+
sources:
|
3
|
+
- name: ruby/gem_rbs_collection
|
4
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
5
|
+
revision: main
|
6
|
+
repo_dir: gems
|
7
|
+
path: ".gem_rbs_collection"
|
8
|
+
gems:
|
9
|
+
- name: actionpack
|
10
|
+
version: '6.0'
|
11
|
+
source:
|
12
|
+
type: git
|
13
|
+
name: ruby/gem_rbs_collection
|
14
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
15
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
16
|
+
repo_dir: gems
|
17
|
+
- name: actionview
|
18
|
+
version: '6.0'
|
19
|
+
source:
|
20
|
+
type: git
|
21
|
+
name: ruby/gem_rbs_collection
|
22
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
23
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
24
|
+
repo_dir: gems
|
25
|
+
- name: activejob
|
26
|
+
version: '6.0'
|
27
|
+
source:
|
28
|
+
type: git
|
29
|
+
name: ruby/gem_rbs_collection
|
30
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
31
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
32
|
+
repo_dir: gems
|
33
|
+
- name: activemodel
|
34
|
+
version: '6.0'
|
35
|
+
source:
|
36
|
+
type: git
|
37
|
+
name: ruby/gem_rbs_collection
|
38
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
39
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
40
|
+
repo_dir: gems
|
41
|
+
- name: activerecord
|
42
|
+
version: '6.1'
|
43
|
+
source:
|
44
|
+
type: git
|
45
|
+
name: ruby/gem_rbs_collection
|
46
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
47
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
48
|
+
repo_dir: gems
|
49
|
+
- name: activestorage
|
50
|
+
version: '6.1'
|
51
|
+
source:
|
52
|
+
type: git
|
53
|
+
name: ruby/gem_rbs_collection
|
54
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
55
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
56
|
+
repo_dir: gems
|
57
|
+
- name: activesupport
|
58
|
+
version: '6.0'
|
59
|
+
source:
|
60
|
+
type: git
|
61
|
+
name: ruby/gem_rbs_collection
|
62
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
63
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
64
|
+
repo_dir: gems
|
65
|
+
- name: ast
|
66
|
+
version: '2.4'
|
67
|
+
source:
|
68
|
+
type: git
|
69
|
+
name: ruby/gem_rbs_collection
|
70
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
71
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
72
|
+
repo_dir: gems
|
73
|
+
- name: deimos-ruby
|
74
|
+
version: 1.16.4
|
75
|
+
source:
|
76
|
+
type: rubygems
|
77
|
+
- name: i18n
|
78
|
+
version: '1.10'
|
79
|
+
source:
|
80
|
+
type: git
|
81
|
+
name: ruby/gem_rbs_collection
|
82
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
83
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
84
|
+
repo_dir: gems
|
85
|
+
- name: listen
|
86
|
+
version: '3.2'
|
87
|
+
source:
|
88
|
+
type: git
|
89
|
+
name: ruby/gem_rbs_collection
|
90
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
91
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
92
|
+
repo_dir: gems
|
93
|
+
- name: minitest
|
94
|
+
version: '0'
|
95
|
+
source:
|
96
|
+
type: stdlib
|
97
|
+
- name: nokogiri
|
98
|
+
version: '1.11'
|
99
|
+
source:
|
100
|
+
type: git
|
101
|
+
name: ruby/gem_rbs_collection
|
102
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
103
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
104
|
+
repo_dir: gems
|
105
|
+
- name: parallel
|
106
|
+
version: '1.20'
|
107
|
+
source:
|
108
|
+
type: git
|
109
|
+
name: ruby/gem_rbs_collection
|
110
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
111
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
112
|
+
repo_dir: gems
|
113
|
+
- name: phobos
|
114
|
+
version: 2.1.5
|
115
|
+
source:
|
116
|
+
type: rubygems
|
117
|
+
- name: rack
|
118
|
+
version: 2.2.2
|
119
|
+
source:
|
120
|
+
type: git
|
121
|
+
name: ruby/gem_rbs_collection
|
122
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
123
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
124
|
+
repo_dir: gems
|
125
|
+
- name: railties
|
126
|
+
version: '6.0'
|
127
|
+
source:
|
128
|
+
type: git
|
129
|
+
name: ruby/gem_rbs_collection
|
130
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
131
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
132
|
+
repo_dir: gems
|
133
|
+
- name: rainbow
|
134
|
+
version: '3.0'
|
135
|
+
source:
|
136
|
+
type: git
|
137
|
+
name: ruby/gem_rbs_collection
|
138
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
139
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
140
|
+
repo_dir: gems
|
141
|
+
- name: tempfile
|
142
|
+
version: '0'
|
143
|
+
source:
|
144
|
+
type: stdlib
|
145
|
+
- name: monitor
|
146
|
+
version: '0'
|
147
|
+
source:
|
148
|
+
type: stdlib
|
149
|
+
- name: date
|
150
|
+
version: '0'
|
151
|
+
source:
|
152
|
+
type: stdlib
|
153
|
+
- name: singleton
|
154
|
+
version: '0'
|
155
|
+
source:
|
156
|
+
type: stdlib
|
157
|
+
- name: logger
|
158
|
+
version: '0'
|
159
|
+
source:
|
160
|
+
type: stdlib
|
161
|
+
- name: mutex_m
|
162
|
+
version: '0'
|
163
|
+
source:
|
164
|
+
type: stdlib
|
165
|
+
- name: time
|
166
|
+
version: '0'
|
167
|
+
source:
|
168
|
+
type: stdlib
|
169
|
+
- name: pathname
|
170
|
+
version: '0'
|
171
|
+
source:
|
172
|
+
type: stdlib
|
173
|
+
- name: tsort
|
174
|
+
version: '0'
|
175
|
+
source:
|
176
|
+
type: stdlib
|
data/rbs_collection.yaml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
# Download sources
|
2
|
+
sources:
|
3
|
+
- name: ruby/gem_rbs_collection
|
4
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
5
|
+
revision: main
|
6
|
+
repo_dir: gems
|
7
|
+
|
8
|
+
# A directory to install the downloaded RBSs
|
9
|
+
path: .gem_rbs_collection
|
10
|
+
|
11
|
+
gems:
|
12
|
+
# Skip loading rbs gem's RBS.
|
13
|
+
# It's unnecessary if you don't use rbs as a library.
|
14
|
+
- name: rbs
|
15
|
+
ignore: true
|