deimos-ruby 1.16.3 → 1.16.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +3 -3
- data/.gitignore +1 -0
- data/.rubocop.yml +20 -14
- data/.rubocop_todo.yml +364 -0
- data/.ruby-version +2 -1
- data/CHANGELOG.md +9 -0
- data/Gemfile +6 -0
- data/README.md +7 -1
- data/Steepfile +6 -0
- data/deimos-ruby.gemspec +3 -2
- data/lib/deimos/active_record_consume/batch_consumption.rb +7 -2
- data/lib/deimos/active_record_consume/batch_slicer.rb +2 -0
- data/lib/deimos/active_record_consume/message_consumption.rb +8 -4
- data/lib/deimos/active_record_consumer.rb +7 -4
- data/lib/deimos/active_record_producer.rb +3 -0
- data/lib/deimos/backends/base.rb +4 -2
- data/lib/deimos/backends/kafka.rb +1 -0
- data/lib/deimos/backends/kafka_async.rb +1 -0
- data/lib/deimos/config/configuration.rb +4 -0
- data/lib/deimos/config/phobos_config.rb +2 -1
- data/lib/deimos/consume/batch_consumption.rb +8 -1
- data/lib/deimos/consume/message_consumption.rb +4 -1
- data/lib/deimos/instrumentation.rb +11 -4
- data/lib/deimos/kafka_message.rb +1 -0
- data/lib/deimos/kafka_source.rb +5 -0
- data/lib/deimos/kafka_topic_info.rb +4 -0
- data/lib/deimos/message.rb +19 -2
- data/lib/deimos/metrics/datadog.rb +2 -1
- data/lib/deimos/metrics/mock.rb +2 -2
- data/lib/deimos/metrics/provider.rb +6 -0
- data/lib/deimos/monkey_patches/phobos_cli.rb +1 -1
- data/lib/deimos/monkey_patches/phobos_producer.rb +1 -0
- data/lib/deimos/producer.rb +12 -6
- data/lib/deimos/schema_backends/base.rb +31 -17
- data/lib/deimos/schema_backends/mock.rb +2 -2
- data/lib/deimos/schema_class/base.rb +9 -5
- data/lib/deimos/schema_class/enum.rb +4 -2
- data/lib/deimos/schema_class/record.rb +5 -5
- data/lib/deimos/shared_config.rb +6 -2
- data/lib/deimos/test_helpers.rb +21 -4
- data/lib/deimos/tracing/datadog.rb +1 -1
- data/lib/deimos/tracing/mock.rb +4 -3
- data/lib/deimos/tracing/provider.rb +5 -0
- data/lib/deimos/utils/db_poller.rb +9 -1
- data/lib/deimos/utils/db_producer.rb +14 -2
- data/lib/deimos/utils/deadlock_retry.rb +3 -0
- data/lib/deimos/utils/inline_consumer.rb +14 -6
- data/lib/deimos/utils/lag_reporter.rb +11 -0
- data/lib/deimos/utils/schema_controller_mixin.rb +8 -0
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +3 -2
- data/lib/generators/deimos/active_record_generator.rb +1 -1
- data/lib/generators/deimos/db_backend_generator.rb +1 -0
- data/lib/generators/deimos/db_poller_generator.rb +1 -0
- data/lib/generators/deimos/schema_class/templates/schema_record.rb.tt +1 -1
- data/lib/generators/deimos/schema_class_generator.rb +13 -4
- data/rbs_collection.lock.yaml +176 -0
- data/rbs_collection.yaml +15 -0
- data/sig/avro.rbs +14 -0
- data/sig/defs.rbs +1867 -0
- data/sig/fig_tree.rbs +2 -0
- data/spec/consumer_spec.rb +14 -14
- data/spec/generators/schema_class/my_schema_spec.rb +3 -3
- data/spec/generators/schema_class/my_schema_with_complex_types_spec.rb +1 -1
- data/spec/producer_spec.rb +1 -1
- data/spec/schemas/my_namespace/my_schema_with_complex_type.rb +3 -3
- data/spec/snapshots/consumers-no-nest.snap +1 -1
- data/spec/snapshots/consumers.snap +1 -1
- data/spec/snapshots/consumers_and_producers-no-nest.snap +3 -3
- data/spec/snapshots/consumers_and_producers.snap +3 -3
- data/spec/snapshots/consumers_circular-no-nest.snap +1 -1
- data/spec/snapshots/consumers_circular.snap +1 -1
- data/spec/snapshots/consumers_complex_types-no-nest.snap +1 -1
- data/spec/snapshots/consumers_complex_types.snap +1 -1
- data/spec/snapshots/consumers_nested-no-nest.snap +1 -1
- data/spec/snapshots/consumers_nested.snap +1 -1
- data/spec/snapshots/namespace_folders.snap +3 -3
- data/spec/snapshots/producers_with_key-no-nest.snap +1 -1
- data/spec/snapshots/producers_with_key.snap +1 -1
- metadata +39 -21
- data/.gemfiles/avro_turf-0.gemfile +0 -3
- data/.gemfiles/avro_turf-1.gemfile +0 -3
- data/.ruby-gemset +0 -1
@@ -31,6 +31,7 @@ module Deimos
|
|
31
31
|
# @param kwactions [String]
|
32
32
|
# @param request [String]
|
33
33
|
# @param response [String]
|
34
|
+
# @return [void]
|
34
35
|
def schemas(*actions, request: nil, response: nil, **kwactions)
|
35
36
|
actions.each do |action|
|
36
37
|
request ||= action.to_s.titleize
|
@@ -49,6 +50,7 @@ module Deimos
|
|
49
50
|
|
50
51
|
# Set the namespace for both requests and responses.
|
51
52
|
# @param name [String]
|
53
|
+
# @return [void]
|
52
54
|
def namespace(name)
|
53
55
|
request_namespace(name)
|
54
56
|
response_namespace(name)
|
@@ -56,12 +58,14 @@ module Deimos
|
|
56
58
|
|
57
59
|
# Set the namespace for requests.
|
58
60
|
# @param name [String]
|
61
|
+
# @return [void]
|
59
62
|
def request_namespace(name)
|
60
63
|
namespaces[:request] = name
|
61
64
|
end
|
62
65
|
|
63
66
|
# Set the namespace for repsonses.
|
64
67
|
# @param name [String]
|
68
|
+
# @return [void]
|
65
69
|
def response_namespace(name)
|
66
70
|
namespaces[:response] = name
|
67
71
|
end
|
@@ -94,6 +98,7 @@ module Deimos
|
|
94
98
|
end
|
95
99
|
|
96
100
|
# Decode the payload with the parameters.
|
101
|
+
# @return [void]
|
97
102
|
def decode_schema
|
98
103
|
namespace, schema = parse_namespace(:request)
|
99
104
|
decoder = Deimos.schema_backend(schema: schema, namespace: namespace)
|
@@ -109,6 +114,9 @@ module Deimos
|
|
109
114
|
|
110
115
|
# Render a hash into a payload as specified by the configured schema and namespace.
|
111
116
|
# @param payload [Hash]
|
117
|
+
# @param schema [String]
|
118
|
+
# @param namespace [String]
|
119
|
+
# @return [void]
|
112
120
|
def render_schema(payload, schema: nil, namespace: nil)
|
113
121
|
namespace, schema = parse_namespace(:response) if !schema && !namespace
|
114
122
|
encoder = Deimos.schema_backend(schema: schema, namespace: namespace)
|
data/lib/deimos/version.rb
CHANGED
data/lib/deimos.rb
CHANGED
@@ -45,7 +45,7 @@ require 'erb'
|
|
45
45
|
# Parent module.
|
46
46
|
module Deimos
|
47
47
|
class << self
|
48
|
-
# @return [Class
|
48
|
+
# @return [Class<Deimos::SchemaBackends::Base>]
|
49
49
|
def schema_backend_class
|
50
50
|
backend = Deimos.config.schema.backend.to_s
|
51
51
|
|
@@ -54,7 +54,7 @@ module Deimos
|
|
54
54
|
"Deimos::SchemaBackends::#{backend.classify}".constantize
|
55
55
|
end
|
56
56
|
|
57
|
-
# @param schema [String
|
57
|
+
# @param schema [String, Symbol]
|
58
58
|
# @param namespace [String]
|
59
59
|
# @return [Deimos::SchemaBackends::Base]
|
60
60
|
def schema_backend(schema:, namespace:)
|
@@ -81,6 +81,7 @@ module Deimos
|
|
81
81
|
|
82
82
|
# Start the DB producers to send Kafka messages.
|
83
83
|
# @param thread_count [Integer] the number of threads to start.
|
84
|
+
# @return [void]
|
84
85
|
def start_db_backend!(thread_count: 1)
|
85
86
|
Sigurd.exit_on_signal = true
|
86
87
|
if self.config.producers.backend != :db
|
@@ -69,7 +69,7 @@ module Deimos
|
|
69
69
|
end
|
70
70
|
|
71
71
|
desc 'Generate migration for a table based on an existing schema.'
|
72
|
-
#
|
72
|
+
# @return [void]
|
73
73
|
def generate
|
74
74
|
migration_template('migration.rb', "db/migrate/create_#{table_name.underscore}.rb")
|
75
75
|
template('model.rb', "app/models/#{table_name.underscore.singularize}.rb")
|
@@ -11,11 +11,17 @@ module Deimos
|
|
11
11
|
# Generator for Schema Classes used for the IDE and consumer/producer interfaces
|
12
12
|
class SchemaClassGenerator < Rails::Generators::Base
|
13
13
|
|
14
|
+
# @return [Array<Symbol>]
|
14
15
|
SPECIAL_TYPES = %i(record enum).freeze
|
16
|
+
# @return [String]
|
15
17
|
INITIALIZE_WHITESPACE = "\n#{' ' * 19}"
|
18
|
+
# @return [Array<String>]
|
16
19
|
IGNORE_DEFAULTS = %w(message_id timestamp).freeze
|
20
|
+
# @return [String]
|
17
21
|
SCHEMA_CLASS_FILE = 'schema_class.rb'
|
22
|
+
# @return [String]
|
18
23
|
SCHEMA_RECORD_PATH = File.expand_path('schema_class/templates/schema_record.rb.tt', __dir__).freeze
|
24
|
+
# @return [String]
|
19
25
|
SCHEMA_ENUM_PATH = File.expand_path('schema_class/templates/schema_enum.rb.tt', __dir__).freeze
|
20
26
|
|
21
27
|
source_root File.expand_path('schema_class/templates', __dir__)
|
@@ -42,6 +48,7 @@ module Deimos
|
|
42
48
|
# @param schema_name [String]
|
43
49
|
# @param namespace [String]
|
44
50
|
# @param key_config [Hash,nil]
|
51
|
+
# @return [void]
|
45
52
|
def generate_classes(schema_name, namespace, key_config)
|
46
53
|
schema_base = Deimos.schema_backend(schema: schema_name, namespace: namespace)
|
47
54
|
schema_base.load_schema
|
@@ -84,6 +91,7 @@ module Deimos
|
|
84
91
|
|
85
92
|
# @param schema_base [Deimos::SchemaBackends::Base]
|
86
93
|
# @param key_config [Hash,nil]
|
94
|
+
# @return [void]
|
87
95
|
def generate_class_from_schema_base(schema_base, key_config: nil)
|
88
96
|
@discovered_schemas = Set.new
|
89
97
|
@sub_schema_templates = []
|
@@ -105,7 +113,8 @@ module Deimos
|
|
105
113
|
end
|
106
114
|
|
107
115
|
# @param schema [Avro::Schema::NamedSchema]
|
108
|
-
# @param
|
116
|
+
# @param key_config [Hash,nil]
|
117
|
+
# @return [void]
|
109
118
|
def write_file(schema, key_config)
|
110
119
|
class_template = _generate_class_template_from_schema(schema, key_config)
|
111
120
|
@modules = Utils::SchemaClass.modules_for(schema.namespace)
|
@@ -143,7 +152,7 @@ module Deimos
|
|
143
152
|
end
|
144
153
|
|
145
154
|
desc 'Generate a class based on configured consumer and producers.'
|
146
|
-
#
|
155
|
+
# @return [void]
|
147
156
|
def generate
|
148
157
|
_validate
|
149
158
|
Rails.logger.info("Generating schemas from Deimos.config to #{Deimos.config.schema.generated_class_path}")
|
@@ -175,7 +184,7 @@ module Deimos
|
|
175
184
|
def generate_from_schema_files(found_schemas)
|
176
185
|
schema_store = AvroTurf::MutableSchemaStore.new(path: Deimos.config.schema.path)
|
177
186
|
schema_store.load_schemas!
|
178
|
-
schema_store.schemas.values.each do |schema|
|
187
|
+
schema_store.schemas.values.sort_by { |s| "#{s.namespace}#{s.name}" }.each do |schema|
|
179
188
|
name = "#{schema.namespace}.#{schema.name}"
|
180
189
|
next if found_schemas.include?(name)
|
181
190
|
|
@@ -252,7 +261,7 @@ module Deimos
|
|
252
261
|
"#{result})"
|
253
262
|
end
|
254
263
|
|
255
|
-
# @param [SchemaField]
|
264
|
+
# @param field [SchemaField]
|
256
265
|
# @return [String]
|
257
266
|
def _field_default(field)
|
258
267
|
default = field.default
|
@@ -0,0 +1,176 @@
|
|
1
|
+
---
|
2
|
+
sources:
|
3
|
+
- name: ruby/gem_rbs_collection
|
4
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
5
|
+
revision: main
|
6
|
+
repo_dir: gems
|
7
|
+
path: ".gem_rbs_collection"
|
8
|
+
gems:
|
9
|
+
- name: actionpack
|
10
|
+
version: '6.0'
|
11
|
+
source:
|
12
|
+
type: git
|
13
|
+
name: ruby/gem_rbs_collection
|
14
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
15
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
16
|
+
repo_dir: gems
|
17
|
+
- name: actionview
|
18
|
+
version: '6.0'
|
19
|
+
source:
|
20
|
+
type: git
|
21
|
+
name: ruby/gem_rbs_collection
|
22
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
23
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
24
|
+
repo_dir: gems
|
25
|
+
- name: activejob
|
26
|
+
version: '6.0'
|
27
|
+
source:
|
28
|
+
type: git
|
29
|
+
name: ruby/gem_rbs_collection
|
30
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
31
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
32
|
+
repo_dir: gems
|
33
|
+
- name: activemodel
|
34
|
+
version: '6.0'
|
35
|
+
source:
|
36
|
+
type: git
|
37
|
+
name: ruby/gem_rbs_collection
|
38
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
39
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
40
|
+
repo_dir: gems
|
41
|
+
- name: activerecord
|
42
|
+
version: '6.1'
|
43
|
+
source:
|
44
|
+
type: git
|
45
|
+
name: ruby/gem_rbs_collection
|
46
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
47
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
48
|
+
repo_dir: gems
|
49
|
+
- name: activestorage
|
50
|
+
version: '6.1'
|
51
|
+
source:
|
52
|
+
type: git
|
53
|
+
name: ruby/gem_rbs_collection
|
54
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
55
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
56
|
+
repo_dir: gems
|
57
|
+
- name: activesupport
|
58
|
+
version: '6.0'
|
59
|
+
source:
|
60
|
+
type: git
|
61
|
+
name: ruby/gem_rbs_collection
|
62
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
63
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
64
|
+
repo_dir: gems
|
65
|
+
- name: ast
|
66
|
+
version: '2.4'
|
67
|
+
source:
|
68
|
+
type: git
|
69
|
+
name: ruby/gem_rbs_collection
|
70
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
71
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
72
|
+
repo_dir: gems
|
73
|
+
- name: deimos-ruby
|
74
|
+
version: 1.16.4
|
75
|
+
source:
|
76
|
+
type: rubygems
|
77
|
+
- name: i18n
|
78
|
+
version: '1.10'
|
79
|
+
source:
|
80
|
+
type: git
|
81
|
+
name: ruby/gem_rbs_collection
|
82
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
83
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
84
|
+
repo_dir: gems
|
85
|
+
- name: listen
|
86
|
+
version: '3.2'
|
87
|
+
source:
|
88
|
+
type: git
|
89
|
+
name: ruby/gem_rbs_collection
|
90
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
91
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
92
|
+
repo_dir: gems
|
93
|
+
- name: minitest
|
94
|
+
version: '0'
|
95
|
+
source:
|
96
|
+
type: stdlib
|
97
|
+
- name: nokogiri
|
98
|
+
version: '1.11'
|
99
|
+
source:
|
100
|
+
type: git
|
101
|
+
name: ruby/gem_rbs_collection
|
102
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
103
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
104
|
+
repo_dir: gems
|
105
|
+
- name: parallel
|
106
|
+
version: '1.20'
|
107
|
+
source:
|
108
|
+
type: git
|
109
|
+
name: ruby/gem_rbs_collection
|
110
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
111
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
112
|
+
repo_dir: gems
|
113
|
+
- name: phobos
|
114
|
+
version: 2.1.5
|
115
|
+
source:
|
116
|
+
type: rubygems
|
117
|
+
- name: rack
|
118
|
+
version: 2.2.2
|
119
|
+
source:
|
120
|
+
type: git
|
121
|
+
name: ruby/gem_rbs_collection
|
122
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
123
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
124
|
+
repo_dir: gems
|
125
|
+
- name: railties
|
126
|
+
version: '6.0'
|
127
|
+
source:
|
128
|
+
type: git
|
129
|
+
name: ruby/gem_rbs_collection
|
130
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
131
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
132
|
+
repo_dir: gems
|
133
|
+
- name: rainbow
|
134
|
+
version: '3.0'
|
135
|
+
source:
|
136
|
+
type: git
|
137
|
+
name: ruby/gem_rbs_collection
|
138
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
139
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
140
|
+
repo_dir: gems
|
141
|
+
- name: tempfile
|
142
|
+
version: '0'
|
143
|
+
source:
|
144
|
+
type: stdlib
|
145
|
+
- name: monitor
|
146
|
+
version: '0'
|
147
|
+
source:
|
148
|
+
type: stdlib
|
149
|
+
- name: date
|
150
|
+
version: '0'
|
151
|
+
source:
|
152
|
+
type: stdlib
|
153
|
+
- name: singleton
|
154
|
+
version: '0'
|
155
|
+
source:
|
156
|
+
type: stdlib
|
157
|
+
- name: logger
|
158
|
+
version: '0'
|
159
|
+
source:
|
160
|
+
type: stdlib
|
161
|
+
- name: mutex_m
|
162
|
+
version: '0'
|
163
|
+
source:
|
164
|
+
type: stdlib
|
165
|
+
- name: time
|
166
|
+
version: '0'
|
167
|
+
source:
|
168
|
+
type: stdlib
|
169
|
+
- name: pathname
|
170
|
+
version: '0'
|
171
|
+
source:
|
172
|
+
type: stdlib
|
173
|
+
- name: tsort
|
174
|
+
version: '0'
|
175
|
+
source:
|
176
|
+
type: stdlib
|
data/rbs_collection.yaml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
# Download sources
|
2
|
+
sources:
|
3
|
+
- name: ruby/gem_rbs_collection
|
4
|
+
remote: https://github.com/ruby/gem_rbs_collection.git
|
5
|
+
revision: main
|
6
|
+
repo_dir: gems
|
7
|
+
|
8
|
+
# A directory to install the downloaded RBSs
|
9
|
+
path: .gem_rbs_collection
|
10
|
+
|
11
|
+
gems:
|
12
|
+
# Skip loading rbs gem's RBS.
|
13
|
+
# It's unnecessary if you don't use rbs as a library.
|
14
|
+
- name: rbs
|
15
|
+
ignore: true
|