deimos-ruby 1.14.3 → 1.14.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +13 -0
- data/README.md +1 -1
- data/docs/CONFIGURATION.md +2 -0
- data/lib/deimos/active_record_consume/batch_consumption.rb +4 -3
- data/lib/deimos/config/configuration.rb +4 -0
- data/lib/deimos/consume/batch_consumption.rb +12 -12
- data/lib/deimos/version.rb +1 -1
- data/lib/generators/deimos/schema_class/templates/schema_class.rb.tt +2 -2
- data/lib/generators/deimos/schema_class_generator.rb +28 -1
- data/spec/generators/schema_class_generator_spec.rb +10 -2
- data/spec/schemas/com/my-namespace/MySchemaId_key.avsc +1 -1
- data/spec/snapshots/consumers-no-nest.snap +1150 -0
- data/spec/snapshots/consumers.snap +1177 -0
- data/spec/snapshots/consumers_and_producers-no-nest.snap +941 -0
- data/spec/snapshots/consumers_and_producers.snap +974 -0
- data/spec/snapshots/consumers_circular-no-nest.snap +1255 -15
- data/spec/snapshots/consumers_circular.snap +1280 -21
- data/spec/snapshots/consumers_complex_types-no-nest.snap +1138 -84
- data/spec/snapshots/consumers_complex_types.snap +1208 -111
- data/spec/snapshots/consumers_nested-no-nest.snap +1200 -0
- data/spec/snapshots/consumers_nested.snap +1219 -0
- data/spec/snapshots/namespace_folders.snap +1429 -0
- data/spec/snapshots/producers_with_key-no-nest.snap +1250 -7
- data/spec/snapshots/producers_with_key.snap +1256 -0
- metadata +5 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a0c3d1a3e60a5e71ea4fd21de8b8f2ed9db064f8ea4f26891169f5e7690ad75c
|
4
|
+
data.tar.gz: afe45152f6834e529d1d9e63b5fd6e2f307d072464224f9e29d8d098d7053ca2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b1359ca3c178171f7999edb2dd43385a04a119489bfc4efa4f5811462c23203b88a1ea5661727718a6142dfc725137592166bef5df36d22fc1a4a6a7ff4a6bdc
|
7
|
+
data.tar.gz: 7629b9f1e75dad6ef0862958e448e7b750da434e62d0be754def299919a49f6ac3d0937588bf004c08ef331ea66aba9dcad989906781465c4a4c51c801acca51
|
data/CHANGELOG.md
CHANGED
@@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
7
7
|
|
8
8
|
## UNRELEASED
|
9
9
|
|
10
|
+
# 1.14.6 - 2022-06-21
|
11
|
+
|
12
|
+
- Add `generate_namespace_folders` to configuration; this will automatically generate subfolders to the `schemas` folder so that you can have different schemas with the same name but different namespaces generate separate classes.
|
13
|
+
|
14
|
+
# 1.14.5 - 2022-06-21
|
15
|
+
|
16
|
+
- Fix crash with the tracer when error happens in decoding a message during batch consuming
|
17
|
+
- Generate schema classes for all schemas, even without a consumer/producer set
|
18
|
+
|
19
|
+
# 1.14.4 - 2022-06-18
|
20
|
+
|
21
|
+
- Fix import in ActiveRecordConsumer on mysql
|
22
|
+
|
10
23
|
# 1.14.3 - 2022-06-17
|
11
24
|
|
12
25
|
- Fix issue with ActiveRecordConsumer double-decoding keys in batch mode.
|
data/README.md
CHANGED
@@ -818,7 +818,7 @@ Add the following configurations to start using generated schema classes in your
|
|
818
818
|
Additionally, you can enable or disable the usage of schema classes for a particular consumer or producer with the
|
819
819
|
`use_schema_classes` config. See [Configuration](./docs/CONFIGURATION.md#defining-producers).
|
820
820
|
|
821
|
-
Note that if you have a schema in your repo but have not configured a producer or consumer, the generator will
|
821
|
+
Note that if you have a schema in your repo but have not configured a producer or consumer, the generator will generate a schema class without a key schema.
|
822
822
|
|
823
823
|
One additional configuration option indicates whether nested records should be generated as top-level classes or would remain nested inside the generated class for its parent schema. The default is to nest them, as a flattened structure can have one sub-schema clobber another sub-schema defined in a different top-level schema.
|
824
824
|
|
data/docs/CONFIGURATION.md
CHANGED
@@ -194,6 +194,8 @@ schema.password|nil|Basic auth password.
|
|
194
194
|
schema.path|nil|Local path to find your schemas.
|
195
195
|
schema.use_schema_classes|false|Set this to true to use generated schema classes in your application.
|
196
196
|
schema.generated_class_path|`app/lib/schema_classes`|Local path to generated schema classes.
|
197
|
+
schema.nest_child_schemas|false|Set to true to nest subschemas within the generated class for the parent schema.
|
198
|
+
schema.generate_namespace_folders|false|Set to true to generate folders for schemas matching the last part of the namespace.
|
197
199
|
|
198
200
|
## Database Producer Configuration
|
199
201
|
|
@@ -100,12 +100,13 @@ module Deimos
|
|
100
100
|
|
101
101
|
options = if key_cols.empty?
|
102
102
|
{} # Can't upsert with no key, just do regular insert
|
103
|
+
elsif ActiveRecord::Base.connection.adapter_name.downcase =~ /mysql/
|
104
|
+
{
|
105
|
+
on_duplicate_key_update: :all
|
106
|
+
}
|
103
107
|
else
|
104
108
|
{
|
105
109
|
on_duplicate_key_update: {
|
106
|
-
# conflict_target must explicitly list the columns for
|
107
|
-
# Postgres and SQLite. Not required for MySQL, but this
|
108
|
-
# ensures consistent behaviour.
|
109
110
|
conflict_target: key_cols,
|
110
111
|
columns: :all
|
111
112
|
}
|
@@ -347,6 +347,10 @@ module Deimos
|
|
347
347
|
# Set to false to generate child schemas as their own files.
|
348
348
|
# @return [Boolean]
|
349
349
|
setting :nest_child_schemas, true
|
350
|
+
|
351
|
+
# Set to true to generate folders matching the last part of the schema namespace.
|
352
|
+
# @return [Boolean]
|
353
|
+
setting :generate_namespace_folders, false
|
350
354
|
end
|
351
355
|
|
352
356
|
# The configured metrics provider.
|
@@ -12,23 +12,23 @@ module Deimos
|
|
12
12
|
# :nodoc:
|
13
13
|
def around_consume_batch(batch, metadata)
|
14
14
|
payloads = []
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
15
|
+
_with_span do
|
16
|
+
benchmark = Benchmark.measure do
|
17
|
+
if self.class.config[:key_configured]
|
18
|
+
metadata[:keys] = batch.map do |message|
|
19
|
+
decode_key(message.key)
|
20
|
+
end
|
19
21
|
end
|
20
|
-
|
21
|
-
metadata[:first_offset] = batch.first&.offset
|
22
|
+
metadata[:first_offset] = batch.first&.offset
|
22
23
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
_with_span do
|
24
|
+
payloads = batch.map do |message|
|
25
|
+
decode_message(message.payload)
|
26
|
+
end
|
27
|
+
_received_batch(payloads, metadata)
|
28
28
|
yield(payloads, metadata)
|
29
29
|
end
|
30
|
+
_handle_batch_success(benchmark.real, payloads, metadata)
|
30
31
|
end
|
31
|
-
_handle_batch_success(benchmark.real, payloads, metadata)
|
32
32
|
rescue StandardError => e
|
33
33
|
_handle_batch_error(e, payloads, metadata)
|
34
34
|
end
|
data/lib/deimos/version.rb
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
# This file is autogenerated by Deimos, Do NOT modify
|
4
|
-
module
|
4
|
+
<%= @modules.map { |m| "module #{m}"}.join('; ') %>
|
5
5
|
### Primary Schema Class ###
|
6
6
|
<%=- @main_class_definition -%>
|
7
7
|
|
8
|
-
end
|
8
|
+
<%= @modules.map { "end" }.join('; ') %>
|
@@ -41,7 +41,7 @@ module Deimos
|
|
41
41
|
# Deimos Consumer or Producer Configuration object
|
42
42
|
# @param schema_name [String]
|
43
43
|
# @param namespace [String]
|
44
|
-
# @param key_schema_name [String]
|
44
|
+
# @param key_schema_name [String,nil]
|
45
45
|
def generate_classes(schema_name, namespace, key_schema_name)
|
46
46
|
schema_base = Deimos.schema_backend(schema: schema_name, namespace: namespace)
|
47
47
|
schema_base.load_schema
|
@@ -108,9 +108,17 @@ module Deimos
|
|
108
108
|
# @param key_schema_base [Avro::Schema::NamedSchema, nil]
|
109
109
|
def write_file(schema, key_schema_base)
|
110
110
|
class_template = _generate_class_template_from_schema(schema, key_schema_base)
|
111
|
+
@modules = ['Schemas']
|
112
|
+
namespace_folder = schema.namespace.split('.').last
|
113
|
+
if Deimos.config.schema.generate_namespace_folders
|
114
|
+
@modules.push(namespace_folder.underscore.classify)
|
115
|
+
end
|
111
116
|
@main_class_definition = class_template
|
112
117
|
|
113
118
|
file_prefix = schema.name.underscore.singularize
|
119
|
+
if Deimos.config.schema.generate_namespace_folders
|
120
|
+
file_prefix = "#{namespace_folder}/#{file_prefix}"
|
121
|
+
end
|
114
122
|
filename = "#{Deimos.config.schema.generated_class_path}/#{file_prefix}.rb"
|
115
123
|
template(SCHEMA_CLASS_FILE, filename, force: true)
|
116
124
|
end
|
@@ -143,10 +151,13 @@ module Deimos
|
|
143
151
|
def generate
|
144
152
|
_validate
|
145
153
|
Rails.logger.info("Generating schemas from Deimos.config to #{Deimos.config.schema.generated_class_path}")
|
154
|
+
found_schemas = Set.new
|
146
155
|
Deimos.config.producer_objects.each do |config|
|
147
156
|
schema_name = config.schema
|
148
157
|
namespace = config.namespace || Deimos.config.producers.schema_namespace
|
149
158
|
key_schema_name = config.key_config[:schema]
|
159
|
+
found_schemas.add("#{namespace}.#{schema_name}")
|
160
|
+
found_schemas.add("#{namespace}.#{key_schema_name}") if key_schema_name
|
150
161
|
generate_classes(schema_name, namespace, key_schema_name)
|
151
162
|
end
|
152
163
|
|
@@ -154,12 +165,28 @@ module Deimos
|
|
154
165
|
schema_name = config.schema
|
155
166
|
namespace = config.namespace
|
156
167
|
key_schema_name = config.key_config[:schema]
|
168
|
+
found_schemas.add("#{namespace}.#{schema_name}")
|
169
|
+
found_schemas.add("#{namespace}.#{key_schema_name}") if key_schema_name
|
157
170
|
generate_classes(schema_name, namespace, key_schema_name)
|
158
171
|
end
|
172
|
+
|
173
|
+
generate_from_schema_files(found_schemas)
|
174
|
+
|
159
175
|
end
|
160
176
|
|
161
177
|
private
|
162
178
|
|
179
|
+
def generate_from_schema_files(found_schemas)
|
180
|
+
schema_store = AvroTurf::MutableSchemaStore.new(path: Deimos.config.schema.path)
|
181
|
+
schema_store.load_schemas!
|
182
|
+
schema_store.schemas.values.each do |schema|
|
183
|
+
name = "#{schema.namespace}.#{schema.name}"
|
184
|
+
next if found_schemas.include?(name)
|
185
|
+
|
186
|
+
generate_classes(schema.name, schema.namespace, nil)
|
187
|
+
end
|
188
|
+
end
|
189
|
+
|
163
190
|
# Determines if Schema Class Generation can be run.
|
164
191
|
# @raise if Schema Backend is not of a Avro-based class
|
165
192
|
def _validate
|
@@ -11,7 +11,7 @@ end
|
|
11
11
|
|
12
12
|
RSpec.describe Deimos::Generators::SchemaClassGenerator do
|
13
13
|
let(:schema_class_path) { 'spec/app/lib/schema_classes' }
|
14
|
-
let(:files) { Dir["#{schema_class_path}
|
14
|
+
let(:files) { Dir["#{schema_class_path}/**/*.rb"].map { |f| [f, File.read(f)]}.to_h }
|
15
15
|
|
16
16
|
before(:each) do
|
17
17
|
Deimos.config.reset!
|
@@ -56,7 +56,6 @@ RSpec.describe Deimos::Generators::SchemaClassGenerator do
|
|
56
56
|
end
|
57
57
|
end
|
58
58
|
end
|
59
|
-
|
60
59
|
end
|
61
60
|
|
62
61
|
context 'with a Consumers Schema with Complex types' do
|
@@ -216,6 +215,15 @@ RSpec.describe Deimos::Generators::SchemaClassGenerator do
|
|
216
215
|
end
|
217
216
|
end
|
218
217
|
|
218
|
+
context 'with namespace folders' do
|
219
|
+
it 'should generate the correct classes' do
|
220
|
+
Deimos.with_config('schema.generate_namespace_folders' => true) do
|
221
|
+
described_class.start
|
222
|
+
expect(files).to match_snapshot('namespace_folders', snapshot_serializer: MultiFileSerializer)
|
223
|
+
end
|
224
|
+
end
|
225
|
+
end
|
226
|
+
|
219
227
|
context 'nested true' do
|
220
228
|
it 'should generate the correct classes' do
|
221
229
|
Deimos.with_config('schema.nest_child_schemas' => true) do
|