deimos-ruby 1.14.2 → 1.14.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +13 -0
- data/README.md +1 -1
- data/lib/deimos/active_record_consume/batch_consumption.rb +8 -9
- data/lib/deimos/consume/batch_consumption.rb +12 -12
- data/lib/deimos/version.rb +1 -1
- data/lib/generators/deimos/schema_class_generator.rb +20 -1
- data/spec/generators/schema_class_generator_spec.rb +0 -1
- data/spec/schemas/com/my-namespace/MySchemaId_key.avsc +1 -1
- data/spec/snapshots/consumers-no-nest.snap +1150 -0
- data/spec/snapshots/consumers.snap +1177 -0
- data/spec/snapshots/consumers_and_producers-no-nest.snap +941 -0
- data/spec/snapshots/consumers_and_producers.snap +974 -0
- data/spec/snapshots/consumers_circular-no-nest.snap +1255 -15
- data/spec/snapshots/consumers_circular.snap +1280 -21
- data/spec/snapshots/consumers_complex_types-no-nest.snap +1138 -84
- data/spec/snapshots/consumers_complex_types.snap +1208 -111
- data/spec/snapshots/consumers_nested-no-nest.snap +1200 -0
- data/spec/snapshots/consumers_nested.snap +1219 -0
- data/spec/snapshots/producers_with_key-no-nest.snap +1250 -7
- data/spec/snapshots/producers_with_key.snap +1256 -0
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 872f449652ea05dfe9aabf1b144391b98d63fa03b193c7160931748b514ff534
|
4
|
+
data.tar.gz: cf6e3b54b909d1731a06d68fbfa1f7dbad5d78ce4e7de01bf712161061515ceb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e2737082fbada9724a9e94615308e4f7cdb17a36b8202dd1bf1f83c46e6378f5a6724eeb9e4cae5b86beabff94246ae90619b4b8e55f06064adf8dcd42e2ef14
|
7
|
+
data.tar.gz: 70b73b45dcb4b881bfa2c50aae105af62caf2d4aad2b103a807c1d776477c49621650434ae5aba9358b7cb1f977fe246e831f655bc7bfde902417bcf9a0b9ce5
|
data/CHANGELOG.md
CHANGED
@@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
7
7
|
|
8
8
|
## UNRELEASED
|
9
9
|
|
10
|
+
# 1.14.5 - 2022-06-21
|
11
|
+
|
12
|
+
- Fix crash with the tracer when error happens in decoding a message during batch consuming
|
13
|
+
- Generate schema classes for all schemas, even without a consumer/producer set
|
14
|
+
|
15
|
+
# 1.14.4 - 2022-06-18
|
16
|
+
|
17
|
+
- Fix import in ActiveRecordConsumer on mysql
|
18
|
+
|
19
|
+
# 1.14.3 - 2022-06-17
|
20
|
+
|
21
|
+
- Fix issue with ActiveRecordConsumer double-decoding keys in batch mode.
|
22
|
+
|
10
23
|
# 1.14.2 - 2022-05-26
|
11
24
|
|
12
25
|
- Fix crash with `test_consume_message` when passing in an instance of a schema class instead of a hash.
|
data/README.md
CHANGED
@@ -818,7 +818,7 @@ Add the following configurations to start using generated schema classes in your
|
|
818
818
|
Additionally, you can enable or disable the usage of schema classes for a particular consumer or producer with the
|
819
819
|
`use_schema_classes` config. See [Configuration](./docs/CONFIGURATION.md#defining-producers).
|
820
820
|
|
821
|
-
Note that if you have a schema in your repo but have not configured a producer or consumer, the generator will
|
821
|
+
Note that if you have a schema in your repo but have not configured a producer or consumer, the generator will generate a schema class without a key schema.
|
822
822
|
|
823
823
|
One additional configuration option indicates whether nested records should be generated as top-level classes or would remain nested inside the generated class for its parent schema. The default is to nest them, as a flattened structure can have one sub-schema clobber another sub-schema defined in a different top-level schema.
|
824
824
|
|
@@ -44,14 +44,12 @@ module Deimos
|
|
44
44
|
# @param key [String] The encoded key.
|
45
45
|
# @return [Hash] The key attributes.
|
46
46
|
def record_key(key)
|
47
|
-
|
48
|
-
|
49
|
-
if decoded_key.nil?
|
47
|
+
if key.nil?
|
50
48
|
{}
|
51
|
-
elsif
|
52
|
-
@key_converter.convert(
|
49
|
+
elsif key.is_a?(Hash)
|
50
|
+
@key_converter.convert(key)
|
53
51
|
else
|
54
|
-
{ @klass.primary_key =>
|
52
|
+
{ @klass.primary_key => key }
|
55
53
|
end
|
56
54
|
end
|
57
55
|
|
@@ -102,12 +100,13 @@ module Deimos
|
|
102
100
|
|
103
101
|
options = if key_cols.empty?
|
104
102
|
{} # Can't upsert with no key, just do regular insert
|
103
|
+
elsif ActiveRecord::Base.connection.adapter_name.downcase =~ /mysql/
|
104
|
+
{
|
105
|
+
on_duplicate_key_update: :all
|
106
|
+
}
|
105
107
|
else
|
106
108
|
{
|
107
109
|
on_duplicate_key_update: {
|
108
|
-
# conflict_target must explicitly list the columns for
|
109
|
-
# Postgres and SQLite. Not required for MySQL, but this
|
110
|
-
# ensures consistent behaviour.
|
111
110
|
conflict_target: key_cols,
|
112
111
|
columns: :all
|
113
112
|
}
|
@@ -12,23 +12,23 @@ module Deimos
|
|
12
12
|
# :nodoc:
|
13
13
|
def around_consume_batch(batch, metadata)
|
14
14
|
payloads = []
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
15
|
+
_with_span do
|
16
|
+
benchmark = Benchmark.measure do
|
17
|
+
if self.class.config[:key_configured]
|
18
|
+
metadata[:keys] = batch.map do |message|
|
19
|
+
decode_key(message.key)
|
20
|
+
end
|
19
21
|
end
|
20
|
-
|
21
|
-
metadata[:first_offset] = batch.first&.offset
|
22
|
+
metadata[:first_offset] = batch.first&.offset
|
22
23
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
_with_span do
|
24
|
+
payloads = batch.map do |message|
|
25
|
+
decode_message(message.payload)
|
26
|
+
end
|
27
|
+
_received_batch(payloads, metadata)
|
28
28
|
yield(payloads, metadata)
|
29
29
|
end
|
30
|
+
_handle_batch_success(benchmark.real, payloads, metadata)
|
30
31
|
end
|
31
|
-
_handle_batch_success(benchmark.real, payloads, metadata)
|
32
32
|
rescue StandardError => e
|
33
33
|
_handle_batch_error(e, payloads, metadata)
|
34
34
|
end
|
data/lib/deimos/version.rb
CHANGED
@@ -41,7 +41,7 @@ module Deimos
|
|
41
41
|
# Deimos Consumer or Producer Configuration object
|
42
42
|
# @param schema_name [String]
|
43
43
|
# @param namespace [String]
|
44
|
-
# @param key_schema_name [String]
|
44
|
+
# @param key_schema_name [String,nil]
|
45
45
|
def generate_classes(schema_name, namespace, key_schema_name)
|
46
46
|
schema_base = Deimos.schema_backend(schema: schema_name, namespace: namespace)
|
47
47
|
schema_base.load_schema
|
@@ -143,10 +143,13 @@ module Deimos
|
|
143
143
|
def generate
|
144
144
|
_validate
|
145
145
|
Rails.logger.info("Generating schemas from Deimos.config to #{Deimos.config.schema.generated_class_path}")
|
146
|
+
found_schemas = Set.new
|
146
147
|
Deimos.config.producer_objects.each do |config|
|
147
148
|
schema_name = config.schema
|
148
149
|
namespace = config.namespace || Deimos.config.producers.schema_namespace
|
149
150
|
key_schema_name = config.key_config[:schema]
|
151
|
+
found_schemas.add("#{namespace}.#{schema_name}")
|
152
|
+
found_schemas.add("#{namespace}.#{key_schema_name}") if key_schema_name
|
150
153
|
generate_classes(schema_name, namespace, key_schema_name)
|
151
154
|
end
|
152
155
|
|
@@ -154,12 +157,28 @@ module Deimos
|
|
154
157
|
schema_name = config.schema
|
155
158
|
namespace = config.namespace
|
156
159
|
key_schema_name = config.key_config[:schema]
|
160
|
+
found_schemas.add("#{namespace}.#{schema_name}")
|
161
|
+
found_schemas.add("#{namespace}.#{key_schema_name}") if key_schema_name
|
157
162
|
generate_classes(schema_name, namespace, key_schema_name)
|
158
163
|
end
|
164
|
+
|
165
|
+
generate_from_schema_files(found_schemas)
|
166
|
+
|
159
167
|
end
|
160
168
|
|
161
169
|
private
|
162
170
|
|
171
|
+
def generate_from_schema_files(found_schemas)
|
172
|
+
schema_store = AvroTurf::MutableSchemaStore.new(path: Deimos.config.schema.path)
|
173
|
+
schema_store.load_schemas!
|
174
|
+
schema_store.schemas.values.each do |schema|
|
175
|
+
name = "#{schema.namespace}.#{schema.name}"
|
176
|
+
next if found_schemas.include?(name)
|
177
|
+
|
178
|
+
generate_classes(schema.name, schema.namespace, nil)
|
179
|
+
end
|
180
|
+
end
|
181
|
+
|
163
182
|
# Determines if Schema Class Generation can be run.
|
164
183
|
# @raise if Schema Backend is not of a Avro-based class
|
165
184
|
def _validate
|