deimos-ruby 1.14.2 → 1.14.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 7052c11e5277eb8b2844e56f13c08921d1be7e3da28db5307b2b64e17b8b918a
4
- data.tar.gz: 02fee15de11ebe50518ae96c91649d9d2341613967462348b4cfcebde3f8218d
3
+ metadata.gz: 872f449652ea05dfe9aabf1b144391b98d63fa03b193c7160931748b514ff534
4
+ data.tar.gz: cf6e3b54b909d1731a06d68fbfa1f7dbad5d78ce4e7de01bf712161061515ceb
5
5
  SHA512:
6
- metadata.gz: 2f9f38797fbd74e5958f3910d4886144ca4ae329cbadaa5cb6474acdb4eeefdfd6edac59d79f3a1081df4fd24ae5766329b0d00649b613c13c44204a85e3b6b7
7
- data.tar.gz: c5dd68694b23e7cdb10a2a2b6fe1163fe6f84f4c3b9de64eaea61c6b50f3ffc08642d72764e5550cf19f56bb14da7913611f45f6cd48a132e4707d5b6a201816
6
+ metadata.gz: e2737082fbada9724a9e94615308e4f7cdb17a36b8202dd1bf1f83c46e6378f5a6724eeb9e4cae5b86beabff94246ae90619b4b8e55f06064adf8dcd42e2ef14
7
+ data.tar.gz: 70b73b45dcb4b881bfa2c50aae105af62caf2d4aad2b103a807c1d776477c49621650434ae5aba9358b7cb1f977fe246e831f655bc7bfde902417bcf9a0b9ce5
data/CHANGELOG.md CHANGED
@@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
7
7
 
8
8
  ## UNRELEASED
9
9
 
10
+ # 1.14.5 - 2022-06-21
11
+
12
+ - Fix crash with the tracer when error happens in decoding a message during batch consuming
13
+ - Generate schema classes for all schemas, even without a consumer/producer set
14
+
15
+ # 1.14.4 - 2022-06-18
16
+
17
+ - Fix import in ActiveRecordConsumer on mysql
18
+
19
+ # 1.14.3 - 2022-06-17
20
+
21
+ - Fix issue with ActiveRecordConsumer double-decoding keys in batch mode.
22
+
10
23
  # 1.14.2 - 2022-05-26
11
24
 
12
25
  - Fix crash with `test_consume_message` when passing in an instance of a schema class instead of a hash.
data/README.md CHANGED
@@ -818,7 +818,7 @@ Add the following configurations to start using generated schema classes in your
818
818
  Additionally, you can enable or disable the usage of schema classes for a particular consumer or producer with the
819
819
  `use_schema_classes` config. See [Configuration](./docs/CONFIGURATION.md#defining-producers).
820
820
 
821
- Note that if you have a schema in your repo but have not configured a producer or consumer, the generator will **not** generate a schema class for it. This is because you need to supply the `key_config` for the generator to know how to handle the schema key.
821
+ Note that if you have a schema in your repo but have not configured a producer or consumer, the generator will generate a schema class without a key schema.
822
822
 
823
823
  One additional configuration option indicates whether nested records should be generated as top-level classes or would remain nested inside the generated class for its parent schema. The default is to nest them, as a flattened structure can have one sub-schema clobber another sub-schema defined in a different top-level schema.
824
824
 
@@ -44,14 +44,12 @@ module Deimos
44
44
  # @param key [String] The encoded key.
45
45
  # @return [Hash] The key attributes.
46
46
  def record_key(key)
47
- decoded_key = decode_key(key)
48
-
49
- if decoded_key.nil?
47
+ if key.nil?
50
48
  {}
51
- elsif decoded_key.is_a?(Hash)
52
- @key_converter.convert(decoded_key)
49
+ elsif key.is_a?(Hash)
50
+ @key_converter.convert(key)
53
51
  else
54
- { @klass.primary_key => decoded_key }
52
+ { @klass.primary_key => key }
55
53
  end
56
54
  end
57
55
 
@@ -102,12 +100,13 @@ module Deimos
102
100
 
103
101
  options = if key_cols.empty?
104
102
  {} # Can't upsert with no key, just do regular insert
103
+ elsif ActiveRecord::Base.connection.adapter_name.downcase =~ /mysql/
104
+ {
105
+ on_duplicate_key_update: :all
106
+ }
105
107
  else
106
108
  {
107
109
  on_duplicate_key_update: {
108
- # conflict_target must explicitly list the columns for
109
- # Postgres and SQLite. Not required for MySQL, but this
110
- # ensures consistent behaviour.
111
110
  conflict_target: key_cols,
112
111
  columns: :all
113
112
  }
@@ -12,23 +12,23 @@ module Deimos
12
12
  # :nodoc:
13
13
  def around_consume_batch(batch, metadata)
14
14
  payloads = []
15
- benchmark = Benchmark.measure do
16
- if self.class.config[:key_configured]
17
- metadata[:keys] = batch.map do |message|
18
- decode_key(message.key)
15
+ _with_span do
16
+ benchmark = Benchmark.measure do
17
+ if self.class.config[:key_configured]
18
+ metadata[:keys] = batch.map do |message|
19
+ decode_key(message.key)
20
+ end
19
21
  end
20
- end
21
- metadata[:first_offset] = batch.first&.offset
22
+ metadata[:first_offset] = batch.first&.offset
22
23
 
23
- payloads = batch.map do |message|
24
- decode_message(message.payload)
25
- end
26
- _received_batch(payloads, metadata)
27
- _with_span do
24
+ payloads = batch.map do |message|
25
+ decode_message(message.payload)
26
+ end
27
+ _received_batch(payloads, metadata)
28
28
  yield(payloads, metadata)
29
29
  end
30
+ _handle_batch_success(benchmark.real, payloads, metadata)
30
31
  end
31
- _handle_batch_success(benchmark.real, payloads, metadata)
32
32
  rescue StandardError => e
33
33
  _handle_batch_error(e, payloads, metadata)
34
34
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Deimos
4
- VERSION = '1.14.2'
4
+ VERSION = '1.14.5'
5
5
  end
@@ -41,7 +41,7 @@ module Deimos
41
41
  # Deimos Consumer or Producer Configuration object
42
42
  # @param schema_name [String]
43
43
  # @param namespace [String]
44
- # @param key_schema_name [String]
44
+ # @param key_schema_name [String,nil]
45
45
  def generate_classes(schema_name, namespace, key_schema_name)
46
46
  schema_base = Deimos.schema_backend(schema: schema_name, namespace: namespace)
47
47
  schema_base.load_schema
@@ -143,10 +143,13 @@ module Deimos
143
143
  def generate
144
144
  _validate
145
145
  Rails.logger.info("Generating schemas from Deimos.config to #{Deimos.config.schema.generated_class_path}")
146
+ found_schemas = Set.new
146
147
  Deimos.config.producer_objects.each do |config|
147
148
  schema_name = config.schema
148
149
  namespace = config.namespace || Deimos.config.producers.schema_namespace
149
150
  key_schema_name = config.key_config[:schema]
151
+ found_schemas.add("#{namespace}.#{schema_name}")
152
+ found_schemas.add("#{namespace}.#{key_schema_name}") if key_schema_name
150
153
  generate_classes(schema_name, namespace, key_schema_name)
151
154
  end
152
155
 
@@ -154,12 +157,28 @@ module Deimos
154
157
  schema_name = config.schema
155
158
  namespace = config.namespace
156
159
  key_schema_name = config.key_config[:schema]
160
+ found_schemas.add("#{namespace}.#{schema_name}")
161
+ found_schemas.add("#{namespace}.#{key_schema_name}") if key_schema_name
157
162
  generate_classes(schema_name, namespace, key_schema_name)
158
163
  end
164
+
165
+ generate_from_schema_files(found_schemas)
166
+
159
167
  end
160
168
 
161
169
  private
162
170
 
171
+ def generate_from_schema_files(found_schemas)
172
+ schema_store = AvroTurf::MutableSchemaStore.new(path: Deimos.config.schema.path)
173
+ schema_store.load_schemas!
174
+ schema_store.schemas.values.each do |schema|
175
+ name = "#{schema.namespace}.#{schema.name}"
176
+ next if found_schemas.include?(name)
177
+
178
+ generate_classes(schema.name, schema.namespace, nil)
179
+ end
180
+ end
181
+
163
182
  # Determines if Schema Class Generation can be run.
164
183
  # @raise if Schema Backend is not of a Avro-based class
165
184
  def _validate
@@ -56,7 +56,6 @@ RSpec.describe Deimos::Generators::SchemaClassGenerator do
56
56
  end
57
57
  end
58
58
  end
59
-
60
59
  end
61
60
 
62
61
  context 'with a Consumers Schema with Complex types' do
@@ -6,7 +6,7 @@
6
6
  "fields": [
7
7
  {
8
8
  "name": "id",
9
- "type": "integer"
9
+ "type": "int"
10
10
  }
11
11
  ]
12
12
  }