deimos-ruby 1.0.0.pre.beta22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. checksums.yaml +7 -0
  2. data/.circleci/config.yml +74 -0
  3. data/.gitignore +41 -0
  4. data/.gitmodules +0 -0
  5. data/.rspec +1 -0
  6. data/.rubocop.yml +321 -0
  7. data/.ruby-gemset +1 -0
  8. data/.ruby-version +1 -0
  9. data/CHANGELOG.md +32 -0
  10. data/CODE_OF_CONDUCT.md +77 -0
  11. data/Dockerfile +23 -0
  12. data/Gemfile +6 -0
  13. data/Gemfile.lock +165 -0
  14. data/Guardfile +22 -0
  15. data/LICENSE.md +195 -0
  16. data/README.md +752 -0
  17. data/Rakefile +13 -0
  18. data/bin/deimos +4 -0
  19. data/deimos-kafka.gemspec +42 -0
  20. data/docker-compose.yml +71 -0
  21. data/docs/DATABASE_BACKEND.md +147 -0
  22. data/docs/PULL_REQUEST_TEMPLATE.md +34 -0
  23. data/lib/deimos/active_record_consumer.rb +81 -0
  24. data/lib/deimos/active_record_producer.rb +64 -0
  25. data/lib/deimos/avro_data_coder.rb +89 -0
  26. data/lib/deimos/avro_data_decoder.rb +36 -0
  27. data/lib/deimos/avro_data_encoder.rb +51 -0
  28. data/lib/deimos/backends/db.rb +27 -0
  29. data/lib/deimos/backends/kafka.rb +27 -0
  30. data/lib/deimos/backends/kafka_async.rb +27 -0
  31. data/lib/deimos/configuration.rb +90 -0
  32. data/lib/deimos/consumer.rb +164 -0
  33. data/lib/deimos/instrumentation.rb +71 -0
  34. data/lib/deimos/kafka_message.rb +27 -0
  35. data/lib/deimos/kafka_source.rb +126 -0
  36. data/lib/deimos/kafka_topic_info.rb +86 -0
  37. data/lib/deimos/message.rb +74 -0
  38. data/lib/deimos/metrics/datadog.rb +47 -0
  39. data/lib/deimos/metrics/mock.rb +39 -0
  40. data/lib/deimos/metrics/provider.rb +38 -0
  41. data/lib/deimos/monkey_patches/phobos_cli.rb +35 -0
  42. data/lib/deimos/monkey_patches/phobos_producer.rb +51 -0
  43. data/lib/deimos/monkey_patches/ruby_kafka_heartbeat.rb +85 -0
  44. data/lib/deimos/monkey_patches/schema_store.rb +19 -0
  45. data/lib/deimos/producer.rb +218 -0
  46. data/lib/deimos/publish_backend.rb +30 -0
  47. data/lib/deimos/railtie.rb +8 -0
  48. data/lib/deimos/schema_coercer.rb +108 -0
  49. data/lib/deimos/shared_config.rb +59 -0
  50. data/lib/deimos/test_helpers.rb +356 -0
  51. data/lib/deimos/tracing/datadog.rb +35 -0
  52. data/lib/deimos/tracing/mock.rb +40 -0
  53. data/lib/deimos/tracing/provider.rb +31 -0
  54. data/lib/deimos/utils/db_producer.rb +122 -0
  55. data/lib/deimos/utils/executor.rb +117 -0
  56. data/lib/deimos/utils/inline_consumer.rb +144 -0
  57. data/lib/deimos/utils/lag_reporter.rb +182 -0
  58. data/lib/deimos/utils/platform_schema_validation.rb +0 -0
  59. data/lib/deimos/utils/signal_handler.rb +68 -0
  60. data/lib/deimos/version.rb +5 -0
  61. data/lib/deimos.rb +133 -0
  62. data/lib/generators/deimos/db_backend/templates/migration +24 -0
  63. data/lib/generators/deimos/db_backend/templates/rails3_migration +30 -0
  64. data/lib/generators/deimos/db_backend_generator.rb +48 -0
  65. data/lib/tasks/deimos.rake +27 -0
  66. data/spec/active_record_consumer_spec.rb +81 -0
  67. data/spec/active_record_producer_spec.rb +107 -0
  68. data/spec/avro_data_decoder_spec.rb +18 -0
  69. data/spec/avro_data_encoder_spec.rb +37 -0
  70. data/spec/backends/db_spec.rb +35 -0
  71. data/spec/backends/kafka_async_spec.rb +11 -0
  72. data/spec/backends/kafka_spec.rb +11 -0
  73. data/spec/consumer_spec.rb +169 -0
  74. data/spec/deimos_spec.rb +120 -0
  75. data/spec/kafka_source_spec.rb +168 -0
  76. data/spec/kafka_topic_info_spec.rb +88 -0
  77. data/spec/phobos.bad_db.yml +73 -0
  78. data/spec/phobos.yml +73 -0
  79. data/spec/producer_spec.rb +397 -0
  80. data/spec/publish_backend_spec.rb +10 -0
  81. data/spec/schemas/com/my-namespace/MySchema-key.avsc +13 -0
  82. data/spec/schemas/com/my-namespace/MySchema.avsc +18 -0
  83. data/spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc +18 -0
  84. data/spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc +33 -0
  85. data/spec/schemas/com/my-namespace/MySchemaWithId.avsc +28 -0
  86. data/spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc +32 -0
  87. data/spec/schemas/com/my-namespace/Widget.avsc +27 -0
  88. data/spec/schemas/com/my-namespace/WidgetTheSecond.avsc +27 -0
  89. data/spec/spec_helper.rb +207 -0
  90. data/spec/updateable_schema_store_spec.rb +36 -0
  91. data/spec/utils/db_producer_spec.rb +259 -0
  92. data/spec/utils/executor_spec.rb +42 -0
  93. data/spec/utils/lag_reporter_spec.rb +69 -0
  94. data/spec/utils/platform_schema_validation_spec.rb +0 -0
  95. data/spec/utils/signal_handler_spec.rb +16 -0
  96. data/support/deimos-solo.png +0 -0
  97. data/support/deimos-with-name-next.png +0 -0
  98. data/support/deimos-with-name.png +0 -0
  99. data/support/flipp-logo.png +0 -0
  100. metadata +452 -0
@@ -0,0 +1,397 @@
1
+ # frozen_string_literal: true
2
+
3
+ # :nodoc:
4
+ module ProducerTest
5
+ describe Deimos::Producer do
6
+
7
+ prepend_before(:each) do
8
+ producer_class = Class.new(Deimos::Producer) do
9
+ schema 'MySchema'
10
+ namespace 'com.my-namespace'
11
+ topic 'my-topic'
12
+ key_config field: 'test_id'
13
+ end
14
+ stub_const('MyProducer', producer_class)
15
+
16
+ producer_class = Class.new(Deimos::Producer) do
17
+ schema 'MySchemaWithId'
18
+ namespace 'com.my-namespace'
19
+ topic 'my-topic'
20
+ key_config plain: true
21
+ end
22
+ stub_const('MyProducerWithID', producer_class)
23
+
24
+ producer_class = Class.new(Deimos::Producer) do
25
+ schema 'MySchema'
26
+ namespace 'com.my-namespace'
27
+ topic 'my-topic'
28
+ key_config plain: true
29
+ # :nodoc:
30
+ def self.partition_key(payload)
31
+ payload[:payload_key] ? payload[:payload_key] + '1' : nil
32
+ end
33
+ end
34
+ stub_const('MyNonEncodedProducer', producer_class)
35
+
36
+ producer_class = Class.new(Deimos::Producer) do
37
+ schema 'MySchema'
38
+ namespace 'com.my-namespace'
39
+ topic 'my-topic2'
40
+ key_config none: true
41
+ end
42
+ stub_const('MyNoKeyProducer', producer_class)
43
+
44
+ producer_class = Class.new(Deimos::Producer) do
45
+ schema 'MySchema'
46
+ namespace 'com.my-namespace'
47
+ topic 'my-topic2'
48
+ key_config schema: 'MySchema-key'
49
+ end
50
+ stub_const('MySchemaProducer', producer_class)
51
+
52
+ producer_class = Class.new(Deimos::Producer) do
53
+ schema 'MySchema'
54
+ namespace 'com.my-namespace'
55
+ topic 'my-topic'
56
+ end
57
+ stub_const('MyErrorProducer', producer_class)
58
+
59
+ end
60
+
61
+ it 'should fail on invalid message with error handler' do
62
+ subscriber = Deimos.subscribe('produce') do |event|
63
+ expect(event.payload[:payloads]).to eq([{ 'invalid' => 'key' }])
64
+ end
65
+ expect { MyProducer.publish('invalid' => 'key', :payload_key => 'key') }.
66
+ to raise_error(Avro::SchemaValidator::ValidationError)
67
+ Deimos.unsubscribe(subscriber)
68
+ end
69
+
70
+ it 'should produce a message' do
71
+ expect(described_class).to receive(:produce_batch).once.with(
72
+ Deimos::Backends::KafkaAsync,
73
+ [
74
+ Deimos::Message.new({ 'test_id' => 'foo', 'some_int' => 123 },
75
+ MyProducer,
76
+ topic: 'my-topic',
77
+ partition_key: 'foo',
78
+ key: 'foo'),
79
+ Deimos::Message.new({ 'test_id' => 'bar', 'some_int' => 124 },
80
+ MyProducer,
81
+ topic: 'my-topic',
82
+ partition_key: 'bar',
83
+ key: 'bar')
84
+ ]
85
+ )
86
+
87
+ MyProducer.publish_list(
88
+ [{ 'test_id' => 'foo', 'some_int' => 123 },
89
+ { 'test_id' => 'bar', 'some_int' => 124 }]
90
+ )
91
+ expect(
92
+ was_message_sent?({ 'test_id' => 'foo', 'some_int' => 123 }, 'my-topic')
93
+ ).to be(true)
94
+ expect(
95
+ was_message_sent?({ 'test_id' => 'foo', 'some_int' => 123 }, 'your-topic')
96
+ ).to be(false)
97
+ expect(
98
+ was_message_sent?({ 'test_id' => 'foo2', 'some_int' => 123 }, 'my-topic')
99
+ ).to be(false)
100
+ end
101
+
102
+ it 'should add a message ID' do
103
+ payload = { 'test_id' => 'foo',
104
+ 'some_int' => 123,
105
+ 'message_id' => a_kind_of(String),
106
+ 'timestamp' => a_kind_of(String) }
107
+ expect(described_class).to receive(:produce_batch).once do |_, messages|
108
+ expect(messages.size).to eq(1)
109
+ expect(messages[0].to_h).
110
+ to match(
111
+ payload: payload,
112
+ topic: 'my-topic',
113
+ partition_key: 'key',
114
+ metadata: {
115
+ producer_name: 'MyProducerWithID',
116
+ decoded_payload: payload
117
+ },
118
+ key: 'key'
119
+ )
120
+ end
121
+ MyProducerWithID.publish_list(
122
+ [{ 'test_id' => 'foo', 'some_int' => 123, :payload_key => 'key' }]
123
+ )
124
+ end
125
+
126
+ it 'should not publish if publish disabled' do
127
+ expect(described_class).not_to receive(:produce_batch)
128
+ Deimos.configure { |c| c.disable_producers = true }
129
+ MyProducer.publish_list(
130
+ [{ 'test_id' => 'foo', 'some_int' => 123 },
131
+ { 'test_id' => 'bar', 'some_int' => 124 }]
132
+ )
133
+ expect(MyProducer.topic).not_to have_sent(anything)
134
+ end
135
+
136
+ it 'should not send messages if inside a disable_producers block' do
137
+ Deimos.disable_producers do
138
+ MyProducer.publish_list(
139
+ [{ 'test_id' => 'foo', 'some_int' => 123 },
140
+ { 'test_id' => 'bar', 'some_int' => 124 }]
141
+ )
142
+ end
143
+ expect(MyProducer.topic).not_to have_sent(anything)
144
+ MyProducer.publish_list(
145
+ [{ 'test_id' => 'foo', 'some_int' => 123 },
146
+ { 'test_id' => 'bar', 'some_int' => 124 }]
147
+ )
148
+ expect(MyProducer.topic).to have_sent(anything)
149
+ end
150
+
151
+ it 'should produce to a prefixed topic' do
152
+ Deimos.configure { |c| c.producer_topic_prefix = 'prefix.' }
153
+ payload = { 'test_id' => 'foo', 'some_int' => 123 }
154
+ expect(described_class).to receive(:produce_batch).once do |_, messages|
155
+ expect(messages.size).to eq(1)
156
+ expect(messages[0].to_h).
157
+ to eq(
158
+ payload: payload,
159
+ topic: 'prefix.my-topic',
160
+ partition_key: 'foo',
161
+ metadata: {
162
+ producer_name: 'MyProducer',
163
+ decoded_payload: payload
164
+ },
165
+ key: 'foo'
166
+ )
167
+ end
168
+
169
+ MyProducer.publish_list([payload])
170
+ Deimos.configure { |c| c.producer_topic_prefix = nil }
171
+ expect(described_class).to receive(:produce_batch).once do |_, messages|
172
+ expect(messages.size).to eq(1)
173
+ expect(messages[0].to_h).
174
+ to eq(
175
+ payload: payload,
176
+ topic: 'my-topic',
177
+ partition_key: 'foo',
178
+ metadata: {
179
+ producer_name: 'MyProducer',
180
+ decoded_payload: payload
181
+ },
182
+ key: 'foo'
183
+ )
184
+ end
185
+
186
+ MyProducer.publish_list(
187
+ [{ 'test_id' => 'foo', 'some_int' => 123 }]
188
+ )
189
+ end
190
+
191
+ it 'should encode the key' do
192
+ encoder = instance_double(Deimos::AvroDataEncoder)
193
+ allow(Deimos::Message).to receive(:new).and_wrap_original do |m, hash, producer|
194
+ message = m.call(hash, producer)
195
+ allow(message).to receive(:add_fields)
196
+ allow(message).to receive(:coerce_fields)
197
+ message
198
+ end
199
+ allow(MyProducer).to receive(:encoder).and_return(encoder).at_least(:once)
200
+ allow(encoder).to receive(:avro_schema)
201
+ expect(encoder).to receive(:encode_key).with('test_id', 'foo', 'my-topic-key')
202
+ expect(encoder).to receive(:encode_key).with('test_id', 'bar', 'my-topic-key')
203
+ expect(encoder).to receive(:encode).with({
204
+ 'test_id' => 'foo',
205
+ 'some_int' => 123
206
+ }, { topic: 'my-topic-value' })
207
+ expect(encoder).to receive(:encode).with({
208
+ 'test_id' => 'bar',
209
+ 'some_int' => 124
210
+ }, { topic: 'my-topic-value' })
211
+
212
+ MyProducer.publish_list(
213
+ [{ 'test_id' => 'foo', 'some_int' => 123 },
214
+ { 'test_id' => 'bar', 'some_int' => 124 }]
215
+ )
216
+ end
217
+
218
+ it 'should not encode with plaintext key' do
219
+ key_encoder = Deimos::AvroDataEncoder.new(
220
+ schema: 'MySchema',
221
+ namespace: 'com.my-namespace'
222
+ )
223
+ allow(key_encoder).to receive(:encode)
224
+ allow(MyNonEncodedProducer).to receive(:encoder).and_return(key_encoder)
225
+ expect(key_encoder).not_to receive(:encode_key)
226
+
227
+ MyNonEncodedProducer.publish_list(
228
+ [{ 'test_id' => 'foo', 'some_int' => 123, :payload_key => 'foo_key' },
229
+ { 'test_id' => 'bar', 'some_int' => 124, :payload_key => 'bar_key' }]
230
+ )
231
+ end
232
+
233
+ it 'should encode with a schema' do
234
+
235
+ encoder = instance_double(Deimos::AvroDataEncoder)
236
+ expect(MySchemaProducer).to receive(:key_encoder).and_return(encoder).
237
+ at_least(:once)
238
+ expect(encoder).to receive(:encode).with({ 'test_id' => 'foo_key' },
239
+ { topic: 'my-topic2-key' })
240
+ expect(encoder).to receive(:encode).with({ 'test_id' => 'bar_key' },
241
+ { topic: 'my-topic2-key' })
242
+
243
+ MySchemaProducer.publish_list(
244
+ [{ 'test_id' => 'foo', 'some_int' => 123,
245
+ :payload_key => { 'test_id' => 'foo_key' } },
246
+ { 'test_id' => 'bar', 'some_int' => 124,
247
+ :payload_key => { 'test_id' => 'bar_key' } }]
248
+ )
249
+ end
250
+
251
+ it 'should error with nothing set' do
252
+ expect {
253
+ MyErrorProducer.publish_list(
254
+ [{ 'test_id' => 'foo', 'some_int' => 123, :payload_key => '123' }]
255
+ )
256
+ }.to raise_error('No key config given - if you are not encoding keys, please use `key_config plain: true`')
257
+ end
258
+
259
+ it 'should error if no key given and none is not the config' do
260
+ expect {
261
+ MyNonEncodedProducer.publish_list(
262
+ [{ 'test_id' => 'foo', 'some_int' => 123 }]
263
+ )
264
+ } .to raise_error('No key given but a key is required! Use `key_config none: true` to avoid using keys.')
265
+ end
266
+
267
+ it 'should allow nil keys if none: true is configured' do
268
+ expect {
269
+ MyNoKeyProducer.publish_list(
270
+ [{ 'test_id' => 'foo', 'some_int' => 123 }]
271
+ )
272
+ } .not_to raise_error
273
+ end
274
+
275
+ it 'should use a partition key' do
276
+ MyNonEncodedProducer.publish_list([{
277
+ 'test_id' => 'foo',
278
+ 'some_int' => 123,
279
+ :payload_key => '123'
280
+ },
281
+ {
282
+ 'test_id' => 'bar',
283
+ 'some_int' => 456,
284
+ :payload_key => '456'
285
+ }])
286
+ expect(MyNonEncodedProducer.topic).to have_sent({
287
+ 'test_id' => 'foo',
288
+ 'some_int' => 123
289
+ }, '123', '1231')
290
+ expect(MyNonEncodedProducer.topic).to have_sent({
291
+ 'test_id' => 'bar',
292
+ 'some_int' => 456
293
+ }, '456', '4561')
294
+ end
295
+
296
+ describe 'disabling' do
297
+ it 'should disable globally' do
298
+ Deimos.disable_producers do
299
+ Deimos.disable_producers do # test nested
300
+ MyProducer.publish(
301
+ 'test_id' => 'foo',
302
+ 'some_int' => 123,
303
+ :payload_key => '123'
304
+ )
305
+ MyProducerWithID.publish(
306
+ 'test_id' => 'foo', 'some_int' => 123
307
+ )
308
+ expect('my-topic').not_to have_sent(anything)
309
+ expect(Deimos).to be_producers_disabled
310
+ expect(Deimos).to be_producers_disabled([MyProducer])
311
+ end
312
+ end
313
+
314
+ MyProducerWithID.publish(
315
+ 'test_id' => 'foo', 'some_int' => 123, :payload_key => 123
316
+ )
317
+ expect('my-topic').
318
+ to have_sent('test_id' => 'foo', 'some_int' => 123,
319
+ 'message_id' => anything, 'timestamp' => anything)
320
+ expect(Deimos).not_to be_producers_disabled
321
+ expect(Deimos).not_to be_producers_disabled([MyProducer])
322
+ end
323
+
324
+ it 'should disable a single producer' do
325
+ Deimos.disable_producers(MyProducer) do # test nested
326
+ Deimos.disable_producers(MyProducer) do
327
+ MySchemaProducer.publish(
328
+ 'test_id' => 'foo', 'some_int' => 123,
329
+ :payload_key => { 'test_id' => 'foo_key' }
330
+ )
331
+ MyProducer.publish(
332
+ 'test_id' => 'foo',
333
+ 'some_int' => 123,
334
+ :payload_key => '123'
335
+ )
336
+ expect('my-topic').not_to have_sent(anything)
337
+ expect('my-topic2').to have_sent('test_id' => 'foo', 'some_int' => 123)
338
+ expect(Deimos).not_to be_producers_disabled
339
+ expect(Deimos).to be_producers_disabled(MyProducer)
340
+ expect(Deimos).not_to be_producers_disabled(MySchemaProducer)
341
+ end
342
+ end
343
+ expect(Deimos).not_to be_producers_disabled
344
+ expect(Deimos).not_to be_producers_disabled(MyProducer)
345
+ expect(Deimos).not_to be_producers_disabled(MySchemaProducer)
346
+ MyProducer.publish(
347
+ 'test_id' => 'foo',
348
+ 'some_int' => 123,
349
+ :payload_key => '123'
350
+ )
351
+ expect('my-topic').
352
+ to have_sent('test_id' => 'foo', 'some_int' => 123)
353
+ end
354
+
355
+ end
356
+
357
+ describe '#determine_backend_class' do
358
+ it 'should return kafka_async if sync is false' do
359
+ expect(described_class.determine_backend_class(false, false)).
360
+ to eq(Deimos::Backends::KafkaAsync)
361
+ expect(described_class.determine_backend_class(nil, false)).
362
+ to eq(Deimos::Backends::KafkaAsync)
363
+ end
364
+
365
+ it 'should return kafka if sync is true' do
366
+ expect(described_class.determine_backend_class(true, false)).
367
+ to eq(Deimos::Backends::Kafka)
368
+ end
369
+
370
+ it 'should return db if db is set' do
371
+ allow(Deimos.config).to receive(:publish_backend).and_return(:db)
372
+ expect(described_class.determine_backend_class(true, false)).
373
+ to eq(Deimos::Backends::Db)
374
+ expect(described_class.determine_backend_class(false, false)).
375
+ to eq(Deimos::Backends::Db)
376
+ end
377
+
378
+ it 'should return kafka if force_send is true' do
379
+ allow(Deimos.config).to receive(:publish_backend).and_return(:db)
380
+ expect(described_class.determine_backend_class(true, true)).
381
+ to eq(Deimos::Backends::Kafka)
382
+ expect(described_class.determine_backend_class(false, true)).
383
+ to eq(Deimos::Backends::KafkaAsync)
384
+ end
385
+
386
+ it 'should use the default sync if set' do
387
+ expect(described_class.determine_backend_class(true, true)).
388
+ to eq(Deimos::Backends::Kafka)
389
+ expect(described_class.determine_backend_class(false, true)).
390
+ to eq(Deimos::Backends::KafkaAsync)
391
+ expect(described_class.determine_backend_class(nil, true)).
392
+ to eq(Deimos::Backends::Kafka)
393
+ end
394
+ end
395
+
396
+ end
397
+ end
@@ -0,0 +1,10 @@
1
+ # frozen_string_literal: true
2
+
3
+ RSpec.describe Deimos::PublishBackend do
4
+ include_context 'with publish_backend'
5
+ it 'should call execute' do
6
+ expect(described_class).to receive(:execute).
7
+ with(messages: messages, producer_class: MyProducer)
8
+ described_class.publish(producer_class: MyProducer, messages: messages)
9
+ end
10
+ end
@@ -0,0 +1,13 @@
1
+ {
2
+ "namespace": "com.my-namespace",
3
+ "name": "MySchema-key",
4
+ "type": "record",
5
+ "doc": "Test schema",
6
+ "fields": [
7
+ {
8
+ "name": "test_id",
9
+ "type": "string",
10
+ "doc": "test string"
11
+ }
12
+ ]
13
+ }
@@ -0,0 +1,18 @@
1
+ {
2
+ "namespace": "com.my-namespace",
3
+ "name": "MySchema",
4
+ "type": "record",
5
+ "doc": "Test schema",
6
+ "fields": [
7
+ {
8
+ "name": "test_id",
9
+ "type": "string",
10
+ "doc": "test string"
11
+ },
12
+ {
13
+ "name": "some_int",
14
+ "type": "int",
15
+ "doc": "test int"
16
+ }
17
+ ]
18
+ }
@@ -0,0 +1,18 @@
1
+ {
2
+ "namespace": "com.my-namespace",
3
+ "name": "MySchemaWithBooleans",
4
+ "type": "record",
5
+ "doc": "Test schema",
6
+ "fields": [
7
+ {
8
+ "name": "test_id",
9
+ "type": "string",
10
+ "doc": "test string"
11
+ },
12
+ {
13
+ "name": "some_bool",
14
+ "type": "boolean",
15
+ "doc": "test bool"
16
+ }
17
+ ]
18
+ }
@@ -0,0 +1,33 @@
1
+ {
2
+ "namespace": "com.my-namespace",
3
+ "name": "MySchemaWithDateTimes",
4
+ "type": "record",
5
+ "doc": "Test schema",
6
+ "fields": [
7
+ {
8
+ "name": "test_id",
9
+ "type": "string",
10
+ "doc": "test string"
11
+ },
12
+ {
13
+ "name": "updated_at",
14
+ "type": ["int", "null"]
15
+ },
16
+ {
17
+ "name": "some_int",
18
+ "type": ["null", "int"],
19
+ "doc": "test int"
20
+ },
21
+ {
22
+ "name": "some_datetime_int",
23
+ "type": ["null", "int"],
24
+ "doc": "test datetime"
25
+ },
26
+ {
27
+ "name": "timestamp",
28
+ "type": "string",
29
+ "doc": "ISO timestamp",
30
+ "default": ""
31
+ }
32
+ ]
33
+ }
@@ -0,0 +1,28 @@
1
+ {
2
+ "namespace": "com.my-namespace",
3
+ "name": "MySchemaWithId",
4
+ "type": "record",
5
+ "doc": "Test schema",
6
+ "fields": [
7
+ {
8
+ "name": "test_id",
9
+ "type": "string",
10
+ "doc": "test string"
11
+ },
12
+ {
13
+ "name": "some_int",
14
+ "type": "int",
15
+ "doc": "test int"
16
+ },
17
+ {
18
+ "name": "message_id",
19
+ "type": "string",
20
+ "doc": "UUID"
21
+ },
22
+ {
23
+ "name": "timestamp",
24
+ "type": "string",
25
+ "doc": "timestamp"
26
+ }
27
+ ]
28
+ }
@@ -0,0 +1,32 @@
1
+ {
2
+ "namespace": "com.my-namespace",
3
+ "name": "MySchemaWithUniqueId",
4
+ "type": "record",
5
+ "doc": "Test schema",
6
+ "fields": [
7
+ {
8
+ "name": "id",
9
+ "type": "int"
10
+ },
11
+ {
12
+ "name": "test_id",
13
+ "type": "string",
14
+ "doc": "test string"
15
+ },
16
+ {
17
+ "name": "some_int",
18
+ "type": "int",
19
+ "doc": "test int"
20
+ },
21
+ {
22
+ "name": "message_id",
23
+ "type": "string",
24
+ "doc": "UUID"
25
+ },
26
+ {
27
+ "name": "timestamp",
28
+ "type": "string",
29
+ "doc": "timestamp"
30
+ }
31
+ ]
32
+ }
@@ -0,0 +1,27 @@
1
+ {
2
+ "namespace": "com.my-namespace",
3
+ "name": "Widget",
4
+ "type": "record",
5
+ "fields": [
6
+ {
7
+ "name": "id",
8
+ "type": "long"
9
+ },
10
+ {
11
+ "name": "widget_id",
12
+ "type": "long"
13
+ },
14
+ {
15
+ "name": "name",
16
+ "type": "string"
17
+ },
18
+ {
19
+ "name": "updated_at",
20
+ "type": "long"
21
+ },
22
+ {
23
+ "name": "created_at",
24
+ "type": "long"
25
+ }
26
+ ]
27
+ }
@@ -0,0 +1,27 @@
1
+ {
2
+ "namespace": "com.my-namespace",
3
+ "name": "WidgetTheSecond",
4
+ "type": "record",
5
+ "fields": [
6
+ {
7
+ "name": "id",
8
+ "type": "long"
9
+ },
10
+ {
11
+ "name": "widget_id",
12
+ "type": "long"
13
+ },
14
+ {
15
+ "name": "model_id",
16
+ "type": "string"
17
+ },
18
+ {
19
+ "name": "updated_at",
20
+ "type": "long"
21
+ },
22
+ {
23
+ "name": "created_at",
24
+ "type": "long"
25
+ }
26
+ ]
27
+ }