avro_turf 1.19.0 → 1.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +20 -11
- data/CHANGELOG.md +6 -0
- data/Gemfile +5 -2
- data/Rakefile +2 -1
- data/avro_turf.gemspec +16 -16
- data/lib/avro_turf/cached_confluent_schema_registry.rb +9 -8
- data/lib/avro_turf/cached_schema_registry.rb +3 -1
- data/lib/avro_turf/confluent_schema_registry.rb +23 -17
- data/lib/avro_turf/core_ext/date.rb +2 -0
- data/lib/avro_turf/core_ext/enumerable.rb +2 -0
- data/lib/avro_turf/core_ext/false_class.rb +2 -0
- data/lib/avro_turf/core_ext/hash.rb +4 -2
- data/lib/avro_turf/core_ext/nil_class.rb +2 -0
- data/lib/avro_turf/core_ext/numeric.rb +2 -0
- data/lib/avro_turf/core_ext/string.rb +2 -0
- data/lib/avro_turf/core_ext/symbol.rb +2 -0
- data/lib/avro_turf/core_ext/time.rb +2 -0
- data/lib/avro_turf/core_ext/true_class.rb +2 -0
- data/lib/avro_turf/core_ext.rb +12 -10
- data/lib/avro_turf/disk_cache.rb +13 -12
- data/lib/avro_turf/in_memory_cache.rb +2 -0
- data/lib/avro_turf/messaging.rb +22 -14
- data/lib/avro_turf/mutable_schema_store.rb +25 -4
- data/lib/avro_turf/schema_registry.rb +3 -1
- data/lib/avro_turf/schema_store.rb +3 -2
- data/lib/avro_turf/schema_to_avro_patch.rb +14 -12
- data/lib/avro_turf/test/fake_confluent_schema_registry_server.rb +24 -23
- data/lib/avro_turf/test/fake_prefixed_confluent_schema_registry_server.rb +12 -10
- data/lib/avro_turf/test/fake_schema_registry_server.rb +3 -1
- data/lib/avro_turf/version.rb +3 -1
- data/lib/avro_turf.rb +15 -13
- data/perf/encoding_size.rb +4 -2
- data/perf/encoding_speed.rb +4 -2
- data/spec/avro_turf_spec.rb +24 -23
- data/spec/cached_confluent_schema_registry_spec.rb +9 -7
- data/spec/confluent_schema_registry_spec.rb +31 -10
- data/spec/core_ext/date_spec.rb +2 -0
- data/spec/core_ext/enumerable_spec.rb +2 -0
- data/spec/core_ext/false_class_spec.rb +2 -0
- data/spec/core_ext/hash_spec.rb +3 -1
- data/spec/core_ext/nil_class_spec.rb +2 -0
- data/spec/core_ext/numeric_spec.rb +2 -0
- data/spec/core_ext/string_spec.rb +2 -0
- data/spec/core_ext/symbol_spec.rb +2 -0
- data/spec/core_ext/time_spec.rb +2 -0
- data/spec/core_ext/true_class_spec.rb +2 -0
- data/spec/disk_cached_confluent_schema_registry_spec.rb +23 -21
- data/spec/messaging_spec.rb +124 -99
- data/spec/mutable_schema_store_spec.rb +134 -0
- data/spec/schema_store_spec.rb +23 -21
- data/spec/schema_to_avro_patch_spec.rb +8 -7
- data/spec/spec_helper.rb +9 -9
- data/spec/support/authorized_fake_confluent_schema_registry_server.rb +4 -2
- data/spec/support/authorized_fake_prefixed_confluent_schema_registry_server.rb +4 -2
- data/spec/support/confluent_schema_registry_context.rb +32 -30
- data/spec/test/fake_confluent_schema_registry_server_spec.rb +97 -94
- metadata +5 -26
@@ -1,36 +1,38 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
class AvroTurf
|
2
4
|
module AvroGemPatch
|
3
5
|
module RecordSchema
|
4
6
|
module ClassMethods
|
5
|
-
def make_field_objects(field_data, names, namespace=nil)
|
7
|
+
def make_field_objects(field_data, names, namespace = nil)
|
6
8
|
new_field_data = []
|
7
9
|
field_data.each do |field|
|
8
|
-
if field.respond_to?(:[]) && !field.key?(
|
10
|
+
if field.respond_to?(:[]) && !field.key?("default")
|
9
11
|
field = field.clone
|
10
|
-
field[
|
12
|
+
field["default"] = :no_default
|
11
13
|
end
|
12
14
|
new_field_data << field
|
13
15
|
end
|
14
16
|
super(new_field_data, names, namespace)
|
15
17
|
end
|
16
18
|
end
|
17
|
-
|
19
|
+
|
18
20
|
def self.prepended(base)
|
19
21
|
class << base
|
20
22
|
prepend ClassMethods
|
21
23
|
end
|
22
24
|
end
|
23
25
|
end
|
24
|
-
|
26
|
+
|
25
27
|
module Field
|
26
|
-
def initialize(type, name, default
|
27
|
-
super
|
28
|
+
def initialize(type, name, default = :no_default, order = nil, names = nil, namespace = nil)
|
29
|
+
super
|
28
30
|
end
|
29
|
-
|
30
|
-
def to_avro(names=Set.new)
|
31
|
-
{
|
32
|
-
avro[
|
33
|
-
avro[
|
31
|
+
|
32
|
+
def to_avro(names = Set.new)
|
33
|
+
{"name" => name, "type" => type.to_avro(names)}.tap do |avro|
|
34
|
+
avro["default"] = default unless default == :no_default
|
35
|
+
avro["order"] = order if order
|
34
36
|
end
|
35
37
|
end
|
36
38
|
end
|
@@ -1,18 +1,20 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "sinatra/base"
|
2
4
|
|
3
5
|
class FakeConfluentSchemaRegistryServer < Sinatra::Base
|
4
6
|
QUALIFIED_SUBJECT = /
|
5
7
|
:(?<context>\.[^:]*)
|
6
8
|
:(?<subject>.*)
|
7
9
|
/x
|
8
|
-
DEFAULT_CONTEXT =
|
9
|
-
SUBJECTS = Hash.new { |hash, key| hash[key] = Hash.new {
|
10
|
-
SCHEMAS = Hash.new { |hash, key| hash[key] =
|
11
|
-
CONFIGS =
|
12
|
-
SUBJECT_NOT_FOUND = {
|
13
|
-
VERSION_NOT_FOUND = {
|
14
|
-
SCHEMA_NOT_FOUND = {
|
15
|
-
DEFAULT_GLOBAL_CONFIG = {
|
10
|
+
DEFAULT_CONTEXT = "."
|
11
|
+
SUBJECTS = Hash.new { |hash, key| hash[key] = Hash.new { [] } }
|
12
|
+
SCHEMAS = Hash.new { |hash, key| hash[key] = [] }
|
13
|
+
CONFIGS = {}
|
14
|
+
SUBJECT_NOT_FOUND = {error_code: 40401, message: "Subject not found"}.to_json.freeze
|
15
|
+
VERSION_NOT_FOUND = {error_code: 40402, message: "Version not found"}.to_json.freeze
|
16
|
+
SCHEMA_NOT_FOUND = {error_code: 40403, message: "Schema not found"}.to_json.freeze
|
17
|
+
DEFAULT_GLOBAL_CONFIG = {"compatibility" => "BACKWARD"}.freeze
|
16
18
|
|
17
19
|
@global_config = DEFAULT_GLOBAL_CONFIG.dup
|
18
20
|
|
@@ -51,7 +53,7 @@ class FakeConfluentSchemaRegistryServer < Sinatra::Base
|
|
51
53
|
SUBJECTS[context][subject] = SUBJECTS[context][subject] << schema_id
|
52
54
|
end
|
53
55
|
|
54
|
-
{
|
56
|
+
{id: schema_id}.to_json
|
55
57
|
end
|
56
58
|
|
57
59
|
get "/schemas/ids/:schema_id/versions" do
|
@@ -60,7 +62,7 @@ class FakeConfluentSchemaRegistryServer < Sinatra::Base
|
|
60
62
|
schema = SCHEMAS[context].at(schema_id)
|
61
63
|
halt(404, SCHEMA_NOT_FOUND) unless schema
|
62
64
|
|
63
|
-
related_subjects = SUBJECTS[context].select {|_, vs| vs.include? schema_id }
|
65
|
+
related_subjects = SUBJECTS[context].select { |_, vs| vs.include? schema_id }
|
64
66
|
|
65
67
|
related_subjects.map do |subject, versions|
|
66
68
|
{
|
@@ -74,14 +76,13 @@ class FakeConfluentSchemaRegistryServer < Sinatra::Base
|
|
74
76
|
context, _subject = parse_qualified_subject(params[:subject])
|
75
77
|
schema = SCHEMAS[context].at(params[:schema_id].to_i)
|
76
78
|
halt(404, SCHEMA_NOT_FOUND) unless schema
|
77
|
-
{
|
79
|
+
{schema: schema}.to_json
|
78
80
|
end
|
79
81
|
|
80
82
|
get "/subjects" do
|
81
|
-
subject_names = SUBJECTS.
|
83
|
+
subject_names = SUBJECTS.each_with_object([]) do |args, acc|
|
82
84
|
context, subjects = args
|
83
|
-
subjects.keys.each { |subject| acc << (context ==
|
84
|
-
acc
|
85
|
+
subjects.keys.each { |subject| acc << ((context == ".") ? subject : ":#{context}:#{subject}") }
|
85
86
|
end
|
86
87
|
subject_names.to_json
|
87
88
|
end
|
@@ -98,11 +99,11 @@ class FakeConfluentSchemaRegistryServer < Sinatra::Base
|
|
98
99
|
schema_ids = SUBJECTS[context][subject]
|
99
100
|
halt(404, SUBJECT_NOT_FOUND) if schema_ids.empty?
|
100
101
|
|
101
|
-
schema_id = if params[:version] ==
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
102
|
+
schema_id = if params[:version] == "latest"
|
103
|
+
schema_ids.last
|
104
|
+
else
|
105
|
+
schema_ids.at(Integer(params[:version]) - 1)
|
106
|
+
end
|
106
107
|
halt(404, VERSION_NOT_FOUND) unless schema_id
|
107
108
|
|
108
109
|
schema = SCHEMAS[context].at(schema_id)
|
@@ -171,13 +172,13 @@ class FakeConfluentSchemaRegistryServer < Sinatra::Base
|
|
171
172
|
def parse_qualified_subject(qualified_subject)
|
172
173
|
match = QUALIFIED_SUBJECT.match(qualified_subject)
|
173
174
|
if !match.nil?
|
174
|
-
match.named_captures.values_at(
|
175
|
+
match.named_captures.values_at("context", "subject")
|
175
176
|
else
|
176
|
-
[
|
177
|
+
[DEFAULT_CONTEXT, qualified_subject]
|
177
178
|
end
|
178
179
|
end
|
179
180
|
|
180
181
|
def qualify_subject(context, subject)
|
181
|
-
context == "." ? subject : ":#{context}:#{subject}"
|
182
|
+
(context == ".") ? subject : ":#{context}:#{subject}"
|
182
183
|
end
|
183
184
|
end
|
@@ -1,7 +1,9 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "sinatra/base"
|
2
4
|
|
3
5
|
class FakePrefixedConfluentSchemaRegistryServer < FakeConfluentSchemaRegistryServer
|
4
|
-
DEFAULT_CONTEXT=
|
6
|
+
DEFAULT_CONTEXT = "."
|
5
7
|
|
6
8
|
post "/prefix/subjects/:subject/versions" do
|
7
9
|
schema = parse_schema
|
@@ -9,7 +11,7 @@ class FakePrefixedConfluentSchemaRegistryServer < FakeConfluentSchemaRegistrySer
|
|
9
11
|
|
10
12
|
schemas_for_subject =
|
11
13
|
SCHEMAS[DEFAULT_CONTEXT].select
|
12
|
-
|
14
|
+
.with_index { |_, i| ids_for_subject.include?(i) }
|
13
15
|
|
14
16
|
if schemas_for_subject.include?(schema)
|
15
17
|
schema_id = SCHEMAS[DEFAULT_CONTEXT].index(schema)
|
@@ -19,13 +21,13 @@ class FakePrefixedConfluentSchemaRegistryServer < FakeConfluentSchemaRegistrySer
|
|
19
21
|
SUBJECTS[DEFAULT_CONTEXT][params[:subject]] = SUBJECTS[DEFAULT_CONTEXT][params[:subject]] << schema_id
|
20
22
|
end
|
21
23
|
|
22
|
-
{
|
24
|
+
{id: schema_id}.to_json
|
23
25
|
end
|
24
26
|
|
25
27
|
get "/prefix/schemas/ids/:schema_id" do
|
26
28
|
schema = SCHEMAS[DEFAULT_CONTEXT].at(params[:schema_id].to_i)
|
27
29
|
halt(404, SCHEMA_NOT_FOUND) unless schema
|
28
|
-
{
|
30
|
+
{schema: schema}.to_json
|
29
31
|
end
|
30
32
|
|
31
33
|
get "/prefix/subjects" do
|
@@ -42,11 +44,11 @@ class FakePrefixedConfluentSchemaRegistryServer < FakeConfluentSchemaRegistrySer
|
|
42
44
|
schema_ids = SUBJECTS[DEFAULT_CONTEXT][params[:subject]]
|
43
45
|
halt(404, SUBJECT_NOT_FOUND) if schema_ids.empty?
|
44
46
|
|
45
|
-
schema_id = if params[:version] ==
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
47
|
+
schema_id = if params[:version] == "latest"
|
48
|
+
schema_ids.last
|
49
|
+
else
|
50
|
+
schema_ids.at(Integer(params[:version]) - 1)
|
51
|
+
end
|
50
52
|
halt(404, VERSION_NOT_FOUND) unless schema_id
|
51
53
|
|
52
54
|
schema = SCHEMAS[DEFAULT_CONTEXT].at(schema_id)
|
@@ -1,4 +1,6 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "avro_turf/test/fake_confluent_schema_registry_server"
|
2
4
|
|
3
5
|
# FakeSchemaRegistryServer is deprecated and will be removed in a future release.
|
4
6
|
# Use FakeConfluentSchemaRegistryServer instead.
|
data/lib/avro_turf/version.rb
CHANGED
data/lib/avro_turf.rb
CHANGED
@@ -1,17 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
begin
|
2
|
-
require
|
4
|
+
require "avro-patches"
|
3
5
|
rescue LoadError
|
4
6
|
false
|
5
7
|
end
|
6
|
-
require
|
7
|
-
require
|
8
|
-
require
|
9
|
-
require
|
10
|
-
require
|
8
|
+
require "avro_turf/version"
|
9
|
+
require "avro"
|
10
|
+
require "json"
|
11
|
+
require "avro_turf/schema_store"
|
12
|
+
require "avro_turf/core_ext"
|
11
13
|
|
12
14
|
# check for something that indicates Avro v1.9.0 or later
|
13
15
|
unless defined?(::Avro::LogicalTypes)
|
14
|
-
require
|
16
|
+
require "avro_turf/schema_to_avro_patch"
|
15
17
|
end
|
16
18
|
|
17
19
|
class AvroTurf
|
@@ -69,10 +71,10 @@ class AvroTurf
|
|
69
71
|
#
|
70
72
|
# Returns nothing.
|
71
73
|
def encode_to_stream(data, schema_name: nil, stream: nil, namespace: @namespace,
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
74
|
+
validate: false,
|
75
|
+
validate_options: {recursive: true,
|
76
|
+
encoded: false,
|
77
|
+
fail_on_extra_fields: true})
|
76
78
|
schema = @schema_store.find(schema_name, namespace)
|
77
79
|
writer = Avro::IO::DatumWriter.new(schema)
|
78
80
|
|
@@ -98,7 +100,7 @@ class AvroTurf
|
|
98
100
|
decode_stream(stream, schema_name: schema_name, namespace: namespace)
|
99
101
|
end
|
100
102
|
|
101
|
-
|
103
|
+
alias_method :decode, :decode_first
|
102
104
|
|
103
105
|
# Returns all entries encoded in the data.
|
104
106
|
def decode_all(encoded_data, schema_name: nil, namespace: @namespace)
|
@@ -119,7 +121,7 @@ class AvroTurf
|
|
119
121
|
data.first
|
120
122
|
end
|
121
123
|
|
122
|
-
|
124
|
+
alias_method :decode_stream, :decode_first_from_stream
|
123
125
|
|
124
126
|
# Returns all entries encoded in the stream.
|
125
127
|
def decode_all_from_stream(stream, schema_name: nil, namespace: @namespace)
|
data/perf/encoding_size.rb
CHANGED
@@ -1,11 +1,13 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
2
4
|
#
|
3
5
|
# Measures the encoded size of messages of increasing size.
|
4
6
|
|
5
7
|
$LOAD_PATH.unshift(File.expand_path("../lib", File.dirname(__FILE__)))
|
6
8
|
|
7
|
-
require
|
8
|
-
require
|
9
|
+
require "benchmark"
|
10
|
+
require "avro_turf"
|
9
11
|
|
10
12
|
sizes = [1, 10, 100, 1_000, 10_000]
|
11
13
|
avro = AvroTurf.new(schemas_path: File.dirname(__FILE__))
|
data/perf/encoding_speed.rb
CHANGED
@@ -1,11 +1,13 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
2
4
|
#
|
3
5
|
# Measures the time it takes to encode messages of increasing size.
|
4
6
|
|
5
7
|
$LOAD_PATH.unshift(File.expand_path("../lib", File.dirname(__FILE__)))
|
6
8
|
|
7
|
-
require
|
8
|
-
require
|
9
|
+
require "benchmark"
|
10
|
+
require "avro_turf"
|
9
11
|
|
10
12
|
# Number of iterations per run.
|
11
13
|
N = 10_000
|
data/spec/avro_turf_spec.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
describe AvroTurf do
|
2
4
|
let(:avro) { AvroTurf.new(schemas_path: "spec/schemas/") }
|
3
5
|
|
@@ -56,7 +58,7 @@ describe AvroTurf do
|
|
56
58
|
end
|
57
59
|
end
|
58
60
|
|
59
|
-
context
|
61
|
+
context "when using nested schemas" do
|
60
62
|
before do
|
61
63
|
define_schema "post.avsc", <<-AVSC
|
62
64
|
{
|
@@ -138,7 +140,6 @@ describe AvroTurf do
|
|
138
140
|
expect(avro.decode(encoded_data)).to eq(data)
|
139
141
|
end
|
140
142
|
end
|
141
|
-
|
142
143
|
end
|
143
144
|
|
144
145
|
describe "#decode" do
|
@@ -179,22 +180,22 @@ describe AvroTurf do
|
|
179
180
|
}
|
180
181
|
AVSC
|
181
182
|
|
182
|
-
encoded_data = avro.encode({
|
183
|
+
encoded_data = avro.encode({"x" => 42, "y" => 13}, schema_name: "point")
|
183
184
|
reader_avro = AvroTurf.new(schemas_path: "spec/schemas/reader")
|
184
185
|
|
185
|
-
expect(reader_avro.decode(encoded_data, schema_name: "point")).to eq({
|
186
|
+
expect(reader_avro.decode(encoded_data, schema_name: "point")).to eq({"x" => 42})
|
186
187
|
end
|
187
188
|
end
|
188
189
|
|
189
190
|
describe "#decode_all" do
|
190
191
|
context "when data contains multiple entries" do
|
191
|
-
let(:encoded_data) {
|
192
|
+
let(:encoded_data) { "Obj\u0001\u0004\u0014avro.codec\bnull\u0016avro.schema\xB6\u0004[{\"type\": \"record\", \"name\": \"address\", \"fields\": [{\"type\": \"string\", \"name\": \"street\"}, {\"type\": \"string\", \"name\": \"city\"}]}, {\"type\": \"record\", \"name\": \"person\", \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, {\"type\": \"int\", \"name\": \"age\"}, {\"type\": \"address\", \"name\": \"address\"}]}]\u0000\xF9u\x84\xA1c\u0010\x82B\xE2\xCF\xF1\x98\xF7\xF1JH\u0004\x96\u0001\u0002\u0014Python🐍\x80\u0004\u0018Green Street\u001ASan Francisco\u0002\u0010Mojo🐍\u0002\u0016Blue Street\u0014Saturn🪐\xF9u\x84\xA1c\u0010\x82B\xE2\xCF\xF1\x98\xF7\xF1JH" }
|
192
193
|
|
193
194
|
it "returns array of entries decoded using the inlined writer's schema " do
|
194
195
|
expect(avro.decode_all(encoded_data).entries).to eq(
|
195
196
|
[
|
196
|
-
{"name"=>"Python🐍", "age"=>256, "address"=>{"street"=>"Green Street", "city"=>"San Francisco"}},
|
197
|
-
{"name"=>"Mojo🐍", "age"=>1, "address"=>{"street"=>"Blue Street", "city"=>"Saturn🪐"}}
|
197
|
+
{"name" => "Python🐍", "age" => 256, "address" => {"street" => "Green Street", "city" => "San Francisco"}},
|
198
|
+
{"name" => "Mojo🐍", "age" => 1, "address" => {"street" => "Blue Street", "city" => "Saturn🪐"}}
|
198
199
|
]
|
199
200
|
)
|
200
201
|
end
|
@@ -219,8 +220,8 @@ describe AvroTurf do
|
|
219
220
|
.decode_all(encoded_data, schema_name: "person").entries
|
220
221
|
).to eq(
|
221
222
|
[
|
222
|
-
{"name"=>"Python🐍", "age"=>256, "fav_color"=>"red🟥"},
|
223
|
-
{"name"=>"Mojo🐍", "age"=>1, "fav_color"=>"red🟥"}
|
223
|
+
{"name" => "Python🐍", "age" => 256, "fav_color" => "red🟥"},
|
224
|
+
{"name" => "Mojo🐍", "age" => 1, "fav_color" => "red🟥"}
|
224
225
|
]
|
225
226
|
)
|
226
227
|
end
|
@@ -249,7 +250,7 @@ describe AvroTurf do
|
|
249
250
|
end
|
250
251
|
|
251
252
|
context "with a valid message" do
|
252
|
-
let(:message) { {
|
253
|
+
let(:message) { {"full_name" => "John Doe"} }
|
253
254
|
|
254
255
|
it "does not raise any error" do
|
255
256
|
define_schema "message.avsc", <<-AVSC
|
@@ -267,7 +268,7 @@ describe AvroTurf do
|
|
267
268
|
end
|
268
269
|
|
269
270
|
context "when message has wrong type" do
|
270
|
-
let(:message) { {
|
271
|
+
let(:message) { {"full_name" => 123} }
|
271
272
|
|
272
273
|
it "raises Avro::SchemaValidator::ValidationError with a message about type mismatch" do
|
273
274
|
define_schema "message.avsc", <<-AVSC
|
@@ -285,7 +286,7 @@ describe AvroTurf do
|
|
285
286
|
end
|
286
287
|
|
287
288
|
context "when message contains extra fields (typo in key)" do
|
288
|
-
let(:message) { {
|
289
|
+
let(:message) { {"fulll_name" => "John Doe"} }
|
289
290
|
|
290
291
|
it "raises Avro::SchemaValidator::ValidationError with a message about extra field" do
|
291
292
|
define_schema "message.avsc", <<-AVSC
|
@@ -303,13 +304,12 @@ describe AvroTurf do
|
|
303
304
|
end
|
304
305
|
|
305
306
|
context "when the `fail_on_extra_fields` validation option is disabled" do
|
306
|
-
let(:message) { {
|
307
|
+
let(:message) { {"full_name" => "John Doe", "first_name" => "John", "last_name" => "Doe"} }
|
307
308
|
subject(:encode_to_stream) do
|
308
309
|
stream = StringIO.new
|
309
310
|
avro.encode_to_stream(message, stream: stream, schema_name: "message",
|
310
|
-
|
311
|
-
|
312
|
-
)
|
311
|
+
validate: true,
|
312
|
+
validate_options: {recursive: true, encoded: false, fail_on_extra_fields: false})
|
313
313
|
end
|
314
314
|
|
315
315
|
it "should not raise Avro::SchemaValidator::ValidationError with a message about extra field" do
|
@@ -352,8 +352,8 @@ describe AvroTurf do
|
|
352
352
|
|
353
353
|
expect(avro.decode_all_from_stream(stream).entries).to eq(
|
354
354
|
[
|
355
|
-
{"name"=>"Python🐍", "age"=>256, "address"=>{"street"=>"Green Street", "city"=>"San Francisco"}},
|
356
|
-
{"name"=>"Mojo🐍", "age"=>1, "address"=>{"street"=>"Blue Street", "city"=>"Saturn🪐"}}
|
355
|
+
{"name" => "Python🐍", "age" => 256, "address" => {"street" => "Green Street", "city" => "San Francisco"}},
|
356
|
+
{"name" => "Mojo🐍", "age" => 1, "address" => {"street" => "Blue Street", "city" => "Saturn🪐"}}
|
357
357
|
]
|
358
358
|
)
|
359
359
|
end
|
@@ -390,7 +390,7 @@ describe AvroTurf do
|
|
390
390
|
}
|
391
391
|
AVSC
|
392
392
|
|
393
|
-
datum = {
|
393
|
+
datum = {message: "hello"}
|
394
394
|
expect(avro.valid?(datum, schema_name: "postcard")).to eq true
|
395
395
|
end
|
396
396
|
|
@@ -423,7 +423,7 @@ describe AvroTurf do
|
|
423
423
|
end
|
424
424
|
|
425
425
|
context "when message contains extra fields (typo in key)" do
|
426
|
-
let(:message) { {
|
426
|
+
let(:message) { {"fulll_name" => "John Doe"} }
|
427
427
|
|
428
428
|
before do
|
429
429
|
define_schema "message.avsc", <<-AVSC
|
@@ -438,16 +438,17 @@ describe AvroTurf do
|
|
438
438
|
end
|
439
439
|
|
440
440
|
it "is valid" do
|
441
|
-
datum = {
|
441
|
+
datum = {"full_name" => "John Doe", "extra" => "extra"}
|
442
442
|
expect(avro.valid?(datum, schema_name: "message")).to eq true
|
443
443
|
end
|
444
444
|
|
445
445
|
it "is invalid when passing fail_on_extra_fields" do
|
446
|
-
datum = {
|
446
|
+
datum = {"full_name" => "John Doe", "extra" => "extra"}
|
447
447
|
validate_options = {
|
448
448
|
recursive: true,
|
449
449
|
encoded: false,
|
450
|
-
fail_on_extra_fields: true
|
450
|
+
fail_on_extra_fields: true
|
451
|
+
}
|
451
452
|
valid = avro.valid?(datum, schema_name: "message", validate_options: validate_options)
|
452
453
|
expect(valid).to eq false
|
453
454
|
end
|
@@ -1,17 +1,19 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
require
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "webmock/rspec"
|
4
|
+
require "avro_turf/cached_confluent_schema_registry"
|
5
|
+
require "avro_turf/test/fake_confluent_schema_registry_server"
|
4
6
|
|
5
7
|
describe AvroTurf::CachedConfluentSchemaRegistry do
|
6
8
|
let(:upstream) { instance_double(AvroTurf::ConfluentSchemaRegistry) }
|
7
9
|
let(:registry) { described_class.new(upstream) }
|
8
10
|
let(:id) { rand(999) }
|
9
|
-
let(:subject_name) {
|
11
|
+
let(:subject_name) { "a_subject" }
|
10
12
|
let(:schema) do
|
11
13
|
{
|
12
14
|
type: "record",
|
13
15
|
name: "person",
|
14
|
-
fields: [{
|
16
|
+
fields: [{name: "name", type: "string"}]
|
15
17
|
}.to_json
|
16
18
|
end
|
17
19
|
|
@@ -57,7 +59,7 @@ describe AvroTurf::CachedConfluentSchemaRegistry do
|
|
57
59
|
end
|
58
60
|
end
|
59
61
|
|
60
|
-
describe
|
62
|
+
describe "#subject_version" do
|
61
63
|
let(:version) { 1 }
|
62
64
|
let(:schema_with_meta) do
|
63
65
|
{
|
@@ -68,7 +70,7 @@ describe AvroTurf::CachedConfluentSchemaRegistry do
|
|
68
70
|
}
|
69
71
|
end
|
70
72
|
|
71
|
-
it
|
73
|
+
it "caches the result of subject_version" do
|
72
74
|
allow(upstream).to receive(:subject_version).with(subject_name, version).and_return(schema_with_meta)
|
73
75
|
registry.subject_version(subject_name, version)
|
74
76
|
registry.subject_version(subject_name, version)
|
@@ -1,22 +1,40 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
require
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "webmock/rspec"
|
4
|
+
require "avro_turf/confluent_schema_registry"
|
5
|
+
require "avro_turf/test/fake_confluent_schema_registry_server"
|
4
6
|
|
5
7
|
describe AvroTurf::ConfluentSchemaRegistry do
|
6
8
|
let(:user) { "abc" }
|
7
9
|
let(:password) { "xxyyzz" }
|
8
10
|
let(:client_cert) { "test client cert" }
|
11
|
+
let(:client_chain) { "test client cert chain" }
|
9
12
|
let(:client_key) { "test client key" }
|
10
13
|
let(:client_key_pass) { "test client key password" }
|
11
14
|
let(:connect_timeout) { 10 }
|
12
15
|
|
13
|
-
context
|
16
|
+
context "authenticated by cert" do
|
17
|
+
it_behaves_like "a confluent schema registry client" do
|
18
|
+
let(:registry) {
|
19
|
+
described_class.new(
|
20
|
+
registry_url,
|
21
|
+
logger: logger,
|
22
|
+
client_cert: client_cert,
|
23
|
+
client_key: client_key,
|
24
|
+
client_key_pass: client_key_pass
|
25
|
+
)
|
26
|
+
}
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
context "authenticated by cert with chain" do
|
14
31
|
it_behaves_like "a confluent schema registry client" do
|
15
32
|
let(:registry) {
|
16
33
|
described_class.new(
|
17
34
|
registry_url,
|
18
35
|
logger: logger,
|
19
36
|
client_cert: client_cert,
|
37
|
+
client_chain: client_chain,
|
20
38
|
client_key: client_key,
|
21
39
|
client_key_pass: client_key_pass
|
22
40
|
)
|
@@ -24,23 +42,25 @@ describe AvroTurf::ConfluentSchemaRegistry do
|
|
24
42
|
end
|
25
43
|
end
|
26
44
|
|
27
|
-
context
|
45
|
+
context "authenticated by basic auth" do
|
28
46
|
it_behaves_like "a confluent schema registry client" do
|
29
47
|
let(:registry) {
|
30
48
|
described_class.new(
|
31
49
|
registry_url,
|
50
|
+
logger: logger,
|
32
51
|
user: user,
|
33
|
-
password: password
|
52
|
+
password: password
|
34
53
|
)
|
35
54
|
}
|
36
55
|
end
|
37
56
|
end
|
38
57
|
|
39
|
-
context
|
58
|
+
context "with connect_timeout" do
|
40
59
|
it_behaves_like "a confluent schema registry client" do
|
41
60
|
let(:registry) {
|
42
61
|
described_class.new(
|
43
62
|
registry_url,
|
63
|
+
logger: logger,
|
44
64
|
user: user,
|
45
65
|
password: password,
|
46
66
|
connect_timeout: connect_timeout
|
@@ -49,12 +69,13 @@ describe AvroTurf::ConfluentSchemaRegistry do
|
|
49
69
|
end
|
50
70
|
end
|
51
71
|
|
52
|
-
context
|
53
|
-
it_behaves_like "a confluent schema registry client", schema_context:
|
72
|
+
context "with non default schema_context" do
|
73
|
+
it_behaves_like "a confluent schema registry client", schema_context: "other" do
|
54
74
|
let(:registry) {
|
55
75
|
described_class.new(
|
56
76
|
registry_url,
|
57
|
-
|
77
|
+
logger: logger,
|
78
|
+
schema_context: "other",
|
58
79
|
user: user,
|
59
80
|
password: password,
|
60
81
|
connect_timeout: connect_timeout
|
data/spec/core_ext/date_spec.rb
CHANGED
data/spec/core_ext/hash_spec.rb
CHANGED
@@ -1,8 +1,10 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
describe Hash, "#as_avro" do
|
2
4
|
it "coerces the keys and values to Avro" do
|
3
5
|
x = double(as_avro: "x")
|
4
6
|
y = double(as_avro: "y")
|
5
7
|
|
6
|
-
expect({
|
8
|
+
expect({x => y}.as_avro).to eq({"x" => "y"})
|
7
9
|
end
|
8
10
|
end
|