avro_turf 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +3 -0
- data/lib/avro_turf.rb +44 -44
- data/lib/avro_turf/schema_store.rb +58 -0
- data/lib/avro_turf/version.rb +1 -1
- data/perf/address.avsc +14 -0
- data/perf/encoding_size.rb +26 -0
- data/perf/encoding_speed.rb +30 -0
- data/perf/person.avsc +14 -0
- data/spec/avro_turf_spec.rb +125 -0
- data/spec/schema_store_spec.rb +253 -0
- data/spec/spec_helper.rb +14 -0
- metadata +11 -4
- data/spec/avro_spec.rb +0 -227
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0845fc0e81baaa72df5abd34b47a0ce6d06a853f
|
4
|
+
data.tar.gz: 434aaa74efb2f96437045a8705c7889348bec9c3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ef023ba8d9fb413e69a467c1ecb4a4ef73c6c1f1bb86f1f362b6f351abcd6ed8b2fae965d7b50b4d3ba6e81513c30dcffbe1e5d688044efa872f4b65f85ff39a
|
7
|
+
data.tar.gz: e095b79ce3f37a69efac8537e6d59fcb5c8d66350604eba7f00166911801699eccf945552fcbc5c950223f7c14ddfddd2f58157b70aff2466e6d2ea157485eab
|
data/Gemfile
CHANGED
data/lib/avro_turf.rb
CHANGED
@@ -1,15 +1,17 @@
|
|
1
1
|
require 'avro_turf/version'
|
2
2
|
require 'avro'
|
3
3
|
require 'json'
|
4
|
+
require 'avro_turf/schema_store'
|
4
5
|
|
5
6
|
class AvroTurf
|
6
7
|
class Error < StandardError; end
|
7
8
|
class SchemaError < Error; end
|
9
|
+
class SchemaNotFoundError < Error; end
|
8
10
|
|
9
|
-
def initialize(schemas_path:, namespace: nil)
|
10
|
-
@schemas_path = schemas_path or raise "Please specify a schema path"
|
11
|
-
@schemas = Hash.new
|
11
|
+
def initialize(schemas_path:, namespace: nil, codec: nil)
|
12
12
|
@namespace = namespace
|
13
|
+
@schema_store = SchemaStore.new(path: schemas_path)
|
14
|
+
@codec = codec
|
13
15
|
end
|
14
16
|
|
15
17
|
# Encodes data to Avro using the specified schema.
|
@@ -19,15 +21,31 @@ class AvroTurf
|
|
19
21
|
#
|
20
22
|
# Returns a String containing the encoded data.
|
21
23
|
def encode(data, schema_name:, namespace: @namespace)
|
22
|
-
|
24
|
+
stream = StringIO.new
|
25
|
+
|
26
|
+
encode_to_stream(data, stream: stream, schema_name: schema_name, namespace: namespace)
|
27
|
+
|
28
|
+
stream.string
|
29
|
+
end
|
30
|
+
|
31
|
+
# Encodes data to Avro using the specified schema and writes it to the
|
32
|
+
# specified stream.
|
33
|
+
#
|
34
|
+
# data - The data that should be encoded.
|
35
|
+
# schema_name - The name of a schema in the `schemas_path`.
|
36
|
+
# stream - An IO object that the encoded data should be written to (optional).
|
37
|
+
# codec - The String name of a codec that should be used to compress messages (optional).
|
38
|
+
#
|
39
|
+
# Currently, the only valid codec name is `deflate`.
|
40
|
+
#
|
41
|
+
# Returns nothing.
|
42
|
+
def encode_to_stream(data, schema_name:, stream:, namespace: @namespace)
|
43
|
+
schema = @schema_store.find(schema_name, namespace)
|
23
44
|
writer = Avro::IO::DatumWriter.new(schema)
|
24
45
|
|
25
|
-
|
26
|
-
dw = Avro::DataFile::Writer.new(io, writer, schema)
|
46
|
+
dw = Avro::DataFile::Writer.new(stream, writer, schema, @codec)
|
27
47
|
dw << data
|
28
48
|
dw.close
|
29
|
-
|
30
|
-
io.string
|
31
49
|
end
|
32
50
|
|
33
51
|
# Decodes Avro data.
|
@@ -35,49 +53,31 @@ class AvroTurf
|
|
35
53
|
# encoded_data - A String containing Avro-encoded data.
|
36
54
|
# schema_name - The String name of the schema that should be used to read
|
37
55
|
# the data. If nil, the writer schema will be used.
|
56
|
+
# namespace - The namespace of the Avro schema used to decode the data.
|
38
57
|
#
|
39
58
|
# Returns whatever is encoded in the data.
|
40
59
|
def decode(encoded_data, schema_name: nil, namespace: @namespace)
|
41
|
-
|
42
|
-
|
43
|
-
reader = Avro::IO::DatumReader.new(nil, schema)
|
44
|
-
dr = Avro::DataFile::Reader.new(io, reader)
|
45
|
-
dr.first
|
60
|
+
stream = StringIO.new(encoded_data)
|
61
|
+
decode_stream(stream, schema_name: schema_name, namespace: namespace)
|
46
62
|
end
|
47
63
|
|
48
|
-
|
49
|
-
|
50
|
-
# Resolves and returns a schema.
|
64
|
+
# Decodes Avro data from an IO stream.
|
51
65
|
#
|
52
|
-
#
|
66
|
+
# stream - An IO object containing Avro data.
|
67
|
+
# schema_name - The String name of the schema that should be used to read
|
68
|
+
# the data. If nil, the writer schema will be used.
|
69
|
+
# namespace - The namespace of the Avro schema used to decode the data.
|
53
70
|
#
|
54
|
-
# Returns
|
55
|
-
def
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
schema_path = File.join(@schemas_path, *namespace, schema_name + ".avsc")
|
62
|
-
schema_json = JSON.parse(File.read(schema_path))
|
63
|
-
schema = Avro::Schema.real_parse(schema_json, @schemas)
|
64
|
-
|
65
|
-
if schema.respond_to?(:fullname) && schema.fullname != fullname
|
66
|
-
raise SchemaError, "expected schema `#{schema_path}' to define type `#{fullname}'"
|
67
|
-
end
|
68
|
-
|
69
|
-
schema
|
70
|
-
rescue ::Avro::SchemaParseError => e
|
71
|
-
# This is a hack in order to figure out exactly which type was missing. The
|
72
|
-
# Avro gem ought to provide this data directly.
|
73
|
-
if e.to_s =~ /"([\w\.]+)" is not a schema we know about/
|
74
|
-
resolve_schema($1)
|
71
|
+
# Returns whatever is encoded in the stream.
|
72
|
+
def decode_stream(stream, schema_name: nil, namespace: @namespace)
|
73
|
+
schema = schema_name && @schema_store.find(schema_name, namespace)
|
74
|
+
reader = Avro::IO::DatumReader.new(nil, schema)
|
75
|
+
dr = Avro::DataFile::Reader.new(stream, reader)
|
76
|
+
dr.first
|
77
|
+
end
|
75
78
|
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
else
|
80
|
-
raise
|
81
|
-
end
|
79
|
+
# Loads all schema definition files in the `schemas_dir`.
|
80
|
+
def load_schemas!
|
81
|
+
@schema_store.load_schemas!
|
82
82
|
end
|
83
83
|
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
class AvroTurf::SchemaStore
|
2
|
+
def initialize(path:)
|
3
|
+
@path = path or raise "Please specify a schema path"
|
4
|
+
@schemas = Hash.new
|
5
|
+
end
|
6
|
+
|
7
|
+
# Resolves and returns a schema.
|
8
|
+
#
|
9
|
+
# schema_name - The String name of the schema to resolve.
|
10
|
+
#
|
11
|
+
# Returns an Avro::Schema.
|
12
|
+
def find(name, namespace = nil)
|
13
|
+
fullname = Avro::Name.make_fullname(name, namespace)
|
14
|
+
|
15
|
+
return @schemas[fullname] if @schemas.key?(fullname)
|
16
|
+
|
17
|
+
*namespace, schema_name = fullname.split(".")
|
18
|
+
schema_path = File.join(@path, *namespace, schema_name + ".avsc")
|
19
|
+
schema_json = JSON.parse(File.read(schema_path))
|
20
|
+
schema = Avro::Schema.real_parse(schema_json, @schemas)
|
21
|
+
|
22
|
+
if schema.respond_to?(:fullname) && schema.fullname != fullname
|
23
|
+
raise AvroTurf::SchemaError, "expected schema `#{schema_path}' to define type `#{fullname}'"
|
24
|
+
end
|
25
|
+
|
26
|
+
schema
|
27
|
+
rescue ::Avro::SchemaParseError => e
|
28
|
+
# This is a hack in order to figure out exactly which type was missing. The
|
29
|
+
# Avro gem ought to provide this data directly.
|
30
|
+
if e.to_s =~ /"([\w\.]+)" is not a schema we know about/
|
31
|
+
find($1)
|
32
|
+
|
33
|
+
# Re-resolve the original schema now that the dependency has been resolved.
|
34
|
+
@schemas.delete(fullname)
|
35
|
+
find(fullname)
|
36
|
+
else
|
37
|
+
raise
|
38
|
+
end
|
39
|
+
rescue Errno::ENOENT
|
40
|
+
raise AvroTurf::SchemaNotFoundError, "could not find Avro schema at `#{schema_path}'"
|
41
|
+
end
|
42
|
+
|
43
|
+
# Loads all schema definition files in the `schemas_dir`.
|
44
|
+
def load_schemas!
|
45
|
+
pattern = [@path, "**", "*.avsc"].join("/")
|
46
|
+
|
47
|
+
Dir.glob(pattern) do |schema_path|
|
48
|
+
# Remove the path prefix.
|
49
|
+
schema_path.sub!(/^\/#{@path}\//, "")
|
50
|
+
|
51
|
+
# Replace `/` with `.` and chop off the file extension.
|
52
|
+
schema_name = File.basename(schema_path.tr("/", "."), ".avsc")
|
53
|
+
|
54
|
+
# Load and cache the schema.
|
55
|
+
find(schema_name)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
data/lib/avro_turf/version.rb
CHANGED
data/perf/address.avsc
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
#
|
3
|
+
# Measures the encoded size of messages of increasing size.
|
4
|
+
|
5
|
+
$LOAD_PATH.unshift(File.expand_path("../lib", File.dirname(__FILE__)))
|
6
|
+
|
7
|
+
require 'benchmark'
|
8
|
+
require 'avro_turf'
|
9
|
+
|
10
|
+
sizes = [1, 10, 100, 1_000, 10_000]
|
11
|
+
avro = AvroTurf.new(schemas_path: File.dirname(__FILE__))
|
12
|
+
|
13
|
+
sizes.each do |size|
|
14
|
+
data = {
|
15
|
+
"name" => "John" * size,
|
16
|
+
"address" => {
|
17
|
+
"street" => "1st st." * size,
|
18
|
+
"city" => "Citytown" * size
|
19
|
+
}
|
20
|
+
}
|
21
|
+
|
22
|
+
result = avro.encode(data, schema_name: "person")
|
23
|
+
encoded_size = result.bytesize
|
24
|
+
encode_factor = result.bytesize / size.to_f
|
25
|
+
puts "size #{size}: #{encoded_size} bytes (encoding factor #{encode_factor})"
|
26
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
#
|
3
|
+
# Measures the time it takes to encode messages of increasing size.
|
4
|
+
|
5
|
+
$LOAD_PATH.unshift(File.expand_path("../lib", File.dirname(__FILE__)))
|
6
|
+
|
7
|
+
require 'benchmark'
|
8
|
+
require 'avro_turf'
|
9
|
+
|
10
|
+
# Number of iterations per run.
|
11
|
+
N = 10_000
|
12
|
+
|
13
|
+
Benchmark.bm(15) do |x|
|
14
|
+
sizes = [1, 10, 100, 1_000, 10_000]
|
15
|
+
avro = AvroTurf.new(schemas_path: File.dirname(__FILE__))
|
16
|
+
|
17
|
+
sizes.each do |size|
|
18
|
+
data = {
|
19
|
+
"name" => "John" * size,
|
20
|
+
"address" => {
|
21
|
+
"street" => "1st st." * size,
|
22
|
+
"city" => "Citytown" * size
|
23
|
+
}
|
24
|
+
}
|
25
|
+
|
26
|
+
x.report("size #{size}:") do
|
27
|
+
N.times { avro.encode(data, schema_name: "person") }
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
data/perf/person.avsc
ADDED
@@ -0,0 +1,125 @@
|
|
1
|
+
describe AvroTurf do
|
2
|
+
let(:avro) { AvroTurf.new(schemas_path: "spec/schemas/") }
|
3
|
+
|
4
|
+
before do
|
5
|
+
FileUtils.mkdir_p("spec/schemas")
|
6
|
+
end
|
7
|
+
|
8
|
+
describe "#encode" do
|
9
|
+
before do
|
10
|
+
define_schema "person.avsc", <<-AVSC
|
11
|
+
{
|
12
|
+
"name": "person",
|
13
|
+
"type": "record",
|
14
|
+
"fields": [
|
15
|
+
{
|
16
|
+
"type": "string",
|
17
|
+
"name": "full_name"
|
18
|
+
}
|
19
|
+
]
|
20
|
+
}
|
21
|
+
AVSC
|
22
|
+
end
|
23
|
+
|
24
|
+
it "encodes data with Avro" do
|
25
|
+
data = {
|
26
|
+
"full_name" => "John Doe"
|
27
|
+
}
|
28
|
+
|
29
|
+
encoded_data = avro.encode(data, schema_name: "person")
|
30
|
+
|
31
|
+
expect(avro.decode(encoded_data)).to eq(data)
|
32
|
+
end
|
33
|
+
|
34
|
+
it "allows specifying a codec that should be used to compress messages" do
|
35
|
+
compressed_avro = AvroTurf.new(schemas_path: "spec/schemas/", codec: "deflate")
|
36
|
+
|
37
|
+
data = {
|
38
|
+
"full_name" => "John Doe" * 100
|
39
|
+
}
|
40
|
+
|
41
|
+
uncompressed_data = avro.encode(data, schema_name: "person")
|
42
|
+
compressed_data = compressed_avro.encode(data, schema_name: "person")
|
43
|
+
|
44
|
+
expect(compressed_data.bytesize).to be < uncompressed_data.bytesize
|
45
|
+
expect(compressed_avro.decode(compressed_data)).to eq(data)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
describe "#decode" do
|
50
|
+
it "decodes Avro data using the inlined writer's schema" do
|
51
|
+
define_schema "message.avsc", <<-AVSC
|
52
|
+
{
|
53
|
+
"name": "message",
|
54
|
+
"type": "string"
|
55
|
+
}
|
56
|
+
AVSC
|
57
|
+
|
58
|
+
encoded_data = avro.encode("hello, world", schema_name: "message")
|
59
|
+
|
60
|
+
expect(avro.decode(encoded_data)).to eq "hello, world"
|
61
|
+
end
|
62
|
+
|
63
|
+
it "decodes Avro data using a specified reader's schema" do
|
64
|
+
FileUtils.mkdir_p("spec/schemas/reader")
|
65
|
+
|
66
|
+
define_schema "point.avsc", <<-AVSC
|
67
|
+
{
|
68
|
+
"name": "point",
|
69
|
+
"type": "record",
|
70
|
+
"fields": [
|
71
|
+
{ "name": "x", "type": "long" },
|
72
|
+
{ "name": "y", "type": "long" }
|
73
|
+
]
|
74
|
+
}
|
75
|
+
AVSC
|
76
|
+
|
77
|
+
define_schema "reader/point.avsc", <<-AVSC
|
78
|
+
{
|
79
|
+
"name": "point",
|
80
|
+
"type": "record",
|
81
|
+
"fields": [
|
82
|
+
{ "name": "x", "type": "long" }
|
83
|
+
]
|
84
|
+
}
|
85
|
+
AVSC
|
86
|
+
|
87
|
+
encoded_data = avro.encode({ "x" => 42, "y" => 13 }, schema_name: "point")
|
88
|
+
reader_avro = AvroTurf.new(schemas_path: "spec/schemas/reader")
|
89
|
+
|
90
|
+
expect(reader_avro.decode(encoded_data, schema_name: "point")).to eq({ "x" => 42 })
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
describe "#encode_to_stream" do
|
95
|
+
it "writes encoded data to an existing stream" do
|
96
|
+
define_schema "message.avsc", <<-AVSC
|
97
|
+
{
|
98
|
+
"name": "message",
|
99
|
+
"type": "string"
|
100
|
+
}
|
101
|
+
AVSC
|
102
|
+
|
103
|
+
stream = StringIO.new
|
104
|
+
avro.encode_to_stream("hello", stream: stream, schema_name: "message")
|
105
|
+
|
106
|
+
expect(avro.decode(stream.string)).to eq "hello"
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
describe "#decode_stream" do
|
111
|
+
it "decodes Avro data from a stream" do
|
112
|
+
define_schema "message.avsc", <<-AVSC
|
113
|
+
{
|
114
|
+
"name": "message",
|
115
|
+
"type": "string"
|
116
|
+
}
|
117
|
+
AVSC
|
118
|
+
|
119
|
+
encoded_data = avro.encode("hello", schema_name: "message")
|
120
|
+
stream = StringIO.new(encoded_data)
|
121
|
+
|
122
|
+
expect(avro.decode_stream(stream)).to eq "hello"
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
@@ -0,0 +1,253 @@
|
|
1
|
+
require 'avro_turf/schema_store'
|
2
|
+
|
3
|
+
describe AvroTurf::SchemaStore do
|
4
|
+
let(:store) { AvroTurf::SchemaStore.new(path: "spec/schemas") }
|
5
|
+
|
6
|
+
before do
|
7
|
+
FileUtils.mkdir_p("spec/schemas")
|
8
|
+
end
|
9
|
+
|
10
|
+
describe "#find" do
|
11
|
+
it "finds schemas on the file system" do
|
12
|
+
define_schema "message.avsc", <<-AVSC
|
13
|
+
{
|
14
|
+
"name": "message",
|
15
|
+
"type": "record",
|
16
|
+
"fields": [
|
17
|
+
{
|
18
|
+
"type": "string",
|
19
|
+
"name": "message"
|
20
|
+
}
|
21
|
+
]
|
22
|
+
}
|
23
|
+
AVSC
|
24
|
+
|
25
|
+
schema = store.find("message")
|
26
|
+
expect(schema.fullname).to eq "message"
|
27
|
+
end
|
28
|
+
|
29
|
+
it "resolves missing references" do
|
30
|
+
define_schema "person.avsc", <<-AVSC
|
31
|
+
{
|
32
|
+
"name": "person",
|
33
|
+
"type": "record",
|
34
|
+
"fields": [
|
35
|
+
{
|
36
|
+
"name": "address",
|
37
|
+
"type": "address"
|
38
|
+
}
|
39
|
+
]
|
40
|
+
}
|
41
|
+
AVSC
|
42
|
+
|
43
|
+
define_schema "address.avsc", <<-AVSC
|
44
|
+
{
|
45
|
+
"type": "record",
|
46
|
+
"name": "address",
|
47
|
+
"fields": []
|
48
|
+
}
|
49
|
+
AVSC
|
50
|
+
|
51
|
+
schema = store.find("person")
|
52
|
+
|
53
|
+
expect(schema.fullname).to eq "person"
|
54
|
+
end
|
55
|
+
|
56
|
+
it "finds namespaced schemas" do
|
57
|
+
FileUtils.mkdir_p("spec/schemas/test/people")
|
58
|
+
|
59
|
+
define_schema "test/people/person.avsc", <<-AVSC
|
60
|
+
{
|
61
|
+
"name": "person",
|
62
|
+
"namespace": "test.people",
|
63
|
+
"type": "record",
|
64
|
+
"fields": [
|
65
|
+
{
|
66
|
+
"name": "address",
|
67
|
+
"type": "test.people.address"
|
68
|
+
}
|
69
|
+
]
|
70
|
+
}
|
71
|
+
AVSC
|
72
|
+
|
73
|
+
define_schema "test/people/address.avsc", <<-AVSC
|
74
|
+
{
|
75
|
+
"name": "address",
|
76
|
+
"namespace": "test.people",
|
77
|
+
"type": "record",
|
78
|
+
"fields": []
|
79
|
+
}
|
80
|
+
AVSC
|
81
|
+
|
82
|
+
schema = store.find("person", "test.people")
|
83
|
+
|
84
|
+
expect(schema.fullname).to eq "test.people.person"
|
85
|
+
end
|
86
|
+
|
87
|
+
it "ignores the namespace when the name contains a dot" do
|
88
|
+
FileUtils.mkdir_p("spec/schemas/test/acme")
|
89
|
+
|
90
|
+
define_schema "test/acme/message.avsc", <<-AVSC
|
91
|
+
{
|
92
|
+
"name": "message",
|
93
|
+
"namespace": "test.acme",
|
94
|
+
"type": "record",
|
95
|
+
"fields": []
|
96
|
+
}
|
97
|
+
AVSC
|
98
|
+
|
99
|
+
schema = store.find("test.acme.message", "test.yolo")
|
100
|
+
|
101
|
+
expect(schema.fullname).to eq "test.acme.message"
|
102
|
+
end
|
103
|
+
|
104
|
+
it "raises AvroTurf::SchemaNotFoundError if there's no schema file matching the name" do
|
105
|
+
expect {
|
106
|
+
store.find("not_there")
|
107
|
+
}.to raise_error(AvroTurf::SchemaNotFoundError, "could not find Avro schema at `spec/schemas/not_there.avsc'")
|
108
|
+
end
|
109
|
+
|
110
|
+
it "raises AvroTurf::SchemaNotFoundError if a type reference cannot be resolved" do
|
111
|
+
define_schema "person.avsc", <<-AVSC
|
112
|
+
{
|
113
|
+
"name": "person",
|
114
|
+
"type": "record",
|
115
|
+
"fields": [
|
116
|
+
{
|
117
|
+
"name": "address",
|
118
|
+
"type": "address"
|
119
|
+
}
|
120
|
+
]
|
121
|
+
}
|
122
|
+
AVSC
|
123
|
+
|
124
|
+
expect {
|
125
|
+
store.find("person")
|
126
|
+
}.to raise_exception(AvroTurf::SchemaNotFoundError)
|
127
|
+
end
|
128
|
+
|
129
|
+
it "raises AvroTurf::SchemaError if the schema's namespace doesn't match the file location" do
|
130
|
+
FileUtils.mkdir_p("spec/schemas/test/people")
|
131
|
+
|
132
|
+
define_schema "test/people/person.avsc", <<-AVSC
|
133
|
+
{
|
134
|
+
"name": "person",
|
135
|
+
"namespace": "yoyoyo.nanana",
|
136
|
+
"type": "record",
|
137
|
+
"fields": []
|
138
|
+
}
|
139
|
+
AVSC
|
140
|
+
|
141
|
+
expect {
|
142
|
+
store.find("test.people.person")
|
143
|
+
}.to raise_error(AvroTurf::SchemaError, "expected schema `spec/schemas/test/people/person.avsc' to define type `test.people.person'")
|
144
|
+
end
|
145
|
+
|
146
|
+
it "handles circular dependencies" do
|
147
|
+
define_schema "a.avsc", <<-AVSC
|
148
|
+
{
|
149
|
+
"name": "a",
|
150
|
+
"type": "record",
|
151
|
+
"fields": [
|
152
|
+
{
|
153
|
+
"type": "b",
|
154
|
+
"name": "b"
|
155
|
+
}
|
156
|
+
]
|
157
|
+
}
|
158
|
+
AVSC
|
159
|
+
|
160
|
+
define_schema "b.avsc", <<-AVSC
|
161
|
+
{
|
162
|
+
"name": "b",
|
163
|
+
"type": "record",
|
164
|
+
"fields": [
|
165
|
+
{
|
166
|
+
"type": "a",
|
167
|
+
"name": "a"
|
168
|
+
}
|
169
|
+
]
|
170
|
+
}
|
171
|
+
AVSC
|
172
|
+
|
173
|
+
schema = store.find("a")
|
174
|
+
expect(schema.fullname).to eq "a"
|
175
|
+
end
|
176
|
+
|
177
|
+
it "caches schemas in memory" do
|
178
|
+
define_schema "person.avsc", <<-AVSC
|
179
|
+
{
|
180
|
+
"name": "person",
|
181
|
+
"type": "record",
|
182
|
+
"fields": [
|
183
|
+
{
|
184
|
+
"type": "string",
|
185
|
+
"name": "full_name"
|
186
|
+
}
|
187
|
+
]
|
188
|
+
}
|
189
|
+
AVSC
|
190
|
+
|
191
|
+
# Warm the schema cache.
|
192
|
+
store.find("person")
|
193
|
+
|
194
|
+
# Force a failure if the schema file is read again.
|
195
|
+
FileUtils.rm("spec/schemas/person.avsc")
|
196
|
+
|
197
|
+
schema = store.find("person")
|
198
|
+
expect(schema.fullname).to eq "person"
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
describe "#load_schemas!" do
|
203
|
+
it "loads schemas defined in the `schemas_path` directory" do
|
204
|
+
define_schema "person.avsc", <<-AVSC
|
205
|
+
{
|
206
|
+
"name": "person",
|
207
|
+
"type": "record",
|
208
|
+
"fields": [
|
209
|
+
{
|
210
|
+
"type": "string",
|
211
|
+
"name": "full_name"
|
212
|
+
}
|
213
|
+
]
|
214
|
+
}
|
215
|
+
AVSC
|
216
|
+
|
217
|
+
# Warm the schema cache.
|
218
|
+
store.load_schemas!
|
219
|
+
|
220
|
+
# Force a failure if the schema file is read again.
|
221
|
+
FileUtils.rm("spec/schemas/person.avsc")
|
222
|
+
|
223
|
+
schema = store.find("person")
|
224
|
+
expect(schema.fullname).to eq "person"
|
225
|
+
end
|
226
|
+
|
227
|
+
it "recursively finds schema definitions in subdirectories" do
|
228
|
+
FileUtils.mkdir_p("spec/schemas/foo/bar")
|
229
|
+
|
230
|
+
define_schema "foo/bar/person.avsc", <<-AVSC
|
231
|
+
{
|
232
|
+
"name": "foo.bar.person",
|
233
|
+
"type": "record",
|
234
|
+
"fields": [
|
235
|
+
{
|
236
|
+
"type": "string",
|
237
|
+
"name": "full_name"
|
238
|
+
}
|
239
|
+
]
|
240
|
+
}
|
241
|
+
AVSC
|
242
|
+
|
243
|
+
# Warm the schema cache.
|
244
|
+
store.load_schemas!
|
245
|
+
|
246
|
+
# Force a failure if the schema file is read again.
|
247
|
+
FileUtils.rm("spec/schemas/foo/bar/person.avsc")
|
248
|
+
|
249
|
+
schema = store.find("foo.bar.person")
|
250
|
+
expect(schema.fullname).to eq "foo.bar.person"
|
251
|
+
end
|
252
|
+
end
|
253
|
+
end
|
data/spec/spec_helper.rb
CHANGED
@@ -1,2 +1,16 @@
|
|
1
1
|
require 'bundler/setup'
|
2
|
+
require 'fakefs/spec_helpers'
|
2
3
|
require 'avro_turf'
|
4
|
+
|
5
|
+
module Helpers
|
6
|
+
def define_schema(path, content)
|
7
|
+
File.open(File.join("spec/schemas", path), "w") do |f|
|
8
|
+
f.write(content)
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
RSpec.configure do |config|
|
14
|
+
config.include FakeFS::SpecHelpers
|
15
|
+
config.include Helpers
|
16
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: avro_turf
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Schierbeck
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-06-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: avro
|
@@ -96,8 +96,14 @@ files:
|
|
96
96
|
- avro_turf.gemspec
|
97
97
|
- circle.yml
|
98
98
|
- lib/avro_turf.rb
|
99
|
+
- lib/avro_turf/schema_store.rb
|
99
100
|
- lib/avro_turf/version.rb
|
100
|
-
-
|
101
|
+
- perf/address.avsc
|
102
|
+
- perf/encoding_size.rb
|
103
|
+
- perf/encoding_speed.rb
|
104
|
+
- perf/person.avsc
|
105
|
+
- spec/avro_turf_spec.rb
|
106
|
+
- spec/schema_store_spec.rb
|
101
107
|
- spec/spec_helper.rb
|
102
108
|
homepage: https://github.com/dasch/avro_turf
|
103
109
|
licenses:
|
@@ -125,6 +131,7 @@ specification_version: 4
|
|
125
131
|
summary: A library that makes it easier to use the Avro serialization format from
|
126
132
|
Ruby
|
127
133
|
test_files:
|
128
|
-
- spec/
|
134
|
+
- spec/avro_turf_spec.rb
|
135
|
+
- spec/schema_store_spec.rb
|
129
136
|
- spec/spec_helper.rb
|
130
137
|
has_rdoc:
|
data/spec/avro_spec.rb
DELETED
@@ -1,227 +0,0 @@
|
|
1
|
-
require 'fakefs/spec_helpers'
|
2
|
-
|
3
|
-
describe AvroTurf do
|
4
|
-
include FakeFS::SpecHelpers
|
5
|
-
|
6
|
-
let(:avro) { AvroTurf.new(schemas_path: "spec/schemas/") }
|
7
|
-
|
8
|
-
before do
|
9
|
-
FileUtils.mkdir_p("spec/schemas")
|
10
|
-
end
|
11
|
-
|
12
|
-
it "encodes and decodes data using a named schema" do
|
13
|
-
define_schema "person.avsc", <<-AVSC
|
14
|
-
{
|
15
|
-
"name": "person",
|
16
|
-
"type": "record",
|
17
|
-
"fields": [
|
18
|
-
{
|
19
|
-
"type": "string",
|
20
|
-
"name": "full_name"
|
21
|
-
},
|
22
|
-
{
|
23
|
-
"name": "address",
|
24
|
-
"type": {
|
25
|
-
"type": "record",
|
26
|
-
"name": "address",
|
27
|
-
"fields": [
|
28
|
-
{
|
29
|
-
"type": "string",
|
30
|
-
"name": "street"
|
31
|
-
},
|
32
|
-
{
|
33
|
-
"type": "string",
|
34
|
-
"name": "city"
|
35
|
-
}
|
36
|
-
]
|
37
|
-
}
|
38
|
-
}
|
39
|
-
]
|
40
|
-
}
|
41
|
-
AVSC
|
42
|
-
|
43
|
-
data = {
|
44
|
-
"full_name" => "John Doe",
|
45
|
-
"address" => {
|
46
|
-
"street" => "Market st. 989",
|
47
|
-
"city" => "San Francisco"
|
48
|
-
}
|
49
|
-
}
|
50
|
-
|
51
|
-
encoded_data = avro.encode(data, schema_name: "person")
|
52
|
-
|
53
|
-
expect(avro.decode(encoded_data, schema_name: "person")).to eq(data)
|
54
|
-
end
|
55
|
-
|
56
|
-
it "resolves named types" do
|
57
|
-
define_schema "person.avsc", <<-AVSC
|
58
|
-
{
|
59
|
-
"name": "person",
|
60
|
-
"type": "record",
|
61
|
-
"fields": [
|
62
|
-
{
|
63
|
-
"type": "string",
|
64
|
-
"name": "full_name"
|
65
|
-
},
|
66
|
-
{
|
67
|
-
"name": "address",
|
68
|
-
"type": "address"
|
69
|
-
}
|
70
|
-
]
|
71
|
-
}
|
72
|
-
AVSC
|
73
|
-
|
74
|
-
define_schema "address.avsc", <<-AVSC
|
75
|
-
{
|
76
|
-
"type": "record",
|
77
|
-
"name": "address",
|
78
|
-
"fields": [
|
79
|
-
{
|
80
|
-
"type": "string",
|
81
|
-
"name": "street"
|
82
|
-
},
|
83
|
-
{
|
84
|
-
"type": "string",
|
85
|
-
"name": "city"
|
86
|
-
}
|
87
|
-
]
|
88
|
-
}
|
89
|
-
AVSC
|
90
|
-
|
91
|
-
data = {
|
92
|
-
"full_name" => "John Doe",
|
93
|
-
"address" => {
|
94
|
-
"street" => "Market st. 989",
|
95
|
-
"city" => "San Francisco"
|
96
|
-
}
|
97
|
-
}
|
98
|
-
|
99
|
-
encoded_data = avro.encode(data, schema_name: "person")
|
100
|
-
|
101
|
-
expect(avro.decode(encoded_data, schema_name: "person")).to eq(data)
|
102
|
-
end
|
103
|
-
|
104
|
-
it "allows decoding without specifying a reader schema" do
|
105
|
-
define_schema "message.avsc", <<-AVSC
|
106
|
-
{
|
107
|
-
"name": "message",
|
108
|
-
"type": "string"
|
109
|
-
}
|
110
|
-
AVSC
|
111
|
-
|
112
|
-
encoded_data = avro.encode("hello, world", schema_name: "message")
|
113
|
-
|
114
|
-
expect(avro.decode(encoded_data)).to eq "hello, world"
|
115
|
-
end
|
116
|
-
|
117
|
-
it "allows using namespaces in schemas" do
|
118
|
-
FileUtils.mkdir_p("spec/schemas/test/people")
|
119
|
-
|
120
|
-
define_schema "test/people/person.avsc", <<-AVSC
|
121
|
-
{
|
122
|
-
"name": "person",
|
123
|
-
"namespace": "test.people",
|
124
|
-
"type": "record",
|
125
|
-
"fields": [
|
126
|
-
{
|
127
|
-
"type": "string",
|
128
|
-
"name": "full_name"
|
129
|
-
},
|
130
|
-
{
|
131
|
-
"name": "address",
|
132
|
-
"type": "test.people.address"
|
133
|
-
}
|
134
|
-
]
|
135
|
-
}
|
136
|
-
AVSC
|
137
|
-
|
138
|
-
define_schema "test/people/address.avsc", <<-AVSC
|
139
|
-
{
|
140
|
-
"name": "address",
|
141
|
-
"namespace": "test.people",
|
142
|
-
"type": "record",
|
143
|
-
"fields": [
|
144
|
-
{
|
145
|
-
"type": "string",
|
146
|
-
"name": "street"
|
147
|
-
},
|
148
|
-
{
|
149
|
-
"type": "string",
|
150
|
-
"name": "city"
|
151
|
-
}
|
152
|
-
]
|
153
|
-
}
|
154
|
-
AVSC
|
155
|
-
|
156
|
-
data = {
|
157
|
-
"full_name" => "John Doe",
|
158
|
-
"address" => {
|
159
|
-
"street" => "Market st. 989",
|
160
|
-
"city" => "San Francisco"
|
161
|
-
}
|
162
|
-
}
|
163
|
-
|
164
|
-
encoded_data = avro.encode(data, schema_name: "person", namespace: "test.people")
|
165
|
-
|
166
|
-
expect(avro.decode(encoded_data, schema_name: "person", namespace: "test.people")).to eq(data)
|
167
|
-
end
|
168
|
-
|
169
|
-
it "raises AvroTurf::SchemaError if the schema's namespace doesn't match the file location" do
|
170
|
-
FileUtils.mkdir_p("spec/schemas/test/people")
|
171
|
-
|
172
|
-
define_schema "test/people/person.avsc", <<-AVSC
|
173
|
-
{
|
174
|
-
"name": "person",
|
175
|
-
"namespace": "yoyoyo.nanana",
|
176
|
-
"type": "record",
|
177
|
-
"fields": [
|
178
|
-
{
|
179
|
-
"type": "string",
|
180
|
-
"name": "full_name"
|
181
|
-
}
|
182
|
-
]
|
183
|
-
}
|
184
|
-
AVSC
|
185
|
-
|
186
|
-
data = { "full_name" => "John Doe" }
|
187
|
-
|
188
|
-
expect {
|
189
|
-
avro.encode(data, schema_name: "test.people.person")
|
190
|
-
}.to raise_error(AvroTurf::SchemaError, "expected schema `spec/schemas/test/people/person.avsc' to define type `test.people.person'")
|
191
|
-
end
|
192
|
-
|
193
|
-
it "caches schemas in memory" do
|
194
|
-
define_schema "person.avsc", <<-AVSC
|
195
|
-
{
|
196
|
-
"name": "person",
|
197
|
-
"type": "record",
|
198
|
-
"fields": [
|
199
|
-
{
|
200
|
-
"type": "string",
|
201
|
-
"name": "full_name"
|
202
|
-
}
|
203
|
-
]
|
204
|
-
}
|
205
|
-
AVSC
|
206
|
-
|
207
|
-
data = {
|
208
|
-
"full_name" => "John Doe"
|
209
|
-
}
|
210
|
-
|
211
|
-
# Warm the schema cache.
|
212
|
-
avro.encode(data, schema_name: "person")
|
213
|
-
|
214
|
-
# Force a failure if the schema file is read again.
|
215
|
-
FileUtils.rm("spec/schemas/person.avsc")
|
216
|
-
|
217
|
-
encoded_data = avro.encode(data, schema_name: "person")
|
218
|
-
|
219
|
-
expect(avro.decode(encoded_data, schema_name: "person")).to eq(data)
|
220
|
-
end
|
221
|
-
|
222
|
-
def define_schema(path, content)
|
223
|
-
File.open(File.join("spec/schemas", path), "w") do |f|
|
224
|
-
f.write(content)
|
225
|
-
end
|
226
|
-
end
|
227
|
-
end
|