avro_turf 0.2.0 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 64620bcf8c0a04337ecf571abe2d0e51098fce4a
4
- data.tar.gz: cc45ee47d74ea7ee364463b105f8cfc92345058b
3
+ metadata.gz: 0845fc0e81baaa72df5abd34b47a0ce6d06a853f
4
+ data.tar.gz: 434aaa74efb2f96437045a8705c7889348bec9c3
5
5
  SHA512:
6
- metadata.gz: 090cf294b97ea232c5ec8372eced0244d87f87a6fd3d9cdffdfb2ad73e467e52be0ae1ab3108e53a3e5aabfdd7a3bd2a35aae70c89ca1b448a74305d61d48964
7
- data.tar.gz: 657f8b32c84d5b6e01a614bab06a4d17987e3c1832641e62bac8548c139810e60febfc72684983b1342e4839c889b667d037fcbb09932c940366636e32198c5f
6
+ metadata.gz: ef023ba8d9fb413e69a467c1ecb4a4ef73c6c1f1bb86f1f362b6f351abcd6ed8b2fae965d7b50b4d3ba6e81513c30dcffbe1e5d688044efa872f4b65f85ff39a
7
+ data.tar.gz: e095b79ce3f37a69efac8537e6d59fcb5c8d66350604eba7f00166911801699eccf945552fcbc5c950223f7c14ddfddd2f58157b70aff2466e6d2ea157485eab
data/Gemfile CHANGED
@@ -2,3 +2,6 @@ source 'https://rubygems.org'
2
2
 
3
3
  # Specify your gem's dependencies in avro_turf.gemspec
4
4
  gemspec
5
+
6
+ # Used by CircleCI to format RSpec results.
7
+ gem 'rspec_junit_formatter', :git => 'git@github.com:circleci/rspec_junit_formatter.git'
data/lib/avro_turf.rb CHANGED
@@ -1,15 +1,17 @@
1
1
  require 'avro_turf/version'
2
2
  require 'avro'
3
3
  require 'json'
4
+ require 'avro_turf/schema_store'
4
5
 
5
6
  class AvroTurf
6
7
  class Error < StandardError; end
7
8
  class SchemaError < Error; end
9
+ class SchemaNotFoundError < Error; end
8
10
 
9
- def initialize(schemas_path:, namespace: nil)
10
- @schemas_path = schemas_path or raise "Please specify a schema path"
11
- @schemas = Hash.new
11
+ def initialize(schemas_path:, namespace: nil, codec: nil)
12
12
  @namespace = namespace
13
+ @schema_store = SchemaStore.new(path: schemas_path)
14
+ @codec = codec
13
15
  end
14
16
 
15
17
  # Encodes data to Avro using the specified schema.
@@ -19,15 +21,31 @@ class AvroTurf
19
21
  #
20
22
  # Returns a String containing the encoded data.
21
23
  def encode(data, schema_name:, namespace: @namespace)
22
- schema = resolve_schema(schema_name, namespace)
24
+ stream = StringIO.new
25
+
26
+ encode_to_stream(data, stream: stream, schema_name: schema_name, namespace: namespace)
27
+
28
+ stream.string
29
+ end
30
+
31
+ # Encodes data to Avro using the specified schema and writes it to the
32
+ # specified stream.
33
+ #
34
+ # data - The data that should be encoded.
35
+ # schema_name - The name of a schema in the `schemas_path`.
36
+ # stream - An IO object that the encoded data should be written to (optional).
37
+ # codec - The String name of a codec that should be used to compress messages (optional).
38
+ #
39
+ # Currently, the only valid codec name is `deflate`.
40
+ #
41
+ # Returns nothing.
42
+ def encode_to_stream(data, schema_name:, stream:, namespace: @namespace)
43
+ schema = @schema_store.find(schema_name, namespace)
23
44
  writer = Avro::IO::DatumWriter.new(schema)
24
45
 
25
- io = StringIO.new
26
- dw = Avro::DataFile::Writer.new(io, writer, schema)
46
+ dw = Avro::DataFile::Writer.new(stream, writer, schema, @codec)
27
47
  dw << data
28
48
  dw.close
29
-
30
- io.string
31
49
  end
32
50
 
33
51
  # Decodes Avro data.
@@ -35,49 +53,31 @@ class AvroTurf
35
53
  # encoded_data - A String containing Avro-encoded data.
36
54
  # schema_name - The String name of the schema that should be used to read
37
55
  # the data. If nil, the writer schema will be used.
56
+ # namespace - The namespace of the Avro schema used to decode the data.
38
57
  #
39
58
  # Returns whatever is encoded in the data.
40
59
  def decode(encoded_data, schema_name: nil, namespace: @namespace)
41
- io = StringIO.new(encoded_data)
42
- schema = schema_name && resolve_schema(schema_name, namespace)
43
- reader = Avro::IO::DatumReader.new(nil, schema)
44
- dr = Avro::DataFile::Reader.new(io, reader)
45
- dr.first
60
+ stream = StringIO.new(encoded_data)
61
+ decode_stream(stream, schema_name: schema_name, namespace: namespace)
46
62
  end
47
63
 
48
- private
49
-
50
- # Resolves and returns a schema.
64
+ # Decodes Avro data from an IO stream.
51
65
  #
52
- # schema_name - The String name of the schema to resolve.
66
+ # stream - An IO object containing Avro data.
67
+ # schema_name - The String name of the schema that should be used to read
68
+ # the data. If nil, the writer schema will be used.
69
+ # namespace - The namespace of the Avro schema used to decode the data.
53
70
  #
54
- # Returns an Avro::Schema.
55
- def resolve_schema(name, namespace = nil)
56
- fullname = Avro::Name.make_fullname(name, namespace)
57
-
58
- return @schemas[fullname] if @schemas.key?(fullname)
59
-
60
- *namespace, schema_name = fullname.split(".")
61
- schema_path = File.join(@schemas_path, *namespace, schema_name + ".avsc")
62
- schema_json = JSON.parse(File.read(schema_path))
63
- schema = Avro::Schema.real_parse(schema_json, @schemas)
64
-
65
- if schema.respond_to?(:fullname) && schema.fullname != fullname
66
- raise SchemaError, "expected schema `#{schema_path}' to define type `#{fullname}'"
67
- end
68
-
69
- schema
70
- rescue ::Avro::SchemaParseError => e
71
- # This is a hack in order to figure out exactly which type was missing. The
72
- # Avro gem ought to provide this data directly.
73
- if e.to_s =~ /"([\w\.]+)" is not a schema we know about/
74
- resolve_schema($1)
71
+ # Returns whatever is encoded in the stream.
72
+ def decode_stream(stream, schema_name: nil, namespace: @namespace)
73
+ schema = schema_name && @schema_store.find(schema_name, namespace)
74
+ reader = Avro::IO::DatumReader.new(nil, schema)
75
+ dr = Avro::DataFile::Reader.new(stream, reader)
76
+ dr.first
77
+ end
75
78
 
76
- # Re-resolve the original schema now that the dependency has been resolved.
77
- @schemas.delete(fullname)
78
- resolve_schema(fullname)
79
- else
80
- raise
81
- end
79
+ # Loads all schema definition files in the `schemas_dir`.
80
+ def load_schemas!
81
+ @schema_store.load_schemas!
82
82
  end
83
83
  end
@@ -0,0 +1,58 @@
1
+ class AvroTurf::SchemaStore
2
+ def initialize(path:)
3
+ @path = path or raise "Please specify a schema path"
4
+ @schemas = Hash.new
5
+ end
6
+
7
+ # Resolves and returns a schema.
8
+ #
9
+ # schema_name - The String name of the schema to resolve.
10
+ #
11
+ # Returns an Avro::Schema.
12
+ def find(name, namespace = nil)
13
+ fullname = Avro::Name.make_fullname(name, namespace)
14
+
15
+ return @schemas[fullname] if @schemas.key?(fullname)
16
+
17
+ *namespace, schema_name = fullname.split(".")
18
+ schema_path = File.join(@path, *namespace, schema_name + ".avsc")
19
+ schema_json = JSON.parse(File.read(schema_path))
20
+ schema = Avro::Schema.real_parse(schema_json, @schemas)
21
+
22
+ if schema.respond_to?(:fullname) && schema.fullname != fullname
23
+ raise AvroTurf::SchemaError, "expected schema `#{schema_path}' to define type `#{fullname}'"
24
+ end
25
+
26
+ schema
27
+ rescue ::Avro::SchemaParseError => e
28
+ # This is a hack in order to figure out exactly which type was missing. The
29
+ # Avro gem ought to provide this data directly.
30
+ if e.to_s =~ /"([\w\.]+)" is not a schema we know about/
31
+ find($1)
32
+
33
+ # Re-resolve the original schema now that the dependency has been resolved.
34
+ @schemas.delete(fullname)
35
+ find(fullname)
36
+ else
37
+ raise
38
+ end
39
+ rescue Errno::ENOENT
40
+ raise AvroTurf::SchemaNotFoundError, "could not find Avro schema at `#{schema_path}'"
41
+ end
42
+
43
+ # Loads all schema definition files in the `schemas_dir`.
44
+ def load_schemas!
45
+ pattern = [@path, "**", "*.avsc"].join("/")
46
+
47
+ Dir.glob(pattern) do |schema_path|
48
+ # Remove the path prefix.
49
+ schema_path.sub!(/^\/#{@path}\//, "")
50
+
51
+ # Replace `/` with `.` and chop off the file extension.
52
+ schema_name = File.basename(schema_path.tr("/", "."), ".avsc")
53
+
54
+ # Load and cache the schema.
55
+ find(schema_name)
56
+ end
57
+ end
58
+ end
@@ -1,3 +1,3 @@
1
1
  class AvroTurf
2
- VERSION = "0.2.0"
2
+ VERSION = "0.3.0"
3
3
  end
data/perf/address.avsc ADDED
@@ -0,0 +1,14 @@
1
+ {
2
+ "name": "address",
3
+ "type": "record",
4
+ "fields": [
5
+ {
6
+ "name": "street",
7
+ "type": "string"
8
+ },
9
+ {
10
+ "name": "city",
11
+ "type": "string"
12
+ }
13
+ ]
14
+ }
@@ -0,0 +1,26 @@
1
+ #!/usr/bin/env ruby
2
+ #
3
+ # Measures the encoded size of messages of increasing size.
4
+
5
+ $LOAD_PATH.unshift(File.expand_path("../lib", File.dirname(__FILE__)))
6
+
7
+ require 'benchmark'
8
+ require 'avro_turf'
9
+
10
+ sizes = [1, 10, 100, 1_000, 10_000]
11
+ avro = AvroTurf.new(schemas_path: File.dirname(__FILE__))
12
+
13
+ sizes.each do |size|
14
+ data = {
15
+ "name" => "John" * size,
16
+ "address" => {
17
+ "street" => "1st st." * size,
18
+ "city" => "Citytown" * size
19
+ }
20
+ }
21
+
22
+ result = avro.encode(data, schema_name: "person")
23
+ encoded_size = result.bytesize
24
+ encode_factor = result.bytesize / size.to_f
25
+ puts "size #{size}: #{encoded_size} bytes (encoding factor #{encode_factor})"
26
+ end
@@ -0,0 +1,30 @@
1
+ #!/usr/bin/env ruby
2
+ #
3
+ # Measures the time it takes to encode messages of increasing size.
4
+
5
+ $LOAD_PATH.unshift(File.expand_path("../lib", File.dirname(__FILE__)))
6
+
7
+ require 'benchmark'
8
+ require 'avro_turf'
9
+
10
+ # Number of iterations per run.
11
+ N = 10_000
12
+
13
+ Benchmark.bm(15) do |x|
14
+ sizes = [1, 10, 100, 1_000, 10_000]
15
+ avro = AvroTurf.new(schemas_path: File.dirname(__FILE__))
16
+
17
+ sizes.each do |size|
18
+ data = {
19
+ "name" => "John" * size,
20
+ "address" => {
21
+ "street" => "1st st." * size,
22
+ "city" => "Citytown" * size
23
+ }
24
+ }
25
+
26
+ x.report("size #{size}:") do
27
+ N.times { avro.encode(data, schema_name: "person") }
28
+ end
29
+ end
30
+ end
data/perf/person.avsc ADDED
@@ -0,0 +1,14 @@
1
+ {
2
+ "name": "person",
3
+ "type": "record",
4
+ "fields": [
5
+ {
6
+ "name": "name",
7
+ "type": "string"
8
+ },
9
+ {
10
+ "name": "address",
11
+ "type": "address"
12
+ }
13
+ ]
14
+ }
@@ -0,0 +1,125 @@
1
+ describe AvroTurf do
2
+ let(:avro) { AvroTurf.new(schemas_path: "spec/schemas/") }
3
+
4
+ before do
5
+ FileUtils.mkdir_p("spec/schemas")
6
+ end
7
+
8
+ describe "#encode" do
9
+ before do
10
+ define_schema "person.avsc", <<-AVSC
11
+ {
12
+ "name": "person",
13
+ "type": "record",
14
+ "fields": [
15
+ {
16
+ "type": "string",
17
+ "name": "full_name"
18
+ }
19
+ ]
20
+ }
21
+ AVSC
22
+ end
23
+
24
+ it "encodes data with Avro" do
25
+ data = {
26
+ "full_name" => "John Doe"
27
+ }
28
+
29
+ encoded_data = avro.encode(data, schema_name: "person")
30
+
31
+ expect(avro.decode(encoded_data)).to eq(data)
32
+ end
33
+
34
+ it "allows specifying a codec that should be used to compress messages" do
35
+ compressed_avro = AvroTurf.new(schemas_path: "spec/schemas/", codec: "deflate")
36
+
37
+ data = {
38
+ "full_name" => "John Doe" * 100
39
+ }
40
+
41
+ uncompressed_data = avro.encode(data, schema_name: "person")
42
+ compressed_data = compressed_avro.encode(data, schema_name: "person")
43
+
44
+ expect(compressed_data.bytesize).to be < uncompressed_data.bytesize
45
+ expect(compressed_avro.decode(compressed_data)).to eq(data)
46
+ end
47
+ end
48
+
49
+ describe "#decode" do
50
+ it "decodes Avro data using the inlined writer's schema" do
51
+ define_schema "message.avsc", <<-AVSC
52
+ {
53
+ "name": "message",
54
+ "type": "string"
55
+ }
56
+ AVSC
57
+
58
+ encoded_data = avro.encode("hello, world", schema_name: "message")
59
+
60
+ expect(avro.decode(encoded_data)).to eq "hello, world"
61
+ end
62
+
63
+ it "decodes Avro data using a specified reader's schema" do
64
+ FileUtils.mkdir_p("spec/schemas/reader")
65
+
66
+ define_schema "point.avsc", <<-AVSC
67
+ {
68
+ "name": "point",
69
+ "type": "record",
70
+ "fields": [
71
+ { "name": "x", "type": "long" },
72
+ { "name": "y", "type": "long" }
73
+ ]
74
+ }
75
+ AVSC
76
+
77
+ define_schema "reader/point.avsc", <<-AVSC
78
+ {
79
+ "name": "point",
80
+ "type": "record",
81
+ "fields": [
82
+ { "name": "x", "type": "long" }
83
+ ]
84
+ }
85
+ AVSC
86
+
87
+ encoded_data = avro.encode({ "x" => 42, "y" => 13 }, schema_name: "point")
88
+ reader_avro = AvroTurf.new(schemas_path: "spec/schemas/reader")
89
+
90
+ expect(reader_avro.decode(encoded_data, schema_name: "point")).to eq({ "x" => 42 })
91
+ end
92
+ end
93
+
94
+ describe "#encode_to_stream" do
95
+ it "writes encoded data to an existing stream" do
96
+ define_schema "message.avsc", <<-AVSC
97
+ {
98
+ "name": "message",
99
+ "type": "string"
100
+ }
101
+ AVSC
102
+
103
+ stream = StringIO.new
104
+ avro.encode_to_stream("hello", stream: stream, schema_name: "message")
105
+
106
+ expect(avro.decode(stream.string)).to eq "hello"
107
+ end
108
+ end
109
+
110
+ describe "#decode_stream" do
111
+ it "decodes Avro data from a stream" do
112
+ define_schema "message.avsc", <<-AVSC
113
+ {
114
+ "name": "message",
115
+ "type": "string"
116
+ }
117
+ AVSC
118
+
119
+ encoded_data = avro.encode("hello", schema_name: "message")
120
+ stream = StringIO.new(encoded_data)
121
+
122
+ expect(avro.decode_stream(stream)).to eq "hello"
123
+ end
124
+ end
125
+ end
@@ -0,0 +1,253 @@
1
+ require 'avro_turf/schema_store'
2
+
3
+ describe AvroTurf::SchemaStore do
4
+ let(:store) { AvroTurf::SchemaStore.new(path: "spec/schemas") }
5
+
6
+ before do
7
+ FileUtils.mkdir_p("spec/schemas")
8
+ end
9
+
10
+ describe "#find" do
11
+ it "finds schemas on the file system" do
12
+ define_schema "message.avsc", <<-AVSC
13
+ {
14
+ "name": "message",
15
+ "type": "record",
16
+ "fields": [
17
+ {
18
+ "type": "string",
19
+ "name": "message"
20
+ }
21
+ ]
22
+ }
23
+ AVSC
24
+
25
+ schema = store.find("message")
26
+ expect(schema.fullname).to eq "message"
27
+ end
28
+
29
+ it "resolves missing references" do
30
+ define_schema "person.avsc", <<-AVSC
31
+ {
32
+ "name": "person",
33
+ "type": "record",
34
+ "fields": [
35
+ {
36
+ "name": "address",
37
+ "type": "address"
38
+ }
39
+ ]
40
+ }
41
+ AVSC
42
+
43
+ define_schema "address.avsc", <<-AVSC
44
+ {
45
+ "type": "record",
46
+ "name": "address",
47
+ "fields": []
48
+ }
49
+ AVSC
50
+
51
+ schema = store.find("person")
52
+
53
+ expect(schema.fullname).to eq "person"
54
+ end
55
+
56
+ it "finds namespaced schemas" do
57
+ FileUtils.mkdir_p("spec/schemas/test/people")
58
+
59
+ define_schema "test/people/person.avsc", <<-AVSC
60
+ {
61
+ "name": "person",
62
+ "namespace": "test.people",
63
+ "type": "record",
64
+ "fields": [
65
+ {
66
+ "name": "address",
67
+ "type": "test.people.address"
68
+ }
69
+ ]
70
+ }
71
+ AVSC
72
+
73
+ define_schema "test/people/address.avsc", <<-AVSC
74
+ {
75
+ "name": "address",
76
+ "namespace": "test.people",
77
+ "type": "record",
78
+ "fields": []
79
+ }
80
+ AVSC
81
+
82
+ schema = store.find("person", "test.people")
83
+
84
+ expect(schema.fullname).to eq "test.people.person"
85
+ end
86
+
87
+ it "ignores the namespace when the name contains a dot" do
88
+ FileUtils.mkdir_p("spec/schemas/test/acme")
89
+
90
+ define_schema "test/acme/message.avsc", <<-AVSC
91
+ {
92
+ "name": "message",
93
+ "namespace": "test.acme",
94
+ "type": "record",
95
+ "fields": []
96
+ }
97
+ AVSC
98
+
99
+ schema = store.find("test.acme.message", "test.yolo")
100
+
101
+ expect(schema.fullname).to eq "test.acme.message"
102
+ end
103
+
104
+ it "raises AvroTurf::SchemaNotFoundError if there's no schema file matching the name" do
105
+ expect {
106
+ store.find("not_there")
107
+ }.to raise_error(AvroTurf::SchemaNotFoundError, "could not find Avro schema at `spec/schemas/not_there.avsc'")
108
+ end
109
+
110
+ it "raises AvroTurf::SchemaNotFoundError if a type reference cannot be resolved" do
111
+ define_schema "person.avsc", <<-AVSC
112
+ {
113
+ "name": "person",
114
+ "type": "record",
115
+ "fields": [
116
+ {
117
+ "name": "address",
118
+ "type": "address"
119
+ }
120
+ ]
121
+ }
122
+ AVSC
123
+
124
+ expect {
125
+ store.find("person")
126
+ }.to raise_exception(AvroTurf::SchemaNotFoundError)
127
+ end
128
+
129
+ it "raises AvroTurf::SchemaError if the schema's namespace doesn't match the file location" do
130
+ FileUtils.mkdir_p("spec/schemas/test/people")
131
+
132
+ define_schema "test/people/person.avsc", <<-AVSC
133
+ {
134
+ "name": "person",
135
+ "namespace": "yoyoyo.nanana",
136
+ "type": "record",
137
+ "fields": []
138
+ }
139
+ AVSC
140
+
141
+ expect {
142
+ store.find("test.people.person")
143
+ }.to raise_error(AvroTurf::SchemaError, "expected schema `spec/schemas/test/people/person.avsc' to define type `test.people.person'")
144
+ end
145
+
146
+ it "handles circular dependencies" do
147
+ define_schema "a.avsc", <<-AVSC
148
+ {
149
+ "name": "a",
150
+ "type": "record",
151
+ "fields": [
152
+ {
153
+ "type": "b",
154
+ "name": "b"
155
+ }
156
+ ]
157
+ }
158
+ AVSC
159
+
160
+ define_schema "b.avsc", <<-AVSC
161
+ {
162
+ "name": "b",
163
+ "type": "record",
164
+ "fields": [
165
+ {
166
+ "type": "a",
167
+ "name": "a"
168
+ }
169
+ ]
170
+ }
171
+ AVSC
172
+
173
+ schema = store.find("a")
174
+ expect(schema.fullname).to eq "a"
175
+ end
176
+
177
+ it "caches schemas in memory" do
178
+ define_schema "person.avsc", <<-AVSC
179
+ {
180
+ "name": "person",
181
+ "type": "record",
182
+ "fields": [
183
+ {
184
+ "type": "string",
185
+ "name": "full_name"
186
+ }
187
+ ]
188
+ }
189
+ AVSC
190
+
191
+ # Warm the schema cache.
192
+ store.find("person")
193
+
194
+ # Force a failure if the schema file is read again.
195
+ FileUtils.rm("spec/schemas/person.avsc")
196
+
197
+ schema = store.find("person")
198
+ expect(schema.fullname).to eq "person"
199
+ end
200
+ end
201
+
202
+ describe "#load_schemas!" do
203
+ it "loads schemas defined in the `schemas_path` directory" do
204
+ define_schema "person.avsc", <<-AVSC
205
+ {
206
+ "name": "person",
207
+ "type": "record",
208
+ "fields": [
209
+ {
210
+ "type": "string",
211
+ "name": "full_name"
212
+ }
213
+ ]
214
+ }
215
+ AVSC
216
+
217
+ # Warm the schema cache.
218
+ store.load_schemas!
219
+
220
+ # Force a failure if the schema file is read again.
221
+ FileUtils.rm("spec/schemas/person.avsc")
222
+
223
+ schema = store.find("person")
224
+ expect(schema.fullname).to eq "person"
225
+ end
226
+
227
+ it "recursively finds schema definitions in subdirectories" do
228
+ FileUtils.mkdir_p("spec/schemas/foo/bar")
229
+
230
+ define_schema "foo/bar/person.avsc", <<-AVSC
231
+ {
232
+ "name": "foo.bar.person",
233
+ "type": "record",
234
+ "fields": [
235
+ {
236
+ "type": "string",
237
+ "name": "full_name"
238
+ }
239
+ ]
240
+ }
241
+ AVSC
242
+
243
+ # Warm the schema cache.
244
+ store.load_schemas!
245
+
246
+ # Force a failure if the schema file is read again.
247
+ FileUtils.rm("spec/schemas/foo/bar/person.avsc")
248
+
249
+ schema = store.find("foo.bar.person")
250
+ expect(schema.fullname).to eq "foo.bar.person"
251
+ end
252
+ end
253
+ end
data/spec/spec_helper.rb CHANGED
@@ -1,2 +1,16 @@
1
1
  require 'bundler/setup'
2
+ require 'fakefs/spec_helpers'
2
3
  require 'avro_turf'
4
+
5
+ module Helpers
6
+ def define_schema(path, content)
7
+ File.open(File.join("spec/schemas", path), "w") do |f|
8
+ f.write(content)
9
+ end
10
+ end
11
+ end
12
+
13
+ RSpec.configure do |config|
14
+ config.include FakeFS::SpecHelpers
15
+ config.include Helpers
16
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: avro_turf
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Daniel Schierbeck
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-03-09 00:00:00.000000000 Z
11
+ date: 2015-06-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: avro
@@ -96,8 +96,14 @@ files:
96
96
  - avro_turf.gemspec
97
97
  - circle.yml
98
98
  - lib/avro_turf.rb
99
+ - lib/avro_turf/schema_store.rb
99
100
  - lib/avro_turf/version.rb
100
- - spec/avro_spec.rb
101
+ - perf/address.avsc
102
+ - perf/encoding_size.rb
103
+ - perf/encoding_speed.rb
104
+ - perf/person.avsc
105
+ - spec/avro_turf_spec.rb
106
+ - spec/schema_store_spec.rb
101
107
  - spec/spec_helper.rb
102
108
  homepage: https://github.com/dasch/avro_turf
103
109
  licenses:
@@ -125,6 +131,7 @@ specification_version: 4
125
131
  summary: A library that makes it easier to use the Avro serialization format from
126
132
  Ruby
127
133
  test_files:
128
- - spec/avro_spec.rb
134
+ - spec/avro_turf_spec.rb
135
+ - spec/schema_store_spec.rb
129
136
  - spec/spec_helper.rb
130
137
  has_rdoc:
data/spec/avro_spec.rb DELETED
@@ -1,227 +0,0 @@
1
- require 'fakefs/spec_helpers'
2
-
3
- describe AvroTurf do
4
- include FakeFS::SpecHelpers
5
-
6
- let(:avro) { AvroTurf.new(schemas_path: "spec/schemas/") }
7
-
8
- before do
9
- FileUtils.mkdir_p("spec/schemas")
10
- end
11
-
12
- it "encodes and decodes data using a named schema" do
13
- define_schema "person.avsc", <<-AVSC
14
- {
15
- "name": "person",
16
- "type": "record",
17
- "fields": [
18
- {
19
- "type": "string",
20
- "name": "full_name"
21
- },
22
- {
23
- "name": "address",
24
- "type": {
25
- "type": "record",
26
- "name": "address",
27
- "fields": [
28
- {
29
- "type": "string",
30
- "name": "street"
31
- },
32
- {
33
- "type": "string",
34
- "name": "city"
35
- }
36
- ]
37
- }
38
- }
39
- ]
40
- }
41
- AVSC
42
-
43
- data = {
44
- "full_name" => "John Doe",
45
- "address" => {
46
- "street" => "Market st. 989",
47
- "city" => "San Francisco"
48
- }
49
- }
50
-
51
- encoded_data = avro.encode(data, schema_name: "person")
52
-
53
- expect(avro.decode(encoded_data, schema_name: "person")).to eq(data)
54
- end
55
-
56
- it "resolves named types" do
57
- define_schema "person.avsc", <<-AVSC
58
- {
59
- "name": "person",
60
- "type": "record",
61
- "fields": [
62
- {
63
- "type": "string",
64
- "name": "full_name"
65
- },
66
- {
67
- "name": "address",
68
- "type": "address"
69
- }
70
- ]
71
- }
72
- AVSC
73
-
74
- define_schema "address.avsc", <<-AVSC
75
- {
76
- "type": "record",
77
- "name": "address",
78
- "fields": [
79
- {
80
- "type": "string",
81
- "name": "street"
82
- },
83
- {
84
- "type": "string",
85
- "name": "city"
86
- }
87
- ]
88
- }
89
- AVSC
90
-
91
- data = {
92
- "full_name" => "John Doe",
93
- "address" => {
94
- "street" => "Market st. 989",
95
- "city" => "San Francisco"
96
- }
97
- }
98
-
99
- encoded_data = avro.encode(data, schema_name: "person")
100
-
101
- expect(avro.decode(encoded_data, schema_name: "person")).to eq(data)
102
- end
103
-
104
- it "allows decoding without specifying a reader schema" do
105
- define_schema "message.avsc", <<-AVSC
106
- {
107
- "name": "message",
108
- "type": "string"
109
- }
110
- AVSC
111
-
112
- encoded_data = avro.encode("hello, world", schema_name: "message")
113
-
114
- expect(avro.decode(encoded_data)).to eq "hello, world"
115
- end
116
-
117
- it "allows using namespaces in schemas" do
118
- FileUtils.mkdir_p("spec/schemas/test/people")
119
-
120
- define_schema "test/people/person.avsc", <<-AVSC
121
- {
122
- "name": "person",
123
- "namespace": "test.people",
124
- "type": "record",
125
- "fields": [
126
- {
127
- "type": "string",
128
- "name": "full_name"
129
- },
130
- {
131
- "name": "address",
132
- "type": "test.people.address"
133
- }
134
- ]
135
- }
136
- AVSC
137
-
138
- define_schema "test/people/address.avsc", <<-AVSC
139
- {
140
- "name": "address",
141
- "namespace": "test.people",
142
- "type": "record",
143
- "fields": [
144
- {
145
- "type": "string",
146
- "name": "street"
147
- },
148
- {
149
- "type": "string",
150
- "name": "city"
151
- }
152
- ]
153
- }
154
- AVSC
155
-
156
- data = {
157
- "full_name" => "John Doe",
158
- "address" => {
159
- "street" => "Market st. 989",
160
- "city" => "San Francisco"
161
- }
162
- }
163
-
164
- encoded_data = avro.encode(data, schema_name: "person", namespace: "test.people")
165
-
166
- expect(avro.decode(encoded_data, schema_name: "person", namespace: "test.people")).to eq(data)
167
- end
168
-
169
- it "raises AvroTurf::SchemaError if the schema's namespace doesn't match the file location" do
170
- FileUtils.mkdir_p("spec/schemas/test/people")
171
-
172
- define_schema "test/people/person.avsc", <<-AVSC
173
- {
174
- "name": "person",
175
- "namespace": "yoyoyo.nanana",
176
- "type": "record",
177
- "fields": [
178
- {
179
- "type": "string",
180
- "name": "full_name"
181
- }
182
- ]
183
- }
184
- AVSC
185
-
186
- data = { "full_name" => "John Doe" }
187
-
188
- expect {
189
- avro.encode(data, schema_name: "test.people.person")
190
- }.to raise_error(AvroTurf::SchemaError, "expected schema `spec/schemas/test/people/person.avsc' to define type `test.people.person'")
191
- end
192
-
193
- it "caches schemas in memory" do
194
- define_schema "person.avsc", <<-AVSC
195
- {
196
- "name": "person",
197
- "type": "record",
198
- "fields": [
199
- {
200
- "type": "string",
201
- "name": "full_name"
202
- }
203
- ]
204
- }
205
- AVSC
206
-
207
- data = {
208
- "full_name" => "John Doe"
209
- }
210
-
211
- # Warm the schema cache.
212
- avro.encode(data, schema_name: "person")
213
-
214
- # Force a failure if the schema file is read again.
215
- FileUtils.rm("spec/schemas/person.avsc")
216
-
217
- encoded_data = avro.encode(data, schema_name: "person")
218
-
219
- expect(avro.decode(encoded_data, schema_name: "person")).to eq(data)
220
- end
221
-
222
- def define_schema(path, content)
223
- File.open(File.join("spec/schemas", path), "w") do |f|
224
- f.write(content)
225
- end
226
- end
227
- end