avro_turf 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 64620bcf8c0a04337ecf571abe2d0e51098fce4a
4
- data.tar.gz: cc45ee47d74ea7ee364463b105f8cfc92345058b
3
+ metadata.gz: 0845fc0e81baaa72df5abd34b47a0ce6d06a853f
4
+ data.tar.gz: 434aaa74efb2f96437045a8705c7889348bec9c3
5
5
  SHA512:
6
- metadata.gz: 090cf294b97ea232c5ec8372eced0244d87f87a6fd3d9cdffdfb2ad73e467e52be0ae1ab3108e53a3e5aabfdd7a3bd2a35aae70c89ca1b448a74305d61d48964
7
- data.tar.gz: 657f8b32c84d5b6e01a614bab06a4d17987e3c1832641e62bac8548c139810e60febfc72684983b1342e4839c889b667d037fcbb09932c940366636e32198c5f
6
+ metadata.gz: ef023ba8d9fb413e69a467c1ecb4a4ef73c6c1f1bb86f1f362b6f351abcd6ed8b2fae965d7b50b4d3ba6e81513c30dcffbe1e5d688044efa872f4b65f85ff39a
7
+ data.tar.gz: e095b79ce3f37a69efac8537e6d59fcb5c8d66350604eba7f00166911801699eccf945552fcbc5c950223f7c14ddfddd2f58157b70aff2466e6d2ea157485eab
data/Gemfile CHANGED
@@ -2,3 +2,6 @@ source 'https://rubygems.org'
2
2
 
3
3
  # Specify your gem's dependencies in avro_turf.gemspec
4
4
  gemspec
5
+
6
+ # Used by CircleCI to format RSpec results.
7
+ gem 'rspec_junit_formatter', :git => 'git@github.com:circleci/rspec_junit_formatter.git'
data/lib/avro_turf.rb CHANGED
@@ -1,15 +1,17 @@
1
1
  require 'avro_turf/version'
2
2
  require 'avro'
3
3
  require 'json'
4
+ require 'avro_turf/schema_store'
4
5
 
5
6
  class AvroTurf
6
7
  class Error < StandardError; end
7
8
  class SchemaError < Error; end
9
+ class SchemaNotFoundError < Error; end
8
10
 
9
- def initialize(schemas_path:, namespace: nil)
10
- @schemas_path = schemas_path or raise "Please specify a schema path"
11
- @schemas = Hash.new
11
+ def initialize(schemas_path:, namespace: nil, codec: nil)
12
12
  @namespace = namespace
13
+ @schema_store = SchemaStore.new(path: schemas_path)
14
+ @codec = codec
13
15
  end
14
16
 
15
17
  # Encodes data to Avro using the specified schema.
@@ -19,15 +21,31 @@ class AvroTurf
19
21
  #
20
22
  # Returns a String containing the encoded data.
21
23
  def encode(data, schema_name:, namespace: @namespace)
22
- schema = resolve_schema(schema_name, namespace)
24
+ stream = StringIO.new
25
+
26
+ encode_to_stream(data, stream: stream, schema_name: schema_name, namespace: namespace)
27
+
28
+ stream.string
29
+ end
30
+
31
+ # Encodes data to Avro using the specified schema and writes it to the
32
+ # specified stream.
33
+ #
34
+ # data - The data that should be encoded.
35
+ # schema_name - The name of a schema in the `schemas_path`.
36
+ # stream - An IO object that the encoded data should be written to (optional).
37
+ # codec - The String name of a codec that should be used to compress messages (optional).
38
+ #
39
+ # Currently, the only valid codec name is `deflate`.
40
+ #
41
+ # Returns nothing.
42
+ def encode_to_stream(data, schema_name:, stream:, namespace: @namespace)
43
+ schema = @schema_store.find(schema_name, namespace)
23
44
  writer = Avro::IO::DatumWriter.new(schema)
24
45
 
25
- io = StringIO.new
26
- dw = Avro::DataFile::Writer.new(io, writer, schema)
46
+ dw = Avro::DataFile::Writer.new(stream, writer, schema, @codec)
27
47
  dw << data
28
48
  dw.close
29
-
30
- io.string
31
49
  end
32
50
 
33
51
  # Decodes Avro data.
@@ -35,49 +53,31 @@ class AvroTurf
35
53
  # encoded_data - A String containing Avro-encoded data.
36
54
  # schema_name - The String name of the schema that should be used to read
37
55
  # the data. If nil, the writer schema will be used.
56
+ # namespace - The namespace of the Avro schema used to decode the data.
38
57
  #
39
58
  # Returns whatever is encoded in the data.
40
59
  def decode(encoded_data, schema_name: nil, namespace: @namespace)
41
- io = StringIO.new(encoded_data)
42
- schema = schema_name && resolve_schema(schema_name, namespace)
43
- reader = Avro::IO::DatumReader.new(nil, schema)
44
- dr = Avro::DataFile::Reader.new(io, reader)
45
- dr.first
60
+ stream = StringIO.new(encoded_data)
61
+ decode_stream(stream, schema_name: schema_name, namespace: namespace)
46
62
  end
47
63
 
48
- private
49
-
50
- # Resolves and returns a schema.
64
+ # Decodes Avro data from an IO stream.
51
65
  #
52
- # schema_name - The String name of the schema to resolve.
66
+ # stream - An IO object containing Avro data.
67
+ # schema_name - The String name of the schema that should be used to read
68
+ # the data. If nil, the writer schema will be used.
69
+ # namespace - The namespace of the Avro schema used to decode the data.
53
70
  #
54
- # Returns an Avro::Schema.
55
- def resolve_schema(name, namespace = nil)
56
- fullname = Avro::Name.make_fullname(name, namespace)
57
-
58
- return @schemas[fullname] if @schemas.key?(fullname)
59
-
60
- *namespace, schema_name = fullname.split(".")
61
- schema_path = File.join(@schemas_path, *namespace, schema_name + ".avsc")
62
- schema_json = JSON.parse(File.read(schema_path))
63
- schema = Avro::Schema.real_parse(schema_json, @schemas)
64
-
65
- if schema.respond_to?(:fullname) && schema.fullname != fullname
66
- raise SchemaError, "expected schema `#{schema_path}' to define type `#{fullname}'"
67
- end
68
-
69
- schema
70
- rescue ::Avro::SchemaParseError => e
71
- # This is a hack in order to figure out exactly which type was missing. The
72
- # Avro gem ought to provide this data directly.
73
- if e.to_s =~ /"([\w\.]+)" is not a schema we know about/
74
- resolve_schema($1)
71
+ # Returns whatever is encoded in the stream.
72
+ def decode_stream(stream, schema_name: nil, namespace: @namespace)
73
+ schema = schema_name && @schema_store.find(schema_name, namespace)
74
+ reader = Avro::IO::DatumReader.new(nil, schema)
75
+ dr = Avro::DataFile::Reader.new(stream, reader)
76
+ dr.first
77
+ end
75
78
 
76
- # Re-resolve the original schema now that the dependency has been resolved.
77
- @schemas.delete(fullname)
78
- resolve_schema(fullname)
79
- else
80
- raise
81
- end
79
+ # Loads all schema definition files in the `schemas_dir`.
80
+ def load_schemas!
81
+ @schema_store.load_schemas!
82
82
  end
83
83
  end
@@ -0,0 +1,58 @@
1
+ class AvroTurf::SchemaStore
2
+ def initialize(path:)
3
+ @path = path or raise "Please specify a schema path"
4
+ @schemas = Hash.new
5
+ end
6
+
7
+ # Resolves and returns a schema.
8
+ #
9
+ # schema_name - The String name of the schema to resolve.
10
+ #
11
+ # Returns an Avro::Schema.
12
+ def find(name, namespace = nil)
13
+ fullname = Avro::Name.make_fullname(name, namespace)
14
+
15
+ return @schemas[fullname] if @schemas.key?(fullname)
16
+
17
+ *namespace, schema_name = fullname.split(".")
18
+ schema_path = File.join(@path, *namespace, schema_name + ".avsc")
19
+ schema_json = JSON.parse(File.read(schema_path))
20
+ schema = Avro::Schema.real_parse(schema_json, @schemas)
21
+
22
+ if schema.respond_to?(:fullname) && schema.fullname != fullname
23
+ raise AvroTurf::SchemaError, "expected schema `#{schema_path}' to define type `#{fullname}'"
24
+ end
25
+
26
+ schema
27
+ rescue ::Avro::SchemaParseError => e
28
+ # This is a hack in order to figure out exactly which type was missing. The
29
+ # Avro gem ought to provide this data directly.
30
+ if e.to_s =~ /"([\w\.]+)" is not a schema we know about/
31
+ find($1)
32
+
33
+ # Re-resolve the original schema now that the dependency has been resolved.
34
+ @schemas.delete(fullname)
35
+ find(fullname)
36
+ else
37
+ raise
38
+ end
39
+ rescue Errno::ENOENT
40
+ raise AvroTurf::SchemaNotFoundError, "could not find Avro schema at `#{schema_path}'"
41
+ end
42
+
43
+ # Loads all schema definition files in the `schemas_dir`.
44
+ def load_schemas!
45
+ pattern = [@path, "**", "*.avsc"].join("/")
46
+
47
+ Dir.glob(pattern) do |schema_path|
48
+ # Remove the path prefix.
49
+ schema_path.sub!(/^\/#{@path}\//, "")
50
+
51
+ # Replace `/` with `.` and chop off the file extension.
52
+ schema_name = File.basename(schema_path.tr("/", "."), ".avsc")
53
+
54
+ # Load and cache the schema.
55
+ find(schema_name)
56
+ end
57
+ end
58
+ end
@@ -1,3 +1,3 @@
1
1
  class AvroTurf
2
- VERSION = "0.2.0"
2
+ VERSION = "0.3.0"
3
3
  end
data/perf/address.avsc ADDED
@@ -0,0 +1,14 @@
1
+ {
2
+ "name": "address",
3
+ "type": "record",
4
+ "fields": [
5
+ {
6
+ "name": "street",
7
+ "type": "string"
8
+ },
9
+ {
10
+ "name": "city",
11
+ "type": "string"
12
+ }
13
+ ]
14
+ }
@@ -0,0 +1,26 @@
1
+ #!/usr/bin/env ruby
2
+ #
3
+ # Measures the encoded size of messages of increasing size.
4
+
5
+ $LOAD_PATH.unshift(File.expand_path("../lib", File.dirname(__FILE__)))
6
+
7
+ require 'benchmark'
8
+ require 'avro_turf'
9
+
10
+ sizes = [1, 10, 100, 1_000, 10_000]
11
+ avro = AvroTurf.new(schemas_path: File.dirname(__FILE__))
12
+
13
+ sizes.each do |size|
14
+ data = {
15
+ "name" => "John" * size,
16
+ "address" => {
17
+ "street" => "1st st." * size,
18
+ "city" => "Citytown" * size
19
+ }
20
+ }
21
+
22
+ result = avro.encode(data, schema_name: "person")
23
+ encoded_size = result.bytesize
24
+ encode_factor = result.bytesize / size.to_f
25
+ puts "size #{size}: #{encoded_size} bytes (encoding factor #{encode_factor})"
26
+ end
@@ -0,0 +1,30 @@
1
+ #!/usr/bin/env ruby
2
+ #
3
+ # Measures the time it takes to encode messages of increasing size.
4
+
5
+ $LOAD_PATH.unshift(File.expand_path("../lib", File.dirname(__FILE__)))
6
+
7
+ require 'benchmark'
8
+ require 'avro_turf'
9
+
10
+ # Number of iterations per run.
11
+ N = 10_000
12
+
13
+ Benchmark.bm(15) do |x|
14
+ sizes = [1, 10, 100, 1_000, 10_000]
15
+ avro = AvroTurf.new(schemas_path: File.dirname(__FILE__))
16
+
17
+ sizes.each do |size|
18
+ data = {
19
+ "name" => "John" * size,
20
+ "address" => {
21
+ "street" => "1st st." * size,
22
+ "city" => "Citytown" * size
23
+ }
24
+ }
25
+
26
+ x.report("size #{size}:") do
27
+ N.times { avro.encode(data, schema_name: "person") }
28
+ end
29
+ end
30
+ end
data/perf/person.avsc ADDED
@@ -0,0 +1,14 @@
1
+ {
2
+ "name": "person",
3
+ "type": "record",
4
+ "fields": [
5
+ {
6
+ "name": "name",
7
+ "type": "string"
8
+ },
9
+ {
10
+ "name": "address",
11
+ "type": "address"
12
+ }
13
+ ]
14
+ }
@@ -0,0 +1,125 @@
1
+ describe AvroTurf do
2
+ let(:avro) { AvroTurf.new(schemas_path: "spec/schemas/") }
3
+
4
+ before do
5
+ FileUtils.mkdir_p("spec/schemas")
6
+ end
7
+
8
+ describe "#encode" do
9
+ before do
10
+ define_schema "person.avsc", <<-AVSC
11
+ {
12
+ "name": "person",
13
+ "type": "record",
14
+ "fields": [
15
+ {
16
+ "type": "string",
17
+ "name": "full_name"
18
+ }
19
+ ]
20
+ }
21
+ AVSC
22
+ end
23
+
24
+ it "encodes data with Avro" do
25
+ data = {
26
+ "full_name" => "John Doe"
27
+ }
28
+
29
+ encoded_data = avro.encode(data, schema_name: "person")
30
+
31
+ expect(avro.decode(encoded_data)).to eq(data)
32
+ end
33
+
34
+ it "allows specifying a codec that should be used to compress messages" do
35
+ compressed_avro = AvroTurf.new(schemas_path: "spec/schemas/", codec: "deflate")
36
+
37
+ data = {
38
+ "full_name" => "John Doe" * 100
39
+ }
40
+
41
+ uncompressed_data = avro.encode(data, schema_name: "person")
42
+ compressed_data = compressed_avro.encode(data, schema_name: "person")
43
+
44
+ expect(compressed_data.bytesize).to be < uncompressed_data.bytesize
45
+ expect(compressed_avro.decode(compressed_data)).to eq(data)
46
+ end
47
+ end
48
+
49
+ describe "#decode" do
50
+ it "decodes Avro data using the inlined writer's schema" do
51
+ define_schema "message.avsc", <<-AVSC
52
+ {
53
+ "name": "message",
54
+ "type": "string"
55
+ }
56
+ AVSC
57
+
58
+ encoded_data = avro.encode("hello, world", schema_name: "message")
59
+
60
+ expect(avro.decode(encoded_data)).to eq "hello, world"
61
+ end
62
+
63
+ it "decodes Avro data using a specified reader's schema" do
64
+ FileUtils.mkdir_p("spec/schemas/reader")
65
+
66
+ define_schema "point.avsc", <<-AVSC
67
+ {
68
+ "name": "point",
69
+ "type": "record",
70
+ "fields": [
71
+ { "name": "x", "type": "long" },
72
+ { "name": "y", "type": "long" }
73
+ ]
74
+ }
75
+ AVSC
76
+
77
+ define_schema "reader/point.avsc", <<-AVSC
78
+ {
79
+ "name": "point",
80
+ "type": "record",
81
+ "fields": [
82
+ { "name": "x", "type": "long" }
83
+ ]
84
+ }
85
+ AVSC
86
+
87
+ encoded_data = avro.encode({ "x" => 42, "y" => 13 }, schema_name: "point")
88
+ reader_avro = AvroTurf.new(schemas_path: "spec/schemas/reader")
89
+
90
+ expect(reader_avro.decode(encoded_data, schema_name: "point")).to eq({ "x" => 42 })
91
+ end
92
+ end
93
+
94
+ describe "#encode_to_stream" do
95
+ it "writes encoded data to an existing stream" do
96
+ define_schema "message.avsc", <<-AVSC
97
+ {
98
+ "name": "message",
99
+ "type": "string"
100
+ }
101
+ AVSC
102
+
103
+ stream = StringIO.new
104
+ avro.encode_to_stream("hello", stream: stream, schema_name: "message")
105
+
106
+ expect(avro.decode(stream.string)).to eq "hello"
107
+ end
108
+ end
109
+
110
+ describe "#decode_stream" do
111
+ it "decodes Avro data from a stream" do
112
+ define_schema "message.avsc", <<-AVSC
113
+ {
114
+ "name": "message",
115
+ "type": "string"
116
+ }
117
+ AVSC
118
+
119
+ encoded_data = avro.encode("hello", schema_name: "message")
120
+ stream = StringIO.new(encoded_data)
121
+
122
+ expect(avro.decode_stream(stream)).to eq "hello"
123
+ end
124
+ end
125
+ end
@@ -0,0 +1,253 @@
1
+ require 'avro_turf/schema_store'
2
+
3
+ describe AvroTurf::SchemaStore do
4
+ let(:store) { AvroTurf::SchemaStore.new(path: "spec/schemas") }
5
+
6
+ before do
7
+ FileUtils.mkdir_p("spec/schemas")
8
+ end
9
+
10
+ describe "#find" do
11
+ it "finds schemas on the file system" do
12
+ define_schema "message.avsc", <<-AVSC
13
+ {
14
+ "name": "message",
15
+ "type": "record",
16
+ "fields": [
17
+ {
18
+ "type": "string",
19
+ "name": "message"
20
+ }
21
+ ]
22
+ }
23
+ AVSC
24
+
25
+ schema = store.find("message")
26
+ expect(schema.fullname).to eq "message"
27
+ end
28
+
29
+ it "resolves missing references" do
30
+ define_schema "person.avsc", <<-AVSC
31
+ {
32
+ "name": "person",
33
+ "type": "record",
34
+ "fields": [
35
+ {
36
+ "name": "address",
37
+ "type": "address"
38
+ }
39
+ ]
40
+ }
41
+ AVSC
42
+
43
+ define_schema "address.avsc", <<-AVSC
44
+ {
45
+ "type": "record",
46
+ "name": "address",
47
+ "fields": []
48
+ }
49
+ AVSC
50
+
51
+ schema = store.find("person")
52
+
53
+ expect(schema.fullname).to eq "person"
54
+ end
55
+
56
+ it "finds namespaced schemas" do
57
+ FileUtils.mkdir_p("spec/schemas/test/people")
58
+
59
+ define_schema "test/people/person.avsc", <<-AVSC
60
+ {
61
+ "name": "person",
62
+ "namespace": "test.people",
63
+ "type": "record",
64
+ "fields": [
65
+ {
66
+ "name": "address",
67
+ "type": "test.people.address"
68
+ }
69
+ ]
70
+ }
71
+ AVSC
72
+
73
+ define_schema "test/people/address.avsc", <<-AVSC
74
+ {
75
+ "name": "address",
76
+ "namespace": "test.people",
77
+ "type": "record",
78
+ "fields": []
79
+ }
80
+ AVSC
81
+
82
+ schema = store.find("person", "test.people")
83
+
84
+ expect(schema.fullname).to eq "test.people.person"
85
+ end
86
+
87
+ it "ignores the namespace when the name contains a dot" do
88
+ FileUtils.mkdir_p("spec/schemas/test/acme")
89
+
90
+ define_schema "test/acme/message.avsc", <<-AVSC
91
+ {
92
+ "name": "message",
93
+ "namespace": "test.acme",
94
+ "type": "record",
95
+ "fields": []
96
+ }
97
+ AVSC
98
+
99
+ schema = store.find("test.acme.message", "test.yolo")
100
+
101
+ expect(schema.fullname).to eq "test.acme.message"
102
+ end
103
+
104
+ it "raises AvroTurf::SchemaNotFoundError if there's no schema file matching the name" do
105
+ expect {
106
+ store.find("not_there")
107
+ }.to raise_error(AvroTurf::SchemaNotFoundError, "could not find Avro schema at `spec/schemas/not_there.avsc'")
108
+ end
109
+
110
+ it "raises AvroTurf::SchemaNotFoundError if a type reference cannot be resolved" do
111
+ define_schema "person.avsc", <<-AVSC
112
+ {
113
+ "name": "person",
114
+ "type": "record",
115
+ "fields": [
116
+ {
117
+ "name": "address",
118
+ "type": "address"
119
+ }
120
+ ]
121
+ }
122
+ AVSC
123
+
124
+ expect {
125
+ store.find("person")
126
+ }.to raise_exception(AvroTurf::SchemaNotFoundError)
127
+ end
128
+
129
+ it "raises AvroTurf::SchemaError if the schema's namespace doesn't match the file location" do
130
+ FileUtils.mkdir_p("spec/schemas/test/people")
131
+
132
+ define_schema "test/people/person.avsc", <<-AVSC
133
+ {
134
+ "name": "person",
135
+ "namespace": "yoyoyo.nanana",
136
+ "type": "record",
137
+ "fields": []
138
+ }
139
+ AVSC
140
+
141
+ expect {
142
+ store.find("test.people.person")
143
+ }.to raise_error(AvroTurf::SchemaError, "expected schema `spec/schemas/test/people/person.avsc' to define type `test.people.person'")
144
+ end
145
+
146
+ it "handles circular dependencies" do
147
+ define_schema "a.avsc", <<-AVSC
148
+ {
149
+ "name": "a",
150
+ "type": "record",
151
+ "fields": [
152
+ {
153
+ "type": "b",
154
+ "name": "b"
155
+ }
156
+ ]
157
+ }
158
+ AVSC
159
+
160
+ define_schema "b.avsc", <<-AVSC
161
+ {
162
+ "name": "b",
163
+ "type": "record",
164
+ "fields": [
165
+ {
166
+ "type": "a",
167
+ "name": "a"
168
+ }
169
+ ]
170
+ }
171
+ AVSC
172
+
173
+ schema = store.find("a")
174
+ expect(schema.fullname).to eq "a"
175
+ end
176
+
177
+ it "caches schemas in memory" do
178
+ define_schema "person.avsc", <<-AVSC
179
+ {
180
+ "name": "person",
181
+ "type": "record",
182
+ "fields": [
183
+ {
184
+ "type": "string",
185
+ "name": "full_name"
186
+ }
187
+ ]
188
+ }
189
+ AVSC
190
+
191
+ # Warm the schema cache.
192
+ store.find("person")
193
+
194
+ # Force a failure if the schema file is read again.
195
+ FileUtils.rm("spec/schemas/person.avsc")
196
+
197
+ schema = store.find("person")
198
+ expect(schema.fullname).to eq "person"
199
+ end
200
+ end
201
+
202
+ describe "#load_schemas!" do
203
+ it "loads schemas defined in the `schemas_path` directory" do
204
+ define_schema "person.avsc", <<-AVSC
205
+ {
206
+ "name": "person",
207
+ "type": "record",
208
+ "fields": [
209
+ {
210
+ "type": "string",
211
+ "name": "full_name"
212
+ }
213
+ ]
214
+ }
215
+ AVSC
216
+
217
+ # Warm the schema cache.
218
+ store.load_schemas!
219
+
220
+ # Force a failure if the schema file is read again.
221
+ FileUtils.rm("spec/schemas/person.avsc")
222
+
223
+ schema = store.find("person")
224
+ expect(schema.fullname).to eq "person"
225
+ end
226
+
227
+ it "recursively finds schema definitions in subdirectories" do
228
+ FileUtils.mkdir_p("spec/schemas/foo/bar")
229
+
230
+ define_schema "foo/bar/person.avsc", <<-AVSC
231
+ {
232
+ "name": "foo.bar.person",
233
+ "type": "record",
234
+ "fields": [
235
+ {
236
+ "type": "string",
237
+ "name": "full_name"
238
+ }
239
+ ]
240
+ }
241
+ AVSC
242
+
243
+ # Warm the schema cache.
244
+ store.load_schemas!
245
+
246
+ # Force a failure if the schema file is read again.
247
+ FileUtils.rm("spec/schemas/foo/bar/person.avsc")
248
+
249
+ schema = store.find("foo.bar.person")
250
+ expect(schema.fullname).to eq "foo.bar.person"
251
+ end
252
+ end
253
+ end
data/spec/spec_helper.rb CHANGED
@@ -1,2 +1,16 @@
1
1
  require 'bundler/setup'
2
+ require 'fakefs/spec_helpers'
2
3
  require 'avro_turf'
4
+
5
+ module Helpers
6
+ def define_schema(path, content)
7
+ File.open(File.join("spec/schemas", path), "w") do |f|
8
+ f.write(content)
9
+ end
10
+ end
11
+ end
12
+
13
+ RSpec.configure do |config|
14
+ config.include FakeFS::SpecHelpers
15
+ config.include Helpers
16
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: avro_turf
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Daniel Schierbeck
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-03-09 00:00:00.000000000 Z
11
+ date: 2015-06-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: avro
@@ -96,8 +96,14 @@ files:
96
96
  - avro_turf.gemspec
97
97
  - circle.yml
98
98
  - lib/avro_turf.rb
99
+ - lib/avro_turf/schema_store.rb
99
100
  - lib/avro_turf/version.rb
100
- - spec/avro_spec.rb
101
+ - perf/address.avsc
102
+ - perf/encoding_size.rb
103
+ - perf/encoding_speed.rb
104
+ - perf/person.avsc
105
+ - spec/avro_turf_spec.rb
106
+ - spec/schema_store_spec.rb
101
107
  - spec/spec_helper.rb
102
108
  homepage: https://github.com/dasch/avro_turf
103
109
  licenses:
@@ -125,6 +131,7 @@ specification_version: 4
125
131
  summary: A library that makes it easier to use the Avro serialization format from
126
132
  Ruby
127
133
  test_files:
128
- - spec/avro_spec.rb
134
+ - spec/avro_turf_spec.rb
135
+ - spec/schema_store_spec.rb
129
136
  - spec/spec_helper.rb
130
137
  has_rdoc:
data/spec/avro_spec.rb DELETED
@@ -1,227 +0,0 @@
1
- require 'fakefs/spec_helpers'
2
-
3
- describe AvroTurf do
4
- include FakeFS::SpecHelpers
5
-
6
- let(:avro) { AvroTurf.new(schemas_path: "spec/schemas/") }
7
-
8
- before do
9
- FileUtils.mkdir_p("spec/schemas")
10
- end
11
-
12
- it "encodes and decodes data using a named schema" do
13
- define_schema "person.avsc", <<-AVSC
14
- {
15
- "name": "person",
16
- "type": "record",
17
- "fields": [
18
- {
19
- "type": "string",
20
- "name": "full_name"
21
- },
22
- {
23
- "name": "address",
24
- "type": {
25
- "type": "record",
26
- "name": "address",
27
- "fields": [
28
- {
29
- "type": "string",
30
- "name": "street"
31
- },
32
- {
33
- "type": "string",
34
- "name": "city"
35
- }
36
- ]
37
- }
38
- }
39
- ]
40
- }
41
- AVSC
42
-
43
- data = {
44
- "full_name" => "John Doe",
45
- "address" => {
46
- "street" => "Market st. 989",
47
- "city" => "San Francisco"
48
- }
49
- }
50
-
51
- encoded_data = avro.encode(data, schema_name: "person")
52
-
53
- expect(avro.decode(encoded_data, schema_name: "person")).to eq(data)
54
- end
55
-
56
- it "resolves named types" do
57
- define_schema "person.avsc", <<-AVSC
58
- {
59
- "name": "person",
60
- "type": "record",
61
- "fields": [
62
- {
63
- "type": "string",
64
- "name": "full_name"
65
- },
66
- {
67
- "name": "address",
68
- "type": "address"
69
- }
70
- ]
71
- }
72
- AVSC
73
-
74
- define_schema "address.avsc", <<-AVSC
75
- {
76
- "type": "record",
77
- "name": "address",
78
- "fields": [
79
- {
80
- "type": "string",
81
- "name": "street"
82
- },
83
- {
84
- "type": "string",
85
- "name": "city"
86
- }
87
- ]
88
- }
89
- AVSC
90
-
91
- data = {
92
- "full_name" => "John Doe",
93
- "address" => {
94
- "street" => "Market st. 989",
95
- "city" => "San Francisco"
96
- }
97
- }
98
-
99
- encoded_data = avro.encode(data, schema_name: "person")
100
-
101
- expect(avro.decode(encoded_data, schema_name: "person")).to eq(data)
102
- end
103
-
104
- it "allows decoding without specifying a reader schema" do
105
- define_schema "message.avsc", <<-AVSC
106
- {
107
- "name": "message",
108
- "type": "string"
109
- }
110
- AVSC
111
-
112
- encoded_data = avro.encode("hello, world", schema_name: "message")
113
-
114
- expect(avro.decode(encoded_data)).to eq "hello, world"
115
- end
116
-
117
- it "allows using namespaces in schemas" do
118
- FileUtils.mkdir_p("spec/schemas/test/people")
119
-
120
- define_schema "test/people/person.avsc", <<-AVSC
121
- {
122
- "name": "person",
123
- "namespace": "test.people",
124
- "type": "record",
125
- "fields": [
126
- {
127
- "type": "string",
128
- "name": "full_name"
129
- },
130
- {
131
- "name": "address",
132
- "type": "test.people.address"
133
- }
134
- ]
135
- }
136
- AVSC
137
-
138
- define_schema "test/people/address.avsc", <<-AVSC
139
- {
140
- "name": "address",
141
- "namespace": "test.people",
142
- "type": "record",
143
- "fields": [
144
- {
145
- "type": "string",
146
- "name": "street"
147
- },
148
- {
149
- "type": "string",
150
- "name": "city"
151
- }
152
- ]
153
- }
154
- AVSC
155
-
156
- data = {
157
- "full_name" => "John Doe",
158
- "address" => {
159
- "street" => "Market st. 989",
160
- "city" => "San Francisco"
161
- }
162
- }
163
-
164
- encoded_data = avro.encode(data, schema_name: "person", namespace: "test.people")
165
-
166
- expect(avro.decode(encoded_data, schema_name: "person", namespace: "test.people")).to eq(data)
167
- end
168
-
169
- it "raises AvroTurf::SchemaError if the schema's namespace doesn't match the file location" do
170
- FileUtils.mkdir_p("spec/schemas/test/people")
171
-
172
- define_schema "test/people/person.avsc", <<-AVSC
173
- {
174
- "name": "person",
175
- "namespace": "yoyoyo.nanana",
176
- "type": "record",
177
- "fields": [
178
- {
179
- "type": "string",
180
- "name": "full_name"
181
- }
182
- ]
183
- }
184
- AVSC
185
-
186
- data = { "full_name" => "John Doe" }
187
-
188
- expect {
189
- avro.encode(data, schema_name: "test.people.person")
190
- }.to raise_error(AvroTurf::SchemaError, "expected schema `spec/schemas/test/people/person.avsc' to define type `test.people.person'")
191
- end
192
-
193
- it "caches schemas in memory" do
194
- define_schema "person.avsc", <<-AVSC
195
- {
196
- "name": "person",
197
- "type": "record",
198
- "fields": [
199
- {
200
- "type": "string",
201
- "name": "full_name"
202
- }
203
- ]
204
- }
205
- AVSC
206
-
207
- data = {
208
- "full_name" => "John Doe"
209
- }
210
-
211
- # Warm the schema cache.
212
- avro.encode(data, schema_name: "person")
213
-
214
- # Force a failure if the schema file is read again.
215
- FileUtils.rm("spec/schemas/person.avsc")
216
-
217
- encoded_data = avro.encode(data, schema_name: "person")
218
-
219
- expect(avro.decode(encoded_data, schema_name: "person")).to eq(data)
220
- end
221
-
222
- def define_schema(path, content)
223
- File.open(File.join("spec/schemas", path), "w") do |f|
224
- f.write(content)
225
- end
226
- end
227
- end