avro_turf 1.3.1 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a4e9638703b961c348d06adcfa3e34ac2bd908e68f9cf7762a550ae4ef453b8a
-  data.tar.gz: 6d53b1f9b5e4b9e5a3aaeb13207bf350e2b56c8db138cb499643180a94a684b7
+  metadata.gz: ebbfdd602e4d6c23894efe960ef21e01bc6bbc770c6076105260fc36de3c8e1e
+  data.tar.gz: 1113f99b28e58f641a13c9cec7b8b314bb113b34e07e3c27aab5559c524568b4
 SHA512:
-  metadata.gz: 7045f852f20d3ddeca3256724c918ed255722a1556f662d159b805921670d0a2af3e574e3c967bee7622d5d3e05f5348d6023b250006c3651cb7b1f9cec059bf
-  data.tar.gz: 50ca19fb058246ba19b28472e03bedcfbc09ddcd336d46deb29dec4b406d81a65c2b0f6ca84ec36d5227da035906976af133cf757cbfc2f20abda116c8e17d8c
+  metadata.gz: 56c4235851a345cf81abfd7170961f973eed218437187a20ef1b717e600ad19f6cbea8df41a755af5d54f2c98adac0ea3d5c236b822d4fdda10e17c1f6987d5c
+  data.tar.gz: 8caefcd9b974120c84f0a7910f26aff6106578eb80623119ac48c0c25c1775092c6e18b55a47cb97a2d000e7be7e110905cf0fb2d30a5f80e6bb0a800f98bc6c

data/.github/workflows/ruby.yml

@@ -8,15 +8,14 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        ruby-version: [1.8.x, 1.9.x, 2.0.x, 2.1.x, 2.2.x,
-          2.3.x, 2.4.x, 2.5.x, 2.6.x, 2.7.x, 3.0.x]
+        ruby: [2.3, 2.4, 2.5, 2.6, 2.7, 3.0]
 
     steps:
-    - uses: actions/checkout@v1
-    - name: Set up Ruby 2.6
-      uses: actions/setup-ruby@v1
+    - uses: actions/checkout@v2
+    - name: Set up Ruby ${{ matrix.ruby }}
+      uses: ruby/setup-ruby@v1
       with:
-        ruby-version: 2.6.x
+        ruby-version: ${{ matrix.ruby }}
     - name: Build and test with RSpec
       run: |
         gem install bundler

data/CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## Unreleased
 
+## v1.4.0
+
+- Add support for Ruby 3 (#146)
+- Add ability to validate message before encoding in `AvroTurf#encode` interface
+
 ## v1.3.1
 
 - Prevent CachedConfluentSchemaRegistry from caching the 'latest' version (#140)

data/README.md

@@ -18,7 +18,7 @@ The aliases for the original names will be removed in a future release.
 
 ## Note about finding nested schemas
 
-As of AvroTurf version 0.12.0, only top-level schemas that have their own .avsc file will be loaded and resolvable by the `AvroTurf::SchemaStore#find` method. This change will likely not affect most users. However, if you use `AvroTurf::SchemaStore#load_schemas!` to pre-cache all your schemas and then rely on `AvroTurf::SchemaStore#find` to access nested schemas that are not defined by their own .avsc files, your code may stop working when you upgrade to v0.12.0.
+As of AvroTurf version 1.0.0, only top-level schemas that have their own .avsc file will be loaded and resolvable by the `AvroTurf::SchemaStore#find` method. This change will likely not affect most users. However, if you use `AvroTurf::SchemaStore#load_schemas!` to pre-cache all your schemas and then rely on `AvroTurf::SchemaStore#find` to access nested schemas that are not defined by their own .avsc files, your code may stop working when you upgrade to v1.0.0.
 
 As an example, if you have a `person` schema (defined in `my/schemas/contacts/person.avsc`) that defines a nested `address` schema like this:
 
@@ -44,7 +44,7 @@ As an example, if you have a `person` schema (defined in `my/schemas/contacts/pe
   ]
 }
 ```
-...this will no longer work in v0.12.0:
+...this will no longer work in v1.0.0:
 ```ruby
 store = AvroTurf::SchemaStore.new(path: 'my/schemas')
 store.load_schemas!
@@ -88,6 +88,10 @@ avro.decode(encoded_data, schema_name: "person")
 
 # Encode some data using the named schema.
 avro.encode({ "name" => "Jane", "age" => 28 }, schema_name: "person")
+
+# Data can be validated before encoding to get a description of problem through
+# Avro::SchemaValidator::ValidationError exception
+avro.encode({ "titl" => "hello, world" }, schema_name: "person", validate: true)
 ```
 
 ### Inter-schema references

data/lib/avro_turf.rb

@@ -40,12 +40,15 @@ class AvroTurf
   #
   # data        - The data that should be encoded.
   # schema_name - The name of a schema in the `schemas_path`.
+  # validate    - The boolean for performing complete data validation before
+  #               encoding it, Avro::SchemaValidator::ValidationError with
+  #               a descriptive message will be raised in case of invalid message.
   #
   # Returns a String containing the encoded data.
-  def encode(data, schema_name: nil, namespace: @namespace)
+  def encode(data, schema_name: nil, namespace: @namespace, validate: false)
     stream = StringIO.new
 
-    encode_to_stream(data, stream: stream, schema_name: schema_name, namespace: namespace)
+    encode_to_stream(data, stream: stream, schema_name: schema_name, namespace: namespace, validate: validate)
 
     stream.string
   end
@@ -56,12 +59,19 @@ class AvroTurf
   # data        - The data that should be encoded.
   # schema_name - The name of a schema in the `schemas_path`.
   # stream      - An IO object that the encoded data should be written to (optional).
+  # validate    - The boolean for performing complete data validation before
+  #               encoding it, Avro::SchemaValidator::ValidationError with
+  #               a descriptive message will be raised in case of invalid message.
   #
   # Returns nothing.
-  def encode_to_stream(data, schema_name: nil, stream: nil, namespace: @namespace)
+  def encode_to_stream(data, schema_name: nil, stream: nil, namespace: @namespace, validate: false)
     schema = @schema_store.find(schema_name, namespace)
     writer = Avro::IO::DatumWriter.new(schema)
 
+    if validate
+      Avro::SchemaValidator.validate!(schema, data, recursive: true, encoded: false, fail_on_extra_fields: true)
+    end
+
     dw = Avro::DataFile::Writer.new(stream, writer, schema, @codec)
     dw << data.as_avro
     dw.close

data/lib/avro_turf/confluent_schema_registry.rb

@@ -40,9 +40,7 @@ class AvroTurf::ConfluentSchemaRegistry
   end
 
   def register(subject, schema)
-    data = post("/subjects/#{subject}/versions", body: {
-      schema: schema.to_s
-    }.to_json)
+    data = post("/subjects/#{subject}/versions", body: { schema: schema.to_s }.to_json)
 
     id = data.fetch("id")
 
@@ -82,8 +80,7 @@ class AvroTurf::ConfluentSchemaRegistry
   # http://docs.confluent.io/3.1.2/schema-registry/docs/api.html#compatibility
   def compatible?(subject, schema, version = 'latest')
     data = post("/compatibility/subjects/#{subject}/versions/#{version}",
-                expects: [200, 404],
-                body: { schema: schema.to_s }.to_json)
+                expects: [200, 404], body: { schema: schema.to_s }.to_json)
     data.fetch('is_compatible', false) unless data.has_key?('error_code')
   end
 
@@ -94,7 +91,7 @@ class AvroTurf::ConfluentSchemaRegistry
 
   # Update global config
   def update_global_config(config)
-    put("/config", { body: config.to_json })
+    put("/config", body: config.to_json)
   end
 
   # Get config for subject
@@ -104,7 +101,7 @@ class AvroTurf::ConfluentSchemaRegistry
 
   # Update config for subject
   def update_subject_config(subject, config)
-    put("/config/#{subject}", { body: config.to_json })
+    put("/config/#{subject}", body: config.to_json)
   end
 
   private

data/lib/avro_turf/schema_store.rb

@@ -83,6 +83,11 @@ class AvroTurf::SchemaStore
       # has been resolved and use the now-updated local_schemas_cache to
       # pick up where we left off.
       local_schemas_cache.delete(fullname)
+      # Ensure enum schemas are cleaned up to avoid conflicts when re-parsing
+      # schema.
+      local_schemas_cache.each do |schema_name, schema|
+        local_schemas_cache.delete(schema_name) if schema.type_sym == :enum
+      end
       load_schema!(fullname, local_schemas_cache)
     else
       raise

data/lib/avro_turf/version.rb

@@ -1,3 +1,3 @@
 class AvroTurf
-  VERSION = "1.3.1"
+  VERSION = "1.4.0"
 end

data/spec/avro_turf_spec.rb

@@ -6,44 +6,112 @@ describe AvroTurf do
   end
 
   describe "#encode" do
-    before do
-      define_schema "person.avsc", <<-AVSC
-        {
-          "name": "person",
-          "type": "record",
-          "fields": [
-            {
-              "type": "string",
-              "name": "full_name"
-            }
-          ]
+    context "when using plain schema" do
+      before do
+        define_schema "person.avsc", <<-AVSC
+          {
+            "name": "person",
+            "type": "record",
+            "fields": [
+              {
+                "type": "string",
+                "name": "full_name"
+              }
+            ]
+          }
+        AVSC
+      end
+
+      it "encodes data with Avro" do
+        data = {
+          "full_name" => "John Doe"
         }
-      AVSC
-    end
 
-    it "encodes data with Avro" do
-      data = {
-        "full_name" => "John Doe"
-      }
+        encoded_data = avro.encode(data, schema_name: "person")
 
-      encoded_data = avro.encode(data, schema_name: "person")
+        expect(avro.decode(encoded_data)).to eq(data)
+      end
 
-      expect(avro.decode(encoded_data)).to eq(data)
+      it "allows specifying a codec that should be used to compress messages" do
+        compressed_avro = AvroTurf.new(schemas_path: "spec/schemas/", codec: "deflate")
+
+        data = {
+          "full_name" => "John Doe" * 100
+        }
+
+        uncompressed_data = avro.encode(data, schema_name: "person")
+        compressed_data = compressed_avro.encode(data, schema_name: "person")
+
+        expect(compressed_data.bytesize).to be < uncompressed_data.bytesize
+        expect(compressed_avro.decode(compressed_data)).to eq(data)
+      end
     end
 
-    it "allows specifying a codec that should be used to compress messages" do
-      compressed_avro = AvroTurf.new(schemas_path: "spec/schemas/", codec: "deflate")
+    context 'when using nested schemas' do
+      before do
+        define_schema "post.avsc", <<-AVSC
+          {
+            "name": "post",
+            "type": "record",
+            "fields": [
+              {
+                "name": "tag",
+                "type": {
+                  "type": "enum",
+                  "name": "tag",
+                  "symbols": ["foo", "bar"]
+                }
+              },
+              {
+                "name": "messages",
+                "type": {
+                  "type": "array",
+                  "items": "message"
+                }
+              }
+            ]
+          }
+        AVSC
+
+        define_schema "message.avsc", <<-AVSC
+          {
+            "name": "message",
+            "type": "record",
+            "fields": [
+              {
+                "type": "string",
+                "name": "content"
+              },
+              {
+                "name": "label",
+                "type": {
+                  "type": "enum",
+                  "name": "label",
+                  "symbols": ["foo", "bar"]
+                }
+              }
+            ]
+          }
+        AVSC
+      end
 
-      data = {
-        "full_name" => "John Doe" * 100
-      }
+      it "encodes data with Avro" do
+        data = {
+          "tag" => "foo",
+          "messages" => [
+            {
+              "content" => "hello",
+              "label" => "bar"
+            }
+          ]
+        }
 
-      uncompressed_data = avro.encode(data, schema_name: "person")
-      compressed_data = compressed_avro.encode(data, schema_name: "person")
+        encoded_data = avro.encode(data, schema_name: "post")
 
-      expect(compressed_data.bytesize).to be < uncompressed_data.bytesize
-      expect(compressed_avro.decode(compressed_data)).to eq(data)
+        expect(avro.decode(encoded_data)).to eq(data)
+      end
     end
+
   end
 
   describe "#decode" do
@@ -105,6 +173,67 @@ describe AvroTurf do
 
       expect(avro.decode(stream.string)).to eq "hello"
     end
+
+    context "validating" do
+      subject(:encode_to_stream) do
+        stream = StringIO.new
+        avro.encode_to_stream(message, stream: stream, schema_name: "message", validate: true) 
+      end
+
+      context "with a valid message" do
+        let(:message) { { "full_name" => "John Doe" } }
+
+        it "does not raise any error" do
+          define_schema "message.avsc", <<-AVSC
+            {
+              "name": "message",
+              "type": "record",
+              "fields": [
+                { "name": "full_name", "type": "string" }
+              ]
+            }
+          AVSC
+
+          expect { encode_to_stream }.not_to raise_error
+        end
+      end
+
+      context "when message has wrong type" do
+        let(:message) { { "full_name" => 123 } }
+
+        it "raises Avro::SchemaValidator::ValidationError with a message about type mismatch" do
+          define_schema "message.avsc", <<-AVSC
+            {
+              "name": "message",
+              "type": "record",
+              "fields": [
+                { "name": "full_name", "type": "string" }
+              ]
+            }
+          AVSC
+
+          expect { encode_to_stream }.to raise_error(Avro::SchemaValidator::ValidationError, /\.full_name expected type string, got int/)
+        end
+      end
+
+      context "when message contains extra fields (typo in key)" do
+        let(:message) { { "fulll_name" => "John Doe" } }
+
+        it "raises Avro::SchemaValidator::ValidationError with a message about extra field" do
+          define_schema "message.avsc", <<-AVSC
+            {
+              "name": "message",
+              "type": "record",
+              "fields": [
+                { "name": "full_name", "type": "string" }
+              ]
+            }
+          AVSC
+
+          expect { encode_to_stream }.to raise_error(Avro::SchemaValidator::ValidationError, /extra field 'fulll_name'/)
+        end
+      end
+    end
   end
 
   describe "#decode_stream" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: avro_turf
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.4.0
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-16 00:00:00.000000000 Z
+date: 2021-06-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro