avro-gen-ruby 0.1.0

data/lib/avro_gen/schema_validator.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require 'avro'
+require 'schema_registry_client'
+require_relative 'schema_field'
+
+module AvroGen
+  # Loads an Avro schema from the configured schema path and exposes its fields.
+  # Used at runtime by generated Record classes (for #schema_fields) and as the
+  # schema loader for the generator.
+  class SchemaValidator
+    # @return [String]
+    attr_reader :schema
+    # @return [String]
+    attr_reader :namespace
+
+    @stores = {}
+
+    class << self
+      # A schema store is an in-memory cache of parsed Avro schemas. Reuse one per
+      # path so repeated runtime lookups (e.g. Record#schema_fields across many
+      # records) don't re-read and re-parse the .avsc files each time.
+      # @param path [String]
+      # @return [SchemaRegistry::AvroSchemaStore]
+      def store_for(path)
+        (@stores ||= {})[path] ||= SchemaRegistry::AvroSchemaStore.new(path: path)
+      end
+
+      # Drop cached stores; call when the schemas on disk may have changed.
+      # @return [void]
+      def clear_store_cache!
+        @stores = {}
+      end
+    end
+
+    # @param schema [String]
+    # @param namespace [String]
+    # @param path [String] location of .avsc files; defaults to the configured schema_path
+    # @param store [SchemaRegistry::AvroSchemaStore] an existing store to reuse; when
+    #   omitted a fresh, isolated store is created (the generator relies on this).
+    def initialize(schema:, namespace:, path: nil, store: nil)
+      @schema = schema
+      @namespace = namespace
+      @path = path || AvroGen.config.schema_path
+      @schema_store = store
+    end
+
+    # @return [SchemaRegistry::AvroSchemaStore]
+    def schema_store
+      @schema_store ||= SchemaRegistry::AvroSchemaStore.new(path: @path)
+    end
+
+    # Forcefully loads the schema into memory.
+    # @return [Avro::Schema]
+    def load_schema
+      avro_schema
+    end
+
+    # @return [Avro::Schema::NamedSchema]
+    def avro_schema(schema=nil)
+      schema ||= @schema
+      schema_store.find("#{@namespace}.#{schema}")
+    end
+
+    # @return [Array<AvroGen::SchemaField>]
+    def schema_fields
+      avro_schema.fields.map do |field|
+        enum_values = field.type.type == 'enum' ? field.type.symbols : []
+        AvroGen::SchemaField.new(field.name, field.type, enum_values, field.default)
+      end
+    end
+
+    # @return [void]
+    def validate(payload, schema: nil)
+      Avro::SchemaValidator.validate!(avro_schema(schema), payload,
+                                      recursive: true,
+                                      fail_on_extra_fields: true)
+    end
+  end
+end

data/lib/avro_gen/upgrader.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require_relative 'configuration'
+
+module AvroGen
+  # Rewrites references in checked-in generated schema-class files from the old
+  # Deimos constants to their AvroGen equivalents, so users can migrate off the
+  # deprecated `const_missing` fallback in one shot.
+  module Upgrader
+    REPLACEMENTS = {
+      'Deimos::SchemaClass::Record' => 'AvroGen::SchemaClass::Record',
+      'Deimos::SchemaClass::Enum' => 'AvroGen::SchemaClass::Enum',
+      'Deimos::SchemaClass::Base' => 'AvroGen::SchemaClass::Base',
+      'Deimos::Utils::SchemaClass' => 'AvroGen::SchemaClass'
+    }.freeze
+
+    class << self
+      # @param path [String] directory containing generated classes; defaults to config
+      # @return [Array<String>] the list of files that were rewritten
+      def run(path: nil)
+        path ||= AvroGen.config.generated_class_path
+        changed = []
+        Dir["#{path}/**/*.rb"].each do |file|
+          original = File.read(file)
+          updated = rewrite(original)
+          next if updated == original
+
+          File.write(file, updated)
+          changed << file
+        end
+        changed
+      end
+
+      # @param contents [String]
+      # @return [String]
+      def rewrite(contents)
+        REPLACEMENTS.reduce(contents) do |text, (from, to)|
+          text.gsub(from, to)
+        end
+      end
+    end
+  end
+end

data/lib/avro_gen/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module AvroGen
+  VERSION = '0.1.0'
+end

data/lib/avro_gen.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'avro_gen/version'
+require 'avro_gen/errors'
+require 'avro_gen/configuration'
+require 'avro_gen/schema_field'
+require 'avro_gen/avro_parser'
+require 'avro_gen/schema_validator'
+require 'avro_gen/schema_class'
+require 'avro_gen/schema_class/base'
+require 'avro_gen/schema_class/enum'
+require 'avro_gen/schema_class/record'
+
+require 'avro_gen/railtie' if defined?(Rails::Railtie)
+
+# Top-level namespace for the Avro schema-class generator.
+module AvroGen
+end

data/lib/tasks/avro.rake

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+namespace :avro do
+  desc 'Generate Ruby schema classes from Avro schemas'
+  task generate: :environment do
+    require 'avro_gen/generator'
+    Rails.logger&.info('Running avro:generate')
+    AvroGen::Generator.new.generate_from_path
+  end
+
+  desc 'Rewrite generated schema classes to use AvroGen instead of Deimos constants'
+  task upgrade: :environment do
+    require 'avro_gen/upgrader'
+    changed = AvroGen::Upgrader.run
+    puts "Upgraded #{changed.size} file(s):"
+    changed.each { |f| puts "  #{f}" }
+  end
+end

data/regenerate_test_schema_classes.rb

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Regenerates the committed runtime schema-class fixtures under spec/schemas/my_namespace.
+# These are loaded by the behaviour specs (e.g. spec/my_schema_spec.rb).
+require 'avro_gen'
+require 'avro_gen/generator'
+
+AvroGen.configure do |config|
+  config.schema_path = 'spec/schemas'
+  config.generated_class_path = 'spec/schemas'
+  config.nest_child_schemas = true
+  config.use_full_namespace = true
+  config.schema_namespace_map = {
+    'com' => 'Schemas',
+    'com.my-namespace.my-suborg' => %w(Schemas MyNamespace)
+  }
+end
+
+# Key configs mirror the Kafka topic configs Deimos used to generate these fixtures,
+# so that keyed records get their tombstone/payload_key helpers.
+ns = 'com.my-namespace'
+configs = [
+  { schema: 'Generated', namespace: ns, key_config: { field: :a_string } },
+  { schema: 'MyNestedSchema', namespace: ns, key_config: { field: :test_id } },
+  { schema: 'MySchema', namespace: ns, key_config: { schema: 'MySchema_key' } },
+  { schema: 'MySchemaWithComplexTypes', namespace: ns, key_config: { field: :test_id } },
+  { schema: 'MySchemaWithCircularReference', namespace: ns, key_config: { none: true } },
+  { schema: 'MyLongNamespaceSchema', namespace: "#{ns}.my-suborg", key_config: { field: :test_id } }
+]
+
+AvroGen::Generator.new.generate_from_configs(configs)

data/spec/generator_spec.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+
+# Serializes a hash of { filename => contents } into a stable snapshot string.
+class MultiFileSerializer
+  def process_string(str)
+    # Ruby 3.4 changes how hashes are printed
+    if Gem::Version.new(RUBY_VERSION) > Gem::Version.new('3.4.0')
+      str.gsub(/{"(.*)" => /, '{"\1"=>')
+    else
+      str
+    end
+  end
+
+  def dump(value)
+    value.keys.sort.map { |k| "#{k}:\n#{process_string(value[k])}\n" }.join("\n")
+  end
+end
+
+RSpec.describe AvroGen::Generator do
+  let(:schema_class_path) { 'spec/app/lib/schema_classes' }
+  let(:files) { Dir["#{schema_class_path}/**/*.rb"].to_h { |f| [f, File.read(f)] } }
+
+  # A schema with a field-based key, plus every other schema in the path
+  # generated as a plain class. This exercises records, enums, nested records,
+  # complex types (arrays/maps/unions) and circular references all at once.
+  let(:configs) do
+    [{ schema: 'Generated', namespace: 'com.my-namespace', key_config: { field: :a_string } }]
+  end
+
+  before(:each) do
+    AvroGen.config.schema_path = 'spec/schemas'
+    AvroGen.config.generated_class_path = schema_class_path
+  end
+
+  after(:each) do
+    FileUtils.rm_rf('spec/app')
+  end
+
+  it 'generates the correct classes with child schemas nested' do
+    AvroGen.config.nest_child_schemas = true
+    described_class.new.generate_from_configs(configs)
+    expect(files).to match_snapshot('consumers', snapshot_serializer: MultiFileSerializer)
+  end
+
+  it 'generates the correct classes with child schemas in their own files' do
+    AvroGen.config.nest_child_schemas = false
+    described_class.new.generate_from_configs(configs)
+    expect(files).to match_snapshot('consumers-no-nest', snapshot_serializer: MultiFileSerializer)
+  end
+
+  it 'generates folders matching the full namespace' do
+    AvroGen.config.use_full_namespace = true
+    described_class.new.generate_from_configs(configs)
+    expect(files).to match_snapshot('namespace_folders', snapshot_serializer: MultiFileSerializer)
+  end
+
+  it 'generates modules according to the namespace map' do
+    AvroGen.config.use_full_namespace = true
+    AvroGen.config.schema_namespace_map = {
+      'com' => 'Schemas',
+      'com.my-namespace.my-suborg' => %w(Schemas MyNamespace)
+    }
+    described_class.new.generate_from_configs(configs)
+    expect(files).to match_snapshot('namespace_map', snapshot_serializer: MultiFileSerializer)
+  end
+
+  # The snapshots above cover the full output, but these focused cases make it
+  # explicit that each notable Avro construct generates correctly (these mirror
+  # the per-schema scenarios that previously lived in Deimos).
+  describe 'specific Avro constructs' do
+    before(:each) do
+      AvroGen.config.nest_child_schemas = true
+      described_class.new.generate_from_path
+    end
+
+    {
+      'records with complex types (arrays, maps, nested records and enums)' => 'my_schema_with_complex_type',
+      'records with a circular reference' => 'my_schema_with_circular_reference',
+      'records with union types' => 'my_schema_with_union_type',
+      'records with nested child records' => 'my_nested_schema',
+      'records with date/time logical types' => 'my_schema_with_date_time',
+      'records with boolean fields' => 'my_schema_with_boolean'
+    }.each do |description, file|
+      it "generates #{description}" do
+        expect(files.slice("#{schema_class_path}/#{file}.rb")).
+          to match_snapshot(file, snapshot_serializer: MultiFileSerializer)
+      end
+    end
+  end
+end

data/spec/my_schema_spec.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+RSpec.describe Schemas::MyNamespace::MySchema do
+  let(:key) { Schemas::MyNamespace::MySchemaKey.new(test_id: 123) }
+
+  it 'should produce a tombstone with a hash' do
+    result = described_class.tombstone({ test_id: 123 })
+    expect(result.payload_key).to eq(key)
+    expect(result.to_h).to eq({ payload_key: { 'test_id' => 123 } })
+  end
+
+  it 'should work with a record' do
+    key = Schemas::MyNamespace::MySchemaKey.new(test_id: 123)
+    result = described_class.tombstone(key)
+    expect(result.payload_key).to eq(key)
+    expect(result.to_h).to eq({ payload_key: { 'test_id' => 123 } })
+  end
+end

data/spec/my_schema_with_circular_reference_spec.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+# For testing the generated class.
+RSpec.describe Schemas::MyNamespace::MySchemaWithCircularReference do
+  let(:payload_hash) do
+    {
+      properties: {
+        a_boolean: {
+          'property' => true
+        },
+        an_integer: {
+          'property' => 1
+        },
+        a_float: {
+          'property' => 4.5
+        },
+        a_string: {
+          'property' => 'string'
+        },
+        an_array: {
+          'property' => [1, 2, 3]
+        },
+        an_hash: {
+          'property' => {
+            'a_key' => 'a_value'
+          }
+        }
+      }
+    }
+  end
+
+  describe 'class initialization' do
+    it 'should initialize the class from keyword arguments' do
+      klass = described_class.new(properties: payload_hash[:properties])
+      expect(klass).to be_instance_of(described_class)
+    end
+
+    it 'should initialize the class from a hash with symbols as keys' do
+      klass = described_class.new(**payload_hash)
+      expect(klass).to be_instance_of(described_class)
+    end
+
+    it 'should initialize the class when missing attributes' do
+      payload_hash.delete(:properties)
+      klass = described_class.new(**payload_hash)
+      expect(klass).to be_instance_of(described_class)
+    end
+  end
+
+  describe 'base class methods' do
+    let(:klass) do
+      described_class.new(**payload_hash)
+    end
+
+    it 'should return the name of the schema and namespace' do
+      expect(klass.schema).to eq('MySchemaWithCircularReference')
+      expect(klass.namespace).to eq('com.my-namespace')
+      expect(klass.full_schema).to eq('com.my-namespace.MySchemaWithCircularReference')
+    end
+
+    it 'should return a json version of the payload' do
+      described_class.new(**payload_hash)
+      payload_h = {
+        'properties' => {
+          a_boolean: {
+            'property' => true
+          },
+          an_integer: {
+            'property' => 1
+          },
+          a_float: {
+            'property' => 4.5
+          },
+          a_string: {
+            'property' => 'string'
+          },
+          an_array: {
+            'property' => [1, 2, 3]
+          },
+          an_hash: {
+            'property' => {
+              'a_key' => 'a_value'
+            }
+          }
+        }
+      }
+
+      expect(klass.as_json).to eq(payload_h)
+    end
+
+    it 'should return a JSON string of the payload' do
+      s = '{"properties":{"a_boolean":{"property":true},"an_integer":{"property":1},"a_float":{"property":4.5},"a_str' \
+          'ing":{"property":"string"},"an_array":{"property":[1,2,3]},"an_hash":{"property":{"a_key":"a_value"}}}}'
+      expect(klass.to_json).to eq(s)
+    end
+  end
+end

data/spec/my_schema_with_complex_types_spec.rb

@@ -0,0 +1,235 @@
+# frozen_string_literal: true
+
+# For testing the generated class.
+RSpec.describe Schemas::MyNamespace::MySchemaWithComplexType do
+  let(:payload_hash) do
+    {
+      test_id: 'test id',
+      test_float: 1.2,
+      test_string_array: %w(abc def),
+      test_int_array: [123, 456],
+      some_integer_map: { 'int_1' => 1, 'int_2' => 2 },
+      some_record: described_class::ARecord.new(a_record_field: 'field 1'),
+      some_optional_record: described_class::ARecord.new(a_record_field: 'field 2'),
+      some_record_array: [described_class::ARecord.new(a_record_field: 'field 3'),
+                          described_class::ARecord.new(a_record_field: 'field 4')],
+      some_record_map: {
+        'record_1' => described_class::ARecord.new(a_record_field: 'field 5'),
+        'record_2' => described_class::ARecord.new(a_record_field: 'field 6')
+      },
+      some_enum_array: [described_class::AnEnum.new('sym1'),
+                        described_class::AnEnum.new('sym2')]
+    }
+  end
+
+  describe 'class initialization' do
+    it 'should initialize the class from keyword arguments' do
+      klass = described_class.new(
+        test_id: payload_hash[:test_id],
+        test_float: payload_hash[:test_float],
+        test_string_array: payload_hash[:test_string_array],
+        some_record: payload_hash[:some_record],
+        some_optional_record: payload_hash[:some_optional_record],
+        some_record_array: payload_hash[:some_record_array],
+        some_record_map: payload_hash[:some_record_map],
+        some_enum_array: payload_hash[:some_enum_array]
+      )
+      expect(klass).to be_instance_of(described_class)
+    end
+
+    it 'should initialize the class from a hash with symbols as keys' do
+      klass = described_class.new(**payload_hash)
+      expect(klass).to be_instance_of(described_class)
+    end
+
+    it 'should initialize the class when missing attributes' do
+      payload_hash.delete(:test_id)
+      klass = described_class.new(**payload_hash)
+      expect(klass).to be_instance_of(described_class)
+    end
+  end
+
+  describe '.tombstone' do
+    it 'should return a tombstone' do
+      record = described_class.tombstone('foo')
+      expect(record.tombstone_key).to eq('foo')
+      expect(record.to_h).to eq({ payload_key: 'foo' })
+    end
+  end
+
+  describe 'base class methods' do
+    let(:klass) do
+      described_class.new(**payload_hash)
+    end
+
+    let(:schema_fields) do
+      %w(test_id test_float test_int_array test_optional_int test_string_array some_integer_map
+         some_record some_optional_record some_record_array some_record_map some_enum_array
+         some_optional_enum some_enum_with_default union_string)
+    end
+
+    it 'should return the name of the schema and namespace' do
+      expect(klass.schema).to eq('MySchemaWithComplexTypes')
+      expect(klass.namespace).to eq('com.my-namespace')
+      expect(klass.full_schema).to eq('com.my-namespace.MySchemaWithComplexTypes')
+    end
+
+    it 'should return an array of all fields in the schema' do
+      expect(klass.schema_fields).to match_array(schema_fields)
+    end
+
+    it 'should return a json version of the payload' do
+      described_class.new(**payload_hash)
+      payload_h = {
+        'test_id' => 'test id',
+        'test_float' => 1.2,
+        'test_string_array' => %w(abc def),
+        'test_int_array' => [123, 456],
+        'some_optional_enum' => nil,
+        'test_optional_int' => 123,
+        'some_integer_map' => { 'int_1' => 1, 'int_2' => 2 },
+        'some_record' => { 'a_record_field' => 'field 1' },
+        'some_optional_record' => { 'a_record_field' => 'field 2' },
+        'some_record_array' => [
+          { 'a_record_field' => 'field 3' },
+          { 'a_record_field' => 'field 4' }
+        ],
+        'some_record_map' => {
+          'record_1' => { 'a_record_field' => 'field 5' },
+          'record_2' => { 'a_record_field' => 'field 6' }
+        },
+        'some_enum_array' => %w(sym1 sym2),
+        'some_enum_with_default' => 'sym6',
+        'union_string' => ''
+      }
+
+      expect(klass.as_json).to eq(payload_h)
+    end
+
+    it 'should return a JSON string of the payload' do
+      s = '{"test_id":"test ' \
+          'id","union_string":"","test_float":1.2,"test_string_array":["abc","def"],"test_int_array":[123,456],"test_' \
+          'optional_i' \
+          'nt":123,"some_integer_map":{"int_1":1,"int_2":2},"some_record":{"a_record_field":"field ' \
+          '1"},"some_optional_record":{"a_record_field":"field 2"},"some_record_array":[{"a_record_field":"field ' \
+          '3"},{"a_record_field":"field 4"}],"some_record_map":{"record_1":{"a_record_field":"field ' \
+          '5"},"record_2":{"a_record_field":"field ' \
+          '6"}},"some_enum_array":["sym1","sym2"],"some_optional_enum":null,"some_enum_with_default":"sym6"}'
+      expect(klass.to_json).to eq(s)
+    end
+  end
+
+  describe 'defaults' do
+    it 'should set an_optional_int if it is not provided' do
+      payload_hash.delete(:an_optional_int)
+      klass = described_class.new(**payload_hash)
+      expect(klass.test_optional_int).to eq(123)
+    end
+
+    it 'should set some_record if it is not provided' do
+      payload_hash.delete(:some_record)
+      klass = described_class.new(**payload_hash)
+      expect(klass.some_record).
+        to eq(described_class::ARecord.new(a_record_field: 'Test String'))
+    end
+
+    it 'should set some_record to nil' do
+      klass = described_class.new(**payload_hash, some_record: nil)
+      expect(klass.some_record).to be_nil
+    end
+  end
+
+  describe 'getters and setters' do
+    let(:klass) do
+      described_class.new(**payload_hash)
+    end
+
+    context 'when getting attributes' do
+      it 'should get of values of primitive types' do
+        expect(klass.test_id).to eq('test id')
+        expect(klass.test_float).to eq(1.2)
+        expect(klass.test_string_array).to eq(%w(abc def))
+      end
+
+      it 'should get the value of some_record_array' do
+        some_record_array = klass.some_record_array
+        expect(some_record_array.first).to be_instance_of(described_class::ARecord)
+        expect(some_record_array.first.a_record_field).to eq('field 3')
+      end
+
+      it 'should get the value of some_record_map' do
+        some_record_map = klass.some_record_map
+        expect(some_record_map['record_1']).
+          to be_instance_of(described_class::ARecord)
+        expect(some_record_map['record_1'].a_record_field).to eq('field 5')
+      end
+
+      it 'should get the value of some_enum_array' do
+        some_enum_array = klass.some_enum_array
+        expect(some_enum_array.first).to be_instance_of(described_class::AnEnum)
+        expect(some_enum_array.first.value).to eq('sym1')
+      end
+
+      it 'should get the value of some_record' do
+        record = klass.some_record
+        expect(record).to be_instance_of(described_class::ARecord)
+        expect(record.a_record_field).to eq('field 1')
+        expect(record.to_h).to eq({ 'a_record_field' => 'field 1' })
+      end
+
+      it 'should support Hash-style element access of values' do
+        expect(klass['test_id']).to eq('test id')
+        expect(klass['test_float']).to eq(1.2)
+        expect(klass['test_string_array']).to eq(%w(abc def))
+      end
+    end
+
+    context 'when setting attributes' do
+      it 'should modify the value of test_id' do
+        expect(klass.test_id).to eq('test id')
+
+        klass.test_id = 'something different'
+        expect(klass.test_id).to eq('something different')
+      end
+
+      it 'should modify the value of some_optional_record' do
+        expect(klass.some_optional_record).
+          to eq(described_class::ARecord.new(a_record_field: 'field 2'))
+        klass.some_optional_record = described_class::ARecord.
+          new(a_record_field: 'new field')
+
+        expect(klass.some_optional_record).to eq(described_class::ARecord.
+          new(a_record_field: 'new field'))
+        expect(klass.some_optional_record.as_json).to eq({ 'a_record_field' => 'new field' })
+      end
+
+      it 'should accept a hash object inner records' do
+        klass.some_optional_record = { a_record_field: 'new field' }
+        expect(klass.some_optional_record).
+          to eq(described_class::ARecord.new(a_record_field: 'new field'))
+        expect(klass.some_optional_record.as_json).to eq({ 'a_record_field' => 'new field' })
+      end
+
+      it 'should modify the value of some_enum_array' do
+        klass.some_enum_array.first.value = 'new_sym'
+        expect(klass.some_enum_array.first).
+          to eq(described_class::AnEnum.new('new_sym'))
+
+        klass.some_enum_array.second.value = described_class::AnEnum.
+          new('other_sym')
+        expect(klass.some_enum_array.second.value).to eq('other_sym')
+      end
+
+      it 'should modify the value of some_record_map' do
+        klass.some_record_map['record_1'].a_record_field = 'new field'
+        expect(klass.some_record_map['record_1']).to eq(described_class::ARecord.
+          new(a_record_field: 'new field'))
+
+        klass.some_record_map['record_2'] = described_class::ARecord.
+          new(a_record_field: 'other field')
+        expect(klass.some_record_map['record_2']).to eq(described_class::ARecord.
+          new(a_record_field: 'other field'))
+      end
+    end
+  end
+end