avrocadabra 0.0.1

data/lib/avrocadabra/avro_turf/ractor_support.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+module Avrocadabra
+  module AvroTurf
+    module RactorSupport
+      JSON_ERROR = ::MultiJson::ParseError
+      EXCON_DEFAULTS = Ractor.make_shareable(::Excon::DEFAULTS, copy: true)
+      LOGICAL_TYPES = Ractor.make_shareable(::Avro::LogicalTypes::TYPES, copy: true)
+      REQUEST_KEYS = Ractor.make_shareable(::Excon::VALID_REQUEST_KEYS, copy: true)
+      CONNECTION_KEYS = Ractor.make_shareable(::Excon::VALID_CONNECTION_KEYS, copy: true)
+      STATUS_ERRORS = Ractor.make_shareable(::Excon::Error.status_errors, copy: true)
+
+      module LogicalTypes
+        def type_adapter(type, logical_type, schema = nil)
+          return super if Ractor.main?
+          return unless logical_type
+
+          adapter = LOGICAL_TYPES.fetch(type, {}).fetch(logical_type, ::Avro::LogicalTypes::Identity)
+          adapter.is_a?(Class) ? adapter.new(schema) : adapter
+        end
+      end
+
+      module StatusErrors
+        def status_errors
+          return super if Ractor.main?
+
+          Ractor.store_if_absent(:avrocadabra_status_errors) { STATUS_ERRORS.transform_values(&:dup) }
+        end
+      end
+
+      module ConnectionKeys
+        def valid_request_keys(middlewares)
+          return super if Ractor.main?
+
+          valid_middleware_keys(middlewares) + REQUEST_KEYS
+        end
+
+        private
+
+        def validate_params(validation, params, middlewares)
+          return super if Ractor.main?
+
+          keys = case validation
+                 when :connection then valid_middleware_keys(middlewares) + CONNECTION_KEYS
+                 when :request then valid_request_keys(middlewares)
+                 else raise ArgumentError, "Invalid validation type '#{validation}'"
+                 end
+          if validation == :connection && params[:omit_default_port] != true
+            ::Excon.display_warning("The `omit_default_port` connection option is deprecated, " \
+                                    "please use `include_default_port` instead.")
+          end
+          invalid = params.keys - keys
+          return if invalid.empty?
+
+          ::Excon.display_warning("Invalid Excon #{validation} keys: #{invalid.map(&:inspect).join(", ")}")
+          return unless validation == :request
+
+          warn_deprecated_keys(invalid)
+        end
+
+        def warn_deprecated_keys(invalid)
+          messages = (invalid & ::Excon::DEPRECATED_VALID_REQUEST_KEYS.keys).map do |key|
+            "#{key}: #{::Excon::DEPRECATED_VALID_REQUEST_KEYS[key]}"
+          end
+          ::Excon.display_warning("The following request keys are only valid with the associated middleware: " \
+                                  "#{messages.join(", ")}")
+        end
+      end
+
+      module Parsing
+        def parse(json)
+          return super if Ractor.main?
+
+          real_parse(RactorSupport.parse_json(json), {})
+        end
+      end
+
+      module Serialization
+        def to_s
+          return super if Ractor.main?
+
+          JSON.generate(to_avro)
+        end
+      end
+
+      module Configuration
+        def disable_enum_symbol_validation
+          return super if Ractor.main?
+
+          @disable_enum_symbol_validation || ENV.fetch("AVRO_DISABLE_ENUM_SYMBOL_VALIDATION", "") != ""
+        end
+
+        def disable_field_default_validation
+          return super if Ractor.main?
+
+          @disable_field_default_validation || ENV.fetch("AVRO_DISABLE_FIELD_DEFAULT_VALIDATION", "") != ""
+        end
+
+        def disable_schema_name_validation
+          return super if Ractor.main?
+
+          @disable_schema_name_validation || ENV.fetch("AVRO_DISABLE_SCHEMA_NAME_VALIDATION", "") != ""
+        end
+      end
+
+      module ExconState
+        def defaults
+          return super if Ractor.main?
+
+          Ractor.store_if_absent(:avrocadabra_excon_defaults) { RactorSupport.excon_defaults }
+        end
+
+        def defaults=(value)
+          if Ractor.main?
+            super
+          else
+            Ractor[:avrocadabra_excon_defaults] = value || RactorSupport.excon_defaults
+          end
+        end
+      end
+
+      class << self
+        def excon_defaults
+          EXCON_DEFAULTS.transform_values do |value|
+            value.is_a?(Hash) || value.is_a?(Array) ? value.dup : value
+          end
+        end
+
+        def prepare
+          %i[PRIMITIVE_TYPES NAMED_TYPES VALID_TYPES PRIMITIVE_TYPES_SYM NAMED_TYPES_SYM VALID_TYPES_SYM].each do |name|
+            Ractor.make_shareable(::Avro::Schema.const_get(name))
+          end
+          Ractor.make_shareable(::Avro::LogicalTypes::IntDate::EPOCH_START)
+          ::Avro::LogicalTypes.singleton_class.prepend(LogicalTypes)
+          ::Excon::Error.singleton_class.prepend(StatusErrors)
+          ::Excon::Connection.prepend(ConnectionKeys)
+          ::Avro::Schema.singleton_class.prepend(Parsing)
+          ::Avro::Schema.prepend(Serialization)
+          ::Avro.singleton_class.prepend(Configuration)
+          ::Excon.singleton_class.prepend(ExconState)
+        end
+
+        def parse_json(json)
+          json = json.read if json.respond_to?(:read)
+          return if json.nil? || json.strip.empty?
+
+          JSON.parse(json)
+        rescue JSON::ParserError => e
+          raise JSON_ERROR.build(e, json)
+        end
+      end
+    end
+  end
+end

data/lib/avrocadabra/avro_turf/routing.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Avrocadabra
+  module AvroTurf
+    module Routing
+      def encode(...)
+        AvroTurf.with_codecs(@avrocadabra_codecs) { super }
+      end
+
+      def decode_message(...)
+        AvroTurf.with_codecs(@avrocadabra_codecs) { super }
+      end
+    end
+  end
+end

data/lib/avrocadabra/avro_turf/schema_state.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Avrocadabra
+  module AvroTurf
+    class SchemaState
+      FIELD_ATTRIBUTES = %i[name type default default? aliases].freeze
+
+      attr_reader :schemas
+
+      def initialize(source)
+        @schemas = Set.new.compare_by_identity
+        @attributes = []
+        @containers = []
+        visit(source, Set.new.compare_by_identity)
+      end
+
+      def current?
+        NativeSchema.unchanged?(@attributes, @containers)
+      end
+
+      private
+
+      def visit(schema, observed)
+        return if @schemas.include?(schema)
+
+        @schemas.add(schema)
+        %i[type_sym logical_type type_adapter name namespace aliases precision scale size symbols
+           default].each do |attribute|
+          observe(schema, attribute, observed) if schema.respond_to?(attribute)
+        end
+        case schema.type_sym
+        when :record, :error
+          observe(schema, :fields, observed).each do |field|
+            FIELD_ATTRIBUTES.each { observe(field, it, observed) }
+            visit(field.type, observed)
+          end
+        when :array then visit(observe(schema, :items, observed), observed)
+        when :map then visit(observe(schema, :values, observed), observed)
+        when :union then observe(schema, :schemas, observed).each { visit(it, observed) }
+        end
+      end
+
+      def observe(object, attribute, observed)
+        value = object.public_send(attribute)
+        @attributes.push(object, attribute, value)
+        watch(value, observed)
+        value
+      end
+
+      def watch(value, observed)
+        return if observed.include?(value)
+
+        observed.add(value)
+        case value
+        when String
+          @containers.push(value, value.dup.freeze) unless value.frozen?
+        when Array
+          @containers.push(value, value.dup.freeze) unless value.frozen?
+          value.each { watch(it, observed) }
+        when Hash
+          @containers.push(value, value.to_a.flatten(1).freeze) unless value.frozen?
+          value.each do |key, child|
+            watch(key, observed)
+            watch(child, observed)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/avrocadabra/avro_turf/validation.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Avrocadabra
+  module AvroTurf
+    module Validation
+      def validate!(schema, value, options = ::Avro::SchemaValidator::DEFAULT_VALIDATION_OPTIONS)
+        if Thread.current[:avrocadabra_codecs] && !options.key?(:avrocadabra_budget)
+          options = options.merge(avrocadabra_budget: [Schema::MAX_ITEMS, Schema::MAX_DEPTH + 1])
+        end
+        super
+      rescue EncodeError => e
+        raise ::Avro::IO::AvroTypeError.new(schema, value), cause: e
+      end
+
+      private
+
+      def validate_recursive(schema, value, path, result, options)
+        budget = options[:avrocadabra_budget]
+        return super unless budget
+
+        depth = schema.type_sym == :union ? 0 : 1
+        budget[0] -= 1
+        budget[1] -= depth
+        raise EncodeError, "#{path}: union search exceeds maximum item count" if budget[0].negative?
+        raise EncodeError, "#{path}: union search exceeds maximum depth" if budget[1].negative?
+
+        super
+      ensure
+        budget[1] += depth if budget
+      end
+    end
+  end
+end

data/lib/avrocadabra/avro_turf.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "avrocadabra"
+require "avro_turf/messaging"
+
+module Avrocadabra
+  module AvroTurf
+    RactorSupport.prepare
+
+    class << self
+      def with_codecs(cache)
+        previous = Thread.current[:avrocadabra_codecs]
+        Thread.current[:avrocadabra_codecs] = cache
+        yield
+      ensure
+        Thread.current[:avrocadabra_codecs] = previous
+      end
+    end
+
+    ::AvroTurf::Messaging.prepend(Routing)
+    ::Avro::IO::DatumWriter.prepend(DatumWriter)
+    ::Avro::IO::DatumReader.prepend(DatumReader)
+    ::Avro::SchemaValidator.singleton_class.prepend(Validation)
+  end
+end

data/lib/avrocadabra/logical.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Avrocadabra
+  module Logical
+    EPOCH_JD = 2_440_588
+
+    class << self
+      def decimal_unscaled(value, precision, scale)
+        sign, digits, _base, exponent = decimal_input(value).split
+        return "0" if digits == "0"
+
+        shift = exponent - digits.length + scale
+        raise EncodeError, "decimal has excess scale (maximum #{scale})" if shift.negative?
+        raise EncodeError, "decimal exceeds precision #{precision}" if digits.length + shift > precision
+
+        "#{"-" if sign.negative?}#{digits}#{"0" * shift}"
+      end
+
+      def decimal_value(unscaled, scale)
+        value = BigDecimal("#{unscaled}e#{-scale}")
+        unless value.finite? && (!value.zero? || unscaled == "0")
+          raise DecodeError, "decimal exponent exceeds Ruby BigDecimal range"
+        end
+
+        value
+      end
+
+      def big_decimal_parts(value)
+        sign, digits, _base, exponent = decimal_input(value).split
+        return ["0", 0] if digits == "0"
+
+        [sign.negative? ? "-#{digits}" : digits, digits.length - exponent]
+      end
+
+      def date_days(value)
+        return value.to_i if value.is_a?(Numeric)
+        raise EncodeError, "date requires Date or Numeric" unless value.is_a?(Date)
+
+        value.jd - EPOCH_JD
+      end
+
+      def date_value(days)
+        Date.jd(EPOCH_JD + days, Date::GREGORIAN)
+      end
+
+      def timestamp_ticks(value, units)
+        return value.to_i if value.is_a?(Numeric)
+        raise EncodeError, "timestamp requires Time, Date or Numeric" unless value.respond_to?(:to_time)
+
+        (value.to_time.to_r * units).floor
+      end
+
+      def timestamp_value(ticks, units)
+        Time.at(Rational(ticks, units)).utc
+      end
+
+      private
+
+      def decimal_input(value)
+        unless value.is_a?(BigDecimal) || value.is_a?(Integer) || value.is_a?(Float)
+          raise EncodeError, "decimal requires BigDecimal, Integer or Float"
+        end
+
+        value = value.to_d
+        raise EncodeError, "decimal must be finite" unless value.finite?
+
+        value
+      end
+    end
+  end
+end

data/lib/avrocadabra/schema.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Avrocadabra
+  class Schema
+    MAX_DEPTH = 64
+    MAX_ITEMS = 1_000_000
+
+    def initialize(schema, references: [], max_depth: MAX_DEPTH, max_bytes: 16 * 1024 * 1024, max_items: MAX_ITEMS)
+      raise SchemaError, "references must be an Array of schemas" unless references.is_a?(Array)
+
+      @native = NativeSchema.new(
+        schema_json(schema), references.map { schema_json(it) }, max_depth, max_bytes, max_items
+      )
+      Ractor.make_shareable(self)
+    rescue JSON::JSONError, TypeError, ArgumentError, RangeError => e
+      raise SchemaError, e.message
+    end
+
+    def encode(data, release_gvl: false)
+      @native.encode(data, release_gvl, nil)
+    rescue TypeError, ArgumentError, RangeError, EncodingError => e
+      raise EncodeError, e.message
+    end
+
+    def decode(bytes, reader_schema: nil, release_gvl: false, tagged_unions: false)
+      unless reader_schema.nil? || reader_schema.is_a?(Schema)
+        raise DecodeError, "reader_schema must be an Avrocadabra::Schema"
+      end
+
+      @native.decode(bytes, reader_schema&.native, release_gvl, tagged_unions, nil)
+    rescue TypeError, ArgumentError, RangeError, EncodingError => e
+      raise DecodeError, e.message
+    end
+
+    protected
+
+    attr_reader :native
+
+    private
+
+    def schema_json(schema)
+      return schema if schema.is_a?(String)
+      raise SchemaError, "schema must be JSON, a Hash, or an Array" unless schema.is_a?(Hash) || schema.is_a?(Array)
+
+      JSON.generate(schema, max_nesting: 128)
+    end
+  end
+end

data/lib/avrocadabra/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Avrocadabra
+  VERSION = "0.0.1"
+end

data/lib/avrocadabra.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "json"
+require "bigdecimal"
+require "bigdecimal/util"
+require "date"
+require "stringio"
+require "zeitwerk"
+
+loader = Zeitwerk::Loader.for_gem
+loader.inflector.inflect("avro_turf" => "AvroTurf")
+loader.do_not_eager_load("#{__dir__}/avrocadabra/avro_turf.rb", "#{__dir__}/avrocadabra/avro_turf")
+loader.setup
+
+module Avrocadabra
+  class Error < StandardError; end
+  class SchemaError < Error; end
+  class EncodeError < Error; end
+  class DecodeError < Error; end
+  class ResolutionError < DecodeError; end
+
+  Union = Data.define(:branch, :value)
+  Duration = Data.define(:months, :days, :milliseconds)
+end
+
+begin
+  require "avrocadabra/#{RUBY_VERSION.split(".").first(2).join(".")}/avrocadabra"
+rescue LoadError => e
+  raise unless e.path == "avrocadabra/#{RUBY_VERSION.split(".").first(2).join(".")}/avrocadabra"
+
+  require "avrocadabra/avrocadabra"
+end
+
+loader.eager_load

metadata

@@ -0,0 +1,183 @@
+--- !ruby/object:Gem::Specification
+name: avrocadabra
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Alexander Grebennik
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bigdecimal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: date
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.4'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.9'
+- !ruby/object:Gem::Dependency
+  name: rb_sys
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.8'
+description: Reusable Avro schemas, raw datum encoding and decoding, and schema resolution
+  through Apache Avro
+email:
+- slbug@users.noreply.github.com
+- sl.bug.sl@gmail.com
+executables: []
+extensions:
+- ext/avrocadabra/extconf.rb
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- Cargo.lock
+- Cargo.toml
+- LICENSE.txt
+- README.md
+- docs/avro_turf.md
+- docs/licenses/crates/quad-rand-0.2.3/DECLARED-LICENSE.txt
+- docs/licenses/gcc-16.2.0/COPYING
+- docs/licenses/gcc-16.2.0/COPYING.RUNTIME
+- docs/licenses/mingw-w64-14.0.0/AUTHORS
+- docs/licenses/mingw-w64-14.0.0/COPYING
+- docs/licenses/rust-1.99.0/COMPILER-BUILTINS.txt
+- docs/licenses/rust-1.99.0/COPYRIGHT-library.html
+- docs/licenses/rust-1.99.0/LIBM.txt
+- docs/licenses/rust-1.99.0/LLVM-LIBUNWIND.txt
+- docs/licenses/rust-1.99.0/RUST-LICENSE-APACHE.txt
+- docs/licenses/rust-1.99.0/RUST-LICENSE-MIT.txt
+- docs/licenses/rust-1.99.0/STDLIB-BACKTRACE-MIT.txt
+- docs/licenses/rust-1.99.0/STDLIB-PORTABLE-SIMD-MIT.txt
+- docs/licenses/rust-1.99.0/STDLIB-STDARCH-MIT.txt
+- docs/licenses/rust-1.99.0/licenses/Apache-2.0.txt
+- docs/licenses/rust-1.99.0/licenses/BSD-2-Clause.txt
+- docs/licenses/rust-1.99.0/licenses/CC-BY-SA-4.0.txt
+- docs/licenses/rust-1.99.0/licenses/GCC-exception-3.1.txt
+- docs/licenses/rust-1.99.0/licenses/GPL-2.0-only.txt
+- docs/licenses/rust-1.99.0/licenses/GPL-3.0-or-later.txt
+- docs/licenses/rust-1.99.0/licenses/ISC.txt
+- docs/licenses/rust-1.99.0/licenses/LLVM-exception.txt
+- docs/licenses/rust-1.99.0/licenses/MIT.txt
+- docs/licenses/rust-1.99.0/licenses/NCSA.txt
+- docs/licenses/rust-1.99.0/licenses/OFL-1.1.txt
+- docs/licenses/rust-1.99.0/licenses/Unicode-3.0.txt
+- docs/releasing.md
+- docs/third_party.md
+- ext/avrocadabra/Cargo.toml
+- ext/avrocadabra/build.rs
+- ext/avrocadabra/extconf.rb
+- ext/avrocadabra/src/big_decimal.rs
+- ext/avrocadabra/src/callback.rs
+- ext/avrocadabra/src/convert.rs
+- ext/avrocadabra/src/decode_value.rs
+- ext/avrocadabra/src/guard.rs
+- ext/avrocadabra/src/lib.rs
+- ext/avrocadabra/src/mapping.rs
+- ext/avrocadabra/src/memory.rs
+- ext/avrocadabra/src/prepare.rs
+- ext/avrocadabra/src/resolution.rs
+- ext/avrocadabra/src/schema_state.rs
+- ext/avrocadabra/src/validation.rs
+- ext/avrocadabra/src/wire.rs
+- lib/avrocadabra.rb
+- lib/avrocadabra/avro_turf.rb
+- lib/avrocadabra/avro_turf/cache.rb
+- lib/avrocadabra/avro_turf/codec.rb
+- lib/avrocadabra/avro_turf/datum_reader.rb
+- lib/avrocadabra/avro_turf/datum_writer.rb
+- lib/avrocadabra/avro_turf/mapping.rb
+- lib/avrocadabra/avro_turf/messaging.rb
+- lib/avrocadabra/avro_turf/ractor_support.rb
+- lib/avrocadabra/avro_turf/routing.rb
+- lib/avrocadabra/avro_turf/schema_state.rb
+- lib/avrocadabra/avro_turf/validation.rb
+- lib/avrocadabra/logical.rb
+- lib/avrocadabra/schema.rb
+- lib/avrocadabra/version.rb
+homepage: https://github.com/slbug/avrocadabra
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/slbug/avrocadabra
+  source_code_uri: https://github.com/slbug/avrocadabra/tree/main
+  changelog_uri: https://github.com/slbug/avrocadabra/blob/main/CHANGELOG.md
+  rubygems_mfa_required: 'true'
+  cargo_crate_name: avrocadabra
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '4.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.21
+specification_version: 4
+summary: Avro binary encoding and decoding with a Rust native extension
+test_files: []