schema_registry_client 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 19d8a1a9d31add74657d9dcf9102bf5b05cca485f15fbaf50a3d4490ce517a2a
-  data.tar.gz: 413516fa57982d58ebacd97bfcc087842aba1192e5f7ef622a4082e4c9b09b60
+  metadata.gz: 5bd3f6c9d59bc29bfbcc32577761bd90b17282dc63b7423a1d74978235b316c8
+  data.tar.gz: fde47855e0974f734724d0c9fbe36b80644be5ed2f42215cbfa389e4951affdb
 SHA512:
-  metadata.gz: 5e345a78b8ea4495f1d8a09edc61aed48e0b5c39c9f1e6ca5df69027982916ea4fa5be192e6e99527ee318203ded60ed0a76398843f135541858a978443530b1
-  data.tar.gz: '086b30ff39befcc21e1e0842809187c81e6adc704b069a38d89213ca1f8865c26c57f6a36e36c886cfa5d59f4f35d02c9aaac4090469524741d9679b17b24653'
+  metadata.gz: dd97621d317f2f951c960b2f928832004c8d38b8c4f0ebdf291bbdb68ed7d873f83f4a789426479ead787f77f1dc00c2a5c1898407373a1fdc27b448c6ed1f97
+  data.tar.gz: fe7f21af487e6e8294977e5b50e77bd2a77648df7eb1b9bd4cb56a709fe09e0401ecba9e44ba35854b6383d35fcb2baf3530cf485963f7887a4e1fa8c0fc6dc4

data/.github/workflows/lint.yml

@@ -0,0 +1,18 @@
+name: Lint
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Ruby ${{ matrix.ruby }}
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 3.4
+        bundler-cache: true
+    - name: Run standardrb
+      run: bundle exec standardrb

data/.github/workflows/release.yml

@@ -0,0 +1,31 @@
+name: Release Gem
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - 'v*.*.*' # Matches semantic versioning tags like v1.0.0
+  workflow_dispatch: # Allows manual triggering of the workflow
+
+jobs:
+  push:
+    name: Push gem to RubyGems.org
+    runs-on: ubuntu-latest
+
+    permissions:
+      id-token: write # IMPORTANT: this permission is mandatory for trusted publishing
+      contents: write # IMPORTANT: this permission is required for `rake release` to push the release tag
+
+    steps:
+      # Set up
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+          ruby-version: 3.4
+
+      # Release
+      - uses: rubygems/release-gem@v1

data/.github/workflows/test.yml

@@ -0,0 +1,22 @@
+name: Test
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [3.1, 3.2, 3.3, 3.4]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Ruby ${{ matrix.ruby }}
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+        bundler-cache: true
+    - name: Build and test with RSpec
+      run: bundle exec rspec

data/.gitignore

@@ -0,0 +1 @@
+coverage

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,39 @@
+AllCops:
+  Exclude:
+    - app/gen/**/*
+    - spec/gen/**/*
+
+Metrics/MethodLength:
+  Severity: refactor
+  Max: 60
+
+Metrics/ModuleLength:
+  Severity: refactor
+  Max: 800
+
+Metrics/CyclomaticComplexity:
+  Severity: refactor
+  Max: 25
+
+Metrics/ClassLength:
+  Severity: refactor
+  Max: 800
+
+
+Metrics/BlockLength:
+  Enabled: false
+
+Metrics/AbcSize:
+  Severity: refactor
+  Max: 60
+
+Metrics/PerceivedComplexity:
+  Severity: refactor
+  Max: 15
+
+Style/Documentation:
+  Enabled: false
+
+Lint/UnusedMethodArgument:
+  AllowUnusedKeywordArguments: true
+

data/CHANGELOG.md

@@ -0,0 +1,12 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## UNRELEASED
+
+# 0.0.6 - 2026-01-02
+
+* Initial release.

data/Gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,150 @@
+PATH
+  remote: .
+  specs:
+    schema_registry_client (0.0.6)
+      avro
+      excon
+      google-protobuf
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.8.8)
+      public_suffix (>= 2.0.2, < 8.0)
+    ast (2.4.3)
+    avro (1.12.1)
+      multi_json (~> 1.0)
+    bigdecimal (4.0.1)
+    crack (1.0.1)
+      bigdecimal
+      rexml
+    diff-lcs (1.6.2)
+    docile (1.4.1)
+    excon (1.3.2)
+      logger
+    google-protobuf (4.33.2)
+      bigdecimal
+      rake (>= 13)
+    google-protobuf (4.33.2-aarch64-linux-gnu)
+      bigdecimal
+      rake (>= 13)
+    google-protobuf (4.33.2-aarch64-linux-musl)
+      bigdecimal
+      rake (>= 13)
+    google-protobuf (4.33.2-arm64-darwin)
+      bigdecimal
+      rake (>= 13)
+    google-protobuf (4.33.2-x86-linux-gnu)
+      bigdecimal
+      rake (>= 13)
+    google-protobuf (4.33.2-x86-linux-musl)
+      bigdecimal
+      rake (>= 13)
+    google-protobuf (4.33.2-x86_64-darwin)
+      bigdecimal
+      rake (>= 13)
+    google-protobuf (4.33.2-x86_64-linux-gnu)
+      bigdecimal
+      rake (>= 13)
+    google-protobuf (4.33.2-x86_64-linux-musl)
+      bigdecimal
+      rake (>= 13)
+    hashdiff (1.2.1)
+    json (2.18.0)
+    language_server-protocol (3.17.0.5)
+    lint_roller (1.1.0)
+    logger (1.7.0)
+    multi_json (1.19.1)
+    parallel (1.27.0)
+    parser (3.3.10.0)
+      ast (~> 2.4.1)
+      racc
+    prism (1.7.0)
+    public_suffix (7.0.0)
+    racc (1.8.1)
+    rainbow (3.1.1)
+    rake (13.3.1)
+    regexp_parser (2.11.3)
+    rexml (3.4.4)
+    rspec (3.13.2)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.6)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.5)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.7)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.6)
+    rubocop (1.81.7)
+      json (~> 2.3)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.1.0)
+      parallel (~> 1.10)
+      parser (>= 3.3.0.2)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 2.9.3, < 3.0)
+      rubocop-ast (>= 1.47.1, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 4.0)
+    rubocop-ast (1.49.0)
+      parser (>= 3.3.7.2)
+      prism (~> 1.7)
+    rubocop-performance (1.26.1)
+      lint_roller (~> 1.1)
+      rubocop (>= 1.75.0, < 2.0)
+      rubocop-ast (>= 1.47.1, < 2.0)
+    ruby-progressbar (1.13.0)
+    simplecov (0.22.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.13.2)
+    simplecov_json_formatter (0.1.4)
+    standard (1.52.0)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.0)
+      rubocop (~> 1.81.7)
+      standard-custom (~> 1.0.0)
+      standard-performance (~> 1.8)
+    standard-custom (1.0.2)
+      lint_roller (~> 1.0)
+      rubocop (~> 1.50)
+    standard-performance (1.9.0)
+      lint_roller (~> 1.1)
+      rubocop-performance (~> 1.26.0)
+    standardrb (1.0.1)
+      standard
+    unicode-display_width (3.2.0)
+      unicode-emoji (~> 4.1)
+    unicode-emoji (4.2.0)
+    webmock (3.26.1)
+      addressable (>= 2.8.0)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
+
+PLATFORMS
+  aarch64-linux-gnu
+  aarch64-linux-musl
+  arm64-darwin
+  ruby
+  x86-linux-gnu
+  x86-linux-musl
+  x86_64-darwin
+  x86_64-linux-gnu
+  x86_64-linux-musl
+
+DEPENDENCIES
+  bundler (~> 2.0)
+  rake (~> 13.0)
+  rspec (~> 3.2)
+  schema_registry_client!
+  simplecov
+  standardrb
+  webmock
+
+BUNDLED WITH
+   2.6.9

data/LICENSE

@@ -0,0 +1,20 @@
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,48 @@
+# schema_registry_client
+
+`schema_registry_client` is a library to interact with the Confluent Schema Registry using Google Protobuf. It is inspired by and based off of [avro_turf](https://github.com/dasch/avro_turf).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'schema_registry_client'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install schema_registry_client
+
+## Usage
+
+SchemaRegistry interacts with the Confluent Schema Registry, and caches all results. When you first encode a message, it will register the message and all dependencies with the Schema Registry. When decoding, it will look up the schema in the Schema Registry and use the associated local generated code to decode the message.
+
+Example usage:
+
+```ruby
+require 'schema_registry_client'
+
+schema_registry_client = SchemaRegistry.new(registry_url: 'http://localhost:8081', schema_paths: ['path/to/protos'])
+message = MyProto::MyMessage.new(field1: 'value1', field2: 42)
+encoded = schema_registry_client.encode(message, subject: 'my-subject')
+
+# Decoding
+
+decoded_proto_message = schema_registry_client.decode(encoded_string)
+```
+
+## Notes about usage
+
+* When decoding, this library does *not* attempt to fully parse the Proto definition stored on the schema registry and generate dynamic classes. Instead, it simply parses out the package and message and assumes that the reader has the message available in the descriptor pool. Any compatibility issues should be detected through normal means, i.e. just by instantiating the message and seeing if any errors are raised.
+
+### Regenerating test protos
+Run the following to regenerate:
+
+```sh
+protoc -I spec/schemas --ruby_out=spec/gen --ruby_opt=paths=source_relative spec/schemas/**/*.proto
+```

data/Rakefile

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'

data/lib/schema_registry_client/avro_schema_store.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require 'avro'
+
+class SchemaRegistry
+  class AvroSchemaStore
+    def initialize(path: nil)
+      @path = path or raise 'Please specify a schema path'
+      @schemas = {}
+      @schema_text = {}
+      @mutex = Mutex.new
+    end
+
+    attr_accessor :schemas
+
+    def find_text(name)
+      @schema_text[name]
+    end
+
+    # Resolves and returns a schema.
+    #
+    # schema_name - The String name of the schema to resolve.
+    #
+    # Returns an Avro::Schema.
+    def find(name)
+      # Optimistic non-blocking read from @schemas
+      # No sense to lock the resource when all the schemas already loaded
+      return @schemas[name] if @schemas.key?(name)
+
+      # Pessimistic blocking write to @schemas
+      @mutex.synchronize do
+        # Still need to check is the schema already loaded
+        return @schemas[name] if @schemas.key?(name)
+
+        load_schema!(name, @schemas.dup)
+      end
+    end
+
+    # Loads all schema definition files in the `schemas_dir`.
+    def load_schemas!
+      pattern = [@path, '**', '*.avsc'].join('/')
+
+      Dir.glob(pattern) do |schema_path|
+        # Remove the path prefix.
+        schema_path.sub!(%r{^/?#{@path}/}, '')
+
+        # Replace `/` with `.` and chop off the file extension.
+        schema_name = File.basename(schema_path.tr('/', '.'), '.avsc')
+
+        # Load and cache the schema.
+        find(schema_name)
+      end
+    end
+
+    # @param schema_hash [Hash]
+    def add_schema(schema_hash)
+      name = schema_hash['name']
+      namespace = schema_hash['namespace']
+      full_name = Avro::Name.make_fullname(name, namespace)
+      return if @schemas.key?(full_name)
+
+      # We pass in copy of @schemas which Avro can freely modify
+      # and register the sub-schema. It doesn't matter because
+      # we will discard it.
+      schema = Avro::Schema.real_parse(schema_hash, @schemas.dup)
+      @schemas[full_name] = schema
+      @schema_text[full_name] = JSON.pretty_generate(schema_hash)
+
+      schema
+    end
+
+    protected
+
+    # Loads single schema
+    # Such method is not thread-safe, do not call it of from mutex synchronization routine
+    def load_schema!(fullname, local_schemas_cache = {})
+      schema_path = build_schema_path(fullname)
+      schema_text = File.read(schema_path)
+      schema_json = JSON.parse(schema_text)
+
+      schema = Avro::Schema.real_parse(schema_json, local_schemas_cache)
+
+      # Don't cache the parsed schema until after its fullname is validated
+      if schema.respond_to?(:fullname) && schema.fullname != fullname
+        raise SchemaRegistry::SchemaError, "expected schema `#{schema_path}' to define type `#{fullname}'"
+      end
+
+      # Cache only this new top-level schema by its fullname. It's critical
+      # not to make every sub-schema resolvable at the top level here because
+      # multiple different avsc files may define the same sub-schema, and
+      # if we share the @schemas cache across all parsing contexts, the Avro
+      # gem will raise an Avro::SchemaParseError when parsing another avsc
+      # file that contains a subschema with the same fullname as one
+      # encountered previously in a different file:
+      # <Avro::SchemaParseError: The name "foo.bar" is already in use.>
+      # Essentially, the only schemas that should be resolvable in @schemas
+      # are those that have their own .avsc files on disk.
+      @schemas[fullname] = schema
+      @schema_text[fullname] = schema_text
+
+      schema
+    rescue ::Avro::UnknownSchemaError => e
+      # Try to first resolve a referenced schema from disk.
+      # If this is successful, the Avro gem will have mutated the
+      # local_schemas_cache, adding all the new schemas it found.
+      load_schema!(::Avro::Name.make_fullname(e.type_name, e.default_namespace), local_schemas_cache)
+
+      # Attempt to re-parse the original schema now that the dependency
+      # has been resolved and use the now-updated local_schemas_cache to
+      # pick up where we left off.
+      local_schemas_cache.delete(fullname)
+      # Ensure all sub-schemas are cleaned up to avoid conflicts when re-parsing
+      # schema.
+      local_schemas_cache.each_key do |schema_name|
+        local_schemas_cache.delete(schema_name) unless File.exist?(build_schema_path(schema_name))
+      end
+      load_schema!(fullname, @schemas.dup)
+    rescue Errno::ENOENT, Errno::ENAMETOOLONG
+      raise "could not find Avro schema at `#{schema_path}'"
+    end
+
+    def build_schema_path(fullname)
+      *namespace, schema_name = fullname.split('.')
+      File.join(@path, *namespace, "#{schema_name}.avsc")
+    end
+  end
+end

data/lib/schema_registry_client/cached_confluent_schema_registry.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+class SchemaRegistry
+  class CachedConfluentSchemaRegistry
+    # @param upstream [SchemaRegistry::ConfluentSchemaRegistry]
+    def initialize(upstream)
+      @upstream = upstream
+      @schemas_by_id = {}
+      @ids_by_schema = {}
+      @versions_by_subject_and_id = {}
+    end
+
+    # Delegate the following methods to the upstream
+    %i[subject_versions schema_subject_versions].each do |name|
+      define_method(name) do |*args|
+        instance_variable_get(:@upstream).send(name, *args)
+      end
+    end
+
+    # @param id [Integer] the schema ID to fetch
+    # @return [String] the schema string stored in the registry for the given id
+    def fetch(id)
+      @schemas_by_id[id] ||= @upstream.fetch(id)
+    end
+
+    # @param id [Integer] the schema ID to fetch
+    # @param subject [String] the subject to fetch the version for
+    # @return [Integer, nil] the version of the schema for the given subject and id, or nil if not found
+    def fetch_version(id, subject)
+      key = [subject, id]
+      return @versions_by_subject_and_id[key] if @versions_by_subject_and_id[key]
+
+      results = @upstream.schema_subject_versions(id)
+      @versions_by_subject_and_id[key] = results&.find { |r| r['subject'] == subject }&.dig('version')
+    end
+
+    # @param subject [String] the subject to check
+    # @param schema [String] the schema text to check
+    # @return [Boolean] true if we know the schema has been registered for that subject.
+    def registered?(subject, schema)
+      @ids_by_schema[[subject, schema]] && !@ids_by_schema[[subject, schema]].empty?
+    end
+
+    # @param subject [String] the subject to register the schema under
+    # @param schema [String] the schema text to register
+    # @param references [Array<Hash>] optional references to other schemas
+    # @param schema_type [String]
+    def register(subject, schema, references: [], schema_type: 'PROTOBUF')
+      key = [subject, schema]
+
+      @ids_by_schema[key] ||= @upstream.register(subject,
+                                                 schema,
+                                                 references: references,
+                                                 schema_type: schema_type)
+    end
+  end
+end

data/lib/schema_registry_client/confluent_schema_registry.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require 'excon'
+
+class SchemaRegistry
+  class ConfluentSchemaRegistry
+    CONTENT_TYPE = 'application/vnd.schemaregistry.v1+json'
+
+    def initialize( # rubocop:disable Metrics/ParameterLists
+      url,
+      schema_context: nil,
+      logger: Logger.new($stdout),
+      proxy: nil,
+      user: nil,
+      password: nil,
+      ssl_ca_file: nil,
+      client_cert: nil,
+      client_key: nil,
+      client_key_pass: nil,
+      client_cert_data: nil,
+      client_key_data: nil,
+      path_prefix: nil,
+      connect_timeout: nil,
+      resolv_resolver: nil,
+      retry_limit: nil
+    )
+      @path_prefix = path_prefix
+      @schema_context_prefix = schema_context.nil? ? '' : ":.#{schema_context}:"
+      @schema_context_options = schema_context.nil? ? {} : { query: { subject: @schema_context_prefix } }
+      @logger = logger
+      headers = Excon.defaults[:headers].merge(
+        'Content-Type' => CONTENT_TYPE
+      )
+      params = {
+        headers: headers,
+        user: user,
+        password: password,
+        proxy: proxy,
+        ssl_ca_file: ssl_ca_file,
+        client_cert: client_cert,
+        client_key: client_key,
+        client_key_pass: client_key_pass,
+        client_cert_data: client_cert_data,
+        client_key_data: client_key_data,
+        resolv_resolver: resolv_resolver,
+        connect_timeout: connect_timeout,
+        retry_limit: retry_limit
+      }
+      # Remove nil params to allow Excon to use its default values
+      params.reject! { |_, v| v.nil? }
+      @connection = Excon.new(
+        url,
+        params
+      )
+    end
+
+    # @param id [Integer] the schema ID to fetch
+    # @return [String] the schema string stored in the registry for the given id
+    def fetch(id)
+      @logger.info "Fetching schema with id #{id}"
+      data = get("/schemas/ids/#{id}", idempotent: true, **@schema_context_options)
+      data.fetch('schema')
+    end
+
+    # @param schema_id [Integer] the schema ID to fetch versions for
+    # @return [Array<Hash>] an array of versions for the given schema ID
+    def schema_subject_versions(schema_id)
+      get("/schemas/ids/#{schema_id}/versions", idempotent: true, **@schema_context_options)
+    end
+
+    # @param subject [String] the subject to check
+    # @param schema [String] the schema text to check
+    # @param references [Array<Hash>] optional references to other schemas
+    # @return [Integer] the ID of the registered schema
+    def register(subject, schema, references: [], schema_type: 'PROTOBUF')
+      data = post("/subjects/#{@schema_context_prefix}#{CGI.escapeURIComponent(subject)}/versions",
+                  body: { schemaType: schema_type,
+                          references: references,
+                          schema: schema.to_s }.to_json)
+
+      id = data.fetch('id')
+
+      @logger.info "Registered schema for subject `#{@schema_context_prefix}#{subject}`; id = #{id}"
+
+      id
+    end
+
+    # @param subject [String]
+    # @return [Array<Hash>] an array of versions for the given subject
+    def subject_versions(subject)
+      get("/subjects/#{@schema_context_prefix}#{CGI.escapeURIComponent(subject)}/versions", idempotent: true)
+    end
+
+    private
+
+    def get(path, **options)
+      request(path, method: :get, **options)
+    end
+
+    def put(path, **options)
+      request(path, method: :put, **options)
+    end
+
+    def post(path, **options)
+      request(path, method: :post, **options)
+    end
+
+    def request(path, **options)
+      options = { expects: 200 }.merge!(options)
+      path = File.join(@path_prefix, path) unless @path_prefix.nil?
+      response = @connection.request(path: path, **options)
+      JSON.parse(response.body)
+    rescue Excon::Error => e
+      @logger.error("Error while requesting #{path}: #{e.response.body}")
+      raise
+    end
+  end
+end