proto_turf 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 748636abfd68b4fbf48cb1786c3d2617ae0b48400c23e14c9ff03786601a4dd8
+  data.tar.gz: 9b3e76306ecefa2cfe6ca7b097b366d8145ec4c3a2c1dbe59b05625b1d1b37e7
+SHA512:
+  metadata.gz: 58e153ed45709e81d7b1d6d2b35523282d866a20f925e549ace5930daf29d1f2afafadcdd48b383bc7f27e662e401aa72cabc7cbd9e4a3d5bc3abad3f121f0d2
+  data.tar.gz: 5715d4b57c35db06292f30ec6228a19d4e089e65558cf0e2c0c698ec930fc471541a1b92d9743344a173a6b7b67ba2c8593bbe198bca5eb58841112d8f5339ce

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Flipp
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,48 @@
+# proto_turf
+
+`proto_turf` is a library to interact with the Confluent Schema Registry using Google Protobuf. It is inspired by and based off of [avro_turf](https://github.com/dasch/avro_turf).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'proto_turf'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install proto_turf
+
+## Usage
+
+ProtoTurf interacts with the Confluent Schema Registry, and caches all results. When you first encode a message, it will register the message and all dependencies with the Schema Registry. When decoding, it will look up the schema in the Schema Registry and use the associated local generated code to decode the message.
+
+Example usage:
+
+```ruby
+require 'proto_turf'
+
+proto_turf = ProtoTurf.new(registry_url: 'http://localhost:8081')
+message = MyProto::MyMessage.new(field1: 'value1', field2: 42)
+encoded = proto_turf.encode(message, subject: 'my-subject')
+
+# Decoding
+
+decoded_proto_message = proto_turf.decode(encoded_string)
+```
+
+If you're using [buf](https://buf.build/) to manage your Protobuf definitions, you should run `buf export` before using `proto_turf` to ensure that all the dependencies are available as `.proto` files in your project. The actual proto text is needed when registering the schema with the Schema Registry.
+
+## Notes about usage
+
+* For now, this library only supports a single message per `.proto` file that is registered with the Schema Registry. You can reference as many other files and messages as you like from that message, but keeping it to one message per file simplifies the workflow significantly.
+* When decoding, this library does *not* attempt to fully parse the Proto definition stored on the schema registry and generate dynamic classes. Instead, it simply parses out the package and message and assumes that the reader has the message available in the descriptor pool. Any compatibility issues should be detected through normal means, i.e. just by instantiating the message and seeing if any errors are raised.
+
+## TODO
+
+* Support multi-message proto files

data/lib/proto_turf/cached_confluent_schema_registry.rb

@@ -0,0 +1,51 @@
+class ProtoTurf::CachedConfluentSchemaRegistry
+
+  # @param upstream [ProtoTurf::ConfluentSchemaRegistry]
+  def initialize(upstream)
+    @upstream = upstream
+    @schemas_by_id = {}
+    @ids_by_schema = {}
+    @versions_by_subject_and_id = {}
+  end
+
+  # Delegate the following methods to the upstream
+  %i(subject_versions schema_subject_versions).each do |name|
+    define_method(name) do |*args|
+      instance_variable_get(:@upstream).send(name, *args)
+    end
+  end
+
+  # @param id [Integer] the schema ID to fetch
+  # @return [String] the schema string stored in the registry for the given id
+  def fetch(id)
+    @schemas_by_id[id] ||= @upstream.fetch(id)
+  end
+
+  # @param id [Integer] the schema ID to fetch
+  # @param subject [String] the subject to fetch the version for
+  # @return [Integer, nil] the version of the schema for the given subject and id, or nil if not found
+  def fetch_version(id, subject)
+    key = [subject, id]
+    return @versions_by_subject_and_id[key] if @versions_by_subject_and_id[key]
+
+    results = @upstream.schema_subject_versions(id)
+    @versions_by_subject_and_id[key] = results&.find { |r| r['subject'] == subject }&.dig('version')
+  end
+
+  # @param subject [String] the subject to check
+  # @param schema [String] the schema text to check
+  # @return [Boolean] true if we know the schema has been registered for that subject.
+  def registered?(subject, schema)
+    @ids_by_schema[[subject, schema]].present?
+  end
+
+  # @param subject [String] the subject to register the schema under
+  # @param schema [String] the schema text to register
+  # @param references [Array<Hash>] optional references to other schemas
+  def register(subject, schema, references: [])
+    key = [subject, schema]
+
+    @ids_by_schema[key] ||= @upstream.register(subject, schema, references: references)
+  end
+
+end

data/lib/proto_turf/confluent_schema_registry.rb

@@ -0,0 +1,118 @@
+require 'excon'
+
+class ProtoTurf
+  class ConfluentSchemaRegistry
+
+    CONTENT_TYPE = "application/vnd.schemaregistry.v1+json".freeze
+
+    def initialize(
+      url,
+      schema_context: nil,
+      logger: Logger.new($stdout),
+      proxy: nil,
+      user: nil,
+      password: nil,
+      ssl_ca_file: nil,
+      client_cert: nil,
+      client_key: nil,
+      client_key_pass: nil,
+      client_cert_data: nil,
+      client_key_data: nil,
+      path_prefix: nil,
+      connect_timeout: nil,
+      resolv_resolver: nil,
+      retry_limit: nil
+    )
+      @path_prefix = path_prefix
+      @schema_context_prefix = schema_context.nil? ? '' : ":.#{schema_context}:"
+      @schema_context_options = schema_context.nil? ? {} : {query: {subject: @schema_context_prefix}}
+      @logger = logger
+      headers = Excon.defaults[:headers].merge(
+        "Content-Type" => CONTENT_TYPE
+      )
+      params = {
+        headers: headers,
+        user: user,
+        password: password,
+        proxy: proxy,
+        ssl_ca_file: ssl_ca_file,
+        client_cert: client_cert,
+        client_key: client_key,
+        client_key_pass: client_key_pass,
+        client_cert_data: client_cert_data,
+        client_key_data: client_key_data,
+        resolv_resolver: resolv_resolver,
+        connect_timeout: connect_timeout,
+        retry_limit: retry_limit
+      }
+      # Remove nil params to allow Excon to use its default values
+      params.reject! { |_, v| v.nil? }
+      @connection = Excon.new(
+        url,
+        params
+      )
+    end
+
+    # @param id [Integer] the schema ID to fetch
+    # @return [String] the schema string stored in the registry for the given id
+    def fetch(id)
+      @logger.info "Fetching schema with id #{id}"
+      data = get("/schemas/ids/#{id}", idempotent: true, **@schema_context_options, )
+      data.fetch("schema")
+    end
+
+    # @param schema_id [Integer] the schema ID to fetch versions for
+    # @return [Array<Hash>] an array of versions for the given schema ID
+    def schema_subject_versions(schema_id)
+      get("/schemas/ids/#{schema_id}/versions", idempotent: true, **@schema_context_options)
+    end
+
+    # @param subject [String] the subject to check
+    # @param schema [String] the schema text to check
+    # @param references [Array<Hash>] optional references to other schemas
+    # @return [Integer] the ID of the registered schema
+    def register(subject, schema, references: [])
+      data = post("/subjects/#{@schema_context_prefix}#{CGI.escapeURIComponent(subject)}/versions",
+                  body: { schemaType: 'PROTOBUF',
+                          references: references,
+                          schema: schema.to_s }.to_json)
+
+      id = data.fetch("id")
+
+      @logger.info "Registered schema for subject `#{@schema_context_prefix}#{subject}`; id = #{id}"
+
+      id
+    end
+
+    # @param subject [String]
+    # @return [Array<Hash>] an array of versions for the given subject
+    def subject_versions(subject)
+      get("/subjects/#{@schema_context_prefix}#{CGI.escapeURIComponent(subject)}/versions", idempotent: true)
+    end
+
+    private
+
+    def get(path, **options)
+      request(path, method: :get, **options)
+    end
+
+    def put(path, **options)
+      request(path, method: :put, **options)
+    end
+
+    def post(path, **options)
+      request(path, method: :post, **options)
+    end
+
+    def request(path, **options)
+      options = { expects: 200 }.merge!(options)
+      path = File.join(@path_prefix, path) unless @path_prefix.nil?
+      response = @connection.request(path: path, **options)
+      JSON.parse(response.body)
+    rescue Excon::Error => e
+      @logger.error("Error while requesting #{path}: #{e.response.body}")
+      raise
+    end
+
+  end
+end

data/lib/proto_turf/version.rb

@@ -0,0 +1,3 @@
+class ProtoTurf
+  VERSION = "0.0.1"
+end

data/lib/proto_turf.rb

@@ -0,0 +1,229 @@
+require 'logger'
+require 'proto_turf/confluent_schema_registry'
+require 'proto_turf/cached_confluent_schema_registry'
+
+class ProtoTurf
+  # Provides a way to encode and decode messages without having to embed schemas
+  # in the encoded data. Confluent's Schema Registry[1] is used to register
+  # a schema when encoding a message -- the registry will issue a schema id that
+  # will be included in the encoded data alongside the actual message. When
+  # decoding the data, the schema id will be used to look up the writer's schema
+  # from the registry.
+  #
+  # 1: https://github.com/confluentinc/schema-registry
+  # https://docs.confluent.io/platform/current/schema-registry/fundamentals/serdes-develop/serdes-protobuf.html
+  # https://docs.confluent.io/platform/current/schema-registry/fundamentals/serdes-develop/index.html#wire-format
+  MAGIC_BYTE = [0].pack("C").freeze
+
+
+  # Instantiate a new ProtoTurf instance with the given configuration.
+  #
+  # registry             - A schema registry object that responds to all methods in the
+  #                        ProtoTurf::ConfluentSchemaRegistry interface.
+  # registry_url         - The String URL of the schema registry that should be used.
+  # schema_context       - Schema registry context name (optional)
+  # schemas_path         - The String file system path where local schemas are stored.
+  # registry_path_prefix - The String URL path prefix used to namespace schema registry requests (optional).
+  # logger               - The Logger that should be used to log information (optional).
+  # proxy                - Forward the request via  proxy (optional).
+  # user                 - User for basic auth (optional).
+  # password             - Password for basic auth (optional).
+  # ssl_ca_file          - Name of file containing CA certificate (optional).
+  # client_cert          - Name of file containing client certificate (optional).
+  # client_key           - Name of file containing client private key to go with client_cert (optional).
+  # client_key_pass      - Password to go with client_key (optional).
+  # client_cert_data     - In-memory client certificate (optional).
+  # client_key_data      - In-memory client private key to go with client_cert_data (optional).
+  # connect_timeout      - Timeout to use in the connection with the schema registry (optional).
+  # resolv_resolver      - Custom domain name resolver (optional).
+  def initialize(
+    registry: nil,
+    registry_url: nil,
+    schema_context: nil,
+    schemas_path: nil,
+    registry_path_prefix: nil,
+    logger: nil,
+    proxy: nil,
+    user: nil,
+    password: nil,
+    ssl_ca_file: nil,
+    client_cert: nil,
+    client_key: nil,
+    client_key_pass: nil,
+    client_cert_data: nil,
+    client_key_data: nil,
+    connect_timeout: nil,
+    resolv_resolver: nil,
+    retry_limit: nil
+  )
+    @logger = logger || Logger.new($stderr)
+    @path = schemas_path
+    @registry = registry || ProtoTurf::CachedConfluentSchemaRegistry.new(
+      ProtoTurf::ConfluentSchemaRegistry.new(
+        registry_url,
+        schema_context: schema_context,
+        logger: @logger,
+        proxy: proxy,
+        user: user,
+        password: password,
+        ssl_ca_file: ssl_ca_file,
+        client_cert: client_cert,
+        client_key: client_key,
+        client_key_pass: client_key_pass,
+        client_cert_data: client_cert_data,
+        client_key_data: client_key_data,
+        path_prefix: registry_path_prefix,
+        connect_timeout: connect_timeout,
+        resolv_resolver: resolv_resolver
+      )
+    )
+    @all_schemas = {}
+  end
+
+  # Encodes a message using the specified schema.
+  #
+  # message           - The message that should be encoded. Must be compatible with
+  #                     the schema.
+  # subject           - The subject name the schema should be registered under in
+  #                     the schema registry (optional).
+  # Returns the encoded data as a String.
+  def encode(message, subject: nil)
+    load_schemas! if @all_schemas.empty?
+
+    id = register_schema(message.class.descriptor.file_descriptor, subject: subject)
+
+    stream = StringIO.new
+    # Always start with the magic byte.
+    stream.write(MAGIC_BYTE)
+
+    # The schema id is encoded as a 4-byte big-endian integer.
+    stream.write([id].pack("N"))
+
+    # For now, we're only going to support a single message per schema. See
+    # https://docs.confluent.io/platform/current/schema-registry/fundamentals/serdes-develop/index.html#wire-format
+    write_int(stream, 0)
+
+    # Now we write the actual message.
+    stream.write(message.to_proto)
+
+    stream.string
+  rescue Excon::Error::NotFound
+    if schema_id
+      raise SchemaNotFoundError.new("Schema with id: #{schema_id} is not found on registry")
+    else
+      raise SchemaNotFoundError.new("Schema with subject: `#{subject}` version: `#{version}` is not found on registry")
+    end
+  end
+
+  # Decodes data into the original message.
+  #
+  # data        - A String containing encoded data.
+  #
+  # Returns a Protobuf AbstractMessage object instantiated with the decoded data.
+  def decode(data)
+    stream = StringIO.new(data)
+
+    # The first byte is MAGIC!!!
+    magic_byte = stream.read(1)
+
+    if magic_byte != MAGIC_BYTE
+      raise "Expected data to begin with a magic byte, got `#{magic_byte.inspect}`"
+    end
+
+    # The schema id is a 4-byte big-endian integer.
+    schema_id = stream.read(4).unpack("N").first
+
+    # For now, we're only going to support a single message per schema. See
+    # https://docs.confluent.io/platform/current/schema-registry/fundamentals/serdes-develop/index.html#wire-format
+    read_int(stream)
+
+    schema = @registry.fetch(schema_id)
+    encoded = stream.read
+    decode_protobuf(schema, encoded)
+  rescue Excon::Error::NotFound
+    raise SchemaNotFoundError.new("Schema with id: #{schema_id} is not found on registry")
+  end
+
+  private
+
+  # Write an int with zig-zag encoding. Copied from Avro.
+  def write_int(stream, n)
+    n = (n << 1) ^ (n >> 63)
+    while (n & ~0x7F) != 0
+      stream.write(((n & 0x7f) | 0x80).chr)
+      n >>= 7
+    end
+    stream.write(n.chr)
+  end
+
+  # Read an int with zig-zag encoding. Copied from Avro.
+  def read_int(stream)
+    b = stream.readbyte
+    n = b & 0x7F
+    shift = 7
+    while (b & 0x80) != 0
+      b = stream.readbyte
+      n |= (b & 0x7F) << shift
+      shift += 7
+    end
+    (n >> 1) ^ -(n & 1)
+  end
+
+  def decode_protobuf(schema, encoded)
+    # get the package
+    package = schema.match(/package (\S+);/)[1]
+    # get the first message in the protobuf text
+    # TODO - get the correct message based on schema index
+    message_name = schema.match(/message (\w+) {/)[1]
+    # look up the descriptor
+    full_name = "#{package}.#{message_name}"
+    descriptor = Google::Protobuf::DescriptorPool.generated_pool.lookup(full_name)
+    unless descriptor
+      raise "Could not find schema for #{full_name}. Make sure the corresponding .proto file has been compiled and loaded."
+    end
+    descriptor.msgclass.decode(encoded)
+  end
+
+  def register_schema(file_descriptor, subject: nil)
+    subject ||= file_descriptor.name
+    return if @registry.registered?(file_descriptor.name, subject)
+
+    # register dependencies first
+    dependencies = file_descriptor.to_proto.dependency.to_a.reject { |d| d.start_with?('google/protobuf/') }
+    versions = dependencies.map do |dependency|
+      dependency_descriptor = @all_schemas[dependency]
+      result = register_schema(dependency_descriptor, subject: dependency_descriptor.name)
+      @registry.fetch_version(result, dependency_descriptor.name)
+    end
+
+    @registry.register(subject,
+                       schema_text(file_descriptor),
+                       references: dependencies.map.with_index do |dependency, i|
+                         {
+                           name: dependency,
+                           subject: dependency,
+                           version: versions[i]
+                         }
+                       end
+    )
+  end
+
+  def schema_text(file_descriptor)
+    filename = "#{@path}/#{file_descriptor.name}"
+    File.exist?(filename) ? File.read(filename) : ""
+  end
+
+  def load_schemas!
+    all_messages = ObjectSpace.each_object(Class).select do |o|
+      o < Google::Protobuf.const_get(:AbstractMessage)
+    end.to_a
+    all_messages.each do |m|
+      file_desc = m.descriptor.file_descriptor
+      file_path = file_desc.name
+      next if file_path.start_with?('google/protobuf/') # skip built-in protos
+
+      @all_schemas[file_path] = file_desc
+    end
+  end
+
+end

data/proto_turf.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'proto_turf/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "proto_turf"
+  spec.version       = ProtoTurf::VERSION
+  spec.authors       = ["Daniel Orner"]
+  spec.email         = ["daniel.orner@flipp.com"]
+  spec.summary       = "Support for Protobuf files in Confluent Schema Registry"
+  spec.homepage      = "https://github.com/flipp-oss/proto_turf"
+  spec.license       = "MIT"
+
+  spec.metadata["rubygems_mfa_required"] = "true"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "google-protobuf"
+
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "rspec", "~> 3.2"
+end

metadata

@@ -0,0 +1,107 @@
+--- !ruby/object:Gem::Specification
+name: proto_turf
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Daniel Orner
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2025-08-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: google-protobuf
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+description:
+email:
+- daniel.orner@flipp.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- lib/proto_turf.rb
+- lib/proto_turf/cached_confluent_schema_registry.rb
+- lib/proto_turf/confluent_schema_registry.rb
+- lib/proto_turf/version.rb
+- proto_turf.gemspec
+homepage: https://github.com/flipp-oss/proto_turf
+licenses:
+- MIT
+metadata:
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.10
+signing_key:
+specification_version: 4
+summary: Support for Protobuf files in Confluent Schema Registry
+test_files: []