unibuf 0.1.0 → 0.1.2

data/docs/TXTPROTO.adoc

@@ -0,0 +1,369 @@
+= Protocol Buffers Text Format (Textproto) Support
+
+:toc:
+:toclevels: 3
+
+== Purpose
+
+Unibuf provides complete support for Protocol Buffers text format (textproto), a human-readable representation of protobuf messages.
+
+Features:
+
+* Parse `.txtpb` and `.textproto` files
+* No schema required for parsing (schema recommended for validation)
+* Support all Protocol Buffers field types
+* Nested messages and repeated fields
+* Comments and whitespace handling
+* Round-trip serialization
+* Rich domain models
+
+== Text Format Overview
+
+Text format (textproto) is:
+
+Human-readable::
+Easy to read and edit by humans
+
+Self-documenting::
+Field names make structure obvious
+
+Configuration-friendly::
+Ideal for configuration files
+
+Debug-friendly::
+Easy to inspect message contents
+
+== Parsing Text Format
+
+=== Basic parsing
+
+[source,ruby]
+----
+require "unibuf"
+
+# Parse from file
+message = Unibuf.parse_textproto_file("config.txtpb")  # <1>
+
+# Parse from string
+text = <<~TEXTPROTO
+  name: "Alice"
+  id: 123
+TEXTPROTO
+
+message = Unibuf.parse_textproto(text)  # <2>
+
+# Access fields
+puts message.find_field("name").value  # => "Alice"  # <3>
+puts message.find_field("id").value    # => 123  # <4>
+----
+<1> Parse from file
+<2> Parse from string
+<3> Access string field
+<4> Access integer field
+
+=== Parsing nested messages
+
+[source,ruby]
+----
+text = <<~TEXTPROTO
+  name: "Alice"
+  address {
+    street: "123 Main St"
+    city: "Springfield"
+  }
+TEXTPROTO
+
+message = Unibuf.parse_textproto(text)  # <1>
+
+# Access nested message
+address = message.find_field("address").value  # <2>
+puts address.find_field("street").value  # => "123 Main St"  # <3>
+----
+<1> Parse text with nested message
+<2> Get nested message field
+<3> Access field in nested message
+
+=== Parsing repeated fields
+
+[source,ruby]
+----
+text = <<~TEXTPROTO
+  name: "Alice"
+  tags: "important"
+  tags: "customer"
+  tags: "vip"
+TEXTPROTO
+
+message = Unibuf.parse_textproto(text)  # <1>
+
+# Access repeated field
+tags = message.find_fields("tags")  # <2>
+tag_values = tags.map(&:value)  # => ["important", "customer", "vip"]  # <3>
+----
+<1> Parse text with repeated fields
+<2> Find all fields with name "tags"
+<3> Extract values
+
+== Text Format Syntax
+
+=== Field types
+
+String fields::
++
+[source,textproto]
+----
+name: "Alice"
+description: "A person"
+----
+
+Integer fields::
++
+[source,textproto]
+----
+id: 123
+count: -42
+large: 9223372036854775807
+----
+
+Float fields::
++
+[source,textproto]
+----
+price: 19.99
+ratio: 0.5
+scientific: 1.23e-4
+----
+
+Boolean fields::
++
+[source,textproto]
+----
+enabled: true
+active: false
+----
+
+=== Nested messages
+
+[source,textproto]
+----
+person {
+  name: "Alice"
+  address {
+    street: "123 Main St"
+    city: "Springfield"
+  }
+}
+----
+
+=== Repeated fields
+
+[source,textproto]
+----
+# Multiple values for same field
+tags: "tag1"
+tags: "tag2"
+tags: "tag3"
+
+# Or as nested messages
+people {
+  name: "Alice"
+}
+people {
+  name: "Bob"
+}
+----
+
+=== Comments
+
+[source,textproto]
+----
+# Line comment
+name: "Alice"  # Inline comment
+
+/*
+ * Block comment
+ */
+id: 123
+----
+
+== Domain Models
+
+=== Message model
+
+[source,ruby]
+----
+message = Unibuf.parse_textproto_file("data.txtpb")
+
+# Classification
+message.nested?           # Has nested messages?
+message.scalar_only?      # Only scalar values?
+message.repeated_fields?  # Has repeated fields?
+
+# Queries
+message.find_field("name")        # Find single field
+message.find_fields("tags")       # Find all with name
+message.field_names               # All field names
+message.repeated_field_names      # Repeated field names
+
+# Traversal
+message.traverse_depth_first { |field| puts field.name }
+message.traverse_breadth_first { |field| puts field.value }
+message.depth  # Maximum nesting depth
+----
+
+=== Field model
+
+[source,ruby]
+----
+field = message.find_field("address")
+
+# Properties
+field.name       # Field name
+field.value      # Field value
+field.repeated?  # Is repeated field?
+
+# Type classification
+field.scalar?    # Scalar value?
+field.message?   # Nested message?
+field.list?      # List value?
+field.map?       # Map value?
+----
+
+=== Value types
+
+ScalarValue::
+Strings, integers, floats, booleans
+
+MessageValue::
+Nested messages
+
+ListValue::
+Repeated fields
+
+MapValue::
+Key-value maps
+
+== Serialization
+
+=== Serializing to text format
+
+[source,ruby]
+----
+# Parse message
+message = Unibuf.parse_textproto_file("input.txtpb")  # <1>
+
+# Modify if needed
+# ... modifications ...
+
+# Serialize back to text format
+output = message.to_textproto  # <2>
+
+# Write to file
+File.write("output.txtpb", output)  # <3>
+
+# Verify round-trip
+reparsed = Unibuf.parse_textproto(output)  # <4>
+puts message == reparsed  # => true  # <5>
+----
+<1> Parse original
+<2> Serialize to text
+<3> Write output
+<4> Parse again
+<5> Verify equivalence
+
+== With Schema Validation
+
+=== Parsing with validation
+
+[source,ruby]
+----
+# Load schema
+schema = Unibuf.parse_schema("schema.proto")  # <1>
+
+# Parse text format
+message = Unibuf.parse_textproto_file("data.txtpb")  # <2>
+
+# Validate
+validator = Unibuf::Validators::SchemaValidator.new(schema)  # <3>
+validator.validate!(message, "Person")  # <4>
+----
+<1> Load schema
+<2> Parse text format
+<3> Create validator
+<4> Validate against schema
+
+== Testing
+
+=== Test coverage
+
+Text format implementation includes:
+
+Grammar tests (60+ tests)::
+All syntax elements, strings, numbers, booleans, nested messages
+
+Parser tests (40+ tests)::
+Field parsing, message building, error handling
+
+Integration tests (40+ tests)::
+Real-world Protocol Buffer files (Google Fonts metadata)
+
+Serialization tests (20+ tests)::
+Round-trip verification, formatting
+
+**Total: 160+ tests, 100% passing**
+
+=== Running tests
+
+[source,shell]
+----
+# Run textproto tests
+bundle exec rspec spec/unibuf/parsers/textproto/
+
+# Run specific test
+bundle exec rspec spec/unibuf/parsers/textproto/grammar_spec.rb
+----
+
+== Real-World Examples
+
+=== Google Fonts metadata
+
+Unibuf successfully parses Google Fonts METADATA.pb files:
+
+[source,ruby]
+----
+# Parse Google Font metadata
+message = Unibuf.parse_file("METADATA.pb")
+
+# Access font information
+name = message.find_field("name")&.value
+category = message.find_field("category")&.value
+designers = message.find_fields("designer").map(&:value)
+
+puts "Font: #{name}"
+puts "Category: #{category}"
+puts "Designers: #{designers.join(', ')}"
+----
+
+== References
+
+Protocol Buffers text format::
+https://protobuf.dev/reference/protobuf/textformat-spec/
+
+Proto3 language guide::
+https://protobuf.dev/programming-guides/proto3/
+
+Encoding specification::
+https://protobuf.dev/programming-guides/encoding/
+
+== Support
+
+For issues, questions, or contributions related to Protocol Buffers text format:
+
+* GitHub Issues: https://github.com/lutaml/unibuf/issues
+* Documentation: https://github.com/lutaml/unibuf/tree/main/docs
+
+== Copyright and License
+
+Copyright https://www.ribose.com[Ribose Inc.]
+
+Licensed under the 3-clause BSD License.

data/lib/unibuf/commands/convert.rb

@@ -42,12 +42,24 @@ module Unibuf
                 "Target format required"
         end
 
-        valid_formats = %w[json yaml textproto]
+        valid_formats = %w[json yaml textproto binpb]
         unless valid_formats.include?(target_format)
           raise InvalidArgumentError,
                 "Invalid format '#{target_format}'. " \
                 "Valid formats: #{valid_formats.join(', ')}"
         end
+
+        # Binary format requires schema
+        if target_format == "binpb" && !schema_file
+          raise InvalidArgumentError,
+                "Binary format requires --schema option"
+        end
+
+        # Binary input requires schema
+        if binary_file?(file) && !schema_file
+          raise InvalidArgumentError,
+                "Binary input requires --schema option"
+        end
       end
 
       def load_message(file)
@@ -56,6 +68,8 @@ module Unibuf
           load_from_json(file)
         elsif yaml_file?(file)
           load_from_yaml(file)
+        elsif binary_file?(file)
+          load_from_binary(file)
         else
           Unibuf.parse_file(file)
         end
@@ -73,6 +87,11 @@ module Unibuf
         Unibuf::Models::Message.from_hash(data)
       end
 
+      def load_from_binary(file)
+        schema = load_schema
+        Unibuf.parse_binary_file(file, schema: schema)
+      end
+
       def convert_message(message)
         case target_format
         when "json"
@@ -81,18 +100,45 @@ module Unibuf
           message.to_yaml
         when "textproto"
           message.to_textproto
+        when "binpb"
+          schema = load_schema
+          message.to_binary(schema: schema, message_type: message_type)
         end
       end
 
       def write_output(content)
         if output_file
-          File.write(output_file, content)
+          if target_format == "binpb"
+            File.binwrite(output_file, content)
+          else
+            File.write(output_file, content)
+          end
           puts "Output written to #{output_file}" if verbose?
+        elsif target_format == "binpb"
+          $stdout.binmode
+          $stdout.write(content)
+        # Binary output to stdout
         else
           puts content
         end
       end
 
+      def load_schema
+        return @schema if @schema
+
+        unless schema_file
+          raise InvalidArgumentError,
+                "Schema required for binary format"
+        end
+
+        unless File.exist?(schema_file)
+          raise FileNotFoundError,
+                "Schema file not found: #{schema_file}"
+        end
+
+        @schema = Unibuf.parse_schema(schema_file)
+      end
+
       def json_file?(file)
         file.end_with?(".json")
       end
@@ -101,6 +147,10 @@ module Unibuf
         file.end_with?(".yaml", ".yml")
       end
 
+      def binary_file?(file)
+        file.end_with?(".binpb", ".bin", ".pb")
+      end
+
       def target_format
         options[:to]
       end
@@ -109,6 +159,14 @@ module Unibuf
         options[:output]
       end
 
+      def schema_file
+        options[:schema]
+      end
+
+      def message_type
+        options[:message_type]
+      end
+
       def verbose?
         options[:verbose]
       end

data/lib/unibuf/commands/schema.rb

@@ -43,6 +43,8 @@ module Unibuf
           Unibuf.parse_schema(file)
         when ".fbs"
           Unibuf.parse_flatbuffers_schema(file)
+        when ".capnp"
+          Unibuf.parse_capnproto_schema(file)
         else
           raise InvalidArgumentError,
                 "Unknown schema format: #{File.extname(file)}"
@@ -77,20 +79,75 @@ module Unibuf
 
       def format_text(schema)
         lines = []
-        lines << "Package: #{schema.package}" if schema.package
-        lines << "Syntax: #{schema.syntax}"
-        lines << ""
-        lines << "Messages (#{schema.messages.size}):"
-        schema.messages.each do |msg|
-          lines << "  #{msg.name} (#{msg.fields.size} fields)"
-        end
-        if schema.enums.any?
+
+        # Handle different schema types
+        if schema.respond_to?(:package)
+          # Proto3 schema
+          lines << "Package: #{schema.package}" if schema.package
+          lines << "Syntax: #{schema.syntax}"
           lines << ""
-          lines << "Enums (#{schema.enums.size}):"
-          schema.enums.each do |enum|
-            lines << "  #{enum.name} (#{enum.values.size} values)"
+          lines << "Messages (#{schema.messages.size}):"
+          schema.messages.each do |msg|
+            lines << "  #{msg.name} (#{msg.fields.size} fields)"
+          end
+          if schema.enums.any?
+            lines << ""
+            lines << "Enums (#{schema.enums.size}):"
+            schema.enums.each do |enum|
+              lines << "  #{enum.name} (#{enum.values.size} values)"
+            end
+          end
+        elsif schema.respond_to?(:file_id)
+          # Cap'n Proto schema
+          lines << "File ID: #{schema.file_id}" if schema.file_id
+          lines << ""
+          if schema.structs.any?
+            lines << "Structs (#{schema.structs.size}):"
+            schema.structs.each do |struct|
+              lines << "  #{struct.name} (#{struct.fields.size} fields)"
+            end
+          end
+          if schema.enums.any?
+            lines << ""
+            lines << "Enums (#{schema.enums.size}):"
+            schema.enums.each do |enum|
+              lines << "  #{enum.name} (#{enum.values.size} values)"
+            end
+          end
+          if schema.interfaces.any?
+            lines << ""
+            lines << "Interfaces (#{schema.interfaces.size}):"
+            schema.interfaces.each do |interface|
+              lines << "  #{interface.name} (#{interface.methods.size} methods)"
+            end
+          end
+        else
+          # FlatBuffers schema
+          lines << "Namespace: #{schema.namespace}" if schema.namespace
+          lines << "Root Type: #{schema.root_type}" if schema.root_type
+          lines << ""
+          if schema.tables.any?
+            lines << "Tables (#{schema.tables.size}):"
+            schema.tables.each do |table|
+              lines << "  #{table.name} (#{table.fields.size} fields)"
+            end
+          end
+          if schema.structs.any?
+            lines << ""
+            lines << "Structs (#{schema.structs.size}):"
+            schema.structs.each do |struct|
+              lines << "  #{struct.name} (#{struct.fields.size} fields)"
+            end
+          end
+          if schema.enums.any?
+            lines << ""
+            lines << "Enums (#{schema.enums.size}):"
+            schema.enums.each do |enum|
+              lines << "  #{enum.name} (#{enum.values.size} values)"
+            end
           end
         end
+
         lines.join("\n")
       end
 

data/lib/unibuf/errors.rb

@@ -1,36 +1,33 @@
 # frozen_string_literal: true
 
 module Unibuf
-  # Base error class for all Unibuf errors
+  # Base error class
   class Error < StandardError; end
 
-  # Parsing errors
+  # Parse error
   class ParseError < Error; end
-  class SyntaxError < ParseError; end
-  class UnexpectedTokenError < ParseError; end
-  class UnterminatedStringError < ParseError; end
 
-  # Validation errors
+  # Serialization error
+  class SerializationError < Error; end
+
+  # Validation error
   class ValidationError < Error; end
-  class TypeValidationError < ValidationError; end
+
+  # Schema validation error
   class SchemaValidationError < ValidationError; end
-  class ReferenceValidationError < ValidationError; end
-  class RequiredFieldError < ValidationError; end
-
-  # Model errors
-  class ModelError < Error; end
-  class InvalidFieldError < ModelError; end
-  class InvalidValueError < ModelError; end
-  class TypeCoercionError < ModelError; end
-
-  # File errors
-  class FileError < Error; end
-  class FileNotFoundError < FileError; end
-  class FileReadError < FileError; end
-  class FileWriteError < FileError; end
-
-  # CLI errors
-  class CLIError < Error; end
-  class InvalidArgumentError < CLIError; end
-  class CommandExecutionError < CLIError; end
+
+  # Type validation error
+  class TypeValidationError < ValidationError; end
+
+  # Invalid value error
+  class InvalidValueError < ValidationError; end
+
+  # Type coercion error
+  class TypeCoercionError < Error; end
+
+  # File not found error
+  class FileNotFoundError < Error; end
+
+  # Invalid argument error
+  class InvalidArgumentError < Error; end
 end

data/lib/unibuf/models/capnproto/enum_definition.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Unibuf
+  module Models
+    module Capnproto
+      # Represents a Cap'n Proto enum definition
+      class EnumDefinition
+        attr_reader :name, :values, :annotations
+
+        def initialize(attributes = {})
+          @name = attributes[:name] || attributes["name"]
+          @values = attributes[:values] || attributes["values"] || {}
+          @annotations = Array(
+            attributes[:annotations] || attributes["annotations"],
+          )
+        end
+
+        # Queries
+        def value_names
+          values.keys
+        end
+
+        def ordinals
+          values.values
+        end
+
+        def find_value(name)
+          values[name]
+        end
+
+        def find_name_by_ordinal(ordinal)
+          values.find { |_name, ord| ord == ordinal }&.first
+        end
+
+        # Validation
+        def valid?
+          validate!
+          true
+        rescue ValidationError
+          false
+        end
+
+        def validate!
+          raise ValidationError, "Enum name required" unless name
+
+          if values.empty?
+            raise ValidationError,
+                  "Enum must have at least one value"
+          end
+
+          # Check for duplicate ordinals
+          ordinal_counts = ordinals.tally
+          duplicates = ordinal_counts.select { |_ord, count| count > 1 }
+          unless duplicates.empty?
+            raise ValidationError,
+                  "Duplicate ordinals in enum '#{name}': #{duplicates.keys.join(', ')}"
+          end
+
+          true
+        end
+
+        def to_h
+          {
+            name: name,
+            values: values,
+            annotations: annotations,
+          }
+        end
+      end
+    end
+  end
+end