parquet 0.4.2-x86_64-linux-musl → 0.5.1-x86_64-linux-musl

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 687cb5045bff37ffa78b8a3bc156e6c01dd260b3c44ed77e7466c42c10a969e8
-  data.tar.gz: 651a435619d83e4365bed81287c519d68a0a605a804f2e9a7147029750d582a4
+  metadata.gz: 0d6605ede3c285ff55f311a9aabf082324fb0c55bcd251561c8d384116ae6e84
+  data.tar.gz: e788dd632b8cacfb6caa25f4c248d33e9aafdeece5ea44f3f783a77edbbdd9c3
 SHA512:
-  metadata.gz: 9de6a46c718ac607118350d2592ed29b289d315f8d78f6a3ba63952349f4337072fc5532c288beb27f03b4599cebefb6d63aae5f0a92c2ba64e79d01fdaae328
-  data.tar.gz: 1761fcd3dec2fcf6fdd935a1443d58e18c36daa5fa2cf5b804c238216e5bb79e3fdfe54a0511ad12c7b3bfc77cc7e5134c58f621cb7e496b7bf85f89a599e228
+  metadata.gz: 5fffd806cb18001d4e5bfa39cb4edd5ec19fac0f87cb01782630bccc4929c9a96d4f85f15598597040930e9facc065b238470db5862518f5cf330944598ac842
+  data.tar.gz: c8031944b16aea4c0a789a9741fc15ee612424350e00b7cedfba2dfa8095e3f0aa1bfd8b995ddf54a46a68ba474ad101decb410c241f5a862e1e3706ee09df42

data/README.md

@@ -194,4 +194,108 @@ The following data types are supported in the schema:
 - `date32`
 - `timestamp_millis`, `timestamp_micros`
 
-Note: Writing of List and Map types is not currently supported.
+### Schema DSL for Complex Data Types
+
+In addition to the hash-based schema definition shown above, this library provides a more expressive DSL for defining complex schemas with nested structures:
+
+```ruby
+require "parquet"
+
+# Define a complex schema using the Schema DSL
+schema = Parquet::Schema.define do
+  field :id, :int64, nullable: false  # Required field
+  field :name, :string                # Optional field (nullable: true is default)
+
+  # Nested struct
+  field :address, :struct do
+    field :street, :string
+    field :city, :string
+    field :zip, :string
+    field :coordinates, :struct do
+      field :latitude, :double
+      field :longitude, :double
+    end
+  end
+
+  # List of primitives
+  field :scores, :list, item: :float
+
+  # List of structs
+  field :contacts, :list, item: :struct do
+    field :name, :string
+    field :phone, :string
+    field :primary, :boolean
+  end
+
+  # Map with string values
+  field :metadata, :map, key: :string, value: :string
+
+  # Map with struct values
+  field :properties, :map, key: :string, value: :struct do
+    field :count, :int32
+    field :description, :string
+  end
+
+  # Nested lists
+  field :nested_lists, :list, item: :list do
+    field :item, :string  # For nested lists, inner item must be named 'item'
+  end
+
+  # Map of lists
+  field :map_of_lists, :map, key: :string, value: :list do
+    field :item, :int32  # For list items in maps, item must be named 'item'
+  end
+end
+
+# Sample data with nested structures
+data = [
+  [
+    1,                            # id
+    "John Doe",                   # name
+    {                             # address (struct)
+      "street" => "123 Main St",
+      "city" => "Springfield",
+      "zip" => "12345",
+      "coordinates" => {
+        "latitude" => 37.7749,
+        "longitude" => -122.4194
+      }
+    },
+    [85.5, 92.0, 78.5],          # scores (list of floats)
+    [                             # contacts (list of structs)
+      { "name" => "Contact 1", "phone" => "555-1234", "primary" => true },
+      { "name" => "Contact 2", "phone" => "555-5678", "primary" => false }
+    ],
+    { "created" => "2023-01-01", "status" => "active" },  # metadata (map)
+    {                             # properties (map of structs)
+      "feature1" => { "count" => 5, "description" => "Main feature" },
+      "feature2" => { "count" => 3, "description" => "Secondary feature" }
+    },
+    [["a", "b"], ["c", "d", "e"]],  # nested_lists
+    {                                # map_of_lists
+      "group1" => [1, 2, 3],
+      "group2" => [4, 5, 6]
+    }
+  ]
+]
+
+# Write to a parquet file using the schema
+Parquet.write_rows(data.each, schema: schema, write_to: "complex_data.parquet")
+
+# Read back the data
+Parquet.each_row("complex_data.parquet") do |row|
+  puts row.inspect
+end
+```
+
+The Schema DSL supports:
+
+- **Primitive types**: All standard Parquet types (`int32`, `string`, etc.)
+- **Complex types**: Structs, lists, and maps with arbitrary nesting
+- **Nullability control**: Specify which fields can contain null values with `nullable: false/true`
+- **List item nullability**: Control whether list items can be null with `item_nullable: false/true`
+- **Map key/value nullability**: Control whether map keys or values can be null with `value_nullable: false/true`
+
+Note: When using List and Map types, you need to provide at least:
+- For lists: The `item:` parameter specifying the item type
+- For maps: Both `key:` and `value:` parameters specifying key and value types

data/lib/parquet/schema.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+module Parquet
+  # Schema definition for Parquet files
+  class Schema
+    # Define a new schema using the DSL
+    # @return [Hash] schema definition hash
+    #
+    # @example Define a schema with nullable and non-nullable fields
+    #   Parquet::Schema.define do
+    #     field :id, :int64, nullable: false  # ID cannot be null
+    #     field :name, :string  # Default nullable: true
+    #
+    #     # List with non-nullable items
+    #     field :scores, :list, item: :float, item_nullable: false
+    #
+    #     # Map with nullable values
+    #     field :metadata, :map,
+    #           key: :string,
+    #           value: :string,
+    #           value_nullable: true
+    #
+    #     # Nested struct with non-nullable fields
+    #     field :address, :struct, nullable: true do
+    #       field :street, :string, nullable: false
+    #       field :city, :string, nullable: false
+    #       field :zip, :string, nullable: false
+    #     end
+    #   end
+    def self.define(&block)
+      builder = SchemaBuilder.new
+      builder.instance_eval(&block)
+
+      # Return a structured hash representing the schema
+      { type: :struct, fields: builder.fields }
+    end
+
+    # Internal builder class that provides the DSL methods
+    class SchemaBuilder
+      attr_reader :fields
+
+      def initialize
+        @fields = []
+      end
+
+      # Define a field in the schema
+      # @param name [String, Symbol] field name
+      # @param type [Symbol] data type (:int32, :int64, :string, :list, :map, :struct, etc)
+      # @param nullable [Boolean] whether the field can be null (default: true)
+      # @param kwargs [Hash] additional options depending on type
+      #
+      # Additional keyword args:
+      #   - `item:` if type == :list
+      #   - `item_nullable:` controls nullability of list items (default: true)
+      #   - `key:, value:` if type == :map
+      #   - `key_nullable:, value_nullable:` controls nullability of map keys/values (default: true)
+      #   - `format:` if you want to store some format string
+      #   - `nullable:` default to true if not specified
+      def field(name, type, nullable: true, **kwargs, &block)
+        field_hash = { name: name.to_s, type: type, nullable: !!nullable }
+
+        # Possibly store a format if provided
+        field_hash[:format] = kwargs[:format] if kwargs.key?(:format)
+
+        case type
+        when :struct
+          # We'll parse subfields from the block
+          sub_builder = SchemaBuilder.new
+          sub_builder.instance_eval(&block) if block
+          field_hash[:fields] = sub_builder.fields
+        when :list
+          item_type = kwargs[:item]
+          raise ArgumentError, "list field `#{name}` requires `item:` type" unless item_type
+          # Pass item_nullable if provided, otherwise use true as default
+          item_nullable = kwargs[:item_nullable].nil? ? true : !!kwargs[:item_nullable]
+          field_hash[:item] = wrap_subtype(item_type, nullable: item_nullable, &block)
+        when :map
+          # user must specify key:, value:
+          key_type = kwargs[:key]
+          value_type = kwargs[:value]
+          raise ArgumentError, "map field `#{name}` requires `key:` and `value:`" if key_type.nil? || value_type.nil?
+          # Pass key_nullable and value_nullable if provided, otherwise use true as default
+          key_nullable = kwargs[:key_nullable].nil? ? true : !!kwargs[:key_nullable]
+          value_nullable = kwargs[:value_nullable].nil? ? true : !!kwargs[:value_nullable]
+          field_hash[:key] = wrap_subtype(key_type, nullable: key_nullable)
+          field_hash[:value] = wrap_subtype(value_type, nullable: value_nullable, &block)
+        else
+          # primitive type: :int32, :int64, :string, etc.
+          # do nothing else special
+        end
+
+        @fields << field_hash
+      end
+
+      def build_map(key_type, value_type, key_nullable: false, value_nullable: true, nullable: true, &block)
+        # Wrap the key type (maps typically use non-nullable keys)
+        key = wrap_subtype(key_type, nullable: key_nullable)
+
+        # Handle the case where value_type is a complex type (:struct or :list) and a block is provided
+        value =
+          if (value_type == :struct || value_type == :list) && block
+            wrap_subtype(value_type, nullable: value_nullable, &block)
+          else
+            wrap_subtype(value_type, nullable: value_nullable)
+          end
+
+        # Map is represented as a list of key/value pairs in Parquet
+        {
+          type: :map,
+          nullable: nullable,
+          item: {
+            type: :struct,
+            nullable: false,
+            name: "key_value",
+            fields: [key, value]
+          }
+        }
+      end
+
+      private
+
+      # If user said: field "something", :list, item: :struct do ... end
+      # we want to recursively parse that sub-struct from the block.
+      # So wrap_subtype might be:
+      def wrap_subtype(t, nullable: true, &block)
+        if t == :struct
+          sub_builder = SchemaBuilder.new
+          sub_builder.instance_eval(&block) if block
+
+          # Validate that the struct has at least one field
+          if sub_builder.fields.empty?
+            raise ArgumentError, "Cannot create a struct with zero fields. Parquet doesn't support empty structs."
+          end
+
+          { type: :struct, nullable: nullable, name: "item", fields: sub_builder.fields }
+        elsif t == :list && block
+          # Handle nested lists by processing the block to define the item type
+          sub_builder = SchemaBuilder.new
+          sub_builder.instance_eval(&block) if block
+
+          # We expect a single field named "item" that defines the inner list's item type
+          if sub_builder.fields.empty? || sub_builder.fields.length > 1 || sub_builder.fields[0][:name] != "item"
+            raise ArgumentError, "Nested list must define exactly one field named 'item' for the inner list's item type"
+          end
+
+          { type: :list, nullable: nullable, name: "item", item: sub_builder.fields[0] }
+        else
+          # e.g. :int32 => { type: :int32, nullable: true }
+          { type: t, nullable: nullable, name: "item" }
+        end
+      end
+    end
+  end
+end

data/lib/parquet/version.rb

@@ -1,3 +1,3 @@
 module Parquet
-  VERSION = "0.4.2"
+  VERSION = "0.5.1"
 end

data/lib/parquet.rb

@@ -1,4 +1,5 @@
 require_relative "parquet/version"
+require_relative "parquet/schema"
 
 begin
   require "parquet/#{RUBY_VERSION.to_f}/parquet"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parquet
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.5.1
 platform: x86_64-linux-musl
 authors:
 - Nathan Jaremko
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2025-02-21 00:00:00.000000000 Z
+date: 2025-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake-compiler
@@ -42,6 +42,7 @@ files:
 - lib/parquet/3.2/parquet.so
 - lib/parquet/3.3/parquet.so
 - lib/parquet/3.4/parquet.so
+- lib/parquet/schema.rb
 - lib/parquet/version.rb
 homepage: https://github.com/njaremko/parquet-ruby
 licenses: