iceberg 0.11.2 → 0.12.0

data/lib/iceberg/result.rb

@@ -0,0 +1,22 @@
+module Iceberg
+  class Result
+    include Enumerable
+
+    attr_reader :columns, :rows
+
+    def initialize(columns, rows)
+      @columns = columns
+      @rows = rows
+    end
+
+    def each
+      @rows.each do |row|
+        yield @columns.zip(row).to_h
+      end
+    end
+
+    def empty?
+      rows.empty?
+    end
+  end
+end

data/lib/iceberg/s3_tables_catalog.rb

@@ -1,7 +1,10 @@
 module Iceberg
   class S3TablesCatalog < Catalog
-    def initialize(arn:)
-      @catalog = RbCatalog.new_s3tables(arn)
+    def initialize(arn:, default_namespace: nil)
+      _initialize(
+        RbCatalog.new_s3tables(arn),
+        default_namespace:
+      )
     end
   end
 end

data/lib/iceberg/sql_catalog.rb

@@ -2,8 +2,11 @@ module Iceberg
   class SqlCatalog < Catalog
     # warehouse is default storage location
     # name is stored in SQL table
-    def initialize(uri:, warehouse:, name: "main", properties: {})
-      @catalog = RbCatalog.new_sql(uri, warehouse, name, properties)
+    def initialize(uri:, warehouse:, name: "main", properties: {}, default_namespace: nil)
+      _initialize(
+        RbCatalog.new_sql(uri, warehouse, name, properties),
+        default_namespace:
+      )
     end
   end
 end

data/lib/iceberg/table.rb

@@ -5,93 +5,154 @@ module Iceberg
       @catalog = catalog
     end
 
+    def refresh
+      @table = @catalog.load_table(@table.identifier)
+    end
+
     def format_version
-      @table.format_version
+      metadata.format_version
     end
 
     def uuid
-      @table.uuid
+      metadata.uuid
     end
 
     def location
-      @table.location
+      metadata.location
     end
 
     def last_sequence_number
-      @table.last_sequence_number
+      metadata.last_sequence_number
     end
 
     def next_sequence_number
-      @table.next_sequence_number
+      metadata.next_sequence_number
     end
 
     def last_column_id
-      @table.last_column_id
+      metadata.last_column_id
     end
 
     def last_partition_id
-      @table.last_partition_id
+      metadata.last_partition_id
+    end
+
+    def last_updated_at
+      ms = metadata.last_updated_ms
+      Time.at(ms / 1000, ms % 1000, :millisecond)
     end
 
     def schemas
-      @table.schemas
+      metadata.schemas
     end
 
     def schema_by_id(schema_id)
-      @table.schema_by_id(schema_id)
+      metadata.schema_by_id(schema_id)
     end
 
     def current_schema
-      @table.current_schema
+      metadata.current_schema
     end
     alias_method :schema, :current_schema
 
     def current_schema_id
-      @table.current_schema_id
+      metadata.current_schema_id
     end
     alias_method :schema_id, :current_schema_id
 
+    def partition_specs
+      metadata.partition_specs
+    end
+
+    def partition_spec_by_id(partition_spec_id)
+      metadata.partition_spec_by_id(partition_spec_id)
+    end
+
+    def default_partition_spec
+      metadata.default_partition_spec
+    end
+
     def default_partition_spec_id
-      @table.default_partition_spec_id
+      metadata.default_partition_spec_id
     end
 
     def snapshots
-      @table.snapshots
+      metadata.snapshots
     end
 
     def snapshot_by_id(snapshot_id)
-      @table.snapshot_by_id(snapshot_id)
+      metadata.snapshot_by_id(snapshot_id)
     end
 
     def history
-      @table.history
+      metadata.history
     end
 
     def metadata_log
-      @table.metadata_log
+      metadata.metadata_log
     end
 
     def current_snapshot
-      @table.current_snapshot
+      metadata.current_snapshot
     end
 
     def current_snapshot_id
-      @table.current_snapshot_id
+      metadata.current_snapshot_id
+    end
+
+    def sort_orders
+      metadata.sort_orders
+    end
+
+    def default_sort_order
+      metadata.default_sort_order
+    end
+
+    def default_sort_order_id
+      metadata.default_sort_order_id
     end
 
     def properties
-      @table.properties
+      metadata.properties
+    end
+
+    def statistics
+      metadata.statistics
+    end
+
+    def partition_statistics
+      metadata.partition_statistics
+    end
+
+    def statistics_for_snapshot(snapshot_id)
+      metadata.statistics_for_snapshot(snapshot_id)
+    end
+
+    def partition_statistics_for_snapshot(snapshot_id)
+      metadata.partition_statistics_for_snapshot(snapshot_id)
+    end
+
+    def encryption_keys
+      metadata.encryption_keys
+    end
+
+    def next_row_id
+      metadata.next_row_id
     end
 
     def scan(snapshot_id: nil)
       TableScan.new(@table.scan(snapshot_id), self)
     end
 
+    def to_a(snapshot_id: nil)
+      scan(snapshot_id: snapshot_id).to_a
+    end
+
     def to_polars(snapshot_id: nil, storage_options: nil)
       require "polars-df"
 
-      if Gem::Version.new(Polars::VERSION) < Gem::Version.new("0.23")
-        raise "Requires polars-df >= 0.23"
+      if Gem::Version.new(Polars::VERSION) < Gem::Version.new("0.26.1")
+        raise "Requires polars-df >= 0.26.1"
       end
 
       Polars.scan_iceberg(self, snapshot_id:, storage_options:)
@@ -99,6 +160,7 @@ module Iceberg
 
     def append(df)
       check_catalog
+      df = ArrowRecordBatch.new(df, schema.arrow_c_schema) if df.is_a?(Array)
       @table = @table.append(df.arrow_c_stream, @catalog)
       nil
     end
@@ -110,6 +172,10 @@ module Iceberg
 
     private
 
+    def metadata
+      @table.metadata
+    end
+
     def check_catalog
       raise Error, "Read-only table" if @catalog.nil?
     end

data/lib/iceberg/table_definition.rb

@@ -1,7 +1,9 @@
 module Iceberg
   class TableDefinition
     TYPES = %w[
-      boolean int long float double date timestamp timestamptz string uuid binary
+      boolean int long float double decimal date time
+      timestamp timestamptz timestamp_nano timestamptz_nano
+      string uuid fixed binary
     ]
 
     TYPE_ALIASES = {
@@ -21,17 +23,57 @@ module Iceberg
       end
     end
 
-    def column(name, type, null: true, default: nil, comment: nil)
-      type = type.to_s
-      @fields << {
-        id: @fields.size + 1,
+    def column(name, type, null: true, default: nil, comment: nil, limit: nil, precision: nil, scale: nil)
+      unless type.is_a?(Type)
+        type = type.to_s
+        type =
+          case TYPE_ALIASES.fetch(type, type)
+          when "boolean"
+            BooleanType.new
+          when "int"
+            IntType.new
+          when "long"
+            LongType.new
+          when "float"
+            FloatType.new
+          when "double"
+            DoubleType.new
+          when "decimal"
+            DecimalType.new(precision, scale)
+          when "date"
+            DateType.new
+          when "time"
+            TimeType.new
+          when "timestamp"
+            TimestampType.new
+          when "timestamptz"
+            TimestamptzType.new
+          when "timestamp_nano"
+            TimestampNanoType.new
+          when "timestamptz_nano"
+            TimestamptzNanoType.new
+          when "string"
+            StringType.new
+          when "uuid"
+            UUIDType.new
+          when "fixed"
+            FixedType.new(limit)
+          when "binary"
+            BinaryType.new
+          else
+            type
+          end
+      end
+
+      @fields << NestedField.new(
+        field_id: @fields.size + 1,
         name: name.to_s,
-        type: TYPE_ALIASES.fetch(type, type),
+        field_type: type,
         required: !null,
         doc: comment,
         # no need for initial default (and not supported until v3)
         write_default: default
-      }
+      )
     end
   end
 end

data/lib/iceberg/table_scan.rb

@@ -14,5 +14,19 @@ module Iceberg
     def snapshot
       @scan.snapshot
     end
+
+    def collect
+      @scan.collect
+    end
+
+    def to_a
+      collect.to_a
+    end
+
+    def to_arrow
+      require "nanoarrow"
+
+      Nanoarrow::Array.new(@scan)
+    end
   end
 end

data/lib/iceberg/transforms.rb

@@ -0,0 +1,64 @@
+module Iceberg
+  class Transform
+    def ==(other)
+      other.is_a?(self.class)
+    end
+
+    def inspect
+      "#<#{self.class.name}>"
+    end
+  end
+
+  class IdentityTransform < Transform
+  end
+
+  class BucketTransform < Transform
+    attr_reader :num_buckets
+
+    def initialize(num_buckets)
+      @num_buckets = num_buckets
+    end
+
+    def ==(other)
+      other.is_a?(self.class) && other.num_buckets == @num_buckets
+    end
+
+    def inspect
+      "#<#{self.class.name} num_buckets=#{@num_buckets.inspect}>"
+    end
+  end
+
+  class TruncateTransform < Transform
+    attr_reader :width
+
+    def initialize(width)
+      @width = width
+    end
+
+    def ==(other)
+      other.is_a?(self.class) && other.width == @width
+    end
+
+    def inspect
+      "#<#{self.class.name} width=#{@width.inspect}>"
+    end
+  end
+
+  class YearTransform < Transform
+  end
+
+  class MonthTransform < Transform
+  end
+
+  class DayTransform < Transform
+  end
+
+  class HourTransform < Transform
+  end
+
+  class VoidTransform < Transform
+  end
+
+  class UnknownTransform < Transform
+  end
+end

data/lib/iceberg/types.rb

@@ -0,0 +1,137 @@
+module Iceberg
+  class Type
+    def inspect
+      "#<#{self.class.name}>"
+    end
+  end
+
+  class PrimitiveType < Type
+    def ==(other)
+      other.is_a?(self.class)
+    end
+  end
+
+  class BooleanType < PrimitiveType
+  end
+
+  class IntType < PrimitiveType
+  end
+
+  class LongType < PrimitiveType
+  end
+
+  class FloatType < PrimitiveType
+  end
+
+  class DoubleType < PrimitiveType
+  end
+
+  class DecimalType < PrimitiveType
+    attr_reader :precision, :scale
+
+    def initialize(precision, scale)
+      @precision = precision
+      @scale = scale
+    end
+
+    def ==(other)
+      other.is_a?(self.class) && other.precision == @precision && other.scale == @scale
+    end
+
+    def inspect
+      "#<#{self.class.name} precision=#{@precision.inspect}, scale=#{@scale.inspect}>"
+    end
+  end
+
+  class DateType < PrimitiveType
+  end
+
+  class TimeType < PrimitiveType
+  end
+
+  class TimestampType < PrimitiveType
+  end
+
+  class TimestamptzType < PrimitiveType
+  end
+
+  class TimestampNanoType < PrimitiveType
+  end
+
+  class TimestamptzNanoType < PrimitiveType
+  end
+
+  class StringType < PrimitiveType
+  end
+
+  class UUIDType < PrimitiveType
+  end
+
+  class FixedType < PrimitiveType
+    attr_reader :length
+
+    def initialize(length)
+      @length = length
+    end
+
+    def ==(other)
+      other.is_a?(self.class) && other.length == @length
+    end
+
+    def inspect
+      "#<#{self.class.name} length=#{@length.inspect}>"
+    end
+  end
+
+  class BinaryType < PrimitiveType
+  end
+
+  class StructType < Type
+    attr_reader :fields
+
+    def initialize(*fields)
+      @fields = fields
+    end
+
+    def ==(other)
+      other.is_a?(self.class) && other.fields == @fields
+    end
+
+    def inspect
+      "#<#{self.class.name} fields=#{@fields.inspect}>"
+    end
+  end
+
+  class ListType < Type
+    attr_reader :element_field
+
+    def initialize(element_field)
+      @element_field = element_field
+    end
+
+    def ==(other)
+      other.is_a?(self.class) && other.element_field == @element_field
+    end
+
+    def inspect
+      "#<#{self.class.name} element_field=#{@element_field.inspect}>"
+    end
+  end
+
+  class MapType < Type
+    attr_reader :key_field, :value_field
+
+    def initialize(key_field, value_field)
+      @key_field = key_field
+      @value_field = value_field
+    end
+
+    def ==(other)
+      other.is_a?(self.class) && other.key_field == @key_field && other.value_field == @value_field
+    end
+
+    def inspect
+      "#<#{self.class.name} key_field=#{@key_field.inspect}, value_field=#{@value_field.inspect}>"
+    end
+  end
+end

data/lib/iceberg/version.rb

@@ -1,3 +1,3 @@
 module Iceberg
-  VERSION = "0.11.2"
+  VERSION = "0.12.0"
 end

data/lib/iceberg.rb

@@ -5,13 +5,18 @@ rescue LoadError
   require "iceberg/iceberg"
 end
 
+# stdlib
+require "date"
+
 # modules
 require_relative "iceberg/catalog"
-require_relative "iceberg/schema"
+require_relative "iceberg/result"
 require_relative "iceberg/table"
 require_relative "iceberg/table_scan"
 require_relative "iceberg/static_table"
 require_relative "iceberg/table_definition"
+require_relative "iceberg/transforms"
+require_relative "iceberg/types"
 require_relative "iceberg/version"
 
 # catalogs
@@ -25,14 +30,14 @@ module Iceberg
   class Error < StandardError; end
   class InvalidDataError < Error; end
   class NamespaceAlreadyExistsError < Error; end
-  class NamespaceNotFoundError < Error; end
+  class NoSuchNamespaceError < Error; end
+  class NoSuchTableError < Error; end
   class TableAlreadyExistsError < Error; end
-  class TableNotFoundError < Error; end
   class UnsupportedFeatureError < Error; end
 
   class Todo < Error
-    def message
-      "not implemented yet"
+    def initialize(message = "not implemented yet")
+      super(message)
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: iceberg
 version: !ruby/object:Gem::Version
-  version: 0.11.2
+  version: 0.12.0
 platform: ruby
 authors:
 - Andrew Kane
@@ -38,12 +38,21 @@ files:
 - ext/iceberg/Cargo.toml
 - ext/iceberg/extconf.rb
 - ext/iceberg/src/arrow.rs
+- ext/iceberg/src/batch.rs
+- ext/iceberg/src/capsule.rs
 - ext/iceberg/src/catalog.rs
+- ext/iceberg/src/encryption.rs
 - ext/iceberg/src/error.rs
 - ext/iceberg/src/lib.rs
+- ext/iceberg/src/partitioning.rs
+- ext/iceberg/src/result.rs
 - ext/iceberg/src/ruby.rs
 - ext/iceberg/src/runtime.rs
 - ext/iceberg/src/scan.rs
+- ext/iceberg/src/schema.rs
+- ext/iceberg/src/snapshot.rs
+- ext/iceberg/src/sorting.rs
+- ext/iceberg/src/statistics.rs
 - ext/iceberg/src/table.rs
 - ext/iceberg/src/utils.rs
 - lib/iceberg.rb
@@ -51,13 +60,15 @@ files:
 - lib/iceberg/glue_catalog.rb
 - lib/iceberg/memory_catalog.rb
 - lib/iceberg/rest_catalog.rb
+- lib/iceberg/result.rb
 - lib/iceberg/s3_tables_catalog.rb
-- lib/iceberg/schema.rb
 - lib/iceberg/sql_catalog.rb
 - lib/iceberg/static_table.rb
 - lib/iceberg/table.rb
 - lib/iceberg/table_definition.rb
 - lib/iceberg/table_scan.rb
+- lib/iceberg/transforms.rb
+- lib/iceberg/types.rb
 - lib/iceberg/version.rb
 homepage: https://github.com/ankane/iceberg-ruby
 licenses:
@@ -77,7 +88,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.10
+rubygems_version: 4.0.14
 specification_version: 4
 summary: Apache Iceberg for Ruby
 test_files: []

data/lib/iceberg/schema.rb

@@ -1,10 +0,0 @@
-module Iceberg
-  class Schema
-    attr_reader :fields, :schema_id
-
-    def initialize(fields, schema_id: nil)
-      @fields = fields
-      @schema_id = schema_id
-    end
-  end
-end