iceberg 0.11.2-aarch64-linux → 0.12.0-aarch64-linux

data/NOTICE.txt

@@ -1,6 +1,6 @@
 
 Apache Iceberg
-Copyright 2017-2025 The Apache Software Foundation
+Copyright 2017-2026 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
@@ -8,7 +8,7 @@ The Apache Software Foundation (http://www.apache.org/).
 --------------------------------------------------------------------------------
 
 Apache Iceberg Rust
-Copyright 2023-2024 The Apache Software Foundation
+Copyright 2023-2026 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).

data/README.md

@@ -19,10 +19,10 @@ gem "iceberg"
 Create a client for an Iceberg catalog
 
 ```ruby
-catalog = Iceberg::RestCatalog.new(uri: "http://localhost:8181")
+catalog = Iceberg::RestCatalog.new(uri: "http://localhost:8181", default_namespace: "main")
 ```
 
-Create a namespace
+Create a namespace if needed
 
 ```ruby
 catalog.create_namespace("main")
@@ -31,7 +31,7 @@ catalog.create_namespace("main")
 Create a table
 
 ```ruby
-catalog.create_table("main.events") do |t|
+catalog.create_table("events") do |t|
   t.bigint "id"
   t.float "value"
 end
@@ -41,14 +41,14 @@ Or with [Polars](https://github.com/ankane/ruby-polars)
 
 ```ruby
 df = Polars::DataFrame.new({"id" => [1, 2], "value" => [3.0, 4.0]})
-table = catalog.create_table("main.events", schema: df.schema)
+table = catalog.create_table("events", schema: df.schema)
 table.append(df)
 ```
 
 Load a table
 
 ```ruby
-table = catalog.load_table("main.events")
+table = catalog.load_table("events")
 ```
 
 Query a table
@@ -67,6 +67,26 @@ Iceberg::RestCatalog.new(
 )
 ```
 
+### S3 Tables
+
+[unreleased]
+
+```ruby
+Iceberg::S3TablesCatalog.new(
+  arn: "arn:aws:s3tables:..."
+)
+```
+
+### Glue
+
+[unreleased]
+
+```ruby
+Iceberg::GlueCatalog.new(
+  warehouse: "s3://my-bucket"
+)
+```
+
 ### SQL
 
 ```ruby
@@ -133,7 +153,7 @@ catalog.list_tables("main")
 Create a table
 
 ```ruby
-catalog.create_table("main.events") do |t|
+catalog.create_table("events") do |t|
   t.integer "id"
   t.float "value"
 end
@@ -142,31 +162,31 @@ end
 Load a table
 
 ```ruby
-catalog.load_table("main.events")
+catalog.load_table("events")
 ```
 
 Check if a table exists
 
 ```ruby
-catalog.table_exists?("main.events")
+catalog.table_exists?("events")
 ```
 
 Rename a table
 
 ```ruby
-catalog.rename_table("main.events", "main.events2")
+catalog.rename_table("events", "events2")
 ```
 
 Register a table
 
 ```ruby
-catalog.register_table("main.events", "metadata.json")
+catalog.register_table("events", "metadata.json")
 ```
 
 Drop a table
 
 ```ruby
-catalog.drop_table("main.events")
+catalog.drop_table("events")
 ```
 
 ## Static Tables
@@ -196,7 +216,7 @@ To get started with development:
 git clone https://github.com/ankane/iceberg-ruby.git
 cd iceberg-ruby
 bundle install
-bundle exec rake compile
+bundle exec rake compile:dev
 
 # memory catalog
 bundle exec rake test:memory

data/lib/iceberg/catalog.rb

@@ -1,14 +1,18 @@
 module Iceberg
   class Catalog
+    def _initialize(catalog, default_namespace:)
+      @catalog = catalog
+      @default_namespace = default_namespace
+    end
+
     def list_namespaces(parent = nil)
       @catalog.list_namespaces(parent)
     end
 
     def create_namespace(namespace, properties: {}, if_not_exists: nil)
       @catalog.create_namespace(namespace, properties)
-    rescue Error => e
-      # ideally all catalogs would use NamespaceAlreadyExistsError
-      if !if_not_exists || (e.message != "Cannot create namespace" && !e.message.include?("already exists"))
+    rescue NamespaceAlreadyExistsError => e
+      if !if_not_exists
         raise e
       end
       nil
@@ -28,19 +32,18 @@ module Iceberg
 
     def drop_namespace(namespace, if_exists: nil)
       @catalog.drop_namespace(namespace)
-    rescue Error => e
-      # ideally all catalogs would use NamespaceNotFoundError
-      if !if_exists || (e.message != "Tried to drop a namespace that does not exist" && !e.message.include?("No such namespace") && !e.message.include?("The specified namespace does not exist") && !e.message.include?("not found"))
+    rescue NoSuchNamespaceError => e
+      if !if_exists
         raise e
       end
       nil
     end
 
-    def list_tables(namespace)
-      @catalog.list_tables(namespace)
+    def list_tables(namespace = nil)
+      @catalog.list_tables(namespace || @default_namespace)
     end
 
-    def create_table(table_name, schema: nil, location: nil)
+    def create_table(table_name, schema: nil, location: nil, partition_spec: nil, sort_order: nil, properties: {})
       if !schema.nil? && block_given?
         raise ArgumentError, "Must pass schema or block"
       end
@@ -48,63 +51,79 @@ module Iceberg
       if block_given?
         table_definition = TableDefinition.new
         yield table_definition
-        schema = Schema.new(table_definition.fields)
-      elsif schema.is_a?(Hash) || (defined?(Polars::Schema) && schema.is_a?(Polars::Schema))
-        fields =
-          schema.to_h.map.with_index do |(k, v), i|
-            {
-              id: i + 1,
-              name: k.is_a?(Symbol) ? k.to_s : k,
-              type: v,
-              required: false
-            }
-          end
-        schema = Schema.new(fields)
+        schema = Schema.new(*table_definition.fields)
+      elsif schema.is_a?(Schema)
+        # do nothing
+      elsif schema.respond_to?(:arrow_c_schema)
+        schema = Schema.new(schema)
+      elsif schema.is_a?(Hash)
+        table_definition = TableDefinition.new
+        schema.each do |k, v|
+          table_definition.column(k, v)
+        end
+        schema = Schema.new(*table_definition.fields)
       elsif schema.nil?
-        schema = Schema.new([])
+        schema = Schema.new
       end
 
-      Table.new(@catalog.create_table(table_name, schema, location), @catalog)
+      Table.new(@catalog.create_table(with_namespace(table_name), schema, location, partition_spec, sort_order, properties), @catalog)
     end
 
     def load_table(table_name)
-      Table.new(@catalog.load_table(table_name), @catalog)
+      Table.new(@catalog.load_table(with_namespace(table_name)), @catalog)
     end
 
     def drop_table(table_name, if_exists: nil)
-      @catalog.drop_table(table_name)
-    rescue Error => e
-      # ideally all catalogs would use TableNotFoundError
-      if !if_exists || (e.message != "Tried to drop a table that does not exist" && !e.message.include?("No such table") && !e.message.include?("The specified table does not exist") && !e.message.include?("not found"))
+      @catalog.drop_table(with_namespace(table_name))
+    rescue NoSuchTableError => e
+      if !if_exists
         raise e
       end
       nil
     end
 
+    def purge_table(table_name)
+      @catalog.purge_table(with_namespace(table_name))
+    end
+
     def table_exists?(table_name)
-      @catalog.table_exists?(table_name)
-    rescue NamespaceNotFoundError
+      @catalog.table_exists?(with_namespace(table_name))
+    rescue NoSuchNamespaceError
       false
     end
 
     def rename_table(table_name, new_name)
-      @catalog.rename_table(table_name, new_name)
+      @catalog.rename_table(with_namespace(table_name), with_namespace(new_name))
     end
 
     def register_table(table_name, metadata_location)
-      @catalog.register_table(table_name, metadata_location)
+      @catalog.register_table(with_namespace(table_name), metadata_location)
     end
 
-    def sql(sql)
+    def sql(sql, params = [])
       # requires datafusion feature
-      raise Todo unless @catalog.respond_to?(:sql)
+      raise Todo unless @catalog.respond_to?(:session_context)
 
-      @catalog.sql(sql)
+      session_context.sql(sql, params)
     end
 
     # hide internal state
     def inspect
       to_s
     end
+
+    private
+
+    def with_namespace(table_name)
+      if @default_namespace && table_name.is_a?(String) && !table_name.include?(".")
+        [@default_namespace, table_name]
+      else
+        table_name
+      end
+    end
+
+    def session_context
+      @session_context ||= @catalog.session_context(@default_namespace)
+    end
   end
 end

data/lib/iceberg/glue_catalog.rb

@@ -1,8 +1,11 @@
 module Iceberg
   class GlueCatalog < Catalog
     # warehouse is URI of S3 storage bucket
-    def initialize(warehouse:)
-      @catalog = RbCatalog.new_glue(warehouse)
+    def initialize(warehouse:, default_namespace: nil)
+      _initialize(
+        RbCatalog.new_glue(warehouse),
+        default_namespace:
+      )
     end
   end
 end

data/lib/iceberg/memory_catalog.rb

@@ -1,8 +1,11 @@
 module Iceberg
   class MemoryCatalog < Catalog
     # warehouse is default storage location
-    def initialize(warehouse: nil)
-      @catalog = RbCatalog.new_memory(warehouse)
+    def initialize(warehouse: nil, default_namespace: nil)
+      _initialize(
+        RbCatalog.new_memory(warehouse),
+        default_namespace:
+      )
     end
   end
 end

data/lib/iceberg/rest_catalog.rb

@@ -1,8 +1,11 @@
 module Iceberg
   class RestCatalog < Catalog
     # warehouse is passed to REST server
-    def initialize(uri:, warehouse: nil, properties: {})
-      @catalog = RbCatalog.new_rest(uri, warehouse, properties)
+    def initialize(uri:, warehouse: nil, properties: {}, default_namespace: nil)
+      _initialize(
+        RbCatalog.new_rest(uri, warehouse, properties),
+        default_namespace:
+      )
     end
   end
 end

data/lib/iceberg/result.rb

@@ -0,0 +1,22 @@
+module Iceberg
+  class Result
+    include Enumerable
+
+    attr_reader :columns, :rows
+
+    def initialize(columns, rows)
+      @columns = columns
+      @rows = rows
+    end
+
+    def each
+      @rows.each do |row|
+        yield @columns.zip(row).to_h
+      end
+    end
+
+    def empty?
+      rows.empty?
+    end
+  end
+end

data/lib/iceberg/s3_tables_catalog.rb

@@ -1,7 +1,10 @@
 module Iceberg
   class S3TablesCatalog < Catalog
-    def initialize(arn:)
-      @catalog = RbCatalog.new_s3tables(arn)
+    def initialize(arn:, default_namespace: nil)
+      _initialize(
+        RbCatalog.new_s3tables(arn),
+        default_namespace:
+      )
     end
   end
 end

data/lib/iceberg/sql_catalog.rb

@@ -2,8 +2,11 @@ module Iceberg
   class SqlCatalog < Catalog
     # warehouse is default storage location
     # name is stored in SQL table
-    def initialize(uri:, warehouse:, name: "main", properties: {})
-      @catalog = RbCatalog.new_sql(uri, warehouse, name, properties)
+    def initialize(uri:, warehouse:, name: "main", properties: {}, default_namespace: nil)
+      _initialize(
+        RbCatalog.new_sql(uri, warehouse, name, properties),
+        default_namespace:
+      )
     end
   end
 end

data/lib/iceberg/table.rb

@@ -5,93 +5,154 @@ module Iceberg
       @catalog = catalog
     end
 
+    def refresh
+      @table = @catalog.load_table(@table.identifier)
+    end
+
     def format_version
-      @table.format_version
+      metadata.format_version
     end
 
     def uuid
-      @table.uuid
+      metadata.uuid
     end
 
     def location
-      @table.location
+      metadata.location
     end
 
     def last_sequence_number
-      @table.last_sequence_number
+      metadata.last_sequence_number
     end
 
     def next_sequence_number
-      @table.next_sequence_number
+      metadata.next_sequence_number
     end
 
     def last_column_id
-      @table.last_column_id
+      metadata.last_column_id
     end
 
     def last_partition_id
-      @table.last_partition_id
+      metadata.last_partition_id
+    end
+
+    def last_updated_at
+      ms = metadata.last_updated_ms
+      Time.at(ms / 1000, ms % 1000, :millisecond)
     end
 
     def schemas
-      @table.schemas
+      metadata.schemas
     end
 
     def schema_by_id(schema_id)
-      @table.schema_by_id(schema_id)
+      metadata.schema_by_id(schema_id)
     end
 
     def current_schema
-      @table.current_schema
+      metadata.current_schema
     end
     alias_method :schema, :current_schema
 
     def current_schema_id
-      @table.current_schema_id
+      metadata.current_schema_id
     end
     alias_method :schema_id, :current_schema_id
 
+    def partition_specs
+      metadata.partition_specs
+    end
+
+    def partition_spec_by_id(partition_spec_id)
+      metadata.partition_spec_by_id(partition_spec_id)
+    end
+
+    def default_partition_spec
+      metadata.default_partition_spec
+    end
+
     def default_partition_spec_id
-      @table.default_partition_spec_id
+      metadata.default_partition_spec_id
     end
 
     def snapshots
-      @table.snapshots
+      metadata.snapshots
     end
 
     def snapshot_by_id(snapshot_id)
-      @table.snapshot_by_id(snapshot_id)
+      metadata.snapshot_by_id(snapshot_id)
     end
 
     def history
-      @table.history
+      metadata.history
     end
 
     def metadata_log
-      @table.metadata_log
+      metadata.metadata_log
     end
 
     def current_snapshot
-      @table.current_snapshot
+      metadata.current_snapshot
     end
 
     def current_snapshot_id
-      @table.current_snapshot_id
+      metadata.current_snapshot_id
+    end
+
+    def sort_orders
+      metadata.sort_orders
+    end
+
+    def default_sort_order
+      metadata.default_sort_order
+    end
+
+    def default_sort_order_id
+      metadata.default_sort_order_id
     end
 
     def properties
-      @table.properties
+      metadata.properties
+    end
+
+    def statistics
+      metadata.statistics
+    end
+
+    def partition_statistics
+      metadata.partition_statistics
+    end
+
+    def statistics_for_snapshot(snapshot_id)
+      metadata.statistics_for_snapshot(snapshot_id)
+    end
+
+    def partition_statistics_for_snapshot(snapshot_id)
+      metadata.partition_statistics_for_snapshot(snapshot_id)
+    end
+
+    def encryption_keys
+      metadata.encryption_keys
+    end
+
+    def next_row_id
+      metadata.next_row_id
     end
 
     def scan(snapshot_id: nil)
       TableScan.new(@table.scan(snapshot_id), self)
     end
 
+    def to_a(snapshot_id: nil)
+      scan(snapshot_id: snapshot_id).to_a
+    end
+
     def to_polars(snapshot_id: nil, storage_options: nil)
       require "polars-df"
 
-      if Gem::Version.new(Polars::VERSION) < Gem::Version.new("0.23")
-        raise "Requires polars-df >= 0.23"
+      if Gem::Version.new(Polars::VERSION) < Gem::Version.new("0.26.1")
+        raise "Requires polars-df >= 0.26.1"
       end
 
       Polars.scan_iceberg(self, snapshot_id:, storage_options:)
@@ -99,6 +160,7 @@ module Iceberg
 
     def append(df)
       check_catalog
+      df = ArrowRecordBatch.new(df, schema.arrow_c_schema) if df.is_a?(Array)
       @table = @table.append(df.arrow_c_stream, @catalog)
       nil
     end
@@ -110,6 +172,10 @@ module Iceberg
 
     private
 
+    def metadata
+      @table.metadata
+    end
+
     def check_catalog
       raise Error, "Read-only table" if @catalog.nil?
     end

data/lib/iceberg/table_definition.rb

@@ -1,7 +1,9 @@
 module Iceberg
   class TableDefinition
     TYPES = %w[
-      boolean int long float double date timestamp timestamptz string uuid binary
+      boolean int long float double decimal date time
+      timestamp timestamptz timestamp_nano timestamptz_nano
+      string uuid fixed binary
     ]
 
     TYPE_ALIASES = {
@@ -21,17 +23,57 @@ module Iceberg
       end
     end
 
-    def column(name, type, null: true, default: nil, comment: nil)
-      type = type.to_s
-      @fields << {
-        id: @fields.size + 1,
+    def column(name, type, null: true, default: nil, comment: nil, limit: nil, precision: nil, scale: nil)
+      unless type.is_a?(Type)
+        type = type.to_s
+        type =
+          case TYPE_ALIASES.fetch(type, type)
+          when "boolean"
+            BooleanType.new
+          when "int"
+            IntType.new
+          when "long"
+            LongType.new
+          when "float"
+            FloatType.new
+          when "double"
+            DoubleType.new
+          when "decimal"
+            DecimalType.new(precision, scale)
+          when "date"
+            DateType.new
+          when "time"
+            TimeType.new
+          when "timestamp"
+            TimestampType.new
+          when "timestamptz"
+            TimestamptzType.new
+          when "timestamp_nano"
+            TimestampNanoType.new
+          when "timestamptz_nano"
+            TimestamptzNanoType.new
+          when "string"
+            StringType.new
+          when "uuid"
+            UUIDType.new
+          when "fixed"
+            FixedType.new(limit)
+          when "binary"
+            BinaryType.new
+          else
+            type
+          end
+      end
+
+      @fields << NestedField.new(
+        field_id: @fields.size + 1,
         name: name.to_s,
-        type: TYPE_ALIASES.fetch(type, type),
+        field_type: type,
         required: !null,
         doc: comment,
         # no need for initial default (and not supported until v3)
         write_default: default
-      }
+      )
     end
   end
 end

data/lib/iceberg/table_scan.rb

@@ -14,5 +14,19 @@ module Iceberg
     def snapshot
       @scan.snapshot
     end
+
+    def collect
+      @scan.collect
+    end
+
+    def to_a
+      collect.to_a
+    end
+
+    def to_arrow
+      require "nanoarrow"
+
+      Nanoarrow::Array.new(@scan)
+    end
   end
 end