iceberg 0.11.2-aarch64-linux → 0.12.1-aarch64-linux

data/NOTICE.txt

@@ -1,6 +1,6 @@
 
 Apache Iceberg
-Copyright 2017-2025 The Apache Software Foundation
+Copyright 2017-2026 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
@@ -8,7 +8,7 @@ The Apache Software Foundation (http://www.apache.org/).
 --------------------------------------------------------------------------------
 
 Apache Iceberg Rust
-Copyright 2023-2024 The Apache Software Foundation
+Copyright 2023-2026 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).

data/README.md

@@ -19,10 +19,10 @@ gem "iceberg"
 Create a client for an Iceberg catalog
 
 ```ruby
-catalog = Iceberg::RestCatalog.new(uri: "http://localhost:8181")
+catalog = Iceberg::RestCatalog.new(uri: "http://localhost:8181", default_namespace: "main")
 ```
 
-Create a namespace
+Create a namespace if needed
 
 ```ruby
 catalog.create_namespace("main")
@@ -31,30 +31,53 @@ catalog.create_namespace("main")
 Create a table
 
 ```ruby
-catalog.create_table("main.events") do |t|
-  t.bigint "id"
-  t.float "value"
-end
+table =
+  catalog.create_table("events") do |t|
+    t.bigint "id"
+    t.float "value"
+  end
+```
+
+Append data
+
+```ruby
+table.append([{"id" => 1, "value" => 3.0}, {"id" => 2, "value" => 4.0}])
 ```
 
 Or with [Polars](https://github.com/ankane/ruby-polars)
 
 ```ruby
 df = Polars::DataFrame.new({"id" => [1, 2], "value" => [3.0, 4.0]})
-table = catalog.create_table("main.events", schema: df.schema)
+table = catalog.create_table("events", schema: df.schema)
 table.append(df)
 ```
 
+Or with [Nanoarrow](https://github.com/ankane/nanoarrow-ruby)
+
+```ruby
+arr =
+  Nanoarrow::Array.new(
+    [{"id" => 1, "value" => 3.0}, {"id" => 2, "value" => 4.0}],
+    Nanoarrow.struct({"a" => Nanoarrow.int64, "b" => Nanoarrow.string})
+  )
+table = catalog.create_table("events", schema: arr.schema)
+table.append(arr)
+```
+
 Load a table
 
 ```ruby
-table = catalog.load_table("main.events")
+table = catalog.load_table("events")
 ```
 
 Query a table
 
 ```ruby
+table.to_a
+# or
 table.to_polars.collect
+# or
+table.scan.to_arrow
 ```
 
 ## Catalog Types
@@ -67,6 +90,22 @@ Iceberg::RestCatalog.new(
 )
 ```
 
+### S3 Tables
+
+```ruby
+Iceberg::S3TablesCatalog.new(
+  arn: "arn:aws:s3tables:..."
+)
+```
+
+### Glue
+
+```ruby
+Iceberg::GlueCatalog.new(
+  warehouse: "s3://my-bucket"
+)
+```
+
 ### SQL
 
 ```ruby
@@ -133,7 +172,7 @@ catalog.list_tables("main")
 Create a table
 
 ```ruby
-catalog.create_table("main.events") do |t|
+catalog.create_table("events") do |t|
   t.integer "id"
   t.float "value"
 end
@@ -142,31 +181,31 @@ end
 Load a table
 
 ```ruby
-catalog.load_table("main.events")
+catalog.load_table("events")
 ```
 
 Check if a table exists
 
 ```ruby
-catalog.table_exists?("main.events")
+catalog.table_exists?("events")
 ```
 
 Rename a table
 
 ```ruby
-catalog.rename_table("main.events", "main.events2")
+catalog.rename_table("events", "events2")
 ```
 
 Register a table
 
 ```ruby
-catalog.register_table("main.events", "metadata.json")
+catalog.register_table("events", "metadata.json")
 ```
 
 Drop a table
 
 ```ruby
-catalog.drop_table("main.events")
+catalog.drop_table("events")
 ```
 
 ## Static Tables
@@ -196,7 +235,7 @@ To get started with development:
 git clone https://github.com/ankane/iceberg-ruby.git
 cd iceberg-ruby
 bundle install
-bundle exec rake compile
+bundle exec rake compile:dev
 
 # memory catalog
 bundle exec rake test:memory
@@ -208,4 +247,10 @@ bundle exec rake test:rest
 # SQL catalog
 createdb iceberg_ruby_test
 bundle exec rake test:sql
+
+# S3 Tables catalog
+bundle exec rake test:s3tables
+
+# Glue catalog
+bundle exec rake test:glue
 ```

data/lib/iceberg/catalog.rb

@@ -1,14 +1,18 @@
 module Iceberg
   class Catalog
+    def _initialize(catalog, default_namespace:)
+      @catalog = catalog
+      @default_namespace = default_namespace
+    end
+
     def list_namespaces(parent = nil)
       @catalog.list_namespaces(parent)
     end
 
     def create_namespace(namespace, properties: {}, if_not_exists: nil)
       @catalog.create_namespace(namespace, properties)
-    rescue Error => e
-      # ideally all catalogs would use NamespaceAlreadyExistsError
-      if !if_not_exists || (e.message != "Cannot create namespace" && !e.message.include?("already exists"))
+    rescue NamespaceAlreadyExistsError => e
+      if !if_not_exists
         raise e
       end
       nil
@@ -28,19 +32,18 @@ module Iceberg
 
     def drop_namespace(namespace, if_exists: nil)
       @catalog.drop_namespace(namespace)
-    rescue Error => e
-      # ideally all catalogs would use NamespaceNotFoundError
-      if !if_exists || (e.message != "Tried to drop a namespace that does not exist" && !e.message.include?("No such namespace") && !e.message.include?("The specified namespace does not exist") && !e.message.include?("not found"))
+    rescue NoSuchNamespaceError => e
+      if !if_exists
         raise e
       end
       nil
     end
 
-    def list_tables(namespace)
-      @catalog.list_tables(namespace)
+    def list_tables(namespace = nil)
+      @catalog.list_tables(namespace || @default_namespace)
     end
 
-    def create_table(table_name, schema: nil, location: nil)
+    def create_table(table_name, schema: nil, location: nil, partition_spec: nil, sort_order: nil, properties: {})
       if !schema.nil? && block_given?
         raise ArgumentError, "Must pass schema or block"
       end
@@ -48,63 +51,84 @@ module Iceberg
       if block_given?
         table_definition = TableDefinition.new
         yield table_definition
-        schema = Schema.new(table_definition.fields)
-      elsif schema.is_a?(Hash) || (defined?(Polars::Schema) && schema.is_a?(Polars::Schema))
-        fields =
-          schema.to_h.map.with_index do |(k, v), i|
-            {
-              id: i + 1,
-              name: k.is_a?(Symbol) ? k.to_s : k,
-              type: v,
-              required: false
-            }
-          end
-        schema = Schema.new(fields)
+        schema = Schema.new(*table_definition.fields)
+      elsif schema.is_a?(Schema)
+        # do nothing
+      elsif schema.respond_to?(:arrow_c_schema)
+        schema = Schema.new(schema)
+      elsif schema.is_a?(Hash)
+        table_definition = TableDefinition.new
+        schema.each do |k, v|
+          table_definition.column(k, v)
+        end
+        schema = Schema.new(*table_definition.fields)
       elsif schema.nil?
-        schema = Schema.new([])
+        schema = Schema.new
       end
 
-      Table.new(@catalog.create_table(table_name, schema, location), @catalog)
+      format_version = (properties["format-version"] || 2).to_i
+      if properties["format-version"] && (is_a?(MemoryCatalog) || is_a?(SqlCatalog))
+        properties = properties.except("format-version")
+      end
+
+      Table.new(@catalog.create_table(with_namespace(table_name), schema, location, partition_spec, sort_order, properties, format_version), @catalog)
     end
 
     def load_table(table_name)
-      Table.new(@catalog.load_table(table_name), @catalog)
+      Table.new(@catalog.load_table(with_namespace(table_name)), @catalog)
     end
 
     def drop_table(table_name, if_exists: nil)
-      @catalog.drop_table(table_name)
-    rescue Error => e
-      # ideally all catalogs would use TableNotFoundError
-      if !if_exists || (e.message != "Tried to drop a table that does not exist" && !e.message.include?("No such table") && !e.message.include?("The specified table does not exist") && !e.message.include?("not found"))
+      @catalog.drop_table(with_namespace(table_name))
+    rescue NoSuchTableError => e
+      if !if_exists
         raise e
       end
       nil
     end
 
+    def purge_table(table_name)
+      @catalog.purge_table(with_namespace(table_name))
+    end
+
     def table_exists?(table_name)
-      @catalog.table_exists?(table_name)
-    rescue NamespaceNotFoundError
+      @catalog.table_exists?(with_namespace(table_name))
+    rescue NoSuchNamespaceError
       false
     end
 
     def rename_table(table_name, new_name)
-      @catalog.rename_table(table_name, new_name)
+      @catalog.rename_table(with_namespace(table_name), with_namespace(new_name))
     end
 
     def register_table(table_name, metadata_location)
-      @catalog.register_table(table_name, metadata_location)
+      @catalog.register_table(with_namespace(table_name), metadata_location)
     end
 
-    def sql(sql)
+    def sql(sql, params = [])
       # requires datafusion feature
-      raise Todo unless @catalog.respond_to?(:sql)
+      raise Todo unless @catalog.respond_to?(:session_context)
 
-      @catalog.sql(sql)
+      session_context.sql(sql, params)
     end
 
     # hide internal state
     def inspect
       to_s
     end
+
+    private
+
+    def with_namespace(table_name)
+      if @default_namespace && table_name.is_a?(String) && !table_name.include?(".")
+        [@default_namespace, table_name]
+      else
+        table_name
+      end
+    end
+
+    def session_context
+      @session_context ||= @catalog.session_context(@default_namespace)
+    end
   end
 end

data/lib/iceberg/glue_catalog.rb

@@ -1,8 +1,11 @@
 module Iceberg
   class GlueCatalog < Catalog
     # warehouse is URI of S3 storage bucket
-    def initialize(warehouse:)
-      @catalog = RbCatalog.new_glue(warehouse)
+    def initialize(warehouse:, default_namespace: nil)
+      _initialize(
+        RbCatalog.new_glue(warehouse),
+        default_namespace:
+      )
     end
   end
 end

data/lib/iceberg/memory_catalog.rb

@@ -1,8 +1,11 @@
 module Iceberg
   class MemoryCatalog < Catalog
     # warehouse is default storage location
-    def initialize(warehouse: nil)
-      @catalog = RbCatalog.new_memory(warehouse)
+    def initialize(warehouse: nil, default_namespace: nil)
+      _initialize(
+        RbCatalog.new_memory(warehouse),
+        default_namespace:
+      )
     end
   end
 end

data/lib/iceberg/rest_catalog.rb

@@ -1,8 +1,11 @@
 module Iceberg
   class RestCatalog < Catalog
     # warehouse is passed to REST server
-    def initialize(uri:, warehouse: nil, properties: {})
-      @catalog = RbCatalog.new_rest(uri, warehouse, properties)
+    def initialize(uri:, warehouse: nil, properties: {}, default_namespace: nil)
+      _initialize(
+        RbCatalog.new_rest(uri, warehouse, properties),
+        default_namespace:
+      )
     end
   end
 end

data/lib/iceberg/result.rb

@@ -0,0 +1,22 @@
+module Iceberg
+  class Result
+    include Enumerable
+
+    attr_reader :columns, :rows
+
+    def initialize(columns, rows)
+      @columns = columns
+      @rows = rows
+    end
+
+    def each
+      @rows.each do |row|
+        yield @columns.zip(row).to_h
+      end
+    end
+
+    def empty?
+      rows.empty?
+    end
+  end
+end

data/lib/iceberg/s3_tables_catalog.rb

@@ -1,7 +1,10 @@
 module Iceberg
   class S3TablesCatalog < Catalog
-    def initialize(arn:)
-      @catalog = RbCatalog.new_s3tables(arn)
+    def initialize(arn:, default_namespace: nil)
+      _initialize(
+        RbCatalog.new_s3tables(arn),
+        default_namespace:
+      )
     end
   end
 end

data/lib/iceberg/sql_catalog.rb

@@ -2,8 +2,11 @@ module Iceberg
   class SqlCatalog < Catalog
     # warehouse is default storage location
     # name is stored in SQL table
-    def initialize(uri:, warehouse:, name: "main", properties: {})
-      @catalog = RbCatalog.new_sql(uri, warehouse, name, properties)
+    def initialize(uri:, warehouse:, name: "main", properties: {}, default_namespace: nil)
+      _initialize(
+        RbCatalog.new_sql(uri, warehouse, name, properties),
+        default_namespace:
+      )
     end
   end
 end

data/lib/iceberg/table.rb

@@ -5,93 +5,154 @@ module Iceberg
       @catalog = catalog
     end
 
+    def refresh
+      @table = @catalog.load_table(@table.identifier)
+    end
+
     def format_version
-      @table.format_version
+      metadata.format_version
     end
 
     def uuid
-      @table.uuid
+      metadata.uuid
     end
 
     def location
-      @table.location
+      metadata.location
     end
 
     def last_sequence_number
-      @table.last_sequence_number
+      metadata.last_sequence_number
     end
 
     def next_sequence_number
-      @table.next_sequence_number
+      metadata.next_sequence_number
     end
 
     def last_column_id
-      @table.last_column_id
+      metadata.last_column_id
     end
 
     def last_partition_id
-      @table.last_partition_id
+      metadata.last_partition_id
+    end
+
+    def last_updated_at
+      ms = metadata.last_updated_ms
+      Time.at(ms / 1000, ms % 1000, :millisecond)
     end
 
     def schemas
-      @table.schemas
+      metadata.schemas
     end
 
     def schema_by_id(schema_id)
-      @table.schema_by_id(schema_id)
+      metadata.schema_by_id(schema_id)
     end
 
     def current_schema
-      @table.current_schema
+      metadata.current_schema
     end
     alias_method :schema, :current_schema
 
     def current_schema_id
-      @table.current_schema_id
+      metadata.current_schema_id
     end
     alias_method :schema_id, :current_schema_id
 
+    def partition_specs
+      metadata.partition_specs
+    end
+
+    def partition_spec_by_id(partition_spec_id)
+      metadata.partition_spec_by_id(partition_spec_id)
+    end
+
+    def default_partition_spec
+      metadata.default_partition_spec
+    end
+
     def default_partition_spec_id
-      @table.default_partition_spec_id
+      metadata.default_partition_spec_id
     end
 
     def snapshots
-      @table.snapshots
+      metadata.snapshots
     end
 
     def snapshot_by_id(snapshot_id)
-      @table.snapshot_by_id(snapshot_id)
+      metadata.snapshot_by_id(snapshot_id)
     end
 
     def history
-      @table.history
+      metadata.history
     end
 
     def metadata_log
-      @table.metadata_log
+      metadata.metadata_log
     end
 
     def current_snapshot
-      @table.current_snapshot
+      metadata.current_snapshot
     end
 
     def current_snapshot_id
-      @table.current_snapshot_id
+      metadata.current_snapshot_id
+    end
+
+    def sort_orders
+      metadata.sort_orders
+    end
+
+    def default_sort_order
+      metadata.default_sort_order
+    end
+
+    def default_sort_order_id
+      metadata.default_sort_order_id
     end
 
     def properties
-      @table.properties
+      metadata.properties
+    end
+
+    def statistics
+      metadata.statistics
+    end
+
+    def partition_statistics
+      metadata.partition_statistics
+    end
+
+    def statistics_for_snapshot(snapshot_id)
+      metadata.statistics_for_snapshot(snapshot_id)
+    end
+
+    def partition_statistics_for_snapshot(snapshot_id)
+      metadata.partition_statistics_for_snapshot(snapshot_id)
+    end
+
+    def encryption_keys
+      metadata.encryption_keys
+    end
+
+    def next_row_id
+      metadata.next_row_id
     end
 
     def scan(snapshot_id: nil)
       TableScan.new(@table.scan(snapshot_id), self)
     end
 
+    def to_a(snapshot_id: nil)
+      scan(snapshot_id: snapshot_id).to_a
+    end
+
     def to_polars(snapshot_id: nil, storage_options: nil)
       require "polars-df"
 
-      if Gem::Version.new(Polars::VERSION) < Gem::Version.new("0.23")
-        raise "Requires polars-df >= 0.23"
+      if Gem::Version.new(Polars::VERSION) < Gem::Version.new("0.26.1")
+        raise "Requires polars-df >= 0.26.1"
       end
 
       Polars.scan_iceberg(self, snapshot_id:, storage_options:)
@@ -99,6 +160,7 @@ module Iceberg
 
     def append(df)
       check_catalog
+      df = ArrowRecordBatch.new(df, schema.arrow_c_schema) if df.is_a?(Array)
       @table = @table.append(df.arrow_c_stream, @catalog)
       nil
     end
@@ -110,6 +172,10 @@ module Iceberg
 
     private
 
+    def metadata
+      @table.metadata
+    end
+
     def check_catalog
       raise Error, "Read-only table" if @catalog.nil?
     end