iceberg 0.10.3 → 0.11.0

data/README.md

@@ -2,6 +2,8 @@
 
 [Apache Iceberg](https://iceberg.apache.org/) for Ruby
 
+:duck: Also check out [SeaDuck](https://github.com/ankane/seaduck)
+
 [![Build Status](https://github.com/ankane/iceberg-ruby/actions/workflows/build.yml/badge.svg)](https://github.com/ankane/iceberg-ruby/actions)
 
 ## Installation
@@ -35,7 +37,7 @@ catalog.create_table("main.events") do |t|
 end
 ```
 
-Or
+Or with [Polars](https://github.com/ankane/ruby-polars)
 
 ```ruby
 df = Polars::DataFrame.new({"id" => [1, 2], "value" => [3.0, 4.0]})

data/ext/iceberg/Cargo.toml

@@ -1,27 +1,29 @@
 [package]
-name = "iceberg"
-version = "0.10.3"
+name = "iceberg-ruby"
+version = "0.11.0"
 license = "Apache-2.0"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2024"
-rust-version = "1.87"
+rust-version = "1.88"
 publish = false
 
 [lib]
+name = "iceberg"
 crate-type = ["cdylib"]
 
 [dependencies]
-arrow-array = { version = "55", features = ["ffi"] }
-arrow-schema = "55"
-datafusion = { version = "48", optional = true }
+arrow-array = { version = "57", features = ["ffi"] }
+arrow-schema = "57"
+datafusion = { version = "51", optional = true }
 futures = "0.3"
-iceberg = "0.7"
-iceberg-catalog-glue = { version = "0.7", optional = true }
-iceberg-catalog-rest = { version = "0.7", optional = true }
-iceberg-catalog-sql = { version = "0.7", optional = true }
-iceberg-datafusion = { version = "0.7", optional = true }
+iceberg = "0.8"
+iceberg-catalog-glue = { version = "0.8", optional = true }
+iceberg-catalog-rest = { version = "0.8", optional = true }
+iceberg-catalog-s3tables = { version = "0.8", optional = true }
+iceberg-catalog-sql = { version = "0.8", optional = true }
+iceberg-datafusion = { version = "0.8", optional = true }
 magnus = "0.8"
-parquet = "55"
+parquet = "57"
 sqlx = { version = "0.8", features = ["postgres", "runtime-tokio", "sqlite"], default-features = false, optional = true }
 tokio = { version = "1", features = ["rt-multi-thread"] }
 uuid = { version = "1", features = ["v4"] }
@@ -31,4 +33,5 @@ default = ["rest", "sql"]
 datafusion = ["dep:datafusion", "dep:iceberg-datafusion"]
 glue = ["dep:iceberg-catalog-glue"]
 rest = ["dep:iceberg-catalog-rest"]
+s3tables = ["dep:iceberg-catalog-s3tables"]
 sql = ["dep:iceberg-catalog-sql", "dep:sqlx"]

data/ext/iceberg/src/catalog.rs

@@ -1,6 +1,5 @@
 #[cfg(feature = "datafusion")]
 use datafusion::execution::context::SessionContext;
-use iceberg::io::FileIO;
 use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder};
 use iceberg::spec::Schema;
 use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, NamespaceIdent, TableCreation, TableIdent};
@@ -10,8 +9,15 @@ use iceberg_catalog_glue::{GLUE_CATALOG_PROP_WAREHOUSE, GlueCatalog, GlueCatalog
 use iceberg_catalog_rest::{
     REST_CATALOG_PROP_URI, REST_CATALOG_PROP_WAREHOUSE, RestCatalog, RestCatalogBuilder,
 };
+#[cfg(feature = "s3tables")]
+use iceberg_catalog_s3tables::{
+    S3TABLES_CATALOG_PROP_TABLE_BUCKET_ARN, S3TablesCatalog, S3TablesCatalogBuilder,
+};
 #[cfg(feature = "sql")]
-use iceberg_catalog_sql::{SqlBindStyle, SqlCatalog, SqlCatalogConfig};
+use iceberg_catalog_sql::{
+    SQL_CATALOG_PROP_BIND_STYLE, SQL_CATALOG_PROP_URI, SQL_CATALOG_PROP_WAREHOUSE, SqlBindStyle,
+    SqlCatalog, SqlCatalogBuilder,
+};
 #[cfg(feature = "datafusion")]
 use iceberg_datafusion::IcebergCatalogProvider;
 use std::cell::RefCell;
@@ -29,6 +35,8 @@ pub enum RbCatalogType {
     Memory(Arc<MemoryCatalog>),
     #[cfg(feature = "rest")]
     Rest(Arc<RestCatalog>),
+    #[cfg(feature = "s3tables")]
+    S3Tables(Arc<S3TablesCatalog>),
     #[cfg(feature = "sql")]
     Sql(Arc<SqlCatalog>),
 }
@@ -41,6 +49,8 @@ impl RbCatalogType {
             RbCatalogType::Memory(v) => v.as_ref(),
             #[cfg(feature = "rest")]
             RbCatalogType::Rest(v) => v.as_ref(),
+            #[cfg(feature = "s3tables")]
+            RbCatalogType::S3Tables(v) => v.as_ref(),
             #[cfg(feature = "sql")]
             RbCatalogType::Sql(v) => v.as_ref(),
         }
@@ -54,6 +64,8 @@ impl RbCatalogType {
             RbCatalogType::Memory(v) => v.clone(),
             #[cfg(feature = "rest")]
             RbCatalogType::Rest(v) => v.clone(),
+            #[cfg(feature = "s3tables")]
+            RbCatalogType::S3Tables(v) => v.clone(),
             #[cfg(feature = "sql")]
             RbCatalogType::Sql(v) => v.clone(),
         }
@@ -109,6 +121,18 @@ impl RbCatalog {
         })
     }
 
+    #[cfg(feature = "s3tables")]
+    pub fn new_s3tables(arn: String) -> RbResult<Self> {
+        let mut props = HashMap::new();
+        props.insert(S3TABLES_CATALOG_PROP_TABLE_BUCKET_ARN.to_string(), arn);
+        let catalog = runtime()
+            .block_on(S3TablesCatalogBuilder::default().load("s3tables", props))
+            .map_err(to_rb_err)?;
+        Ok(Self {
+            catalog: RbCatalogType::S3Tables(catalog.into()).into(),
+        })
+    }
+
     #[cfg(feature = "sql")]
     pub fn new_sql(
         uri: String,
@@ -116,20 +140,15 @@ impl RbCatalog {
         name: String,
         props: HashMap<String, String>,
     ) -> RbResult<Self> {
-        let file_io = FileIO::from_path(&warehouse)
-            .map_err(to_rb_err)?
-            .build()
-            .map_err(to_rb_err)?;
-        let config = SqlCatalogConfig::builder()
-            .uri(uri)
-            .warehouse_location(warehouse)
-            .name(name)
-            .file_io(file_io)
-            .sql_bind_style(SqlBindStyle::DollarNumeric)
-            .props(props)
-            .build();
+        let mut props = props;
+        props.insert(SQL_CATALOG_PROP_URI.to_string(), uri);
+        props.insert(SQL_CATALOG_PROP_WAREHOUSE.to_string(), warehouse);
+        props.insert(
+            SQL_CATALOG_PROP_BIND_STYLE.to_string(),
+            SqlBindStyle::DollarNumeric.to_string(),
+        );
         let catalog = runtime()
-            .block_on(SqlCatalog::new(config))
+            .block_on(SqlCatalogBuilder::default().load(name, props))
             .map_err(to_rb_err)?;
         Ok(Self {
             catalog: RbCatalogType::Sql(catalog.into()).into(),
@@ -297,7 +316,7 @@ impl RbCatalog {
     }
 
     #[cfg(feature = "datafusion")]
-    pub fn query(&self, sql: String) -> RbResult<()> {
+    pub fn sql(&self, sql: String) -> RbResult<()> {
         let runtime = runtime();
 
         // TODO only create context once
@@ -311,8 +330,7 @@ impl RbCatalog {
         let df = runtime.block_on(ctx.sql(&sql)).unwrap();
         let _results = runtime.block_on(df.collect()).unwrap();
 
-        // println!("{:?}", df.schema().fields());
-        // println!("{:?}", results);
+        // println!("{:?}", _results);
 
         Ok(())
     }

data/ext/iceberg/src/lib.rs

@@ -14,7 +14,7 @@ use crate::table::RbTable;
 
 type RbResult<T> = Result<T, RbErr>;
 
-#[magnus::init]
+#[magnus::init(name = "iceberg")]
 fn init(ruby: &Ruby) -> RbResult<()> {
     let module = ruby.define_module("Iceberg")?;
 
@@ -24,6 +24,8 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_singleton_method("new_memory", function!(RbCatalog::new_memory, 1))?;
     #[cfg(feature = "rest")]
     class.define_singleton_method("new_rest", function!(RbCatalog::new_rest, 3))?;
+    #[cfg(feature = "s3tables")]
+    class.define_singleton_method("new_s3tables", function!(RbCatalog::new_s3tables, 1))?;
     #[cfg(feature = "sql")]
     class.define_singleton_method("new_sql", function!(RbCatalog::new_sql, 4))?;
     class.define_method("list_namespaces", method!(RbCatalog::list_namespaces, 1))?;
@@ -43,7 +45,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("rename_table", method!(RbCatalog::rename_table, 2))?;
     class.define_method("register_table", method!(RbCatalog::register_table, 2))?;
     #[cfg(feature = "datafusion")]
-    class.define_method("query", method!(RbCatalog::query, 1))?;
+    class.define_method("sql", method!(RbCatalog::sql, 1))?;
 
     let class = module.define_class("RbTable", ruby.class_object())?;
     class.define_method("scan", method!(RbTable::scan, 1))?;

data/ext/iceberg/src/table.rs

@@ -9,6 +9,7 @@ use iceberg::writer::file_writer::ParquetWriterBuilder;
 use iceberg::writer::file_writer::location_generator::{
     DefaultFileNameGenerator, DefaultLocationGenerator,
 };
+use iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder;
 use iceberg::writer::{IcebergWriter, IcebergWriterBuilder};
 use magnus::{Error as RbErr, RArray, Ruby, Value};
 use parquet::file::properties::WriterProperties;
@@ -72,14 +73,16 @@ impl RbTable {
         let parquet_writer_builder = ParquetWriterBuilder::new(
             WriterProperties::default(),
             table.metadata().current_schema().clone(),
-            None,
+        );
+        let rolling_file_writer_builder = RollingFileWriterBuilder::new_with_default_file_size(
+            parquet_writer_builder,
             table.file_io().clone(),
             location_generator.clone(),
             file_name_generator.clone(),
         );
-        let data_file_writer_builder = DataFileWriterBuilder::new(parquet_writer_builder, None, 0);
+        let data_file_writer_builder = DataFileWriterBuilder::new(rolling_file_writer_builder);
         let mut data_file_writer = runtime
-            .block_on(data_file_writer_builder.build())
+            .block_on(data_file_writer_builder.build(None))
             .map_err(to_rb_err)?;
 
         for batch in data.0 {
@@ -112,6 +115,7 @@ impl RbTable {
         match self.table.borrow().metadata().format_version() {
             FormatVersion::V1 => 1,
             FormatVersion::V2 => 2,
+            FormatVersion::V3 => 3,
         }
     }
 
@@ -357,18 +361,18 @@ impl RbTable {
 
     pub fn encryption_keys(ruby: &Ruby, rb_self: &Self) -> RbResult<RArray> {
         let encryption_keys = ruby.ary_new();
-        for (k, v) in rb_self.table.borrow().metadata().encryption_keys_iter() {
-            encryption_keys.push((ruby.str_new(k), ruby.str_new(v)))?;
+        for k in rb_self.table.borrow().metadata().encryption_keys_iter() {
+            encryption_keys.push(rb_encrypted_key(k)?)?;
         }
         Ok(encryption_keys)
     }
 
-    pub fn encryption_key(&self, key_id: String) -> Option<String> {
-        self.table
-            .borrow()
-            .metadata()
-            .encryption_key(&key_id)
-            .cloned()
+    pub fn encryption_key(&self, key_id: String) -> RbResult<Option<Value>> {
+        let key = match self.table.borrow().metadata().encryption_key(&key_id) {
+            Some(k) => Some(rb_encrypted_key(k)?),
+            None => None,
+        };
+        Ok(key)
     }
 
     pub fn from_metadata_file(location: String) -> RbResult<Self> {

data/ext/iceberg/src/utils.rs

@@ -1,6 +1,6 @@
 use iceberg::spec::{
-    Literal, NestedField, PartitionSpec, PartitionStatisticsFile, PrimitiveLiteral, PrimitiveType,
-    Schema, Snapshot, SortOrder, StatisticsFile, Type,
+    EncryptedKey, Literal, NestedField, PartitionSpec, PartitionStatisticsFile, PrimitiveLiteral,
+    PrimitiveType, Schema, Snapshot, SortOrder, StatisticsFile, Type,
 };
 use iceberg::{NamespaceIdent, TableIdent};
 use magnus::{
@@ -238,6 +238,10 @@ pub fn rb_partition_statistics_file(
     todo!();
 }
 
+pub fn rb_encrypted_key(_encrypted_key: &EncryptedKey) -> RbResult<Value> {
+    todo!();
+}
+
 pub fn rb_literal(ruby: &Ruby, literal: &Literal) -> Value {
     match literal {
         Literal::Primitive(pl) => match pl {

data/lib/iceberg/catalog.rb

@@ -7,6 +7,7 @@ module Iceberg
     def create_namespace(namespace, properties: {}, if_not_exists: nil)
       @catalog.create_namespace(namespace, properties)
     rescue Error => e
+      # ideally all catalogs would use NamespaceAlreadyExistsError
       if !if_not_exists || (e.message != "Cannot create namespace" && !e.message.include?("already exists"))
         raise e
       end
@@ -28,7 +29,8 @@ module Iceberg
     def drop_namespace(namespace, if_exists: nil)
       @catalog.drop_namespace(namespace)
     rescue Error => e
-      if !if_exists || (e.message != "Tried to drop a namespace that does not exist" && !e.message.include?("No such namespace"))
+      # ideally all catalogs would use NamespaceNotFoundError
+      if !if_exists || (e.message != "Tried to drop a namespace that does not exist" && !e.message.include?("No such namespace") && !e.message.include?("The specified namespace does not exist") && !e.message.include?("not found"))
         raise e
       end
       nil
@@ -47,9 +49,9 @@ module Iceberg
         table_definition = TableDefinition.new
         yield table_definition
         schema = Schema.new(table_definition.fields)
-      elsif schema.is_a?(Hash)
+      elsif schema.is_a?(Hash) || (defined?(Polars::Schema) && schema.is_a?(Polars::Schema))
         fields =
-          schema.map.with_index do |(k, v), i|
+          schema.to_h.map.with_index do |(k, v), i|
             {
               id: i + 1,
               name: k.is_a?(Symbol) ? k.to_s : k,
@@ -72,7 +74,8 @@ module Iceberg
     def drop_table(table_name, if_exists: nil)
       @catalog.drop_table(table_name)
     rescue Error => e
-      if !if_exists || (e.message != "Tried to drop a table that does not exist" && !e.message.include?("No such table"))
+      # ideally all catalogs would use TableNotFoundError
+      if !if_exists || (e.message != "Tried to drop a table that does not exist" && !e.message.include?("No such table") && !e.message.include?("The specified table does not exist") && !e.message.include?("not found"))
         raise e
       end
       nil
@@ -92,11 +95,11 @@ module Iceberg
       @catalog.register_table(table_name, metadata_location)
     end
 
-    def query(sql)
+    def sql(sql)
       # requires datafusion feature
-      raise Todo unless @catalog.respond_to?(:query)
+      raise Todo unless @catalog.respond_to?(:sql)
 
-      @catalog.query(sql)
+      @catalog.sql(sql)
     end
 
     # hide internal state

data/lib/iceberg/glue_catalog.rb

@@ -2,9 +2,6 @@ module Iceberg
   class GlueCatalog < Catalog
     # warehouse is URI of S3 storage bucket
     def initialize(warehouse:)
-      # requires glue feature
-      raise Error, "Feature not enabled" unless RbCatalog.respond_to?(:new_glue)
-
       @catalog = RbCatalog.new_glue(warehouse)
     end
   end

data/lib/iceberg/s3_tables_catalog.rb

@@ -0,0 +1,7 @@
+module Iceberg
+  class S3TablesCatalog < Catalog
+    def initialize(arn:)
+      @catalog = RbCatalog.new_s3tables(arn)
+    end
+  end
+end

data/lib/iceberg/table.rb

@@ -87,61 +87,14 @@ module Iceberg
       TableScan.new(@table.scan(snapshot_id), self)
     end
 
-    def to_polars(snapshot_id: nil, storage_options: nil, _schema_changes: false)
+    def to_polars(snapshot_id: nil, storage_options: nil)
       require "polars-df"
 
-      # TODO always take this path in 0.2.0
-      if _schema_changes
-        return Polars.scan_iceberg(self, snapshot_id:, storage_options:)
+      if Gem::Version.new(Polars::VERSION) < Gem::Version.new("0.23")
+        raise "Requires polars-df >= 0.23"
       end
 
-      scan = scan(snapshot_id:)
-      files = scan.plan_files
-
-      if files.empty?
-        snapshot = scan.snapshot
-        scan_schema = snapshot ? schema_by_id(snapshot[:schema_id]) : current_schema
-
-        # TODO improve
-        schema =
-          scan_schema.fields.to_h do |field|
-            dtype =
-              case field[:type]
-              when "int"
-                Polars::Int32
-              when "long"
-                Polars::Int64
-              when "double"
-                Polars::Float64
-              when "string"
-                Polars::String
-              when "timestamp"
-                Polars::Datetime
-              else
-                raise Todo
-              end
-
-            [field[:name], dtype]
-          end
-
-        Polars::LazyFrame.new(schema: schema)
-      else
-        sources = files.map { |v| v[:data_file_path] }
-
-        deletion_files = [
-          "iceberg-position-delete",
-          files.map.with_index
-            .select { |v, i| v[:deletes].any? }
-            .to_h { |v, i| [i, v[:deletes].map { |d| d[:file_path] }] }
-        ]
-
-        scan_options = {
-          storage_options: storage_options,
-          _deletion_files: deletion_files,
-        }
-
-        Polars.scan_parquet(sources, **scan_options)
-      end
+      Polars.scan_iceberg(self, snapshot_id:, storage_options:)
     end
 
     def append(df)

data/lib/iceberg/version.rb

@@ -1,3 +1,3 @@
 module Iceberg
-  VERSION = "0.10.3"
+  VERSION = "0.11.0"
 end

data/lib/iceberg.rb

@@ -18,6 +18,7 @@ require_relative "iceberg/version"
 require_relative "iceberg/glue_catalog"
 require_relative "iceberg/memory_catalog"
 require_relative "iceberg/rest_catalog"
+require_relative "iceberg/s3_tables_catalog"
 require_relative "iceberg/sql_catalog"
 
 module Iceberg

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: iceberg
 version: !ruby/object:Gem::Version
-  version: 0.10.3
+  version: 0.11.0
 platform: ruby
 authors:
 - Andrew Kane
@@ -50,6 +50,7 @@ files:
 - lib/iceberg/glue_catalog.rb
 - lib/iceberg/memory_catalog.rb
 - lib/iceberg/rest_catalog.rb
+- lib/iceberg/s3_tables_catalog.rb
 - lib/iceberg/schema.rb
 - lib/iceberg/sql_catalog.rb
 - lib/iceberg/static_table.rb
@@ -68,7 +69,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.2'
+      version: '3.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="