iceberg 0.10.2 → 0.11.0

data/README.md

@@ -2,6 +2,8 @@
 
 [Apache Iceberg](https://iceberg.apache.org/) for Ruby
 
+:duck: Also check out [SeaDuck](https://github.com/ankane/seaduck)
+
 [![Build Status](https://github.com/ankane/iceberg-ruby/actions/workflows/build.yml/badge.svg)](https://github.com/ankane/iceberg-ruby/actions)
 
 ## Installation
@@ -35,7 +37,7 @@ catalog.create_table("main.events") do |t|
 end
 ```
 
-Or
+Or with [Polars](https://github.com/ankane/ruby-polars)
 
 ```ruby
 df = Polars::DataFrame.new({"id" => [1, 2], "value" => [3.0, 4.0]})
@@ -82,9 +84,7 @@ Iceberg::MemoryCatalog.new(
 )
 ```
 
-## Reference
-
-### Namespaces
+## Namespaces
 
 List namespaces
 
@@ -122,7 +122,7 @@ Drop a namespace
 catalog.drop_namespace("main")
 ```
 
-### Tables
+## Tables
 
 List tables
 
@@ -169,7 +169,7 @@ Drop a table
 catalog.drop_table("main.events")
 ```
 
-### Static Tables
+## Static Tables
 
 Load a static table
 

data/ext/iceberg/Cargo.toml

@@ -1,27 +1,29 @@
 [package]
-name = "iceberg"
-version = "0.10.2"
+name = "iceberg-ruby"
+version = "0.11.0"
 license = "Apache-2.0"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2024"
-rust-version = "1.86"
+rust-version = "1.88"
 publish = false
 
 [lib]
+name = "iceberg"
 crate-type = ["cdylib"]
 
 [dependencies]
-arrow-array = { version = "55", features = ["ffi"] }
-arrow-schema = "55"
-datafusion = { version = "47", optional = true }
+arrow-array = { version = "57", features = ["ffi"] }
+arrow-schema = "57"
+datafusion = { version = "51", optional = true }
 futures = "0.3"
-iceberg = "0.6"
-iceberg-catalog-glue = { version = "0.6", optional = true }
-iceberg-catalog-rest = { version = "0.6", optional = true }
-iceberg-catalog-sql = { version = "0.6", optional = true }
-iceberg-datafusion = { version = "0.6", optional = true }
+iceberg = "0.8"
+iceberg-catalog-glue = { version = "0.8", optional = true }
+iceberg-catalog-rest = { version = "0.8", optional = true }
+iceberg-catalog-s3tables = { version = "0.8", optional = true }
+iceberg-catalog-sql = { version = "0.8", optional = true }
+iceberg-datafusion = { version = "0.8", optional = true }
 magnus = "0.8"
-parquet = "55"
+parquet = "57"
 sqlx = { version = "0.8", features = ["postgres", "runtime-tokio", "sqlite"], default-features = false, optional = true }
 tokio = { version = "1", features = ["rt-multi-thread"] }
 uuid = { version = "1", features = ["v4"] }
@@ -31,4 +33,5 @@ default = ["rest", "sql"]
 datafusion = ["dep:datafusion", "dep:iceberg-datafusion"]
 glue = ["dep:iceberg-catalog-glue"]
 rest = ["dep:iceberg-catalog-rest"]
+s3tables = ["dep:iceberg-catalog-s3tables"]
 sql = ["dep:iceberg-catalog-sql", "dep:sqlx"]

data/ext/iceberg/src/catalog.rs

@@ -1,14 +1,23 @@
 #[cfg(feature = "datafusion")]
 use datafusion::execution::context::SessionContext;
-use iceberg::io::{FileIO, FileIOBuilder};
+use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder};
 use iceberg::spec::Schema;
-use iceberg::{Catalog, MemoryCatalog, NamespaceIdent, TableCreation, TableIdent};
+use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, NamespaceIdent, TableCreation, TableIdent};
 #[cfg(feature = "glue")]
-use iceberg_catalog_glue::{GlueCatalog, GlueCatalogConfig};
+use iceberg_catalog_glue::{GLUE_CATALOG_PROP_WAREHOUSE, GlueCatalog, GlueCatalogBuilder};
 #[cfg(feature = "rest")]
-use iceberg_catalog_rest::{RestCatalog, RestCatalogConfig};
+use iceberg_catalog_rest::{
+    REST_CATALOG_PROP_URI, REST_CATALOG_PROP_WAREHOUSE, RestCatalog, RestCatalogBuilder,
+};
+#[cfg(feature = "s3tables")]
+use iceberg_catalog_s3tables::{
+    S3TABLES_CATALOG_PROP_TABLE_BUCKET_ARN, S3TablesCatalog, S3TablesCatalogBuilder,
+};
 #[cfg(feature = "sql")]
-use iceberg_catalog_sql::{SqlBindStyle, SqlCatalog, SqlCatalogConfig};
+use iceberg_catalog_sql::{
+    SQL_CATALOG_PROP_BIND_STYLE, SQL_CATALOG_PROP_URI, SQL_CATALOG_PROP_WAREHOUSE, SqlBindStyle,
+    SqlCatalog, SqlCatalogBuilder,
+};
 #[cfg(feature = "datafusion")]
 use iceberg_datafusion::IcebergCatalogProvider;
 use std::cell::RefCell;
@@ -26,6 +35,8 @@ pub enum RbCatalogType {
     Memory(Arc<MemoryCatalog>),
     #[cfg(feature = "rest")]
     Rest(Arc<RestCatalog>),
+    #[cfg(feature = "s3tables")]
+    S3Tables(Arc<S3TablesCatalog>),
     #[cfg(feature = "sql")]
     Sql(Arc<SqlCatalog>),
 }
@@ -38,6 +49,8 @@ impl RbCatalogType {
             RbCatalogType::Memory(v) => v.as_ref(),
             #[cfg(feature = "rest")]
             RbCatalogType::Rest(v) => v.as_ref(),
+            #[cfg(feature = "s3tables")]
+            RbCatalogType::S3Tables(v) => v.as_ref(),
             #[cfg(feature = "sql")]
             RbCatalogType::Sql(v) => v.as_ref(),
         }
@@ -51,6 +64,8 @@ impl RbCatalogType {
             RbCatalogType::Memory(v) => v.clone(),
             #[cfg(feature = "rest")]
             RbCatalogType::Rest(v) => v.clone(),
+            #[cfg(feature = "s3tables")]
+            RbCatalogType::S3Tables(v) => v.clone(),
             #[cfg(feature = "sql")]
             RbCatalogType::Sql(v) => v.clone(),
         }
@@ -65,9 +80,9 @@ pub struct RbCatalog {
 impl RbCatalog {
     #[cfg(feature = "glue")]
     pub fn new_glue(warehouse: String) -> RbResult<Self> {
-        let config = GlueCatalogConfig::builder().warehouse(warehouse).build();
+        let props = HashMap::from([(GLUE_CATALOG_PROP_WAREHOUSE.to_string(), warehouse)]);
         let catalog = runtime()
-            .block_on(GlueCatalog::new(config))
+            .block_on(GlueCatalogBuilder::default().load("glue", props))
             .map_err(to_rb_err)?;
         Ok(Self {
             catalog: RbCatalogType::Glue(catalog.into()).into(),
@@ -75,14 +90,13 @@ impl RbCatalog {
     }
 
     pub fn new_memory(warehouse: Option<String>) -> RbResult<Self> {
-        let file_io = match warehouse {
-            Some(ref v) => FileIO::from_path(v)
-                .map_err(to_rb_err)?
-                .build()
-                .map_err(to_rb_err)?,
-            None => FileIOBuilder::new_fs_io().build().map_err(to_rb_err)?,
-        };
-        let catalog = MemoryCatalog::new(file_io, warehouse);
+        let mut props = HashMap::new();
+        if let Some(v) = warehouse {
+            props.insert(MEMORY_CATALOG_WAREHOUSE.to_string(), v);
+        }
+        let catalog = runtime()
+            .block_on(MemoryCatalogBuilder::default().load("memory", props))
+            .map_err(to_rb_err)?;
         Ok(Self {
             catalog: RbCatalogType::Memory(catalog.into()).into(),
         })
@@ -93,16 +107,30 @@ impl RbCatalog {
         uri: String,
         warehouse: Option<String>,
         props: HashMap<String, String>,
-    ) -> Self {
-        let config = RestCatalogConfig::builder()
-            .uri(uri)
-            .warehouse_opt(warehouse)
-            .props(props)
-            .build();
-        let catalog = RestCatalog::new(config);
-        Self {
-            catalog: RbCatalogType::Rest(catalog.into()).into(),
+    ) -> RbResult<Self> {
+        let mut props = props;
+        props.insert(REST_CATALOG_PROP_URI.to_string(), uri);
+        if let Some(v) = warehouse {
+            props.insert(REST_CATALOG_PROP_WAREHOUSE.to_string(), v);
         }
+        let catalog = runtime()
+            .block_on(RestCatalogBuilder::default().load("rest", props))
+            .map_err(to_rb_err)?;
+        Ok(Self {
+            catalog: RbCatalogType::Rest(catalog.into()).into(),
+        })
+    }
+
+    #[cfg(feature = "s3tables")]
+    pub fn new_s3tables(arn: String) -> RbResult<Self> {
+        let mut props = HashMap::new();
+        props.insert(S3TABLES_CATALOG_PROP_TABLE_BUCKET_ARN.to_string(), arn);
+        let catalog = runtime()
+            .block_on(S3TablesCatalogBuilder::default().load("s3tables", props))
+            .map_err(to_rb_err)?;
+        Ok(Self {
+            catalog: RbCatalogType::S3Tables(catalog.into()).into(),
+        })
     }
 
     #[cfg(feature = "sql")]
@@ -112,20 +140,15 @@ impl RbCatalog {
         name: String,
         props: HashMap<String, String>,
     ) -> RbResult<Self> {
-        let file_io = FileIO::from_path(&warehouse)
-            .map_err(to_rb_err)?
-            .build()
-            .map_err(to_rb_err)?;
-        let config = SqlCatalogConfig::builder()
-            .uri(uri)
-            .warehouse_location(warehouse)
-            .name(name)
-            .file_io(file_io)
-            .sql_bind_style(SqlBindStyle::DollarNumeric)
-            .props(props)
-            .build();
+        let mut props = props;
+        props.insert(SQL_CATALOG_PROP_URI.to_string(), uri);
+        props.insert(SQL_CATALOG_PROP_WAREHOUSE.to_string(), warehouse);
+        props.insert(
+            SQL_CATALOG_PROP_BIND_STYLE.to_string(),
+            SqlBindStyle::DollarNumeric.to_string(),
+        );
         let catalog = runtime()
-            .block_on(SqlCatalog::new(config))
+            .block_on(SqlCatalogBuilder::default().load(name, props))
             .map_err(to_rb_err)?;
         Ok(Self {
             catalog: RbCatalogType::Sql(catalog.into()).into(),
@@ -293,7 +316,7 @@ impl RbCatalog {
     }
 
     #[cfg(feature = "datafusion")]
-    pub fn query(&self, sql: String) -> RbResult<()> {
+    pub fn sql(&self, sql: String) -> RbResult<()> {
         let runtime = runtime();
 
         // TODO only create context once
@@ -307,8 +330,7 @@ impl RbCatalog {
         let df = runtime.block_on(ctx.sql(&sql)).unwrap();
         let _results = runtime.block_on(df.collect()).unwrap();
 
-        // println!("{:?}", df.schema().fields());
-        // println!("{:?}", results);
+        // println!("{:?}", _results);
 
         Ok(())
     }

data/ext/iceberg/src/lib.rs

@@ -14,7 +14,7 @@ use crate::table::RbTable;
 
 type RbResult<T> = Result<T, RbErr>;
 
-#[magnus::init]
+#[magnus::init(name = "iceberg")]
 fn init(ruby: &Ruby) -> RbResult<()> {
     let module = ruby.define_module("Iceberg")?;
 
@@ -24,6 +24,8 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_singleton_method("new_memory", function!(RbCatalog::new_memory, 1))?;
     #[cfg(feature = "rest")]
     class.define_singleton_method("new_rest", function!(RbCatalog::new_rest, 3))?;
+    #[cfg(feature = "s3tables")]
+    class.define_singleton_method("new_s3tables", function!(RbCatalog::new_s3tables, 1))?;
     #[cfg(feature = "sql")]
     class.define_singleton_method("new_sql", function!(RbCatalog::new_sql, 4))?;
     class.define_method("list_namespaces", method!(RbCatalog::list_namespaces, 1))?;
@@ -43,7 +45,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("rename_table", method!(RbCatalog::rename_table, 2))?;
     class.define_method("register_table", method!(RbCatalog::register_table, 2))?;
     #[cfg(feature = "datafusion")]
-    class.define_method("query", method!(RbCatalog::query, 1))?;
+    class.define_method("sql", method!(RbCatalog::sql, 1))?;
 
     let class = module.define_class("RbTable", ruby.class_object())?;
     class.define_method("scan", method!(RbTable::scan, 1))?;

data/ext/iceberg/src/table.rs

@@ -9,6 +9,7 @@ use iceberg::writer::file_writer::ParquetWriterBuilder;
 use iceberg::writer::file_writer::location_generator::{
     DefaultFileNameGenerator, DefaultLocationGenerator,
 };
+use iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder;
 use iceberg::writer::{IcebergWriter, IcebergWriterBuilder};
 use magnus::{Error as RbErr, RArray, Ruby, Value};
 use parquet::file::properties::WriterProperties;
@@ -72,13 +73,16 @@ impl RbTable {
         let parquet_writer_builder = ParquetWriterBuilder::new(
             WriterProperties::default(),
             table.metadata().current_schema().clone(),
+        );
+        let rolling_file_writer_builder = RollingFileWriterBuilder::new_with_default_file_size(
+            parquet_writer_builder,
             table.file_io().clone(),
             location_generator.clone(),
             file_name_generator.clone(),
         );
-        let data_file_writer_builder = DataFileWriterBuilder::new(parquet_writer_builder, None, 0);
+        let data_file_writer_builder = DataFileWriterBuilder::new(rolling_file_writer_builder);
         let mut data_file_writer = runtime
-            .block_on(data_file_writer_builder.build())
+            .block_on(data_file_writer_builder.build(None))
             .map_err(to_rb_err)?;
 
         for batch in data.0 {
@@ -111,6 +115,7 @@ impl RbTable {
         match self.table.borrow().metadata().format_version() {
             FormatVersion::V1 => 1,
             FormatVersion::V2 => 2,
+            FormatVersion::V3 => 3,
         }
     }
 
@@ -356,18 +361,18 @@ impl RbTable {
 
     pub fn encryption_keys(ruby: &Ruby, rb_self: &Self) -> RbResult<RArray> {
         let encryption_keys = ruby.ary_new();
-        for (k, v) in rb_self.table.borrow().metadata().encryption_keys_iter() {
-            encryption_keys.push((ruby.str_new(k), ruby.str_new(v)))?;
+        for k in rb_self.table.borrow().metadata().encryption_keys_iter() {
+            encryption_keys.push(rb_encrypted_key(k)?)?;
         }
         Ok(encryption_keys)
     }
 
-    pub fn encryption_key(&self, key_id: String) -> Option<String> {
-        self.table
-            .borrow()
-            .metadata()
-            .encryption_key(&key_id)
-            .cloned()
+    pub fn encryption_key(&self, key_id: String) -> RbResult<Option<Value>> {
+        let key = match self.table.borrow().metadata().encryption_key(&key_id) {
+            Some(k) => Some(rb_encrypted_key(k)?),
+            None => None,
+        };
+        Ok(key)
     }
 
     pub fn from_metadata_file(location: String) -> RbResult<Self> {

data/ext/iceberg/src/utils.rs

@@ -1,6 +1,6 @@
 use iceberg::spec::{
-    Literal, NestedField, PartitionSpec, PartitionStatisticsFile, PrimitiveLiteral, PrimitiveType,
-    Schema, Snapshot, SortOrder, StatisticsFile, Type,
+    EncryptedKey, Literal, NestedField, PartitionSpec, PartitionStatisticsFile, PrimitiveLiteral,
+    PrimitiveType, Schema, Snapshot, SortOrder, StatisticsFile, Type,
 };
 use iceberg::{NamespaceIdent, TableIdent};
 use magnus::{
@@ -238,6 +238,10 @@ pub fn rb_partition_statistics_file(
     todo!();
 }
 
+pub fn rb_encrypted_key(_encrypted_key: &EncryptedKey) -> RbResult<Value> {
+    todo!();
+}
+
 pub fn rb_literal(ruby: &Ruby, literal: &Literal) -> Value {
     match literal {
         Literal::Primitive(pl) => match pl {

data/lib/iceberg/catalog.rb

@@ -7,6 +7,7 @@ module Iceberg
     def create_namespace(namespace, properties: {}, if_not_exists: nil)
       @catalog.create_namespace(namespace, properties)
     rescue Error => e
+      # ideally all catalogs would use NamespaceAlreadyExistsError
       if !if_not_exists || (e.message != "Cannot create namespace" && !e.message.include?("already exists"))
         raise e
       end
@@ -28,7 +29,8 @@ module Iceberg
     def drop_namespace(namespace, if_exists: nil)
       @catalog.drop_namespace(namespace)
     rescue Error => e
-      if !if_exists || (e.message != "Tried to drop a namespace that does not exist" && !e.message.include?("No such namespace"))
+      # ideally all catalogs would use NamespaceNotFoundError
+      if !if_exists || (e.message != "Tried to drop a namespace that does not exist" && !e.message.include?("No such namespace") && !e.message.include?("The specified namespace does not exist") && !e.message.include?("not found"))
         raise e
       end
       nil
@@ -47,9 +49,9 @@ module Iceberg
         table_definition = TableDefinition.new
         yield table_definition
         schema = Schema.new(table_definition.fields)
-      elsif schema.is_a?(Hash)
+      elsif schema.is_a?(Hash) || (defined?(Polars::Schema) && schema.is_a?(Polars::Schema))
         fields =
-          schema.map.with_index do |(k, v), i|
+          schema.to_h.map.with_index do |(k, v), i|
             {
               id: i + 1,
               name: k.is_a?(Symbol) ? k.to_s : k,
@@ -72,7 +74,8 @@ module Iceberg
     def drop_table(table_name, if_exists: nil)
       @catalog.drop_table(table_name)
     rescue Error => e
-      if !if_exists || (e.message != "Tried to drop a table that does not exist" && !e.message.include?("No such table"))
+      # ideally all catalogs would use TableNotFoundError
+      if !if_exists || (e.message != "Tried to drop a table that does not exist" && !e.message.include?("No such table") && !e.message.include?("The specified table does not exist") && !e.message.include?("not found"))
         raise e
       end
       nil
@@ -92,11 +95,11 @@ module Iceberg
       @catalog.register_table(table_name, metadata_location)
     end
 
-    def query(sql)
+    def sql(sql)
       # requires datafusion feature
-      raise Todo unless @catalog.respond_to?(:query)
+      raise Todo unless @catalog.respond_to?(:sql)
 
-      @catalog.query(sql)
+      @catalog.sql(sql)
     end
 
     # hide internal state

data/lib/iceberg/glue_catalog.rb

@@ -2,9 +2,6 @@ module Iceberg
   class GlueCatalog < Catalog
     # warehouse is URI of S3 storage bucket
     def initialize(warehouse:)
-      # requires glue feature
-      raise Error, "Feature not enabled" unless RbCatalog.respond_to?(:new_glue)
-
       @catalog = RbCatalog.new_glue(warehouse)
     end
   end

data/lib/iceberg/s3_tables_catalog.rb

@@ -0,0 +1,7 @@
+module Iceberg
+  class S3TablesCatalog < Catalog
+    def initialize(arn:)
+      @catalog = RbCatalog.new_s3tables(arn)
+    end
+  end
+end

data/lib/iceberg/table.rb

@@ -83,56 +83,18 @@ module Iceberg
       @table.properties
     end
 
+    def scan(snapshot_id: nil)
+      TableScan.new(@table.scan(snapshot_id), self)
+    end
+
     def to_polars(snapshot_id: nil, storage_options: nil)
       require "polars-df"
 
-      files = @table.scan(snapshot_id).plan_files
-      if files.empty?
-        # TODO improve
-        schema =
-          # TODO use schema from snapshot_id
-          current_schema.fields.to_h do |field|
-            dtype =
-              case field[:type]
-              when "int"
-                Polars::Int32
-              when "long"
-                Polars::Int64
-              when "double"
-                Polars::Float64
-              when "string"
-                Polars::String
-              when "timestamp"
-                Polars::Datetime
-              else
-                raise Todo
-              end
-
-            [field[:name], dtype]
-          end
-
-        Polars::LazyFrame.new(schema: schema)
-      else
-        sources = files.map { |v| v[:data_file_path] }
-
-        deletion_files = [
-          "iceberg-position-delete",
-          files.map.with_index
-            .select { |v, i| v[:deletes].any? }
-            .to_h { |v, i| [i, v[:deletes].map { |d| d[:file_path] }] }
-        ]
-
-        Polars.scan_parquet(
-          sources,
-          storage_options: storage_options,
-          # TODO
-          # cast_options: Polars::ScanCastOptions._default_iceberg,
-          # allow_missing_columns: true,
-          # extra_columns: "ignore",
-          # _column_mapping: column_mapping,
-          _deletion_files: deletion_files
-        )
+      if Gem::Version.new(Polars::VERSION) < Gem::Version.new("0.23")
+        raise "Requires polars-df >= 0.23"
       end
+
+      Polars.scan_iceberg(self, snapshot_id:, storage_options:)
     end
 
     def append(df)

data/lib/iceberg/table_scan.rb

@@ -0,0 +1,18 @@
+module Iceberg
+  class TableScan
+    attr_reader :table
+
+    def initialize(scan, table)
+      @scan = scan
+      @table = table
+    end
+
+    def plan_files
+      @scan.plan_files
+    end
+
+    def snapshot
+      @scan.snapshot
+    end
+  end
+end

data/lib/iceberg/version.rb

@@ -1,3 +1,3 @@
 module Iceberg
-  VERSION = "0.10.2"
+  VERSION = "0.11.0"
 end

data/lib/iceberg.rb

@@ -9,6 +9,7 @@ end
 require_relative "iceberg/catalog"
 require_relative "iceberg/schema"
 require_relative "iceberg/table"
+require_relative "iceberg/table_scan"
 require_relative "iceberg/static_table"
 require_relative "iceberg/table_definition"
 require_relative "iceberg/version"
@@ -17,6 +18,7 @@ require_relative "iceberg/version"
 require_relative "iceberg/glue_catalog"
 require_relative "iceberg/memory_catalog"
 require_relative "iceberg/rest_catalog"
+require_relative "iceberg/s3_tables_catalog"
 require_relative "iceberg/sql_catalog"
 
 module Iceberg

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: iceberg
 version: !ruby/object:Gem::Version
-  version: 0.10.2
+  version: 0.11.0
 platform: ruby
 authors:
 - Andrew Kane
@@ -50,11 +50,13 @@ files:
 - lib/iceberg/glue_catalog.rb
 - lib/iceberg/memory_catalog.rb
 - lib/iceberg/rest_catalog.rb
+- lib/iceberg/s3_tables_catalog.rb
 - lib/iceberg/schema.rb
 - lib/iceberg/sql_catalog.rb
 - lib/iceberg/static_table.rb
 - lib/iceberg/table.rb
 - lib/iceberg/table_definition.rb
+- lib/iceberg/table_scan.rb
 - lib/iceberg/version.rb
 homepage: https://github.com/ankane/iceberg-ruby
 licenses:
@@ -67,7 +69,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.2'
+      version: '3.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="