iceberg 0.11.0 → 0.11.2

data/ext/iceberg/Cargo.toml

@@ -1,10 +1,10 @@
 [package]
 name = "iceberg-ruby"
-version = "0.11.0"
+version = "0.11.2"
 license = "Apache-2.0"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2024"
-rust-version = "1.88"
+rust-version = "1.92"
 publish = false
 
 [lib]
@@ -14,16 +14,17 @@ crate-type = ["cdylib"]
 [dependencies]
 arrow-array = { version = "57", features = ["ffi"] }
 arrow-schema = "57"
-datafusion = { version = "51", optional = true }
+datafusion = { version = "52", optional = true }
 futures = "0.3"
-iceberg = "0.8"
-iceberg-catalog-glue = { version = "0.8", optional = true }
-iceberg-catalog-rest = { version = "0.8", optional = true }
-iceberg-catalog-s3tables = { version = "0.8", optional = true }
-iceberg-catalog-sql = { version = "0.8", optional = true }
-iceberg-datafusion = { version = "0.8", optional = true }
+iceberg = "=0.9.1"
+iceberg-catalog-glue = { version = "=0.9.1", optional = true }
+iceberg-catalog-rest = { version = "=0.9.1", optional = true }
+iceberg-catalog-s3tables = { version = "=0.9.1", optional = true }
+iceberg-catalog-sql = { version = "=0.9.1", optional = true }
+iceberg-datafusion = { version = "=0.9.1", optional = true }
 magnus = "0.8"
 parquet = "57"
+rb-sys = "0.9"
 sqlx = { version = "0.8", features = ["postgres", "runtime-tokio", "sqlite"], default-features = false, optional = true }
 tokio = { version = "1", features = ["rt-multi-thread"] }
 uuid = { version = "1", features = ["v4"] }

data/ext/iceberg/src/catalog.rs

@@ -1,5 +1,6 @@
 #[cfg(feature = "datafusion")]
 use datafusion::execution::context::SessionContext;
+use iceberg::io::LocalFsStorageFactory;
 use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder};
 use iceberg::spec::Schema;
 use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, NamespaceIdent, TableCreation, TableIdent};
@@ -20,9 +21,8 @@ use iceberg_catalog_sql::{
 };
 #[cfg(feature = "datafusion")]
 use iceberg_datafusion::IcebergCatalogProvider;
-use std::cell::RefCell;
 use std::collections::HashMap;
-use std::sync::Arc;
+use std::sync::{Arc, RwLock};
 
 use crate::error::to_rb_err;
 use crate::runtime::runtime;
@@ -74,7 +74,7 @@ impl RbCatalogType {
 
 #[magnus::wrap(class = "Iceberg::RbCatalog")]
 pub struct RbCatalog {
-    pub catalog: RefCell<RbCatalogType>,
+    pub catalog: RwLock<RbCatalogType>,
 }
 
 impl RbCatalog {
@@ -95,7 +95,11 @@ impl RbCatalog {
             props.insert(MEMORY_CATALOG_WAREHOUSE.to_string(), v);
         }
         let catalog = runtime()
-            .block_on(MemoryCatalogBuilder::default().load("memory", props))
+            .block_on(
+                MemoryCatalogBuilder::default()
+                    .with_storage_factory(Arc::new(LocalFsStorageFactory))
+                    .load("memory", props),
+            )
             .map_err(to_rb_err)?;
         Ok(Self {
             catalog: RbCatalogType::Memory(catalog.into()).into(),
@@ -114,7 +118,11 @@ impl RbCatalog {
             props.insert(REST_CATALOG_PROP_WAREHOUSE.to_string(), v);
         }
         let catalog = runtime()
-            .block_on(RestCatalogBuilder::default().load("rest", props))
+            .block_on(
+                RestCatalogBuilder::default()
+                    .with_storage_factory(Arc::new(LocalFsStorageFactory))
+                    .load("rest", props),
+            )
             .map_err(to_rb_err)?;
         Ok(Self {
             catalog: RbCatalogType::Rest(catalog.into()).into(),
@@ -148,7 +156,11 @@ impl RbCatalog {
             SqlBindStyle::DollarNumeric.to_string(),
         );
         let catalog = runtime()
-            .block_on(SqlCatalogBuilder::default().load(name, props))
+            .block_on(
+                SqlCatalogBuilder::default()
+                    .with_storage_factory(Arc::new(LocalFsStorageFactory))
+                    .load(name, props),
+            )
             .map_err(to_rb_err)?;
         Ok(Self {
             catalog: RbCatalogType::Sql(catalog.into()).into(),
@@ -162,7 +174,8 @@ impl RbCatalog {
         let namespaces = runtime()
             .block_on(
                 self.catalog
-                    .borrow()
+                    .read()
+                    .unwrap()
                     .as_catalog()
                     .list_namespaces(parent.map(|v| v.0).as_ref()),
             )
@@ -178,7 +191,8 @@ impl RbCatalog {
         runtime()
             .block_on(
                 self.catalog
-                    .borrow()
+                    .read()
+                    .unwrap()
                     .as_catalog()
                     .create_namespace(&name.0, props),
             )
@@ -188,7 +202,13 @@ impl RbCatalog {
 
     pub fn namespace_exists(&self, name: Wrap<NamespaceIdent>) -> RbResult<bool> {
         let exists = runtime()
-            .block_on(self.catalog.borrow().as_catalog().namespace_exists(&name.0))
+            .block_on(
+                self.catalog
+                    .read()
+                    .unwrap()
+                    .as_catalog()
+                    .namespace_exists(&name.0),
+            )
             .map_err(to_rb_err)?;
         Ok(exists)
     }
@@ -198,7 +218,13 @@ impl RbCatalog {
         name: Wrap<NamespaceIdent>,
     ) -> RbResult<HashMap<String, String>> {
         let namespace = runtime()
-            .block_on(self.catalog.borrow().as_catalog().get_namespace(&name.0))
+            .block_on(
+                self.catalog
+                    .read()
+                    .unwrap()
+                    .as_catalog()
+                    .get_namespace(&name.0),
+            )
             .map_err(to_rb_err)?;
         Ok(namespace.properties().clone())
     }
@@ -211,7 +237,8 @@ impl RbCatalog {
         runtime()
             .block_on(
                 self.catalog
-                    .borrow()
+                    .read()
+                    .unwrap()
                     .as_catalog()
                     .update_namespace(&name.0, props),
             )
@@ -221,14 +248,26 @@ impl RbCatalog {
 
     pub fn drop_namespace(&self, name: Wrap<NamespaceIdent>) -> RbResult<()> {
         runtime()
-            .block_on(self.catalog.borrow().as_catalog().drop_namespace(&name.0))
+            .block_on(
+                self.catalog
+                    .read()
+                    .unwrap()
+                    .as_catalog()
+                    .drop_namespace(&name.0),
+            )
             .map_err(to_rb_err)?;
         Ok(())
     }
 
     pub fn list_tables(&self, namespace: Wrap<NamespaceIdent>) -> RbResult<Vec<Vec<String>>> {
         let tables = runtime()
-            .block_on(self.catalog.borrow().as_catalog().list_tables(&namespace.0))
+            .block_on(
+                self.catalog
+                    .read()
+                    .unwrap()
+                    .as_catalog()
+                    .list_tables(&namespace.0),
+            )
             .map_err(to_rb_err)?;
         Ok(tables
             .iter()
@@ -254,7 +293,8 @@ impl RbCatalog {
         let table = runtime()
             .block_on(
                 self.catalog
-                    .borrow()
+                    .read()
+                    .unwrap()
                     .as_catalog()
                     .create_table(&name.0.namespace, creation),
             )
@@ -266,7 +306,13 @@ impl RbCatalog {
 
     pub fn load_table(&self, name: Wrap<TableIdent>) -> RbResult<RbTable> {
         let table = runtime()
-            .block_on(self.catalog.borrow().as_catalog().load_table(&name.0))
+            .block_on(
+                self.catalog
+                    .read()
+                    .unwrap()
+                    .as_catalog()
+                    .load_table(&name.0),
+            )
             .map_err(to_rb_err)?;
         Ok(RbTable {
             table: table.into(),
@@ -275,14 +321,26 @@ impl RbCatalog {
 
     pub fn drop_table(&self, name: Wrap<TableIdent>) -> RbResult<()> {
         runtime()
-            .block_on(self.catalog.borrow().as_catalog().drop_table(&name.0))
+            .block_on(
+                self.catalog
+                    .read()
+                    .unwrap()
+                    .as_catalog()
+                    .drop_table(&name.0),
+            )
             .map_err(to_rb_err)?;
         Ok(())
     }
 
     pub fn table_exists(&self, name: Wrap<TableIdent>) -> RbResult<bool> {
         let exists = runtime()
-            .block_on(self.catalog.borrow().as_catalog().table_exists(&name.0))
+            .block_on(
+                self.catalog
+                    .read()
+                    .unwrap()
+                    .as_catalog()
+                    .table_exists(&name.0),
+            )
             .map_err(to_rb_err)?;
         Ok(exists)
     }
@@ -291,7 +349,8 @@ impl RbCatalog {
         runtime()
             .block_on(
                 self.catalog
-                    .borrow()
+                    .read()
+                    .unwrap()
                     .as_catalog()
                     .rename_table(&name.0, &new_name.0),
             )
@@ -307,7 +366,8 @@ impl RbCatalog {
         runtime()
             .block_on(
                 self.catalog
-                    .borrow()
+                    .read()
+                    .unwrap()
                     .as_catalog()
                     .register_table(&name.0, metadata_location),
             )
@@ -320,7 +380,7 @@ impl RbCatalog {
         let runtime = runtime();
 
         // TODO only create context once
-        let catalog = self.catalog.borrow().as_arc();
+        let catalog = self.catalog.read().unwrap().as_arc();
         let provider = runtime
             .block_on(IcebergCatalogProvider::try_new(catalog))
             .unwrap();

data/ext/iceberg/src/error.rs

@@ -11,7 +11,7 @@ pub fn to_rb_err(err: iceberg::Error) -> RbErr {
         _ => "Error",
     };
 
-    let class = Ruby::get()
+    let mut class = Ruby::get()
         .unwrap()
         .class_object()
         .const_get::<_, RModule>("Iceberg")
@@ -22,6 +22,12 @@ pub fn to_rb_err(err: iceberg::Error) -> RbErr {
     // no way to get context separately
     // https://github.com/apache/iceberg-rust/issues/1071
     let message = err.to_string();
+
+    // TODO remove in 0.12.0
+    if message.contains("target schema is not superset of current schema") {
+        class = Ruby::get().unwrap().exception_arg_error();
+    }
+
     let message = message
         // TODO improve
         .strip_prefix("Unexpected => ")

data/ext/iceberg/src/lib.rs

@@ -1,6 +1,7 @@
 mod arrow;
 mod catalog;
 mod error;
+mod ruby;
 mod runtime;
 mod scan;
 mod table;

data/ext/iceberg/src/ruby.rs

@@ -0,0 +1,51 @@
+use std::ffi::c_void;
+use std::ptr::null_mut;
+
+use magnus::Ruby;
+use rb_sys::rb_thread_call_without_gvl;
+
+pub trait GvlExt {
+    fn detach<T, F>(&self, func: F) -> T
+    where
+        F: Send + FnOnce() -> T,
+        T: Send;
+}
+
+impl GvlExt for Ruby {
+    fn detach<T, F>(&self, func: F) -> T
+    where
+        F: Send + FnOnce() -> T,
+        T: Send,
+    {
+        let mut data = CallbackData {
+            func: Some(func),
+            result: None,
+        };
+
+        unsafe {
+            rb_thread_call_without_gvl(
+                Some(call_without_gvl::<F, T>),
+                &mut data as *mut _ as *mut c_void,
+                None,
+                null_mut(),
+            );
+        }
+
+        data.result.unwrap()
+    }
+}
+
+struct CallbackData<F, T> {
+    func: Option<F>,
+    result: Option<T>,
+}
+
+extern "C" fn call_without_gvl<F, T>(data: *mut c_void) -> *mut c_void
+where
+    F: FnOnce() -> T,
+{
+    let data = unsafe { &mut *(data as *mut CallbackData<F, T>) };
+    let func = data.func.take().unwrap();
+    data.result = Some(func());
+    null_mut()
+}

data/ext/iceberg/src/scan.rs

@@ -1,27 +1,30 @@
 use futures::TryStreamExt;
 use iceberg::scan::TableScan;
 use magnus::{RArray, Ruby, Value};
-use std::cell::RefCell;
+use std::sync::RwLock;
 
 use crate::RbResult;
 use crate::error::to_rb_err;
+use crate::ruby::GvlExt;
 use crate::runtime::runtime;
 use crate::utils::rb_snapshot;
 
 #[magnus::wrap(class = "Iceberg::RbTableScan")]
 pub struct RbTableScan {
-    pub scan: RefCell<TableScan>,
+    pub scan: RwLock<TableScan>,
 }
 
 impl RbTableScan {
     pub fn plan_files(ruby: &Ruby, rb_self: &Self) -> RbResult<RArray> {
-        let scan = rb_self.scan.borrow();
-
-        let runtime = runtime();
-        let plan_files = runtime.block_on(scan.plan_files()).map_err(to_rb_err)?;
-        let plan_files: Vec<_> = runtime
-            .block_on(plan_files.try_collect())
+        let plan_files: Vec<_> = ruby
+            .detach(|| {
+                let scan = rb_self.scan.read().unwrap();
+                let runtime = runtime();
+                let plan_files = runtime.block_on(scan.plan_files())?;
+                runtime.block_on(plan_files.try_collect())
+            })
             .map_err(to_rb_err)?;
+
         let files = ruby.ary_new();
         for v in plan_files {
             let file = ruby.hash_new();
@@ -47,7 +50,7 @@ impl RbTableScan {
     }
 
     pub fn snapshot(ruby: &Ruby, rb_self: &Self) -> RbResult<Option<Value>> {
-        match rb_self.scan.borrow().snapshot() {
+        match rb_self.scan.read().unwrap().snapshot() {
             Some(s) => Ok(Some(rb_snapshot(ruby, s)?)),
             None => Ok(None),
         }