iceberg 0.11.1 → 0.11.2

data/ext/iceberg/src/catalog.rs

@@ -21,9 +21,8 @@ use iceberg_catalog_sql::{
 };
 #[cfg(feature = "datafusion")]
 use iceberg_datafusion::IcebergCatalogProvider;
-use std::cell::RefCell;
 use std::collections::HashMap;
-use std::sync::Arc;
+use std::sync::{Arc, RwLock};
 
 use crate::error::to_rb_err;
 use crate::runtime::runtime;
@@ -75,7 +74,7 @@ impl RbCatalogType {
 
 #[magnus::wrap(class = "Iceberg::RbCatalog")]
 pub struct RbCatalog {
-    pub catalog: RefCell<RbCatalogType>,
+    pub catalog: RwLock<RbCatalogType>,
 }
 
 impl RbCatalog {
@@ -175,7 +174,8 @@ impl RbCatalog {
         let namespaces = runtime()
             .block_on(
                 self.catalog
-                    .borrow()
+                    .read()
+                    .unwrap()
                     .as_catalog()
                     .list_namespaces(parent.map(|v| v.0).as_ref()),
             )
@@ -191,7 +191,8 @@ impl RbCatalog {
         runtime()
             .block_on(
                 self.catalog
-                    .borrow()
+                    .read()
+                    .unwrap()
                     .as_catalog()
                     .create_namespace(&name.0, props),
             )
@@ -201,7 +202,13 @@ impl RbCatalog {
 
     pub fn namespace_exists(&self, name: Wrap<NamespaceIdent>) -> RbResult<bool> {
         let exists = runtime()
-            .block_on(self.catalog.borrow().as_catalog().namespace_exists(&name.0))
+            .block_on(
+                self.catalog
+                    .read()
+                    .unwrap()
+                    .as_catalog()
+                    .namespace_exists(&name.0),
+            )
             .map_err(to_rb_err)?;
         Ok(exists)
     }
@@ -211,7 +218,13 @@ impl RbCatalog {
         name: Wrap<NamespaceIdent>,
     ) -> RbResult<HashMap<String, String>> {
         let namespace = runtime()
-            .block_on(self.catalog.borrow().as_catalog().get_namespace(&name.0))
+            .block_on(
+                self.catalog
+                    .read()
+                    .unwrap()
+                    .as_catalog()
+                    .get_namespace(&name.0),
+            )
             .map_err(to_rb_err)?;
         Ok(namespace.properties().clone())
     }
@@ -224,7 +237,8 @@ impl RbCatalog {
         runtime()
             .block_on(
                 self.catalog
-                    .borrow()
+                    .read()
+                    .unwrap()
                     .as_catalog()
                     .update_namespace(&name.0, props),
             )
@@ -234,14 +248,26 @@ impl RbCatalog {
 
     pub fn drop_namespace(&self, name: Wrap<NamespaceIdent>) -> RbResult<()> {
         runtime()
-            .block_on(self.catalog.borrow().as_catalog().drop_namespace(&name.0))
+            .block_on(
+                self.catalog
+                    .read()
+                    .unwrap()
+                    .as_catalog()
+                    .drop_namespace(&name.0),
+            )
             .map_err(to_rb_err)?;
         Ok(())
     }
 
     pub fn list_tables(&self, namespace: Wrap<NamespaceIdent>) -> RbResult<Vec<Vec<String>>> {
         let tables = runtime()
-            .block_on(self.catalog.borrow().as_catalog().list_tables(&namespace.0))
+            .block_on(
+                self.catalog
+                    .read()
+                    .unwrap()
+                    .as_catalog()
+                    .list_tables(&namespace.0),
+            )
             .map_err(to_rb_err)?;
         Ok(tables
             .iter()
@@ -267,7 +293,8 @@ impl RbCatalog {
         let table = runtime()
             .block_on(
                 self.catalog
-                    .borrow()
+                    .read()
+                    .unwrap()
                     .as_catalog()
                     .create_table(&name.0.namespace, creation),
             )
@@ -279,7 +306,13 @@ impl RbCatalog {
 
     pub fn load_table(&self, name: Wrap<TableIdent>) -> RbResult<RbTable> {
         let table = runtime()
-            .block_on(self.catalog.borrow().as_catalog().load_table(&name.0))
+            .block_on(
+                self.catalog
+                    .read()
+                    .unwrap()
+                    .as_catalog()
+                    .load_table(&name.0),
+            )
             .map_err(to_rb_err)?;
         Ok(RbTable {
             table: table.into(),
@@ -288,14 +321,26 @@ impl RbCatalog {
 
     pub fn drop_table(&self, name: Wrap<TableIdent>) -> RbResult<()> {
         runtime()
-            .block_on(self.catalog.borrow().as_catalog().drop_table(&name.0))
+            .block_on(
+                self.catalog
+                    .read()
+                    .unwrap()
+                    .as_catalog()
+                    .drop_table(&name.0),
+            )
             .map_err(to_rb_err)?;
         Ok(())
     }
 
     pub fn table_exists(&self, name: Wrap<TableIdent>) -> RbResult<bool> {
         let exists = runtime()
-            .block_on(self.catalog.borrow().as_catalog().table_exists(&name.0))
+            .block_on(
+                self.catalog
+                    .read()
+                    .unwrap()
+                    .as_catalog()
+                    .table_exists(&name.0),
+            )
             .map_err(to_rb_err)?;
         Ok(exists)
     }
@@ -304,7 +349,8 @@ impl RbCatalog {
         runtime()
             .block_on(
                 self.catalog
-                    .borrow()
+                    .read()
+                    .unwrap()
                     .as_catalog()
                     .rename_table(&name.0, &new_name.0),
             )
@@ -320,7 +366,8 @@ impl RbCatalog {
         runtime()
             .block_on(
                 self.catalog
-                    .borrow()
+                    .read()
+                    .unwrap()
                     .as_catalog()
                     .register_table(&name.0, metadata_location),
             )
@@ -333,7 +380,7 @@ impl RbCatalog {
         let runtime = runtime();
 
         // TODO only create context once
-        let catalog = self.catalog.borrow().as_arc();
+        let catalog = self.catalog.read().unwrap().as_arc();
         let provider = runtime
             .block_on(IcebergCatalogProvider::try_new(catalog))
             .unwrap();

data/ext/iceberg/src/error.rs

@@ -11,7 +11,7 @@ pub fn to_rb_err(err: iceberg::Error) -> RbErr {
         _ => "Error",
     };
 
-    let class = Ruby::get()
+    let mut class = Ruby::get()
         .unwrap()
         .class_object()
         .const_get::<_, RModule>("Iceberg")
@@ -22,6 +22,12 @@ pub fn to_rb_err(err: iceberg::Error) -> RbErr {
     // no way to get context separately
     // https://github.com/apache/iceberg-rust/issues/1071
     let message = err.to_string();
+
+    // TODO remove in 0.12.0
+    if message.contains("target schema is not superset of current schema") {
+        class = Ruby::get().unwrap().exception_arg_error();
+    }
+
     let message = message
         // TODO improve
         .strip_prefix("Unexpected => ")

data/ext/iceberg/src/lib.rs

@@ -1,6 +1,7 @@
 mod arrow;
 mod catalog;
 mod error;
+mod ruby;
 mod runtime;
 mod scan;
 mod table;

data/ext/iceberg/src/ruby.rs

@@ -0,0 +1,51 @@
+use std::ffi::c_void;
+use std::ptr::null_mut;
+
+use magnus::Ruby;
+use rb_sys::rb_thread_call_without_gvl;
+
+pub trait GvlExt {
+    fn detach<T, F>(&self, func: F) -> T
+    where
+        F: Send + FnOnce() -> T,
+        T: Send;
+}
+
+impl GvlExt for Ruby {
+    fn detach<T, F>(&self, func: F) -> T
+    where
+        F: Send + FnOnce() -> T,
+        T: Send,
+    {
+        let mut data = CallbackData {
+            func: Some(func),
+            result: None,
+        };
+
+        unsafe {
+            rb_thread_call_without_gvl(
+                Some(call_without_gvl::<F, T>),
+                &mut data as *mut _ as *mut c_void,
+                None,
+                null_mut(),
+            );
+        }
+
+        data.result.unwrap()
+    }
+}
+
+struct CallbackData<F, T> {
+    func: Option<F>,
+    result: Option<T>,
+}
+
+extern "C" fn call_without_gvl<F, T>(data: *mut c_void) -> *mut c_void
+where
+    F: FnOnce() -> T,
+{
+    let data = unsafe { &mut *(data as *mut CallbackData<F, T>) };
+    let func = data.func.take().unwrap();
+    data.result = Some(func());
+    null_mut()
+}

data/ext/iceberg/src/scan.rs

@@ -1,27 +1,30 @@
 use futures::TryStreamExt;
 use iceberg::scan::TableScan;
 use magnus::{RArray, Ruby, Value};
-use std::cell::RefCell;
+use std::sync::RwLock;
 
 use crate::RbResult;
 use crate::error::to_rb_err;
+use crate::ruby::GvlExt;
 use crate::runtime::runtime;
 use crate::utils::rb_snapshot;
 
 #[magnus::wrap(class = "Iceberg::RbTableScan")]
 pub struct RbTableScan {
-    pub scan: RefCell<TableScan>,
+    pub scan: RwLock<TableScan>,
 }
 
 impl RbTableScan {
     pub fn plan_files(ruby: &Ruby, rb_self: &Self) -> RbResult<RArray> {
-        let scan = rb_self.scan.borrow();
-
-        let runtime = runtime();
-        let plan_files = runtime.block_on(scan.plan_files()).map_err(to_rb_err)?;
-        let plan_files: Vec<_> = runtime
-            .block_on(plan_files.try_collect())
+        let plan_files: Vec<_> = ruby
+            .detach(|| {
+                let scan = rb_self.scan.read().unwrap();
+                let runtime = runtime();
+                let plan_files = runtime.block_on(scan.plan_files())?;
+                runtime.block_on(plan_files.try_collect())
+            })
             .map_err(to_rb_err)?;
+
         let files = ruby.ary_new();
         for v in plan_files {
             let file = ruby.hash_new();
@@ -47,7 +50,7 @@ impl RbTableScan {
     }
 
     pub fn snapshot(ruby: &Ruby, rb_self: &Self) -> RbResult<Option<Value>> {
-        match rb_self.scan.borrow().snapshot() {
+        match rb_self.scan.read().unwrap().snapshot() {
             Some(s) => Ok(Some(rb_snapshot(ruby, s)?)),
             None => Ok(None),
         }