polars-df 0.20.0 → 0.21.0

data/ext/polars/src/series/general.rs

@@ -1,10 +1,8 @@
-use magnus::{exception, Error, IntoValue, Value};
+use magnus::{Error, IntoValue, Value, exception};
 use polars::prelude::*;
 use polars::series::IsSorted;
 
-use crate::apply_method_all_arrow_series2;
 use crate::conversion::*;
-use crate::map::series::{call_lambda_and_extract, ApplyLambda};
 use crate::{RbDataFrame, RbPolarsErr, RbResult, RbSeries};
 
 impl RbSeries {
@@ -42,21 +40,15 @@ impl RbSeries {
     }
 
     pub fn cat_uses_lexical_ordering(&self) -> RbResult<bool> {
-        let binding = self.series.borrow();
-        let ca = binding.categorical().map_err(RbPolarsErr::from)?;
-        Ok(ca.uses_lexical_ordering())
+        Ok(true)
     }
 
     pub fn cat_is_local(&self) -> RbResult<bool> {
-        let binding = self.series.borrow();
-        let ca = binding.categorical().map_err(RbPolarsErr::from)?;
-        Ok(ca.get_rev_map().is_local())
+        Ok(false)
     }
 
     pub fn cat_to_local(&self) -> RbResult<Self> {
-        let binding = self.series.borrow();
-        let ca = binding.categorical().map_err(RbPolarsErr::from)?;
-        Ok(ca.to_local().into_series().into())
+        Ok(self.clone())
     }
 
     pub fn estimated_size(&self) -> usize {
@@ -75,7 +67,7 @@ impl RbSeries {
             if val == v_trunc {
                 val
             } else {
-                format!("{}…", v_trunc)
+                format!("{v_trunc}…")
             }
         } else {
             val
@@ -305,198 +297,6 @@ impl RbSeries {
         RbSeries::new(self.series.borrow().clone())
     }
 
-    pub fn apply_lambda(
-        &self,
-        lambda: Value,
-        output_type: Option<Wrap<DataType>>,
-        skip_nulls: bool,
-    ) -> RbResult<Self> {
-        let series = &self.series.borrow();
-
-        let output_type = output_type.map(|dt| dt.0);
-
-        macro_rules! dispatch_apply {
-            ($self:expr, $method:ident, $($args:expr),*) => {
-                if matches!($self.dtype(), DataType::Object(_)) {
-                    // let ca = $self.0.unpack::<ObjectType<ObjectValue>>().unwrap();
-                    // ca.$method($($args),*)
-                    todo!()
-                } else {
-                    apply_method_all_arrow_series2!(
-                        $self,
-                        $method,
-                        $($args),*
-                    )
-                }
-
-            }
-
-        }
-
-        if matches!(
-            series.dtype(),
-            DataType::Datetime(_, _)
-                | DataType::Date
-                | DataType::Duration(_)
-                | DataType::Categorical(_, _)
-                | DataType::Time
-        ) || !skip_nulls
-        {
-            let mut avs = Vec::with_capacity(series.len());
-            let iter = series.iter().map(|av| {
-                let input = Wrap(av);
-                call_lambda_and_extract::<_, Wrap<AnyValue>>(lambda, input)
-                    .unwrap()
-                    .0
-            });
-            avs.extend(iter);
-            return Ok(Series::new(self.name().into(), &avs).into());
-        }
-
-        let out = match output_type {
-            Some(DataType::Int8) => {
-                let ca: Int8Chunked = dispatch_apply!(
-                    series,
-                    apply_lambda_with_primitive_out_type,
-                    lambda,
-                    0,
-                    None
-                )?;
-                ca.into_series()
-            }
-            Some(DataType::Int16) => {
-                let ca: Int16Chunked = dispatch_apply!(
-                    series,
-                    apply_lambda_with_primitive_out_type,
-                    lambda,
-                    0,
-                    None
-                )?;
-                ca.into_series()
-            }
-            Some(DataType::Int32) => {
-                let ca: Int32Chunked = dispatch_apply!(
-                    series,
-                    apply_lambda_with_primitive_out_type,
-                    lambda,
-                    0,
-                    None
-                )?;
-                ca.into_series()
-            }
-            Some(DataType::Int64) => {
-                let ca: Int64Chunked = dispatch_apply!(
-                    series,
-                    apply_lambda_with_primitive_out_type,
-                    lambda,
-                    0,
-                    None
-                )?;
-                ca.into_series()
-            }
-            Some(DataType::UInt8) => {
-                let ca: UInt8Chunked = dispatch_apply!(
-                    series,
-                    apply_lambda_with_primitive_out_type,
-                    lambda,
-                    0,
-                    None
-                )?;
-                ca.into_series()
-            }
-            Some(DataType::UInt16) => {
-                let ca: UInt16Chunked = dispatch_apply!(
-                    series,
-                    apply_lambda_with_primitive_out_type,
-                    lambda,
-                    0,
-                    None
-                )?;
-                ca.into_series()
-            }
-            Some(DataType::UInt32) => {
-                let ca: UInt32Chunked = dispatch_apply!(
-                    series,
-                    apply_lambda_with_primitive_out_type,
-                    lambda,
-                    0,
-                    None
-                )?;
-                ca.into_series()
-            }
-            Some(DataType::UInt64) => {
-                let ca: UInt64Chunked = dispatch_apply!(
-                    series,
-                    apply_lambda_with_primitive_out_type,
-                    lambda,
-                    0,
-                    None
-                )?;
-                ca.into_series()
-            }
-            Some(DataType::Float32) => {
-                let ca: Float32Chunked = dispatch_apply!(
-                    series,
-                    apply_lambda_with_primitive_out_type,
-                    lambda,
-                    0,
-                    None
-                )?;
-                ca.into_series()
-            }
-            Some(DataType::Float64) => {
-                let ca: Float64Chunked = dispatch_apply!(
-                    series,
-                    apply_lambda_with_primitive_out_type,
-                    lambda,
-                    0,
-                    None
-                )?;
-                ca.into_series()
-            }
-            Some(DataType::Boolean) => {
-                let ca: BooleanChunked =
-                    dispatch_apply!(series, apply_lambda_with_bool_out_type, lambda, 0, None)?;
-                ca.into_series()
-            }
-            Some(DataType::Date) => {
-                let ca: Int32Chunked = dispatch_apply!(
-                    series,
-                    apply_lambda_with_primitive_out_type,
-                    lambda,
-                    0,
-                    None
-                )?;
-                ca.into_date().into_series()
-            }
-            Some(DataType::Datetime(tu, tz)) => {
-                let ca: Int64Chunked = dispatch_apply!(
-                    series,
-                    apply_lambda_with_primitive_out_type,
-                    lambda,
-                    0,
-                    None
-                )?;
-                ca.into_datetime(tu, tz).into_series()
-            }
-            Some(DataType::String) => {
-                let ca = dispatch_apply!(series, apply_lambda_with_utf8_out_type, lambda, 0, None)?;
-
-                ca.into_series()
-            }
-            Some(DataType::Object(_)) => {
-                let ca =
-                    dispatch_apply!(series, apply_lambda_with_object_out_type, lambda, 0, None)?;
-                ca.into_series()
-            }
-            None => return dispatch_apply!(series, apply_lambda_unknown, lambda),
-
-            _ => return dispatch_apply!(series, apply_lambda_unknown, lambda),
-        };
-
-        Ok(RbSeries::new(out))
-    }
-
     pub fn zip_with(&self, mask: &RbSeries, other: &RbSeries) -> RbResult<Self> {
         let binding = mask.series.borrow();
         let mask = binding.bool().map_err(RbPolarsErr::from)?;
@@ -508,11 +308,16 @@ impl RbSeries {
         Ok(RbSeries::new(s))
     }
 
-    pub fn to_dummies(&self, sep: Option<String>, drop_first: bool) -> RbResult<RbDataFrame> {
+    pub fn to_dummies(
+        &self,
+        sep: Option<String>,
+        drop_first: bool,
+        drop_nulls: bool,
+    ) -> RbResult<RbDataFrame> {
         let df = self
             .series
             .borrow()
-            .to_dummies(sep.as_deref(), drop_first)
+            .to_dummies(sep.as_deref(), drop_first, drop_nulls)
             .map_err(RbPolarsErr::from)?;
         Ok(df.into())
     }

data/ext/polars/src/series/import.rs

@@ -1,13 +1,13 @@
 use arrow::array::Array;
 use arrow::ffi::{ArrowArrayStream, ArrowArrayStreamReader};
-use magnus::prelude::*;
 use magnus::Value;
+use magnus::prelude::*;
 use polars::prelude::*;
 
 use super::RbSeries;
 
-use crate::exceptions::RbValueError;
 use crate::RbResult;
+use crate::exceptions::RbValueError;
 
 /// Import `arrow_c_stream` across Ruby boundary.
 fn call_arrow_c_stream(ob: Value) -> RbResult<Value> {

data/ext/polars/src/series/map.rs

@@ -0,0 +1,227 @@
+use magnus::Value;
+
+use super::RbSeries;
+use crate::map::check_nested_object;
+use crate::map::series::{ApplyLambda, call_lambda_and_extract};
+use crate::prelude::*;
+use crate::{RbPolarsErr, RbResult};
+use crate::{apply_method_all_arrow_series2, raise_err};
+
+impl RbSeries {
+    pub fn map_elements(
+        &self,
+        function: Value,
+        return_dtype: Option<Wrap<DataType>>,
+        skip_nulls: bool,
+    ) -> RbResult<Self> {
+        let series = &self.series.borrow();
+
+        if return_dtype.is_none() {
+            polars_warn!(
+                MapWithoutReturnDtypeWarning,
+                "Calling `map_elements` without specifying `return_dtype` can lead to unpredictable results. \
+                Specify `return_dtype` to silence this warning."
+            )
+        }
+
+        if skip_nulls && (series.null_count() == series.len()) {
+            if let Some(return_dtype) = return_dtype {
+                return Ok(
+                    Series::full_null(series.name().clone(), series.len(), &return_dtype.0).into(),
+                );
+            }
+            let msg = "The output type of the 'map_elements' function cannot be determined.\n\
+            The function was never called because 'skip_nulls: true' and all values are null.\n\
+            Consider setting 'skip_nulls: false' or setting the 'return_dtype'.";
+            raise_err!(msg, ComputeError)
+        }
+
+        let return_dtype = return_dtype.map(|dt| dt.0);
+
+        macro_rules! dispatch_apply {
+            ($self:expr, $method:ident, $($args:expr),*) => {
+                match $self.dtype() {
+                    DataType::Object(_) => {
+                        // let ca = $self.0.unpack::<ObjectType<ObjectValue>>().unwrap();
+                        // ca.$method($($args),*)
+                        todo!()
+                    }
+                    _ => {
+                        apply_method_all_arrow_series2!(
+                            $self,
+                            $method,
+                            $($args),*
+                        )
+                    }
+                }
+            }
+
+        }
+
+        if matches!(
+            series.dtype(),
+            DataType::Datetime(_, _)
+                | DataType::Date
+                | DataType::Duration(_)
+                | DataType::Categorical(_, _)
+                | DataType::Enum(_, _)
+                | DataType::Binary
+                | DataType::Array(_, _)
+                | DataType::Time
+                | DataType::Decimal(_, _)
+        ) || !skip_nulls
+        {
+            let mut avs = Vec::with_capacity(series.len());
+            let s = series.rechunk();
+
+            for av in s.iter() {
+                let out = match (skip_nulls, av) {
+                    (true, AnyValue::Null) => AnyValue::Null,
+                    (_, av) => {
+                        let av: Option<Wrap<AnyValue>> =
+                            call_lambda_and_extract(function, Wrap(av))?;
+                        match av {
+                            None => AnyValue::Null,
+                            Some(av) => av.0,
+                        }
+                    }
+                };
+                avs.push(out)
+            }
+            let out = Series::new(series.name().clone(), &avs);
+            let dtype = out.dtype();
+            if dtype.is_nested() {
+                check_nested_object(dtype)?;
+            }
+
+            return Ok(out.into());
+        }
+
+        let out = match return_dtype {
+            Some(DataType::Int8) => {
+                let ca: Int8Chunked = dispatch_apply!(
+                    series,
+                    apply_lambda_with_primitive_out_type,
+                    function,
+                    0,
+                    None
+                )?;
+                ca.into_series()
+            }
+            Some(DataType::Int16) => {
+                let ca: Int16Chunked = dispatch_apply!(
+                    series,
+                    apply_lambda_with_primitive_out_type,
+                    function,
+                    0,
+                    None
+                )?;
+                ca.into_series()
+            }
+            Some(DataType::Int32) => {
+                let ca: Int32Chunked = dispatch_apply!(
+                    series,
+                    apply_lambda_with_primitive_out_type,
+                    function,
+                    0,
+                    None
+                )?;
+                ca.into_series()
+            }
+            Some(DataType::Int64) => {
+                let ca: Int64Chunked = dispatch_apply!(
+                    series,
+                    apply_lambda_with_primitive_out_type,
+                    function,
+                    0,
+                    None
+                )?;
+                ca.into_series()
+            }
+            Some(DataType::UInt8) => {
+                let ca: UInt8Chunked = dispatch_apply!(
+                    series,
+                    apply_lambda_with_primitive_out_type,
+                    function,
+                    0,
+                    None
+                )?;
+                ca.into_series()
+            }
+            Some(DataType::UInt16) => {
+                let ca: UInt16Chunked = dispatch_apply!(
+                    series,
+                    apply_lambda_with_primitive_out_type,
+                    function,
+                    0,
+                    None
+                )?;
+                ca.into_series()
+            }
+            Some(DataType::UInt32) => {
+                let ca: UInt32Chunked = dispatch_apply!(
+                    series,
+                    apply_lambda_with_primitive_out_type,
+                    function,
+                    0,
+                    None
+                )?;
+                ca.into_series()
+            }
+            Some(DataType::UInt64) => {
+                let ca: UInt64Chunked = dispatch_apply!(
+                    series,
+                    apply_lambda_with_primitive_out_type,
+                    function,
+                    0,
+                    None
+                )?;
+                ca.into_series()
+            }
+            Some(DataType::Float32) => {
+                let ca: Float32Chunked = dispatch_apply!(
+                    series,
+                    apply_lambda_with_primitive_out_type,
+                    function,
+                    0,
+                    None
+                )?;
+                ca.into_series()
+            }
+            Some(DataType::Float64) => {
+                let ca: Float64Chunked = dispatch_apply!(
+                    series,
+                    apply_lambda_with_primitive_out_type,
+                    function,
+                    0,
+                    None
+                )?;
+                ca.into_series()
+            }
+            Some(DataType::Boolean) => {
+                let ca: BooleanChunked =
+                    dispatch_apply!(series, apply_lambda_with_bool_out_type, function, 0, None)?;
+                ca.into_series()
+            }
+            Some(DataType::String) => {
+                let ca =
+                    dispatch_apply!(series, apply_lambda_with_utf8_out_type, function, 0, None)?;
+
+                ca.into_series()
+            }
+            Some(DataType::List(_inner)) => {
+                todo!()
+            }
+            Some(DataType::Object(_)) => {
+                let ca =
+                    dispatch_apply!(series, apply_lambda_with_object_out_type, function, 0, None)?;
+                ca.into_series()
+            }
+            None => return dispatch_apply!(series, apply_lambda_unknown, function),
+
+            _ => return dispatch_apply!(series, apply_lambda_unknown, function),
+        };
+
+        Ok(RbSeries::new(out))
+    }
+}

data/ext/polars/src/series/mod.rs

@@ -5,9 +5,10 @@ mod construction;
 mod export;
 mod general;
 mod import;
+mod map;
 mod scatter;
 
-use magnus::{prelude::*, RArray};
+use magnus::{RArray, prelude::*};
 use polars::prelude::*;
 use std::cell::RefCell;
 

data/ext/polars/src/series/scatter.rs

@@ -100,7 +100,7 @@ fn scatter(mut s: Series, idx: &Series, values: &Series) -> PolarsResult<Series>
             let values = values.str()?;
             ca.scatter(idx, values)
         }
-        _ => panic!("not yet implemented for dtype: {}", logical_dtype),
+        _ => panic!("not yet implemented for dtype: {logical_dtype}"),
     };
 
     s.and_then(|s| s.cast(&logical_dtype))

data/ext/polars/src/utils.rs

@@ -1,3 +1,5 @@
+use crate::{RbErr, RbPolarsErr};
+
 #[macro_export]
 macro_rules! apply_method_all_arrow_series2 {
     ($self:expr, $method:ident, $($args:expr),*) => {
@@ -14,11 +16,17 @@ macro_rules! apply_method_all_arrow_series2 {
             DataType::Int64 => $self.i64().unwrap().$method($($args),*),
             DataType::Float32 => $self.f32().unwrap().$method($($args),*),
             DataType::Float64 => $self.f64().unwrap().$method($($args),*),
-            DataType::Date => $self.date().unwrap().$method($($args),*),
-            DataType::Datetime(_, _) => $self.datetime().unwrap().$method($($args),*),
+            DataType::Date => $self.date().unwrap().physical().$method($($args),*),
+            DataType::Datetime(_, _) => $self.datetime().unwrap().physical().$method($($args),*),
             // DataType::List(_) => $self.list().unwrap().$method($($args),*),
             DataType::Struct(_) => $self.struct_().unwrap().$method($($args),*),
             dt => panic!("dtype {:?} not supported", dt)
         }
     }
 }
+
+/// Boilerplate for `|e| RbPolarsErr::from(e).into()`
+#[allow(unused)]
+pub(crate) fn to_rb_err<E: Into<RbPolarsErr>>(e: E) -> RbErr {
+    e.into().into()
+}

data/lib/polars/cat_name_space.rb

@@ -31,56 +31,16 @@ module Polars
 
     # Return whether or not the column is a local categorical.
     #
-    # @return [Boolean]
-    #
-    # @example Categoricals constructed without a string cache are considered local.
-    #   s = Polars::Series.new(["a", "b", "a"], dtype: Polars::Categorical)
-    #   s.cat.is_local
-    #   # => true
+    # Always returns false.
     #
-    # @example Categoricals constructed with a string cache are considered global.
-    #   s = nil
-    #   Polars::StringCache.new do
-    #     s = Polars::Series.new(["a", "b", "a"], dtype: Polars::Categorical)
-    #   end
-    #   s.cat.is_local
-    #   # => false
+    # @return [Boolean]
     def is_local
       _s.cat_is_local
     end
 
-    # Convert a categorical column to its local representation.
-    #
-    # This may change the underlying physical representation of the column.
+    # Simply returns the column as-is, local representations are deprecated.
     #
     # @return [Series]
-    #
-    # @example Compare the global and local representations of a categorical.
-    #   s = nil
-    #   Polars::StringCache.new do
-    #     _ = Polars::Series.new("x", ["a", "b", "a"], dtype: Polars::Categorical)
-    #     s = Polars::Series.new("y", ["c", "b", "d"], dtype: Polars::Categorical)
-    #   end
-    #   s.to_physical
-    #   # =>
-    #   # shape: (3,)
-    #   # Series: 'y' [u32]
-    #   # [
-    #   #         2
-    #   #         1
-    #   #         3
-    #   # ]
-    #
-    # @example
-    #   s.cat.to_local.to_physical
-    #   # =>
-    #   # shape: (3,)
-    #   # Series: 'y' [u32]
-    #   # [
-    #   #         0
-    #   #         1
-    #   #         2
-    #   # ]
     def to_local
       Utils.wrap_s(_s.cat_to_local)
     end

data/lib/polars/catalog/unity/catalog_info.rb

@@ -0,0 +1,20 @@
+module Polars
+  class Catalog
+    module Unity
+      # Information for a catalog within a metastore.
+      CatalogInfo =
+        ::Struct.new(
+          :name,
+          :comment,
+          :properties,
+          :options,
+          :storage_location,
+          :created_at,
+          :created_by,
+          :updated_at,
+          :updated_by,
+          keyword_init: true
+        )
+    end
+  end
+end

data/lib/polars/catalog/unity/column_info.rb

@@ -0,0 +1,31 @@
+module Polars
+  class Catalog
+    module Unity
+      # Information for a column within a catalog table.
+      ColumnInfo =
+        ::Struct.new(
+          :name,
+          :type_name,
+          :type_text,
+          :type_json,
+          :position,
+          :comment,
+          :partition_index,
+          keyword_init: true
+        )
+
+      class ColumnInfo
+        # Get the native polars datatype of this column.
+        #
+        # @note
+        #   This functionality is considered **unstable**. It may be changed
+        #   at any point without it being considered a breaking change.
+        #
+        # @return [Object]
+        def get_polars_dtype
+          RbCatalogClient.type_json_to_polars_type(type_json)
+        end
+      end
+    end
+  end
+end

data/lib/polars/catalog/unity/namespace_info.rb

@@ -0,0 +1,21 @@
+module Polars
+  class Catalog
+    module Unity
+      # Information for a namespace within a catalog.
+      #
+      # This is also known by the name "schema" in unity catalog terminology.
+      NamespaceInfo =
+        ::Struct.new(
+          :name,
+          :comment,
+          :properties,
+          :storage_location,
+          :created_at,
+          :created_by,
+          :updated_at,
+          :updated_by,
+          keyword_init: true
+        )
+    end
+  end
+end