polars-df 0.19.0 → 0.21.0

data/ext/polars/src/series/map.rs

@@ -0,0 +1,227 @@
+use magnus::Value;
+
+use super::RbSeries;
+use crate::map::check_nested_object;
+use crate::map::series::{ApplyLambda, call_lambda_and_extract};
+use crate::prelude::*;
+use crate::{RbPolarsErr, RbResult};
+use crate::{apply_method_all_arrow_series2, raise_err};
+
+impl RbSeries {
+    pub fn map_elements(
+        &self,
+        function: Value,
+        return_dtype: Option<Wrap<DataType>>,
+        skip_nulls: bool,
+    ) -> RbResult<Self> {
+        let series = &self.series.borrow();
+
+        if return_dtype.is_none() {
+            polars_warn!(
+                MapWithoutReturnDtypeWarning,
+                "Calling `map_elements` without specifying `return_dtype` can lead to unpredictable results. \
+                Specify `return_dtype` to silence this warning."
+            )
+        }
+
+        if skip_nulls && (series.null_count() == series.len()) {
+            if let Some(return_dtype) = return_dtype {
+                return Ok(
+                    Series::full_null(series.name().clone(), series.len(), &return_dtype.0).into(),
+                );
+            }
+            let msg = "The output type of the 'map_elements' function cannot be determined.\n\
+            The function was never called because 'skip_nulls: true' and all values are null.\n\
+            Consider setting 'skip_nulls: false' or setting the 'return_dtype'.";
+            raise_err!(msg, ComputeError)
+        }
+
+        let return_dtype = return_dtype.map(|dt| dt.0);
+
+        macro_rules! dispatch_apply {
+            ($self:expr, $method:ident, $($args:expr),*) => {
+                match $self.dtype() {
+                    DataType::Object(_) => {
+                        // let ca = $self.0.unpack::<ObjectType<ObjectValue>>().unwrap();
+                        // ca.$method($($args),*)
+                        todo!()
+                    }
+                    _ => {
+                        apply_method_all_arrow_series2!(
+                            $self,
+                            $method,
+                            $($args),*
+                        )
+                    }
+                }
+            }
+
+        }
+
+        if matches!(
+            series.dtype(),
+            DataType::Datetime(_, _)
+                | DataType::Date
+                | DataType::Duration(_)
+                | DataType::Categorical(_, _)
+                | DataType::Enum(_, _)
+                | DataType::Binary
+                | DataType::Array(_, _)
+                | DataType::Time
+                | DataType::Decimal(_, _)
+        ) || !skip_nulls
+        {
+            let mut avs = Vec::with_capacity(series.len());
+            let s = series.rechunk();
+
+            for av in s.iter() {
+                let out = match (skip_nulls, av) {
+                    (true, AnyValue::Null) => AnyValue::Null,
+                    (_, av) => {
+                        let av: Option<Wrap<AnyValue>> =
+                            call_lambda_and_extract(function, Wrap(av))?;
+                        match av {
+                            None => AnyValue::Null,
+                            Some(av) => av.0,
+                        }
+                    }
+                };
+                avs.push(out)
+            }
+            let out = Series::new(series.name().clone(), &avs);
+            let dtype = out.dtype();
+            if dtype.is_nested() {
+                check_nested_object(dtype)?;
+            }
+
+            return Ok(out.into());
+        }
+
+        let out = match return_dtype {
+            Some(DataType::Int8) => {
+                let ca: Int8Chunked = dispatch_apply!(
+                    series,
+                    apply_lambda_with_primitive_out_type,
+                    function,
+                    0,
+                    None
+                )?;
+                ca.into_series()
+            }
+            Some(DataType::Int16) => {
+                let ca: Int16Chunked = dispatch_apply!(
+                    series,
+                    apply_lambda_with_primitive_out_type,
+                    function,
+                    0,
+                    None
+                )?;
+                ca.into_series()
+            }
+            Some(DataType::Int32) => {
+                let ca: Int32Chunked = dispatch_apply!(
+                    series,
+                    apply_lambda_with_primitive_out_type,
+                    function,
+                    0,
+                    None
+                )?;
+                ca.into_series()
+            }
+            Some(DataType::Int64) => {
+                let ca: Int64Chunked = dispatch_apply!(
+                    series,
+                    apply_lambda_with_primitive_out_type,
+                    function,
+                    0,
+                    None
+                )?;
+                ca.into_series()
+            }
+            Some(DataType::UInt8) => {
+                let ca: UInt8Chunked = dispatch_apply!(
+                    series,
+                    apply_lambda_with_primitive_out_type,
+                    function,
+                    0,
+                    None
+                )?;
+                ca.into_series()
+            }
+            Some(DataType::UInt16) => {
+                let ca: UInt16Chunked = dispatch_apply!(
+                    series,
+                    apply_lambda_with_primitive_out_type,
+                    function,
+                    0,
+                    None
+                )?;
+                ca.into_series()
+            }
+            Some(DataType::UInt32) => {
+                let ca: UInt32Chunked = dispatch_apply!(
+                    series,
+                    apply_lambda_with_primitive_out_type,
+                    function,
+                    0,
+                    None
+                )?;
+                ca.into_series()
+            }
+            Some(DataType::UInt64) => {
+                let ca: UInt64Chunked = dispatch_apply!(
+                    series,
+                    apply_lambda_with_primitive_out_type,
+                    function,
+                    0,
+                    None
+                )?;
+                ca.into_series()
+            }
+            Some(DataType::Float32) => {
+                let ca: Float32Chunked = dispatch_apply!(
+                    series,
+                    apply_lambda_with_primitive_out_type,
+                    function,
+                    0,
+                    None
+                )?;
+                ca.into_series()
+            }
+            Some(DataType::Float64) => {
+                let ca: Float64Chunked = dispatch_apply!(
+                    series,
+                    apply_lambda_with_primitive_out_type,
+                    function,
+                    0,
+                    None
+                )?;
+                ca.into_series()
+            }
+            Some(DataType::Boolean) => {
+                let ca: BooleanChunked =
+                    dispatch_apply!(series, apply_lambda_with_bool_out_type, function, 0, None)?;
+                ca.into_series()
+            }
+            Some(DataType::String) => {
+                let ca =
+                    dispatch_apply!(series, apply_lambda_with_utf8_out_type, function, 0, None)?;
+
+                ca.into_series()
+            }
+            Some(DataType::List(_inner)) => {
+                todo!()
+            }
+            Some(DataType::Object(_)) => {
+                let ca =
+                    dispatch_apply!(series, apply_lambda_with_object_out_type, function, 0, None)?;
+                ca.into_series()
+            }
+            None => return dispatch_apply!(series, apply_lambda_unknown, function),
+
+            _ => return dispatch_apply!(series, apply_lambda_unknown, function),
+        };
+
+        Ok(RbSeries::new(out))
+    }
+}

data/ext/polars/src/series/mod.rs

@@ -5,9 +5,10 @@ mod construction;
 mod export;
 mod general;
 mod import;
+mod map;
 mod scatter;
 
-use magnus::{prelude::*, RArray};
+use magnus::{RArray, prelude::*};
 use polars::prelude::*;
 use std::cell::RefCell;
 

data/ext/polars/src/series/scatter.rs

@@ -100,7 +100,7 @@ fn scatter(mut s: Series, idx: &Series, values: &Series) -> PolarsResult<Series>
             let values = values.str()?;
             ca.scatter(idx, values)
         }
-        _ => panic!("not yet implemented for dtype: {}", logical_dtype),
+        _ => panic!("not yet implemented for dtype: {logical_dtype}"),
     };
 
     s.and_then(|s| s.cast(&logical_dtype))

data/ext/polars/src/utils.rs

@@ -1,3 +1,5 @@
+use crate::{RbErr, RbPolarsErr};
+
 #[macro_export]
 macro_rules! apply_method_all_arrow_series2 {
     ($self:expr, $method:ident, $($args:expr),*) => {
@@ -14,11 +16,17 @@ macro_rules! apply_method_all_arrow_series2 {
             DataType::Int64 => $self.i64().unwrap().$method($($args),*),
             DataType::Float32 => $self.f32().unwrap().$method($($args),*),
             DataType::Float64 => $self.f64().unwrap().$method($($args),*),
-            DataType::Date => $self.date().unwrap().$method($($args),*),
-            DataType::Datetime(_, _) => $self.datetime().unwrap().$method($($args),*),
+            DataType::Date => $self.date().unwrap().physical().$method($($args),*),
+            DataType::Datetime(_, _) => $self.datetime().unwrap().physical().$method($($args),*),
             // DataType::List(_) => $self.list().unwrap().$method($($args),*),
             DataType::Struct(_) => $self.struct_().unwrap().$method($($args),*),
             dt => panic!("dtype {:?} not supported", dt)
         }
     }
 }
+
+/// Boilerplate for `|e| RbPolarsErr::from(e).into()`
+#[allow(unused)]
+pub(crate) fn to_rb_err<E: Into<RbPolarsErr>>(e: E) -> RbErr {
+    e.into().into()
+}

data/lib/polars/cat_name_space.rb

@@ -31,56 +31,16 @@ module Polars
 
     # Return whether or not the column is a local categorical.
     #
-    # @return [Boolean]
-    #
-    # @example Categoricals constructed without a string cache are considered local.
-    #   s = Polars::Series.new(["a", "b", "a"], dtype: Polars::Categorical)
-    #   s.cat.is_local
-    #   # => true
+    # Always returns false.
     #
-    # @example Categoricals constructed with a string cache are considered global.
-    #   s = nil
-    #   Polars::StringCache.new do
-    #     s = Polars::Series.new(["a", "b", "a"], dtype: Polars::Categorical)
-    #   end
-    #   s.cat.is_local
-    #   # => false
+    # @return [Boolean]
     def is_local
       _s.cat_is_local
     end
 
-    # Convert a categorical column to its local representation.
-    #
-    # This may change the underlying physical representation of the column.
+    # Simply returns the column as-is, local representations are deprecated.
     #
     # @return [Series]
-    #
-    # @example Compare the global and local representations of a categorical.
-    #   s = nil
-    #   Polars::StringCache.new do
-    #     _ = Polars::Series.new("x", ["a", "b", "a"], dtype: Polars::Categorical)
-    #     s = Polars::Series.new("y", ["c", "b", "d"], dtype: Polars::Categorical)
-    #   end
-    #   s.to_physical
-    #   # =>
-    #   # shape: (3,)
-    #   # Series: 'y' [u32]
-    #   # [
-    #   #         2
-    #   #         1
-    #   #         3
-    #   # ]
-    #
-    # @example
-    #   s.cat.to_local.to_physical
-    #   # =>
-    #   # shape: (3,)
-    #   # Series: 'y' [u32]
-    #   # [
-    #   #         0
-    #   #         1
-    #   #         2
-    #   # ]
     def to_local
       Utils.wrap_s(_s.cat_to_local)
     end

data/lib/polars/catalog/unity/catalog_info.rb

@@ -0,0 +1,20 @@
+module Polars
+  class Catalog
+    module Unity
+      # Information for a catalog within a metastore.
+      CatalogInfo =
+        ::Struct.new(
+          :name,
+          :comment,
+          :properties,
+          :options,
+          :storage_location,
+          :created_at,
+          :created_by,
+          :updated_at,
+          :updated_by,
+          keyword_init: true
+        )
+    end
+  end
+end

data/lib/polars/catalog/unity/column_info.rb

@@ -0,0 +1,31 @@
+module Polars
+  class Catalog
+    module Unity
+      # Information for a column within a catalog table.
+      ColumnInfo =
+        ::Struct.new(
+          :name,
+          :type_name,
+          :type_text,
+          :type_json,
+          :position,
+          :comment,
+          :partition_index,
+          keyword_init: true
+        )
+
+      class ColumnInfo
+        # Get the native polars datatype of this column.
+        #
+        # @note
+        #   This functionality is considered **unstable**. It may be changed
+        #   at any point without it being considered a breaking change.
+        #
+        # @return [Object]
+        def get_polars_dtype
+          RbCatalogClient.type_json_to_polars_type(type_json)
+        end
+      end
+    end
+  end
+end

data/lib/polars/catalog/unity/namespace_info.rb

@@ -0,0 +1,21 @@
+module Polars
+  class Catalog
+    module Unity
+      # Information for a namespace within a catalog.
+      #
+      # This is also known by the name "schema" in unity catalog terminology.
+      NamespaceInfo =
+        ::Struct.new(
+          :name,
+          :comment,
+          :properties,
+          :storage_location,
+          :created_at,
+          :created_by,
+          :updated_at,
+          :updated_by,
+          keyword_init: true
+        )
+    end
+  end
+end

data/lib/polars/catalog/unity/table_info.rb

@@ -0,0 +1,50 @@
+module Polars
+  class Catalog
+    module Unity
+      # Information for a catalog table.
+      TableInfo =
+        ::Struct.new(
+          :name,
+          :comment,
+          :table_id,
+          :table_type,
+          :storage_location,
+          :data_source_format,
+          :columns,
+          :properties,
+          :created_at,
+          :created_by,
+          :updated_at,
+          :updated_by,
+          keyword_init: true
+        )
+
+      class TableInfo
+        # Get the native polars schema of this table.
+        #
+        # @note
+        #   This functionality is considered **unstable**. It may be changed
+        #   at any point without it being considered a breaking change.
+        #
+        # @return [Schema]
+        def get_polars_schema
+          if columns.nil?
+            return nil
+          end
+
+          schema = Schema.new(check_dtypes: false)
+
+          columns.each do |column_info|
+            if schema[column_info.name]
+              msg = "duplicate column name: #{column_info.name}"
+              raise DuplicateError, msg
+            end
+            schema[column_info.name] = column_info.get_polars_dtype
+          end
+
+          schema
+        end
+      end
+    end
+  end
+end