polars-df 0.18.0 → 0.19.0

data/ext/polars/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "polars"
-version = "0.18.0"
+version = "0.19.0"
 license = "MIT"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"
@@ -12,22 +12,22 @@ crate-type = ["cdylib"]
 
 [dependencies]
 ahash = "0.8"
-arrow = { package = "polars-arrow", version = "=0.47.1" }
+arrow = { package = "polars-arrow", version = "=0.48.0" }
 bytes = "1"
 chrono = "0.4"
 either = "1.8"
 magnus = "0.7"
 num-traits = "0.2"
-polars-core = "=0.47.1"
-polars-plan = "=0.47.1"
-polars-parquet = "=0.47.1"
-polars-utils = "=0.47.1"
+polars-core = "=0.48.0"
+polars-plan = "=0.48.0"
+polars-parquet = "=0.48.0"
+polars-utils = "=0.48.0"
 rayon = "1.9"
 regex = "1"
 serde_json = "1"
 
 [dependencies.polars]
-version = "=0.47.1"
+version = "=0.48.0"
 features = [
     "abs",
     "approx_unique",

data/ext/polars/src/conversion/any_value.rs

@@ -173,7 +173,7 @@ pub(crate) fn rb_object_to_any_value<'s>(ob: Value, strict: bool) -> RbResult<An
     }
 
     fn get_struct(ob: Value, _strict: bool) -> RbResult<AnyValue<'static>> {
-        let dict = RHash::from_value(ob).unwrap();
+        let dict = RHash::try_convert(ob)?;
         let len = dict.len();
         let mut keys = Vec::with_capacity(len);
         let mut vals = Vec::with_capacity(len);

data/ext/polars/src/conversion/mod.rs

@@ -210,7 +210,10 @@ impl IntoValue for Wrap<DataType> {
             DataType::Datetime(tu, tz) => {
                 let datetime_class = pl.const_get::<_, Value>("Datetime").unwrap();
                 datetime_class
-                    .funcall::<_, _, Value>("new", (tu.to_ascii(), tz.as_deref()))
+                    .funcall::<_, _, Value>(
+                        "new",
+                        (tu.to_ascii(), tz.as_deref().map(|x| x.as_str())),
+                    )
                     .unwrap()
             }
             DataType::Duration(tu) => {
@@ -375,7 +378,10 @@ impl TryConvert for Wrap<DataType> {
                     let time_unit: Value = ob.funcall("time_unit", ()).unwrap();
                     let time_unit = Wrap::<TimeUnit>::try_convert(time_unit)?.0;
                     let time_zone: Option<String> = ob.funcall("time_zone", ())?;
-                    DataType::Datetime(time_unit, time_zone.as_deref().map(|x| x.into()))
+                    DataType::Datetime(
+                        time_unit,
+                        TimeZone::opt_try_new(time_zone.as_deref()).map_err(RbPolarsErr::from)?,
+                    )
                 }
                 "Polars::Duration" => {
                     let time_unit: Value = ob.funcall("time_unit", ()).unwrap();
@@ -1229,3 +1235,13 @@ impl TryConvert for RbCompatLevel {
         }))
     }
 }
+
+impl TryConvert for Wrap<Option<TimeZone>> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let tz = Option::<Wrap<PlSmallStr>>::try_convert(ob)?;
+
+        let tz = tz.map(|x| x.0);
+
+        Ok(Wrap(TimeZone::opt_try_new(tz).map_err(RbPolarsErr::from)?))
+    }
+}

data/ext/polars/src/expr/array.rs

@@ -75,11 +75,11 @@ impl RbExpr {
             .into()
     }
 
-    pub fn arr_contains(&self, other: &RbExpr) -> Self {
+    pub fn arr_contains(&self, other: &RbExpr, nulls_equal: bool) -> Self {
         self.inner
             .clone()
             .arr()
-            .contains(other.inner.clone())
+            .contains(other.inner.clone(), nulls_equal)
             .into()
     }
 

data/ext/polars/src/expr/datetime.rs

@@ -1,7 +1,7 @@
 use polars::prelude::*;
 
 use crate::conversion::Wrap;
-use crate::RbExpr;
+use crate::{RbExpr, RbPolarsErr, RbResult};
 
 impl RbExpr {
     pub fn dt_to_string(&self, format: String) -> Self {
@@ -30,12 +30,17 @@ impl RbExpr {
         self.inner.clone().dt().with_time_unit(tu.0).into()
     }
 
-    pub fn dt_convert_time_zone(&self, time_zone: String) -> Self {
-        self.inner
+    pub fn dt_convert_time_zone(&self, time_zone: String) -> RbResult<Self> {
+        Ok(self
+            .inner
             .clone()
             .dt()
-            .convert_time_zone(time_zone.into())
-            .into()
+            .convert_time_zone(
+                TimeZone::opt_try_new(Some(PlSmallStr::from(time_zone)))
+                    .map_err(RbPolarsErr::from)?
+                    .unwrap_or(TimeZone::UTC),
+            )
+            .into())
     }
 
     pub fn dt_cast_time_unit(&self, tu: Wrap<TimeUnit>) -> Self {
@@ -47,16 +52,18 @@ impl RbExpr {
         time_zone: Option<String>,
         ambiguous: &Self,
         non_existent: Wrap<NonExistent>,
-    ) -> Self {
-        self.inner
+    ) -> RbResult<Self> {
+        Ok(self
+            .inner
             .clone()
             .dt()
             .replace_time_zone(
-                time_zone.map(|x| x.into()),
+                TimeZone::opt_try_new(time_zone.map(PlSmallStr::from_string))
+                    .map_err(RbPolarsErr::from)?,
                 ambiguous.inner.clone(),
                 non_existent.0,
             )
-            .into()
+            .into())
     }
 
     pub fn dt_truncate(&self, every: &Self) -> Self {

data/ext/polars/src/expr/list.rs

@@ -23,11 +23,11 @@ impl RbExpr {
         self.inner.clone().list().arg_min().into()
     }
 
-    pub fn list_contains(&self, other: &RbExpr) -> Self {
+    pub fn list_contains(&self, other: &RbExpr, nulls_equal: bool) -> Self {
         self.inner
             .clone()
             .list()
-            .contains(other.inner.clone())
+            .contains(other.inner.clone(), nulls_equal)
             .into()
     }
 
@@ -180,7 +180,7 @@ impl RbExpr {
         &self,
         width_strat: Wrap<ListToStructWidthStrategy>,
         name_gen: Option<Value>,
-        upper_bound: usize,
+        upper_bound: Option<usize>,
     ) -> RbResult<Self> {
         let name_gen = name_gen.map(|lambda| {
             let lambda = Opaque::from(lambda);

data/ext/polars/src/expr/string.rs

@@ -35,14 +35,14 @@ impl RbExpr {
         &self,
         format: Option<String>,
         time_unit: Option<Wrap<TimeUnit>>,
-        time_zone: Option<Wrap<TimeZone>>,
+        time_zone: Wrap<Option<TimeZone>>,
         strict: bool,
         exact: bool,
         cache: bool,
         ambiguous: &Self,
     ) -> Self {
         let format = format.map(|x| x.into());
-        let time_zone = time_zone.map(|x| x.0);
+        let time_zone = time_zone.0;
 
         let options = StrptimeOptions {
             format,

data/ext/polars/src/functions/lazy.rs

@@ -310,7 +310,7 @@ pub fn last() -> RbExpr {
     dsl::last().into()
 }
 
-pub fn lit(value: Value, allow_object: bool) -> RbResult<RbExpr> {
+pub fn lit(value: Value, allow_object: bool, is_scalar: bool) -> RbResult<RbExpr> {
     if value.is_kind_of(class::true_class()) || value.is_kind_of(class::false_class()) {
         Ok(dsl::lit(bool::try_convert(value)?).into())
     } else if let Some(v) = Integer::from_value(value) {
@@ -336,7 +336,16 @@ pub fn lit(value: Value, allow_object: bool) -> RbResult<RbExpr> {
             Ok(dsl::lit(unsafe { v.as_slice() }).into())
         }
     } else if let Ok(series) = Obj::<RbSeries>::try_convert(value) {
-        Ok(dsl::lit(series.series.borrow().clone()).into())
+        let s = series.series.borrow();
+        if is_scalar {
+            let av = s
+                .get(0)
+                .map_err(|_| RbValueError::new_err("expected at least 1 value"))?;
+            let av = av.into_static();
+            Ok(dsl::lit(Scalar::new(s.dtype().clone(), av)).into())
+        } else {
+            Ok(dsl::lit(s.clone()).into())
+        }
     } else if value.is_nil() {
         Ok(dsl::lit(Null {}).into())
     } else if allow_object {

data/ext/polars/src/functions/range.rs

@@ -56,14 +56,14 @@ pub fn datetime_range(
     every: String,
     closed: Wrap<ClosedWindow>,
     time_unit: Option<Wrap<TimeUnit>>,
-    time_zone: Option<Wrap<TimeZone>>,
+    time_zone: Wrap<Option<TimeZone>>,
 ) -> RbExpr {
     let start = start.inner.clone();
     let end = end.inner.clone();
     let every = Duration::parse(&every);
     let closed = closed.0;
     let time_unit = time_unit.map(|x| x.0);
-    let time_zone = time_zone.map(|x| x.0);
+    let time_zone = time_zone.0;
     dsl::datetime_range(start, end, every, closed, time_unit, time_zone).into()
 }
 
@@ -73,14 +73,14 @@ pub fn datetime_ranges(
     every: String,
     closed: Wrap<ClosedWindow>,
     time_unit: Option<Wrap<TimeUnit>>,
-    time_zone: Option<Wrap<TimeZone>>,
+    time_zone: Wrap<Option<TimeZone>>,
 ) -> RbExpr {
     let start = start.inner.clone();
     let end = end.inner.clone();
     let every = Duration::parse(&every);
     let closed = closed.0;
     let time_unit = time_unit.map(|x| x.0);
-    let time_zone = time_zone.map(|x| x.0);
+    let time_zone = time_zone.0;
     dsl::datetime_ranges(start, end, every, closed, time_unit, time_zone).into()
 }
 

data/ext/polars/src/lazyframe/general.rs

@@ -392,6 +392,8 @@ impl RbLazyFrame {
             statistics: statistics.0,
             row_group_size,
             data_page_size,
+            key_value_metadata: None,
+            field_overwrites: Vec::new(),
         };
 
         let cloud_options = match target.base_path() {

data/ext/polars/src/lib.rs

@@ -320,7 +320,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("arr_arg_max", method!(RbExpr::arr_arg_max, 0))?;
     class.define_method("arr_get", method!(RbExpr::arr_get, 2))?;
     class.define_method("arr_join", method!(RbExpr::arr_join, 2))?;
-    class.define_method("arr_contains", method!(RbExpr::arr_contains, 1))?;
+    class.define_method("arr_contains", method!(RbExpr::arr_contains, 2))?;
     class.define_method("arr_count_matches", method!(RbExpr::arr_count_matches, 1))?;
     class.define_method("binary_contains", method!(RbExpr::bin_contains, 1))?;
     class.define_method("binary_ends_with", method!(RbExpr::bin_ends_with, 1))?;
@@ -365,7 +365,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("str_contains_any", method!(RbExpr::str_contains_any, 2))?;
     class.define_method("str_replace_many", method!(RbExpr::str_replace_many, 3))?;
     class.define_method("list_len", method!(RbExpr::list_len, 0))?;
-    class.define_method("list_contains", method!(RbExpr::list_contains, 1))?;
+    class.define_method("list_contains", method!(RbExpr::list_contains, 2))?;
     class.define_method("list_count_matches", method!(RbExpr::list_count_matches, 1))?;
     class.define_method("dt_year", method!(RbExpr::dt_year, 0))?;
     class.define_method("dt_is_leap_year", method!(RbExpr::dt_is_leap_year, 0))?;
@@ -559,7 +559,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_singleton_method("cols", function!(functions::lazy::cols, 1))?;
     class.define_singleton_method("fold", function!(functions::lazy::fold, 5))?;
     class.define_singleton_method("cum_fold", function!(functions::lazy::cum_fold, 4))?;
-    class.define_singleton_method("lit", function!(functions::lazy::lit, 2))?;
+    class.define_singleton_method("lit", function!(functions::lazy::lit, 3))?;
     class.define_singleton_method("int_range", function!(functions::range::int_range, 4))?;
     class.define_singleton_method("int_ranges", function!(functions::range::int_ranges, 4))?;
     class.define_singleton_method("repeat", function!(functions::lazy::repeat, 3))?;

data/lib/polars/array_expr.rb

@@ -481,6 +481,8 @@ module Polars
     #
     # @param item [Object]
     #   Item that will be checked for membership
+    # @param nulls_equal [Boolean]
+    #   If true, treat null as a distinct value. Null values will not propagate.
     #
     # @return [Expr]
     #
@@ -501,9 +503,9 @@ module Polars
     #   # │ ["x", "y"]    ┆ false    │
     #   # │ ["a", "c"]    ┆ true     │
     #   # └───────────────┴──────────┘
-    def contains(item)
+    def contains(item, nulls_equal: true)
       item = Utils.parse_into_expression(item, str_as_lit: true)
-      Utils.wrap_expr(_rbexpr.arr_contains(item))
+      Utils.wrap_expr(_rbexpr.arr_contains(item, nulls_equal))
     end
 
     # Count how often the value produced by `element` occurs.

data/lib/polars/expr.rb

@@ -3734,15 +3734,7 @@ module Polars
     #   # │ [9, 10]   ┆ 3                ┆ false    │
     #   # └───────────┴──────────────────┴──────────┘
     def is_in(other, nulls_equal: false)
-      if other.is_a?(::Array)
-        if other.length == 0
-          other = Polars.lit(nil)._rbexpr
-        else
-          other = Polars.lit(Series.new(other))._rbexpr
-        end
-      else
-        other = Utils.parse_into_expression(other, str_as_lit: false)
-      end
+      other = Utils.parse_into_expression(other)
       _from_rbexpr(_rbexpr.is_in(other, nulls_equal))
     end
     alias_method :in?, :is_in

data/lib/polars/functions/lit.rb

@@ -16,20 +16,15 @@ module Polars
       elsif value.is_a?(::Date)
         return lit(::Time.utc(value.year, value.month, value.day)).cast(Date)
       elsif value.is_a?(Polars::Series)
-        name = value.name
         value = value._s
-        e = Utils.wrap_expr(Plr.lit(value, allow_object))
-        if name == ""
-          return e
-        end
-        return e.alias(name)
+        return Utils.wrap_expr(Plr.lit(value, allow_object, false))
       elsif (defined?(Numo::NArray) && value.is_a?(Numo::NArray)) || value.is_a?(::Array)
-        return lit(Series.new("", value))
+        return Utils.wrap_expr(Plr.lit(Series.new("literal", [value.to_a], dtype: dtype)._s, allow_object, true))
       elsif dtype
-        return Utils.wrap_expr(Plr.lit(value, allow_object)).cast(dtype)
+        return Utils.wrap_expr(Plr.lit(value, allow_object, true)).cast(dtype)
       end
 
-      Utils.wrap_expr(Plr.lit(value, allow_object))
+      Utils.wrap_expr(Plr.lit(value, allow_object, true))
     end
   end
 end

data/lib/polars/list_expr.rb

@@ -481,6 +481,8 @@ module Polars
     #
     # @param item [Object]
     #   Item that will be checked for membership
+    # @param nulls_equal [Boolean]
+    #   If true, treat null as a distinct value. Null values will not propagate.
     #
     # @return [Expr]
     #
@@ -498,8 +500,8 @@ module Polars
     #   # │ false │
     #   # │ true  │
     #   # └───────┘
-    def contains(item)
-      Utils.wrap_expr(_rbexpr.list_contains(Utils.parse_into_expression(item)))
+    def contains(item, nulls_equal: true)
+      Utils.wrap_expr(_rbexpr.list_contains(Utils.parse_into_expression(item), nulls_equal))
     end
 
     # Join all string items in a sublist and place a separator between them.
@@ -746,9 +748,9 @@ module Polars
     #   # │ {1,2,3}    │
     #   # │ {1,2,null} │
     #   # └────────────┘
-    def to_struct(n_field_strategy: "first_non_null", name_generator: nil)
+    def to_struct(n_field_strategy: "first_non_null", name_generator: nil, upper_bound: nil)
       raise Todo if name_generator
-      Utils.wrap_expr(_rbexpr.list_to_struct(n_field_strategy, name_generator, 0))
+      Utils.wrap_expr(_rbexpr.list_to_struct(n_field_strategy, name_generator, nil))
     end
 
     # Run any polars expression against the lists' elements.

data/lib/polars/series.rb

@@ -2144,18 +2144,33 @@ module Polars
 
     # Check if elements of this Series are in the other Series.
     #
+    # @param nulls_equal [Boolean]
+    #   If true, treat null as a distinct value. Null values will not propagate.
+    #
     # @return [Series]
     #
     # @example
     #   s = Polars::Series.new("a", [1, 2, 3])
-    #   s2 = Polars::Series.new("b", [2, 4])
+    #   s2 = Polars::Series.new("b", [2, 4, nil])
     #   s2.is_in(s)
     #   # =>
-    #   # shape: (2,)
+    #   # shape: (3,)
+    #   # Series: 'b' [bool]
+    #   # [
+    #   #         true
+    #   #         false
+    #   #         null
+    #   # ]
+    #
+    # @example
+    #   s2.is_in(s, nulls_equal: true)
+    #   # =>
+    #   # shape: (3,)
     #   # Series: 'b' [bool]
     #   # [
     #   #         true
     #   #         false
+    #   #         false
     #   # ]
     #
     # @example
@@ -2190,7 +2205,7 @@ module Polars
     #   #         true
     #   #         false
     #   # ]
-    def is_in(other)
+    def is_in(other, nulls_equal: false)
       super
     end
     alias_method :in?, :is_in

data/lib/polars/string_expr.rb

@@ -1403,7 +1403,7 @@ module Polars
     #   # │ Can you feel the love tonight   ┆ true         │
     #   # └─────────────────────────────────┴──────────────┘
     def contains_any(patterns, ascii_case_insensitive: false)
-      patterns = Utils.parse_into_expression(patterns, str_as_lit: false, list_as_series: true)
+      patterns = Utils.parse_into_expression(patterns, str_as_lit: false)
       Utils.wrap_expr(
         _rbexpr.str_contains_any(patterns, ascii_case_insensitive)
       )
@@ -1474,9 +1474,9 @@ module Polars
     #   # │ Can you feel the love tonight   ┆ Can me feel the love tonight    │
     #   # └─────────────────────────────────┴─────────────────────────────────┘
     def replace_many(patterns, replace_with, ascii_case_insensitive: false)
-      patterns = Utils.parse_into_expression(patterns, str_as_lit: false, list_as_series: true)
+      patterns = Utils.parse_into_expression(patterns, str_as_lit: false)
       replace_with = Utils.parse_into_expression(
-        replace_with, str_as_lit: true, list_as_series: true
+        replace_with, str_as_lit: true
       )
       Utils.wrap_expr(
         _rbexpr.str_replace_many(

data/lib/polars/version.rb

@@ -1,4 +1,4 @@
 module Polars
   # @private
-  VERSION = "0.18.0"
+  VERSION = "0.19.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: polars-df
 version: !ruby/object:Gem::Version
-  version: 0.18.0
+  version: 0.19.0
 platform: ruby
 authors:
 - Andrew Kane