polars-df 0.13.0 → 0.15.0

data/LICENSE.txt

@@ -1,5 +1,6 @@
 Copyright (c) 2020 Ritchie Vink
 Copyright (c) 2022-2024 Andrew Kane
+Some portions Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -17,8 +17,7 @@ gem "polars-df"
 This library follows the [Polars Python API](https://pola-rs.github.io/polars/py-polars/html/reference/index.html).
 
 ```ruby
-Polars.read_csv("iris.csv")
-  .lazy
+Polars.scan_csv("iris.csv")
   .filter(Polars.col("sepal_length") > 5)
   .group_by("species")
   .agg(Polars.all.sum)

data/ext/polars/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "polars"
-version = "0.13.0"
+version = "0.15.0"
 license = "MIT"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"
@@ -12,17 +12,20 @@ crate-type = ["cdylib"]
 
 [dependencies]
 ahash = "0.8"
+arrow = { package = "polars-arrow", version = "=0.44.2" }
+bytes = "1"
 chrono = "0.4"
 either = "1.8"
 magnus = "0.7"
-polars-core = "=0.42.0"
-polars-parquet = "=0.42.0"
-polars-utils = "=0.42.0"
+polars-core = "=0.44.2"
+polars-plan = "=0.44.2"
+polars-parquet = "=0.44.2"
+polars-utils = "=0.44.2"
+regex = "1"
 serde_json = "1"
-smartstring = "1"
 
 [dependencies.polars]
-version = "=0.42.0"
+version = "=0.44.2"
 features = [
     "abs",
     "approx_unique",
@@ -31,7 +34,11 @@ features = [
     "array_count",
     "asof_join",
     "avro",
+    "aws",
+    "azure",
     "binary_encoding",
+    "business",
+    "cloud",
     "concat_str",
     "cov",
     "cross_join",
@@ -52,6 +59,8 @@ features = [
     "extract_jsonpath",
     "find_many",
     "fmt",
+    "gcp",
+    "http",
     "interpolate",
     "ipc",
     "ipc_streaming",

data/ext/polars/src/batched_csv.rs

@@ -34,8 +34,7 @@ impl RbBatchedCsv {
         let n_threads = Option::<usize>::try_convert(arguments[11])?;
         let path = PathBuf::try_convert(arguments[12])?;
         let overwrite_dtype = Option::<Vec<(String, Wrap<DataType>)>>::try_convert(arguments[13])?;
-        // TODO fix
-        let overwrite_dtype_slice = Option::<Vec<Wrap<DataType>>>::None; // Option::<Vec<Wrap<DataType>>>::try_convert(arguments[14])?;
+        let overwrite_dtype_slice = Option::<Vec<Wrap<DataType>>>::try_convert(arguments[14])?;
         let low_memory = bool::try_convert(arguments[15])?;
         let comment_prefix = Option::<String>::try_convert(arguments[16])?;
         let quote_char = Option::<String>::try_convert(arguments[17])?;
@@ -44,17 +43,16 @@ impl RbBatchedCsv {
         let try_parse_dates = bool::try_convert(arguments[20])?;
         let skip_rows_after_header = usize::try_convert(arguments[21])?;
         let row_index = Option::<(String, IdxSize)>::try_convert(arguments[22])?;
-        let sample_size = usize::try_convert(arguments[23])?;
-        let eol_char = String::try_convert(arguments[24])?;
-        let raise_if_empty = bool::try_convert(arguments[25])?;
-        let truncate_ragged_lines = bool::try_convert(arguments[26])?;
-        let decimal_comma = bool::try_convert(arguments[27])?;
+        let eol_char = String::try_convert(arguments[23])?;
+        let raise_if_empty = bool::try_convert(arguments[24])?;
+        let truncate_ragged_lines = bool::try_convert(arguments[25])?;
+        let decimal_comma = bool::try_convert(arguments[26])?;
         // end arguments
 
         let null_values = null_values.map(|w| w.0);
         let eol_char = eol_char.as_bytes()[0];
         let row_index = row_index.map(|(name, offset)| RowIndex {
-            name: Arc::from(name.as_str()),
+            name: name.into(),
             offset,
         });
         let quote_char = if let Some(s) = quote_char {
@@ -72,7 +70,7 @@ impl RbBatchedCsv {
                 .iter()
                 .map(|(name, dtype)| {
                     let dtype = dtype.0.clone();
-                    Field::new(name, dtype)
+                    Field::new((&**name).into(), dtype)
                 })
                 .collect::<Schema>()
         });
@@ -84,7 +82,7 @@ impl RbBatchedCsv {
                 .collect::<Vec<_>>()
         });
 
-        let file = std::fs::File::open(path).map_err(RbPolarsErr::io)?;
+        let file = std::fs::File::open(path).map_err(RbPolarsErr::from)?;
         let reader = Box::new(file) as Box<dyn MmapBytesReader>;
         let reader = CsvReadOptions::default()
             .with_infer_schema_length(infer_schema_length)
@@ -95,13 +93,12 @@ impl RbBatchedCsv {
             .with_projection(projection.map(Arc::new))
             .with_rechunk(rechunk)
             .with_chunk_size(chunk_size)
-            .with_columns(columns.map(Arc::from))
+            .with_columns(columns.map(|x| x.into_iter().map(PlSmallStr::from_string).collect()))
             .with_n_threads(n_threads)
             .with_dtype_overwrite(overwrite_dtype_slice.map(Arc::new))
             .with_low_memory(low_memory)
             .with_skip_rows_after_header(skip_rows_after_header)
             .with_row_index(row_index)
-            .with_sample_size(sample_size)
             .with_raise_if_empty(raise_if_empty)
             .with_parse_options(
                 CsvParseOptions::default()
@@ -132,7 +129,7 @@ impl RbBatchedCsv {
         let batches = reader
             .borrow()
             .lock()
-            .map_err(|e| RbPolarsErr::other(e.to_string()))?
+            .map_err(|e| RbPolarsErr::Other(e.to_string()))?
             .next_batches(n)
             .map_err(RbPolarsErr::from)?;
 

data/ext/polars/src/conversion/any_value.rs

@@ -7,9 +7,9 @@ use polars_core::utils::any_values_to_supertype_and_n_dtypes;
 
 use super::{struct_dict, ObjectValue, Wrap};
 
-use crate::error::RbOverflowError;
+use crate::exceptions::RbOverflowError;
 use crate::rb_modules::utils;
-use crate::{RbPolarsErr, RbResult, RbSeries};
+use crate::{RbErr, RbPolarsErr, RbResult, RbSeries};
 
 impl IntoValue for Wrap<AnyValue<'_>> {
     fn into_value_with(self, ruby: &Ruby) -> Value {
@@ -47,12 +47,20 @@ pub(crate) fn any_value_into_rb_object(av: AnyValue, ruby: &Ruby) -> Value {
             };
             s.into_value()
         }
+        AnyValue::CategoricalOwned(idx, rev, arr) | AnyValue::EnumOwned(idx, rev, arr) => {
+            let s = if arr.is_null() {
+                rev.get(idx)
+            } else {
+                unsafe { arr.deref_unchecked().value(idx as usize) }
+            };
+            s.into_value()
+        }
         AnyValue::Date(v) => utils().funcall("_to_ruby_date", (v,)).unwrap(),
         AnyValue::Datetime(v, time_unit, time_zone) => {
-            let time_unit = time_unit.to_ascii();
-            utils()
-                .funcall("_to_ruby_datetime", (v, time_unit, time_zone.clone()))
-                .unwrap()
+            datetime_to_rb_object(v, time_unit, time_zone)
+        }
+        AnyValue::DatetimeOwned(v, time_unit, time_zone) => {
+            datetime_to_rb_object(v, time_unit, time_zone.as_ref().map(AsRef::as_ref))
         }
         AnyValue::Duration(v, time_unit) => {
             let time_unit = time_unit.to_ascii();
@@ -66,11 +74,11 @@ pub(crate) fn any_value_into_rb_object(av: AnyValue, ruby: &Ruby) -> Value {
         AnyValue::StructOwned(payload) => struct_dict(payload.0.into_iter(), &payload.1),
         AnyValue::Object(v) => {
             let object = v.as_any().downcast_ref::<ObjectValue>().unwrap();
-            object.to_object()
+            object.to_value()
         }
         AnyValue::ObjectOwned(v) => {
             let object = v.0.as_any().downcast_ref::<ObjectValue>().unwrap();
-            object.to_object()
+            object.to_value()
         }
         AnyValue::Binary(v) => RString::from_slice(v).into_value(),
         AnyValue::BinaryOwned(v) => RString::from_slice(&v).into_value(),
@@ -80,6 +88,13 @@ pub(crate) fn any_value_into_rb_object(av: AnyValue, ruby: &Ruby) -> Value {
     }
 }
 
+fn datetime_to_rb_object(v: i64, tu: TimeUnit, tz: Option<&TimeZone>) -> Value {
+    let tu = tu.to_ascii();
+    utils()
+        .funcall("_to_ruby_datetime", (v, tu, tz.map(|v| v.to_string())))
+        .unwrap()
+}
+
 pub(crate) fn rb_object_to_any_value<'s>(ob: Value, strict: bool) -> RbResult<AnyValue<'s>> {
     // Conversion functions.
     fn get_null(_ob: Value, _strict: bool) -> RbResult<AnyValue<'static>> {
@@ -122,7 +137,10 @@ pub(crate) fn rb_object_to_any_value<'s>(ob: Value, strict: bool) -> RbResult<An
     fn get_list(ob: Value, _strict: bool) -> RbResult<AnyValue<'static>> {
         let v = RArray::from_value(ob).unwrap();
         if v.is_empty() {
-            Ok(AnyValue::List(Series::new_empty("", &DataType::Null)))
+            Ok(AnyValue::List(Series::new_empty(
+                PlSmallStr::EMPTY,
+                &DataType::Null,
+            )))
         } else {
             let list = v;
 
@@ -142,7 +160,7 @@ pub(crate) fn rb_object_to_any_value<'s>(ob: Value, strict: bool) -> RbResult<An
                 avs.push(Wrap::<AnyValue>::try_convert(item)?.0)
             }
 
-            let s = Series::from_any_values_and_dtype("", &avs, &dtype, true)
+            let s = Series::from_any_values_and_dtype(PlSmallStr::EMPTY, &avs, &dtype, true)
                 .map_err(RbPolarsErr::from)?;
             Ok(AnyValue::List(s))
         }
@@ -158,11 +176,10 @@ pub(crate) fn rb_object_to_any_value<'s>(ob: Value, strict: bool) -> RbResult<An
         let len = dict.len();
         let mut keys = Vec::with_capacity(len);
         let mut vals = Vec::with_capacity(len);
-        dict.foreach(|k: Value, v: Value| {
-            let key = String::try_convert(k)?;
-            let val = Wrap::<AnyValue>::try_convert(v)?.0;
+        dict.foreach(|key: String, val: Wrap<AnyValue>| {
+            let val = val.0;
             let dtype = DataType::from(&val);
-            keys.push(Field::new(&key, dtype));
+            keys.push(Field::new(key.into(), dtype));
             vals.push(val);
             Ok(ForEach::Continue)
         })?;
@@ -184,7 +201,7 @@ pub(crate) fn rb_object_to_any_value<'s>(ob: Value, strict: bool) -> RbResult<An
         let v = sec * 1_000_000_000 + nsec;
         // TODO support time zone when possible
         // https://github.com/pola-rs/polars/issues/9103
-        Ok(AnyValue::Datetime(v, TimeUnit::Nanoseconds, &None))
+        Ok(AnyValue::Datetime(v, TimeUnit::Nanoseconds, None))
     }
 
     fn get_datetime(ob: Value, _strict: bool) -> RbResult<AnyValue<'static>> {
@@ -193,7 +210,7 @@ pub(crate) fn rb_object_to_any_value<'s>(ob: Value, strict: bool) -> RbResult<An
         Ok(AnyValue::Datetime(
             sec * 1_000_000_000 + nsec,
             TimeUnit::Nanoseconds,
-            &None,
+            None,
         ))
     }
 
@@ -218,7 +235,9 @@ pub(crate) fn rb_object_to_any_value<'s>(ob: Value, strict: bool) -> RbResult<An
 
         let (sign, digits, _, exp): (i8, String, i32, i32) = ob.funcall("split", ()).unwrap();
         let (mut v, scale) = abs_decimal_from_digits(digits, exp).ok_or_else(|| {
-            RbPolarsErr::other("BigDecimal is too large to fit in Decimal128".into())
+            RbErr::from(RbPolarsErr::Other(
+                "BigDecimal is too large to fit in Decimal128".into(),
+            ))
         })?;
         if sign < 0 {
             // TODO better error
@@ -253,9 +272,6 @@ pub(crate) fn rb_object_to_any_value<'s>(ob: Value, strict: bool) -> RbResult<An
     } else if ob.is_kind_of(crate::rb_modules::bigdecimal()) {
         get_decimal(ob, strict)
     } else {
-        Err(RbPolarsErr::other(format!(
-            "object type not supported {:?}",
-            ob
-        )))
+        Err(RbPolarsErr::Other(format!("object type not supported {:?}", ob)).into())
     }
 }

data/ext/polars/src/conversion/chunked_array.rs

@@ -9,7 +9,7 @@ use crate::RbResult;
 impl TryConvert for Wrap<StringChunked> {
     fn try_convert(obj: Value) -> RbResult<Self> {
         let (seq, len) = get_rbseq(obj)?;
-        let mut builder = StringChunkedBuilder::new("", len);
+        let mut builder = StringChunkedBuilder::new(PlSmallStr::EMPTY, len);
 
         for res in seq.into_iter() {
             let item = res;
@@ -25,7 +25,7 @@ impl TryConvert for Wrap<StringChunked> {
 impl TryConvert for Wrap<BinaryChunked> {
     fn try_convert(obj: Value) -> RbResult<Self> {
         let (seq, len) = get_rbseq(obj)?;
-        let mut builder = BinaryChunkedBuilder::new("", len);
+        let mut builder = BinaryChunkedBuilder::new(PlSmallStr::EMPTY, len);
 
         for res in seq.into_iter() {
             let item = res;
@@ -90,7 +90,7 @@ impl IntoValue for Wrap<&DatetimeChunked> {
     fn into_value_with(self, _: &Ruby) -> Value {
         let utils = utils();
         let time_unit = Wrap(self.0.time_unit()).into_value();
-        let time_zone = self.0.time_zone().clone().into_value();
+        let time_zone = self.0.time_zone().as_deref().map(|v| v.into_value());
         let iter = self.0.into_iter().map(|opt_v| {
             opt_v.map(|v| {
                 utils