polars-df 0.14.0 → 0.16.0

data/LICENSE.txt

@@ -1,5 +1,6 @@
 Copyright (c) 2020 Ritchie Vink
 Copyright (c) 2022-2024 Andrew Kane
+Some portions Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -14,18 +14,17 @@ gem "polars-df"
 
 ## Getting Started
 
-This library follows the [Polars Python API](https://pola-rs.github.io/polars/py-polars/html/reference/index.html).
+This library follows the [Polars Python API](https://docs.pola.rs/api/python/stable/reference/index.html).
 
 ```ruby
-Polars.read_csv("iris.csv")
-  .lazy
+Polars.scan_csv("iris.csv")
   .filter(Polars.col("sepal_length") > 5)
   .group_by("species")
   .agg(Polars.all.sum)
   .collect
 ```
 
-You can follow [Polars tutorials](https://pola-rs.github.io/polars-book/user-guide/) and convert the code to Ruby in many cases. Feel free to open an issue if you run into problems.
+You can follow [Polars tutorials](https://docs.pola.rs/user-guide/getting-started/) and convert the code to Ruby in many cases. Feel free to open an issue if you run into problems.
 
 ## Reference
 
@@ -89,6 +88,15 @@ From Avro
 Polars.read_avro("file.avro")
 ```
 
+From Delta Lake (requires [deltalake-rb](https://github.com/ankane/delta-ruby)) [experimental, unreleased]
+
+```ruby
+Polars.read_delta("./table")
+
+# or lazily with
+Polars.scan_delta("./table")
+```
+
 From a hash
 
 ```ruby
@@ -337,6 +345,32 @@ Parquet
 df.write_parquet("file.parquet")
 ```
 
+JSON
+
+```ruby
+df.write_json("file.json")
+# or
+df.write_ndjson("file.ndjson")
+```
+
+Feather / Arrow IPC
+
+```ruby
+df.write_ipc("file.arrow")
+```
+
+Avro
+
+```ruby
+df.write_avro("file.avro")
+```
+
+Delta Lake [experimental, unreleased]
+
+```ruby
+df.write_delta("./table")
+```
+
 Numo array
 
 ```ruby

data/ext/polars/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "polars"
-version = "0.14.0"
+version = "0.16.0"
 license = "MIT"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"
@@ -12,16 +12,20 @@ crate-type = ["cdylib"]
 
 [dependencies]
 ahash = "0.8"
+arrow = { package = "polars-arrow", version = "=0.45.1" }
+bytes = "1"
 chrono = "0.4"
 either = "1.8"
 magnus = "0.7"
-polars-core = "=0.43.1"
-polars-parquet = "=0.43.1"
-polars-utils = "=0.43.1"
+polars-core = "=0.45.1"
+polars-plan = "=0.45.1"
+polars-parquet = "=0.45.1"
+polars-utils = "=0.45.1"
+regex = "1"
 serde_json = "1"
 
 [dependencies.polars]
-version = "=0.43.1"
+version = "=0.45.1"
 features = [
     "abs",
     "approx_unique",
@@ -30,7 +34,11 @@ features = [
     "array_count",
     "asof_join",
     "avro",
+    "aws",
+    "azure",
     "binary_encoding",
+    "business",
+    "cloud",
     "concat_str",
     "cov",
     "cross_join",
@@ -51,6 +59,8 @@ features = [
     "extract_jsonpath",
     "find_many",
     "fmt",
+    "gcp",
+    "http",
     "interpolate",
     "ipc",
     "ipc_streaming",

data/ext/polars/src/batched_csv.rs

@@ -34,8 +34,7 @@ impl RbBatchedCsv {
         let n_threads = Option::<usize>::try_convert(arguments[11])?;
         let path = PathBuf::try_convert(arguments[12])?;
         let overwrite_dtype = Option::<Vec<(String, Wrap<DataType>)>>::try_convert(arguments[13])?;
-        // TODO fix
-        let overwrite_dtype_slice = Option::<Vec<Wrap<DataType>>>::None; // Option::<Vec<Wrap<DataType>>>::try_convert(arguments[14])?;
+        let overwrite_dtype_slice = Option::<Vec<Wrap<DataType>>>::try_convert(arguments[14])?;
         let low_memory = bool::try_convert(arguments[15])?;
         let comment_prefix = Option::<String>::try_convert(arguments[16])?;
         let quote_char = Option::<String>::try_convert(arguments[17])?;
@@ -44,11 +43,10 @@ impl RbBatchedCsv {
         let try_parse_dates = bool::try_convert(arguments[20])?;
         let skip_rows_after_header = usize::try_convert(arguments[21])?;
         let row_index = Option::<(String, IdxSize)>::try_convert(arguments[22])?;
-        let sample_size = usize::try_convert(arguments[23])?;
-        let eol_char = String::try_convert(arguments[24])?;
-        let raise_if_empty = bool::try_convert(arguments[25])?;
-        let truncate_ragged_lines = bool::try_convert(arguments[26])?;
-        let decimal_comma = bool::try_convert(arguments[27])?;
+        let eol_char = String::try_convert(arguments[23])?;
+        let raise_if_empty = bool::try_convert(arguments[24])?;
+        let truncate_ragged_lines = bool::try_convert(arguments[25])?;
+        let decimal_comma = bool::try_convert(arguments[26])?;
         // end arguments
 
         let null_values = null_values.map(|w| w.0);
@@ -84,7 +82,7 @@ impl RbBatchedCsv {
                 .collect::<Vec<_>>()
         });
 
-        let file = std::fs::File::open(path).map_err(RbPolarsErr::io)?;
+        let file = std::fs::File::open(path).map_err(RbPolarsErr::from)?;
         let reader = Box::new(file) as Box<dyn MmapBytesReader>;
         let reader = CsvReadOptions::default()
             .with_infer_schema_length(infer_schema_length)
@@ -101,7 +99,6 @@ impl RbBatchedCsv {
             .with_low_memory(low_memory)
             .with_skip_rows_after_header(skip_rows_after_header)
             .with_row_index(row_index)
-            .with_sample_size(sample_size)
             .with_raise_if_empty(raise_if_empty)
             .with_parse_options(
                 CsvParseOptions::default()
@@ -132,7 +129,7 @@ impl RbBatchedCsv {
         let batches = reader
             .borrow()
             .lock()
-            .map_err(|e| RbPolarsErr::other(e.to_string()))?
+            .map_err(|e| RbPolarsErr::Other(e.to_string()))?
             .next_batches(n)
             .map_err(RbPolarsErr::from)?;
 

data/ext/polars/src/conversion/any_value.rs

@@ -7,9 +7,9 @@ use polars_core::utils::any_values_to_supertype_and_n_dtypes;
 
 use super::{struct_dict, ObjectValue, Wrap};
 
-use crate::error::RbOverflowError;
+use crate::exceptions::RbOverflowError;
 use crate::rb_modules::utils;
-use crate::{RbPolarsErr, RbResult, RbSeries};
+use crate::{RbErr, RbPolarsErr, RbResult, RbSeries};
 
 impl IntoValue for Wrap<AnyValue<'_>> {
     fn into_value_with(self, ruby: &Ruby) -> Value {
@@ -47,15 +47,20 @@ pub(crate) fn any_value_into_rb_object(av: AnyValue, ruby: &Ruby) -> Value {
             };
             s.into_value()
         }
+        AnyValue::CategoricalOwned(idx, rev, arr) | AnyValue::EnumOwned(idx, rev, arr) => {
+            let s = if arr.is_null() {
+                rev.get(idx)
+            } else {
+                unsafe { arr.deref_unchecked().value(idx as usize) }
+            };
+            s.into_value()
+        }
         AnyValue::Date(v) => utils().funcall("_to_ruby_date", (v,)).unwrap(),
         AnyValue::Datetime(v, time_unit, time_zone) => {
-            let time_unit = time_unit.to_ascii();
-            utils()
-                .funcall(
-                    "_to_ruby_datetime",
-                    (v, time_unit, time_zone.as_ref().map(|v| v.to_string())),
-                )
-                .unwrap()
+            datetime_to_rb_object(v, time_unit, time_zone)
+        }
+        AnyValue::DatetimeOwned(v, time_unit, time_zone) => {
+            datetime_to_rb_object(v, time_unit, time_zone.as_ref().map(AsRef::as_ref))
         }
         AnyValue::Duration(v, time_unit) => {
             let time_unit = time_unit.to_ascii();
@@ -69,11 +74,11 @@ pub(crate) fn any_value_into_rb_object(av: AnyValue, ruby: &Ruby) -> Value {
         AnyValue::StructOwned(payload) => struct_dict(payload.0.into_iter(), &payload.1),
         AnyValue::Object(v) => {
             let object = v.as_any().downcast_ref::<ObjectValue>().unwrap();
-            object.to_object()
+            object.to_value()
         }
         AnyValue::ObjectOwned(v) => {
             let object = v.0.as_any().downcast_ref::<ObjectValue>().unwrap();
-            object.to_object()
+            object.to_value()
         }
         AnyValue::Binary(v) => RString::from_slice(v).into_value(),
         AnyValue::BinaryOwned(v) => RString::from_slice(&v).into_value(),
@@ -83,6 +88,13 @@ pub(crate) fn any_value_into_rb_object(av: AnyValue, ruby: &Ruby) -> Value {
     }
 }
 
+fn datetime_to_rb_object(v: i64, tu: TimeUnit, tz: Option<&TimeZone>) -> Value {
+    let tu = tu.to_ascii();
+    utils()
+        .funcall("_to_ruby_datetime", (v, tu, tz.map(|v| v.to_string())))
+        .unwrap()
+}
+
 pub(crate) fn rb_object_to_any_value<'s>(ob: Value, strict: bool) -> RbResult<AnyValue<'s>> {
     // Conversion functions.
     fn get_null(_ob: Value, _strict: bool) -> RbResult<AnyValue<'static>> {
@@ -164,9 +176,8 @@ pub(crate) fn rb_object_to_any_value<'s>(ob: Value, strict: bool) -> RbResult<An
         let len = dict.len();
         let mut keys = Vec::with_capacity(len);
         let mut vals = Vec::with_capacity(len);
-        dict.foreach(|k: Value, v: Value| {
-            let key = String::try_convert(k)?;
-            let val = Wrap::<AnyValue>::try_convert(v)?.0;
+        dict.foreach(|key: String, val: Wrap<AnyValue>| {
+            let val = val.0;
             let dtype = DataType::from(&val);
             keys.push(Field::new(key.into(), dtype));
             vals.push(val);
@@ -190,7 +201,7 @@ pub(crate) fn rb_object_to_any_value<'s>(ob: Value, strict: bool) -> RbResult<An
         let v = sec * 1_000_000_000 + nsec;
         // TODO support time zone when possible
         // https://github.com/pola-rs/polars/issues/9103
-        Ok(AnyValue::Datetime(v, TimeUnit::Nanoseconds, &None))
+        Ok(AnyValue::Datetime(v, TimeUnit::Nanoseconds, None))
     }
 
     fn get_datetime(ob: Value, _strict: bool) -> RbResult<AnyValue<'static>> {
@@ -199,7 +210,7 @@ pub(crate) fn rb_object_to_any_value<'s>(ob: Value, strict: bool) -> RbResult<An
         Ok(AnyValue::Datetime(
             sec * 1_000_000_000 + nsec,
             TimeUnit::Nanoseconds,
-            &None,
+            None,
         ))
     }
 
@@ -224,7 +235,9 @@ pub(crate) fn rb_object_to_any_value<'s>(ob: Value, strict: bool) -> RbResult<An
 
         let (sign, digits, _, exp): (i8, String, i32, i32) = ob.funcall("split", ()).unwrap();
         let (mut v, scale) = abs_decimal_from_digits(digits, exp).ok_or_else(|| {
-            RbPolarsErr::other("BigDecimal is too large to fit in Decimal128".into())
+            RbErr::from(RbPolarsErr::Other(
+                "BigDecimal is too large to fit in Decimal128".into(),
+            ))
         })?;
         if sign < 0 {
             // TODO better error
@@ -259,9 +272,6 @@ pub(crate) fn rb_object_to_any_value<'s>(ob: Value, strict: bool) -> RbResult<An
     } else if ob.is_kind_of(crate::rb_modules::bigdecimal()) {
         get_decimal(ob, strict)
     } else {
-        Err(RbPolarsErr::other(format!(
-            "object type not supported {:?}",
-            ob
-        )))
+        Err(RbPolarsErr::Other(format!("object type not supported {:?}", ob)).into())
     }
 }