polars-df 0.15.0 → 0.16.0

data/README.md

@@ -14,7 +14,7 @@ gem "polars-df"
 
 ## Getting Started
 
-This library follows the [Polars Python API](https://pola-rs.github.io/polars/py-polars/html/reference/index.html).
+This library follows the [Polars Python API](https://docs.pola.rs/api/python/stable/reference/index.html).
 
 ```ruby
 Polars.scan_csv("iris.csv")
@@ -24,7 +24,7 @@ Polars.scan_csv("iris.csv")
   .collect
 ```
 
-You can follow [Polars tutorials](https://pola-rs.github.io/polars-book/user-guide/) and convert the code to Ruby in many cases. Feel free to open an issue if you run into problems.
+You can follow [Polars tutorials](https://docs.pola.rs/user-guide/getting-started/) and convert the code to Ruby in many cases. Feel free to open an issue if you run into problems.
 
 ## Reference
 
@@ -88,6 +88,15 @@ From Avro
 Polars.read_avro("file.avro")
 ```
 
+From Delta Lake (requires [deltalake-rb](https://github.com/ankane/delta-ruby)) [experimental, unreleased]
+
+```ruby
+Polars.read_delta("./table")
+
+# or lazily with
+Polars.scan_delta("./table")
+```
+
 From a hash
 
 ```ruby
@@ -336,6 +345,32 @@ Parquet
 df.write_parquet("file.parquet")
 ```
 
+JSON
+
+```ruby
+df.write_json("file.json")
+# or
+df.write_ndjson("file.ndjson")
+```
+
+Feather / Arrow IPC
+
+```ruby
+df.write_ipc("file.arrow")
+```
+
+Avro
+
+```ruby
+df.write_avro("file.avro")
+```
+
+Delta Lake [experimental, unreleased]
+
+```ruby
+df.write_delta("./table")
+```
+
 Numo array
 
 ```ruby

data/ext/polars/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "polars"
-version = "0.15.0"
+version = "0.16.0"
 license = "MIT"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"
@@ -12,20 +12,20 @@ crate-type = ["cdylib"]
 
 [dependencies]
 ahash = "0.8"
-arrow = { package = "polars-arrow", version = "=0.44.2" }
+arrow = { package = "polars-arrow", version = "=0.45.1" }
 bytes = "1"
 chrono = "0.4"
 either = "1.8"
 magnus = "0.7"
-polars-core = "=0.44.2"
-polars-plan = "=0.44.2"
-polars-parquet = "=0.44.2"
-polars-utils = "=0.44.2"
+polars-core = "=0.45.1"
+polars-plan = "=0.45.1"
+polars-parquet = "=0.45.1"
+polars-utils = "=0.45.1"
 regex = "1"
 serde_json = "1"
 
 [dependencies.polars]
-version = "=0.44.2"
+version = "=0.45.1"
 features = [
     "abs",
     "approx_unique",

data/ext/polars/src/conversion/mod.rs

@@ -15,7 +15,6 @@ use polars::chunked_array::object::PolarsObjectSafe;
 use polars::chunked_array::ops::{FillNullLimit, FillNullStrategy};
 use polars::datatypes::AnyValue;
 use polars::frame::row::Row;
-use polars::frame::NullStrategy;
 use polars::io::avro::AvroCompression;
 use polars::io::cloud::CloudOptions;
 use polars::prelude::*;
@@ -23,6 +22,7 @@ use polars::series::ops::NullBehavior;
 use polars_core::utils::arrow::array::Array;
 use polars_core::utils::materialize_dyn_int;
 use polars_plan::plans::ScanSources;
+use polars_utils::mmap::MemSlice;
 use polars_utils::total_ord::{TotalEq, TotalHash};
 
 use crate::file::{get_ruby_scan_source_input, RubyScanSourceInput};
@@ -304,29 +304,31 @@ impl TryConvert for Wrap<DataType> {
         let dtype = if ob.is_kind_of(class::class()) {
             let name = ob.funcall::<_, _, String>("name", ())?;
             match name.as_str() {
-                "Polars::UInt8" => DataType::UInt8,
-                "Polars::UInt16" => DataType::UInt16,
-                "Polars::UInt32" => DataType::UInt32,
-                "Polars::UInt64" => DataType::UInt64,
                 "Polars::Int8" => DataType::Int8,
                 "Polars::Int16" => DataType::Int16,
                 "Polars::Int32" => DataType::Int32,
                 "Polars::Int64" => DataType::Int64,
+                "Polars::UInt8" => DataType::UInt8,
+                "Polars::UInt16" => DataType::UInt16,
+                "Polars::UInt32" => DataType::UInt32,
+                "Polars::UInt64" => DataType::UInt64,
+                "Polars::Float32" => DataType::Float32,
+                "Polars::Float64" => DataType::Float64,
+                "Polars::Boolean" => DataType::Boolean,
                 "Polars::String" => DataType::String,
                 "Polars::Binary" => DataType::Binary,
-                "Polars::Boolean" => DataType::Boolean,
                 "Polars::Categorical" => DataType::Categorical(None, Default::default()),
                 "Polars::Enum" => DataType::Enum(None, Default::default()),
                 "Polars::Date" => DataType::Date,
-                "Polars::Datetime" => DataType::Datetime(TimeUnit::Microseconds, None),
                 "Polars::Time" => DataType::Time,
+                "Polars::Datetime" => DataType::Datetime(TimeUnit::Microseconds, None),
                 "Polars::Duration" => DataType::Duration(TimeUnit::Microseconds),
                 "Polars::Decimal" => DataType::Decimal(None, None),
-                "Polars::Float32" => DataType::Float32,
-                "Polars::Float64" => DataType::Float64,
-                "Polars::Object" => DataType::Object(OBJECT_NAME, None),
                 "Polars::List" => DataType::List(Box::new(DataType::Null)),
+                "Polars::Array" => DataType::Array(Box::new(DataType::Null), 0),
+                "Polars::Struct" => DataType::Struct(vec![]),
                 "Polars::Null" => DataType::Null,
+                "Polars::Object" => DataType::Object(OBJECT_NAME, None),
                 "Polars::Unknown" => DataType::Unknown(Default::default()),
                 dt => {
                     return Err(RbValueError::new_err(format!(
@@ -345,9 +347,11 @@ impl TryConvert for Wrap<DataType> {
                 "Polars::UInt16" => DataType::UInt16,
                 "Polars::UInt32" => DataType::UInt32,
                 "Polars::UInt64" => DataType::UInt64,
+                "Polars::Float32" => DataType::Float32,
+                "Polars::Float64" => DataType::Float64,
+                "Polars::Boolean" => DataType::Boolean,
                 "Polars::String" => DataType::String,
                 "Polars::Binary" => DataType::Binary,
-                "Polars::Boolean" => DataType::Boolean,
                 "Polars::Categorical" => {
                     let ordering = ob
                         .funcall::<_, _, Wrap<CategoricalOrdering>>("ordering", ())?
@@ -363,21 +367,17 @@ impl TryConvert for Wrap<DataType> {
                 }
                 "Polars::Date" => DataType::Date,
                 "Polars::Time" => DataType::Time,
-                "Polars::Float32" => DataType::Float32,
-                "Polars::Float64" => DataType::Float64,
-                "Polars::Null" => DataType::Null,
-                "Polars::Unknown" => DataType::Unknown(Default::default()),
-                "Polars::Duration" => {
-                    let time_unit: Value = ob.funcall("time_unit", ()).unwrap();
-                    let time_unit = Wrap::<TimeUnit>::try_convert(time_unit)?.0;
-                    DataType::Duration(time_unit)
-                }
                 "Polars::Datetime" => {
                     let time_unit: Value = ob.funcall("time_unit", ()).unwrap();
                     let time_unit = Wrap::<TimeUnit>::try_convert(time_unit)?.0;
                     let time_zone: Option<String> = ob.funcall("time_zone", ())?;
                     DataType::Datetime(time_unit, time_zone.as_deref().map(|x| x.into()))
                 }
+                "Polars::Duration" => {
+                    let time_unit: Value = ob.funcall("time_unit", ()).unwrap();
+                    let time_unit = Wrap::<TimeUnit>::try_convert(time_unit)?.0;
+                    DataType::Duration(time_unit)
+                }
                 "Polars::Decimal" => {
                     let precision = ob.funcall("precision", ())?;
                     let scale = ob.funcall("scale", ())?;
@@ -388,6 +388,13 @@ impl TryConvert for Wrap<DataType> {
                     let inner = Wrap::<DataType>::try_convert(inner)?;
                     DataType::List(Box::new(inner.0))
                 }
+                "Polars::Array" => {
+                    let inner: Value = ob.funcall("inner", ()).unwrap();
+                    let size: Value = ob.funcall("size", ()).unwrap();
+                    let inner = Wrap::<DataType>::try_convert(inner)?;
+                    let size = usize::try_convert(size)?;
+                    DataType::Array(Box::new(inner.0), size)
+                }
                 "Polars::Struct" => {
                     let arr: RArray = ob.funcall("fields", ())?;
                     let mut fields = Vec::with_capacity(arr.len());
@@ -396,6 +403,9 @@ impl TryConvert for Wrap<DataType> {
                     }
                     DataType::Struct(fields)
                 }
+                "Polars::Null" => DataType::Null,
+                "Object" => DataType::Object(OBJECT_NAME, None),
+                "Polars::Unknown" => DataType::Unknown(Default::default()),
                 dt => {
                     return Err(RbTypeError::new_err(format!(
                         "A {dt} object is not a correct polars DataType. \
@@ -501,7 +511,7 @@ impl TryConvert for Wrap<ScanSources> {
         enum MutableSources {
             Paths(Vec<PathBuf>),
             Files(Vec<File>),
-            Buffers(Vec<bytes::Bytes>),
+            Buffers(Vec<MemSlice>),
         }
 
         let num_items = list.len();

data/ext/polars/src/dataframe/general.rs

@@ -1,6 +1,5 @@
 use either::Either;
 use magnus::{prelude::*, typed_data::Obj, IntoValue, RArray, Value};
-use polars::frame::NullStrategy;
 use polars::prelude::pivot::{pivot, pivot_stable};
 use polars::prelude::*;
 
@@ -158,7 +157,7 @@ impl RbDataFrame {
     }
 
     pub fn n_chunks(&self) -> usize {
-        self.df.borrow().n_chunks()
+        self.df.borrow().first_col_n_chunks()
     }
 
     pub fn shape(&self) -> (usize, usize) {
@@ -410,52 +409,6 @@ impl RbDataFrame {
         self.df.borrow().clone().lazy().into()
     }
 
-    pub fn max_horizontal(&self) -> RbResult<Option<RbSeries>> {
-        let s = self
-            .df
-            .borrow()
-            .max_horizontal()
-            .map_err(RbPolarsErr::from)?;
-        Ok(s.map(|s| s.take_materialized_series().into()))
-    }
-
-    pub fn min_horizontal(&self) -> RbResult<Option<RbSeries>> {
-        let s = self
-            .df
-            .borrow()
-            .min_horizontal()
-            .map_err(RbPolarsErr::from)?;
-        Ok(s.map(|s| s.take_materialized_series().into()))
-    }
-
-    pub fn sum_horizontal(&self, ignore_nulls: bool) -> RbResult<Option<RbSeries>> {
-        let null_strategy = if ignore_nulls {
-            NullStrategy::Ignore
-        } else {
-            NullStrategy::Propagate
-        };
-        let s = self
-            .df
-            .borrow()
-            .sum_horizontal(null_strategy)
-            .map_err(RbPolarsErr::from)?;
-        Ok(s.map(|s| s.into()))
-    }
-
-    pub fn mean_horizontal(&self, ignore_nulls: bool) -> RbResult<Option<RbSeries>> {
-        let null_strategy = if ignore_nulls {
-            NullStrategy::Ignore
-        } else {
-            NullStrategy::Propagate
-        };
-        let s = self
-            .df
-            .borrow()
-            .mean_horizontal(null_strategy)
-            .map_err(RbPolarsErr::from)?;
-        Ok(s.map(|s| s.into()))
-    }
-
     pub fn to_dummies(
         &self,
         columns: Option<Vec<String>>,

data/ext/polars/src/dataframe/io.rs

@@ -2,15 +2,13 @@ use magnus::{prelude::*, Value};
 use polars::io::avro::AvroCompression;
 use polars::io::RowIndex;
 use polars::prelude::*;
-use polars_utils::mmap::ensure_not_mapped;
 use std::io::BufWriter;
 use std::num::NonZeroUsize;
 
 use super::*;
 use crate::conversion::*;
 use crate::file::{
-    get_either_file, get_file_like, get_mmap_bytes_reader, get_mmap_bytes_reader_and_path,
-    read_if_bytesio, EitherRustRubyFile,
+    get_file_like, get_mmap_bytes_reader, get_mmap_bytes_reader_and_path, read_if_bytesio,
 };
 use crate::{RbPolarsErr, RbResult};
 
@@ -330,13 +328,19 @@ impl RbDataFrame {
         rb_f: Value,
         compression: Wrap<Option<IpcCompression>>,
         compat_level: RbCompatLevel,
+        cloud_options: Option<Vec<(String, String)>>,
+        retries: usize,
     ) -> RbResult<()> {
-        let either = get_either_file(rb_f, true)?;
-        if let EitherRustRubyFile::Rust(ref f) = either {
-            ensure_not_mapped(f).map_err(RbPolarsErr::from)?;
-        }
-        let mut buf = either.into_dyn();
-        IpcWriter::new(&mut buf)
+        let cloud_options = if let Ok(path) = String::try_convert(rb_f) {
+            let cloud_options = parse_cloud_options(&path, cloud_options.unwrap_or_default())?;
+            Some(cloud_options.with_max_retries(retries))
+        } else {
+            None
+        };
+
+        let f = crate::file::try_get_writeable(rb_f, cloud_options.as_ref())?;
+
+        IpcWriter::new(f)
             .with_compression(compression.0)
             .with_compat_level(compat_level.0)
             .finish(&mut self.df.borrow_mut())

data/ext/polars/src/expr/general.rs

@@ -271,6 +271,7 @@ impl RbExpr {
                 nulls_last,
                 multithreaded: true,
                 maintain_order: false,
+                limit: None,
             })
             .into()
     }
@@ -283,6 +284,7 @@ impl RbExpr {
                 nulls_last,
                 multithreaded: true,
                 maintain_order: false,
+                limit: None,
             })
             .into()
     }
@@ -363,6 +365,7 @@ impl RbExpr {
                     nulls_last,
                     multithreaded,
                     maintain_order,
+                    limit: None,
                 },
             )
             .into())

data/ext/polars/src/expr/meta.rs

@@ -84,13 +84,17 @@ impl RbExpr {
         self.inner.clone().meta()._into_selector().into()
     }
 
-    pub fn meta_tree_format(&self) -> RbResult<String> {
+    fn compute_tree_format(&self, display_as_dot: bool) -> RbResult<String> {
         let e = self
             .inner
             .clone()
             .meta()
-            .into_tree_formatter()
+            .into_tree_formatter(display_as_dot)
             .map_err(RbPolarsErr::from)?;
         Ok(format!("{e}"))
     }
+
+    pub fn meta_tree_format(&self) -> RbResult<String> {
+        self.compute_tree_format(false)
+    }
 }

data/ext/polars/src/file.rs

@@ -4,7 +4,9 @@ use std::io::{Cursor, Read, Seek, SeekFrom, Write};
 use std::path::PathBuf;
 
 use magnus::{exception, prelude::*, Error, RString, Value};
+use polars::io::cloud::CloudOptions;
 use polars::io::mmap::MmapBytesReader;
+use polars_utils::mmap::MemSlice;
 
 use crate::error::RbPolarsErr;
 use crate::prelude::resolve_homedir;
@@ -141,10 +143,17 @@ impl EitherRustRubyFile {
             EitherRustRubyFile::Rust(f) => Box::new(f),
         }
     }
+
+    pub fn into_dyn_writeable(self) -> Box<dyn Write> {
+        match self {
+            EitherRustRubyFile::Rb(f) => Box::new(f),
+            EitherRustRubyFile::Rust(f) => Box::new(f),
+        }
+    }
 }
 
 pub enum RubyScanSourceInput {
-    Buffer(bytes::Bytes),
+    Buffer(MemSlice),
     Path(PathBuf),
     #[allow(dead_code)]
     File(File),
@@ -156,7 +165,9 @@ pub fn get_ruby_scan_source_input(rb_f: Value, write: bool) -> RbResult<RubyScan
         Ok(RubyScanSourceInput::Path(file_path))
     } else {
         let f = RbFileLikeObject::with_requirements(rb_f, !write, write, !write)?;
-        Ok(RubyScanSourceInput::Buffer(f.as_bytes()))
+        Ok(RubyScanSourceInput::Buffer(MemSlice::from_bytes(
+            f.as_bytes(),
+        )))
     }
 }
 
@@ -167,7 +178,7 @@ pub fn get_either_file(rb_f: Value, truncate: bool) -> RbResult<EitherRustRubyFi
     if let Ok(rstring) = RString::try_convert(rb_f) {
         let s = unsafe { rstring.as_str() }?;
         let file_path = std::path::Path::new(&s);
-        let file_path = resolve_homedir(file_path);
+        let file_path = resolve_homedir(&file_path);
         let f = if truncate {
             File::create(file_path).map_err(RbPolarsErr::from)?
         } else {
@@ -212,3 +223,10 @@ pub fn get_mmap_bytes_reader_and_path<'a>(
         }
     }
 }
+
+pub fn try_get_writeable(
+    rb_f: Value,
+    _cloud_options: Option<&CloudOptions>,
+) -> RbResult<Box<dyn Write>> {
+    Ok(get_either_file(rb_f, true)?.into_dyn_writeable())
+}

data/ext/polars/src/functions/aggregation.rs

@@ -28,14 +28,14 @@ pub fn min_horizontal(exprs: RArray) -> RbResult<RbExpr> {
     Ok(e.into())
 }
 
-pub fn sum_horizontal(exprs: RArray) -> RbResult<RbExpr> {
+pub fn sum_horizontal(exprs: RArray, ignore_nulls: bool) -> RbResult<RbExpr> {
     let exprs = rb_exprs_to_exprs(exprs)?;
-    let e = dsl::sum_horizontal(exprs).map_err(RbPolarsErr::from)?;
+    let e = dsl::sum_horizontal(exprs, ignore_nulls).map_err(RbPolarsErr::from)?;
     Ok(e.into())
 }
 
-pub fn mean_horizontal(exprs: RArray) -> RbResult<RbExpr> {
+pub fn mean_horizontal(exprs: RArray, ignore_nulls: bool) -> RbResult<RbExpr> {
     let exprs = rb_exprs_to_exprs(exprs)?;
-    let e = dsl::mean_horizontal(exprs).map_err(RbPolarsErr::from)?;
+    let e = dsl::mean_horizontal(exprs, ignore_nulls).map_err(RbPolarsErr::from)?;
     Ok(e.into())
 }

data/ext/polars/src/functions/io.rs

@@ -1,34 +1,55 @@
+use std::io::BufReader;
+
 use magnus::{RHash, Value};
+use polars::prelude::ArrowSchema;
+use polars_core::datatypes::create_enum_dtype;
+use polars_core::export::arrow::array::Utf8ViewArray;
 
 use crate::conversion::Wrap;
-use crate::file::get_file_like;
-use crate::prelude::DataType;
+use crate::file::{get_either_file, EitherRustRubyFile};
+use crate::prelude::ArrowDataType;
 use crate::{RbPolarsErr, RbResult};
 
 pub fn read_ipc_schema(rb_f: Value) -> RbResult<RHash> {
     use polars_core::export::arrow::io::ipc::read::read_file_metadata;
-    let mut r = get_file_like(rb_f, false)?;
-    let metadata = read_file_metadata(&mut r).map_err(RbPolarsErr::from)?;
+    let metadata = match get_either_file(rb_f, false)? {
+        EitherRustRubyFile::Rust(r) => {
+            read_file_metadata(&mut BufReader::new(r)).map_err(RbPolarsErr::from)?
+        }
+        EitherRustRubyFile::Rb(mut r) => read_file_metadata(&mut r).map_err(RbPolarsErr::from)?,
+    };
 
     let dict = RHash::new();
-    for field in metadata.schema.iter_values() {
-        let dt: Wrap<DataType> = Wrap((&field.dtype).into());
-        dict.aset(field.name.as_str(), dt)?;
-    }
+    fields_to_rbdict(&metadata.schema, &dict)?;
     Ok(dict)
 }
 
 pub fn read_parquet_schema(rb_f: Value) -> RbResult<RHash> {
     use polars_parquet::read::{infer_schema, read_metadata};
 
-    let mut r = get_file_like(rb_f, false)?;
-    let metadata = read_metadata(&mut r).map_err(RbPolarsErr::from)?;
+    let metadata = match get_either_file(rb_f, false)? {
+        EitherRustRubyFile::Rust(r) => {
+            read_metadata(&mut BufReader::new(r)).map_err(RbPolarsErr::from)?
+        }
+        EitherRustRubyFile::Rb(mut r) => read_metadata(&mut r).map_err(RbPolarsErr::from)?,
+    };
     let arrow_schema = infer_schema(&metadata).map_err(RbPolarsErr::from)?;
 
     let dict = RHash::new();
-    for field in arrow_schema.iter_values() {
-        let dt: Wrap<DataType> = Wrap((&field.dtype).into());
+    fields_to_rbdict(&arrow_schema, &dict)?;
+    Ok(dict)
+}
+
+fn fields_to_rbdict(schema: &ArrowSchema, dict: &RHash) -> RbResult<()> {
+    for field in schema.iter_values() {
+        let dt = if field.is_enum() {
+            Wrap(create_enum_dtype(Utf8ViewArray::new_empty(
+                ArrowDataType::Utf8View,
+            )))
+        } else {
+            Wrap(polars::prelude::DataType::from_arrow_field(field))
+        };
         dict.aset(field.name.as_str(), dt)?;
     }
-    Ok(dict)
+    Ok(())
 }

data/ext/polars/src/functions/lazy.rs

@@ -70,6 +70,7 @@ pub fn arg_sort_by(
             nulls_last,
             multithreaded,
             maintain_order,
+            limit: None,
         },
     )
     .into())
@@ -320,8 +321,8 @@ pub fn lit(value: Value, allow_object: bool) -> RbResult<RbExpr> {
     }
 }
 
-pub fn pearson_corr(a: &RbExpr, b: &RbExpr, ddof: u8) -> RbExpr {
-    dsl::pearson_corr(a.inner.clone(), b.inner.clone(), ddof).into()
+pub fn pearson_corr(a: &RbExpr, b: &RbExpr) -> RbExpr {
+    dsl::pearson_corr(a.inner.clone(), b.inner.clone()).into()
 }
 
 pub fn repeat(value: &RbExpr, n: &RbExpr, dtype: Option<Wrap<DataType>>) -> RbResult<RbExpr> {
@@ -345,8 +346,8 @@ pub fn repeat(value: &RbExpr, n: &RbExpr, dtype: Option<Wrap<DataType>>) -> RbRe
     Ok(dsl::repeat(value, n).into())
 }
 
-pub fn spearman_rank_corr(a: &RbExpr, b: &RbExpr, ddof: u8, propagate_nans: bool) -> RbExpr {
-    dsl::spearman_rank_corr(a.inner.clone(), b.inner.clone(), ddof, propagate_nans).into()
+pub fn spearman_rank_corr(a: &RbExpr, b: &RbExpr, propagate_nans: bool) -> RbExpr {
+    dsl::spearman_rank_corr(a.inner.clone(), b.inner.clone(), propagate_nans).into()
 }
 
 pub fn sql_expr(sql: String) -> RbResult<RbExpr> {

data/ext/polars/src/functions/meta.rs

@@ -11,7 +11,7 @@ pub fn get_index_type() -> Value {
     Wrap(IDX_DTYPE).into_value()
 }
 
-pub fn threadpool_size() -> usize {
+pub fn thread_pool_size() -> usize {
     POOL.current_num_threads()
 }
 

data/ext/polars/src/interop/arrow/to_ruby.rs

@@ -9,7 +9,7 @@ use polars_core::utils::arrow;
 
 use crate::RbResult;
 
-#[magnus::wrap(class = "Polars::RbArrowArrayStream")]
+#[magnus::wrap(class = "Polars::ArrowArrayStream")]
 pub struct RbArrowArrayStream {
     stream: ffi::ArrowArrayStream,
 }
@@ -47,7 +47,7 @@ impl DataFrameStreamIterator {
                 .collect(),
             dtype,
             idx: 0,
-            n_chunks: df.n_chunks(),
+            n_chunks: df.first_col_n_chunks(),
         }
     }