polars-df 0.2.5 → 0.3.0

data/Cargo.toml

@@ -4,7 +4,7 @@ members = ["ext/polars"]
 [patch.crates-io]
 jsonpath_lib = { git = "https://github.com/ritchie46/jsonpath", rev = "24eaf0b4416edff38a4d1b6b17bc4b9f3f047b4b" }
 halfbrown = { git = "https://github.com/Licenser/halfbrown", rev = "952023c5dd6461b009bb5ba66b9aa979bd75949f" }
-arrow2 = { git = "https://github.com/ankane/arrow2", rev = "9f36b2b97446e6dd495473e4361a70d863ac8027" }
+arrow2 = { git = "https://github.com/ankane/arrow2", rev = "ef0270922a217070ba9942567c0ff3263ae8c531" }
 
 [profile.release]
 strip = true

data/ext/polars/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "polars"
-version = "0.2.4"
+version = "0.3.0"
 license = "MIT"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"
@@ -11,18 +11,19 @@ crate-type = ["cdylib"]
 
 [dependencies]
 ahash = "0.8"
-magnus = "0.4"
-polars-core = "0.26.1"
+magnus = "0.5"
+polars-core = "0.27.0"
 serde_json = "1"
 
 [dependencies.polars]
-version = "0.26.1"
+version = "0.27.0"
 features = [
     "abs",
     "arange",
     "arg_where",
     "asof_join",
     "avro",
+    "binary_encoding",
     "concat_str",
     "cse",
     "csv-file",

data/ext/polars/src/apply/dataframe.rs

@@ -1,4 +1,4 @@
-use magnus::{class, RArray, TryConvert, Value};
+use magnus::{class, IntoValue, RArray, TryConvert, Value};
 use polars::prelude::*;
 use polars_core::frame::row::{rows_to_schema_first_non_null, Row};
 use polars_core::series::SeriesIter;
@@ -27,7 +27,7 @@ pub fn apply_lambda_unknown<'a>(
 
     for _ in 0..df.height() {
         let iter = iters.iter_mut().map(|it| Wrap(it.next().unwrap()));
-        let arg = (iter.collect::<Vec<Wrap<AnyValue>>>(),);
+        let arg = (RArray::from_iter(iter),);
         let out: Value = lambda.funcall("call", arg)?;
 
         if out.is_nil() {
@@ -141,7 +141,7 @@ where
     let mut iters = get_iters_skip(df, init_null_count + skip);
     ((init_null_count + skip)..df.height()).map(move |_| {
         let iter = iters.iter_mut().map(|it| Wrap(it.next().unwrap()));
-        let tpl = (iter.collect::<Vec<Wrap<AnyValue>>>(),);
+        let tpl = (RArray::from_iter(iter),);
         match lambda.funcall::<_, _, Value>("call", tpl) {
             Ok(val) => val.try_convert::<T>().ok(),
             Err(e) => panic!("ruby function failed {}", e),
@@ -158,7 +158,7 @@ pub fn apply_lambda_with_primitive_out_type<D>(
 ) -> ChunkedArray<D>
 where
     D: RbArrowPrimitiveType,
-    D::Native: Into<Value> + TryConvert,
+    D::Native: IntoValue + TryConvert,
 {
     let skip = usize::from(first_value.is_some());
     if init_null_count == df.height() {
@@ -216,7 +216,7 @@ pub fn apply_lambda_with_list_out_type<'a>(
         let mut iters = get_iters_skip(df, init_null_count + skip);
         let iter = ((init_null_count + skip)..df.height()).map(|_| {
             let iter = iters.iter_mut().map(|it| Wrap(it.next().unwrap()));
-            let tpl = (iter.collect::<Vec<Wrap<AnyValue>>>(),);
+            let tpl = (RArray::from_iter(iter),);
             match lambda.funcall::<_, _, Value>("call", tpl) {
                 Ok(val) => match val.funcall::<_, _, Value>("_s", ()) {
                     Ok(val) => val
@@ -254,7 +254,7 @@ pub fn apply_lambda_with_rows_output<'a>(
     let mut iters = get_iters_skip(df, init_null_count + skip);
     let mut row_iter = ((init_null_count + skip)..df.height()).map(|_| {
         let iter = iters.iter_mut().map(|it| Wrap(it.next().unwrap()));
-        let tpl = (iter.collect::<Vec<Wrap<AnyValue>>>(),);
+        let tpl = (RArray::from_iter(iter),);
         match lambda.funcall::<_, _, Value>("call", tpl) {
             Ok(val) => {
                 match val.try_convert::<RArray>().ok() {

data/ext/polars/src/apply/series.rs

@@ -1,4 +1,4 @@
-use magnus::{class, RHash, TryConvert, Value};
+use magnus::{class, IntoValue, RHash, TryConvert, Value};
 use polars::prelude::*;
 
 use super::*;
@@ -85,7 +85,7 @@ pub trait ApplyLambda<'a> {
     ) -> RbResult<ChunkedArray<D>>
     where
         D: RbArrowPrimitiveType,
-        D::Native: Into<Value> + TryConvert;
+        D::Native: IntoValue + TryConvert;
 
     /// Apply a lambda with a boolean output type
     fn apply_lambda_with_bool_out_type(
@@ -130,14 +130,14 @@ pub trait ApplyLambda<'a> {
 
 pub fn call_lambda<T>(lambda: Value, in_val: T) -> RbResult<Value>
 where
-    T: Into<Value>,
+    T: IntoValue,
 {
     lambda.funcall("call", (in_val,))
 }
 
 pub(crate) fn call_lambda_and_extract<T, S>(lambda: Value, in_val: T) -> RbResult<S>
 where
-    T: Into<Value>,
+    T: IntoValue,
     S: TryConvert,
 {
     match call_lambda(lambda, in_val) {
@@ -148,7 +148,7 @@ where
 
 fn call_lambda_series_out<T>(lambda: Value, in_val: T) -> RbResult<Series>
 where
-    T: Into<Value>,
+    T: IntoValue,
 {
     let out: Value = lambda.funcall("call", (in_val,))?;
     let py_series: Value = out.funcall("_s", ())?;
@@ -216,7 +216,7 @@ impl<'a> ApplyLambda<'a> for BooleanChunked {
     ) -> RbResult<ChunkedArray<D>>
     where
         D: RbArrowPrimitiveType,
-        D::Native: Into<Value> + TryConvert,
+        D::Native: IntoValue + TryConvert,
     {
         let skip = usize::from(first_value.is_some());
         if init_null_count == self.len() {
@@ -435,7 +435,7 @@ impl<'a> ApplyLambda<'a> for BooleanChunked {
 impl<'a, T> ApplyLambda<'a> for ChunkedArray<T>
 where
     T: RbArrowPrimitiveType + PolarsNumericType,
-    T::Native: Into<Value> + TryConvert,
+    T::Native: IntoValue + TryConvert,
     ChunkedArray<T>: IntoSeries,
 {
     fn apply_lambda_unknown(&'a self, lambda: Value) -> RbResult<RbSeries> {
@@ -493,7 +493,7 @@ where
     ) -> RbResult<ChunkedArray<D>>
     where
         D: RbArrowPrimitiveType,
-        D::Native: Into<Value> + TryConvert,
+        D::Native: IntoValue + TryConvert,
     {
         let skip = usize::from(first_value.is_some());
         if init_null_count == self.len() {
@@ -765,7 +765,7 @@ impl<'a> ApplyLambda<'a> for Utf8Chunked {
     ) -> RbResult<ChunkedArray<D>>
     where
         D: RbArrowPrimitiveType,
-        D::Native: Into<Value> + TryConvert,
+        D::Native: IntoValue + TryConvert,
     {
         let skip = usize::from(first_value.is_some());
         if init_null_count == self.len() {
@@ -1036,7 +1036,7 @@ impl<'a> ApplyLambda<'a> for StructChunked {
     ) -> RbResult<ChunkedArray<D>>
     where
         D: RbArrowPrimitiveType,
-        D::Native: Into<Value> + TryConvert,
+        D::Native: IntoValue + TryConvert,
     {
         let names = self.fields().iter().map(|s| s.name()).collect::<Vec<_>>();
 

data/ext/polars/src/batched_csv.rs

@@ -1,4 +1,4 @@
-use magnus::Value;
+use magnus::{RArray, Value};
 use polars::io::mmap::MmapBytesReader;
 use polars::io::RowCount;
 use polars::prelude::read_impl::OwnedBatchedCsvReader;
@@ -84,7 +84,7 @@ impl RbBatchedCsv {
             .with_n_rows(n_rows)
             .with_delimiter(sep.as_bytes()[0])
             .with_skip_rows(skip_rows)
-            .with_ignore_parser_errors(ignore_errors)
+            .with_ignore_errors(ignore_errors)
             .with_projection(projection)
             .with_rechunk(rechunk)
             .with_chunk_size(chunk_size)
@@ -109,12 +109,14 @@ impl RbBatchedCsv {
         })
     }
 
-    pub fn next_batches(&self, n: usize) -> RbResult<Option<Vec<RbDataFrame>>> {
+    pub fn next_batches(&self, n: usize) -> RbResult<Option<RArray>> {
         let batches = self
             .reader
             .borrow_mut()
             .next_batches(n)
             .map_err(RbPolarsErr::from)?;
-        Ok(batches.map(|batches| batches.into_iter().map(|out| out.1.into()).collect()))
+        Ok(batches.map(|batches| {
+            RArray::from_iter(batches.into_iter().map(|out| RbDataFrame::from(out.1)))
+        }))
     }
 }

data/ext/polars/src/conversion.rs

@@ -1,6 +1,6 @@
 use magnus::{
-    class, r_hash::ForEach, Integer, Module, RArray, RFloat, RHash, RString, Symbol, TryConvert,
-    Value, QNIL,
+    class, exception, r_hash::ForEach, ruby_handle::RubyHandle, Integer, IntoValue, Module, RArray,
+    RFloat, RHash, RString, Symbol, TryConvert, Value, QNIL,
 };
 use polars::chunked_array::object::PolarsObjectSafe;
 use polars::chunked_array::ops::{FillNullLimit, FillNullStrategy};
@@ -98,9 +98,9 @@ impl TryConvert for Wrap<NullValues> {
     }
 }
 
-impl From<Wrap<AnyValue<'_>>> for Value {
-    fn from(w: Wrap<AnyValue<'_>>) -> Self {
-        match w.0 {
+impl IntoValue for Wrap<AnyValue<'_>> {
+    fn into_value_with(self, _: &RubyHandle) -> Value {
+        match self.0 {
             AnyValue::UInt8(v) => Value::from(v),
             AnyValue::UInt16(v) => Value::from(v),
             AnyValue::UInt32(v) => Value::from(v),
@@ -114,6 +114,8 @@ impl From<Wrap<AnyValue<'_>>> for Value {
             AnyValue::Null => *QNIL,
             AnyValue::Boolean(v) => Value::from(v),
             AnyValue::Utf8(v) => Value::from(v),
+            AnyValue::Utf8Owned(_v) => todo!(),
+            AnyValue::Categorical(_idx, _rev, _arr) => todo!(),
             AnyValue::Date(v) => class::time()
                 .funcall::<_, _, Value>("at", (v * 86400,))
                 .unwrap()
@@ -140,16 +142,24 @@ impl From<Wrap<AnyValue<'_>>> for Value {
                     t.funcall::<_, _, Value>("utc", ()).unwrap()
                 }
             }
-            _ => todo!(),
+            AnyValue::Duration(_v, _tu) => todo!(),
+            AnyValue::Time(_v) => todo!(),
+            AnyValue::List(v) => RbSeries::new(v).to_a().into_value(),
+            ref _av @ AnyValue::Struct(_, _, _flds) => todo!(),
+            AnyValue::StructOwned(_payload) => todo!(),
+            AnyValue::Object(_v) => todo!(),
+            AnyValue::ObjectOwned(_v) => todo!(),
+            AnyValue::Binary(_v) => todo!(),
+            AnyValue::BinaryOwned(_v) => todo!(),
         }
     }
 }
 
-impl From<Wrap<DataType>> for Value {
-    fn from(w: Wrap<DataType>) -> Self {
+impl IntoValue for Wrap<DataType> {
+    fn into_value_with(self, _: &RubyHandle) -> Value {
         let pl = crate::rb_modules::polars();
 
-        match &w.0 {
+        match self.0 {
             DataType::Int8 => pl.const_get::<_, Value>("Int8").unwrap(),
             DataType::Int16 => pl.const_get::<_, Value>("Int16").unwrap(),
             DataType::Int32 => pl.const_get::<_, Value>("Int32").unwrap(),
@@ -160,6 +170,7 @@ impl From<Wrap<DataType>> for Value {
             DataType::UInt64 => pl.const_get::<_, Value>("UInt64").unwrap(),
             DataType::Float32 => pl.const_get::<_, Value>("Float32").unwrap(),
             DataType::Float64 => pl.const_get::<_, Value>("Float64").unwrap(),
+            DataType::Decimal128(_) => todo!(),
             DataType::Boolean => pl.const_get::<_, Value>("Boolean").unwrap(),
             DataType::Utf8 => pl.const_get::<_, Value>("Utf8").unwrap(),
             DataType::Binary => pl.const_get::<_, Value>("Binary").unwrap(),
@@ -423,9 +434,9 @@ impl ObjectValue {
     }
 }
 
-impl From<ObjectValue> for Value {
-    fn from(val: ObjectValue) -> Self {
-        val.inner
+impl IntoValue for ObjectValue {
+    fn into_value_with(self, _: &RubyHandle) -> Value {
+        self.inner
     }
 }
 
@@ -767,6 +778,22 @@ impl TryConvert for Wrap<UniqueKeepStrategy> {
     }
 }
 
+impl TryConvert for Wrap<SearchSortedSide> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let parsed = match ob.try_convert::<String>()?.as_str() {
+            "any" => SearchSortedSide::Any,
+            "left" => SearchSortedSide::Left,
+            "right" => SearchSortedSide::Right,
+            v => {
+                return Err(RbValueError::new_err(format!(
+                    "side must be one of {{'any', 'left', 'right'}}, got {v}",
+                )))
+            }
+        };
+        Ok(Wrap(parsed))
+    }
+}
+
 pub fn parse_fill_null_strategy(
     strategy: &str,
     limit: FillNullLimit,
@@ -780,7 +807,7 @@ pub fn parse_fill_null_strategy(
         "zero" => FillNullStrategy::Zero,
         "one" => FillNullStrategy::One,
         e => {
-            return Err(magnus::Error::runtime_error(format!(
+            return Err(magnus::Error::new(exception::runtime_error(), format!(
                 "strategy must be one of {{'forward', 'backward', 'min', 'max', 'mean', 'zero', 'one'}}, got {}",
                 e,
             )))

data/ext/polars/src/dataframe.rs

@@ -1,4 +1,4 @@
-use magnus::{r_hash::ForEach, RArray, RHash, RString, Value};
+use magnus::{r_hash::ForEach, IntoValue, RArray, RHash, RString, Value};
 use polars::frame::row::{rows_to_schema_supertypes, Row};
 use polars::frame::NullStrategy;
 use polars::io::avro::AvroCompression;
@@ -68,7 +68,7 @@ impl RbDataFrame {
                         *dtype_ = dtype;
                     }
                 } else {
-                    schema.with_column(name, dtype)
+                    schema.with_column(name, dtype);
                 }
             }
         }
@@ -159,7 +159,7 @@ impl RbDataFrame {
             .with_n_rows(n_rows)
             .with_delimiter(sep.as_bytes()[0])
             .with_skip_rows(skip_rows)
-            .with_ignore_parser_errors(ignore_errors)
+            .with_ignore_errors(ignore_errors)
             .with_projection(projection)
             .with_rechunk(rechunk)
             .with_chunk_size(chunk_size)
@@ -457,7 +457,7 @@ impl RbDataFrame {
         } else {
             idx as usize
         };
-        RArray::from_vec(
+        RArray::from_iter(
             self.df
                 .borrow()
                 .get_columns()
@@ -467,36 +467,29 @@ impl RbDataFrame {
                         let obj: Option<&ObjectValue> = s.get_object(idx).map(|any| any.into());
                         obj.unwrap().to_object()
                     }
-                    _ => Wrap(s.get(idx).unwrap()).into(),
-                })
-                .collect(),
+                    _ => Wrap(s.get(idx).unwrap()).into_value(),
+                }),
         )
         .into()
     }
 
     pub fn row_tuples(&self) -> Value {
         let df = &self.df;
-        RArray::from_vec(
-            (0..df.borrow().height())
-                .map(|idx| {
-                    RArray::from_vec(
-                        self.df
-                            .borrow()
-                            .get_columns()
-                            .iter()
-                            .map(|s| match s.dtype() {
-                                DataType::Object(_) => {
-                                    let obj: Option<&ObjectValue> =
-                                        s.get_object(idx).map(|any| any.into());
-                                    obj.unwrap().to_object()
-                                }
-                                _ => Wrap(s.get(idx).unwrap()).into(),
-                            })
-                            .collect(),
-                    )
-                })
-                .collect(),
-        )
+        RArray::from_iter((0..df.borrow().height()).map(|idx| {
+            RArray::from_iter(
+                self.df
+                    .borrow()
+                    .get_columns()
+                    .iter()
+                    .map(|s| match s.dtype() {
+                        DataType::Object(_) => {
+                            let obj: Option<&ObjectValue> = s.get_object(idx).map(|any| any.into());
+                            obj.unwrap().to_object()
+                        }
+                        _ => Wrap(s.get(idx).unwrap()).into_value(),
+                    }),
+            )
+        }))
         .into()
     }
 
@@ -613,7 +606,7 @@ impl RbDataFrame {
         format!("{}", self.df.borrow())
     }
 
-    pub fn get_columns(&self) -> Vec<RbSeries> {
+    pub fn get_columns(&self) -> RArray {
         let cols = self.df.borrow().get_columns().clone();
         to_rbseries_collection(cols)
     }
@@ -635,12 +628,13 @@ impl RbDataFrame {
         Ok(())
     }
 
-    pub fn dtypes(&self) -> Vec<Value> {
-        self.df
-            .borrow()
-            .iter()
-            .map(|s| Wrap(s.dtype().clone()).into())
-            .collect()
+    pub fn dtypes(&self) -> RArray {
+        RArray::from_iter(
+            self.df
+                .borrow()
+                .iter()
+                .map(|s| Wrap(s.dtype().clone()).into_value()),
+        )
     }
 
     pub fn n_chunks(&self) -> usize {
@@ -777,6 +771,7 @@ impl RbDataFrame {
                 SortOptions {
                     descending: reverse,
                     nulls_last,
+                    multithreaded: true,
                 },
             )
             .map_err(RbPolarsErr::from)?;
@@ -876,6 +871,7 @@ impl RbDataFrame {
         Ok(RbDataFrame::new(df))
     }
 
+    #[allow(clippy::too_many_arguments)]
     pub fn pivot_expr(
         &self,
         values: Vec<String>,
@@ -884,6 +880,7 @@ impl RbDataFrame {
         aggregate_expr: &RbExpr,
         maintain_order: bool,
         sort_columns: bool,
+        separator: Option<String>,
     ) -> RbResult<Self> {
         let fun = match maintain_order {
             true => pivot_stable,
@@ -896,19 +893,20 @@ impl RbDataFrame {
             columns,
             aggregate_expr.inner.clone(),
             sort_columns,
+            separator.as_deref(),
         )
         .map_err(RbPolarsErr::from)?;
         Ok(RbDataFrame::new(df))
     }
 
-    pub fn partition_by(&self, groups: Vec<String>, stable: bool) -> RbResult<Vec<Self>> {
+    pub fn partition_by(&self, groups: Vec<String>, stable: bool) -> RbResult<RArray> {
         let out = if stable {
             self.df.borrow().partition_by_stable(groups)
         } else {
             self.df.borrow().partition_by(groups)
         }
         .map_err(RbPolarsErr::from)?;
-        Ok(out.into_iter().map(RbDataFrame::new).collect())
+        Ok(RArray::from_iter(out.into_iter().map(RbDataFrame::new)))
     }
 
     pub fn shift(&self, periods: i64) -> Self {
@@ -1003,13 +1001,17 @@ impl RbDataFrame {
         Ok(df.into())
     }
 
-    pub fn to_dummies(&self, columns: Option<Vec<String>>) -> RbResult<Self> {
+    pub fn to_dummies(
+        &self,
+        columns: Option<Vec<String>>,
+        separator: Option<String>,
+    ) -> RbResult<Self> {
         let df = match columns {
-            Some(cols) => self
-                .df
-                .borrow()
-                .columns_to_dummies(cols.iter().map(|x| x as &str).collect()),
-            None => self.df.borrow().to_dummies(),
+            Some(cols) => self.df.borrow().columns_to_dummies(
+                cols.iter().map(|x| x as &str).collect(),
+                separator.as_deref(),
+            ),
+            None => self.df.borrow().to_dummies(separator.as_deref()),
         }
         .map_err(RbPolarsErr::from)?;
         Ok(df.into())

data/ext/polars/src/error.rs

@@ -1,4 +1,4 @@
-use magnus::exception::arg_error;
+use magnus::exception;
 use magnus::Error;
 use polars::error::ArrowError;
 use polars::prelude::PolarsError;
@@ -8,23 +8,23 @@ pub struct RbPolarsErr {}
 impl RbPolarsErr {
     // convert to Error instead of Self
     pub fn from(e: PolarsError) -> Error {
-        Error::runtime_error(e.to_string())
+        Error::new(exception::runtime_error(), e.to_string())
     }
 
     pub fn arrow(e: ArrowError) -> Error {
-        Error::runtime_error(e.to_string())
+        Error::new(exception::runtime_error(), e.to_string())
     }
 
     pub fn io(e: std::io::Error) -> Error {
-        Error::runtime_error(e.to_string())
+        Error::new(exception::runtime_error(), e.to_string())
     }
 
     pub fn other(message: String) -> Error {
-        Error::runtime_error(message)
+        Error::new(exception::runtime_error(), message)
     }
 
     pub fn todo() -> Error {
-        Error::runtime_error("not implemented yet")
+        Error::new(exception::runtime_error(), "not implemented yet")
     }
 }
 
@@ -32,7 +32,7 @@ pub struct RbValueError {}
 
 impl RbValueError {
     pub fn new_err(message: String) -> Error {
-        Error::new(arg_error(), message)
+        Error::new(exception::arg_error(), message)
     }
 }
 
@@ -40,6 +40,6 @@ pub struct ComputeError {}
 
 impl ComputeError {
     pub fn new_err(message: String) -> Error {
-        Error::runtime_error(message)
+        Error::new(exception::runtime_error(), message)
     }
 }

data/ext/polars/src/file.rs

@@ -1,4 +1,4 @@
-use magnus::{Error, RString, Value};
+use magnus::{exception, Error, RString, Value};
 use polars::io::mmap::MmapBytesReader;
 use std::fs::File;
 use std::io::Cursor;
@@ -9,9 +9,10 @@ use crate::RbResult;
 pub fn get_file_like(f: Value, truncate: bool) -> RbResult<File> {
     let str_slice = f.try_convert::<PathBuf>()?;
     let f = if truncate {
-        File::create(str_slice).map_err(|e| Error::runtime_error(e.to_string()))?
+        File::create(str_slice)
+            .map_err(|e| Error::new(exception::runtime_error(), e.to_string()))?
     } else {
-        File::open(str_slice).map_err(|e| Error::runtime_error(e.to_string()))?
+        File::open(str_slice).map_err(|e| Error::new(exception::runtime_error(), e.to_string()))?
     };
     Ok(f)
 }
@@ -23,7 +24,7 @@ pub fn get_mmap_bytes_reader(rb_f: Value) -> RbResult<Box<dyn MmapBytesReader>>
         Ok(Box::new(Cursor::new(bytes.to_vec())))
     } else {
         let p = rb_f.try_convert::<PathBuf>()?;
-        let f = File::open(p).map_err(|e| Error::runtime_error(e.to_string()))?;
+        let f = File::open(p).map_err(|e| Error::new(exception::runtime_error(), e.to_string()))?;
         Ok(Box::new(f))
     }
 }

data/ext/polars/src/lazy/apply.rs

@@ -4,7 +4,7 @@ use polars::prelude::*;
 use crate::lazy::dsl::RbExpr;
 use crate::Wrap;
 
-pub fn binary_lambda(_lambda: Value, _a: Series, _b: Series) -> PolarsResult<Series> {
+pub fn binary_lambda(_lambda: Value, _a: Series, _b: Series) -> PolarsResult<Option<Series>> {
     todo!();
 }
 

data/ext/polars/src/lazy/dataframe.rs

@@ -1,4 +1,4 @@
-use magnus::{RArray, RHash, Value};
+use magnus::{IntoValue, RArray, RHash, Value};
 use polars::io::RowCount;
 use polars::lazy::frame::{LazyFrame, LazyGroupBy};
 use polars::prelude::*;
@@ -140,7 +140,7 @@ impl RbLazyFrame {
             .with_infer_schema_length(infer_schema_length)
             .with_delimiter(delimiter)
             .has_header(has_header)
-            .with_ignore_parser_errors(ignore_errors)
+            .with_ignore_errors(ignore_errors)
             .with_skip_rows(skip_rows)
             .with_n_rows(n_rows)
             .with_cache(cache)
@@ -180,6 +180,8 @@ impl RbLazyFrame {
             rechunk,
             row_count,
             low_memory,
+            // TODO support cloud options
+            cloud_options: None,
         };
         let lf = LazyFrame::scan_parquet(path, args).map_err(RbPolarsErr::from)?;
         Ok(lf.into())
@@ -254,6 +256,7 @@ impl RbLazyFrame {
             SortOptions {
                 descending: reverse,
                 nulls_last,
+                multithreaded: true,
             },
         )
         .into()
@@ -597,10 +600,10 @@ impl RbLazyFrame {
         Ok(self.get_schema()?.iter_names().cloned().collect())
     }
 
-    pub fn dtypes(&self) -> RbResult<Vec<Value>> {
+    pub fn dtypes(&self) -> RbResult<RArray> {
         let schema = self.get_schema()?;
-        let iter = schema.iter_dtypes().map(|dt| Wrap(dt.clone()).into());
-        Ok(iter.collect())
+        let iter = schema.iter_dtypes().map(|dt| Wrap(dt.clone()).into_value());
+        Ok(RArray::from_iter(iter))
     }
 
     pub fn schema(&self) -> RbResult<RHash> {
@@ -610,7 +613,10 @@ impl RbLazyFrame {
         schema.iter_fields().for_each(|fld| {
             // TODO remove unwrap
             schema_dict
-                .aset::<String, Value>(fld.name().clone(), Wrap(fld.data_type().clone()).into())
+                .aset::<String, Value>(
+                    fld.name().clone(),
+                    Wrap(fld.data_type().clone()).into_value(),
+                )
                 .unwrap();
         });
         Ok(schema_dict)