polars-df 0.26.0 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/Cargo.lock +300 -141
- data/Cargo.toml +0 -3
- data/LICENSE.txt +1 -1
- data/README.md +6 -0
- data/ext/polars/Cargo.toml +16 -16
- data/ext/polars/src/conversion/mod.rs +53 -60
- data/ext/polars/src/dataframe/general.rs +15 -2
- data/ext/polars/src/dataframe/io.rs +104 -97
- data/ext/polars/src/expr/general.rs +6 -2
- data/ext/polars/src/expr/list.rs +2 -2
- data/ext/polars/src/file.rs +8 -9
- data/ext/polars/src/functions/io.rs +2 -1
- data/ext/polars/src/interop/arrow/to_rb.rs +12 -22
- data/ext/polars/src/lazyframe/general.rs +10 -4
- data/ext/polars/src/lazyframe/sink.rs +1 -1
- data/ext/polars/src/lib.rs +8 -8
- data/ext/polars/src/on_startup.rs +2 -2
- data/ext/polars/src/ruby/capsule.rs +39 -0
- data/ext/polars/src/ruby/mod.rs +1 -0
- data/ext/polars/src/series/export.rs +5 -0
- data/ext/polars/src/utils.rs +7 -3
- data/lib/polars/array_expr.rb +11 -3
- data/lib/polars/array_name_space.rb +2 -2
- data/lib/polars/data_frame.rb +57 -3
- data/lib/polars/expr.rb +74 -6
- data/lib/polars/functions/aggregation/horizontal.rb +4 -4
- data/lib/polars/functions/eager.rb +2 -2
- data/lib/polars/iceberg_dataset.rb +27 -86
- data/lib/polars/io/delta.rb +4 -0
- data/lib/polars/lazy_frame.rb +11 -3
- data/lib/polars/list_expr.rb +10 -2
- data/lib/polars/list_name_space.rb +2 -2
- data/lib/polars/schema.rb +9 -0
- data/lib/polars/series.rb +21 -3
- data/lib/polars/version.rb +1 -1
- data/lib/polars.rb +2 -6
- metadata +3 -2
data/ext/polars/src/file.rs
CHANGED
|
@@ -6,9 +6,8 @@ use std::path::PathBuf;
|
|
|
6
6
|
use magnus::{Error, RString, Ruby, Value, prelude::*, value::Opaque};
|
|
7
7
|
use polars::io::mmap::MmapBytesReader;
|
|
8
8
|
use polars::prelude::PlRefPath;
|
|
9
|
-
use polars::prelude::file::{
|
|
9
|
+
use polars::prelude::file::{Writable, WritableTrait};
|
|
10
10
|
use polars_buffer::Buffer;
|
|
11
|
-
use polars_utils::create_file;
|
|
12
11
|
|
|
13
12
|
use crate::error::RbPolarsErr;
|
|
14
13
|
use crate::prelude::resolve_homedir;
|
|
@@ -23,7 +22,7 @@ pub struct RbFileLikeObject {
|
|
|
23
22
|
has_flush: bool,
|
|
24
23
|
}
|
|
25
24
|
|
|
26
|
-
impl
|
|
25
|
+
impl WritableTrait for RbFileLikeObject {
|
|
27
26
|
fn close(&mut self) -> io::Result<()> {
|
|
28
27
|
Ok(())
|
|
29
28
|
}
|
|
@@ -192,7 +191,7 @@ impl Seek for RbFileLikeObject {
|
|
|
192
191
|
}
|
|
193
192
|
}
|
|
194
193
|
|
|
195
|
-
pub trait FileLike: Read + Write + Seek {}
|
|
194
|
+
pub trait FileLike: Read + Write + Seek + Sync + Send {}
|
|
196
195
|
|
|
197
196
|
impl FileLike for File {}
|
|
198
197
|
impl FileLike for RbFileLikeObject {}
|
|
@@ -217,10 +216,10 @@ impl EitherRustRubyFile {
|
|
|
217
216
|
}
|
|
218
217
|
}
|
|
219
218
|
|
|
220
|
-
pub(crate) fn
|
|
219
|
+
pub(crate) fn into_writable(self) -> Writable {
|
|
221
220
|
match self {
|
|
222
|
-
Self::Rb(f) =>
|
|
223
|
-
Self::Rust(f) =>
|
|
221
|
+
Self::Rb(f) => Writable::Dyn(Box::new(f)),
|
|
222
|
+
Self::Rust(f) => Writable::Local(f),
|
|
224
223
|
}
|
|
225
224
|
}
|
|
226
225
|
}
|
|
@@ -272,9 +271,9 @@ pub fn get_either_buffer_or_path(
|
|
|
272
271
|
let file_path = std::path::Path::new(&s);
|
|
273
272
|
let file_path = resolve_homedir(&file_path);
|
|
274
273
|
let f = if write {
|
|
275
|
-
create_file(&file_path).map_err(RbPolarsErr::from)?
|
|
274
|
+
polars_utils::io::create_file(&file_path).map_err(RbPolarsErr::from)?
|
|
276
275
|
} else {
|
|
277
|
-
polars_utils::open_file(&file_path).map_err(RbPolarsErr::from)?
|
|
276
|
+
polars_utils::io::open_file(&file_path).map_err(RbPolarsErr::from)?
|
|
278
277
|
};
|
|
279
278
|
Ok((EitherRustRubyFile::Rust(f), Some(file_path.into_owned())))
|
|
280
279
|
} else {
|
|
@@ -62,7 +62,8 @@ pub fn read_parquet_metadata(
|
|
|
62
62
|
})
|
|
63
63
|
.map_err(RbPolarsErr::from)?
|
|
64
64
|
} else {
|
|
65
|
-
let file =
|
|
65
|
+
let file =
|
|
66
|
+
polars_utils::io::open_file(p.as_std_path()).map_err(RbPolarsErr::from)?;
|
|
66
67
|
read_metadata(&mut BufReader::new(file)).map_err(RbPolarsErr::from)?
|
|
67
68
|
}
|
|
68
69
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
use arrow::datatypes::ArrowDataType;
|
|
2
2
|
use arrow::ffi;
|
|
3
|
-
use magnus::{
|
|
3
|
+
use magnus::{Ruby, Value};
|
|
4
4
|
use polars::datatypes::CompatLevel;
|
|
5
5
|
use polars::frame::DataFrame;
|
|
6
6
|
use polars::prelude::{ArrayRef, ArrowField, PlSmallStr, PolarsResult, SchemaExt};
|
|
@@ -8,34 +8,24 @@ use polars::series::Series;
|
|
|
8
8
|
use polars_core::utils::arrow;
|
|
9
9
|
|
|
10
10
|
use crate::RbResult;
|
|
11
|
+
use crate::ruby::capsule::RbCapsule;
|
|
11
12
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
pub fn to_i(&self) -> usize {
|
|
19
|
-
(&self.stream as *const _) as usize
|
|
20
|
-
}
|
|
21
|
-
}
|
|
13
|
+
pub(crate) fn series_to_stream(series: &Series, ruby: &Ruby) -> RbResult<Value> {
|
|
14
|
+
let field = series.field().to_arrow(CompatLevel::newest());
|
|
15
|
+
let series = series.clone();
|
|
16
|
+
let iter = Box::new(
|
|
17
|
+
(0..series.n_chunks()).map(move |i| Ok(series.to_arrow(i, CompatLevel::newest()))),
|
|
18
|
+
) as _;
|
|
22
19
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
pub(crate) schema: ffi::ArrowSchema,
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
impl RbArrowSchema {
|
|
29
|
-
pub fn to_i(&self) -> usize {
|
|
30
|
-
(&self.schema as *const _) as usize
|
|
31
|
-
}
|
|
20
|
+
let stream = ffi::export_iterator(iter, field);
|
|
21
|
+
RbCapsule::new_with_value(ruby, stream, c"arrow_array_stream")
|
|
32
22
|
}
|
|
33
23
|
|
|
34
24
|
pub(crate) fn dataframe_to_stream(df: &DataFrame, ruby: &Ruby) -> RbResult<Value> {
|
|
35
25
|
let iter = Box::new(DataFrameStreamIterator::new(df));
|
|
36
26
|
let field = iter.field();
|
|
37
27
|
let stream = ffi::export_iterator(iter, field);
|
|
38
|
-
|
|
28
|
+
RbCapsule::new_with_value(ruby, stream, c"arrow_array_stream")
|
|
39
29
|
}
|
|
40
30
|
|
|
41
31
|
pub(crate) fn polars_schema_to_rbcapsule(
|
|
@@ -54,7 +44,7 @@ pub(crate) fn polars_schema_to_rbcapsule(
|
|
|
54
44
|
false,
|
|
55
45
|
));
|
|
56
46
|
|
|
57
|
-
|
|
47
|
+
RbCapsule::new_with_value(ruby, schema, c"arrow_schema")
|
|
58
48
|
}
|
|
59
49
|
|
|
60
50
|
pub struct DataFrameStreamIterator {
|
|
@@ -10,6 +10,7 @@ use polars::prelude::*;
|
|
|
10
10
|
use polars_core::query_result::QueryResult;
|
|
11
11
|
use polars_plan::dsl::ScanSources;
|
|
12
12
|
use polars_plan::plans::{HintIR, Sorted};
|
|
13
|
+
use std::ffi::CString;
|
|
13
14
|
use std::num::NonZeroUsize;
|
|
14
15
|
|
|
15
16
|
use super::{RbLazyFrame, RbOptFlags};
|
|
@@ -27,8 +28,7 @@ use crate::ruby::ruby_function::RubyObject;
|
|
|
27
28
|
use crate::ruby::utils::TryIntoValue;
|
|
28
29
|
use crate::utils::{EnterPolarsExt, to_rb_err};
|
|
29
30
|
use crate::{
|
|
30
|
-
|
|
31
|
-
RbValueError,
|
|
31
|
+
RbCapsule, RbDataFrame, RbExpr, RbLazyGroupBy, RbPolarsErr, RbResult, RbTypeError, RbValueError,
|
|
32
32
|
};
|
|
33
33
|
|
|
34
34
|
fn rbobject_to_first_path_and_scan_sources(
|
|
@@ -1224,7 +1224,12 @@ impl RbLazyFrame {
|
|
|
1224
1224
|
ldf.count().into()
|
|
1225
1225
|
}
|
|
1226
1226
|
|
|
1227
|
-
pub fn merge_sorted(
|
|
1227
|
+
pub fn merge_sorted(
|
|
1228
|
+
&self,
|
|
1229
|
+
other: &Self,
|
|
1230
|
+
key: Vec<String>,
|
|
1231
|
+
maintain_order: bool,
|
|
1232
|
+
) -> RbResult<Self> {
|
|
1228
1233
|
let out = self
|
|
1229
1234
|
.ldf
|
|
1230
1235
|
.read()
|
|
@@ -1314,7 +1319,8 @@ impl RbCollectBatches {
|
|
|
1314
1319
|
let iter = Box::new(ArrowStreamIterator::new(self_.inner.clone(), dtype.clone()));
|
|
1315
1320
|
let field = ArrowField::new(PlSmallStr::EMPTY, dtype, false);
|
|
1316
1321
|
let stream = export_iterator(iter, field);
|
|
1317
|
-
|
|
1322
|
+
let stream_capsule_name = CString::new("arrow_array_stream").unwrap();
|
|
1323
|
+
Ok(RbCapsule::new(stream, Some(stream_capsule_name)).into_value_with(rb))
|
|
1318
1324
|
}
|
|
1319
1325
|
}
|
|
1320
1326
|
|
data/ext/polars/src/lib.rs
CHANGED
|
@@ -37,10 +37,10 @@ use expr::RbExpr;
|
|
|
37
37
|
use expr::datatype::RbDataTypeExpr;
|
|
38
38
|
use expr::selector::RbSelector;
|
|
39
39
|
use functions::whenthen::{RbChainedThen, RbChainedWhen, RbThen, RbWhen};
|
|
40
|
-
use interop::arrow::to_rb::{RbArrowArrayStream, RbArrowSchema};
|
|
41
40
|
use lazyframe::{RbCollectBatches, RbInProcessQuery, RbLazyFrame, RbOptFlags};
|
|
42
41
|
use lazygroupby::RbLazyGroupBy;
|
|
43
42
|
use magnus::{Ruby, function, method, prelude::*};
|
|
43
|
+
use ruby::capsule::RbCapsule;
|
|
44
44
|
use ruby::exceptions::{RbTypeError, RbValueError};
|
|
45
45
|
use series::RbSeries;
|
|
46
46
|
use sql::RbSQLContext;
|
|
@@ -137,6 +137,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
|
|
|
137
137
|
class.define_method("tail", method!(RbDataFrame::tail, 1))?;
|
|
138
138
|
class.define_method("is_unique", method!(RbDataFrame::is_unique, 0))?;
|
|
139
139
|
class.define_method("is_duplicated", method!(RbDataFrame::is_duplicated, 0))?;
|
|
140
|
+
class.define_method("is_sorted", method!(RbDataFrame::is_sorted, 3))?;
|
|
140
141
|
class.define_method("equals", method!(RbDataFrame::equals, 2))?;
|
|
141
142
|
class.define_method("with_row_index", method!(RbDataFrame::with_row_index, 2))?;
|
|
142
143
|
class.define_method(
|
|
@@ -251,6 +252,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
|
|
|
251
252
|
class.define_method("is_unique", method!(RbExpr::is_unique, 0))?;
|
|
252
253
|
class.define_method("is_between", method!(RbExpr::is_between, 3))?;
|
|
253
254
|
class.define_method("is_close", method!(RbExpr::is_close, 4))?;
|
|
255
|
+
class.define_method("is_sorted", method!(RbExpr::is_sorted, 2))?;
|
|
254
256
|
class.define_method("approx_n_unique", method!(RbExpr::approx_n_unique, 0))?;
|
|
255
257
|
class.define_method("is_first_distinct", method!(RbExpr::is_first_distinct, 0))?;
|
|
256
258
|
class.define_method("is_last_distinct", method!(RbExpr::is_last_distinct, 0))?;
|
|
@@ -1135,6 +1137,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
|
|
|
1135
1137
|
class.define_method("to_s", method!(RbSeries::as_str, 0))?;
|
|
1136
1138
|
class.define_method("len", method!(RbSeries::len, 0))?;
|
|
1137
1139
|
class.define_method("to_a", method!(RbSeries::to_a, 0))?;
|
|
1140
|
+
class.define_method("arrow_c_stream", method!(RbSeries::__arrow_c_stream__, 0))?;
|
|
1138
1141
|
class.define_method("median", method!(RbSeries::median, 0))?;
|
|
1139
1142
|
class.define_method("quantile", method!(RbSeries::quantile, 2))?;
|
|
1140
1143
|
class.define_method("_clone", method!(RbSeries::clone, 0))?;
|
|
@@ -1340,13 +1343,10 @@ fn init(ruby: &Ruby) -> RbResult<()> {
|
|
|
1340
1343
|
class.define_method("register", method!(RbSQLContext::register, 2))?;
|
|
1341
1344
|
class.define_method("unregister", method!(RbSQLContext::unregister, 1))?;
|
|
1342
1345
|
|
|
1343
|
-
//
|
|
1344
|
-
let class = module.define_class("
|
|
1345
|
-
class.define_method("to_i", method!(
|
|
1346
|
-
|
|
1347
|
-
// arrow schema
|
|
1348
|
-
let class = module.define_class("ArrowSchema", ruby.class_object())?;
|
|
1349
|
-
class.define_method("to_i", method!(RbArrowSchema::to_i, 0))?;
|
|
1346
|
+
// capsule
|
|
1347
|
+
let class = module.define_class("Capsule", ruby.class_object())?;
|
|
1348
|
+
class.define_method("to_i", method!(RbCapsule::to_i, 0))?;
|
|
1349
|
+
class.define_method("name", method!(RbCapsule::name, 0))?;
|
|
1350
1350
|
|
|
1351
1351
|
// catalog
|
|
1352
1352
|
let class = module.define_class("RbCatalogClient", ruby.class_object())?;
|
|
@@ -12,7 +12,7 @@ use polars_core::chunked_array::object::registry;
|
|
|
12
12
|
use polars_core::chunked_array::object::registry::AnonymousObjectBuilder;
|
|
13
13
|
use polars_core::prelude::AnyValue;
|
|
14
14
|
use polars_error::PolarsWarning;
|
|
15
|
-
use polars_error::
|
|
15
|
+
use polars_error::abort::register_polars_abort_mechanism;
|
|
16
16
|
|
|
17
17
|
use crate::Wrap;
|
|
18
18
|
use crate::dataframe::RbDataFrame;
|
|
@@ -209,7 +209,7 @@ pub unsafe fn register_startup_deps(catch_keyboard_interrupt: bool) {
|
|
|
209
209
|
polars_error::set_warning_function(warning_function);
|
|
210
210
|
|
|
211
211
|
if catch_keyboard_interrupt {
|
|
212
|
-
|
|
212
|
+
register_polars_abort_mechanism();
|
|
213
213
|
}
|
|
214
214
|
});
|
|
215
215
|
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
use std::any::Any;
|
|
2
|
+
use std::ffi::{CStr, CString};
|
|
3
|
+
|
|
4
|
+
use magnus::{IntoValue, Ruby, Value};
|
|
5
|
+
|
|
6
|
+
use crate::RbResult;
|
|
7
|
+
|
|
8
|
+
#[magnus::wrap(class = "Polars::Capsule")]
|
|
9
|
+
pub struct RbCapsule {
|
|
10
|
+
value: Box<dyn Any + Send>,
|
|
11
|
+
name: Option<CString>,
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
impl RbCapsule {
|
|
15
|
+
pub fn new_with_value<T: 'static + Send>(
|
|
16
|
+
ruby: &Ruby,
|
|
17
|
+
value: T,
|
|
18
|
+
name: &CStr,
|
|
19
|
+
) -> RbResult<Value> {
|
|
20
|
+
Ok(Self::new(value, Some(name.into())).into_value_with(ruby))
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
pub fn new<T: 'static + Send>(value: T, name: Option<CString>) -> Self {
|
|
24
|
+
Self {
|
|
25
|
+
value: Box::new(value),
|
|
26
|
+
name,
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
pub fn to_i(&self) -> usize {
|
|
31
|
+
(&*self.value as *const dyn Any as *const ()) as usize
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// TODO use &CStr when Magnus supports it
|
|
35
|
+
// https://github.com/matsadler/magnus/pull/182
|
|
36
|
+
pub fn name(&self) -> Option<&str> {
|
|
37
|
+
self.name.as_deref().map(|v| v.to_str().unwrap())
|
|
38
|
+
}
|
|
39
|
+
}
|
data/ext/polars/src/ruby/mod.rs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
use magnus::{IntoValue, Ruby, Value, value::ReprValue};
|
|
2
2
|
use polars_core::prelude::*;
|
|
3
3
|
|
|
4
|
+
use crate::interop::arrow::to_rb::series_to_stream;
|
|
4
5
|
use crate::prelude::*;
|
|
5
6
|
use crate::ruby::utils::TryIntoValue;
|
|
6
7
|
use crate::{RbPolarsErr, RbResult, RbSeries};
|
|
@@ -178,4 +179,8 @@ impl RbSeries {
|
|
|
178
179
|
|
|
179
180
|
to_a_recursive(ruby, series)
|
|
180
181
|
}
|
|
182
|
+
|
|
183
|
+
pub fn __arrow_c_stream__(ruby: &Ruby, self_: &Self) -> RbResult<Value> {
|
|
184
|
+
series_to_stream(&self_.series.read(), ruby)
|
|
185
|
+
}
|
|
181
186
|
}
|
data/ext/polars/src/utils.rs
CHANGED
|
@@ -4,7 +4,7 @@ use magnus::Ruby;
|
|
|
4
4
|
use polars::frame::DataFrame;
|
|
5
5
|
use polars::series::IntoSeries;
|
|
6
6
|
use polars_error::PolarsResult;
|
|
7
|
-
use polars_error::
|
|
7
|
+
use polars_error::abort::{QueryAborted, catch_polars_abort};
|
|
8
8
|
|
|
9
9
|
use crate::ruby::exceptions::RbKeyboardInterrupt;
|
|
10
10
|
use crate::ruby::gvl::GvlExt;
|
|
@@ -114,12 +114,16 @@ impl EnterPolarsExt for &Ruby {
|
|
|
114
114
|
E: Send + Into<RbPolarsErr>,
|
|
115
115
|
{
|
|
116
116
|
let timeout = schedule_polars_timeout();
|
|
117
|
-
let ret = self.detach(||
|
|
117
|
+
let ret = self.detach(|| catch_polars_abort(AssertUnwindSafe(f)));
|
|
118
118
|
cancel_polars_timeout(timeout);
|
|
119
119
|
match ret {
|
|
120
120
|
Ok(Ok(ret)) => Ok(ret),
|
|
121
121
|
Ok(Err(err)) => Err(RbErr::from(err.into())),
|
|
122
|
-
Err(KeyboardInterrupt) => Err(RbKeyboardInterrupt::new_err("")),
|
|
122
|
+
Err(QueryAborted::KeyboardInterrupt) => Err(RbKeyboardInterrupt::new_err("")),
|
|
123
|
+
Err(QueryAborted::OocOutOfDisk) => Err(RbPolarsErr::Other(
|
|
124
|
+
"query aborted, raise POLARS_OOC_DISK_BUDGET_MB".to_string(),
|
|
125
|
+
)
|
|
126
|
+
.into()),
|
|
123
127
|
}
|
|
124
128
|
}
|
|
125
129
|
}
|
data/lib/polars/array_expr.rb
CHANGED
|
@@ -776,7 +776,7 @@ module Polars
|
|
|
776
776
|
# df = Polars::DataFrame.new(
|
|
777
777
|
# {"a" => [[1, 2, 3], [4, 5, 6]]}, schema: {"a" => Polars::Array.new(Polars::Int64, 3)}
|
|
778
778
|
# )
|
|
779
|
-
# df.select(Polars.col("a").arr.explode)
|
|
779
|
+
# df.select(Polars.col("a").arr.explode(empty_as_null: false))
|
|
780
780
|
# # =>
|
|
781
781
|
# # shape: (6, 1)
|
|
782
782
|
# # ┌─────┐
|
|
@@ -791,8 +791,16 @@ module Polars
|
|
|
791
791
|
# # │ 5 │
|
|
792
792
|
# # │ 6 │
|
|
793
793
|
# # └─────┘
|
|
794
|
-
def explode(empty_as_null:
|
|
795
|
-
|
|
794
|
+
def explode(empty_as_null: OMITTED, keep_nulls: true)
|
|
795
|
+
if empty_as_null == OMITTED
|
|
796
|
+
Utils.issue_deprecation_warning(
|
|
797
|
+
"The default behavior for `empty_as_null` will change to `false`. " +
|
|
798
|
+
"To keep the current behavior, explicitly set `empty_as_null: true`."
|
|
799
|
+
)
|
|
800
|
+
empty_as_null = true
|
|
801
|
+
end
|
|
802
|
+
|
|
803
|
+
Utils.wrap_expr(_rbexpr.arr_explode(empty_as_null, keep_nulls))
|
|
796
804
|
end
|
|
797
805
|
|
|
798
806
|
# Check if sub-arrays contain the given item.
|
|
@@ -614,7 +614,7 @@ module Polars
|
|
|
614
614
|
#
|
|
615
615
|
# @example
|
|
616
616
|
# s = Polars::Series.new("a", [[1, 2, 3], [4, 5, 6]], dtype: Polars::Array.new(Polars::Int64, 3))
|
|
617
|
-
# s.arr.explode
|
|
617
|
+
# s.arr.explode(empty_as_null: false)
|
|
618
618
|
# # =>
|
|
619
619
|
# # shape: (6,)
|
|
620
620
|
# # Series: 'a' [i64]
|
|
@@ -626,7 +626,7 @@ module Polars
|
|
|
626
626
|
# # 5
|
|
627
627
|
# # 6
|
|
628
628
|
# # ]
|
|
629
|
-
def explode(empty_as_null:
|
|
629
|
+
def explode(empty_as_null: nil, keep_nulls: true)
|
|
630
630
|
super
|
|
631
631
|
end
|
|
632
632
|
|
data/lib/polars/data_frame.rb
CHANGED
|
@@ -771,7 +771,14 @@ module Polars
|
|
|
771
771
|
s.get_index_signed(row)
|
|
772
772
|
end
|
|
773
773
|
|
|
774
|
-
#
|
|
774
|
+
# Collect the underlying arrow arrays in an Arrow Table.
|
|
775
|
+
#
|
|
776
|
+
# @return [Nanoarrow::Array]
|
|
777
|
+
def to_arrow
|
|
778
|
+
require "nanoarrow"
|
|
779
|
+
|
|
780
|
+
Nanoarrow::Array.new(self)
|
|
781
|
+
end
|
|
775
782
|
|
|
776
783
|
# Convert DataFrame to a hash mapping column name to values.
|
|
777
784
|
#
|
|
@@ -810,6 +817,8 @@ module Polars
|
|
|
810
817
|
# df.to_numo.class
|
|
811
818
|
# # => Numo::RObject
|
|
812
819
|
def to_numo
|
|
820
|
+
require "numo/narray"
|
|
821
|
+
|
|
813
822
|
out = _df.to_numo
|
|
814
823
|
if out.nil?
|
|
815
824
|
Numo::NArray.vstack(width.times.map { |i| to_series(i).to_numo }).transpose
|
|
@@ -4526,7 +4535,7 @@ module Polars
|
|
|
4526
4535
|
# "numbers" => [[1], [2, 3], [4, 5], [6, 7, 8]]
|
|
4527
4536
|
# }
|
|
4528
4537
|
# )
|
|
4529
|
-
# df.explode("numbers")
|
|
4538
|
+
# df.explode("numbers", empty_as_null: false)
|
|
4530
4539
|
# # =>
|
|
4531
4540
|
# # shape: (8, 2)
|
|
4532
4541
|
# # ┌─────────┬─────────┐
|
|
@@ -4546,7 +4555,7 @@ module Polars
|
|
|
4546
4555
|
def explode(
|
|
4547
4556
|
columns,
|
|
4548
4557
|
*more_columns,
|
|
4549
|
-
empty_as_null:
|
|
4558
|
+
empty_as_null: OMITTED,
|
|
4550
4559
|
keep_nulls: true
|
|
4551
4560
|
)
|
|
4552
4561
|
lazy
|
|
@@ -4961,6 +4970,51 @@ module Polars
|
|
|
4961
4970
|
lazy.shift(n, fill_value: fill_value).collect(optimizations: QueryOptFlags._eager)
|
|
4962
4971
|
end
|
|
4963
4972
|
|
|
4973
|
+
# Check whether the DataFrame is sorted by the given columns.
|
|
4974
|
+
#
|
|
4975
|
+
# @param by [Object]
|
|
4976
|
+
# Column name(s) to check.
|
|
4977
|
+
# @param more_by [Array]
|
|
4978
|
+
# Additional column names.
|
|
4979
|
+
# @param descending [Boolean]
|
|
4980
|
+
# Sort in descending order. When sorting by multiple columns, can be
|
|
4981
|
+
# specified per column by passing a sequence of booleans.
|
|
4982
|
+
# @param nulls_last [Boolean]
|
|
4983
|
+
# Place null values last. When sorting by multiple columns, can be
|
|
4984
|
+
# specified per column by passing a sequence of booleans.
|
|
4985
|
+
#
|
|
4986
|
+
# @return [Boolean]
|
|
4987
|
+
#
|
|
4988
|
+
# @example
|
|
4989
|
+
# df = Polars::DataFrame.new({"a" => [1, 2, 3], "b" => [5, 4, 3]})
|
|
4990
|
+
# df.is_sorted("a")
|
|
4991
|
+
# # => true
|
|
4992
|
+
#
|
|
4993
|
+
# @example
|
|
4994
|
+
# df.is_sorted("b", descending: true)
|
|
4995
|
+
# # => true
|
|
4996
|
+
#
|
|
4997
|
+
# @example
|
|
4998
|
+
# df.is_sorted("a", "b")
|
|
4999
|
+
# # => true
|
|
5000
|
+
def is_sorted(
|
|
5001
|
+
by,
|
|
5002
|
+
*more_by,
|
|
5003
|
+
descending: false,
|
|
5004
|
+
nulls_last: false
|
|
5005
|
+
)
|
|
5006
|
+
by = Array(by)
|
|
5007
|
+
by.concat(more_by)
|
|
5008
|
+
n = by.length
|
|
5009
|
+
if Utils.bool?(descending)
|
|
5010
|
+
descending = [descending] * n
|
|
5011
|
+
end
|
|
5012
|
+
if Utils.bool?(nulls_last)
|
|
5013
|
+
nulls_last = [nulls_last] * n
|
|
5014
|
+
end
|
|
5015
|
+
_df.is_sorted(by, descending, nulls_last)
|
|
5016
|
+
end
|
|
5017
|
+
|
|
4964
5018
|
# Get a mask of all duplicated rows in this DataFrame.
|
|
4965
5019
|
#
|
|
4966
5020
|
# @return [Series]
|
data/lib/polars/expr.rb
CHANGED
|
@@ -1576,7 +1576,7 @@ module Polars
|
|
|
1576
1576
|
# @example Get the top 2 rows by column `a` in each group.
|
|
1577
1577
|
# df.group_by("c", maintain_order: true)
|
|
1578
1578
|
# .agg(Polars.all.top_k_by("a", k: 2))
|
|
1579
|
-
# .explode(Polars.all.exclude("c"))
|
|
1579
|
+
# .explode(Polars.all.exclude("c"), empty_as_null: false)
|
|
1580
1580
|
# # =>
|
|
1581
1581
|
# # shape: (5, 3)
|
|
1582
1582
|
# # ┌────────┬─────┬─────┐
|
|
@@ -1721,7 +1721,7 @@ module Polars
|
|
|
1721
1721
|
# @example Get the bottom 2 rows by column `a` in each group.
|
|
1722
1722
|
# df.group_by("c", maintain_order: true)
|
|
1723
1723
|
# .agg(Polars.all.bottom_k_by("a", k: 2))
|
|
1724
|
-
# .explode(Polars.all.exclude("c"))
|
|
1724
|
+
# .explode(Polars.all.exclude("c"), empty_as_null: false)
|
|
1725
1725
|
# # =>
|
|
1726
1726
|
# # shape: (5, 3)
|
|
1727
1727
|
# # ┌────────┬─────┬─────┐
|
|
@@ -3677,7 +3677,7 @@ module Polars
|
|
|
3677
3677
|
#
|
|
3678
3678
|
# @example
|
|
3679
3679
|
# df = Polars::DataFrame.new({"b" => [[1, 2, 3], [4, 5, 6]]})
|
|
3680
|
-
# df.select(Polars.col("b").explode)
|
|
3680
|
+
# df.select(Polars.col("b").explode(empty_as_null: false))
|
|
3681
3681
|
# # =>
|
|
3682
3682
|
# # shape: (6, 1)
|
|
3683
3683
|
# # ┌─────┐
|
|
@@ -3692,7 +3692,15 @@ module Polars
|
|
|
3692
3692
|
# # │ 5 │
|
|
3693
3693
|
# # │ 6 │
|
|
3694
3694
|
# # └─────┘
|
|
3695
|
-
def explode(empty_as_null:
|
|
3695
|
+
def explode(empty_as_null: OMITTED, keep_nulls: true)
|
|
3696
|
+
if empty_as_null == OMITTED
|
|
3697
|
+
Utils.issue_deprecation_warning(
|
|
3698
|
+
"The default behavior for `empty_as_null` will change to `false`. " +
|
|
3699
|
+
"To keep the current behavior, explicitly set `empty_as_null: true`."
|
|
3700
|
+
)
|
|
3701
|
+
empty_as_null = true
|
|
3702
|
+
end
|
|
3703
|
+
|
|
3696
3704
|
wrap_expr(_rbexpr.explode(empty_as_null, keep_nulls))
|
|
3697
3705
|
end
|
|
3698
3706
|
|
|
@@ -4604,6 +4612,66 @@ module Polars
|
|
|
4604
4612
|
wrap_expr(_rbexpr.is_close(other, abs_tol, rel_tol, nans_equal))
|
|
4605
4613
|
end
|
|
4606
4614
|
|
|
4615
|
+
# Checks if an expression is sorted.
|
|
4616
|
+
#
|
|
4617
|
+
# If `descending` and/or `nulls_last` are nil, it will check `true` and `false`
|
|
4618
|
+
# for the unspecified option(s), and return `true` if the expression is sorted
|
|
4619
|
+
# under any combination of those settings.
|
|
4620
|
+
#
|
|
4621
|
+
# @param descending [Boolean]
|
|
4622
|
+
# Checks if the expression is sorted in descending order.
|
|
4623
|
+
# Defaults to false.
|
|
4624
|
+
# @param nulls_last [Boolean]
|
|
4625
|
+
# Consider null values as being ordered last when checking sortedness.
|
|
4626
|
+
# Defaults to false.
|
|
4627
|
+
#
|
|
4628
|
+
# @return [Expr]
|
|
4629
|
+
#
|
|
4630
|
+
# @example Check if a column is sorted in ascending order.
|
|
4631
|
+
# df = Polars::DataFrame.new({"a" => [1, 2, 3, 4]})
|
|
4632
|
+
# df.select(Polars.col("a").is_sorted)
|
|
4633
|
+
# # =>
|
|
4634
|
+
# # shape: (1, 1)
|
|
4635
|
+
# # ┌──────┐
|
|
4636
|
+
# # │ a │
|
|
4637
|
+
# # │ --- │
|
|
4638
|
+
# # │ bool │
|
|
4639
|
+
# # ╞══════╡
|
|
4640
|
+
# # │ true │
|
|
4641
|
+
# # └──────┘
|
|
4642
|
+
#
|
|
4643
|
+
# @example Check if a column is sorted in descending order.
|
|
4644
|
+
# df = Polars::DataFrame.new({"a" => [4, 3, 2, 1]})
|
|
4645
|
+
# df.select(Polars.col("a").is_sorted(descending: true))
|
|
4646
|
+
# # =>
|
|
4647
|
+
# # shape: (1, 1)
|
|
4648
|
+
# # ┌──────┐
|
|
4649
|
+
# # │ a │
|
|
4650
|
+
# # │ --- │
|
|
4651
|
+
# # │ bool │
|
|
4652
|
+
# # ╞══════╡
|
|
4653
|
+
# # │ true │
|
|
4654
|
+
# # └──────┘
|
|
4655
|
+
#
|
|
4656
|
+
# @example Check if a column is sorted in either direction.
|
|
4657
|
+
# df = Polars::DataFrame.new({"a" => [4, 3, 2, 1]})
|
|
4658
|
+
# df.select(Polars.col("a").is_sorted(descending: nil))
|
|
4659
|
+
# # =>
|
|
4660
|
+
# # shape: (1, 1)
|
|
4661
|
+
# # ┌──────┐
|
|
4662
|
+
# # │ a │
|
|
4663
|
+
# # │ --- │
|
|
4664
|
+
# # │ bool │
|
|
4665
|
+
# # ╞══════╡
|
|
4666
|
+
# # │ true │
|
|
4667
|
+
# # └──────┘
|
|
4668
|
+
def is_sorted(
|
|
4669
|
+
descending: false,
|
|
4670
|
+
nulls_last: false
|
|
4671
|
+
)
|
|
4672
|
+
wrap_expr(_rbexpr.is_sorted(descending, nulls_last))
|
|
4673
|
+
end
|
|
4674
|
+
|
|
4607
4675
|
# Hash the elements in the selection.
|
|
4608
4676
|
#
|
|
4609
4677
|
# The hash value is of type `UInt64`.
|
|
@@ -7583,7 +7651,7 @@ module Polars
|
|
|
7583
7651
|
# # │ --- │
|
|
7584
7652
|
# # │ f64 │
|
|
7585
7653
|
# # ╞══════════╡
|
|
7586
|
-
# # │
|
|
7654
|
+
# # │ null │
|
|
7587
7655
|
# # │ 0.707107 │
|
|
7588
7656
|
# # │ 0.963624 │
|
|
7589
7657
|
# # └──────────┘
|
|
@@ -7615,7 +7683,7 @@ module Polars
|
|
|
7615
7683
|
# # │ --- │
|
|
7616
7684
|
# # │ f64 │
|
|
7617
7685
|
# # ╞══════════╡
|
|
7618
|
-
# # │
|
|
7686
|
+
# # │ null │
|
|
7619
7687
|
# # │ 0.5 │
|
|
7620
7688
|
# # │ 0.928571 │
|
|
7621
7689
|
# # └──────────┘
|
|
@@ -242,10 +242,10 @@ module Polars
|
|
|
242
242
|
rbexprs = Utils.parse_into_list_of_expressions(*exprs)
|
|
243
243
|
exprs_wrapped = rbexprs.map { |e| Utils.wrap_expr(e) }
|
|
244
244
|
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
)
|
|
245
|
+
Polars.cum_fold(
|
|
246
|
+
Polars.lit(0).cast(Polars.dtype_of(Polars.sum_horizontal(exprs))),
|
|
247
|
+
exprs_wrapped
|
|
248
|
+
) { |a, b| a + b }.alias("cum_sum")
|
|
249
249
|
end
|
|
250
250
|
end
|
|
251
251
|
end
|
|
@@ -95,7 +95,7 @@ module Polars
|
|
|
95
95
|
# # │ 2 ┆ 4 ┆ 5 ┆ null │
|
|
96
96
|
# # │ 3 ┆ null ┆ 6 ┆ 8 │
|
|
97
97
|
# # └─────┴──────┴──────┴──────┘
|
|
98
|
-
def concat(items, rechunk: false, how: "vertical", parallel: true, strict:
|
|
98
|
+
def concat(items, rechunk: false, how: "vertical", parallel: true, strict: nil)
|
|
99
99
|
elems = items.to_a
|
|
100
100
|
|
|
101
101
|
if elems.empty?
|
|
@@ -303,7 +303,7 @@ module Polars
|
|
|
303
303
|
def union(
|
|
304
304
|
items,
|
|
305
305
|
how: "vertical",
|
|
306
|
-
strict:
|
|
306
|
+
strict: nil
|
|
307
307
|
)
|
|
308
308
|
elems = items.to_a
|
|
309
309
|
|