polars-df 0.26.0 → 0.26.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/Cargo.lock +11 -12
- data/Cargo.toml +0 -3
- data/README.md +6 -0
- data/ext/polars/Cargo.toml +1 -1
- data/ext/polars/src/conversion/mod.rs +50 -1
- data/ext/polars/src/interop/arrow/to_rb.rs +17 -12
- data/ext/polars/src/lazyframe/general.rs +4 -3
- data/ext/polars/src/lib.rs +7 -4
- data/ext/polars/src/ruby/capsule.rs +27 -0
- data/ext/polars/src/ruby/mod.rs +1 -0
- data/ext/polars/src/series/export.rs +5 -0
- data/lib/polars/data_frame.rb +10 -1
- data/lib/polars/functions/aggregation/horizontal.rb +4 -4
- data/lib/polars/iceberg_dataset.rb +81 -14
- data/lib/polars/io/delta.rb +4 -0
- data/lib/polars/schema.rb +9 -0
- data/lib/polars/series.rb +18 -0
- data/lib/polars/version.rb +1 -1
- data/lib/polars.rb +4 -0
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6e17df47d46e8c53ba63698ff3d91f67cd4eb4f9d7d1833a676c12c743ca7adb
|
|
4
|
+
data.tar.gz: a5d055718adbcc275b35f8a069be8b25261fb34dcdba3c64dea8d444982e0072
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 7a928541df7f9f4a0cb4cc034935729f99c34ad66444e5ac17b969e1ead01e45077de7b7a75b0bf039e13d055160324374e8deab4ff30b3d2230c061852d11af
|
|
7
|
+
data.tar.gz: 6838a74b47b629afb8f6cf263f41743488ca027ee4ed46d42b19d55a251ad0920c86654cbbacee0245251cabafe317c9475701deb011b2cbfbd7fa7115d0a5eb
|
data/CHANGELOG.md
CHANGED
data/Cargo.lock
CHANGED
|
@@ -530,9 +530,9 @@ dependencies = [
|
|
|
530
530
|
|
|
531
531
|
[[package]]
|
|
532
532
|
name = "crossbeam-epoch"
|
|
533
|
-
version = "0.9.
|
|
533
|
+
version = "0.9.20"
|
|
534
534
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
535
|
-
checksum = "
|
|
535
|
+
checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
|
|
536
536
|
dependencies = [
|
|
537
537
|
"crossbeam-utils",
|
|
538
538
|
]
|
|
@@ -650,9 +650,9 @@ dependencies = [
|
|
|
650
650
|
|
|
651
651
|
[[package]]
|
|
652
652
|
name = "ethnum"
|
|
653
|
-
version = "1.5.
|
|
653
|
+
version = "1.5.3"
|
|
654
654
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
655
|
-
checksum = "
|
|
655
|
+
checksum = "40404c3f5f511ec4da6fe866ddf6a717c309fdbb69fbbad7b0f3edab8f2e835f"
|
|
656
656
|
|
|
657
657
|
[[package]]
|
|
658
658
|
name = "event-listener"
|
|
@@ -1341,12 +1341,11 @@ checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de"
|
|
|
1341
1341
|
|
|
1342
1342
|
[[package]]
|
|
1343
1343
|
name = "libmimalloc-sys"
|
|
1344
|
-
version = "0.1.
|
|
1344
|
+
version = "0.1.49"
|
|
1345
1345
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1346
|
-
checksum = "
|
|
1346
|
+
checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9"
|
|
1347
1347
|
dependencies = [
|
|
1348
1348
|
"cc",
|
|
1349
|
-
"libc",
|
|
1350
1349
|
]
|
|
1351
1350
|
|
|
1352
1351
|
[[package]]
|
|
@@ -1459,9 +1458,9 @@ checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
|
|
|
1459
1458
|
|
|
1460
1459
|
[[package]]
|
|
1461
1460
|
name = "memmap2"
|
|
1462
|
-
version = "0.9.
|
|
1461
|
+
version = "0.9.11"
|
|
1463
1462
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1464
|
-
checksum = "
|
|
1463
|
+
checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0"
|
|
1465
1464
|
dependencies = [
|
|
1466
1465
|
"libc",
|
|
1467
1466
|
]
|
|
@@ -2248,7 +2247,7 @@ dependencies = [
|
|
|
2248
2247
|
|
|
2249
2248
|
[[package]]
|
|
2250
2249
|
name = "polars-ruby"
|
|
2251
|
-
version = "0.26.
|
|
2250
|
+
version = "0.26.1"
|
|
2252
2251
|
dependencies = [
|
|
2253
2252
|
"ahash",
|
|
2254
2253
|
"bytes",
|
|
@@ -2500,9 +2499,9 @@ dependencies = [
|
|
|
2500
2499
|
|
|
2501
2500
|
[[package]]
|
|
2502
2501
|
name = "quinn-proto"
|
|
2503
|
-
version = "0.11.
|
|
2502
|
+
version = "0.11.15"
|
|
2504
2503
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2505
|
-
checksum = "
|
|
2504
|
+
checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e"
|
|
2506
2505
|
dependencies = [
|
|
2507
2506
|
"bytes",
|
|
2508
2507
|
"getrandom 0.3.3",
|
data/Cargo.toml
CHANGED
data/README.md
CHANGED
data/ext/polars/Cargo.toml
CHANGED
|
@@ -39,6 +39,7 @@ use crate::object::OBJECT_NAME;
|
|
|
39
39
|
use crate::rb_modules::pl_series;
|
|
40
40
|
use crate::ruby::gvl::GvlExt;
|
|
41
41
|
use crate::ruby::utils::TryIntoValue;
|
|
42
|
+
use crate::series::import_schema_rbcapsule;
|
|
42
43
|
use crate::utils::to_rb_err;
|
|
43
44
|
use crate::{
|
|
44
45
|
RbDataFrame, RbExpr, RbLazyFrame, RbPolarsErr, RbResult, RbSeries, RbTypeError, RbValueError,
|
|
@@ -515,7 +516,28 @@ impl TryConvert for Wrap<Schema> {
|
|
|
515
516
|
impl TryConvert for Wrap<ArrowSchema> {
|
|
516
517
|
fn try_convert(ob: Value) -> RbResult<Self> {
|
|
517
518
|
let ruby = Ruby::get_with(ob);
|
|
518
|
-
|
|
519
|
+
|
|
520
|
+
if let Ok(schema_capsule) = ob.funcall::<_, _, Value>("arrow_c_schema", ()) {
|
|
521
|
+
let field = import_schema_rbcapsule(schema_capsule)?;
|
|
522
|
+
|
|
523
|
+
let ArrowDataType::Struct(fields) = field.dtype else {
|
|
524
|
+
return Err(RbValueError::new_err(format!(
|
|
525
|
+
"arrow_c_schema of object did not return struct dtype: \
|
|
526
|
+
object: {:?}, dtype: {:?}",
|
|
527
|
+
ob, &field.dtype
|
|
528
|
+
)));
|
|
529
|
+
};
|
|
530
|
+
|
|
531
|
+
let mut schema = ArrowSchema::from_iter_check_duplicates(fields).unwrap();
|
|
532
|
+
|
|
533
|
+
if let Some(md) = field.metadata {
|
|
534
|
+
*schema.metadata_mut() = Arc::unwrap_or_clone(md);
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
return Ok(Wrap(schema));
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
// TODO remove in 0.27.0
|
|
519
541
|
let ob = RHash::try_convert(ob)?;
|
|
520
542
|
let fields: RArray = ob.aref(ruby.to_symbol("fields"))?;
|
|
521
543
|
let mut arrow_schema = ArrowSchema::with_capacity(fields.len());
|
|
@@ -546,6 +568,33 @@ impl TryConvert for Wrap<ArrowSchema> {
|
|
|
546
568
|
"binary_view" => ArrowDataType::BinaryView,
|
|
547
569
|
"string_view" => ArrowDataType::Utf8View,
|
|
548
570
|
"unknown" => ArrowDataType::Unknown,
|
|
571
|
+
"timestamp" => {
|
|
572
|
+
let time_unit: String = f.aref(ruby.to_symbol("time_unit"))?;
|
|
573
|
+
let time_zone: Option<String> = f.aref(ruby.to_symbol("time_zone"))?;
|
|
574
|
+
let arrow_time_unit = match time_unit.as_str() {
|
|
575
|
+
"us" => ArrowTimeUnit::Microsecond,
|
|
576
|
+
"ns" => ArrowTimeUnit::Nanosecond,
|
|
577
|
+
_ => todo!(),
|
|
578
|
+
};
|
|
579
|
+
ArrowDataType::Timestamp(arrow_time_unit, time_zone.map(|v| v.into()))
|
|
580
|
+
}
|
|
581
|
+
"time64" => {
|
|
582
|
+
let time_unit: String = f.aref(ruby.to_symbol("time_unit"))?;
|
|
583
|
+
let arrow_time_unit = match time_unit.as_str() {
|
|
584
|
+
"us" => ArrowTimeUnit::Microsecond,
|
|
585
|
+
_ => todo!(),
|
|
586
|
+
};
|
|
587
|
+
ArrowDataType::Time64(arrow_time_unit)
|
|
588
|
+
}
|
|
589
|
+
"decimal" => {
|
|
590
|
+
let precision: usize = f.aref(ruby.to_symbol("precision"))?;
|
|
591
|
+
let scale: usize = f.aref(ruby.to_symbol("scale"))?;
|
|
592
|
+
ArrowDataType::Decimal(precision, scale)
|
|
593
|
+
}
|
|
594
|
+
"fixed_size_binary" => {
|
|
595
|
+
let limit: usize = f.aref(ruby.to_symbol("limit"))?;
|
|
596
|
+
ArrowDataType::FixedSizeBinary(limit)
|
|
597
|
+
}
|
|
549
598
|
_ => todo!(),
|
|
550
599
|
};
|
|
551
600
|
let is_nullable = f.aref(ruby.to_symbol("nullable"))?;
|
|
@@ -6,20 +6,12 @@ use polars::frame::DataFrame;
|
|
|
6
6
|
use polars::prelude::{ArrayRef, ArrowField, PlSmallStr, PolarsResult, SchemaExt};
|
|
7
7
|
use polars::series::Series;
|
|
8
8
|
use polars_core::utils::arrow;
|
|
9
|
+
use std::ffi::CString;
|
|
9
10
|
|
|
10
11
|
use crate::RbResult;
|
|
12
|
+
use crate::ruby::capsule::RbCapsule;
|
|
11
13
|
|
|
12
|
-
|
|
13
|
-
pub struct RbArrowArrayStream {
|
|
14
|
-
pub(crate) stream: ffi::ArrowArrayStream,
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
impl RbArrowArrayStream {
|
|
18
|
-
pub fn to_i(&self) -> usize {
|
|
19
|
-
(&self.stream as *const _) as usize
|
|
20
|
-
}
|
|
21
|
-
}
|
|
22
|
-
|
|
14
|
+
// TODO switch to RbCapsule in 0.27.0
|
|
23
15
|
#[magnus::wrap(class = "Polars::ArrowSchema")]
|
|
24
16
|
pub struct RbArrowSchema {
|
|
25
17
|
pub(crate) schema: ffi::ArrowSchema,
|
|
@@ -31,11 +23,24 @@ impl RbArrowSchema {
|
|
|
31
23
|
}
|
|
32
24
|
}
|
|
33
25
|
|
|
26
|
+
pub(crate) fn series_to_stream(series: &Series, ruby: &Ruby) -> RbResult<Value> {
|
|
27
|
+
let field = series.field().to_arrow(CompatLevel::newest());
|
|
28
|
+
let series = series.clone();
|
|
29
|
+
let iter = Box::new(
|
|
30
|
+
(0..series.n_chunks()).map(move |i| Ok(series.to_arrow(i, CompatLevel::newest()))),
|
|
31
|
+
) as _;
|
|
32
|
+
|
|
33
|
+
let stream = ffi::export_iterator(iter, field);
|
|
34
|
+
let stream_capsule_name = CString::new("arrow_array_stream").unwrap();
|
|
35
|
+
Ok(RbCapsule::new(stream, Some(stream_capsule_name)).into_value_with(ruby))
|
|
36
|
+
}
|
|
37
|
+
|
|
34
38
|
pub(crate) fn dataframe_to_stream(df: &DataFrame, ruby: &Ruby) -> RbResult<Value> {
|
|
35
39
|
let iter = Box::new(DataFrameStreamIterator::new(df));
|
|
36
40
|
let field = iter.field();
|
|
37
41
|
let stream = ffi::export_iterator(iter, field);
|
|
38
|
-
|
|
42
|
+
let stream_capsule_name = CString::new("arrow_array_stream").unwrap();
|
|
43
|
+
Ok(RbCapsule::new(stream, Some(stream_capsule_name)).into_value_with(ruby))
|
|
39
44
|
}
|
|
40
45
|
|
|
41
46
|
pub(crate) fn polars_schema_to_rbcapsule(
|
|
@@ -10,6 +10,7 @@ use polars::prelude::*;
|
|
|
10
10
|
use polars_core::query_result::QueryResult;
|
|
11
11
|
use polars_plan::dsl::ScanSources;
|
|
12
12
|
use polars_plan::plans::{HintIR, Sorted};
|
|
13
|
+
use std::ffi::CString;
|
|
13
14
|
use std::num::NonZeroUsize;
|
|
14
15
|
|
|
15
16
|
use super::{RbLazyFrame, RbOptFlags};
|
|
@@ -27,8 +28,7 @@ use crate::ruby::ruby_function::RubyObject;
|
|
|
27
28
|
use crate::ruby::utils::TryIntoValue;
|
|
28
29
|
use crate::utils::{EnterPolarsExt, to_rb_err};
|
|
29
30
|
use crate::{
|
|
30
|
-
|
|
31
|
-
RbValueError,
|
|
31
|
+
RbCapsule, RbDataFrame, RbExpr, RbLazyGroupBy, RbPolarsErr, RbResult, RbTypeError, RbValueError,
|
|
32
32
|
};
|
|
33
33
|
|
|
34
34
|
fn rbobject_to_first_path_and_scan_sources(
|
|
@@ -1314,7 +1314,8 @@ impl RbCollectBatches {
|
|
|
1314
1314
|
let iter = Box::new(ArrowStreamIterator::new(self_.inner.clone(), dtype.clone()));
|
|
1315
1315
|
let field = ArrowField::new(PlSmallStr::EMPTY, dtype, false);
|
|
1316
1316
|
let stream = export_iterator(iter, field);
|
|
1317
|
-
|
|
1317
|
+
let stream_capsule_name = CString::new("arrow_array_stream").unwrap();
|
|
1318
|
+
Ok(RbCapsule::new(stream, Some(stream_capsule_name)).into_value_with(rb))
|
|
1318
1319
|
}
|
|
1319
1320
|
}
|
|
1320
1321
|
|
data/ext/polars/src/lib.rs
CHANGED
|
@@ -37,10 +37,11 @@ use expr::RbExpr;
|
|
|
37
37
|
use expr::datatype::RbDataTypeExpr;
|
|
38
38
|
use expr::selector::RbSelector;
|
|
39
39
|
use functions::whenthen::{RbChainedThen, RbChainedWhen, RbThen, RbWhen};
|
|
40
|
-
use interop::arrow::to_rb::
|
|
40
|
+
use interop::arrow::to_rb::RbArrowSchema;
|
|
41
41
|
use lazyframe::{RbCollectBatches, RbInProcessQuery, RbLazyFrame, RbOptFlags};
|
|
42
42
|
use lazygroupby::RbLazyGroupBy;
|
|
43
43
|
use magnus::{Ruby, function, method, prelude::*};
|
|
44
|
+
use ruby::capsule::RbCapsule;
|
|
44
45
|
use ruby::exceptions::{RbTypeError, RbValueError};
|
|
45
46
|
use series::RbSeries;
|
|
46
47
|
use sql::RbSQLContext;
|
|
@@ -1135,6 +1136,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
|
|
|
1135
1136
|
class.define_method("to_s", method!(RbSeries::as_str, 0))?;
|
|
1136
1137
|
class.define_method("len", method!(RbSeries::len, 0))?;
|
|
1137
1138
|
class.define_method("to_a", method!(RbSeries::to_a, 0))?;
|
|
1139
|
+
class.define_method("arrow_c_stream", method!(RbSeries::__arrow_c_stream__, 0))?;
|
|
1138
1140
|
class.define_method("median", method!(RbSeries::median, 0))?;
|
|
1139
1141
|
class.define_method("quantile", method!(RbSeries::quantile, 2))?;
|
|
1140
1142
|
class.define_method("_clone", method!(RbSeries::clone, 0))?;
|
|
@@ -1340,9 +1342,10 @@ fn init(ruby: &Ruby) -> RbResult<()> {
|
|
|
1340
1342
|
class.define_method("register", method!(RbSQLContext::register, 2))?;
|
|
1341
1343
|
class.define_method("unregister", method!(RbSQLContext::unregister, 1))?;
|
|
1342
1344
|
|
|
1343
|
-
//
|
|
1344
|
-
let class = module.define_class("
|
|
1345
|
-
class.define_method("to_i", method!(
|
|
1345
|
+
// capsule
|
|
1346
|
+
let class = module.define_class("Capsule", ruby.class_object())?;
|
|
1347
|
+
class.define_method("to_i", method!(RbCapsule::to_i, 0))?;
|
|
1348
|
+
class.define_method("name", method!(RbCapsule::name, 0))?;
|
|
1346
1349
|
|
|
1347
1350
|
// arrow schema
|
|
1348
1351
|
let class = module.define_class("ArrowSchema", ruby.class_object())?;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
use std::any::Any;
|
|
2
|
+
use std::ffi::CString;
|
|
3
|
+
|
|
4
|
+
#[magnus::wrap(class = "Polars::Capsule")]
|
|
5
|
+
pub struct RbCapsule {
|
|
6
|
+
value: Box<dyn Any + Send>,
|
|
7
|
+
name: Option<CString>,
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
impl RbCapsule {
|
|
11
|
+
pub fn new<T: 'static + Send>(value: T, name: Option<CString>) -> Self {
|
|
12
|
+
Self {
|
|
13
|
+
value: Box::new(value),
|
|
14
|
+
name,
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
pub fn to_i(&self) -> usize {
|
|
19
|
+
(&*self.value as *const dyn Any as *const ()) as usize
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
// TODO use &CStr when Magnus supports it
|
|
23
|
+
// https://github.com/matsadler/magnus/pull/182
|
|
24
|
+
pub fn name(&self) -> Option<&str> {
|
|
25
|
+
self.name.as_deref().map(|v| v.to_str().unwrap())
|
|
26
|
+
}
|
|
27
|
+
}
|
data/ext/polars/src/ruby/mod.rs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
use magnus::{IntoValue, Ruby, Value, value::ReprValue};
|
|
2
2
|
use polars_core::prelude::*;
|
|
3
3
|
|
|
4
|
+
use crate::interop::arrow::to_rb::series_to_stream;
|
|
4
5
|
use crate::prelude::*;
|
|
5
6
|
use crate::ruby::utils::TryIntoValue;
|
|
6
7
|
use crate::{RbPolarsErr, RbResult, RbSeries};
|
|
@@ -178,4 +179,8 @@ impl RbSeries {
|
|
|
178
179
|
|
|
179
180
|
to_a_recursive(ruby, series)
|
|
180
181
|
}
|
|
182
|
+
|
|
183
|
+
pub fn __arrow_c_stream__(ruby: &Ruby, self_: &Self) -> RbResult<Value> {
|
|
184
|
+
series_to_stream(&self_.series.read(), ruby)
|
|
185
|
+
}
|
|
181
186
|
}
|
data/lib/polars/data_frame.rb
CHANGED
|
@@ -771,7 +771,14 @@ module Polars
|
|
|
771
771
|
s.get_index_signed(row)
|
|
772
772
|
end
|
|
773
773
|
|
|
774
|
-
#
|
|
774
|
+
# Collect the underlying arrow arrays in an Arrow Table.
|
|
775
|
+
#
|
|
776
|
+
# @return [Nanoarrow::Array]
|
|
777
|
+
def to_arrow
|
|
778
|
+
require "nanoarrow"
|
|
779
|
+
|
|
780
|
+
Nanoarrow::Array.new(self)
|
|
781
|
+
end
|
|
775
782
|
|
|
776
783
|
# Convert DataFrame to a hash mapping column name to values.
|
|
777
784
|
#
|
|
@@ -810,6 +817,8 @@ module Polars
|
|
|
810
817
|
# df.to_numo.class
|
|
811
818
|
# # => Numo::RObject
|
|
812
819
|
def to_numo
|
|
820
|
+
require "numo/narray"
|
|
821
|
+
|
|
813
822
|
out = _df.to_numo
|
|
814
823
|
if out.nil?
|
|
815
824
|
Numo::NArray.vstack(width.times.map { |i| to_series(i).to_numo }).transpose
|
|
@@ -242,10 +242,10 @@ module Polars
|
|
|
242
242
|
rbexprs = Utils.parse_into_list_of_expressions(*exprs)
|
|
243
243
|
exprs_wrapped = rbexprs.map { |e| Utils.wrap_expr(e) }
|
|
244
244
|
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
)
|
|
245
|
+
Polars.cum_fold(
|
|
246
|
+
Polars.lit(0).cast(Polars.dtype_of(Polars.sum_horizontal(exprs))),
|
|
247
|
+
exprs_wrapped
|
|
248
|
+
) { |a, b| a + b }.alias("cum_sum")
|
|
249
249
|
end
|
|
250
250
|
end
|
|
251
251
|
end
|
|
@@ -17,24 +17,48 @@ module Polars
|
|
|
17
17
|
|
|
18
18
|
table = scan.table
|
|
19
19
|
snapshot = scan.snapshot
|
|
20
|
-
schema = snapshot ? table.schema_by_id(snapshot[:schema_id]) : table.current_schema
|
|
20
|
+
schema = snapshot ? table.schema_by_id(snapshot.is_a?(Hash) ? snapshot[:schema_id] : snapshot.schema_id) : table.current_schema
|
|
21
21
|
|
|
22
|
-
if files.empty?
|
|
23
|
-
# TODO
|
|
22
|
+
if files.empty? && !schema.respond_to?(:arrow_c_schema)
|
|
23
|
+
# TODO remove in 0.27.0
|
|
24
24
|
schema =
|
|
25
25
|
schema.fields.to_h do |field|
|
|
26
26
|
dtype =
|
|
27
27
|
case field[:type]
|
|
28
|
+
when "unknown"
|
|
29
|
+
Unknown
|
|
30
|
+
when "boolean"
|
|
31
|
+
Boolean
|
|
28
32
|
when "int"
|
|
29
|
-
|
|
33
|
+
Int32
|
|
30
34
|
when "long"
|
|
31
|
-
|
|
35
|
+
Int64
|
|
36
|
+
when "float"
|
|
37
|
+
Float32
|
|
32
38
|
when "double"
|
|
33
|
-
|
|
39
|
+
Float64
|
|
40
|
+
when "decimal"
|
|
41
|
+
Decimal.new(field[:precision], field[:scale])
|
|
34
42
|
when "string"
|
|
35
|
-
|
|
43
|
+
String
|
|
44
|
+
when "uuid"
|
|
45
|
+
Binary
|
|
46
|
+
when "fixed"
|
|
47
|
+
Binary
|
|
48
|
+
when "binary"
|
|
49
|
+
Binary
|
|
50
|
+
when "date"
|
|
51
|
+
Date
|
|
52
|
+
when "time"
|
|
53
|
+
Time # ns instead of us since does not support time unit
|
|
36
54
|
when "timestamp"
|
|
37
|
-
|
|
55
|
+
Datetime.new("us")
|
|
56
|
+
when "timestamp_ns"
|
|
57
|
+
Datetime.new("ns")
|
|
58
|
+
when "timestamptz"
|
|
59
|
+
Datetime.new("us", "+00:00")
|
|
60
|
+
when "timestamptz_ns"
|
|
61
|
+
Datetime.new("ns", "+00:00")
|
|
38
62
|
else
|
|
39
63
|
raise Todo
|
|
40
64
|
end
|
|
@@ -44,25 +68,26 @@ module Polars
|
|
|
44
68
|
|
|
45
69
|
LazyFrame.new(schema: schema)
|
|
46
70
|
else
|
|
47
|
-
sources = files.map { |v| v[:data_file_path] }
|
|
71
|
+
sources = files.map { |v| v.is_a?(Hash) ? v[:data_file_path] : v.file.file_path }
|
|
48
72
|
|
|
49
73
|
column_mapping = [
|
|
50
74
|
"iceberg-column-mapping",
|
|
51
|
-
arrow_schema(schema)
|
|
75
|
+
schema.respond_to?(:arrow_c_schema) ? schema : arrow_schema(schema)
|
|
52
76
|
]
|
|
53
77
|
|
|
54
78
|
deletion_files = [
|
|
55
79
|
"iceberg-position-delete",
|
|
56
80
|
files.map.with_index
|
|
57
|
-
.select { |v, i| v[:deletes].any? }
|
|
58
|
-
.to_h { |v, i| [i, v[:deletes].map { |d| d[:file_path] }] }
|
|
81
|
+
.select { |v, i| (v.is_a?(Hash) ? v[:deletes] : v.delete_files).any? }
|
|
82
|
+
.to_h { |v, i| [i, v.is_a?(Hash) ? v[:deletes].map { |d| d[:file_path] } : v.delete_files.map(&:file_path)] }
|
|
59
83
|
]
|
|
60
84
|
|
|
61
85
|
scan_options = {
|
|
62
|
-
|
|
86
|
+
schema: schema.respond_to?(:arrow_c_schema) ? Schema.new(schema) : nil,
|
|
63
87
|
cast_options: Polars::ScanCastOptions._default_iceberg,
|
|
64
88
|
missing_columns: "insert",
|
|
65
89
|
extra_columns: "ignore",
|
|
90
|
+
storage_options: @storage_options,
|
|
66
91
|
_column_mapping: column_mapping,
|
|
67
92
|
_deletion_files: deletion_files
|
|
68
93
|
}
|
|
@@ -73,11 +98,14 @@ module Polars
|
|
|
73
98
|
|
|
74
99
|
private
|
|
75
100
|
|
|
101
|
+
# TODO remove in 0.27.0
|
|
76
102
|
def arrow_schema(schema)
|
|
77
103
|
fields =
|
|
78
104
|
schema.fields.map do |field|
|
|
79
105
|
type =
|
|
80
106
|
case field[:type]
|
|
107
|
+
when "unknown"
|
|
108
|
+
"unknown"
|
|
81
109
|
when "boolean"
|
|
82
110
|
"boolean"
|
|
83
111
|
when "int"
|
|
@@ -88,11 +116,42 @@ module Polars
|
|
|
88
116
|
"float32"
|
|
89
117
|
when "double"
|
|
90
118
|
"float64"
|
|
119
|
+
when "decimal"
|
|
120
|
+
"decimal"
|
|
121
|
+
when "string"
|
|
122
|
+
"string"
|
|
123
|
+
when "uuid"
|
|
124
|
+
limit = 16
|
|
125
|
+
"fixed_size_binary"
|
|
126
|
+
when "fixed"
|
|
127
|
+
limit = field[:limit]
|
|
128
|
+
"fixed_size_binary"
|
|
129
|
+
when "binary"
|
|
130
|
+
"large_binary"
|
|
131
|
+
when "date"
|
|
132
|
+
"date32"
|
|
133
|
+
when "time"
|
|
134
|
+
time_unit = "us"
|
|
135
|
+
"time64"
|
|
136
|
+
when "timestamp"
|
|
137
|
+
time_unit = "us"
|
|
138
|
+
"timestamp"
|
|
139
|
+
when "timestamp_ns"
|
|
140
|
+
time_unit = "ns"
|
|
141
|
+
"timestamp"
|
|
142
|
+
when "timestamptz"
|
|
143
|
+
time_unit = "us"
|
|
144
|
+
time_zone = "+00:00"
|
|
145
|
+
"timestamp"
|
|
146
|
+
when "timestamptz_ns"
|
|
147
|
+
time_unit = "ns"
|
|
148
|
+
time_zone = "+00:00"
|
|
149
|
+
"timestamp"
|
|
91
150
|
else
|
|
92
151
|
raise Todo
|
|
93
152
|
end
|
|
94
153
|
|
|
95
|
-
{
|
|
154
|
+
arrow_field = {
|
|
96
155
|
name: field[:name],
|
|
97
156
|
type: type,
|
|
98
157
|
nullable: !field[:required],
|
|
@@ -100,6 +159,14 @@ module Polars
|
|
|
100
159
|
"PARQUET:field_id" => field[:id].to_s
|
|
101
160
|
}
|
|
102
161
|
}
|
|
162
|
+
arrow_field[:limit] = limit if limit
|
|
163
|
+
if type == "decimal"
|
|
164
|
+
arrow_field[:precision] = field[:precision]
|
|
165
|
+
arrow_field[:scale] = field[:scale]
|
|
166
|
+
end
|
|
167
|
+
arrow_field[:time_unit] = time_unit if time_unit
|
|
168
|
+
arrow_field[:time_zone] = time_zone if time_zone
|
|
169
|
+
arrow_field
|
|
103
170
|
end
|
|
104
171
|
|
|
105
172
|
{fields: fields}
|
data/lib/polars/io/delta.rb
CHANGED
|
@@ -25,6 +25,8 @@ module Polars
|
|
|
25
25
|
storage_options: nil,
|
|
26
26
|
delta_table_options: nil
|
|
27
27
|
)
|
|
28
|
+
require "deltalake-rb"
|
|
29
|
+
|
|
28
30
|
df =
|
|
29
31
|
scan_delta(
|
|
30
32
|
source,
|
|
@@ -62,6 +64,8 @@ module Polars
|
|
|
62
64
|
delta_table_options: nil,
|
|
63
65
|
rechunk: nil
|
|
64
66
|
)
|
|
67
|
+
require "deltalake-rb"
|
|
68
|
+
|
|
65
69
|
dl_tbl =
|
|
66
70
|
_get_delta_lake_table(
|
|
67
71
|
source,
|
data/lib/polars/schema.rb
CHANGED
|
@@ -71,6 +71,15 @@ module Polars
|
|
|
71
71
|
@schema.values
|
|
72
72
|
end
|
|
73
73
|
|
|
74
|
+
# Convert the schema to a nanoarrow schema.
|
|
75
|
+
#
|
|
76
|
+
# @return [Nanoarrow::Schema]
|
|
77
|
+
def to_arrow
|
|
78
|
+
require "nanoarrow"
|
|
79
|
+
|
|
80
|
+
Nanoarrow::Schema.new(self)
|
|
81
|
+
end
|
|
82
|
+
|
|
74
83
|
# Get the number of schema entries.
|
|
75
84
|
#
|
|
76
85
|
# @return [Integer]
|
data/lib/polars/series.rb
CHANGED
|
@@ -73,6 +73,8 @@ module Polars
|
|
|
73
73
|
self._s = Utils.dataframe_to_rbseries(
|
|
74
74
|
original_name, values, dtype: dtype, strict: strict
|
|
75
75
|
)
|
|
76
|
+
elsif values.respond_to?(:arrow_c_stream)
|
|
77
|
+
self._s = RbSeries.from_arrow_c_stream(values)
|
|
76
78
|
else
|
|
77
79
|
raise TypeError, "Series constructor called with unsupported type; got #{values.class.name}"
|
|
78
80
|
end
|
|
@@ -496,6 +498,11 @@ module Polars
|
|
|
496
498
|
end
|
|
497
499
|
end
|
|
498
500
|
|
|
501
|
+
# @private
|
|
502
|
+
def arrow_c_stream
|
|
503
|
+
_s.arrow_c_stream
|
|
504
|
+
end
|
|
505
|
+
|
|
499
506
|
# Return the Series as a scalar, or return the element at the given index.
|
|
500
507
|
#
|
|
501
508
|
# If no index is provided, this is equivalent to `s[0]`, with a check
|
|
@@ -3114,6 +3121,8 @@ module Polars
|
|
|
3114
3121
|
# # Numo::Int64#shape=[3]
|
|
3115
3122
|
# # [1, 2, 3]
|
|
3116
3123
|
def to_numo
|
|
3124
|
+
require "numo/narray"
|
|
3125
|
+
|
|
3117
3126
|
if dtype.temporal?
|
|
3118
3127
|
Numo::RObject.cast(to_a)
|
|
3119
3128
|
else
|
|
@@ -3121,6 +3130,15 @@ module Polars
|
|
|
3121
3130
|
end
|
|
3122
3131
|
end
|
|
3123
3132
|
|
|
3133
|
+
# Return the underlying Arrow array.
|
|
3134
|
+
#
|
|
3135
|
+
# @return [Nanoarrow::Array]
|
|
3136
|
+
def to_arrow
|
|
3137
|
+
require "nanoarrow"
|
|
3138
|
+
|
|
3139
|
+
Nanoarrow::Array.new(self)
|
|
3140
|
+
end
|
|
3141
|
+
|
|
3124
3142
|
# Set masked values.
|
|
3125
3143
|
#
|
|
3126
3144
|
# @param filter [Series]
|
data/lib/polars/version.rb
CHANGED
data/lib/polars.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: polars-df
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.26.
|
|
4
|
+
version: 0.26.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Andrew Kane
|
|
@@ -113,6 +113,7 @@ files:
|
|
|
113
113
|
- ext/polars/src/on_startup.rs
|
|
114
114
|
- ext/polars/src/prelude.rs
|
|
115
115
|
- ext/polars/src/rb_modules.rs
|
|
116
|
+
- ext/polars/src/ruby/capsule.rs
|
|
116
117
|
- ext/polars/src/ruby/exceptions.rs
|
|
117
118
|
- ext/polars/src/ruby/gvl.rs
|
|
118
119
|
- ext/polars/src/ruby/lazy.rs
|
|
@@ -262,7 +263,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
262
263
|
- !ruby/object:Gem::Version
|
|
263
264
|
version: '0'
|
|
264
265
|
requirements: []
|
|
265
|
-
rubygems_version: 4.0.
|
|
266
|
+
rubygems_version: 4.0.14
|
|
266
267
|
specification_version: 4
|
|
267
268
|
summary: Blazingly fast DataFrames for Ruby
|
|
268
269
|
test_files: []
|