polars-df 0.26.0 → 0.26.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 2166cba9f090e8a98fdd28f09ef84bda7130502339522fb724f994581191ffa0
4
- data.tar.gz: bfdd2939e545302b8a050c706112ae59a297054d048daa4551fd1227589eeba6
3
+ metadata.gz: 6e17df47d46e8c53ba63698ff3d91f67cd4eb4f9d7d1833a676c12c743ca7adb
4
+ data.tar.gz: a5d055718adbcc275b35f8a069be8b25261fb34dcdba3c64dea8d444982e0072
5
5
  SHA512:
6
- metadata.gz: 189165a0d01be7174c29ebe5678c9e7bc0cd88e6f0bada0e424967b17b3e56f649645a4bd9de7b409f270390f82be94faf7d12a3bd3ca39de2cc04705382eb78
7
- data.tar.gz: 1daf4e9a15adb6f09497ac729ef5ff4fb33de85df5b65d79bfca5de976c9f09d371e7564c00814c9cf20dc606b413fb708de3ea1aeb44c337e1aacaebd09a457
6
+ metadata.gz: 7a928541df7f9f4a0cb4cc034935729f99c34ad66444e5ac17b969e1ead01e45077de7b7a75b0bf039e13d055160324374e8deab4ff30b3d2230c061852d11af
7
+ data.tar.gz: 6838a74b47b629afb8f6cf263f41743488ca027ee4ed46d42b19d55a251ad0920c86654cbbacee0245251cabafe317c9475701deb011b2cbfbd7fa7115d0a5eb
data/CHANGELOG.md CHANGED
@@ -1,3 +1,8 @@
1
+ ## 0.26.1 (2026-07-22)
2
+
3
+ - Added support for more types to `scan_iceberg` method
4
+ - Fixed return dtype for `cum_sum_horizontal` method
5
+
1
6
  ## 0.26.0 (2026-06-04)
2
7
 
3
8
  - Updated Polars to 0.54.4
data/Cargo.lock CHANGED
@@ -530,9 +530,9 @@ dependencies = [
530
530
 
531
531
  [[package]]
532
532
  name = "crossbeam-epoch"
533
- version = "0.9.18"
533
+ version = "0.9.20"
534
534
  source = "registry+https://github.com/rust-lang/crates.io-index"
535
- checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
535
+ checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
536
536
  dependencies = [
537
537
  "crossbeam-utils",
538
538
  ]
@@ -650,9 +650,9 @@ dependencies = [
650
650
 
651
651
  [[package]]
652
652
  name = "ethnum"
653
- version = "1.5.1"
653
+ version = "1.5.3"
654
654
  source = "registry+https://github.com/rust-lang/crates.io-index"
655
- checksum = "0939f82868b77ef93ce3c3c3daf2b3c526b456741da5a1a4559e590965b6026b"
655
+ checksum = "40404c3f5f511ec4da6fe866ddf6a717c309fdbb69fbbad7b0f3edab8f2e835f"
656
656
 
657
657
  [[package]]
658
658
  name = "event-listener"
@@ -1341,12 +1341,11 @@ checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de"
1341
1341
 
1342
1342
  [[package]]
1343
1343
  name = "libmimalloc-sys"
1344
- version = "0.1.44"
1344
+ version = "0.1.49"
1345
1345
  source = "registry+https://github.com/rust-lang/crates.io-index"
1346
- checksum = "667f4fec20f29dfc6bc7357c582d91796c169ad7e2fce709468aefeb2c099870"
1346
+ checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9"
1347
1347
  dependencies = [
1348
1348
  "cc",
1349
- "libc",
1350
1349
  ]
1351
1350
 
1352
1351
  [[package]]
@@ -1459,9 +1458,9 @@ checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
1459
1458
 
1460
1459
  [[package]]
1461
1460
  name = "memmap2"
1462
- version = "0.9.5"
1461
+ version = "0.9.11"
1463
1462
  source = "registry+https://github.com/rust-lang/crates.io-index"
1464
- checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f"
1463
+ checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0"
1465
1464
  dependencies = [
1466
1465
  "libc",
1467
1466
  ]
@@ -2248,7 +2247,7 @@ dependencies = [
2248
2247
 
2249
2248
  [[package]]
2250
2249
  name = "polars-ruby"
2251
- version = "0.26.0"
2250
+ version = "0.26.1"
2252
2251
  dependencies = [
2253
2252
  "ahash",
2254
2253
  "bytes",
@@ -2500,9 +2499,9 @@ dependencies = [
2500
2499
 
2501
2500
  [[package]]
2502
2501
  name = "quinn-proto"
2503
- version = "0.11.14"
2502
+ version = "0.11.15"
2504
2503
  source = "registry+https://github.com/rust-lang/crates.io-index"
2505
- checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
2504
+ checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e"
2506
2505
  dependencies = [
2507
2506
  "bytes",
2508
2507
  "getrandom 0.3.3",
data/Cargo.toml CHANGED
@@ -4,6 +4,3 @@ resolver = "2"
4
4
 
5
5
  [profile.release]
6
6
  strip = true
7
-
8
- [profile.dev]
9
- strip = true
data/README.md CHANGED
@@ -381,6 +381,12 @@ Delta Lake (experimental)
381
381
  df.write_delta("./table")
382
382
  ```
383
383
 
384
+ Arrow array (experimental, requires [nanoarrow](https://github.com/ankane/nanoarrow-ruby))
385
+
386
+ ```ruby
387
+ df.to_arrow
388
+ ```
389
+
384
390
  Numo array
385
391
 
386
392
  ```ruby
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "polars-ruby"
3
- version = "0.26.0"
3
+ version = "0.26.1"
4
4
  license = "MIT"
5
5
  authors = ["Andrew Kane <andrew@ankane.org>"]
6
6
  edition = "2024"
@@ -39,6 +39,7 @@ use crate::object::OBJECT_NAME;
39
39
  use crate::rb_modules::pl_series;
40
40
  use crate::ruby::gvl::GvlExt;
41
41
  use crate::ruby::utils::TryIntoValue;
42
+ use crate::series::import_schema_rbcapsule;
42
43
  use crate::utils::to_rb_err;
43
44
  use crate::{
44
45
  RbDataFrame, RbExpr, RbLazyFrame, RbPolarsErr, RbResult, RbSeries, RbTypeError, RbValueError,
@@ -515,7 +516,28 @@ impl TryConvert for Wrap<Schema> {
515
516
  impl TryConvert for Wrap<ArrowSchema> {
516
517
  fn try_convert(ob: Value) -> RbResult<Self> {
517
518
  let ruby = Ruby::get_with(ob);
518
- // TODO improve
519
+
520
+ if let Ok(schema_capsule) = ob.funcall::<_, _, Value>("arrow_c_schema", ()) {
521
+ let field = import_schema_rbcapsule(schema_capsule)?;
522
+
523
+ let ArrowDataType::Struct(fields) = field.dtype else {
524
+ return Err(RbValueError::new_err(format!(
525
+ "arrow_c_schema of object did not return struct dtype: \
526
+ object: {:?}, dtype: {:?}",
527
+ ob, &field.dtype
528
+ )));
529
+ };
530
+
531
+ let mut schema = ArrowSchema::from_iter_check_duplicates(fields).unwrap();
532
+
533
+ if let Some(md) = field.metadata {
534
+ *schema.metadata_mut() = Arc::unwrap_or_clone(md);
535
+ }
536
+
537
+ return Ok(Wrap(schema));
538
+ }
539
+
540
+ // TODO remove in 0.27.0
519
541
  let ob = RHash::try_convert(ob)?;
520
542
  let fields: RArray = ob.aref(ruby.to_symbol("fields"))?;
521
543
  let mut arrow_schema = ArrowSchema::with_capacity(fields.len());
@@ -546,6 +568,33 @@ impl TryConvert for Wrap<ArrowSchema> {
546
568
  "binary_view" => ArrowDataType::BinaryView,
547
569
  "string_view" => ArrowDataType::Utf8View,
548
570
  "unknown" => ArrowDataType::Unknown,
571
+ "timestamp" => {
572
+ let time_unit: String = f.aref(ruby.to_symbol("time_unit"))?;
573
+ let time_zone: Option<String> = f.aref(ruby.to_symbol("time_zone"))?;
574
+ let arrow_time_unit = match time_unit.as_str() {
575
+ "us" => ArrowTimeUnit::Microsecond,
576
+ "ns" => ArrowTimeUnit::Nanosecond,
577
+ _ => todo!(),
578
+ };
579
+ ArrowDataType::Timestamp(arrow_time_unit, time_zone.map(|v| v.into()))
580
+ }
581
+ "time64" => {
582
+ let time_unit: String = f.aref(ruby.to_symbol("time_unit"))?;
583
+ let arrow_time_unit = match time_unit.as_str() {
584
+ "us" => ArrowTimeUnit::Microsecond,
585
+ _ => todo!(),
586
+ };
587
+ ArrowDataType::Time64(arrow_time_unit)
588
+ }
589
+ "decimal" => {
590
+ let precision: usize = f.aref(ruby.to_symbol("precision"))?;
591
+ let scale: usize = f.aref(ruby.to_symbol("scale"))?;
592
+ ArrowDataType::Decimal(precision, scale)
593
+ }
594
+ "fixed_size_binary" => {
595
+ let limit: usize = f.aref(ruby.to_symbol("limit"))?;
596
+ ArrowDataType::FixedSizeBinary(limit)
597
+ }
549
598
  _ => todo!(),
550
599
  };
551
600
  let is_nullable = f.aref(ruby.to_symbol("nullable"))?;
@@ -6,20 +6,12 @@ use polars::frame::DataFrame;
6
6
  use polars::prelude::{ArrayRef, ArrowField, PlSmallStr, PolarsResult, SchemaExt};
7
7
  use polars::series::Series;
8
8
  use polars_core::utils::arrow;
9
+ use std::ffi::CString;
9
10
 
10
11
  use crate::RbResult;
12
+ use crate::ruby::capsule::RbCapsule;
11
13
 
12
- #[magnus::wrap(class = "Polars::ArrowArrayStream")]
13
- pub struct RbArrowArrayStream {
14
- pub(crate) stream: ffi::ArrowArrayStream,
15
- }
16
-
17
- impl RbArrowArrayStream {
18
- pub fn to_i(&self) -> usize {
19
- (&self.stream as *const _) as usize
20
- }
21
- }
22
-
14
+ // TODO switch to RbCapsule in 0.27.0
23
15
  #[magnus::wrap(class = "Polars::ArrowSchema")]
24
16
  pub struct RbArrowSchema {
25
17
  pub(crate) schema: ffi::ArrowSchema,
@@ -31,11 +23,24 @@ impl RbArrowSchema {
31
23
  }
32
24
  }
33
25
 
26
+ pub(crate) fn series_to_stream(series: &Series, ruby: &Ruby) -> RbResult<Value> {
27
+ let field = series.field().to_arrow(CompatLevel::newest());
28
+ let series = series.clone();
29
+ let iter = Box::new(
30
+ (0..series.n_chunks()).map(move |i| Ok(series.to_arrow(i, CompatLevel::newest()))),
31
+ ) as _;
32
+
33
+ let stream = ffi::export_iterator(iter, field);
34
+ let stream_capsule_name = CString::new("arrow_array_stream").unwrap();
35
+ Ok(RbCapsule::new(stream, Some(stream_capsule_name)).into_value_with(ruby))
36
+ }
37
+
34
38
  pub(crate) fn dataframe_to_stream(df: &DataFrame, ruby: &Ruby) -> RbResult<Value> {
35
39
  let iter = Box::new(DataFrameStreamIterator::new(df));
36
40
  let field = iter.field();
37
41
  let stream = ffi::export_iterator(iter, field);
38
- Ok(RbArrowArrayStream { stream }.into_value_with(ruby))
42
+ let stream_capsule_name = CString::new("arrow_array_stream").unwrap();
43
+ Ok(RbCapsule::new(stream, Some(stream_capsule_name)).into_value_with(ruby))
39
44
  }
40
45
 
41
46
  pub(crate) fn polars_schema_to_rbcapsule(
@@ -10,6 +10,7 @@ use polars::prelude::*;
10
10
  use polars_core::query_result::QueryResult;
11
11
  use polars_plan::dsl::ScanSources;
12
12
  use polars_plan::plans::{HintIR, Sorted};
13
+ use std::ffi::CString;
13
14
  use std::num::NonZeroUsize;
14
15
 
15
16
  use super::{RbLazyFrame, RbOptFlags};
@@ -27,8 +28,7 @@ use crate::ruby::ruby_function::RubyObject;
27
28
  use crate::ruby::utils::TryIntoValue;
28
29
  use crate::utils::{EnterPolarsExt, to_rb_err};
29
30
  use crate::{
30
- RbArrowArrayStream, RbDataFrame, RbExpr, RbLazyGroupBy, RbPolarsErr, RbResult, RbTypeError,
31
- RbValueError,
31
+ RbCapsule, RbDataFrame, RbExpr, RbLazyGroupBy, RbPolarsErr, RbResult, RbTypeError, RbValueError,
32
32
  };
33
33
 
34
34
  fn rbobject_to_first_path_and_scan_sources(
@@ -1314,7 +1314,8 @@ impl RbCollectBatches {
1314
1314
  let iter = Box::new(ArrowStreamIterator::new(self_.inner.clone(), dtype.clone()));
1315
1315
  let field = ArrowField::new(PlSmallStr::EMPTY, dtype, false);
1316
1316
  let stream = export_iterator(iter, field);
1317
- Ok(RbArrowArrayStream { stream }.into_value_with(rb))
1317
+ let stream_capsule_name = CString::new("arrow_array_stream").unwrap();
1318
+ Ok(RbCapsule::new(stream, Some(stream_capsule_name)).into_value_with(rb))
1318
1319
  }
1319
1320
  }
1320
1321
 
@@ -37,10 +37,11 @@ use expr::RbExpr;
37
37
  use expr::datatype::RbDataTypeExpr;
38
38
  use expr::selector::RbSelector;
39
39
  use functions::whenthen::{RbChainedThen, RbChainedWhen, RbThen, RbWhen};
40
- use interop::arrow::to_rb::{RbArrowArrayStream, RbArrowSchema};
40
+ use interop::arrow::to_rb::RbArrowSchema;
41
41
  use lazyframe::{RbCollectBatches, RbInProcessQuery, RbLazyFrame, RbOptFlags};
42
42
  use lazygroupby::RbLazyGroupBy;
43
43
  use magnus::{Ruby, function, method, prelude::*};
44
+ use ruby::capsule::RbCapsule;
44
45
  use ruby::exceptions::{RbTypeError, RbValueError};
45
46
  use series::RbSeries;
46
47
  use sql::RbSQLContext;
@@ -1135,6 +1136,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
1135
1136
  class.define_method("to_s", method!(RbSeries::as_str, 0))?;
1136
1137
  class.define_method("len", method!(RbSeries::len, 0))?;
1137
1138
  class.define_method("to_a", method!(RbSeries::to_a, 0))?;
1139
+ class.define_method("arrow_c_stream", method!(RbSeries::__arrow_c_stream__, 0))?;
1138
1140
  class.define_method("median", method!(RbSeries::median, 0))?;
1139
1141
  class.define_method("quantile", method!(RbSeries::quantile, 2))?;
1140
1142
  class.define_method("_clone", method!(RbSeries::clone, 0))?;
@@ -1340,9 +1342,10 @@ fn init(ruby: &Ruby) -> RbResult<()> {
1340
1342
  class.define_method("register", method!(RbSQLContext::register, 2))?;
1341
1343
  class.define_method("unregister", method!(RbSQLContext::unregister, 1))?;
1342
1344
 
1343
- // arrow array stream
1344
- let class = module.define_class("ArrowArrayStream", ruby.class_object())?;
1345
- class.define_method("to_i", method!(RbArrowArrayStream::to_i, 0))?;
1345
+ // capsule
1346
+ let class = module.define_class("Capsule", ruby.class_object())?;
1347
+ class.define_method("to_i", method!(RbCapsule::to_i, 0))?;
1348
+ class.define_method("name", method!(RbCapsule::name, 0))?;
1346
1349
 
1347
1350
  // arrow schema
1348
1351
  let class = module.define_class("ArrowSchema", ruby.class_object())?;
@@ -0,0 +1,27 @@
1
+ use std::any::Any;
2
+ use std::ffi::CString;
3
+
4
+ #[magnus::wrap(class = "Polars::Capsule")]
5
+ pub struct RbCapsule {
6
+ value: Box<dyn Any + Send>,
7
+ name: Option<CString>,
8
+ }
9
+
10
+ impl RbCapsule {
11
+ pub fn new<T: 'static + Send>(value: T, name: Option<CString>) -> Self {
12
+ Self {
13
+ value: Box::new(value),
14
+ name,
15
+ }
16
+ }
17
+
18
+ pub fn to_i(&self) -> usize {
19
+ (&*self.value as *const dyn Any as *const ()) as usize
20
+ }
21
+
22
+ // TODO use &CStr when Magnus supports it
23
+ // https://github.com/matsadler/magnus/pull/182
24
+ pub fn name(&self) -> Option<&str> {
25
+ self.name.as_deref().map(|v| v.to_str().unwrap())
26
+ }
27
+ }
@@ -1,3 +1,4 @@
1
+ pub mod capsule;
1
2
  pub mod exceptions;
2
3
  pub mod gvl;
3
4
  pub mod lazy;
@@ -1,6 +1,7 @@
1
1
  use magnus::{IntoValue, Ruby, Value, value::ReprValue};
2
2
  use polars_core::prelude::*;
3
3
 
4
+ use crate::interop::arrow::to_rb::series_to_stream;
4
5
  use crate::prelude::*;
5
6
  use crate::ruby::utils::TryIntoValue;
6
7
  use crate::{RbPolarsErr, RbResult, RbSeries};
@@ -178,4 +179,8 @@ impl RbSeries {
178
179
 
179
180
  to_a_recursive(ruby, series)
180
181
  }
182
+
183
+ pub fn __arrow_c_stream__(ruby: &Ruby, self_: &Self) -> RbResult<Value> {
184
+ series_to_stream(&self_.series.read(), ruby)
185
+ }
181
186
  }
@@ -771,7 +771,14 @@ module Polars
771
771
  s.get_index_signed(row)
772
772
  end
773
773
 
774
- # no to_arrow
774
+ # Collect the underlying arrow arrays in an Arrow Table.
775
+ #
776
+ # @return [Nanoarrow::Array]
777
+ def to_arrow
778
+ require "nanoarrow"
779
+
780
+ Nanoarrow::Array.new(self)
781
+ end
775
782
 
776
783
  # Convert DataFrame to a hash mapping column name to values.
777
784
  #
@@ -810,6 +817,8 @@ module Polars
810
817
  # df.to_numo.class
811
818
  # # => Numo::RObject
812
819
  def to_numo
820
+ require "numo/narray"
821
+
813
822
  out = _df.to_numo
814
823
  if out.nil?
815
824
  Numo::NArray.vstack(width.times.map { |i| to_series(i).to_numo }).transpose
@@ -242,10 +242,10 @@ module Polars
242
242
  rbexprs = Utils.parse_into_list_of_expressions(*exprs)
243
243
  exprs_wrapped = rbexprs.map { |e| Utils.wrap_expr(e) }
244
244
 
245
- # (Expr): use u32 as that will not cast to float as eagerly
246
- Polars.cum_fold(Polars.lit(0).cast(UInt32), exprs_wrapped) { |a, b| a + b }.alias(
247
- "cum_sum"
248
- )
245
+ Polars.cum_fold(
246
+ Polars.lit(0).cast(Polars.dtype_of(Polars.sum_horizontal(exprs))),
247
+ exprs_wrapped
248
+ ) { |a, b| a + b }.alias("cum_sum")
249
249
  end
250
250
  end
251
251
  end
@@ -17,24 +17,48 @@ module Polars
17
17
 
18
18
  table = scan.table
19
19
  snapshot = scan.snapshot
20
- schema = snapshot ? table.schema_by_id(snapshot[:schema_id]) : table.current_schema
20
+ schema = snapshot ? table.schema_by_id(snapshot.is_a?(Hash) ? snapshot[:schema_id] : snapshot.schema_id) : table.current_schema
21
21
 
22
- if files.empty?
23
- # TODO improve
22
+ if files.empty? && !schema.respond_to?(:arrow_c_schema)
23
+ # TODO remove in 0.27.0
24
24
  schema =
25
25
  schema.fields.to_h do |field|
26
26
  dtype =
27
27
  case field[:type]
28
+ when "unknown"
29
+ Unknown
30
+ when "boolean"
31
+ Boolean
28
32
  when "int"
29
- Polars::Int32
33
+ Int32
30
34
  when "long"
31
- Polars::Int64
35
+ Int64
36
+ when "float"
37
+ Float32
32
38
  when "double"
33
- Polars::Float64
39
+ Float64
40
+ when "decimal"
41
+ Decimal.new(field[:precision], field[:scale])
34
42
  when "string"
35
- Polars::String
43
+ String
44
+ when "uuid"
45
+ Binary
46
+ when "fixed"
47
+ Binary
48
+ when "binary"
49
+ Binary
50
+ when "date"
51
+ Date
52
+ when "time"
53
+ Time # ns instead of us since does not support time unit
36
54
  when "timestamp"
37
- Polars::Datetime
55
+ Datetime.new("us")
56
+ when "timestamp_ns"
57
+ Datetime.new("ns")
58
+ when "timestamptz"
59
+ Datetime.new("us", "+00:00")
60
+ when "timestamptz_ns"
61
+ Datetime.new("ns", "+00:00")
38
62
  else
39
63
  raise Todo
40
64
  end
@@ -44,25 +68,26 @@ module Polars
44
68
 
45
69
  LazyFrame.new(schema: schema)
46
70
  else
47
- sources = files.map { |v| v[:data_file_path] }
71
+ sources = files.map { |v| v.is_a?(Hash) ? v[:data_file_path] : v.file.file_path }
48
72
 
49
73
  column_mapping = [
50
74
  "iceberg-column-mapping",
51
- arrow_schema(schema)
75
+ schema.respond_to?(:arrow_c_schema) ? schema : arrow_schema(schema)
52
76
  ]
53
77
 
54
78
  deletion_files = [
55
79
  "iceberg-position-delete",
56
80
  files.map.with_index
57
- .select { |v, i| v[:deletes].any? }
58
- .to_h { |v, i| [i, v[:deletes].map { |d| d[:file_path] }] }
81
+ .select { |v, i| (v.is_a?(Hash) ? v[:deletes] : v.delete_files).any? }
82
+ .to_h { |v, i| [i, v.is_a?(Hash) ? v[:deletes].map { |d| d[:file_path] } : v.delete_files.map(&:file_path)] }
59
83
  ]
60
84
 
61
85
  scan_options = {
62
- storage_options: @storage_options,
86
+ schema: schema.respond_to?(:arrow_c_schema) ? Schema.new(schema) : nil,
63
87
  cast_options: Polars::ScanCastOptions._default_iceberg,
64
88
  missing_columns: "insert",
65
89
  extra_columns: "ignore",
90
+ storage_options: @storage_options,
66
91
  _column_mapping: column_mapping,
67
92
  _deletion_files: deletion_files
68
93
  }
@@ -73,11 +98,14 @@ module Polars
73
98
 
74
99
  private
75
100
 
101
+ # TODO remove in 0.27.0
76
102
  def arrow_schema(schema)
77
103
  fields =
78
104
  schema.fields.map do |field|
79
105
  type =
80
106
  case field[:type]
107
+ when "unknown"
108
+ "unknown"
81
109
  when "boolean"
82
110
  "boolean"
83
111
  when "int"
@@ -88,11 +116,42 @@ module Polars
88
116
  "float32"
89
117
  when "double"
90
118
  "float64"
119
+ when "decimal"
120
+ "decimal"
121
+ when "string"
122
+ "string"
123
+ when "uuid"
124
+ limit = 16
125
+ "fixed_size_binary"
126
+ when "fixed"
127
+ limit = field[:limit]
128
+ "fixed_size_binary"
129
+ when "binary"
130
+ "large_binary"
131
+ when "date"
132
+ "date32"
133
+ when "time"
134
+ time_unit = "us"
135
+ "time64"
136
+ when "timestamp"
137
+ time_unit = "us"
138
+ "timestamp"
139
+ when "timestamp_ns"
140
+ time_unit = "ns"
141
+ "timestamp"
142
+ when "timestamptz"
143
+ time_unit = "us"
144
+ time_zone = "+00:00"
145
+ "timestamp"
146
+ when "timestamptz_ns"
147
+ time_unit = "ns"
148
+ time_zone = "+00:00"
149
+ "timestamp"
91
150
  else
92
151
  raise Todo
93
152
  end
94
153
 
95
- {
154
+ arrow_field = {
96
155
  name: field[:name],
97
156
  type: type,
98
157
  nullable: !field[:required],
@@ -100,6 +159,14 @@ module Polars
100
159
  "PARQUET:field_id" => field[:id].to_s
101
160
  }
102
161
  }
162
+ arrow_field[:limit] = limit if limit
163
+ if type == "decimal"
164
+ arrow_field[:precision] = field[:precision]
165
+ arrow_field[:scale] = field[:scale]
166
+ end
167
+ arrow_field[:time_unit] = time_unit if time_unit
168
+ arrow_field[:time_zone] = time_zone if time_zone
169
+ arrow_field
103
170
  end
104
171
 
105
172
  {fields: fields}
@@ -25,6 +25,8 @@ module Polars
25
25
  storage_options: nil,
26
26
  delta_table_options: nil
27
27
  )
28
+ require "deltalake-rb"
29
+
28
30
  df =
29
31
  scan_delta(
30
32
  source,
@@ -62,6 +64,8 @@ module Polars
62
64
  delta_table_options: nil,
63
65
  rechunk: nil
64
66
  )
67
+ require "deltalake-rb"
68
+
65
69
  dl_tbl =
66
70
  _get_delta_lake_table(
67
71
  source,
data/lib/polars/schema.rb CHANGED
@@ -71,6 +71,15 @@ module Polars
71
71
  @schema.values
72
72
  end
73
73
 
74
+ # Convert the schema to a nanoarrow schema.
75
+ #
76
+ # @return [Nanoarrow::Schema]
77
+ def to_arrow
78
+ require "nanoarrow"
79
+
80
+ Nanoarrow::Schema.new(self)
81
+ end
82
+
74
83
  # Get the number of schema entries.
75
84
  #
76
85
  # @return [Integer]
data/lib/polars/series.rb CHANGED
@@ -73,6 +73,8 @@ module Polars
73
73
  self._s = Utils.dataframe_to_rbseries(
74
74
  original_name, values, dtype: dtype, strict: strict
75
75
  )
76
+ elsif values.respond_to?(:arrow_c_stream)
77
+ self._s = RbSeries.from_arrow_c_stream(values)
76
78
  else
77
79
  raise TypeError, "Series constructor called with unsupported type; got #{values.class.name}"
78
80
  end
@@ -496,6 +498,11 @@ module Polars
496
498
  end
497
499
  end
498
500
 
501
+ # @private
502
+ def arrow_c_stream
503
+ _s.arrow_c_stream
504
+ end
505
+
499
506
  # Return the Series as a scalar, or return the element at the given index.
500
507
  #
501
508
  # If no index is provided, this is equivalent to `s[0]`, with a check
@@ -3114,6 +3121,8 @@ module Polars
3114
3121
  # # Numo::Int64#shape=[3]
3115
3122
  # # [1, 2, 3]
3116
3123
  def to_numo
3124
+ require "numo/narray"
3125
+
3117
3126
  if dtype.temporal?
3118
3127
  Numo::RObject.cast(to_a)
3119
3128
  else
@@ -3121,6 +3130,15 @@ module Polars
3121
3130
  end
3122
3131
  end
3123
3132
 
3133
+ # Return the underlying Arrow array.
3134
+ #
3135
+ # @return [Nanoarrow::Array]
3136
+ def to_arrow
3137
+ require "nanoarrow"
3138
+
3139
+ Nanoarrow::Array.new(self)
3140
+ end
3141
+
3124
3142
  # Set masked values.
3125
3143
  #
3126
3144
  # @param filter [Series]
@@ -1,4 +1,4 @@
1
1
  module Polars
2
2
  # @private
3
- VERSION = "0.26.0"
3
+ VERSION = "0.26.1"
4
4
  end
data/lib/polars.rb CHANGED
@@ -126,6 +126,10 @@ module Polars
126
126
  # @private
127
127
  N_INFER_DEFAULT = 100
128
128
 
129
+ # @private
130
+ # TODO remove in 0.27.0
131
+ ArrowArrayStream = Capsule
132
+
129
133
  # @private
130
134
  class ArrowArrayStream
131
135
  def arrow_c_stream
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: polars-df
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.26.0
4
+ version: 0.26.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Andrew Kane
@@ -113,6 +113,7 @@ files:
113
113
  - ext/polars/src/on_startup.rs
114
114
  - ext/polars/src/prelude.rs
115
115
  - ext/polars/src/rb_modules.rs
116
+ - ext/polars/src/ruby/capsule.rs
116
117
  - ext/polars/src/ruby/exceptions.rs
117
118
  - ext/polars/src/ruby/gvl.rs
118
119
  - ext/polars/src/ruby/lazy.rs
@@ -262,7 +263,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
262
263
  - !ruby/object:Gem::Version
263
264
  version: '0'
264
265
  requirements: []
265
- rubygems_version: 4.0.13
266
+ rubygems_version: 4.0.14
266
267
  specification_version: 4
267
268
  summary: Blazingly fast DataFrames for Ruby
268
269
  test_files: []