polars-df 0.16.0 → 0.17.0

data/LICENSE.txt

@@ -1,5 +1,5 @@
 Copyright (c) 2020 Ritchie Vink
-Copyright (c) 2022-2024 Andrew Kane
+Copyright (c) 2022-2025 Andrew Kane
 Some portions Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy

data/README.md

@@ -88,7 +88,7 @@ From Avro
 Polars.read_avro("file.avro")
 ```
 
-From Delta Lake (requires [deltalake-rb](https://github.com/ankane/delta-ruby)) [experimental, unreleased]
+From Delta Lake (requires [deltalake-rb](https://github.com/ankane/delta-ruby)) [experimental]
 
 ```ruby
 Polars.read_delta("./table")
@@ -365,7 +365,7 @@ Avro
 df.write_avro("file.avro")
 ```
 
-Delta Lake [experimental, unreleased]
+Delta Lake [experimental]
 
 ```ruby
 df.write_delta("./table")

data/ext/polars/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "polars"
-version = "0.16.0"
+version = "0.17.0"
 license = "MIT"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"
@@ -12,20 +12,21 @@ crate-type = ["cdylib"]
 
 [dependencies]
 ahash = "0.8"
-arrow = { package = "polars-arrow", version = "=0.45.1" }
+arrow = { package = "polars-arrow", version = "=0.46.0" }
 bytes = "1"
 chrono = "0.4"
 either = "1.8"
 magnus = "0.7"
-polars-core = "=0.45.1"
-polars-plan = "=0.45.1"
-polars-parquet = "=0.45.1"
-polars-utils = "=0.45.1"
+polars-core = "=0.46.0"
+polars-plan = "=0.46.0"
+polars-parquet = "=0.46.0"
+polars-utils = "=0.46.0"
+rayon = "1.9"
 regex = "1"
 serde_json = "1"
 
 [dependencies.polars]
-version = "=0.45.1"
+version = "=0.46.0"
 features = [
     "abs",
     "approx_unique",

data/ext/polars/src/conversion/any_value.rs

@@ -33,6 +33,7 @@ pub(crate) fn any_value_into_rb_object(av: AnyValue, ruby: &Ruby) -> Value {
         AnyValue::Int16(v) => ruby.into_value(v),
         AnyValue::Int32(v) => ruby.into_value(v),
         AnyValue::Int64(v) => ruby.into_value(v),
+        AnyValue::Int128(_v) => todo!(),
         AnyValue::Float32(v) => ruby.into_value(v),
         AnyValue::Float64(v) => ruby.into_value(v),
         AnyValue::Null => ruby.qnil().as_value(),

data/ext/polars/src/conversion/mod.rs

@@ -146,6 +146,10 @@ impl IntoValue for Wrap<DataType> {
                 let class = pl.const_get::<_, Value>("Int64").unwrap();
                 class.funcall("new", ()).unwrap()
             }
+            DataType::Int128 => {
+                let class = pl.const_get::<_, Value>("Int128").unwrap();
+                class.funcall("new", ()).unwrap()
+            }
             DataType::UInt8 => {
                 let class = pl.const_get::<_, Value>("UInt8").unwrap();
                 class.funcall("new", ()).unwrap()

data/ext/polars/src/dataframe/io.rs

@@ -296,30 +296,24 @@ impl RbDataFrame {
         Ok(())
     }
 
-    pub fn write_json(&self, rb_f: Value, pretty: bool, row_oriented: bool) -> RbResult<()> {
+    pub fn write_json(&self, rb_f: Value) -> RbResult<()> {
         let file = BufWriter::new(get_file_like(rb_f, true)?);
 
-        let r = match (pretty, row_oriented) {
-            (_, true) => JsonWriter::new(file)
-                .with_json_format(JsonFormat::Json)
-                .finish(&mut self.df.borrow_mut()),
-            (true, _) => serde_json::to_writer_pretty(file, &*self.df.borrow())
-                .map_err(|e| PolarsError::ComputeError(format!("{:?}", e).into())),
-            (false, _) => serde_json::to_writer(file, &*self.df.borrow())
-                .map_err(|e| PolarsError::ComputeError(format!("{:?}", e).into())),
-        };
-        r.map_err(|e| RbPolarsErr::Other(format!("{:?}", e)))?;
+        JsonWriter::new(file)
+            .with_json_format(JsonFormat::Json)
+            .finish(&mut self.df.borrow_mut())
+            .map_err(RbPolarsErr::from)?;
         Ok(())
     }
 
     pub fn write_ndjson(&self, rb_f: Value) -> RbResult<()> {
         let file = BufWriter::new(get_file_like(rb_f, true)?);
 
-        let r = JsonWriter::new(file)
+        JsonWriter::new(file)
             .with_json_format(JsonFormat::JsonLines)
-            .finish(&mut self.df.borrow_mut());
+            .finish(&mut self.df.borrow_mut())
+            .map_err(RbPolarsErr::from)?;
 
-        r.map_err(|e| RbPolarsErr::Other(format!("{:?}", e)))?;
         Ok(())
     }
 

data/ext/polars/src/functions/io.rs

@@ -1,9 +1,9 @@
 use std::io::BufReader;
 
+use arrow::array::Utf8ViewArray;
 use magnus::{RHash, Value};
 use polars::prelude::ArrowSchema;
 use polars_core::datatypes::create_enum_dtype;
-use polars_core::export::arrow::array::Utf8ViewArray;
 
 use crate::conversion::Wrap;
 use crate::file::{get_either_file, EitherRustRubyFile};
@@ -11,7 +11,7 @@ use crate::prelude::ArrowDataType;
 use crate::{RbPolarsErr, RbResult};
 
 pub fn read_ipc_schema(rb_f: Value) -> RbResult<RHash> {
-    use polars_core::export::arrow::io::ipc::read::read_file_metadata;
+    use arrow::io::ipc::read::read_file_metadata;
     let metadata = match get_either_file(rb_f, false)? {
         EitherRustRubyFile::Rust(r) => {
             read_file_metadata(&mut BufReader::new(r)).map_err(RbPolarsErr::from)?

data/ext/polars/src/interop/numo/to_numo_series.rs

@@ -23,7 +23,7 @@ impl RbSeries {
                     .const_get::<_, RClass>("RObject")?
                     .funcall("cast", (np_arr,))
             }
-            dt if dt.is_numeric() => {
+            dt if dt.is_primitive_numeric() => {
                 if let Some(BitRepr::Large(_)) = s.bit_repr() {
                     let s = s.cast(&DataType::Float64).unwrap();
                     let ca = s.f64().unwrap();

data/ext/polars/src/lazyframe/general.rs

@@ -633,6 +633,8 @@ impl RbLazyFrame {
         tolerance: Option<Wrap<AnyValue<'_>>>,
         tolerance_str: Option<String>,
         coalesce: bool,
+        allow_eq: bool,
+        check_sortedness: bool,
     ) -> RbResult<Self> {
         let coalesce = if coalesce {
             JoinCoalesce::CoalesceColumns
@@ -657,6 +659,8 @@ impl RbLazyFrame {
                 right_by: right_by.map(strings_to_pl_smallstr),
                 tolerance: tolerance.map(|t| t.0.into_static()),
                 tolerance_str: tolerance_str.map(|s| s.into()),
+                allow_eq,
+                check_sortedness,
             }))
             .suffix(suffix)
             .finish()

data/ext/polars/src/lib.rs

@@ -69,7 +69,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("estimated_size", method!(RbDataFrame::estimated_size, 0))?;
     class.define_method("dtype_strings", method!(RbDataFrame::dtype_strings, 0))?;
     class.define_method("write_avro", method!(RbDataFrame::write_avro, 3))?;
-    class.define_method("write_json", method!(RbDataFrame::write_json, 3))?;
+    class.define_method("write_json", method!(RbDataFrame::write_json, 1))?;
     class.define_method("write_ndjson", method!(RbDataFrame::write_ndjson, 1))?;
     class.define_method("write_csv", method!(RbDataFrame::write_csv, 10))?;
     class.define_method("write_ipc", method!(RbDataFrame::write_ipc, 5))?;
@@ -758,7 +758,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
         method!(RbLazyFrame::group_by_dynamic, 9),
     )?;
     class.define_method("with_context", method!(RbLazyFrame::with_context, 1))?;
-    class.define_method("join_asof", method!(RbLazyFrame::join_asof, 12))?;
+    class.define_method("join_asof", method!(RbLazyFrame::join_asof, 14))?;
     class.define_method("join", method!(RbLazyFrame::join, 10))?;
     class.define_method("with_column", method!(RbLazyFrame::with_column, 1))?;
     class.define_method("with_columns", method!(RbLazyFrame::with_columns, 1))?;

data/ext/polars/src/map/mod.rs

@@ -5,9 +5,9 @@ pub mod series;
 use magnus::{prelude::*, RHash, Value};
 use polars::chunked_array::builder::get_list_builder;
 use polars::prelude::*;
-use polars_core::export::rayon::prelude::*;
 use polars_core::utils::CustomIterTools;
 use polars_core::POOL;
+use rayon::prelude::*;
 
 use crate::{ObjectValue, RbPolarsErr, RbResult, RbSeries, Wrap};
 

data/ext/polars/src/series/export.rs

@@ -21,6 +21,7 @@ impl RbSeries {
                 DataType::Int16 => RArray::from_iter(series.i16().unwrap()).into_value(),
                 DataType::Int32 => RArray::from_iter(series.i32().unwrap()).into_value(),
                 DataType::Int64 => RArray::from_iter(series.i64().unwrap()).into_value(),
+                DataType::Int128 => todo!(),
                 DataType::Float32 => RArray::from_iter(series.f32().unwrap()).into_value(),
                 DataType::Float64 => RArray::from_iter(series.f64().unwrap()).into_value(),
                 DataType::Categorical(_, _) | DataType::Enum(_, _) => {

data/ext/polars/src/series/import.rs

@@ -1,7 +1,7 @@
+use arrow::array::Array;
+use arrow::ffi::{ArrowArrayStream, ArrowArrayStreamReader};
 use magnus::prelude::*;
 use magnus::Value;
-use polars::export::arrow::array::Array;
-use polars::export::arrow::ffi::{ArrowArrayStream, ArrowArrayStreamReader};
 use polars::prelude::*;
 
 use super::RbSeries;

data/ext/polars/src/series/scatter.rs

@@ -1,4 +1,4 @@
-use polars::export::arrow::array::Array;
+use arrow::array::Array;
 use polars::prelude::*;
 
 use crate::error::RbPolarsErr;

data/lib/polars/data_frame.rb

@@ -604,10 +604,6 @@ module Polars
     #
     # @param file [String]
     #   File path to which the result should be written.
-    # @param pretty [Boolean]
-    #   Pretty serialize json.
-    # @param row_oriented [Boolean]
-    #   Write to row oriented json. This is slower, but more common.
     #
     # @return [nil]
     #
@@ -619,16 +615,8 @@ module Polars
     #     }
     #   )
     #   df.write_json
-    #   # => "{\"columns\":[{\"name\":\"foo\",\"datatype\":\"Int64\",\"bit_settings\":\"\",\"values\":[1,2,3]},{\"name\":\"bar\",\"datatype\":\"Int64\",\"bit_settings\":\"\",\"values\":[6,7,8]}]}"
-    #
-    # @example
-    #   df.write_json(row_oriented: true)
     #   # => "[{\"foo\":1,\"bar\":6},{\"foo\":2,\"bar\":7},{\"foo\":3,\"bar\":8}]"
-    def write_json(
-      file = nil,
-      pretty: false,
-      row_oriented: false
-    )
+    def write_json(file = nil)
       if Utils.pathlike?(file)
         file = Utils.normalize_filepath(file)
       end
@@ -636,7 +624,7 @@ module Polars
       if file.nil? || to_string_io
         buf = StringIO.new
         buf.set_encoding(Encoding::BINARY)
-        _df.write_json(buf, pretty, row_oriented)
+        _df.write_json(buf)
         json_bytes = buf.string
 
         json_str = json_bytes.force_encoding(Encoding::UTF_8)
@@ -646,7 +634,7 @@ module Polars
           return json_str
         end
       else
-        _df.write_json(file, pretty, row_oriented)
+        _df.write_json(file)
       end
       nil
     end
@@ -2294,6 +2282,14 @@ module Polars
     #   keys are within this distance. If an asof join is done on columns of dtype
     #   "Date", "Datetime", "Duration" or "Time" you use the following string
     #   language:
+    # @param allow_exact_matches [Boolean]
+    #   Whether exact matches are valid join predicates.
+    #     - If true, allow matching with the same `on` value (i.e. less-than-or-equal-to / greater-than-or-equal-to).
+    #     - If false, don't match the same `on` value (i.e., strictly less-than / strictly greater-than).
+    # @param check_sortedness [Boolean]
+    #   Check the sortedness of the asof keys. If the keys are not sorted Polars
+    #   will error, or in case of 'by' argument raise a warning. This might become
+    #   a hard error in the future.
     #
     #    - 1ns   (1 nanosecond)
     #    - 1us   (1 microsecond)
@@ -2375,7 +2371,9 @@ module Polars
       tolerance: nil,
       allow_parallel: true,
       force_parallel: false,
-      coalesce: true
+      coalesce: true,
+      allow_exact_matches: true,
+      check_sortedness: true
     )
       lazy
         .join_asof(
@@ -2391,7 +2389,9 @@ module Polars
           tolerance: tolerance,
           allow_parallel: allow_parallel,
           force_parallel: force_parallel,
-          coalesce: coalesce
+          coalesce: coalesce,
+          allow_exact_matches: allow_exact_matches,
+          check_sortedness: check_sortedness
         )
         .collect(no_optimization: true)
     end

data/lib/polars/data_types.rb

@@ -167,6 +167,10 @@ module Polars
   class Int64 < SignedIntegerType
   end
 
+  # 128-bit signed integer type.
+  class Int128 < SignedIntegerType
+  end
+
   # 8-bit unsigned integer type.
   class UInt8 < UnsignedIntegerType
   end

data/lib/polars/lazy_frame.rb

@@ -1616,6 +1616,14 @@ module Polars
     #     - true: -> Always coalesce join columns.
     #     - false: -> Never coalesce join columns.
     #   Note that joining on any other expressions than `col` will turn off coalescing.
+    # @param allow_exact_matches [Boolean]
+    #   Whether exact matches are valid join predicates.
+    #     - If true, allow matching with the same `on` value (i.e. less-than-or-equal-to / greater-than-or-equal-to).
+    #     - If false, don't match the same `on` value (i.e., strictly less-than / strictly greater-than).
+    # @param check_sortedness [Boolean]
+    #   Check the sortedness of the asof keys. If the keys are not sorted Polars
+    #   will error, or in case of 'by' argument raise a warning. This might become
+    #   a hard error in the future.
     #
     # @return [LazyFrame]
     #
@@ -1815,7 +1823,9 @@ module Polars
       tolerance: nil,
       allow_parallel: true,
       force_parallel: false,
-      coalesce: true
+      coalesce: true,
+      allow_exact_matches: true,
+      check_sortedness: true
     )
       if !other.is_a?(LazyFrame)
         raise ArgumentError, "Expected a `LazyFrame` as join table, got #{other.class.name}"
@@ -1871,7 +1881,9 @@ module Polars
           strategy,
           tolerance_num,
           tolerance_str,
-          coalesce
+          coalesce,
+          allow_exact_matches,
+          check_sortedness
         )
       )
     end

data/lib/polars/series.rb

@@ -4696,7 +4696,12 @@ module Polars
         end
 
         constructor = polars_type_to_constructor(dtype)
-        rbseries = constructor.call(name, values, strict)
+        rbseries =
+          if dtype == Array
+            constructor.call(name, values, strict)
+          else
+            construct_series_with_fallbacks(constructor, name, values, dtype, strict: strict)
+          end
 
         base_type = dtype.is_a?(DataType) ? dtype.class : dtype
         if [Date, Datetime, Duration, Time, Categorical, Boolean, Enum, Decimal].include?(base_type)

data/lib/polars/version.rb

@@ -1,4 +1,4 @@
 module Polars
   # @private
-  VERSION = "0.16.0"
+  VERSION = "0.17.0"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: polars-df
 version: !ruby/object:Gem::Version
-  version: 0.16.0
+  version: 0.17.0
 platform: ruby
 authors:
 - Andrew Kane
 bindir: bin
 cert_chain: []
-date: 2024-12-29 00:00:00.000000000 Z
+date: 2025-01-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bigdecimal