polars-df 0.3.1-x86_64-linux → 0.4.0-x86_64-linux

data/README.md

@@ -50,6 +50,9 @@ From Parquet
 
 ```ruby
 Polars.read_parquet("file.parquet")
+
+# or lazily with
+Polars.scan_parquet("file.parquet")
 ```
 
 From Active Record
@@ -60,6 +63,32 @@ Polars.read_sql(User.all)
 Polars.read_sql("SELECT * FROM users")
 ```
 
+From JSON
+
+```ruby
+Polars.read_json("file.json")
+# or
+Polars.read_ndjson("file.ndjson")
+
+# or lazily with
+Polars.scan_ndjson("file.ndjson")
+```
+
+From Feather / Arrow IPC
+
+```ruby
+Polars.read_ipc("file.arrow")
+
+# or lazily with
+Polars.scan_ipc("file.arrow")
+```
+
+From Avro
+
+```ruby
+Polars.read_avro("file.avro")
+```
+
 From a hash
 
 ```ruby

data/lib/polars/batched_csv_reader.rb

@@ -30,7 +30,7 @@ module Polars
       new_columns: nil
     )
       if Utils.pathlike?(file)
-        path = Utils.format_path(file)
+        path = Utils.normalise_filepath(file)
       end
 
       dtype_list = nil

data/lib/polars/binary_expr.rb

@@ -0,0 +1,77 @@
+module Polars
+  # Namespace for binary related expressions.
+  class BinaryExpr
+    # @private
+    attr_accessor :_rbexpr
+
+    # @private
+    def initialize(expr)
+      self._rbexpr = expr._rbexpr
+    end
+
+    # Check if binaries in Series contain a binary substring.
+    #
+    # @param lit [String]
+    #   The binary substring to look for
+    #
+    # @return [Expr]
+    def contains(lit)
+      Utils.wrap_expr(_rbexpr.binary_contains(lit))
+    end
+
+    # Check if string values end with a binary substring.
+    #
+    # @param sub [String]
+    #   Suffix substring.
+    #
+    # @return [Expr]
+    def ends_with(sub)
+      Utils.wrap_expr(_rbexpr.binary_ends_with(sub))
+    end
+
+    # Check if values start with a binary substring.
+    #
+    # @param sub [String]
+    #   Prefix substring.
+    #
+    # @return [Expr]
+    def starts_with(sub)
+      Utils.wrap_expr(_rbexpr.binary_starts_with(sub))
+    end
+
+    # Decode a value using the provided encoding.
+    #
+    # @param encoding ["hex", "base64"]
+    #   The encoding to use.
+    # @param strict [Boolean]
+    #   Raise an error if the underlying value cannot be decoded,
+    #   otherwise mask out with a null value.
+    #
+    # @return [Expr]
+    def decode(encoding, strict: true)
+      if encoding == "hex"
+        Utils.wrap_expr(_rbexpr.binary_hex_decode(strict))
+      elsif encoding == "base64"
+        Utils.wrap_expr(_rbexpr.binary_base64_decode(strict))
+      else
+        raise ArgumentError, "encoding must be one of {{'hex', 'base64'}}, got #{encoding}"
+      end
+    end
+
+    # Encode a value using the provided encoding.
+    #
+    # @param encoding ["hex", "base64"]
+    #   The encoding to use.
+    #
+    # @return [Expr]
+    def encode(encoding)
+      if encoding == "hex"
+        Utils.wrap_expr(_rbexpr.binary_hex_encode)
+      elsif encoding == "base64"
+        Utils.wrap_expr(_rbexpr.binary_base64_encode)
+      else
+        raise ArgumentError, "encoding must be one of {{'hex', 'base64'}}, got #{encoding}"
+      end
+    end
+  end
+end

data/lib/polars/binary_name_space.rb

@@ -0,0 +1,66 @@
+module Polars
+  # Series.bin namespace.
+  class BinaryNameSpace
+    include ExprDispatch
+
+    self._accessor = "bin"
+
+    # @private
+    def initialize(series)
+      self._s = series._s
+    end
+
+    # Check if binaries in Series contain a binary substring.
+    #
+    # @param lit [String]
+    #   The binary substring to look for
+    #
+    # @return [Series]
+    def contains(lit)
+      super
+    end
+
+    # Check if string values end with a binary substring.
+    #
+    # @param sub [String]
+    #   Suffix substring.
+    #
+    # @return [Series]
+    def ends_with(sub)
+      super
+    end
+
+    # Check if values start with a binary substring.
+    #
+    # @param sub [String]
+    #   Prefix substring.
+    #
+    # @return [Series]
+    def starts_with(sub)
+      super
+    end
+
+    # Decode a value using the provided encoding.
+    #
+    # @param encoding ["hex", "base64"]
+    #   The encoding to use.
+    # @param strict [Boolean]
+    #   Raise an error if the underlying value cannot be decoded,
+    #   otherwise mask out with a null value.
+    #
+    # @return [Series]
+    def decode(encoding, strict: true)
+      super
+    end
+
+    # Encode a value using the provided encoding.
+    #
+    # @param encoding ["hex", "base64"]
+    #   The encoding to use.
+    #
+    # @return [Series]
+    def encode(encoding)
+      super
+    end
+  end
+end

data/lib/polars/data_frame.rb

@@ -97,7 +97,7 @@ module Polars
       eol_char: "\n"
     )
       if Utils.pathlike?(file)
-        path = Utils.format_path(file)
+        path = Utils.normalise_filepath(file)
       else
         path = nil
         # if defined?(StringIO) && file.is_a?(StringIO)
@@ -196,32 +196,56 @@ module Polars
 
     # @private
     def self._read_parquet(
-      file,
+      source,
       columns: nil,
       n_rows: nil,
       parallel: "auto",
       row_count_name: nil,
       row_count_offset: 0,
-      low_memory: false
+      low_memory: false,
+      use_statistics: true,
+      rechunk: true
     )
-      if Utils.pathlike?(file)
-        file = Utils.format_path(file)
+      if Utils.pathlike?(source)
+        source = Utils.normalise_filepath(source)
+      end
+      if columns.is_a?(String)
+        columns = [columns]
       end
 
-      if file.is_a?(String) && file.include?("*")
-        raise Todo
+      if source.is_a?(String) && source.include?("*") && Utils.local_file?(source)
+        scan =
+          Polars.scan_parquet(
+            source,
+            n_rows: n_rows,
+            rechunk: true,
+            parallel: parallel,
+            row_count_name: row_count_name,
+            row_count_offset: row_count_offset,
+            low_memory: low_memory
+          )
+
+        if columns.nil?
+          return self._from_rbdf(scan.collect._df)
+        elsif Utils.is_str_sequence(columns, allow_str: false)
+          return self._from_rbdf(scan.select(columns).collect._df)
+        else
+          raise ArgumentError, "cannot use glob patterns and integer based projection as `columns` argument; Use columns: Array[String]"
+        end
       end
 
       projection, columns = Utils.handle_projection_columns(columns)
       _from_rbdf(
         RbDataFrame.read_parquet(
-          file,
+          source,
           columns,
           projection,
           n_rows,
           parallel,
           Utils._prepare_row_count_args(row_count_name, row_count_offset),
-          low_memory
+          low_memory,
+          use_statistics,
+          rechunk
         )
       )
     end
@@ -229,7 +253,7 @@ module Polars
     # @private
     def self._read_avro(file, columns: nil, n_rows: nil)
       if Utils.pathlike?(file)
-        file = Utils.format_path(file)
+        file = Utils.normalise_filepath(file)
       end
       projection, columns = Utils.handle_projection_columns(columns)
       _from_rbdf(RbDataFrame.read_avro(file, columns, projection, n_rows))
@@ -246,7 +270,7 @@ module Polars
       memory_map: true
     )
       if Utils.pathlike?(file)
-        file = Utils.format_path(file)
+        file = Utils.normalise_filepath(file)
       end
       if columns.is_a?(String)
         columns = [columns]
@@ -272,7 +296,7 @@ module Polars
     # @private
     def self._read_json(file)
       if Utils.pathlike?(file)
-        file = Utils.format_path(file)
+        file = Utils.normalise_filepath(file)
       end
 
       _from_rbdf(RbDataFrame.read_json(file))
@@ -281,7 +305,7 @@ module Polars
     # @private
     def self._read_ndjson(file)
       if Utils.pathlike?(file)
-        file = Utils.format_path(file)
+        file = Utils.normalise_filepath(file)
       end
 
       _from_rbdf(RbDataFrame.read_ndjson(file))
@@ -774,7 +798,7 @@ module Polars
       row_oriented: false
     )
       if Utils.pathlike?(file)
-        file = Utils.format_path(file)
+        file = Utils.normalise_filepath(file)
       end
 
       _df.write_json(file, pretty, row_oriented)
@@ -789,7 +813,7 @@ module Polars
     # @return [nil]
     def write_ndjson(file)
       if Utils.pathlike?(file)
-        file = Utils.format_path(file)
+        file = Utils.normalise_filepath(file)
       end
 
       _df.write_ndjson(file)
@@ -879,7 +903,7 @@ module Polars
       end
 
       if Utils.pathlike?(file)
-        file = Utils.format_path(file)
+        file = Utils.normalise_filepath(file)
       end
 
       _df.write_csv(
@@ -917,7 +941,7 @@ module Polars
         compression = "uncompressed"
       end
       if Utils.pathlike?(file)
-        file = Utils.format_path(file)
+        file = Utils.normalise_filepath(file)
       end
 
       _df.write_avro(file, compression)
@@ -936,7 +960,7 @@ module Polars
         compression = "uncompressed"
       end
       if Utils.pathlike?(file)
-        file = Utils.format_path(file)
+        file = Utils.normalise_filepath(file)
       end
 
       _df.write_ipc(file, compression)
@@ -978,7 +1002,7 @@ module Polars
         compression = "uncompressed"
       end
       if Utils.pathlike?(file)
-        file = Utils.format_path(file)
+        file = Utils.normalise_filepath(file)
       end
 
       _df.write_parquet(
@@ -3042,24 +3066,28 @@ module Polars
       if aggregate_fn.is_a?(String)
         case aggregate_fn
         when "first"
-          aggregate_fn = Polars.element.first
+          aggregate_expr = Polars.element.first._rbexpr
         when "sum"
-          aggregate_fn = Polars.element.sum
+          aggregate_expr = Polars.element.sum._rbexpr
         when "max"
-          aggregate_fn = Polars.element.max
+          aggregate_expr = Polars.element.max._rbexpr
         when "min"
-          aggregate_fn = Polars.element.min
+          aggregate_expr = Polars.element.min._rbexpr
         when "mean"
-          aggregate_fn = Polars.element.mean
+          aggregate_expr = Polars.element.mean._rbexpr
         when "median"
-          aggregate_fn = Polars.element.median
+          aggregate_expr = Polars.element.median._rbexpr
         when "last"
-          aggregate_fn = Polars.element.last
+          aggregate_expr = Polars.element.last._rbexpr
         when "count"
-          aggregate_fn = Polars.count
+          aggregate_expr = Polars.count._rbexpr
         else
           raise ArgumentError, "Argument aggregate fn: '#{aggregate_fn}' was not expected."
         end
+      elsif aggregate_fn.nil?
+        aggregate_expr = nil
+      else
+        aggregate_expr = aggregate_function._rbexpr
       end
 
       _from_rbdf(
@@ -3067,9 +3095,9 @@ module Polars
           values,
           index,
           columns,
-          aggregate_fn._rbexpr,
           maintain_order,
           sort_columns,
+          aggregate_expr,
           separator
         )
       )
@@ -3174,7 +3202,7 @@ module Polars
     #   # │ B    ┆ 1    │
     #   # │ C    ┆ 2    │
     #   # │ D    ┆ 3    │
-    #   # │ ...  ┆ ...  │
+    #   # │ …    ┆ …    │
     #   # │ F    ┆ 5    │
     #   # │ G    ┆ 6    │
     #   # │ H    ┆ 7    │
@@ -4053,15 +4081,12 @@ module Polars
     #   # │ 5   ┆ 3.0 ┆ true  │
     #   # └─────┴─────┴───────┘
     def unique(maintain_order: true, subset: nil, keep: "first")
-      if !subset.nil?
-        if subset.is_a?(String)
-          subset = [subset]
-        elsif !subset.is_a?(Array)
-          subset = subset.to_a
-        end
-      end
-
-      _from_rbdf(_df.unique(maintain_order, subset, keep))
+      self._from_rbdf(
+        lazy
+          .unique(maintain_order: maintain_order, subset: subset, keep: keep)
+          .collect(no_optimization: true)
+          ._df
+      )
     end
 
     # Return the number of unique rows, or the number of unique row-subsets.

data/lib/polars/date_time_expr.rb

@@ -1130,7 +1130,7 @@ module Polars
     #     ]
     #   )
     #   # =>
-    #   # shape: (1001, 2)
+    #   # shape: (1_001, 2)
     #   # ┌─────────────────────────┬───────────────────┐
     #   # │ date                    ┆ milliseconds_diff │
     #   # │ ---                     ┆ ---               │
@@ -1140,7 +1140,7 @@ module Polars
     #   # │ 2020-01-01 00:00:00.001 ┆ 1                 │
     #   # │ 2020-01-01 00:00:00.002 ┆ 1                 │
     #   # │ 2020-01-01 00:00:00.003 ┆ 1                 │
-    #   # │ ...                     ┆ ...               │
+    #   # │ …                       ┆ …                 │
     #   # │ 2020-01-01 00:00:00.997 ┆ 1                 │
     #   # │ 2020-01-01 00:00:00.998 ┆ 1                 │
     #   # │ 2020-01-01 00:00:00.999 ┆ 1                 │
@@ -1169,7 +1169,7 @@ module Polars
     #     ]
     #   )
     #   # =>
-    #   # shape: (1001, 2)
+    #   # shape: (1_001, 2)
     #   # ┌─────────────────────────┬───────────────────┐
     #   # │ date                    ┆ microseconds_diff │
     #   # │ ---                     ┆ ---               │
@@ -1179,7 +1179,7 @@ module Polars
     #   # │ 2020-01-01 00:00:00.001 ┆ 1000              │
     #   # │ 2020-01-01 00:00:00.002 ┆ 1000              │
     #   # │ 2020-01-01 00:00:00.003 ┆ 1000              │
-    #   # │ ...                     ┆ ...               │
+    #   # │ …                       ┆ …                 │
     #   # │ 2020-01-01 00:00:00.997 ┆ 1000              │
     #   # │ 2020-01-01 00:00:00.998 ┆ 1000              │
     #   # │ 2020-01-01 00:00:00.999 ┆ 1000              │
@@ -1208,7 +1208,7 @@ module Polars
     #     ]
     #   )
     #   # =>
-    #   # shape: (1001, 2)
+    #   # shape: (1_001, 2)
     #   # ┌─────────────────────────┬──────────────────┐
     #   # │ date                    ┆ nanoseconds_diff │
     #   # │ ---                     ┆ ---              │
@@ -1218,7 +1218,7 @@ module Polars
     #   # │ 2020-01-01 00:00:00.001 ┆ 1000000          │
     #   # │ 2020-01-01 00:00:00.002 ┆ 1000000          │
     #   # │ 2020-01-01 00:00:00.003 ┆ 1000000          │
-    #   # │ ...                     ┆ ...              │
+    #   # │ …                       ┆ …                │
     #   # │ 2020-01-01 00:00:00.997 ┆ 1000000          │
     #   # │ 2020-01-01 00:00:00.998 ┆ 1000000          │
     #   # │ 2020-01-01 00:00:00.999 ┆ 1000000          │

data/lib/polars/expr.rb

@@ -2194,7 +2194,7 @@ module Polars
     #   # │ 4      │
     #   # │ 6      │
     #   # │ 6      │
-    #   # │ ...    │
+    #   # │ …      │
     #   # │ 6      │
     #   # │ 6      │
     #   # │ 6      │
@@ -2571,7 +2571,7 @@ module Polars
     #   # │ e   │
     #   # │ l   │
     #   # │ l   │
-    #   # │ ... │
+    #   # │ …   │
     #   # │ o   │
     #   # │ r   │
     #   # │ l   │
@@ -4962,6 +4962,13 @@ module Polars
       ListExpr.new(self)
     end
 
+    # Create an object namespace of all binary related methods.
+    #
+    # @return [BinaryExpr]
+    def bin
+      BinaryExpr.new(self)
+    end
+
     # Create an object namespace of all categorical related methods.
     #
     # @return [CatExpr]