polars-df 0.3.0-x86_64-linux → 0.4.0-x86_64-linux

data/README.md

@@ -50,6 +50,9 @@ From Parquet
 
 ```ruby
 Polars.read_parquet("file.parquet")
+
+# or lazily with
+Polars.scan_parquet("file.parquet")
 ```
 
 From Active Record
@@ -60,6 +63,32 @@ Polars.read_sql(User.all)
 Polars.read_sql("SELECT * FROM users")
 ```
 
+From JSON
+
+```ruby
+Polars.read_json("file.json")
+# or
+Polars.read_ndjson("file.ndjson")
+
+# or lazily with
+Polars.scan_ndjson("file.ndjson")
+```
+
+From Feather / Arrow IPC
+
+```ruby
+Polars.read_ipc("file.arrow")
+
+# or lazily with
+Polars.scan_ipc("file.arrow")
+```
+
+From Avro
+
+```ruby
+Polars.read_avro("file.avro")
+```
+
 From a hash
 
 ```ruby
@@ -282,10 +311,10 @@ df.to_dummies
 
 ## Conversion
 
-Array of rows
+Array of hashes
 
 ```ruby
-df.rows
+df.rows(named: true)
 ```
 
 Hash of series
@@ -308,6 +337,12 @@ Parquet
 df.write_parquet("file.parquet")
 ```
 
+Numo array
+
+```ruby
+df.to_numo
+```
+
 ## Types
 
 You can specify column types when creating a data frame
@@ -343,6 +378,38 @@ Cast a column
 df["a"].cast(Polars::Int32)
 ```
 
+## Visualization
+
+Add [Vega](https://github.com/ankane/vega-ruby) to your application’s Gemfile:
+
+```ruby
+gem "vega"
+```
+
+And use:
+
+```ruby
+df.plot("a", "b")
+```
+
+Specify the chart type (`line`, `pie`, `column`, `bar`, `area`, or `scatter`)
+
+```ruby
+df.plot("a", "b", type: "pie")
+```
+
+Group data
+
+```ruby
+df.groupby("c").plot("a", "b")
+```
+
+Stacked columns or bars
+
+```ruby
+df.groupby("c").plot("a", "b", stacked: true)
+```
+
 ## History
 
 View the [changelog](CHANGELOG.md)

data/lib/polars/batched_csv_reader.rb

@@ -30,7 +30,7 @@ module Polars
       new_columns: nil
     )
       if Utils.pathlike?(file)
-        path = Utils.format_path(file)
+        path = Utils.normalise_filepath(file)
       end
 
       dtype_list = nil

data/lib/polars/binary_expr.rb

@@ -0,0 +1,77 @@
+module Polars
+  # Namespace for binary related expressions.
+  class BinaryExpr
+    # @private
+    attr_accessor :_rbexpr
+
+    # @private
+    def initialize(expr)
+      self._rbexpr = expr._rbexpr
+    end
+
+    # Check if binaries in Series contain a binary substring.
+    #
+    # @param lit [String]
+    #   The binary substring to look for
+    #
+    # @return [Expr]
+    def contains(lit)
+      Utils.wrap_expr(_rbexpr.binary_contains(lit))
+    end
+
+    # Check if string values end with a binary substring.
+    #
+    # @param sub [String]
+    #   Suffix substring.
+    #
+    # @return [Expr]
+    def ends_with(sub)
+      Utils.wrap_expr(_rbexpr.binary_ends_with(sub))
+    end
+
+    # Check if values start with a binary substring.
+    #
+    # @param sub [String]
+    #   Prefix substring.
+    #
+    # @return [Expr]
+    def starts_with(sub)
+      Utils.wrap_expr(_rbexpr.binary_starts_with(sub))
+    end
+
+    # Decode a value using the provided encoding.
+    #
+    # @param encoding ["hex", "base64"]
+    #   The encoding to use.
+    # @param strict [Boolean]
+    #   Raise an error if the underlying value cannot be decoded,
+    #   otherwise mask out with a null value.
+    #
+    # @return [Expr]
+    def decode(encoding, strict: true)
+      if encoding == "hex"
+        Utils.wrap_expr(_rbexpr.binary_hex_decode(strict))
+      elsif encoding == "base64"
+        Utils.wrap_expr(_rbexpr.binary_base64_decode(strict))
+      else
+        raise ArgumentError, "encoding must be one of {{'hex', 'base64'}}, got #{encoding}"
+      end
+    end
+
+    # Encode a value using the provided encoding.
+    #
+    # @param encoding ["hex", "base64"]
+    #   The encoding to use.
+    #
+    # @return [Expr]
+    def encode(encoding)
+      if encoding == "hex"
+        Utils.wrap_expr(_rbexpr.binary_hex_encode)
+      elsif encoding == "base64"
+        Utils.wrap_expr(_rbexpr.binary_base64_encode)
+      else
+        raise ArgumentError, "encoding must be one of {{'hex', 'base64'}}, got #{encoding}"
+      end
+    end
+  end
+end

data/lib/polars/binary_name_space.rb

@@ -0,0 +1,66 @@
+module Polars
+  # Series.bin namespace.
+  class BinaryNameSpace
+    include ExprDispatch
+
+    self._accessor = "bin"
+
+    # @private
+    def initialize(series)
+      self._s = series._s
+    end
+
+    # Check if binaries in Series contain a binary substring.
+    #
+    # @param lit [String]
+    #   The binary substring to look for
+    #
+    # @return [Series]
+    def contains(lit)
+      super
+    end
+
+    # Check if string values end with a binary substring.
+    #
+    # @param sub [String]
+    #   Suffix substring.
+    #
+    # @return [Series]
+    def ends_with(sub)
+      super
+    end
+
+    # Check if values start with a binary substring.
+    #
+    # @param sub [String]
+    #   Prefix substring.
+    #
+    # @return [Series]
+    def starts_with(sub)
+      super
+    end
+
+    # Decode a value using the provided encoding.
+    #
+    # @param encoding ["hex", "base64"]
+    #   The encoding to use.
+    # @param strict [Boolean]
+    #   Raise an error if the underlying value cannot be decoded,
+    #   otherwise mask out with a null value.
+    #
+    # @return [Series]
+    def decode(encoding, strict: true)
+      super
+    end
+
+    # Encode a value using the provided encoding.
+    #
+    # @param encoding ["hex", "base64"]
+    #   The encoding to use.
+    #
+    # @return [Series]
+    def encode(encoding)
+      super
+    end
+  end
+end

data/lib/polars/data_frame.rb

@@ -1,6 +1,8 @@
 module Polars
   # Two-dimensional data structure representing data as a table with rows and columns.
   class DataFrame
+    include Plot
+
     # @private
     attr_accessor :_df
 
@@ -95,7 +97,7 @@ module Polars
       eol_char: "\n"
     )
       if Utils.pathlike?(file)
-        path = Utils.format_path(file)
+        path = Utils.normalise_filepath(file)
       else
         path = nil
         # if defined?(StringIO) && file.is_a?(StringIO)
@@ -194,32 +196,56 @@ module Polars
 
     # @private
     def self._read_parquet(
-      file,
+      source,
       columns: nil,
       n_rows: nil,
       parallel: "auto",
       row_count_name: nil,
       row_count_offset: 0,
-      low_memory: false
+      low_memory: false,
+      use_statistics: true,
+      rechunk: true
     )
-      if Utils.pathlike?(file)
-        file = Utils.format_path(file)
+      if Utils.pathlike?(source)
+        source = Utils.normalise_filepath(source)
+      end
+      if columns.is_a?(String)
+        columns = [columns]
       end
 
-      if file.is_a?(String) && file.include?("*")
-        raise Todo
+      if source.is_a?(String) && source.include?("*") && Utils.local_file?(source)
+        scan =
+          Polars.scan_parquet(
+            source,
+            n_rows: n_rows,
+            rechunk: true,
+            parallel: parallel,
+            row_count_name: row_count_name,
+            row_count_offset: row_count_offset,
+            low_memory: low_memory
+          )
+
+        if columns.nil?
+          return self._from_rbdf(scan.collect._df)
+        elsif Utils.is_str_sequence(columns, allow_str: false)
+          return self._from_rbdf(scan.select(columns).collect._df)
+        else
+          raise ArgumentError, "cannot use glob patterns and integer based projection as `columns` argument; Use columns: Array[String]"
+        end
       end
 
       projection, columns = Utils.handle_projection_columns(columns)
       _from_rbdf(
         RbDataFrame.read_parquet(
-          file,
+          source,
           columns,
           projection,
           n_rows,
           parallel,
           Utils._prepare_row_count_args(row_count_name, row_count_offset),
-          low_memory
+          low_memory,
+          use_statistics,
+          rechunk
         )
       )
     end
@@ -227,7 +253,7 @@ module Polars
     # @private
     def self._read_avro(file, columns: nil, n_rows: nil)
       if Utils.pathlike?(file)
-        file = Utils.format_path(file)
+        file = Utils.normalise_filepath(file)
       end
       projection, columns = Utils.handle_projection_columns(columns)
       _from_rbdf(RbDataFrame.read_avro(file, columns, projection, n_rows))
@@ -244,7 +270,7 @@ module Polars
       memory_map: true
     )
       if Utils.pathlike?(file)
-        file = Utils.format_path(file)
+        file = Utils.normalise_filepath(file)
       end
       if columns.is_a?(String)
         columns = [columns]
@@ -270,7 +296,7 @@ module Polars
     # @private
     def self._read_json(file)
       if Utils.pathlike?(file)
-        file = Utils.format_path(file)
+        file = Utils.normalise_filepath(file)
       end
 
       _from_rbdf(RbDataFrame.read_json(file))
@@ -279,7 +305,7 @@ module Polars
     # @private
     def self._read_ndjson(file)
       if Utils.pathlike?(file)
-        file = Utils.format_path(file)
+        file = Utils.normalise_filepath(file)
       end
 
       _from_rbdf(RbDataFrame.read_ndjson(file))
@@ -604,10 +630,10 @@ module Polars
           return Slice.new(self).apply(item)
         end
 
-        if Utils.is_str_sequence(item, allow_str: false)
+        if item.is_a?(Array) && item.all? { |v| Utils.strlike?(v) }
           # select multiple columns
           # df[["foo", "bar"]]
-          return _from_rbdf(_df.select(item))
+          return _from_rbdf(_df.select(item.map(&:to_s)))
         end
 
         if Utils.is_int_sequence(item)
@@ -689,7 +715,8 @@ module Polars
     # @example
     #   df = Polars::DataFrame.new({"foo" => [1, 2, 3], "bar" => [4, 5, 6]})
     #   df.to_hashes
-    #   [{'foo': 1, 'bar': 4}, {'foo': 2, 'bar': 5}, {'foo': 3, 'bar': 6}]
+    #   # =>
+    #   # [{"foo"=>1, "bar"=>4}, {"foo"=>2, "bar"=>5}, {"foo"=>3, "bar"=>6}]
     def to_hashes
       rbdf = _df
       names = columns
@@ -699,8 +726,26 @@ module Polars
       end
     end
 
-    # def to_numo
-    # end
+    # Convert DataFrame to a 2D Numo array.
+    #
+    # This operation clones data.
+    #
+    # @return [Numo::NArray]
+    #
+    # @example
+    #   df = Polars::DataFrame.new(
+    #     {"foo" => [1, 2, 3], "bar" => [6, 7, 8], "ham" => ["a", "b", "c"]}
+    #   )
+    #   df.to_numo.class
+    #   # => Numo::RObject
+    def to_numo
+      out = _df.to_numo
+      if out.nil?
+        Numo::NArray.vstack(width.times.map { |i| to_series(i).to_numo }).transpose
+      else
+        out
+      end
+    end
 
     # no to_pandas
 
@@ -753,7 +798,7 @@ module Polars
       row_oriented: false
     )
       if Utils.pathlike?(file)
-        file = Utils.format_path(file)
+        file = Utils.normalise_filepath(file)
       end
 
       _df.write_json(file, pretty, row_oriented)
@@ -768,7 +813,7 @@ module Polars
     # @return [nil]
     def write_ndjson(file)
       if Utils.pathlike?(file)
-        file = Utils.format_path(file)
+        file = Utils.normalise_filepath(file)
       end
 
       _df.write_ndjson(file)
@@ -858,7 +903,7 @@ module Polars
       end
 
       if Utils.pathlike?(file)
-        file = Utils.format_path(file)
+        file = Utils.normalise_filepath(file)
       end
 
       _df.write_csv(
@@ -896,7 +941,7 @@ module Polars
         compression = "uncompressed"
       end
       if Utils.pathlike?(file)
-        file = Utils.format_path(file)
+        file = Utils.normalise_filepath(file)
       end
 
       _df.write_avro(file, compression)
@@ -915,7 +960,7 @@ module Polars
         compression = "uncompressed"
       end
       if Utils.pathlike?(file)
-        file = Utils.format_path(file)
+        file = Utils.normalise_filepath(file)
       end
 
       _df.write_ipc(file, compression)
@@ -957,7 +1002,7 @@ module Polars
         compression = "uncompressed"
       end
       if Utils.pathlike?(file)
-        file = Utils.format_path(file)
+        file = Utils.normalise_filepath(file)
       end
 
       _df.write_parquet(
@@ -3021,24 +3066,28 @@ module Polars
       if aggregate_fn.is_a?(String)
         case aggregate_fn
         when "first"
-          aggregate_fn = Polars.element.first
+          aggregate_expr = Polars.element.first._rbexpr
         when "sum"
-          aggregate_fn = Polars.element.sum
+          aggregate_expr = Polars.element.sum._rbexpr
         when "max"
-          aggregate_fn = Polars.element.max
+          aggregate_expr = Polars.element.max._rbexpr
         when "min"
-          aggregate_fn = Polars.element.min
+          aggregate_expr = Polars.element.min._rbexpr
         when "mean"
-          aggregate_fn = Polars.element.mean
+          aggregate_expr = Polars.element.mean._rbexpr
         when "median"
-          aggregate_fn = Polars.element.median
+          aggregate_expr = Polars.element.median._rbexpr
         when "last"
-          aggregate_fn = Polars.element.last
+          aggregate_expr = Polars.element.last._rbexpr
         when "count"
-          aggregate_fn = Polars.count
+          aggregate_expr = Polars.count._rbexpr
         else
           raise ArgumentError, "Argument aggregate fn: '#{aggregate_fn}' was not expected."
         end
+      elsif aggregate_fn.nil?
+        aggregate_expr = nil
+      else
+        aggregate_expr = aggregate_function._rbexpr
       end
 
       _from_rbdf(
@@ -3046,9 +3095,9 @@ module Polars
           values,
           index,
           columns,
-          aggregate_fn._rbexpr,
           maintain_order,
           sort_columns,
+          aggregate_expr,
           separator
         )
       )
@@ -3153,7 +3202,7 @@ module Polars
     #   # │ B    ┆ 1    │
     #   # │ C    ┆ 2    │
     #   # │ D    ┆ 3    │
-    #   # │ ...  ┆ ...  │
+    #   # │ …    ┆ …    │
     #   # │ F    ┆ 5    │
     #   # │ G    ┆ 6    │
     #   # │ H    ┆ 7    │
@@ -4032,15 +4081,12 @@ module Polars
     #   # │ 5   ┆ 3.0 ┆ true  │
     #   # └─────┴─────┴───────┘
     def unique(maintain_order: true, subset: nil, keep: "first")
-      if !subset.nil?
-        if subset.is_a?(String)
-          subset = [subset]
-        elsif !subset.is_a?(Array)
-          subset = subset.to_a
-        end
-      end
-
-      _from_rbdf(_df.unique(maintain_order, subset, keep))
+      self._from_rbdf(
+        lazy
+          .unique(maintain_order: maintain_order, subset: subset, keep: keep)
+          .collect(no_optimization: true)
+          ._df
+      )
     end
 
     # Return the number of unique rows, or the number of unique row-subsets.

data/lib/polars/data_types.rb

@@ -84,6 +84,8 @@ module Polars
 
   # Calendar date and time type.
   class Datetime < TemporalType
+    attr_reader :tu
+
     def initialize(time_unit = "us", time_zone = nil)
       @tu = time_unit || "us"
       @time_zone = time_zone
@@ -92,6 +94,8 @@ module Polars
 
   # Time duration/delta type.
   class Duration < TemporalType
+    attr_reader :tu
+
     def initialize(time_unit = "us")
       @tu = time_unit
     end

data/lib/polars/date_time_expr.rb

@@ -1130,7 +1130,7 @@ module Polars
     #     ]
     #   )
     #   # =>
-    #   # shape: (1001, 2)
+    #   # shape: (1_001, 2)
     #   # ┌─────────────────────────┬───────────────────┐
     #   # │ date                    ┆ milliseconds_diff │
     #   # │ ---                     ┆ ---               │
@@ -1140,7 +1140,7 @@ module Polars
     #   # │ 2020-01-01 00:00:00.001 ┆ 1                 │
     #   # │ 2020-01-01 00:00:00.002 ┆ 1                 │
     #   # │ 2020-01-01 00:00:00.003 ┆ 1                 │
-    #   # │ ...                     ┆ ...               │
+    #   # │ …                       ┆ …                 │
     #   # │ 2020-01-01 00:00:00.997 ┆ 1                 │
     #   # │ 2020-01-01 00:00:00.998 ┆ 1                 │
     #   # │ 2020-01-01 00:00:00.999 ┆ 1                 │
@@ -1169,7 +1169,7 @@ module Polars
     #     ]
     #   )
     #   # =>
-    #   # shape: (1001, 2)
+    #   # shape: (1_001, 2)
     #   # ┌─────────────────────────┬───────────────────┐
     #   # │ date                    ┆ microseconds_diff │
     #   # │ ---                     ┆ ---               │
@@ -1179,7 +1179,7 @@ module Polars
     #   # │ 2020-01-01 00:00:00.001 ┆ 1000              │
     #   # │ 2020-01-01 00:00:00.002 ┆ 1000              │
     #   # │ 2020-01-01 00:00:00.003 ┆ 1000              │
-    #   # │ ...                     ┆ ...               │
+    #   # │ …                       ┆ …                 │
     #   # │ 2020-01-01 00:00:00.997 ┆ 1000              │
     #   # │ 2020-01-01 00:00:00.998 ┆ 1000              │
     #   # │ 2020-01-01 00:00:00.999 ┆ 1000              │
@@ -1208,7 +1208,7 @@ module Polars
     #     ]
     #   )
     #   # =>
-    #   # shape: (1001, 2)
+    #   # shape: (1_001, 2)
     #   # ┌─────────────────────────┬──────────────────┐
     #   # │ date                    ┆ nanoseconds_diff │
     #   # │ ---                     ┆ ---              │
@@ -1218,7 +1218,7 @@ module Polars
     #   # │ 2020-01-01 00:00:00.001 ┆ 1000000          │
     #   # │ 2020-01-01 00:00:00.002 ┆ 1000000          │
     #   # │ 2020-01-01 00:00:00.003 ┆ 1000000          │
-    #   # │ ...                     ┆ ...              │
+    #   # │ …                       ┆ …                │
     #   # │ 2020-01-01 00:00:00.997 ┆ 1000000          │
     #   # │ 2020-01-01 00:00:00.998 ┆ 1000000          │
     #   # │ 2020-01-01 00:00:00.999 ┆ 1000000          │

data/lib/polars/expr.rb

@@ -2194,7 +2194,7 @@ module Polars
     #   # │ 4      │
     #   # │ 6      │
     #   # │ 6      │
-    #   # │ ...    │
+    #   # │ …      │
     #   # │ 6      │
     #   # │ 6      │
     #   # │ 6      │
@@ -2571,7 +2571,7 @@ module Polars
     #   # │ e   │
     #   # │ l   │
     #   # │ l   │
-    #   # │ ... │
+    #   # │ …   │
     #   # │ o   │
     #   # │ r   │
     #   # │ l   │
@@ -4962,6 +4962,13 @@ module Polars
       ListExpr.new(self)
     end
 
+    # Create an object namespace of all binary related methods.
+    #
+    # @return [BinaryExpr]
+    def bin
+      BinaryExpr.new(self)
+    end
+
     # Create an object namespace of all categorical related methods.
     #
     # @return [CatExpr]

data/lib/polars/group_by.rb

@@ -571,5 +571,16 @@ module Polars
     def agg_list
       agg(Polars.all.list)
     end
+
+    # Plot data.
+    #
+    # @return [Vega::LiteChart]
+    def plot(*args, **options)
+      raise ArgumentError, "Multiple groups not supported" if by.is_a?(Array) && by.size > 1
+      # same message as Ruby
+      raise ArgumentError, "unknown keyword: :group" if options.key?(:group)
+
+      Utils.wrap_df(_df).plot(*args, **options, group: by)
+    end
   end
 end