polars-df 0.19.0-x86_64-darwin → 0.21.0-x86_64-darwin

data/lib/polars/list_name_space.rb

@@ -123,7 +123,7 @@ module Polars
     #   # shape: (2,)
     #   # Series: 'values' [list[i64]]
     #   # [
-    #   #         [2, 1]
+    #   #         [2, 3]
     #   #         [5]
     #   # ]
     def sample(n: nil, fraction: nil, with_replacement: false, shuffle: false, seed: nil)
@@ -586,12 +586,6 @@ module Polars
     # @param expr [Expr]
     #   Expression to run. Note that you can select an element with `Polars.first`, or
     #   `Polars.col`
-    # @param parallel [Boolean]
-    #   Run all expression parallel. Don't activate this blindly.
-    #   Parallelism is worth it if there is enough work to do per thread.
-    #
-    #   This likely should not be use in the group by context, because we already
-    #   parallel execution per group
     #
     # @return [Series]
     #
@@ -611,7 +605,126 @@ module Polars
     #   # │ 8   ┆ 5   ┆ [2.0, 1.0] │
     #   # │ 3   ┆ 2   ┆ [2.0, 1.0] │
     #   # └─────┴─────┴────────────┘
-    def eval(expr, parallel: false)
+    def eval(expr)
+      super
+    end
+
+    # Filter elements in each list by a boolean expression, returning a new Series of lists.
+    #
+    # @param predicate [Object]
+    #   A boolean expression evaluated on each list element.
+    #   Use `Polars.element` to refer to the current element.
+    #
+    # @return [Series]
+    #
+    # @example
+    #   s = Polars::Series.new("a", [[1, 4], [8, 5], [3, 2]])
+    #   s.list.filter(Polars.element % 2 == 0)
+    #   # =>
+    #   # shape: (3,)
+    #   # Series: 'a' [list[i64]]
+    #   # [
+    #   #         [4]
+    #   #         [8]
+    #   #         [2]
+    #   # ]
+    def filter(predicate)
+      super
+    end
+
+    # Compute the SET UNION between the elements in this list and the elements of `other`.
+    #
+    # @param other [Object]
+    #   Right hand side of the set operation.
+    #
+    # @return [Series]
+    #
+    # @example
+    #   a = Polars::Series.new([[1, 2, 3], [], [nil, 3], [5, 6, 7]])
+    #   b = Polars::Series.new([[2, 3, 4], [3], [3, 4, nil], [6, 8]])
+    #   a.list.set_union(b)
+    #   # =>
+    #   # shape: (4,)
+    #   # Series: '' [list[i64]]
+    #   # [
+    #   #         [1, 2, … 4]
+    #   #         [3]
+    #   #         [null, 3, 4]
+    #   #         [5, 6, … 8]
+    #   # ]
+    def set_union(other)
+      super
+    end
+
+    # Compute the SET DIFFERENCE between the elements in this list and the elements of `other`.
+    #
+    # @param other [Object]
+    #   Right hand side of the set operation.
+    #
+    # @return [Series]
+    #
+    # @example
+    #   a = Polars::Series.new([[1, 2, 3], [], [nil, 3], [5, 6, 7]])
+    #   b = Polars::Series.new([[2, 3, 4], [3], [3, 4, nil], [6, 8]])
+    #   a.list.set_difference(b)
+    #   # =>
+    #   # shape: (4,)
+    #   # Series: '' [list[i64]]
+    #   # [
+    #   #         [1]
+    #   #         []
+    #   #         []
+    #   #         [5, 7]
+    #   # ]
+    def set_difference(other)
+      super
+    end
+
+    # Compute the SET INTERSECTION between the elements in this list and the elements of `other`.
+    #
+    # @param other [Object]
+    #   Right hand side of the set operation.
+    #
+    # @return [Series]
+    #
+    # @example
+    #   a = Polars::Series.new([[1, 2, 3], [], [nil, 3], [5, 6, 7]])
+    #   b = Polars::Series.new([[2, 3, 4], [3], [3, 4, nil], [6, 8]])
+    #   a.list.set_intersection(b)
+    #   # =>
+    #   # shape: (4,)
+    #   # Series: '' [list[i64]]
+    #   # [
+    #   #         [2, 3]
+    #   #         []
+    #   #         [null, 3]
+    #   #         [6]
+    #   # ]
+    def set_intersection(other)
+      super
+    end
+
+    # Compute the SET SYMMETRIC DIFFERENCE between the elements in this list and the elements of `other`.
+    #
+    # @param other [Object]
+    #   Right hand side of the set operation.
+    #
+    # @return [Series]
+    #
+    # @example
+    #   a = Polars::Series.new([[1, 2, 3], [], [nil, 3], [5, 6, 7]])
+    #   b = Polars::Series.new([[2, 3, 4], [3], [3, 4, nil], [6, 8]])
+    #   a.list.set_symmetric_difference(b)
+    #   # =>
+    #   # shape: (4,)
+    #   # Series: '' [list[i64]]
+    #   # [
+    #   #         [1, 4]
+    #   #         [3]
+    #   #         [4]
+    #   #         [5, 7, 8]
+    #   # ]
+    def set_symmetric_difference(other)
       super
     end
   end

data/lib/polars/meta_expr.rb

@@ -125,14 +125,14 @@ module Polars
     # @return [Array]
     #
     # @example
-    #   e = Polars.col("foo").alias("bar")
+    #   e = Polars.col("foo") + Polars.col("bar")
     #   first = e.meta.pop[0]
-    #   _ = first.meta == Polars.col("foo")
-    #   # => true
     #   _ = first.meta == Polars.col("bar")
+    #   # => true
+    #   _ = first.meta == Polars.col("foo")
     #   # => false
-    def pop
-      _rbexpr.meta_pop.map { |e| Utils.wrap_expr(e) }
+    def pop(schema: nil)
+      _rbexpr.meta_pop(schema).map { |e| Utils.wrap_expr(e) }
     end
 
     # Get a list with the root column name.
@@ -171,32 +171,17 @@ module Polars
       Utils.wrap_expr(_rbexpr.meta_undo_aliases)
     end
 
-    # Turn this expression in a selector.
+    # Try to turn this expression in a selector.
     #
-    # @return [Expr]
-    def _as_selector
-      Utils.wrap_expr(_rbexpr._meta_as_selector)
-    end
-
-    # Add selectors.
+    # Raises if the underlying expressions is not a column or selector.
     #
     # @return [Expr]
-    def _selector_add(other)
-      Utils.wrap_expr(_rbexpr._meta_selector_add(other._rbexpr))
-    end
-
-    # Subtract selectors.
     #
-    # @return [Expr]
-    def _selector_sub(other)
-      Utils.wrap_expr(_rbexpr._meta_selector_sub(other._rbexpr))
-    end
-
-    # & selectors.
-    #
-    # @return [Expr]
-    def _selector_and(other)
-      Utils.wrap_expr(_rbexpr._meta_selector_and(other._rbexpr))
+    # @note
+    #   This functionality is considered **unstable**. It may be changed
+    #   at any point without it being considered a breaking change.
+    def as_selector
+      Selector._from_rbselector(_rbexpr.into_selector)
     end
 
     # Format the expression as a tree.
@@ -209,8 +194,8 @@ module Polars
     # @example
     #   e = (Polars.col("foo") * Polars.col("bar")).sum.over(Polars.col("ham")) / 2
     #   e.meta.tree_format(return_as_string: true)
-    def tree_format(return_as_string: false)
-      s = _rbexpr.meta_tree_format
+    def tree_format(return_as_string: false, schema: nil)
+      s = _rbexpr.meta_tree_format(schema)
       if return_as_string
         s
       else

data/lib/polars/scan_cast_options.rb

@@ -0,0 +1,64 @@
+module Polars
+  # Options for scanning files.
+  class ScanCastOptions
+    # Common configuration for scanning files.
+    #
+    # @note
+    #   This functionality is considered **unstable**. It may be changed
+    #   at any point without it being considered a breaking change.
+    #
+    # @param integer_cast ['upcast', 'forbid']
+    #   Configuration for casting from integer types:
+    #
+    #   * `upcast`: Allow lossless casting to wider integer types.
+    #   * `forbid`: Raises an error if dtypes do not match.
+    #
+    # @param float_cast ['upcast', 'downcast', 'forbid']
+    #   Configuration for casting from float types:
+    #
+    #   * `upcast`: Allow casting to higher precision float types.
+    #   * `downcast`: Allow casting to lower precision float types.
+    #   * `forbid`: Raises an error if dtypes do not match.
+    #
+    # @param datetime_cast ['nanosecond-downcast', 'convert-timezone', 'forbid']
+    #   Configuration for casting from datetime types:
+    #
+    #   * `nanosecond-downcast`: Allow nanosecond precision datetime to be
+    #     downcasted to any lower precision. This has a similar effect to
+    #     PyArrow's `coerce_int96_timestamp_unit`.
+    #   * `convert-timezone`: Allow casting to a different timezone.
+    #   * `forbid`: Raises an error if dtypes do not match.
+    #
+    # @param missing_struct_fields ['insert', 'raise']
+    #   Configuration for behavior when struct fields defined in the schema
+    #   are missing from the data:
+    #
+    #   * `insert`: Inserts the missing fields.
+    #   * `raise`: Raises an error.
+    #
+    # @param extra_struct_fields ['ignore', 'raise']
+    #   Configuration for behavior when extra struct fields outside of the
+    #   defined schema are encountered in the data:
+    #
+    #   * `ignore`: Silently ignores.
+    #   * `raise`: Raises an error.
+    def initialize(
+      integer_cast: "forbid",
+      float_cast: "forbid",
+      datetime_cast: "forbid",
+      missing_struct_fields: "raise",
+      extra_struct_fields: "raise",
+      _internal_call: false
+    )
+      @integer_cast = integer_cast
+      @float_cast = float_cast
+      @datetime_cast = datetime_cast
+      @missing_struct_fields = missing_struct_fields
+      @extra_struct_fields = extra_struct_fields
+    end
+
+    def self.default
+      new(_internal_call: true)
+    end
+  end
+end

data/lib/polars/schema.rb

@@ -1,6 +1,6 @@
 module Polars
   class Schema
-    def initialize(schema, check_dtypes: true)
+    def initialize(schema = nil, check_dtypes: true)
       raise Todo if check_dtypes
       @schema = schema.to_h
     end
@@ -9,6 +9,11 @@ module Polars
       @schema[key]
     end
 
+    def []=(name, dtype)
+      # TODO check dtype if needed
+      @schema[name] = dtype
+    end
+
     def names
       @schema.keys
     end

data/lib/polars/selector.rb

@@ -0,0 +1,138 @@
+module Polars
+  # Base column selector expression/proxy.
+  class Selector < Expr
+    # @private
+    attr_accessor :_rbselector
+
+    # @private
+    def self._from_rbselector(rbselector)
+      slf = new
+      slf._rbselector = rbselector
+      slf._rbexpr = RbExpr.new_selector(rbselector)
+      slf
+    end
+
+    def inspect
+      Expr._from_rbexpr(_rbexpr).to_s
+    end
+
+    # @private
+    def self._by_dtype(dtypes)
+      selectors = []
+      concrete_dtypes = []
+      dtypes.each do |dt|
+        if Utils.is_polars_dtype(dt)
+          concrete_dtypes += [dt]
+        else
+          raise Todo
+        end
+      end
+
+      dtype_selector = _from_rbselector(RbSelector.by_dtype(concrete_dtypes))
+
+      if selectors.length == 0
+        return dtype_selector
+      end
+
+      selector = selectors[0]
+      selectors[1..].each do |s|
+        selector = selector | s
+      end
+      if concrete_dtypes.length == 0
+        selector
+      else
+        dtype_selector | selector
+      end
+    end
+
+    # @private
+    def self._by_name(names, strict:)
+      _from_rbselector(RbSelector.by_name(names, strict))
+    end
+
+    def ~
+      Selectors.all - self
+    end
+
+    def &(other)
+      if Utils.is_column(other)
+        colname = other.meta.output_name
+        other = by_name(colname)
+      end
+      if Utils.is_selector(other)
+        Selector._from_rbselector(
+          _rbselector.intersect(other._rbselector)
+        )
+      else
+        as_expr & other
+      end
+    end
+
+    def |(other)
+      if Utils.is_column(other)
+        other = by_name(other.meta.output_name)
+      end
+      if Utils.is_selector(other)
+        Selector._from_rbselector(
+          _rbselector.union(other._rbselector)
+        )
+      else
+        as_expr | other
+      end
+    end
+
+    def -(other)
+      if Utils.is_selector(other)
+        Selector._from_rbselector(
+          _rbselector.difference(other._rbselector)
+        )
+      else
+        as_expr - other
+      end
+    end
+
+    def ^(other)
+      if Utils.is_column(other)
+        other = by_name(other.meta.output_name)
+      end
+      if Utils.is_selector(other)
+        Selector._from_rbselector(
+          _rbselector.exclusive_or(other._rbselector)
+        )
+      else
+        as_expr ^ other
+      end
+    end
+
+    def exclude(columns, *more_columns)
+      exclude_cols = []
+      exclude_dtypes = []
+      ((columns.is_a?(::Array) ? columns : [columns]) + more_columns).each do |item|
+        if item.is_a?(::String)
+          exclude_cols << item
+        elsif Utils.is_polars_dtype(item)
+          exclude_dtypes << item
+        else
+          msg = (
+            "invalid input for `exclude`" +
+            "\n\nExpected one or more `str` or `DataType`; found #{item.inspect} instead."
+          )
+          raise TypeError, msg
+        end
+      end
+
+      if exclude_cols.any? && exclude_dtypes.any?
+        msg = "cannot exclude by both column name and dtype; use a selector instead"
+        raise TypeError, msg
+      elsif exclude_dtypes.any?
+        self - Selectors.by_dtype(exclude_dtypes)
+      else
+        self - Selectors.by_name(exclude_cols, require_all: false)
+      end
+    end
+
+    def as_expr
+      Expr._from_rbexpr(_rbexpr)
+    end
+  end
+end