polars-df 0.15.0 → 0.26.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +274 -0
  3. data/Cargo.lock +1465 -867
  4. data/Cargo.toml +3 -0
  5. data/LICENSE.txt +2 -2
  6. data/README.md +87 -37
  7. data/ext/polars/Cargo.toml +47 -16
  8. data/ext/polars/src/c_api/allocator.rs +7 -0
  9. data/ext/polars/src/c_api/mod.rs +1 -0
  10. data/ext/polars/src/catalog/mod.rs +1 -0
  11. data/ext/polars/src/catalog/unity.rs +470 -0
  12. data/ext/polars/src/conversion/any_value.rs +99 -84
  13. data/ext/polars/src/conversion/categorical.rs +30 -0
  14. data/ext/polars/src/conversion/chunked_array.rs +71 -62
  15. data/ext/polars/src/conversion/datetime.rs +63 -0
  16. data/ext/polars/src/conversion/mod.rs +796 -312
  17. data/ext/polars/src/dataframe/construction.rs +6 -18
  18. data/ext/polars/src/dataframe/export.rs +30 -39
  19. data/ext/polars/src/dataframe/general.rs +294 -362
  20. data/ext/polars/src/dataframe/io.rs +33 -150
  21. data/ext/polars/src/dataframe/map.rs +175 -0
  22. data/ext/polars/src/dataframe/mod.rs +37 -5
  23. data/ext/polars/src/dataframe/serde.rs +23 -8
  24. data/ext/polars/src/error.rs +44 -7
  25. data/ext/polars/src/exceptions.rs +21 -8
  26. data/ext/polars/src/expr/array.rs +86 -22
  27. data/ext/polars/src/expr/binary.rs +50 -1
  28. data/ext/polars/src/expr/bitwise.rs +39 -0
  29. data/ext/polars/src/expr/categorical.rs +20 -0
  30. data/ext/polars/src/expr/datatype.rs +51 -0
  31. data/ext/polars/src/expr/datetime.rs +99 -41
  32. data/ext/polars/src/expr/extension.rs +12 -0
  33. data/ext/polars/src/expr/general.rs +252 -128
  34. data/ext/polars/src/expr/list.rs +56 -60
  35. data/ext/polars/src/expr/meta.rs +30 -35
  36. data/ext/polars/src/expr/mod.rs +28 -6
  37. data/ext/polars/src/expr/name.rs +29 -14
  38. data/ext/polars/src/expr/rolling.rs +111 -3
  39. data/ext/polars/src/expr/selector.rs +219 -0
  40. data/ext/polars/src/expr/serde.rs +28 -0
  41. data/ext/polars/src/expr/string.rs +118 -20
  42. data/ext/polars/src/expr/struct.rs +14 -1
  43. data/ext/polars/src/file.rs +194 -86
  44. data/ext/polars/src/functions/aggregation.rs +13 -12
  45. data/ext/polars/src/functions/business.rs +2 -3
  46. data/ext/polars/src/functions/eager.rs +3 -2
  47. data/ext/polars/src/functions/io.rs +90 -18
  48. data/ext/polars/src/functions/lazy.rs +267 -118
  49. data/ext/polars/src/functions/meta.rs +8 -7
  50. data/ext/polars/src/functions/misc.rs +1 -1
  51. data/ext/polars/src/functions/mod.rs +2 -1
  52. data/ext/polars/src/functions/range.rs +88 -31
  53. data/ext/polars/src/functions/strings.rs +6 -0
  54. data/ext/polars/src/functions/utils.rs +8 -0
  55. data/ext/polars/src/interop/arrow/mod.rs +52 -1
  56. data/ext/polars/src/interop/arrow/{to_ruby.rs → to_rb.rs} +37 -7
  57. data/ext/polars/src/interop/arrow/to_rust.rs +43 -0
  58. data/ext/polars/src/interop/numo/to_numo_df.rs +1 -1
  59. data/ext/polars/src/interop/numo/to_numo_series.rs +72 -50
  60. data/ext/polars/src/io/cloud_options.rs +107 -0
  61. data/ext/polars/src/io/mod.rs +4 -0
  62. data/ext/polars/src/io/scan_options.rs +113 -0
  63. data/ext/polars/src/io/sink_options.rs +46 -0
  64. data/ext/polars/src/io/sink_output.rs +21 -0
  65. data/ext/polars/src/lazyframe/exitable.rs +39 -0
  66. data/ext/polars/src/lazyframe/general.rs +846 -368
  67. data/ext/polars/src/lazyframe/mod.rs +58 -5
  68. data/ext/polars/src/lazyframe/optflags.rs +59 -0
  69. data/ext/polars/src/lazyframe/serde.rs +36 -4
  70. data/ext/polars/src/lazyframe/sink.rs +46 -0
  71. data/ext/polars/src/lazygroupby.rs +38 -9
  72. data/ext/polars/src/lib.rs +574 -165
  73. data/ext/polars/src/map/lazy.rs +44 -74
  74. data/ext/polars/src/map/mod.rs +18 -254
  75. data/ext/polars/src/map/series.rs +241 -1087
  76. data/ext/polars/src/on_startup.rs +192 -9
  77. data/ext/polars/src/prelude.rs +1 -0
  78. data/ext/polars/src/rb_modules.rs +10 -57
  79. data/ext/polars/src/ruby/exceptions.rs +26 -0
  80. data/ext/polars/src/ruby/gvl.rs +104 -0
  81. data/ext/polars/src/ruby/lazy.rs +46 -0
  82. data/ext/polars/src/ruby/mod.rs +11 -0
  83. data/ext/polars/src/ruby/numo.rs +52 -0
  84. data/ext/polars/src/ruby/plan_callback.rs +198 -0
  85. data/ext/polars/src/ruby/rb_modules.rs +16 -0
  86. data/ext/polars/src/ruby/ruby_convert_registry.rs +51 -0
  87. data/ext/polars/src/ruby/ruby_function.rs +11 -0
  88. data/ext/polars/src/ruby/ruby_udf.rs +164 -0
  89. data/ext/polars/src/ruby/thread.rs +65 -0
  90. data/ext/polars/src/ruby/utils.rs +39 -0
  91. data/ext/polars/src/series/aggregation.rs +116 -91
  92. data/ext/polars/src/series/arithmetic.rs +16 -22
  93. data/ext/polars/src/series/comparison.rs +101 -222
  94. data/ext/polars/src/series/construction.rs +80 -70
  95. data/ext/polars/src/series/export.rs +98 -56
  96. data/ext/polars/src/series/general.rs +323 -440
  97. data/ext/polars/src/series/import.rs +22 -5
  98. data/ext/polars/src/series/map.rs +103 -0
  99. data/ext/polars/src/series/mod.rs +57 -15
  100. data/ext/polars/src/series/scatter.rs +139 -82
  101. data/ext/polars/src/sql.rs +16 -9
  102. data/ext/polars/src/testing/frame.rs +31 -0
  103. data/ext/polars/src/testing/mod.rs +5 -0
  104. data/ext/polars/src/testing/series.rs +31 -0
  105. data/ext/polars/src/timeout.rs +105 -0
  106. data/ext/polars/src/utils.rs +105 -4
  107. data/lib/polars/array_expr.rb +500 -22
  108. data/lib/polars/array_name_space.rb +384 -10
  109. data/lib/polars/batched_csv_reader.rb +48 -66
  110. data/lib/polars/binary_expr.rb +217 -0
  111. data/lib/polars/binary_name_space.rb +155 -1
  112. data/lib/polars/cat_expr.rb +224 -0
  113. data/lib/polars/cat_name_space.rb +132 -32
  114. data/lib/polars/catalog/unity/catalog_info.rb +20 -0
  115. data/lib/polars/catalog/unity/column_info.rb +31 -0
  116. data/lib/polars/catalog/unity/namespace_info.rb +21 -0
  117. data/lib/polars/catalog/unity/table_info.rb +50 -0
  118. data/lib/polars/catalog.rb +448 -0
  119. data/lib/polars/collect_batches.rb +22 -0
  120. data/lib/polars/config.rb +3 -3
  121. data/lib/polars/convert.rb +201 -36
  122. data/lib/polars/data_frame.rb +2851 -1017
  123. data/lib/polars/data_frame_plot.rb +173 -0
  124. data/lib/polars/data_type_expr.rb +52 -0
  125. data/lib/polars/data_type_group.rb +6 -0
  126. data/lib/polars/data_types.rb +118 -18
  127. data/lib/polars/date_time_expr.rb +426 -84
  128. data/lib/polars/date_time_name_space.rb +384 -111
  129. data/lib/polars/dynamic_group_by.rb +102 -10
  130. data/lib/polars/exceptions.rb +50 -5
  131. data/lib/polars/expr.rb +2159 -915
  132. data/lib/polars/extension_expr.rb +39 -0
  133. data/lib/polars/extension_name_space.rb +39 -0
  134. data/lib/polars/functions/aggregation/horizontal.rb +11 -6
  135. data/lib/polars/functions/aggregation/vertical.rb +2 -3
  136. data/lib/polars/functions/as_datatype.rb +290 -8
  137. data/lib/polars/functions/business.rb +95 -0
  138. data/lib/polars/functions/col.rb +6 -5
  139. data/lib/polars/functions/datatype.rb +62 -0
  140. data/lib/polars/functions/eager.rb +426 -24
  141. data/lib/polars/functions/escape_regex.rb +21 -0
  142. data/lib/polars/functions/lazy.rb +813 -195
  143. data/lib/polars/functions/lit.rb +21 -10
  144. data/lib/polars/functions/range/int_range.rb +74 -2
  145. data/lib/polars/functions/range/linear_space.rb +195 -0
  146. data/lib/polars/functions/range/time_range.rb +1 -1
  147. data/lib/polars/functions/repeat.rb +7 -12
  148. data/lib/polars/functions/whenthen.rb +2 -2
  149. data/lib/polars/group_by.rb +188 -58
  150. data/lib/polars/iceberg_dataset.rb +108 -0
  151. data/lib/polars/in_process_query.rb +37 -0
  152. data/lib/polars/io/cloud.rb +18 -0
  153. data/lib/polars/io/csv.rb +336 -128
  154. data/lib/polars/io/database.rb +19 -4
  155. data/lib/polars/io/delta.rb +134 -0
  156. data/lib/polars/io/iceberg.rb +34 -0
  157. data/lib/polars/io/ipc.rb +63 -63
  158. data/lib/polars/io/json.rb +16 -0
  159. data/lib/polars/io/lines.rb +172 -0
  160. data/lib/polars/io/ndjson.rb +176 -20
  161. data/lib/polars/io/parquet.rb +173 -95
  162. data/lib/polars/io/scan_options.rb +55 -0
  163. data/lib/polars/io/sink_options.rb +27 -0
  164. data/lib/polars/io/utils.rb +17 -0
  165. data/lib/polars/lazy_frame.rb +3017 -622
  166. data/lib/polars/lazy_group_by.rb +436 -2
  167. data/lib/polars/list_expr.rb +551 -59
  168. data/lib/polars/list_name_space.rb +465 -51
  169. data/lib/polars/meta_expr.rb +146 -24
  170. data/lib/polars/name_expr.rb +87 -2
  171. data/lib/polars/query_opt_flags.rb +264 -0
  172. data/lib/polars/rolling_group_by.rb +90 -5
  173. data/lib/polars/scan_cast_options.rb +86 -0
  174. data/lib/polars/schema.rb +128 -0
  175. data/lib/polars/selector.rb +245 -0
  176. data/lib/polars/selectors.rb +1048 -201
  177. data/lib/polars/series.rb +2522 -774
  178. data/lib/polars/series_plot.rb +72 -0
  179. data/lib/polars/slice.rb +1 -1
  180. data/lib/polars/sql_context.rb +13 -6
  181. data/lib/polars/string_cache.rb +19 -72
  182. data/lib/polars/string_expr.rb +561 -107
  183. data/lib/polars/string_name_space.rb +781 -109
  184. data/lib/polars/struct_expr.rb +139 -18
  185. data/lib/polars/struct_name_space.rb +19 -1
  186. data/lib/polars/testing.rb +24 -273
  187. data/lib/polars/utils/constants.rb +2 -0
  188. data/lib/polars/utils/construction/data_frame.rb +410 -0
  189. data/lib/polars/utils/construction/series.rb +350 -0
  190. data/lib/polars/utils/construction/utils.rb +9 -0
  191. data/lib/polars/utils/convert.rb +18 -8
  192. data/lib/polars/utils/deprecation.rb +11 -0
  193. data/lib/polars/utils/parse.rb +62 -9
  194. data/lib/polars/utils/reduce_balanced.rb +43 -0
  195. data/lib/polars/utils/serde.rb +22 -0
  196. data/lib/polars/utils/unstable.rb +19 -0
  197. data/lib/polars/utils/various.rb +86 -1
  198. data/lib/polars/utils.rb +63 -48
  199. data/lib/polars/version.rb +1 -1
  200. data/lib/polars.rb +85 -2
  201. metadata +80 -28
  202. data/ext/polars/src/allocator.rs +0 -13
  203. data/ext/polars/src/batched_csv.rs +0 -138
  204. data/ext/polars/src/functions/string_cache.rs +0 -25
  205. data/ext/polars/src/map/dataframe.rs +0 -338
  206. data/lib/polars/plot.rb +0 -109
@@ -9,6 +9,181 @@ module Polars
9
9
  self._rbexpr = expr._rbexpr
10
10
  end
11
11
 
12
+ # Return the number of elements in each array.
13
+ #
14
+ # @return [Expr]
15
+ #
16
+ # @example
17
+ # df = Polars::DataFrame.new(
18
+ # {"a" => [[1, 2], [4, 3]]},
19
+ # schema: {"a" => Polars::Array.new(Polars::Int64, 2)}
20
+ # )
21
+ # df.select(Polars.col("a").arr.len)
22
+ # # =>
23
+ # # shape: (2, 1)
24
+ # # ┌─────┐
25
+ # # │ a │
26
+ # # │ --- │
27
+ # # │ u32 │
28
+ # # ╞═════╡
29
+ # # │ 2 │
30
+ # # │ 2 │
31
+ # # └─────┘
32
+ def len
33
+ Utils.wrap_expr(_rbexpr.arr_len)
34
+ end
35
+
36
+ # Slice every subarray.
37
+ #
38
+ # @param offset [Integer]
39
+ # Start index. Negative indexing is supported.
40
+ # @param length [Integer]
41
+ # Length of the slice. If set to `nil` (default), the slice is taken to the
42
+ # end of the list.
43
+ # @param as_array [Boolean]
44
+ # Return result as a fixed-length `Array`, otherwise as a `List`.
45
+ # If true `length` and `offset` must be constant values.
46
+ #
47
+ # @return [Expr]
48
+ #
49
+ # @example
50
+ # df = Polars::DataFrame.new(
51
+ # {"a" => [[1, 2], [4, 3]]},
52
+ # schema: {"a" => Polars::Array.new(Polars::Int64, 2)}
53
+ # )
54
+ # df.select(Polars.col("a").arr.slice(0, 1))
55
+ # # =>
56
+ # # shape: (2, 1)
57
+ # # ┌───────────┐
58
+ # # │ a │
59
+ # # │ --- │
60
+ # # │ list[i64] │
61
+ # # ╞═══════════╡
62
+ # # │ [1] │
63
+ # # │ [4] │
64
+ # # └───────────┘
65
+ #
66
+ # @example
67
+ # df = Polars::DataFrame.new(
68
+ # {"a" => [[1, 2], [4, 3]]},
69
+ # schema: {"a" => Polars::Array.new(Polars::Int64, 2)}
70
+ # )
71
+ # df.select(Polars.col("a").arr.slice(0, 1, as_array: true))
72
+ # # =>
73
+ # # shape: (2, 1)
74
+ # # ┌───────────────┐
75
+ # # │ a │
76
+ # # │ --- │
77
+ # # │ array[i64, 1] │
78
+ # # ╞═══════════════╡
79
+ # # │ [1] │
80
+ # # │ [4] │
81
+ # # └───────────────┘
82
+ def slice(
83
+ offset,
84
+ length = nil,
85
+ as_array: false
86
+ )
87
+ offset = Utils.parse_into_expression(offset)
88
+ length = !length.nil? ? Utils.parse_into_expression(length) : nil
89
+ Utils.wrap_expr(_rbexpr.arr_slice(offset, length, as_array))
90
+ end
91
+
92
+ # Get the first `n` elements of the sub-arrays.
93
+ #
94
+ # @param n [Integer]
95
+ # Number of values to return for each sublist.
96
+ # @param as_array [Boolean]
97
+ # Return result as a fixed-length `Array`, otherwise as a `List`.
98
+ # If true `n` must be a constant value.
99
+ #
100
+ # @return [Expr]
101
+ #
102
+ # @example
103
+ # df = Polars::DataFrame.new(
104
+ # {"a" => [[1, 2], [4, 3]]},
105
+ # schema: {"a" => Polars::Array.new(Polars::Int64, 2)}
106
+ # )
107
+ # df.select(Polars.col("a").arr.head(1))
108
+ # # =>
109
+ # # shape: (2, 1)
110
+ # # ┌───────────┐
111
+ # # │ a │
112
+ # # │ --- │
113
+ # # │ list[i64] │
114
+ # # ╞═══════════╡
115
+ # # │ [1] │
116
+ # # │ [4] │
117
+ # # └───────────┘
118
+ #
119
+ # @example
120
+ # df = Polars::DataFrame.new(
121
+ # {"a" => [[1, 2], [4, 3]]},
122
+ # schema: {"a" => Polars::Array.new(Polars::Int64, 2)}
123
+ # )
124
+ # df.select(Polars.col("a").arr.head(1, as_array: true))
125
+ # # =>
126
+ # # shape: (2, 1)
127
+ # # ┌───────────────┐
128
+ # # │ a │
129
+ # # │ --- │
130
+ # # │ array[i64, 1] │
131
+ # # ╞═══════════════╡
132
+ # # │ [1] │
133
+ # # │ [4] │
134
+ # # └───────────────┘
135
+ def head(n = 5, as_array: false)
136
+ slice(0, n, as_array: as_array)
137
+ end
138
+
139
+ # Slice the last `n` values of every sublist.
140
+ #
141
+ # @param n [Integer]
142
+ # Number of values to return for each sublist.
143
+ # @param as_array [Boolean]
144
+ # Return result as a fixed-length `Array`, otherwise as a `List`.
145
+ # If true `n` must be a constant value.
146
+ #
147
+ # @return [Expr]
148
+ #
149
+ # @example
150
+ # df = Polars::DataFrame.new(
151
+ # {"a" => [[1, 2], [4, 3]]},
152
+ # schema: {"a" => Polars::Array.new(Polars::Int64, 2)}
153
+ # )
154
+ # df.select(Polars.col("a").arr.tail(1))
155
+ # # =>
156
+ # # shape: (2, 1)
157
+ # # ┌───────────┐
158
+ # # │ a │
159
+ # # │ --- │
160
+ # # │ list[i64] │
161
+ # # ╞═══════════╡
162
+ # # │ [2] │
163
+ # # │ [3] │
164
+ # # └───────────┘
165
+ #
166
+ # @example
167
+ # df = Polars::DataFrame.new(
168
+ # {"a" => [[1, 2], [4, 3]]},
169
+ # schema: {"a" => Polars::Array.new(Polars::Int64, 2)}
170
+ # )
171
+ # df.select(Polars.col("a").arr.tail(1, as_array: true))
172
+ # # =>
173
+ # # shape: (2, 1)
174
+ # # ┌───────────────┐
175
+ # # │ a │
176
+ # # │ --- │
177
+ # # │ array[i64, 1] │
178
+ # # ╞═══════════════╡
179
+ # # │ [2] │
180
+ # # │ [3] │
181
+ # # └───────────────┘
182
+ def tail(n = 5, as_array: false)
183
+ n = Utils.parse_into_expression(n)
184
+ Utils.wrap_expr(_rbexpr.arr_tail(n, as_array))
185
+ end
186
+
12
187
  # Compute the min values of the sub-arrays.
13
188
  #
14
189
  # @return [Expr]
@@ -16,7 +191,7 @@ module Polars
16
191
  # @example
17
192
  # df = Polars::DataFrame.new(
18
193
  # {"a" => [[1, 2], [4, 3]]},
19
- # schema: {"a" => Polars::Array.new(2, Polars::Int64)}
194
+ # schema: {"a" => Polars::Array.new(Polars::Int64, 2)}
20
195
  # )
21
196
  # df.select(Polars.col("a").arr.min)
22
197
  # # =>
@@ -30,7 +205,7 @@ module Polars
30
205
  # # │ 3 │
31
206
  # # └─────┘
32
207
  def min
33
- Utils.wrap_expr(_rbexpr.array_min)
208
+ Utils.wrap_expr(_rbexpr.arr_min)
34
209
  end
35
210
 
36
211
  # Compute the max values of the sub-arrays.
@@ -40,7 +215,7 @@ module Polars
40
215
  # @example
41
216
  # df = Polars::DataFrame.new(
42
217
  # {"a" => [[1, 2], [4, 3]]},
43
- # schema: {"a" => Polars::Array.new(2, Polars::Int64)}
218
+ # schema: {"a" => Polars::Array.new(Polars::Int64, 2)}
44
219
  # )
45
220
  # df.select(Polars.col("a").arr.max)
46
221
  # # =>
@@ -54,7 +229,7 @@ module Polars
54
229
  # # │ 4 │
55
230
  # # └─────┘
56
231
  def max
57
- Utils.wrap_expr(_rbexpr.array_max)
232
+ Utils.wrap_expr(_rbexpr.arr_max)
58
233
  end
59
234
 
60
235
  # Compute the sum values of the sub-arrays.
@@ -64,7 +239,7 @@ module Polars
64
239
  # @example
65
240
  # df = Polars::DataFrame.new(
66
241
  # {"a" => [[1, 2], [4, 3]]},
67
- # schema: {"a" => Polars::Array.new(2, Polars::Int64)}
242
+ # schema: {"a" => Polars::Array.new(Polars::Int64, 2)}
68
243
  # )
69
244
  # df.select(Polars.col("a").arr.sum)
70
245
  # # =>
@@ -78,7 +253,103 @@ module Polars
78
253
  # # │ 7 │
79
254
  # # └─────┘
80
255
  def sum
81
- Utils.wrap_expr(_rbexpr.array_sum)
256
+ Utils.wrap_expr(_rbexpr.arr_sum)
257
+ end
258
+
259
+ # Compute the std of the values of the sub-arrays.
260
+ #
261
+ # @return [Expr]
262
+ #
263
+ # @example
264
+ # df = Polars::DataFrame.new(
265
+ # {"a" => [[1, 2], [4, 3]]},
266
+ # schema: {"a" => Polars::Array.new(Polars::Int64, 2)}
267
+ # )
268
+ # df.select(Polars.col("a").arr.std)
269
+ # # =>
270
+ # # shape: (2, 1)
271
+ # # ┌──────────┐
272
+ # # │ a │
273
+ # # │ --- │
274
+ # # │ f64 │
275
+ # # ╞══════════╡
276
+ # # │ 0.707107 │
277
+ # # │ 0.707107 │
278
+ # # └──────────┘
279
+ def std(ddof: 1)
280
+ Utils.wrap_expr(_rbexpr.arr_std(ddof))
281
+ end
282
+
283
+ # Compute the var of the values of the sub-arrays.
284
+ #
285
+ # @return [Expr]
286
+ #
287
+ # @example
288
+ # df = Polars::DataFrame.new(
289
+ # {"a" => [[1, 2], [4, 3]]},
290
+ # schema: {"a" => Polars::Array.new(Polars::Int64, 2)}
291
+ # )
292
+ # df.select(Polars.col("a").arr.var)
293
+ # # =>
294
+ # # shape: (2, 1)
295
+ # # ┌─────┐
296
+ # # │ a │
297
+ # # │ --- │
298
+ # # │ f64 │
299
+ # # ╞═════╡
300
+ # # │ 0.5 │
301
+ # # │ 0.5 │
302
+ # # └─────┘
303
+ def var(ddof: 1)
304
+ Utils.wrap_expr(_rbexpr.arr_var(ddof))
305
+ end
306
+
307
+ # Compute the mean of the values of the sub-arrays.
308
+ #
309
+ # @return [Expr]
310
+ #
311
+ # @example
312
+ # df = Polars::DataFrame.new(
313
+ # {"a" => [[1, 2, 3], [1, 1, 16]]},
314
+ # schema: {"a" => Polars::Array.new(Polars::Int64, 3)}
315
+ # )
316
+ # df.select(Polars.col("a").arr.mean)
317
+ # # =>
318
+ # # shape: (2, 1)
319
+ # # ┌─────┐
320
+ # # │ a │
321
+ # # │ --- │
322
+ # # │ f64 │
323
+ # # ╞═════╡
324
+ # # │ 2.0 │
325
+ # # │ 6.0 │
326
+ # # └─────┘
327
+ def mean
328
+ Utils.wrap_expr(_rbexpr.arr_mean)
329
+ end
330
+
331
+ # Compute the median of the values of the sub-arrays.
332
+ #
333
+ # @return [Expr]
334
+ #
335
+ # @example
336
+ # df = Polars::DataFrame.new(
337
+ # {"a" => [[1, 2], [4, 3]]},
338
+ # schema: {"a" => Polars::Array.new(Polars::Int64, 2)}
339
+ # )
340
+ # df.select(Polars.col("a").arr.median)
341
+ # # =>
342
+ # # shape: (2, 1)
343
+ # # ┌─────┐
344
+ # # │ a │
345
+ # # │ --- │
346
+ # # │ f64 │
347
+ # # ╞═════╡
348
+ # # │ 1.5 │
349
+ # # │ 3.5 │
350
+ # # └─────┘
351
+ def median
352
+ Utils.wrap_expr(_rbexpr.arr_median)
82
353
  end
83
354
 
84
355
  # Get the unique/distinct values in the array.
@@ -106,7 +377,35 @@ module Polars
106
377
  # # │ [1, 2] │
107
378
  # # └───────────┘
108
379
  def unique(maintain_order: false)
109
- Utils.wrap_expr(_rbexpr.arr_unique(maintain_order))
380
+ eval(
381
+ F.element.unique(maintain_order: maintain_order), as_list: true
382
+ )
383
+ end
384
+
385
+ # Count the number of unique values in every sub-arrays.
386
+ #
387
+ # @return [Expr]
388
+ #
389
+ # @example
390
+ # df = Polars::DataFrame.new(
391
+ # {
392
+ # "a" => [[1, 1, 2], [2, 3, 4]],
393
+ # },
394
+ # schema: {"a" => Polars::Array.new(Polars::Int64, 3)}
395
+ # )
396
+ # df.with_columns(n_unique: Polars.col("a").arr.n_unique)
397
+ # # =>
398
+ # # shape: (2, 2)
399
+ # # ┌───────────────┬──────────┐
400
+ # # │ a ┆ n_unique │
401
+ # # │ --- ┆ --- │
402
+ # # │ array[i64, 3] ┆ u32 │
403
+ # # ╞═══════════════╪══════════╡
404
+ # # │ [1, 1, 2] ┆ 2 │
405
+ # # │ [2, 3, 4] ┆ 3 │
406
+ # # └───────────────┴──────────┘
407
+ def n_unique
408
+ agg(F.element.n_unique)
110
409
  end
111
410
 
112
411
  # Convert an Array column into a List column with the same inner data type.
@@ -135,12 +434,19 @@ module Polars
135
434
 
136
435
  # Evaluate whether any boolean value is true for every subarray.
137
436
  #
437
+ # @param ignore_nulls [Boolean]
438
+ # * If set to `true` (default), null values are ignored. If there
439
+ # are no non-null values, the output is `false`.
440
+ # * If set to `false`, [Kleene logic](https://en.wikipedia.org/wiki/Three-valued_logic) is used to deal with nulls:
441
+ # if the column contains any null values and no `true` values,
442
+ # the output is null.
443
+ #
138
444
  # @return [Expr]
139
445
  #
140
446
  # @example
141
447
  # df = Polars::DataFrame.new(
142
448
  # {
143
- # "a": [
449
+ # "a" => [
144
450
  # [true, true],
145
451
  # [false, true],
146
452
  # [false, false],
@@ -164,18 +470,25 @@ module Polars
164
470
  # # │ [null, null] ┆ false │
165
471
  # # │ null ┆ null │
166
472
  # # └────────────────┴───────┘
167
- def any
168
- Utils.wrap_expr(_rbexpr.arr_any)
473
+ def any(ignore_nulls: true)
474
+ agg(F.element.any(ignore_nulls: ignore_nulls))
169
475
  end
170
476
 
171
477
  # Evaluate whether all boolean values are true for every subarray.
172
478
  #
479
+ # @param ignore_nulls [Boolean]
480
+ # * If set to `true` (default), null values are ignored. If there
481
+ # are no non-null values, the output is `true`.
482
+ # * If set to `false`, [Kleene logic](https://en.wikipedia.org/wiki/Three-valued_logic) is used to deal with nulls:
483
+ # if the column contains any null values and no `false` values,
484
+ # the output is null.
485
+ #
173
486
  # @return [Expr]
174
487
  #
175
488
  # @example
176
489
  # df = Polars::DataFrame.new(
177
490
  # {
178
- # "a": [
491
+ # "a" => [
179
492
  # [true, true],
180
493
  # [false, true],
181
494
  # [false, false],
@@ -199,8 +512,8 @@ module Polars
199
512
  # # │ [null, null] ┆ true │
200
513
  # # │ null ┆ null │
201
514
  # # └────────────────┴───────┘
202
- def all
203
- Utils.wrap_expr(_rbexpr.arr_all)
515
+ def all(ignore_nulls: true)
516
+ agg(F.element.all(ignore_nulls: ignore_nulls))
204
517
  end
205
518
 
206
519
  # Sort the arrays in this column.
@@ -270,7 +583,7 @@ module Polars
270
583
  # # │ [9, 1, 2] ┆ [2, 1, 9] │
271
584
  # # └───────────────┴───────────────┘
272
585
  def reverse
273
- Utils.wrap_expr(_rbexpr.arr_reverse)
586
+ eval(F.element.reverse)
274
587
  end
275
588
 
276
589
  # Retrieve the index of the minimal value in every sub-array.
@@ -345,7 +658,7 @@ module Polars
345
658
  # {"arr" => [[1, 2, 3], [4, 5, 6], [7, 8, 9]], "idx" => [1, -2, 4]},
346
659
  # schema: {"arr" => Polars::Array.new(Polars::Int32, 3), "idx" => Polars::Int32}
347
660
  # )
348
- # df.with_columns(get: Polars.col("arr").arr.get("idx"))
661
+ # df.with_columns(get: Polars.col("arr").arr.get("idx", null_on_oob: true))
349
662
  # # =>
350
663
  # # shape: (3, 3)
351
664
  # # ┌───────────────┬─────┬──────┐
@@ -357,7 +670,7 @@ module Polars
357
670
  # # │ [4, 5, 6] ┆ -2 ┆ 5 │
358
671
  # # │ [7, 8, 9] ┆ 4 ┆ null │
359
672
  # # └───────────────┴─────┴──────┘
360
- def get(index, null_on_oob: true)
673
+ def get(index, null_on_oob: false)
361
674
  index = Utils.parse_into_expression(index)
362
675
  Utils.wrap_expr(_rbexpr.arr_get(index, null_on_oob))
363
676
  end
@@ -384,7 +697,7 @@ module Polars
384
697
  # # │ [7, 8, 9] ┆ 7 │
385
698
  # # └───────────────┴───────┘
386
699
  def first
387
- get(0)
700
+ get(0, null_on_oob: true)
388
701
  end
389
702
 
390
703
  # Get the last value of the sub-arrays.
@@ -409,7 +722,7 @@ module Polars
409
722
  # # │ [7, 8, 9] ┆ 9 │
410
723
  # # └───────────────┴──────┘
411
724
  def last
412
- get(-1)
725
+ get(-1, null_on_oob: true)
413
726
  end
414
727
 
415
728
  # Join all string items in a sub-array and place a separator between them.
@@ -452,6 +765,11 @@ module Polars
452
765
 
453
766
  # Returns a column with a separate row for every array element.
454
767
  #
768
+ # @param empty_as_null [Boolean]
769
+ # Explode an empty array into a `null`.
770
+ # @param keep_nulls [Boolean]
771
+ # Explode a `null` array into a `null`.
772
+ #
455
773
  # @return [Expr]
456
774
  #
457
775
  # @example
@@ -473,14 +791,16 @@ module Polars
473
791
  # # │ 5 │
474
792
  # # │ 6 │
475
793
  # # └─────┘
476
- def explode
477
- Utils.wrap_expr(_rbexpr.explode)
794
+ def explode(empty_as_null: true, keep_nulls: true)
795
+ Utils.wrap_expr(_rbexpr.explode(empty_as_null, keep_nulls))
478
796
  end
479
797
 
480
798
  # Check if sub-arrays contain the given item.
481
799
  #
482
800
  # @param item [Object]
483
801
  # Item that will be checked for membership
802
+ # @param nulls_equal [Boolean]
803
+ # If true, treat null as a distinct value. Null values will not propagate.
484
804
  #
485
805
  # @return [Expr]
486
806
  #
@@ -501,9 +821,9 @@ module Polars
501
821
  # # │ ["x", "y"] ┆ false │
502
822
  # # │ ["a", "c"] ┆ true │
503
823
  # # └───────────────┴──────────┘
504
- def contains(item)
824
+ def contains(item, nulls_equal: true)
505
825
  item = Utils.parse_into_expression(item, str_as_lit: true)
506
- Utils.wrap_expr(_rbexpr.arr_contains(item))
826
+ Utils.wrap_expr(_rbexpr.arr_contains(item, nulls_equal))
507
827
  end
508
828
 
509
829
  # Count how often the value produced by `element` occurs.
@@ -533,5 +853,163 @@ module Polars
533
853
  element = Utils.parse_into_expression(element, str_as_lit: true)
534
854
  Utils.wrap_expr(_rbexpr.arr_count_matches(element))
535
855
  end
856
+
857
+ # Convert the Series of type `Array` to a Series of type `Struct`.
858
+ #
859
+ # @param fields [Object]
860
+ # If the name and number of the desired fields is known in advance
861
+ # a list of field names can be given, which will be assigned by index.
862
+ # Otherwise, to dynamically assign field names, a custom function can be
863
+ # used; if neither are set, fields will be `field_0, field_1 .. field_n`.
864
+ #
865
+ # @return [Expr]
866
+ #
867
+ # @example Convert array to struct with default field name assignment:
868
+ # s1 = Polars::Series.new("n", [[0, 1, 2], [3, 4, 5]], dtype: Polars::Array.new(Polars::Int8, 3))
869
+ # s2 = s1.arr.to_struct
870
+ # s2.struct.fields
871
+ # # => ["field_0", "field_1", "field_2"]
872
+ #
873
+ # @example Convert array to struct with field name assignment by function/index:
874
+ # s3 = s1.arr.to_struct(fields: ->(idx) { "n%02d" % idx })
875
+ # s3.struct.fields
876
+ # # => ["n00", "n01", "n02"]
877
+ #
878
+ # @example Convert array to struct with field name assignment by index from a list of names:
879
+ # s1.arr.to_struct(fields: ["one", "two", "three"]).struct.unnest
880
+ # # =>
881
+ # # shape: (2, 3)
882
+ # # ┌─────┬─────┬───────┐
883
+ # # │ one ┆ two ┆ three │
884
+ # # │ --- ┆ --- ┆ --- │
885
+ # # │ i8 ┆ i8 ┆ i8 │
886
+ # # ╞═════╪═════╪═══════╡
887
+ # # │ 0 ┆ 1 ┆ 2 │
888
+ # # │ 3 ┆ 4 ┆ 5 │
889
+ # # └─────┴─────┴───────┘
890
+ def to_struct(fields: nil)
891
+ if fields.is_a?(Enumerable)
892
+ field_names = fields.to_a
893
+ rbexpr = _rbexpr.arr_to_struct(nil)
894
+ Utils.wrap_expr(rbexpr).struct.rename_fields(field_names)
895
+ else
896
+ rbexpr = _rbexpr.arr_to_struct(fields)
897
+ Utils.wrap_expr(rbexpr)
898
+ end
899
+ end
900
+
901
+ # Shift array values by the given number of indices.
902
+ #
903
+ # @param n [Integer]
904
+ # Number of indices to shift forward. If a negative value is passed, values
905
+ # are shifted in the opposite direction instead.
906
+ #
907
+ # @return [Expr]
908
+ #
909
+ # @note
910
+ # This method is similar to the `LAG` operation in SQL when the value for `n`
911
+ # is positive. With a negative value for `n`, it is similar to `LEAD`.
912
+ #
913
+ # @example By default, array values are shifted forward by one index.
914
+ # df = Polars::DataFrame.new(
915
+ # {"a" => [[1, 2, 3], [4, 5, 6]]}, schema: {"a" => Polars::Array.new(Polars::Int64, 3)}
916
+ # )
917
+ # df.with_columns(shift: Polars.col("a").arr.shift)
918
+ # # =>
919
+ # # shape: (2, 2)
920
+ # # ┌───────────────┬───────────────┐
921
+ # # │ a ┆ shift │
922
+ # # │ --- ┆ --- │
923
+ # # │ array[i64, 3] ┆ array[i64, 3] │
924
+ # # ╞═══════════════╪═══════════════╡
925
+ # # │ [1, 2, 3] ┆ [null, 1, 2] │
926
+ # # │ [4, 5, 6] ┆ [null, 4, 5] │
927
+ # # └───────────────┴───────────────┘
928
+ #
929
+ # @example Pass a negative value to shift in the opposite direction instead.
930
+ # df.with_columns(shift: Polars.col("a").arr.shift(-2))
931
+ # # =>
932
+ # # shape: (2, 2)
933
+ # # ┌───────────────┬─────────────────┐
934
+ # # │ a ┆ shift │
935
+ # # │ --- ┆ --- │
936
+ # # │ array[i64, 3] ┆ array[i64, 3] │
937
+ # # ╞═══════════════╪═════════════════╡
938
+ # # │ [1, 2, 3] ┆ [3, null, null] │
939
+ # # │ [4, 5, 6] ┆ [6, null, null] │
940
+ # # └───────────────┴─────────────────┘
941
+ def shift(n = 1)
942
+ n = Utils.parse_into_expression(n)
943
+ Utils.wrap_expr(_rbexpr.arr_shift(n))
944
+ end
945
+
946
+ # Run any polars expression against the arrays' elements.
947
+ #
948
+ # @param expr [Expr]
949
+ # Expression to run. Note that you can select an element with `Polars.element`
950
+ # @param as_list [Boolean]
951
+ # Collect the resulting data as a list. This allows for expressions which
952
+ # output a variable amount of data.
953
+ #
954
+ # @return [Expr]
955
+ #
956
+ # @example
957
+ # df = Polars::DataFrame.new({"a" => [1, 8, 3], "b" => [4, 5, 2]})
958
+ # df.with_columns(rank: Polars.concat_arr("a", "b").arr.eval(Polars.element.rank))
959
+ # # =>
960
+ # # shape: (3, 3)
961
+ # # ┌─────┬─────┬───────────────┐
962
+ # # │ a ┆ b ┆ rank │
963
+ # # │ --- ┆ --- ┆ --- │
964
+ # # │ i64 ┆ i64 ┆ array[f64, 2] │
965
+ # # ╞═════╪═════╪═══════════════╡
966
+ # # │ 1 ┆ 4 ┆ [1.0, 2.0] │
967
+ # # │ 8 ┆ 5 ┆ [2.0, 1.0] │
968
+ # # │ 3 ┆ 2 ┆ [2.0, 1.0] │
969
+ # # └─────┴─────┴───────────────┘
970
+ def eval(expr, as_list: false)
971
+ Utils.wrap_expr(_rbexpr.arr_eval(expr._rbexpr, as_list))
972
+ end
973
+
974
+ # Run any polars aggregation expression against the arrays' elements.
975
+ #
976
+ # @param expr [Expr]
977
+ # Expression to run. Note that you can select an element with `Polars.element`.
978
+ #
979
+ # @return [Expr]
980
+ #
981
+ # @example
982
+ # df = Polars::Series.new(
983
+ # "a", [[1, nil], [42, 13], [nil, nil]], dtype: Polars::Array.new(Polars::Int64, 2)
984
+ # ).to_frame
985
+ # df.with_columns(null_count: Polars.col("a").arr.agg(Polars.element.null_count))
986
+ # # =>
987
+ # # shape: (3, 2)
988
+ # # ┌───────────────┬────────────┐
989
+ # # │ a ┆ null_count │
990
+ # # │ --- ┆ --- │
991
+ # # │ array[i64, 2] ┆ u32 │
992
+ # # ╞═══════════════╪════════════╡
993
+ # # │ [1, null] ┆ 1 │
994
+ # # │ [42, 13] ┆ 0 │
995
+ # # │ [null, null] ┆ 2 │
996
+ # # └───────────────┴────────────┘
997
+ #
998
+ # @example
999
+ # df.with_columns(no_nulls: Polars.col("a").arr.agg(Polars.element.drop_nulls))
1000
+ # # =>
1001
+ # # shape: (3, 2)
1002
+ # # ┌───────────────┬───────────┐
1003
+ # # │ a ┆ no_nulls │
1004
+ # # │ --- ┆ --- │
1005
+ # # │ array[i64, 2] ┆ list[i64] │
1006
+ # # ╞═══════════════╪═══════════╡
1007
+ # # │ [1, null] ┆ [1] │
1008
+ # # │ [42, 13] ┆ [42, 13] │
1009
+ # # │ [null, null] ┆ [] │
1010
+ # # └───────────────┴───────────┘
1011
+ def agg(expr)
1012
+ Utils.wrap_expr(_rbexpr.arr_agg(expr._rbexpr))
1013
+ end
536
1014
  end
537
1015
  end