polars-df 0.15.0 → 0.26.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +274 -0
  3. data/Cargo.lock +1465 -867
  4. data/Cargo.toml +3 -0
  5. data/LICENSE.txt +2 -2
  6. data/README.md +87 -37
  7. data/ext/polars/Cargo.toml +47 -16
  8. data/ext/polars/src/c_api/allocator.rs +7 -0
  9. data/ext/polars/src/c_api/mod.rs +1 -0
  10. data/ext/polars/src/catalog/mod.rs +1 -0
  11. data/ext/polars/src/catalog/unity.rs +470 -0
  12. data/ext/polars/src/conversion/any_value.rs +99 -84
  13. data/ext/polars/src/conversion/categorical.rs +30 -0
  14. data/ext/polars/src/conversion/chunked_array.rs +71 -62
  15. data/ext/polars/src/conversion/datetime.rs +63 -0
  16. data/ext/polars/src/conversion/mod.rs +796 -312
  17. data/ext/polars/src/dataframe/construction.rs +6 -18
  18. data/ext/polars/src/dataframe/export.rs +30 -39
  19. data/ext/polars/src/dataframe/general.rs +294 -362
  20. data/ext/polars/src/dataframe/io.rs +33 -150
  21. data/ext/polars/src/dataframe/map.rs +175 -0
  22. data/ext/polars/src/dataframe/mod.rs +37 -5
  23. data/ext/polars/src/dataframe/serde.rs +23 -8
  24. data/ext/polars/src/error.rs +44 -7
  25. data/ext/polars/src/exceptions.rs +21 -8
  26. data/ext/polars/src/expr/array.rs +86 -22
  27. data/ext/polars/src/expr/binary.rs +50 -1
  28. data/ext/polars/src/expr/bitwise.rs +39 -0
  29. data/ext/polars/src/expr/categorical.rs +20 -0
  30. data/ext/polars/src/expr/datatype.rs +51 -0
  31. data/ext/polars/src/expr/datetime.rs +99 -41
  32. data/ext/polars/src/expr/extension.rs +12 -0
  33. data/ext/polars/src/expr/general.rs +252 -128
  34. data/ext/polars/src/expr/list.rs +56 -60
  35. data/ext/polars/src/expr/meta.rs +30 -35
  36. data/ext/polars/src/expr/mod.rs +28 -6
  37. data/ext/polars/src/expr/name.rs +29 -14
  38. data/ext/polars/src/expr/rolling.rs +111 -3
  39. data/ext/polars/src/expr/selector.rs +219 -0
  40. data/ext/polars/src/expr/serde.rs +28 -0
  41. data/ext/polars/src/expr/string.rs +118 -20
  42. data/ext/polars/src/expr/struct.rs +14 -1
  43. data/ext/polars/src/file.rs +194 -86
  44. data/ext/polars/src/functions/aggregation.rs +13 -12
  45. data/ext/polars/src/functions/business.rs +2 -3
  46. data/ext/polars/src/functions/eager.rs +3 -2
  47. data/ext/polars/src/functions/io.rs +90 -18
  48. data/ext/polars/src/functions/lazy.rs +267 -118
  49. data/ext/polars/src/functions/meta.rs +8 -7
  50. data/ext/polars/src/functions/misc.rs +1 -1
  51. data/ext/polars/src/functions/mod.rs +2 -1
  52. data/ext/polars/src/functions/range.rs +88 -31
  53. data/ext/polars/src/functions/strings.rs +6 -0
  54. data/ext/polars/src/functions/utils.rs +8 -0
  55. data/ext/polars/src/interop/arrow/mod.rs +52 -1
  56. data/ext/polars/src/interop/arrow/{to_ruby.rs → to_rb.rs} +37 -7
  57. data/ext/polars/src/interop/arrow/to_rust.rs +43 -0
  58. data/ext/polars/src/interop/numo/to_numo_df.rs +1 -1
  59. data/ext/polars/src/interop/numo/to_numo_series.rs +72 -50
  60. data/ext/polars/src/io/cloud_options.rs +107 -0
  61. data/ext/polars/src/io/mod.rs +4 -0
  62. data/ext/polars/src/io/scan_options.rs +113 -0
  63. data/ext/polars/src/io/sink_options.rs +46 -0
  64. data/ext/polars/src/io/sink_output.rs +21 -0
  65. data/ext/polars/src/lazyframe/exitable.rs +39 -0
  66. data/ext/polars/src/lazyframe/general.rs +846 -368
  67. data/ext/polars/src/lazyframe/mod.rs +58 -5
  68. data/ext/polars/src/lazyframe/optflags.rs +59 -0
  69. data/ext/polars/src/lazyframe/serde.rs +36 -4
  70. data/ext/polars/src/lazyframe/sink.rs +46 -0
  71. data/ext/polars/src/lazygroupby.rs +38 -9
  72. data/ext/polars/src/lib.rs +574 -165
  73. data/ext/polars/src/map/lazy.rs +44 -74
  74. data/ext/polars/src/map/mod.rs +18 -254
  75. data/ext/polars/src/map/series.rs +241 -1087
  76. data/ext/polars/src/on_startup.rs +192 -9
  77. data/ext/polars/src/prelude.rs +1 -0
  78. data/ext/polars/src/rb_modules.rs +10 -57
  79. data/ext/polars/src/ruby/exceptions.rs +26 -0
  80. data/ext/polars/src/ruby/gvl.rs +104 -0
  81. data/ext/polars/src/ruby/lazy.rs +46 -0
  82. data/ext/polars/src/ruby/mod.rs +11 -0
  83. data/ext/polars/src/ruby/numo.rs +52 -0
  84. data/ext/polars/src/ruby/plan_callback.rs +198 -0
  85. data/ext/polars/src/ruby/rb_modules.rs +16 -0
  86. data/ext/polars/src/ruby/ruby_convert_registry.rs +51 -0
  87. data/ext/polars/src/ruby/ruby_function.rs +11 -0
  88. data/ext/polars/src/ruby/ruby_udf.rs +164 -0
  89. data/ext/polars/src/ruby/thread.rs +65 -0
  90. data/ext/polars/src/ruby/utils.rs +39 -0
  91. data/ext/polars/src/series/aggregation.rs +116 -91
  92. data/ext/polars/src/series/arithmetic.rs +16 -22
  93. data/ext/polars/src/series/comparison.rs +101 -222
  94. data/ext/polars/src/series/construction.rs +80 -70
  95. data/ext/polars/src/series/export.rs +98 -56
  96. data/ext/polars/src/series/general.rs +323 -440
  97. data/ext/polars/src/series/import.rs +22 -5
  98. data/ext/polars/src/series/map.rs +103 -0
  99. data/ext/polars/src/series/mod.rs +57 -15
  100. data/ext/polars/src/series/scatter.rs +139 -82
  101. data/ext/polars/src/sql.rs +16 -9
  102. data/ext/polars/src/testing/frame.rs +31 -0
  103. data/ext/polars/src/testing/mod.rs +5 -0
  104. data/ext/polars/src/testing/series.rs +31 -0
  105. data/ext/polars/src/timeout.rs +105 -0
  106. data/ext/polars/src/utils.rs +105 -4
  107. data/lib/polars/array_expr.rb +500 -22
  108. data/lib/polars/array_name_space.rb +384 -10
  109. data/lib/polars/batched_csv_reader.rb +48 -66
  110. data/lib/polars/binary_expr.rb +217 -0
  111. data/lib/polars/binary_name_space.rb +155 -1
  112. data/lib/polars/cat_expr.rb +224 -0
  113. data/lib/polars/cat_name_space.rb +132 -32
  114. data/lib/polars/catalog/unity/catalog_info.rb +20 -0
  115. data/lib/polars/catalog/unity/column_info.rb +31 -0
  116. data/lib/polars/catalog/unity/namespace_info.rb +21 -0
  117. data/lib/polars/catalog/unity/table_info.rb +50 -0
  118. data/lib/polars/catalog.rb +448 -0
  119. data/lib/polars/collect_batches.rb +22 -0
  120. data/lib/polars/config.rb +3 -3
  121. data/lib/polars/convert.rb +201 -36
  122. data/lib/polars/data_frame.rb +2851 -1017
  123. data/lib/polars/data_frame_plot.rb +173 -0
  124. data/lib/polars/data_type_expr.rb +52 -0
  125. data/lib/polars/data_type_group.rb +6 -0
  126. data/lib/polars/data_types.rb +118 -18
  127. data/lib/polars/date_time_expr.rb +426 -84
  128. data/lib/polars/date_time_name_space.rb +384 -111
  129. data/lib/polars/dynamic_group_by.rb +102 -10
  130. data/lib/polars/exceptions.rb +50 -5
  131. data/lib/polars/expr.rb +2159 -915
  132. data/lib/polars/extension_expr.rb +39 -0
  133. data/lib/polars/extension_name_space.rb +39 -0
  134. data/lib/polars/functions/aggregation/horizontal.rb +11 -6
  135. data/lib/polars/functions/aggregation/vertical.rb +2 -3
  136. data/lib/polars/functions/as_datatype.rb +290 -8
  137. data/lib/polars/functions/business.rb +95 -0
  138. data/lib/polars/functions/col.rb +6 -5
  139. data/lib/polars/functions/datatype.rb +62 -0
  140. data/lib/polars/functions/eager.rb +426 -24
  141. data/lib/polars/functions/escape_regex.rb +21 -0
  142. data/lib/polars/functions/lazy.rb +813 -195
  143. data/lib/polars/functions/lit.rb +21 -10
  144. data/lib/polars/functions/range/int_range.rb +74 -2
  145. data/lib/polars/functions/range/linear_space.rb +195 -0
  146. data/lib/polars/functions/range/time_range.rb +1 -1
  147. data/lib/polars/functions/repeat.rb +7 -12
  148. data/lib/polars/functions/whenthen.rb +2 -2
  149. data/lib/polars/group_by.rb +188 -58
  150. data/lib/polars/iceberg_dataset.rb +108 -0
  151. data/lib/polars/in_process_query.rb +37 -0
  152. data/lib/polars/io/cloud.rb +18 -0
  153. data/lib/polars/io/csv.rb +336 -128
  154. data/lib/polars/io/database.rb +19 -4
  155. data/lib/polars/io/delta.rb +134 -0
  156. data/lib/polars/io/iceberg.rb +34 -0
  157. data/lib/polars/io/ipc.rb +63 -63
  158. data/lib/polars/io/json.rb +16 -0
  159. data/lib/polars/io/lines.rb +172 -0
  160. data/lib/polars/io/ndjson.rb +176 -20
  161. data/lib/polars/io/parquet.rb +173 -95
  162. data/lib/polars/io/scan_options.rb +55 -0
  163. data/lib/polars/io/sink_options.rb +27 -0
  164. data/lib/polars/io/utils.rb +17 -0
  165. data/lib/polars/lazy_frame.rb +3017 -622
  166. data/lib/polars/lazy_group_by.rb +436 -2
  167. data/lib/polars/list_expr.rb +551 -59
  168. data/lib/polars/list_name_space.rb +465 -51
  169. data/lib/polars/meta_expr.rb +146 -24
  170. data/lib/polars/name_expr.rb +87 -2
  171. data/lib/polars/query_opt_flags.rb +264 -0
  172. data/lib/polars/rolling_group_by.rb +90 -5
  173. data/lib/polars/scan_cast_options.rb +86 -0
  174. data/lib/polars/schema.rb +128 -0
  175. data/lib/polars/selector.rb +245 -0
  176. data/lib/polars/selectors.rb +1048 -201
  177. data/lib/polars/series.rb +2522 -774
  178. data/lib/polars/series_plot.rb +72 -0
  179. data/lib/polars/slice.rb +1 -1
  180. data/lib/polars/sql_context.rb +13 -6
  181. data/lib/polars/string_cache.rb +19 -72
  182. data/lib/polars/string_expr.rb +561 -107
  183. data/lib/polars/string_name_space.rb +781 -109
  184. data/lib/polars/struct_expr.rb +139 -18
  185. data/lib/polars/struct_name_space.rb +19 -1
  186. data/lib/polars/testing.rb +24 -273
  187. data/lib/polars/utils/constants.rb +2 -0
  188. data/lib/polars/utils/construction/data_frame.rb +410 -0
  189. data/lib/polars/utils/construction/series.rb +350 -0
  190. data/lib/polars/utils/construction/utils.rb +9 -0
  191. data/lib/polars/utils/convert.rb +18 -8
  192. data/lib/polars/utils/deprecation.rb +11 -0
  193. data/lib/polars/utils/parse.rb +62 -9
  194. data/lib/polars/utils/reduce_balanced.rb +43 -0
  195. data/lib/polars/utils/serde.rb +22 -0
  196. data/lib/polars/utils/unstable.rb +19 -0
  197. data/lib/polars/utils/various.rb +86 -1
  198. data/lib/polars/utils.rb +63 -48
  199. data/lib/polars/version.rb +1 -1
  200. data/lib/polars.rb +85 -2
  201. metadata +80 -28
  202. data/ext/polars/src/allocator.rs +0 -13
  203. data/ext/polars/src/batched_csv.rs +0 -138
  204. data/ext/polars/src/functions/string_cache.rs +0 -25
  205. data/ext/polars/src/map/dataframe.rs +0 -338
  206. data/lib/polars/plot.rb +0 -109
@@ -0,0 +1,62 @@
1
+ module Polars
2
+ module Functions
3
+ # Get a lazily evaluated :class:`DataType` of a column or expression.
4
+ #
5
+ # @note
6
+ # This functionality is considered **unstable**. It may be changed
7
+ # at any point without it being considered a breaking change.
8
+ #
9
+ # @return [DataTypeExpr]
10
+ def dtype_of(col_or_expr)
11
+ e = nil
12
+ if col_or_expr.is_a?(::String)
13
+ e = F.col(col_or_expr)
14
+ else
15
+ e = col_or_expr
16
+ end
17
+
18
+ DataTypeExpr._from_rbdatatype_expr(RbDataTypeExpr.of_expr(e._rbexpr))
19
+ end
20
+
21
+ # Get the dtype of `self` in `map_elements` and `map_batches`.
22
+ #
23
+ # @note
24
+ # This functionality is considered **unstable**. It may be changed
25
+ # at any point without it being considered a breaking change.
26
+ #
27
+ # @return [DataTypeExpr]
28
+ def self_dtype
29
+ DataTypeExpr._from_rbdatatype_expr(RbDataTypeExpr.self_dtype)
30
+ end
31
+
32
+ # Create a new datatype expression that represents a Struct datatype.
33
+ #
34
+ # @note
35
+ # This functionality is considered **unstable**. It may be changed
36
+ # at any point without it being considered a breaking change.
37
+ #
38
+ # @return [DataTypeExpr]
39
+ def struct_with_fields(
40
+ mapping
41
+ )
42
+ preprocess = lambda do |dtype_expr|
43
+ if dtype_expr.is_a?(DataType)
44
+ dtype_expr.to_dtype_expr._rbdatatype_expr
45
+ elsif dtype_expr < DataType
46
+ dtype_expr.to_dtype_expr._rbdatatype_expr
47
+ elsif dtype_expr.is_a?(DataTypeExpr)
48
+ dtype_expr._rbdatatype_expr
49
+ else
50
+ msg = "mapping item must be a datatype or datatype expression; found #{dtype_expr.inspect}"
51
+ raise TypeError, msg
52
+ end
53
+ end
54
+
55
+ fields = mapping.map { |name, dtype_expr| [name.to_s, preprocess.(dtype_expr)] }
56
+
57
+ DataTypeExpr._from_rbdatatype_expr(
58
+ RbDataTypeExpr.struct_with_fields(fields)
59
+ )
60
+ end
61
+ end
62
+ end
@@ -6,8 +6,7 @@ module Polars
6
6
  # DataFrames/Series/LazyFrames to concatenate.
7
7
  # @param rechunk [Boolean]
8
8
  # Make sure that all data is in contiguous memory.
9
- # @param how ["vertical", "vertical_relaxed", "diagonal", "horizontal"]
10
- # LazyFrames do not support the `horizontal` strategy.
9
+ # @param how ["vertical", "vertical_relaxed", "diagonal", "diagonal_relaxed", "horizontal"]
11
10
  #
12
11
  # - Vertical: applies multiple `vstack` operations.
13
12
  # - Diagonal: finds a union between the column schemas and fills missing column values with null.
@@ -15,13 +14,15 @@ module Polars
15
14
  # @param parallel [Boolean]
16
15
  # Only relevant for LazyFrames. This determines if the concatenated
17
16
  # lazy computations may be executed in parallel.
17
+ # @param strict [Boolean]
18
+ # When how=`horizontal`, require all DataFrames to be the same height, raising an error if not.
18
19
  #
19
20
  # @return [Object]
20
21
  #
21
22
  # @example
22
23
  # df1 = Polars::DataFrame.new({"a" => [1], "b" => [3]})
23
24
  # df2 = Polars::DataFrame.new({"a" => [2], "b" => [4]})
24
- # Polars.concat([df1, df2])
25
+ # Polars.concat([df1, df2]) # default is 'vertical' strategy
25
26
  # # =>
26
27
  # # shape: (2, 2)
27
28
  # # ┌─────┬─────┐
@@ -32,38 +33,168 @@ module Polars
32
33
  # # │ 1 ┆ 3 │
33
34
  # # │ 2 ┆ 4 │
34
35
  # # └─────┴─────┘
35
- def concat(items, rechunk: true, how: "vertical", parallel: true)
36
- if items.empty?
36
+ #
37
+ # @example
38
+ # df1 = Polars::DataFrame.new({"a" => [1], "b" => [3]})
39
+ # df2 = Polars::DataFrame.new({"a" => [2.5], "b" => [4]})
40
+ # Polars.concat([df1, df2], how: "vertical_relaxed") # 'a' coerced into f64
41
+ # # =>
42
+ # # shape: (2, 2)
43
+ # # ┌─────┬─────┐
44
+ # # │ a ┆ b │
45
+ # # │ --- ┆ --- │
46
+ # # │ f64 ┆ i64 │
47
+ # # ╞═════╪═════╡
48
+ # # │ 1.0 ┆ 3 │
49
+ # # │ 2.5 ┆ 4 │
50
+ # # └─────┴─────┘
51
+ #
52
+ # @example
53
+ # df_h1 = Polars::DataFrame.new({"l1" => [1, 2], "l2" => [3, 4]})
54
+ # df_h2 = Polars::DataFrame.new({"r1" => [5, 6], "r2" => [7, 8], "r3" => [9, 10]})
55
+ # Polars.concat([df_h1, df_h2], how: "horizontal")
56
+ # # =>
57
+ # # shape: (2, 5)
58
+ # # ┌─────┬─────┬─────┬─────┬─────┐
59
+ # # │ l1 ┆ l2 ┆ r1 ┆ r2 ┆ r3 │
60
+ # # │ --- ┆ --- ┆ --- ┆ --- ┆ --- │
61
+ # # │ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 │
62
+ # # ╞═════╪═════╪═════╪═════╪═════╡
63
+ # # │ 1 ┆ 3 ┆ 5 ┆ 7 ┆ 9 │
64
+ # # │ 2 ┆ 4 ┆ 6 ┆ 8 ┆ 10 │
65
+ # # └─────┴─────┴─────┴─────┴─────┘
66
+ #
67
+ # @example
68
+ # df_d1 = Polars::DataFrame.new({"a" => [1], "b" => [3]})
69
+ # df_d2 = Polars::DataFrame.new({"a" => [2], "c" => [4]})
70
+ # Polars.concat([df_d1, df_d2], how: "diagonal")
71
+ # # =>
72
+ # # shape: (2, 3)
73
+ # # ┌─────┬──────┬──────┐
74
+ # # │ a ┆ b ┆ c │
75
+ # # │ --- ┆ --- ┆ --- │
76
+ # # │ i64 ┆ i64 ┆ i64 │
77
+ # # ╞═════╪══════╪══════╡
78
+ # # │ 1 ┆ 3 ┆ null │
79
+ # # │ 2 ┆ null ┆ 4 │
80
+ # # └─────┴──────┴──────┘
81
+ #
82
+ # @example
83
+ # df_a1 = Polars::DataFrame.new({"id" => [1, 2], "x" => [3, 4]})
84
+ # df_a2 = Polars::DataFrame.new({"id" => [2, 3], "y" => [5, 6]})
85
+ # df_a3 = Polars::DataFrame.new({"id" => [1, 3], "z" => [7, 8]})
86
+ # Polars.concat([df_a1, df_a2, df_a3], how: "align")
87
+ # # =>
88
+ # # shape: (3, 4)
89
+ # # ┌─────┬──────┬──────┬──────┐
90
+ # # │ id ┆ x ┆ y ┆ z │
91
+ # # │ --- ┆ --- ┆ --- ┆ --- │
92
+ # # │ i64 ┆ i64 ┆ i64 ┆ i64 │
93
+ # # ╞═════╪══════╪══════╪══════╡
94
+ # # │ 1 ┆ 3 ┆ null ┆ 7 │
95
+ # # │ 2 ┆ 4 ┆ 5 ┆ null │
96
+ # # │ 3 ┆ null ┆ 6 ┆ 8 │
97
+ # # └─────┴──────┴──────┴──────┘
98
+ def concat(items, rechunk: false, how: "vertical", parallel: true, strict: false)
99
+ elems = items.to_a
100
+
101
+ if elems.empty?
37
102
  raise ArgumentError, "cannot concat empty list"
38
103
  end
39
104
 
40
- first = items[0]
105
+ if how == "align"
106
+ if !elems[0].is_a?(DataFrame) && !elems[0].is_a?(LazyFrame)
107
+ msg = "'align' strategy is not supported for #{elems[0].class.name}"
108
+ raise TypeError, msg
109
+ end
110
+
111
+ # establish common columns, maintaining the order in which they appear
112
+ all_columns = elems.flat_map { |e| e.collect_schema.names }
113
+ key = all_columns.uniq.map.with_index.to_h
114
+ common_cols = elems.map { |e| e.collect_schema.names }
115
+ .reduce { |x, y| Set.new(x) & Set.new(y) }
116
+ .sort_by { |k| key[k] }
117
+ # we require at least one key column for 'align'
118
+ if common_cols.empty?
119
+ msg = "'align' strategy requires at least one common column"
120
+ raise InvalidOperationError, msg
121
+ end
122
+
123
+ # align the frame data using a full outer join with no suffix-resolution
124
+ # (so we raise an error in case of column collision, like "horizontal")
125
+ lf = elems.map { |df| df.lazy }.reduce do |x, y|
126
+ x.join(
127
+ y,
128
+ how: "full",
129
+ on: common_cols,
130
+ suffix: "_PL_CONCAT_RIGHT",
131
+ maintain_order: "right_left"
132
+ )
133
+ # Coalesce full outer join columns
134
+ .with_columns(
135
+ common_cols.map { |name| F.coalesce([name, "#{name}_PL_CONCAT_RIGHT"]) }
136
+ )
137
+ .drop(common_cols.map { |name| "#{name}_PL_CONCAT_RIGHT" })
138
+ end.sort(common_cols)
139
+
140
+ eager = elems[0].is_a?(DataFrame)
141
+ return eager ? lf.collect : lf
142
+ end
143
+
144
+ first = elems[0]
145
+
41
146
  if first.is_a?(DataFrame)
42
147
  if how == "vertical"
43
- out = Utils.wrap_df(Plr.concat_df(items))
148
+ out = Utils.wrap_df(Plr.concat_df(elems))
149
+ elsif how == "vertical_relaxed"
150
+ out = Utils.wrap_ldf(
151
+ Plr.concat_lf(
152
+ elems.map { |df| df.lazy },
153
+ rechunk,
154
+ parallel,
155
+ true
156
+ )
157
+ ).collect(optimizations: QueryOptFlags._eager)
44
158
  elsif how == "diagonal"
45
- out = Utils.wrap_df(Plr.concat_df_diagonal(items))
159
+ out = Utils.wrap_df(Plr.concat_df_diagonal(elems))
160
+ elsif how == "diagonal_relaxed"
161
+ out = Utils.wrap_ldf(
162
+ Plr.concat_lf_diagonal(
163
+ elems.map { |df| df.lazy },
164
+ rechunk,
165
+ parallel,
166
+ true
167
+ )
168
+ ).collect(optimizations: QueryOptFlags._eager)
46
169
  elsif how == "horizontal"
47
- out = Utils.wrap_df(Plr.concat_df_horizontal(items))
170
+ out = Utils.wrap_df(Plr.concat_df_horizontal(elems, strict))
48
171
  else
49
- raise ArgumentError, "how must be one of {{'vertical', 'diagonal', 'horizontal'}}, got #{how}"
172
+ raise ArgumentError, "how must be one of {{'vertical', 'vertical_relaxed', 'diagonal', 'diagonal_relaxed', 'horizontal'}}, got #{how}"
50
173
  end
51
174
  elsif first.is_a?(LazyFrame)
52
175
  if how == "vertical"
53
- return Utils.wrap_ldf(Plr.concat_lf(items, rechunk, parallel, false))
176
+ return Utils.wrap_ldf(Plr.concat_lf(elems, rechunk, parallel, false))
54
177
  elsif how == "vertical_relaxed"
55
- return Utils.wrap_ldf(Plr.concat_lf(items, rechunk, parallel, true))
178
+ return Utils.wrap_ldf(Plr.concat_lf(elems, rechunk, parallel, true))
56
179
  elsif how == "diagonal"
57
- return Utils.wrap_ldf(Plr.concat_lf_diagonal(items, rechunk, parallel, false))
180
+ return Utils.wrap_ldf(Plr.concat_lf_diagonal(elems, rechunk, parallel, false))
181
+ elsif how == "diagonal_relaxed"
182
+ return Utils.wrap_ldf(Plr.concat_lf_diagonal(elems, rechunk, parallel, true))
183
+ elsif how == "horizontal"
184
+ return Utils.wrap_ldf(Plr.concat_lf_horizontal(elems, parallel, strict))
58
185
  else
59
- raise ArgumentError, "Lazy only allows 'vertical', 'vertical_relaxed', and 'diagonal' concat strategy."
186
+ raise ArgumentError, "Lazy only allows 'vertical', 'vertical_relaxed', 'diagonal', and 'diagonal_relaxed' concat strategy."
60
187
  end
61
188
  elsif first.is_a?(Series)
62
- # TODO
63
- out = Utils.wrap_s(Plr.concat_series(items))
189
+ if how == "vertical"
190
+ out = Utils.wrap_s(Plr.concat_series(elems))
191
+ else
192
+ msg = "Series only supports 'vertical' concat strategy"
193
+ raise ArgumentError, msg
194
+ end
64
195
  elsif first.is_a?(Expr)
65
196
  out = first
66
- items[1..-1].each do |e|
197
+ elems[1..-1].each do |e|
67
198
  out = out.append(e)
68
199
  end
69
200
  else
@@ -77,7 +208,277 @@ module Polars
77
208
  end
78
209
  end
79
210
 
80
- # Align a sequence of frames using the uique values from one or more columns as a key.
211
+ # Combine multiple DataFrames, LazyFrames, or Series into a single object.
212
+ #
213
+ # @note
214
+ # This function does not guarantee any specific ordering of rows in the result.
215
+ # If you need predictable row ordering, use `Polars.concat` instead.
216
+ #
217
+ # @param items [Array]
218
+ # DataFrames, LazyFrames, or Series to concatenate.
219
+ # @param how ['vertical', 'vertical_relaxed', 'diagonal', 'diagonal_relaxed', 'horizontal', 'align', 'align_full', 'align_inner', 'align_left', 'align_right']
220
+ # Note that `Series` only support the `vertical` strategy.
221
+ #
222
+ # * vertical: Applies multiple `vstack` operations.
223
+ # * vertical_relaxed: Same as `vertical`, but additionally coerces columns to
224
+ # their common supertype *if* they are mismatched (eg: Int32 → Int64).
225
+ # * diagonal: Finds a union between the column schemas and fills missing column
226
+ # values with `null`.
227
+ # * diagonal_relaxed: Same as `diagonal`, but additionally coerces columns to
228
+ # their common supertype *if* they are mismatched (eg: Int32 → Int64).
229
+ # * horizontal: Stacks Series from DataFrames horizontally and fills with `null`
230
+ # if the lengths don't match.
231
+ # * align, align_full, align_left, align_right: Combines frames horizontally,
232
+ # auto-determining the common key columns and aligning rows using the same
233
+ # logic as `align_frames` (note that "align" is an alias for "align_full").
234
+ # The "align" strategy determines the type of join used to align the frames,
235
+ # equivalent to the "how" parameter on `align_frames`. Note that the common
236
+ # join columns are automatically coalesced, but other column collisions
237
+ # will raise an error (if you need more control over this you should use
238
+ # a suitable `join` method directly).
239
+ # @param strict [Boolean]
240
+ # When how=`horizontal`, require all DataFrames to be the same height, raising an error if not.
241
+ #
242
+ # @return [Object]
243
+ #
244
+ # @example
245
+ # df1 = Polars::DataFrame.new({"a" => [1], "b" => [3]})
246
+ # df2 = Polars::DataFrame.new({"a" => [2], "b" => [4]})
247
+ # Polars.union([df1, df2])
248
+ # # =>
249
+ # # shape: (2, 2)
250
+ # # ┌─────┬─────┐
251
+ # # │ a ┆ b │
252
+ # # │ --- ┆ --- │
253
+ # # │ i64 ┆ i64 │
254
+ # # ╞═════╪═════╡
255
+ # # │ 1 ┆ 3 │
256
+ # # │ 2 ┆ 4 │
257
+ # # └─────┴─────┘
258
+ #
259
+ # @example
260
+ # df1 = Polars::DataFrame.new({"a" => [1], "b" => [3]})
261
+ # df2 = Polars::DataFrame.new({"a" => [2.5], "b" => [4]})
262
+ # Polars.union([df1, df2], how: "vertical_relaxed")
263
+ # # =>
264
+ # # shape: (2, 2)
265
+ # # ┌─────┬─────┐
266
+ # # │ a ┆ b │
267
+ # # │ --- ┆ --- │
268
+ # # │ f64 ┆ i64 │
269
+ # # ╞═════╪═════╡
270
+ # # │ 1.0 ┆ 3 │
271
+ # # │ 2.5 ┆ 4 │
272
+ # # └─────┴─────┘
273
+ #
274
+ # @example
275
+ # df_h1 = Polars::DataFrame.new({"l1" => [1, 2], "l2" => [3, 4]})
276
+ # df_h2 = Polars::DataFrame.new({"r1" => [5, 6], "r2" => [7, 8], "r3" => [9, 10]})
277
+ # Polars.union([df_h1, df_h2], how: "horizontal")
278
+ # # =>
279
+ # # shape: (2, 5)
280
+ # # ┌─────┬─────┬─────┬─────┬─────┐
281
+ # # │ l1 ┆ l2 ┆ r1 ┆ r2 ┆ r3 │
282
+ # # │ --- ┆ --- ┆ --- ┆ --- ┆ --- │
283
+ # # │ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 │
284
+ # # ╞═════╪═════╪═════╪═════╪═════╡
285
+ # # │ 1 ┆ 3 ┆ 5 ┆ 7 ┆ 9 │
286
+ # # │ 2 ┆ 4 ┆ 6 ┆ 8 ┆ 10 │
287
+ # # └─────┴─────┴─────┴─────┴─────┘
288
+ #
289
+ # @example The "diagonal" strategy allows for some frames to have missing columns, the values for which are filled with `null`:
290
+ # df_d1 = Polars::DataFrame.new({"a" => [1], "b" => [3]})
291
+ # df_d2 = Polars::DataFrame.new({"a" => [2], "c" => [4]})
292
+ # Polars.union([df_d1, df_d2], how: "diagonal")
293
+ # # =>
294
+ # # shape: (2, 3)
295
+ # # ┌─────┬──────┬──────┐
296
+ # # │ a ┆ b ┆ c │
297
+ # # │ --- ┆ --- ┆ --- │
298
+ # # │ i64 ┆ i64 ┆ i64 │
299
+ # # ╞═════╪══════╪══════╡
300
+ # # │ 1 ┆ 3 ┆ null │
301
+ # # │ 2 ┆ null ┆ 4 │
302
+ # # └─────┴──────┴──────┘
303
+ def union(
304
+ items,
305
+ how: "vertical",
306
+ strict: false
307
+ )
308
+ elems = items.to_a
309
+
310
+ if elems.empty?
311
+ msg = "cannot concat empty list"
312
+ raise ArgumentError, msg
313
+ elsif elems.length == 1 && (elems[0].is_a?(DataFrame) || elems[0].is_a?(Series) || elems[0].is_a?(LazyFrame))
314
+ return elems[0]
315
+ end
316
+
317
+ if how.start_with?("align")
318
+ raise Todo
319
+ end
320
+
321
+ out = nil
322
+ first = elems[0]
323
+
324
+ if first.is_a?(DataFrame)
325
+ if ["vertical", "vertical_relaxed"].include?(how)
326
+ out = Utils.wrap_ldf(
327
+ Plr.concat_lf(
328
+ elems.map { |df| df.lazy },
329
+ false,
330
+ true,
331
+ how.end_with?("relaxed")
332
+ )
333
+ ).collect(optimizations: QueryOptFlags._eager)
334
+ elsif ["diagonal", "diagonal_relaxed"].include?(how)
335
+ out = Utils.wrap_ldf(
336
+ Plr.concat_lf_diagonal(
337
+ elems.map { |df| df.lazy },
338
+ false,
339
+ true,
340
+ how.end_with?("relaxed")
341
+ )
342
+ ).collect(optimizations: QueryOptFlags._eager)
343
+ elsif how == "horizontal"
344
+ out = Utils.wrap_df(Plr.concat_df_horizontal(elems, strict))
345
+ else
346
+ raise Todo
347
+ msg = "DataFrame `how` must be one of {{#{allowed}}}, got #{how.inspect}"
348
+ raise ArgumentError, msg
349
+ end
350
+
351
+ elsif first.is_a?(LazyFrame)
352
+ if ["vertical", "vertical_relaxed"].include?(how)
353
+ return Utils.wrap_ldf(
354
+ Plr.concat_lf(
355
+ elems,
356
+ false,
357
+ true,
358
+ how.end_with?("relaxed")
359
+ )
360
+ )
361
+ elsif ["diagonal", "diagonal_relaxed"].include?(how)
362
+ return Utils.wrap_ldf(
363
+ Plr.concat_lf_diagonal(
364
+ elems,
365
+ false,
366
+ true,
367
+ how.end_with?("relaxed")
368
+ )
369
+ )
370
+ elsif how == "horizontal"
371
+ return Utils.wrap_ldf(
372
+ Plr.concat_lf_horizontal(
373
+ elems,
374
+ true,
375
+ strict
376
+ )
377
+ )
378
+ else
379
+ raise Todo
380
+ msg = "LazyFrame `how` must be one of {{#{allowed}}}, got #{how.inspect}"
381
+ raise ArgumentError, msg
382
+ end
383
+
384
+ elsif first.is_a?(Series)
385
+ if how == "vertical"
386
+ out = Utils.wrap_s(Plr.concat_series(elems))
387
+ else
388
+ msg = "Series only supports 'vertical' concat strategy"
389
+ raise ArgumentError, msg
390
+ end
391
+
392
+ elsif first.is_a?(Expr)
393
+ return Utils.wrap_expr(Plr.concat_expr(elems.map { |e| e._rbexpr }, false))
394
+ else
395
+ msg = "did not expect type: #{first.class.name.inspect} in `concat`"
396
+ raise TypeError, msg
397
+ end
398
+
399
+ out
400
+ end
401
+
402
+ # Merge multiple sorted DataFrames or LazyFrames by the sorted key.
403
+ #
404
+ # The output of this operation will also be sorted.
405
+ # It is the callers responsibility that the frames
406
+ # are sorted in ascending order by that key otherwise
407
+ # the output will not make sense.
408
+ #
409
+ # @note
410
+ # This functionality is considered **unstable**. It may be changed
411
+ # at any point without it being considered a breaking change.
412
+ #
413
+ # @param items [Array]
414
+ # DataFrames or LazyFrames to merge.
415
+ # @param key [String]
416
+ # Key that is sorted.
417
+ # @param maintain_order [Boolean]
418
+ # If `true`, the output is guaranteed to have left-biased ordering
419
+ # for equal keys: rows from the left frame appear before rows from
420
+ # the right frame when their keys are equal.
421
+ #
422
+ # @return [Object]
423
+ #
424
+ # @example
425
+ # df0 = Polars::DataFrame.new(
426
+ # {"name" => ["steve", "elise", "bob"], "age" => [42, 44, 18]}
427
+ # ).sort("age")
428
+ # df1 = Polars::DataFrame.new(
429
+ # {"name" => ["anna", "megan", "steve", "thomas"], "age" => [21, 33, 17, 20]}
430
+ # ).sort("age")
431
+ # df2 = Polars::DataFrame.new({"name" => ["ida", "maya"], "age" => [37, 27]}).sort("age")
432
+ # Polars.merge_sorted([df0, df1, df2], "age")
433
+ # # =>
434
+ # # shape: (9, 2)
435
+ # # ┌────────┬─────┐
436
+ # # │ name ┆ age │
437
+ # # │ --- ┆ --- │
438
+ # # │ str ┆ i64 │
439
+ # # ╞════════╪═════╡
440
+ # # │ steve ┆ 17 │
441
+ # # │ bob ┆ 18 │
442
+ # # │ thomas ┆ 20 │
443
+ # # │ anna ┆ 21 │
444
+ # # │ maya ┆ 27 │
445
+ # # │ megan ┆ 33 │
446
+ # # │ ida ┆ 37 │
447
+ # # │ steve ┆ 42 │
448
+ # # │ elise ┆ 44 │
449
+ # # └────────┴─────┘
450
+ def merge_sorted(
451
+ items,
452
+ key,
453
+ maintain_order: false
454
+ )
455
+ elems = items.to_a
456
+
457
+ if elems.empty?
458
+ msg = "cannot merge_sort empty list"
459
+ raise ArgumentError, msg
460
+ end
461
+ if elems.length == 1 && (elems[0].is_a?(DataFrame) || elems[0].is_a?(LazyFrame))
462
+ return elems[0]
463
+ end
464
+
465
+ if !Utils.is_non_empty_sequence_of(elems, DataFrame) && !Utils.is_non_empty_sequence_of(elems, LazyFrame)
466
+ msg = "merge_sorted is not supported for #{elems[0].inspect}"
467
+ raise TypeError, msg
468
+ end
469
+
470
+ frames = elems.map { |df| df.lazy }
471
+
472
+ reduce_fn = lambda do |x, y|
473
+ x.merge_sorted(y, key, maintain_order: maintain_order)
474
+ end
475
+
476
+ lf = Utils.reduce_balanced(reduce_fn, frames)
477
+ eager = elems[0].is_a?(DataFrame)
478
+ eager ? lf.collect : lf
479
+ end
480
+
481
+ # Align an array of frames using the unique values from one or more columns as a key.
81
482
  #
82
483
  # Frames that do not contain the given key values have rows injected (with nulls
83
484
  # filling the non-key columns), and each resulting frame is sorted by the key.
@@ -90,13 +491,13 @@ module Polars
90
491
  # the same number of rows.
91
492
  #
92
493
  # @param frames [Array]
93
- # Sequence of DataFrames or LazyFrames.
494
+ # Array of DataFrames or LazyFrames.
94
495
  # @param on [Object]
95
496
  # One or more columns whose unique values will be used to align the frames.
96
497
  # @param select [Object]
97
498
  # Optional post-alignment column select to constrain and/or order
98
499
  # the columns returned from the newly aligned frames.
99
- # @param reverse [Object]
500
+ # @param descending [Object]
100
501
  # Sort the alignment column values in descending order; can be a single
101
502
  # boolean or a list of booleans associated with each column in `on`.
102
503
  #
@@ -125,7 +526,7 @@ module Polars
125
526
  # }
126
527
  # )
127
528
  # af1, af2, af3 = Polars.align_frames(
128
- # df1, df2, df3, on: "dt", select: ["x", "y"]
529
+ # df1, df2, df3, on: "dt", how: "left", select: ["x", "y"]
129
530
  # )
130
531
  # (af1 * af2 * af3).fill_null(0).select(Polars.sum_horizontal("*").alias("dot"))
131
532
  # # =>
@@ -142,8 +543,9 @@ module Polars
142
543
  def align_frames(
143
544
  *frames,
144
545
  on:,
546
+ how: "full",
145
547
  select: nil,
146
- reverse: false
548
+ descending: false
147
549
  )
148
550
  if frames.empty?
149
551
  return []
@@ -156,7 +558,7 @@ module Polars
156
558
  alignment_frame = (
157
559
  concat(frames.map { |df| df.lazy.select(on) })
158
560
  .unique(maintain_order: false)
159
- .sort(on, reverse: reverse)
561
+ .sort(on, descending: descending)
160
562
  )
161
563
  alignment_frame = (
162
564
  eager ? alignment_frame.collect.lazy : alignment_frame.cache
@@ -167,7 +569,7 @@ module Polars
167
569
  alignment_frame.join(
168
570
  df.lazy,
169
571
  on: alignment_frame.columns,
170
- how: "left"
572
+ how: how
171
573
  ).select(df.columns)
172
574
  end
173
575
  if !select.nil?
@@ -0,0 +1,21 @@
1
+ module Polars
2
+ module Functions
3
+ # Escapes string regex meta characters.
4
+ #
5
+ # @param s [String]
6
+ # The string whose meta characters will be escaped.
7
+ #
8
+ # @return [String]
9
+ def escape_regex(s)
10
+ if s.is_a?(Expr)
11
+ msg = "escape_regex function is unsupported for `Expr`, you may want use `Expr.str.escape_regex` instead"
12
+ raise TypeError, msg
13
+ elsif !s.is_a?(::String)
14
+ msg = "escape_regex function supports only `String` type, got `#{s.class.name}`"
15
+ raise TypeError, msg
16
+ end
17
+
18
+ Plr.escape_regex(s)
19
+ end
20
+ end
21
+ end