polars-df 0.15.0 → 0.26.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +274 -0
  3. data/Cargo.lock +1465 -867
  4. data/Cargo.toml +3 -0
  5. data/LICENSE.txt +2 -2
  6. data/README.md +87 -37
  7. data/ext/polars/Cargo.toml +47 -16
  8. data/ext/polars/src/c_api/allocator.rs +7 -0
  9. data/ext/polars/src/c_api/mod.rs +1 -0
  10. data/ext/polars/src/catalog/mod.rs +1 -0
  11. data/ext/polars/src/catalog/unity.rs +470 -0
  12. data/ext/polars/src/conversion/any_value.rs +99 -84
  13. data/ext/polars/src/conversion/categorical.rs +30 -0
  14. data/ext/polars/src/conversion/chunked_array.rs +71 -62
  15. data/ext/polars/src/conversion/datetime.rs +63 -0
  16. data/ext/polars/src/conversion/mod.rs +796 -312
  17. data/ext/polars/src/dataframe/construction.rs +6 -18
  18. data/ext/polars/src/dataframe/export.rs +30 -39
  19. data/ext/polars/src/dataframe/general.rs +294 -362
  20. data/ext/polars/src/dataframe/io.rs +33 -150
  21. data/ext/polars/src/dataframe/map.rs +175 -0
  22. data/ext/polars/src/dataframe/mod.rs +37 -5
  23. data/ext/polars/src/dataframe/serde.rs +23 -8
  24. data/ext/polars/src/error.rs +44 -7
  25. data/ext/polars/src/exceptions.rs +21 -8
  26. data/ext/polars/src/expr/array.rs +86 -22
  27. data/ext/polars/src/expr/binary.rs +50 -1
  28. data/ext/polars/src/expr/bitwise.rs +39 -0
  29. data/ext/polars/src/expr/categorical.rs +20 -0
  30. data/ext/polars/src/expr/datatype.rs +51 -0
  31. data/ext/polars/src/expr/datetime.rs +99 -41
  32. data/ext/polars/src/expr/extension.rs +12 -0
  33. data/ext/polars/src/expr/general.rs +252 -128
  34. data/ext/polars/src/expr/list.rs +56 -60
  35. data/ext/polars/src/expr/meta.rs +30 -35
  36. data/ext/polars/src/expr/mod.rs +28 -6
  37. data/ext/polars/src/expr/name.rs +29 -14
  38. data/ext/polars/src/expr/rolling.rs +111 -3
  39. data/ext/polars/src/expr/selector.rs +219 -0
  40. data/ext/polars/src/expr/serde.rs +28 -0
  41. data/ext/polars/src/expr/string.rs +118 -20
  42. data/ext/polars/src/expr/struct.rs +14 -1
  43. data/ext/polars/src/file.rs +194 -86
  44. data/ext/polars/src/functions/aggregation.rs +13 -12
  45. data/ext/polars/src/functions/business.rs +2 -3
  46. data/ext/polars/src/functions/eager.rs +3 -2
  47. data/ext/polars/src/functions/io.rs +90 -18
  48. data/ext/polars/src/functions/lazy.rs +267 -118
  49. data/ext/polars/src/functions/meta.rs +8 -7
  50. data/ext/polars/src/functions/misc.rs +1 -1
  51. data/ext/polars/src/functions/mod.rs +2 -1
  52. data/ext/polars/src/functions/range.rs +88 -31
  53. data/ext/polars/src/functions/strings.rs +6 -0
  54. data/ext/polars/src/functions/utils.rs +8 -0
  55. data/ext/polars/src/interop/arrow/mod.rs +52 -1
  56. data/ext/polars/src/interop/arrow/{to_ruby.rs → to_rb.rs} +37 -7
  57. data/ext/polars/src/interop/arrow/to_rust.rs +43 -0
  58. data/ext/polars/src/interop/numo/to_numo_df.rs +1 -1
  59. data/ext/polars/src/interop/numo/to_numo_series.rs +72 -50
  60. data/ext/polars/src/io/cloud_options.rs +107 -0
  61. data/ext/polars/src/io/mod.rs +4 -0
  62. data/ext/polars/src/io/scan_options.rs +113 -0
  63. data/ext/polars/src/io/sink_options.rs +46 -0
  64. data/ext/polars/src/io/sink_output.rs +21 -0
  65. data/ext/polars/src/lazyframe/exitable.rs +39 -0
  66. data/ext/polars/src/lazyframe/general.rs +846 -368
  67. data/ext/polars/src/lazyframe/mod.rs +58 -5
  68. data/ext/polars/src/lazyframe/optflags.rs +59 -0
  69. data/ext/polars/src/lazyframe/serde.rs +36 -4
  70. data/ext/polars/src/lazyframe/sink.rs +46 -0
  71. data/ext/polars/src/lazygroupby.rs +38 -9
  72. data/ext/polars/src/lib.rs +574 -165
  73. data/ext/polars/src/map/lazy.rs +44 -74
  74. data/ext/polars/src/map/mod.rs +18 -254
  75. data/ext/polars/src/map/series.rs +241 -1087
  76. data/ext/polars/src/on_startup.rs +192 -9
  77. data/ext/polars/src/prelude.rs +1 -0
  78. data/ext/polars/src/rb_modules.rs +10 -57
  79. data/ext/polars/src/ruby/exceptions.rs +26 -0
  80. data/ext/polars/src/ruby/gvl.rs +104 -0
  81. data/ext/polars/src/ruby/lazy.rs +46 -0
  82. data/ext/polars/src/ruby/mod.rs +11 -0
  83. data/ext/polars/src/ruby/numo.rs +52 -0
  84. data/ext/polars/src/ruby/plan_callback.rs +198 -0
  85. data/ext/polars/src/ruby/rb_modules.rs +16 -0
  86. data/ext/polars/src/ruby/ruby_convert_registry.rs +51 -0
  87. data/ext/polars/src/ruby/ruby_function.rs +11 -0
  88. data/ext/polars/src/ruby/ruby_udf.rs +164 -0
  89. data/ext/polars/src/ruby/thread.rs +65 -0
  90. data/ext/polars/src/ruby/utils.rs +39 -0
  91. data/ext/polars/src/series/aggregation.rs +116 -91
  92. data/ext/polars/src/series/arithmetic.rs +16 -22
  93. data/ext/polars/src/series/comparison.rs +101 -222
  94. data/ext/polars/src/series/construction.rs +80 -70
  95. data/ext/polars/src/series/export.rs +98 -56
  96. data/ext/polars/src/series/general.rs +323 -440
  97. data/ext/polars/src/series/import.rs +22 -5
  98. data/ext/polars/src/series/map.rs +103 -0
  99. data/ext/polars/src/series/mod.rs +57 -15
  100. data/ext/polars/src/series/scatter.rs +139 -82
  101. data/ext/polars/src/sql.rs +16 -9
  102. data/ext/polars/src/testing/frame.rs +31 -0
  103. data/ext/polars/src/testing/mod.rs +5 -0
  104. data/ext/polars/src/testing/series.rs +31 -0
  105. data/ext/polars/src/timeout.rs +105 -0
  106. data/ext/polars/src/utils.rs +105 -4
  107. data/lib/polars/array_expr.rb +500 -22
  108. data/lib/polars/array_name_space.rb +384 -10
  109. data/lib/polars/batched_csv_reader.rb +48 -66
  110. data/lib/polars/binary_expr.rb +217 -0
  111. data/lib/polars/binary_name_space.rb +155 -1
  112. data/lib/polars/cat_expr.rb +224 -0
  113. data/lib/polars/cat_name_space.rb +132 -32
  114. data/lib/polars/catalog/unity/catalog_info.rb +20 -0
  115. data/lib/polars/catalog/unity/column_info.rb +31 -0
  116. data/lib/polars/catalog/unity/namespace_info.rb +21 -0
  117. data/lib/polars/catalog/unity/table_info.rb +50 -0
  118. data/lib/polars/catalog.rb +448 -0
  119. data/lib/polars/collect_batches.rb +22 -0
  120. data/lib/polars/config.rb +3 -3
  121. data/lib/polars/convert.rb +201 -36
  122. data/lib/polars/data_frame.rb +2851 -1017
  123. data/lib/polars/data_frame_plot.rb +173 -0
  124. data/lib/polars/data_type_expr.rb +52 -0
  125. data/lib/polars/data_type_group.rb +6 -0
  126. data/lib/polars/data_types.rb +118 -18
  127. data/lib/polars/date_time_expr.rb +426 -84
  128. data/lib/polars/date_time_name_space.rb +384 -111
  129. data/lib/polars/dynamic_group_by.rb +102 -10
  130. data/lib/polars/exceptions.rb +50 -5
  131. data/lib/polars/expr.rb +2159 -915
  132. data/lib/polars/extension_expr.rb +39 -0
  133. data/lib/polars/extension_name_space.rb +39 -0
  134. data/lib/polars/functions/aggregation/horizontal.rb +11 -6
  135. data/lib/polars/functions/aggregation/vertical.rb +2 -3
  136. data/lib/polars/functions/as_datatype.rb +290 -8
  137. data/lib/polars/functions/business.rb +95 -0
  138. data/lib/polars/functions/col.rb +6 -5
  139. data/lib/polars/functions/datatype.rb +62 -0
  140. data/lib/polars/functions/eager.rb +426 -24
  141. data/lib/polars/functions/escape_regex.rb +21 -0
  142. data/lib/polars/functions/lazy.rb +813 -195
  143. data/lib/polars/functions/lit.rb +21 -10
  144. data/lib/polars/functions/range/int_range.rb +74 -2
  145. data/lib/polars/functions/range/linear_space.rb +195 -0
  146. data/lib/polars/functions/range/time_range.rb +1 -1
  147. data/lib/polars/functions/repeat.rb +7 -12
  148. data/lib/polars/functions/whenthen.rb +2 -2
  149. data/lib/polars/group_by.rb +188 -58
  150. data/lib/polars/iceberg_dataset.rb +108 -0
  151. data/lib/polars/in_process_query.rb +37 -0
  152. data/lib/polars/io/cloud.rb +18 -0
  153. data/lib/polars/io/csv.rb +336 -128
  154. data/lib/polars/io/database.rb +19 -4
  155. data/lib/polars/io/delta.rb +134 -0
  156. data/lib/polars/io/iceberg.rb +34 -0
  157. data/lib/polars/io/ipc.rb +63 -63
  158. data/lib/polars/io/json.rb +16 -0
  159. data/lib/polars/io/lines.rb +172 -0
  160. data/lib/polars/io/ndjson.rb +176 -20
  161. data/lib/polars/io/parquet.rb +173 -95
  162. data/lib/polars/io/scan_options.rb +55 -0
  163. data/lib/polars/io/sink_options.rb +27 -0
  164. data/lib/polars/io/utils.rb +17 -0
  165. data/lib/polars/lazy_frame.rb +3017 -622
  166. data/lib/polars/lazy_group_by.rb +436 -2
  167. data/lib/polars/list_expr.rb +551 -59
  168. data/lib/polars/list_name_space.rb +465 -51
  169. data/lib/polars/meta_expr.rb +146 -24
  170. data/lib/polars/name_expr.rb +87 -2
  171. data/lib/polars/query_opt_flags.rb +264 -0
  172. data/lib/polars/rolling_group_by.rb +90 -5
  173. data/lib/polars/scan_cast_options.rb +86 -0
  174. data/lib/polars/schema.rb +128 -0
  175. data/lib/polars/selector.rb +245 -0
  176. data/lib/polars/selectors.rb +1048 -201
  177. data/lib/polars/series.rb +2522 -774
  178. data/lib/polars/series_plot.rb +72 -0
  179. data/lib/polars/slice.rb +1 -1
  180. data/lib/polars/sql_context.rb +13 -6
  181. data/lib/polars/string_cache.rb +19 -72
  182. data/lib/polars/string_expr.rb +561 -107
  183. data/lib/polars/string_name_space.rb +781 -109
  184. data/lib/polars/struct_expr.rb +139 -18
  185. data/lib/polars/struct_name_space.rb +19 -1
  186. data/lib/polars/testing.rb +24 -273
  187. data/lib/polars/utils/constants.rb +2 -0
  188. data/lib/polars/utils/construction/data_frame.rb +410 -0
  189. data/lib/polars/utils/construction/series.rb +350 -0
  190. data/lib/polars/utils/construction/utils.rb +9 -0
  191. data/lib/polars/utils/convert.rb +18 -8
  192. data/lib/polars/utils/deprecation.rb +11 -0
  193. data/lib/polars/utils/parse.rb +62 -9
  194. data/lib/polars/utils/reduce_balanced.rb +43 -0
  195. data/lib/polars/utils/serde.rb +22 -0
  196. data/lib/polars/utils/unstable.rb +19 -0
  197. data/lib/polars/utils/various.rb +86 -1
  198. data/lib/polars/utils.rb +63 -48
  199. data/lib/polars/version.rb +1 -1
  200. data/lib/polars.rb +85 -2
  201. metadata +80 -28
  202. data/ext/polars/src/allocator.rs +0 -13
  203. data/ext/polars/src/batched_csv.rs +0 -138
  204. data/ext/polars/src/functions/string_cache.rs +0 -25
  205. data/ext/polars/src/map/dataframe.rs +0 -338
  206. data/lib/polars/plot.rb +0 -109
@@ -0,0 +1,72 @@
1
+ module Polars
2
+ # Series.plot namespace.
3
+ class SeriesPlot
4
+ # @private
5
+ def initialize(s)
6
+ require "vega"
7
+
8
+ name = s.name || "value"
9
+ @df = s.to_frame(name)
10
+ @series_name = name
11
+ end
12
+
13
+ # Draw histogram.
14
+ #
15
+ # @return [Vega::LiteChart]
16
+ def hist
17
+ encoding = {
18
+ x: {field: @series_name, bin: true},
19
+ y: {aggregate: "count"}
20
+ }
21
+
22
+ Vega.lite
23
+ .data(@df.rows(named: true))
24
+ .mark(type: "bar", tooltip: true)
25
+ .encoding(encoding)
26
+ .config(axis: {labelFontSize: 12})
27
+ end
28
+
29
+ # Draw kernel density estimate plot.
30
+ #
31
+ # @return [Vega::LiteChart]
32
+ def kde
33
+ if @series_name == "density"
34
+ msg = "cannot use `plot.kde` when Series name is `'density'`"
35
+ raise ArgumentError, msg
36
+ end
37
+
38
+ encoding = {
39
+ x: {field: @series_name, type: "quantitative"},
40
+ y: {field: "density", type: "quantitative"}
41
+ }
42
+
43
+ Vega.lite
44
+ .data(@df.rows(named: true))
45
+ .transform(density: @series_name, as: [@series_name, "density"])
46
+ .mark(type: "area", tooltip: true)
47
+ .encoding(encoding)
48
+ .config(axis: {labelFontSize: 12})
49
+ end
50
+
51
+ # Draw line plot.
52
+ #
53
+ # @return [Vega::LiteChart]
54
+ def line
55
+ if @series_name == "index"
56
+ msg = "cannot call `plot.line` when Series name is 'index'"
57
+ raise ArgumentError, msg
58
+ end
59
+
60
+ encoding = {
61
+ x: {field: "index", type: "quantitative"},
62
+ y: {field: @series_name, type: "quantitative"}
63
+ }
64
+
65
+ Vega.lite
66
+ .data(@df.with_row_index.rows(named: true))
67
+ .mark(type: "line", tooltip: true)
68
+ .encoding(encoding)
69
+ .config(axis: {labelFontSize: 12})
70
+ end
71
+ end
72
+ end
data/lib/polars/slice.rb CHANGED
@@ -12,7 +12,7 @@ module Polars
12
12
 
13
13
  # check for fast-paths / single-operation calls
14
14
  if @slice_length == 0
15
- @obj.cleared
15
+ @obj.clear
16
16
  elsif @is_unbounded && [-1, 1].include?(@stride)
17
17
  @stride < 0 ? @obj.reverse : @obj.clone
18
18
  elsif @start >= 0 && @stop >= 0 && @stride == 1
@@ -5,11 +5,17 @@ module Polars
5
5
  attr_accessor :_ctxt, :_eager_execution
6
6
 
7
7
  # Initialize a new `SQLContext`.
8
- def initialize(frames = nil, eager_execution: false, **named_frames)
8
+ def initialize(frames = nil, register_globals: false, eager: false, **named_frames)
9
+ Utils.issue_unstable_warning(
10
+ "`SQLContext` is considered **unstable**, although it is close to being considered stable."
11
+ )
9
12
  self._ctxt = RbSQLContext.new
10
- self._eager_execution = eager_execution
13
+ self._eager_execution = eager
11
14
 
12
15
  frames = (frames || {}).to_h
16
+ if register_globals
17
+ raise Todo
18
+ end
13
19
 
14
20
  if frames.any? || named_frames.any?
15
21
  register_many(frames, **named_frames)
@@ -37,7 +43,8 @@ module Polars
37
43
  # ["Pulp Fiction", 1994, 8_000_000, 107_930_000, 8.9],
38
44
  # ["The Shawshank Redemption", 1994, 25_000_000, 28_341_469, 9.3],
39
45
  # ],
40
- # schema: ["title", "release_year", "budget", "gross", "imdb_score"]
46
+ # schema: ["title", "release_year", "budget", "gross", "imdb_score"],
47
+ # orient: "row"
41
48
  # )
42
49
  # ctx = Polars::SQLContext.new(films: df)
43
50
  # ctx.execute(
@@ -66,11 +73,11 @@ module Polars
66
73
  # ctx.execute(
67
74
  # "
68
75
  # SELECT
69
- # MAX(release_year / 10) * 10 AS decade,
76
+ # MAX(release_year // 10) * 10 AS decade,
70
77
  # SUM(gross) AS total_gross,
71
78
  # COUNT(title) AS n_films,
72
79
  # FROM films
73
- # GROUP BY (release_year / 10) -- decade
80
+ # GROUP BY (release_year // 10) -- decade
74
81
  # ORDER BY total_gross DESC
75
82
  # ",
76
83
  # eager: true
@@ -129,7 +136,7 @@ module Polars
129
136
  # Named eager/lazy frames, provided as kwargs.
130
137
  #
131
138
  # @return [SQLContext]
132
- def register_many(frames, **named_frames)
139
+ def register_many(frames = nil, **named_frames)
133
140
  frames = (frames || {}).to_h
134
141
  frames = frames.merge(named_frames)
135
142
  frames.each do |name, frame|
@@ -1,30 +1,11 @@
1
1
  module Polars
2
- # Class for enabling and disabling the global string cache.
2
+ # Does nothing.
3
3
  #
4
- # @example Construct two Series using the same global string cache.
5
- # s1 = nil
6
- # s2 = nil
7
- # Polars::StringCache.new do
8
- # s1 = Polars::Series.new("color", ["red", "green", "red"], dtype: Polars::Categorical)
9
- # s2 = Polars::Series.new("color", ["blue", "red", "green"], dtype: Polars::Categorical)
10
- # end
11
- #
12
- # @example As both Series are constructed under the same global string cache, they can be concatenated.
13
- # Polars.concat([s1, s2])
14
- # # =>
15
- # # shape: (6,)
16
- # # Series: 'color' [cat]
17
- # # [
18
- # # "red"
19
- # # "green"
20
- # # "red"
21
- # # "blue"
22
- # # "red"
23
- # # "green"
24
- # # ]
4
+ # @deprecated
5
+ # The string cache was used to maintain the mapping for the Categorical
6
+ # dtype, this is now done through `Polars::Categories`.
25
7
  class StringCache
26
8
  def initialize(&block)
27
- RbStringCacheHolder.hold(&block)
28
9
  end
29
10
  end
30
11
 
@@ -33,69 +14,35 @@ module Polars
33
14
  end
34
15
 
35
16
  module Functions
36
- # Enable the global string cache.
17
+ # Does nothing.
37
18
  #
38
- # `Categorical` columns created under the same global string cache have
39
- # the same underlying physical value when string values are equal. This allows the
40
- # columns to be concatenated or used in a join operation, for example.
19
+ # @deprecated
20
+ # The string cache was used to maintain the mapping for the Categorical
21
+ # dtype, this is now done through `Polars::Categories`.
41
22
  #
42
23
  # @return [nil]
43
- #
44
- # @example Construct two Series using the same global string cache.
45
- # Polars.enable_string_cache
46
- # s1 = Polars::Series.new("color", ["red", "green", "red"], dtype: Polars::Categorical)
47
- # s2 = Polars::Series.new("color", ["blue", "red", "green"], dtype: Polars::Categorical)
48
- # Polars.disable_string_cache
49
- #
50
- # @example As both Series are constructed under the same global string cache, they can be concatenated.
51
- # Polars.concat([s1, s2])
52
- # # =>
53
- # # shape: (6,)
54
- # # Series: 'color' [cat]
55
- # # [
56
- # # "red"
57
- # # "green"
58
- # # "red"
59
- # # "blue"
60
- # # "red"
61
- # # "green"
62
- # # ]
63
24
  def enable_string_cache
64
- Plr.enable_string_cache
65
25
  end
66
26
 
67
- # Disable and clear the global string cache.
27
+ # Does nothing.
68
28
  #
69
- # @return [nil]
29
+ # @deprecated
30
+ # The string cache was used to maintain the mapping for the Categorical
31
+ # dtype, this is now done through `Polars::Categories`.
70
32
  #
71
- # @example Construct two Series using the same global string cache.
72
- # Polars.enable_string_cache
73
- # s1 = Polars::Series.new("color", ["red", "green", "red"], dtype: Polars::Categorical)
74
- # s2 = Polars::Series.new("color", ["blue", "red", "green"], dtype: Polars::Categorical)
75
- # Polars.disable_string_cache
76
- #
77
- # @example As both Series are constructed under the same global string cache, they can be concatenated.
78
- # Polars.concat([s1, s2])
79
- # # =>
80
- # # shape: (6,)
81
- # # Series: 'color' [cat]
82
- # # [
83
- # # "red"
84
- # # "green"
85
- # # "red"
86
- # # "blue"
87
- # # "red"
88
- # # "green"
89
- # # ]
33
+ # @return [nil]
90
34
  def disable_string_cache
91
- Plr.disable_string_cache
92
35
  end
93
36
 
94
- # Check whether the global string cache is enabled.
37
+ # Always returns true.
38
+ #
39
+ # @deprecated
40
+ # The string cache was used to maintain the mapping for the Categorical
41
+ # dtype, this is now done through `Polars::Categories`.
95
42
  #
96
43
  # @return [Boolean]
97
44
  def using_string_cache
98
- Plr.using_string_cache
45
+ true
99
46
  end
100
47
  end
101
48
  end