polars-df 0.13.0 → 0.15.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +30 -0
  3. data/Cargo.lock +1368 -319
  4. data/LICENSE.txt +1 -0
  5. data/README.md +1 -2
  6. data/ext/polars/Cargo.toml +15 -6
  7. data/ext/polars/src/batched_csv.rs +10 -13
  8. data/ext/polars/src/conversion/any_value.rs +37 -21
  9. data/ext/polars/src/conversion/chunked_array.rs +3 -3
  10. data/ext/polars/src/conversion/mod.rs +159 -46
  11. data/ext/polars/src/dataframe/construction.rs +4 -7
  12. data/ext/polars/src/dataframe/export.rs +9 -2
  13. data/ext/polars/src/dataframe/general.rs +22 -16
  14. data/ext/polars/src/dataframe/io.rs +78 -174
  15. data/ext/polars/src/dataframe/mod.rs +1 -0
  16. data/ext/polars/src/dataframe/serde.rs +15 -0
  17. data/ext/polars/src/error.rs +31 -48
  18. data/ext/polars/src/exceptions.rs +24 -0
  19. data/ext/polars/src/expr/binary.rs +4 -42
  20. data/ext/polars/src/expr/datetime.rs +16 -7
  21. data/ext/polars/src/expr/general.rs +14 -23
  22. data/ext/polars/src/expr/list.rs +18 -11
  23. data/ext/polars/src/expr/name.rs +3 -2
  24. data/ext/polars/src/expr/rolling.rs +6 -7
  25. data/ext/polars/src/expr/string.rs +17 -37
  26. data/ext/polars/src/file.rs +59 -22
  27. data/ext/polars/src/functions/business.rs +15 -0
  28. data/ext/polars/src/functions/io.rs +6 -6
  29. data/ext/polars/src/functions/lazy.rs +17 -8
  30. data/ext/polars/src/functions/mod.rs +1 -0
  31. data/ext/polars/src/functions/range.rs +4 -2
  32. data/ext/polars/src/interop/arrow/mod.rs +1 -0
  33. data/ext/polars/src/interop/arrow/to_ruby.rs +83 -0
  34. data/ext/polars/src/interop/mod.rs +1 -0
  35. data/ext/polars/src/lazyframe/general.rs +877 -0
  36. data/ext/polars/src/lazyframe/mod.rs +3 -825
  37. data/ext/polars/src/lazyframe/serde.rs +31 -0
  38. data/ext/polars/src/lib.rs +44 -13
  39. data/ext/polars/src/map/dataframe.rs +46 -14
  40. data/ext/polars/src/map/lazy.rs +65 -4
  41. data/ext/polars/src/map/mod.rs +17 -16
  42. data/ext/polars/src/map/series.rs +106 -64
  43. data/ext/polars/src/on_startup.rs +2 -2
  44. data/ext/polars/src/series/aggregation.rs +1 -5
  45. data/ext/polars/src/series/arithmetic.rs +10 -10
  46. data/ext/polars/src/series/construction.rs +52 -25
  47. data/ext/polars/src/series/export.rs +1 -1
  48. data/ext/polars/src/series/general.rs +643 -0
  49. data/ext/polars/src/series/import.rs +55 -0
  50. data/ext/polars/src/series/mod.rs +11 -638
  51. data/ext/polars/src/series/scatter.rs +2 -2
  52. data/ext/polars/src/utils.rs +0 -20
  53. data/lib/polars/batched_csv_reader.rb +0 -2
  54. data/lib/polars/binary_expr.rb +133 -9
  55. data/lib/polars/binary_name_space.rb +101 -6
  56. data/lib/polars/config.rb +4 -0
  57. data/lib/polars/data_frame.rb +285 -62
  58. data/lib/polars/data_type_group.rb +28 -0
  59. data/lib/polars/data_types.rb +2 -0
  60. data/lib/polars/date_time_expr.rb +244 -0
  61. data/lib/polars/date_time_name_space.rb +87 -0
  62. data/lib/polars/expr.rb +109 -8
  63. data/lib/polars/functions/as_datatype.rb +51 -2
  64. data/lib/polars/functions/col.rb +1 -1
  65. data/lib/polars/functions/eager.rb +1 -3
  66. data/lib/polars/functions/lazy.rb +88 -10
  67. data/lib/polars/functions/range/time_range.rb +21 -21
  68. data/lib/polars/io/csv.rb +14 -16
  69. data/lib/polars/io/database.rb +2 -2
  70. data/lib/polars/io/ipc.rb +14 -12
  71. data/lib/polars/io/ndjson.rb +10 -0
  72. data/lib/polars/io/parquet.rb +168 -111
  73. data/lib/polars/lazy_frame.rb +649 -15
  74. data/lib/polars/list_name_space.rb +169 -0
  75. data/lib/polars/selectors.rb +1144 -0
  76. data/lib/polars/series.rb +470 -40
  77. data/lib/polars/string_cache.rb +27 -1
  78. data/lib/polars/string_expr.rb +0 -1
  79. data/lib/polars/string_name_space.rb +73 -3
  80. data/lib/polars/struct_name_space.rb +31 -7
  81. data/lib/polars/utils/various.rb +5 -1
  82. data/lib/polars/utils.rb +45 -10
  83. data/lib/polars/version.rb +1 -1
  84. data/lib/polars.rb +2 -1
  85. metadata +14 -4
  86. data/lib/polars/functions.rb +0 -57
@@ -1,13 +1,17 @@
1
- use magnus::{prelude::*, RString, Value};
1
+ use magnus::{prelude::*, Value};
2
2
  use polars::io::avro::AvroCompression;
3
3
  use polars::io::RowIndex;
4
4
  use polars::prelude::*;
5
- use std::io::{BufWriter, Cursor};
5
+ use polars_utils::mmap::ensure_not_mapped;
6
+ use std::io::BufWriter;
6
7
  use std::num::NonZeroUsize;
7
8
 
8
9
  use super::*;
9
10
  use crate::conversion::*;
10
- use crate::file::{get_either_file, get_file_like, get_mmap_bytes_reader, EitherRustRubyFile};
11
+ use crate::file::{
12
+ get_either_file, get_file_like, get_mmap_bytes_reader, get_mmap_bytes_reader_and_path,
13
+ read_if_bytesio, EitherRustRubyFile,
14
+ };
11
15
  use crate::{RbPolarsErr, RbResult};
12
16
 
13
17
  impl RbDataFrame {
@@ -29,8 +33,7 @@ impl RbDataFrame {
29
33
  let n_threads = Option::<usize>::try_convert(arguments[12])?;
30
34
  let path = Option::<String>::try_convert(arguments[13])?;
31
35
  let overwrite_dtype = Option::<Vec<(String, Wrap<DataType>)>>::try_convert(arguments[14])?;
32
- // TODO fix
33
- let overwrite_dtype_slice = Option::<Vec<Wrap<DataType>>>::None; // Option::<Vec<Wrap<DataType>>>::try_convert(arguments[15])?;
36
+ let overwrite_dtype_slice = Option::<Vec<Wrap<DataType>>>::try_convert(arguments[15])?;
34
37
  let low_memory = bool::try_convert(arguments[16])?;
35
38
  let comment_prefix = Option::<String>::try_convert(arguments[17])?;
36
39
  let quote_char = Option::<String>::try_convert(arguments[18])?;
@@ -39,18 +42,17 @@ impl RbDataFrame {
39
42
  let try_parse_dates = bool::try_convert(arguments[21])?;
40
43
  let skip_rows_after_header = usize::try_convert(arguments[22])?;
41
44
  let row_index = Option::<(String, IdxSize)>::try_convert(arguments[23])?;
42
- let sample_size = usize::try_convert(arguments[24])?;
43
- let eol_char = String::try_convert(arguments[25])?;
44
- let raise_if_empty = bool::try_convert(arguments[26])?;
45
- let truncate_ragged_lines = bool::try_convert(arguments[27])?;
46
- let decimal_comma = bool::try_convert(arguments[28])?;
47
- let schema = Option::<Wrap<Schema>>::try_convert(arguments[29])?;
45
+ let eol_char = String::try_convert(arguments[24])?;
46
+ let raise_if_empty = bool::try_convert(arguments[25])?;
47
+ let truncate_ragged_lines = bool::try_convert(arguments[26])?;
48
+ let decimal_comma = bool::try_convert(arguments[27])?;
49
+ let schema = Option::<Wrap<Schema>>::try_convert(arguments[28])?;
48
50
  // end arguments
49
51
 
50
52
  let null_values = null_values.map(|w| w.0);
51
53
  let eol_char = eol_char.as_bytes()[0];
52
54
  let row_index = row_index.map(|(name, offset)| RowIndex {
53
- name: Arc::from(name.as_str()),
55
+ name: name.into(),
54
56
  offset,
55
57
  });
56
58
  let quote_char = if let Some(s) = quote_char {
@@ -68,7 +70,7 @@ impl RbDataFrame {
68
70
  .iter()
69
71
  .map(|(name, dtype)| {
70
72
  let dtype = dtype.0.clone();
71
- Field::new(name, dtype)
73
+ Field::new((&**name).into(), dtype)
72
74
  })
73
75
  .collect::<Schema>()
74
76
  });
@@ -80,7 +82,8 @@ impl RbDataFrame {
80
82
  .collect::<Vec<_>>()
81
83
  });
82
84
 
83
- let mmap_bytes_r = get_mmap_bytes_reader(rb_f)?;
85
+ let rb_f = read_if_bytesio(rb_f);
86
+ let mmap_bytes_r = get_mmap_bytes_reader(&rb_f)?;
84
87
  let df = CsvReadOptions::default()
85
88
  .with_path(path)
86
89
  .with_infer_schema_length(infer_schema_length)
@@ -91,7 +94,7 @@ impl RbDataFrame {
91
94
  .with_projection(projection.map(Arc::new))
92
95
  .with_rechunk(rechunk)
93
96
  .with_chunk_size(chunk_size)
94
- .with_columns(columns.map(Arc::from))
97
+ .with_columns(columns.map(|x| x.into_iter().map(|x| x.into()).collect()))
95
98
  .with_n_threads(n_threads)
96
99
  .with_schema_overwrite(overwrite_dtype.map(Arc::new))
97
100
  .with_dtype_overwrite(overwrite_dtype_slice.map(Arc::new))
@@ -99,7 +102,6 @@ impl RbDataFrame {
99
102
  .with_low_memory(low_memory)
100
103
  .with_skip_rows_after_header(skip_rows_after_header)
101
104
  .with_row_index(row_index)
102
- .with_sample_size(sample_size)
103
105
  .with_raise_if_empty(raise_if_empty)
104
106
  .with_parse_options(
105
107
  CsvParseOptions::default()
@@ -120,59 +122,14 @@ impl RbDataFrame {
120
122
  Ok(df.into())
121
123
  }
122
124
 
123
- #[allow(clippy::too_many_arguments)]
124
- pub fn read_parquet(
125
- rb_f: Value,
126
- columns: Option<Vec<String>>,
127
- projection: Option<Vec<usize>>,
128
- n_rows: Option<usize>,
129
- parallel: Wrap<ParallelStrategy>,
130
- row_index: Option<(String, IdxSize)>,
131
- low_memory: bool,
132
- use_statistics: bool,
133
- rechunk: bool,
134
- ) -> RbResult<Self> {
135
- use EitherRustRubyFile::*;
136
-
137
- let row_index = row_index.map(|(name, offset)| RowIndex {
138
- name: Arc::from(name.as_str()),
139
- offset,
140
- });
141
- let result = match get_either_file(rb_f, false)? {
142
- Rb(f) => {
143
- let buf = f.as_buffer();
144
- ParquetReader::new(buf)
145
- .with_projection(projection)
146
- .with_columns(columns)
147
- .read_parallel(parallel.0)
148
- .with_slice(n_rows.map(|x| (0, x)))
149
- .with_row_index(row_index)
150
- .set_low_memory(low_memory)
151
- .use_statistics(use_statistics)
152
- .set_rechunk(rechunk)
153
- .finish()
154
- }
155
- Rust(f) => ParquetReader::new(f.into_inner())
156
- .with_projection(projection)
157
- .with_columns(columns)
158
- .read_parallel(parallel.0)
159
- .with_slice(n_rows.map(|x| (0, x)))
160
- .with_row_index(row_index)
161
- .use_statistics(use_statistics)
162
- .set_rechunk(rechunk)
163
- .finish(),
164
- };
165
- let df = result.map_err(RbPolarsErr::from)?;
166
- Ok(RbDataFrame::new(df))
167
- }
168
-
169
125
  pub fn read_json(
170
126
  rb_f: Value,
171
127
  infer_schema_length: Option<usize>,
172
128
  schema: Option<Wrap<Schema>>,
173
129
  schema_overrides: Option<Wrap<Schema>>,
174
130
  ) -> RbResult<Self> {
175
- let mmap_bytes_r = get_mmap_bytes_reader(rb_f)?;
131
+ let rb_f = read_if_bytesio(rb_f);
132
+ let mmap_bytes_r = get_mmap_bytes_reader(&rb_f)?;
176
133
 
177
134
  let mut builder = JsonReader::new(mmap_bytes_r)
178
135
  .with_json_format(JsonFormat::Json)
@@ -196,7 +153,8 @@ impl RbDataFrame {
196
153
  schema: Option<Wrap<Schema>>,
197
154
  schema_overrides: Option<Wrap<Schema>>,
198
155
  ) -> RbResult<Self> {
199
- let mmap_bytes_r = get_mmap_bytes_reader(rb_f)?;
156
+ let rb_f = read_if_bytesio(rb_f);
157
+ let mmap_bytes_r = get_mmap_bytes_reader(&rb_f)?;
200
158
 
201
159
  let mut builder = JsonReader::new(mmap_bytes_r)
202
160
  .with_json_format(JsonFormat::JsonLines)
@@ -212,7 +170,7 @@ impl RbDataFrame {
212
170
 
213
171
  let out = builder
214
172
  .finish()
215
- .map_err(|e| RbPolarsErr::other(format!("{e}")))?;
173
+ .map_err(|e| RbPolarsErr::Other(format!("{e}")))?;
216
174
  Ok(out.into())
217
175
  }
218
176
 
@@ -222,16 +180,16 @@ impl RbDataFrame {
222
180
  projection: Option<Vec<usize>>,
223
181
  n_rows: Option<usize>,
224
182
  row_index: Option<(String, IdxSize)>,
225
- _memory_map: bool,
183
+ memory_map: bool,
226
184
  ) -> RbResult<Self> {
227
185
  let row_index = row_index.map(|(name, offset)| RowIndex {
228
- name: Arc::from(name.as_str()),
186
+ name: name.into(),
229
187
  offset,
230
188
  });
231
- let mmap_bytes_r = get_mmap_bytes_reader(rb_f)?;
189
+ let rb_f = read_if_bytesio(rb_f);
190
+ let (mmap_bytes_r, mmap_path) = get_mmap_bytes_reader_and_path(&rb_f)?;
232
191
 
233
- // TODO fix
234
- let mmap_path = None;
192
+ let mmap_path = if memory_map { mmap_path } else { None };
235
193
  let df = IpcReader::new(mmap_bytes_r)
236
194
  .with_projection(projection)
237
195
  .with_columns(columns)
@@ -252,11 +210,11 @@ impl RbDataFrame {
252
210
  rechunk: bool,
253
211
  ) -> RbResult<Self> {
254
212
  let row_index = row_index.map(|(name, offset)| RowIndex {
255
- name: Arc::from(name.as_str()),
213
+ name: name.into(),
256
214
  offset,
257
215
  });
258
- // rb_f = read_if_bytesio(rb_f);
259
- let mmap_bytes_r = get_mmap_bytes_reader(rb_f)?;
216
+ let rb_f = read_if_bytesio(rb_f);
217
+ let mmap_bytes_r = get_mmap_bytes_reader(&rb_f)?;
260
218
  let df = IpcStreamReader::new(mmap_bytes_r)
261
219
  .with_projection(projection)
262
220
  .with_columns(columns)
@@ -302,41 +260,19 @@ impl RbDataFrame {
302
260
  ) -> RbResult<()> {
303
261
  let batch_size = batch_size.0;
304
262
  let null = null_value.unwrap_or_default();
305
-
306
- if let Ok(s) = String::try_convert(rb_f) {
307
- let f = std::fs::File::create(s).unwrap();
308
- // no need for a buffered writer, because the csv writer does internal buffering
309
- CsvWriter::new(f)
310
- .include_header(include_header)
311
- .with_separator(separator)
312
- .with_quote_char(quote_char)
313
- .with_batch_size(batch_size)
314
- .with_datetime_format(datetime_format)
315
- .with_date_format(date_format)
316
- .with_time_format(time_format)
317
- .with_float_precision(float_precision)
318
- .with_null_value(null)
319
- .finish(&mut self.df.borrow_mut())
320
- .map_err(RbPolarsErr::from)?;
321
- } else {
322
- let mut buf = Cursor::new(Vec::new());
323
- CsvWriter::new(&mut buf)
324
- .include_header(include_header)
325
- .with_separator(separator)
326
- .with_quote_char(quote_char)
327
- .with_batch_size(batch_size)
328
- .with_datetime_format(datetime_format)
329
- .with_date_format(date_format)
330
- .with_time_format(time_format)
331
- .with_float_precision(float_precision)
332
- .with_null_value(null)
333
- .finish(&mut self.df.borrow_mut())
334
- .map_err(RbPolarsErr::from)?;
335
- // TODO less copying
336
- let rb_str = RString::from_slice(&buf.into_inner());
337
- rb_f.funcall::<_, _, Value>("write", (rb_str,))?;
338
- }
339
-
263
+ let mut buf = get_file_like(rb_f, true)?;
264
+ CsvWriter::new(&mut buf)
265
+ .include_header(include_header)
266
+ .with_separator(separator)
267
+ .with_quote_char(quote_char)
268
+ .with_batch_size(batch_size)
269
+ .with_datetime_format(datetime_format)
270
+ .with_date_format(date_format)
271
+ .with_time_format(time_format)
272
+ .with_float_precision(float_precision)
273
+ .with_null_value(null)
274
+ .finish(&mut self.df.borrow_mut())
275
+ .map_err(RbPolarsErr::from)?;
340
276
  Ok(())
341
277
  }
342
278
 
@@ -351,26 +287,14 @@ impl RbDataFrame {
351
287
  ) -> RbResult<()> {
352
288
  let compression = parse_parquet_compression(&compression, compression_level)?;
353
289
 
354
- if let Ok(s) = String::try_convert(rb_f) {
355
- let f = std::fs::File::create(s).unwrap();
356
- ParquetWriter::new(f)
357
- .with_compression(compression)
358
- .with_statistics(statistics.0)
359
- .with_row_group_size(row_group_size)
360
- .with_data_page_size(data_page_size)
361
- .finish(&mut self.df.borrow_mut())
362
- .map_err(RbPolarsErr::from)?;
363
- } else {
364
- let buf = get_file_like(rb_f, true)?;
365
- ParquetWriter::new(buf)
366
- .with_compression(compression)
367
- .with_statistics(statistics.0)
368
- .with_row_group_size(row_group_size)
369
- .with_data_page_size(data_page_size)
370
- .finish(&mut self.df.borrow_mut())
371
- .map_err(RbPolarsErr::from)?;
372
- }
373
-
290
+ let buf = get_file_like(rb_f, true)?;
291
+ ParquetWriter::new(buf)
292
+ .with_compression(compression)
293
+ .with_statistics(statistics.0)
294
+ .with_row_group_size(row_group_size)
295
+ .with_data_page_size(data_page_size)
296
+ .finish(&mut self.df.borrow_mut())
297
+ .map_err(RbPolarsErr::from)?;
374
298
  Ok(())
375
299
  }
376
300
 
@@ -386,7 +310,7 @@ impl RbDataFrame {
386
310
  (false, _) => serde_json::to_writer(file, &*self.df.borrow())
387
311
  .map_err(|e| PolarsError::ComputeError(format!("{:?}", e).into())),
388
312
  };
389
- r.map_err(|e| RbPolarsErr::other(format!("{:?}", e)))?;
313
+ r.map_err(|e| RbPolarsErr::Other(format!("{:?}", e)))?;
390
314
  Ok(())
391
315
  }
392
316
 
@@ -397,7 +321,7 @@ impl RbDataFrame {
397
321
  .with_json_format(JsonFormat::JsonLines)
398
322
  .finish(&mut self.df.borrow_mut());
399
323
 
400
- r.map_err(|e| RbPolarsErr::other(format!("{:?}", e)))?;
324
+ r.map_err(|e| RbPolarsErr::Other(format!("{:?}", e)))?;
401
325
  Ok(())
402
326
  }
403
327
 
@@ -405,23 +329,18 @@ impl RbDataFrame {
405
329
  &self,
406
330
  rb_f: Value,
407
331
  compression: Wrap<Option<IpcCompression>>,
332
+ compat_level: RbCompatLevel,
408
333
  ) -> RbResult<()> {
409
- if let Ok(s) = String::try_convert(rb_f) {
410
- let f = std::fs::File::create(s).unwrap();
411
- IpcWriter::new(f)
412
- .with_compression(compression.0)
413
- .finish(&mut self.df.borrow_mut())
414
- .map_err(RbPolarsErr::from)?;
415
- } else {
416
- let mut buf = Cursor::new(Vec::new());
417
- IpcWriter::new(&mut buf)
418
- .with_compression(compression.0)
419
- .finish(&mut self.df.borrow_mut())
420
- .map_err(RbPolarsErr::from)?;
421
- // TODO less copying
422
- let rb_str = RString::from_slice(&buf.into_inner());
423
- rb_f.funcall::<_, _, Value>("write", (rb_str,))?;
334
+ let either = get_either_file(rb_f, true)?;
335
+ if let EitherRustRubyFile::Rust(ref f) = either {
336
+ ensure_not_mapped(f).map_err(RbPolarsErr::from)?;
424
337
  }
338
+ let mut buf = either.into_dyn();
339
+ IpcWriter::new(&mut buf)
340
+ .with_compression(compression.0)
341
+ .with_compat_level(compat_level.0)
342
+ .finish(&mut self.df.borrow_mut())
343
+ .map_err(RbPolarsErr::from)?;
425
344
  Ok(())
426
345
  }
427
346
 
@@ -429,21 +348,14 @@ impl RbDataFrame {
429
348
  &self,
430
349
  rb_f: Value,
431
350
  compression: Wrap<Option<IpcCompression>>,
351
+ compat_level: RbCompatLevel,
432
352
  ) -> RbResult<()> {
433
- if let Ok(s) = String::try_convert(rb_f) {
434
- let f = std::fs::File::create(s).unwrap();
435
- IpcStreamWriter::new(f)
436
- .with_compression(compression.0)
437
- .finish(&mut self.df.borrow_mut())
438
- .map_err(RbPolarsErr::from)?
439
- } else {
440
- let mut buf = get_file_like(rb_f, true)?;
441
-
442
- IpcStreamWriter::new(&mut buf)
443
- .with_compression(compression.0)
444
- .finish(&mut self.df.borrow_mut())
445
- .map_err(RbPolarsErr::from)?;
446
- }
353
+ let mut buf = get_file_like(rb_f, true)?;
354
+ IpcStreamWriter::new(&mut buf)
355
+ .with_compression(compression.0)
356
+ .with_compat_level(compat_level.0)
357
+ .finish(&mut self.df.borrow_mut())
358
+ .map_err(RbPolarsErr::from)?;
447
359
  Ok(())
448
360
  }
449
361
 
@@ -451,23 +363,15 @@ impl RbDataFrame {
451
363
  &self,
452
364
  rb_f: Value,
453
365
  compression: Wrap<Option<AvroCompression>>,
366
+ name: String,
454
367
  ) -> RbResult<()> {
455
368
  use polars::io::avro::AvroWriter;
456
-
457
- if let Ok(s) = String::try_convert(rb_f) {
458
- let f = std::fs::File::create(s).unwrap();
459
- AvroWriter::new(f)
460
- .with_compression(compression.0)
461
- .finish(&mut self.df.borrow_mut())
462
- .map_err(RbPolarsErr::from)?;
463
- } else {
464
- let mut buf = get_file_like(rb_f, true)?;
465
- AvroWriter::new(&mut buf)
466
- .with_compression(compression.0)
467
- .finish(&mut self.df.borrow_mut())
468
- .map_err(RbPolarsErr::from)?;
469
- }
470
-
369
+ let mut buf = get_file_like(rb_f, true)?;
370
+ AvroWriter::new(&mut buf)
371
+ .with_compression(compression.0)
372
+ .with_name(name)
373
+ .finish(&mut self.df.borrow_mut())
374
+ .map_err(RbPolarsErr::from)?;
471
375
  Ok(())
472
376
  }
473
377
  }
@@ -2,6 +2,7 @@ mod construction;
2
2
  mod export;
3
3
  mod general;
4
4
  mod io;
5
+ mod serde;
5
6
 
6
7
  use polars::prelude::*;
7
8
  use std::cell::RefCell;
@@ -0,0 +1,15 @@
1
+ use crate::exceptions::ComputeError;
2
+ use crate::file::get_file_like;
3
+ use crate::{RbDataFrame, RbResult};
4
+ use magnus::Value;
5
+ use std::io::BufWriter;
6
+
7
+ impl RbDataFrame {
8
+ // TODO add to Ruby
9
+ pub fn serialize_json(&self, rb_f: Value) -> RbResult<()> {
10
+ let file = get_file_like(rb_f, true)?;
11
+ let writer = BufWriter::new(file);
12
+ serde_json::to_writer(writer, &self.df)
13
+ .map_err(|err| ComputeError::new_err(err.to_string()))
14
+ }
15
+ }
@@ -1,67 +1,50 @@
1
- use magnus::exception;
1
+ use std::fmt::{Debug, Formatter};
2
+
2
3
  use magnus::Error;
3
4
  use polars::prelude::PolarsError;
4
5
 
6
+ use crate::exceptions::{ComputeError, InvalidOperationError};
5
7
  use crate::rb_modules;
6
8
 
7
- pub struct RbPolarsErr {}
8
-
9
- impl RbPolarsErr {
10
- // convert to Error instead of Self
11
- pub fn from(e: PolarsError) -> Error {
12
- match e {
13
- PolarsError::ComputeError(err) => ComputeError::new_err(err.to_string()),
14
- PolarsError::InvalidOperation(err) => InvalidOperationError::new_err(err.to_string()),
15
- _ => Error::new(rb_modules::error(), e.to_string()),
16
- }
17
- }
18
-
19
- pub fn io(e: std::io::Error) -> Error {
20
- Error::new(rb_modules::error(), e.to_string())
21
- }
22
-
23
- pub fn other(message: String) -> Error {
24
- Error::new(rb_modules::error(), message)
25
- }
26
- }
27
-
28
- pub struct RbTypeError {}
29
-
30
- impl RbTypeError {
31
- pub fn new_err(message: String) -> Error {
32
- Error::new(exception::type_error(), message)
33
- }
9
+ pub enum RbPolarsErr {
10
+ Polars(PolarsError),
11
+ Other(String),
34
12
  }
35
13
 
36
- pub struct RbValueError {}
37
-
38
- impl RbValueError {
39
- pub fn new_err(message: String) -> Error {
40
- Error::new(exception::arg_error(), message)
14
+ impl From<PolarsError> for RbPolarsErr {
15
+ fn from(err: PolarsError) -> Self {
16
+ RbPolarsErr::Polars(err)
41
17
  }
42
18
  }
43
19
 
44
- pub struct RbOverflowError {}
45
-
46
- impl RbOverflowError {
47
- pub fn new_err(message: String) -> Error {
48
- Error::new(exception::range_error(), message)
20
+ impl From<std::io::Error> for RbPolarsErr {
21
+ fn from(value: std::io::Error) -> Self {
22
+ RbPolarsErr::Other(format!("{value:?}"))
49
23
  }
50
24
  }
51
25
 
52
- pub struct ComputeError {}
53
-
54
- impl ComputeError {
55
- pub fn new_err(message: String) -> Error {
56
- Error::new(rb_modules::compute_error(), message)
26
+ impl From<RbPolarsErr> for Error {
27
+ fn from(err: RbPolarsErr) -> Self {
28
+ match err {
29
+ RbPolarsErr::Polars(err) => match err {
30
+ PolarsError::ComputeError(err) => ComputeError::new_err(err.to_string()),
31
+ PolarsError::InvalidOperation(err) => {
32
+ InvalidOperationError::new_err(err.to_string())
33
+ }
34
+ _ => Error::new(rb_modules::error(), err.to_string()),
35
+ },
36
+ RbPolarsErr::Other(err) => Error::new(rb_modules::error(), err.to_string()),
37
+ }
57
38
  }
58
39
  }
59
40
 
60
- pub struct InvalidOperationError {}
61
-
62
- impl InvalidOperationError {
63
- pub fn new_err(message: String) -> Error {
64
- Error::new(rb_modules::invalid_operation_error(), message)
41
+ impl Debug for RbPolarsErr {
42
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
43
+ use RbPolarsErr::*;
44
+ match self {
45
+ Polars(err) => write!(f, "{err:?}"),
46
+ Other(err) => write!(f, "BindingsError: {err:?}"),
47
+ }
65
48
  }
66
49
  }
67
50
 
@@ -0,0 +1,24 @@
1
+ use crate::rb_modules;
2
+ use magnus::{exception, Error};
3
+ use std::borrow::Cow;
4
+
5
+ macro_rules! create_exception {
6
+ ($type:ident, $cls:expr) => {
7
+ pub struct $type {}
8
+
9
+ impl $type {
10
+ pub fn new_err<T>(message: T) -> Error
11
+ where
12
+ T: Into<Cow<'static, str>>,
13
+ {
14
+ Error::new($cls, message)
15
+ }
16
+ }
17
+ };
18
+ }
19
+
20
+ create_exception!(RbTypeError, exception::type_error());
21
+ create_exception!(RbValueError, exception::arg_error());
22
+ create_exception!(RbOverflowError, exception::range_error());
23
+ create_exception!(ComputeError, rb_modules::compute_error());
24
+ create_exception!(InvalidOperationError, rb_modules::invalid_operation_error());
@@ -1,5 +1,3 @@
1
- use polars::prelude::*;
2
-
3
1
  use crate::RbExpr;
4
2
 
5
3
  impl RbExpr {
@@ -28,54 +26,18 @@ impl RbExpr {
28
26
  }
29
27
 
30
28
  pub fn bin_hex_decode(&self, strict: bool) -> Self {
31
- self.clone()
32
- .inner
33
- .map(
34
- move |s| {
35
- s.binary()?
36
- .hex_decode(strict)
37
- .map(|s| Some(s.into_series()))
38
- },
39
- GetOutput::same_type(),
40
- )
41
- .with_fmt("bin.hex_decode")
42
- .into()
29
+ self.inner.clone().binary().hex_decode(strict).into()
43
30
  }
44
31
 
45
32
  pub fn bin_base64_decode(&self, strict: bool) -> Self {
46
- self.clone()
47
- .inner
48
- .map(
49
- move |s| {
50
- s.binary()?
51
- .base64_decode(strict)
52
- .map(|s| Some(s.into_series()))
53
- },
54
- GetOutput::same_type(),
55
- )
56
- .with_fmt("bin.base64_decode")
57
- .into()
33
+ self.inner.clone().binary().base64_decode(strict).into()
58
34
  }
59
35
 
60
36
  pub fn bin_hex_encode(&self) -> Self {
61
- self.clone()
62
- .inner
63
- .map(
64
- move |s| s.binary().map(|s| Some(s.hex_encode().into_series())),
65
- GetOutput::same_type(),
66
- )
67
- .with_fmt("bin.hex_encode")
68
- .into()
37
+ self.inner.clone().binary().hex_encode().into()
69
38
  }
70
39
 
71
40
  pub fn bin_base64_encode(&self) -> Self {
72
- self.clone()
73
- .inner
74
- .map(
75
- move |s| s.binary().map(|s| Some(s.base64_encode().into_series())),
76
- GetOutput::same_type(),
77
- )
78
- .with_fmt("bin.base64_encode")
79
- .into()
41
+ self.inner.clone().binary().base64_encode().into()
80
42
  }
81
43
  }
@@ -13,12 +13,13 @@ impl RbExpr {
13
13
  }
14
14
 
15
15
  pub fn dt_epoch_seconds(&self) -> Self {
16
- self.clone()
17
- .inner
16
+ self.inner
17
+ .clone()
18
18
  .map(
19
19
  |s| {
20
- s.timestamp(TimeUnit::Milliseconds)
21
- .map(|ca| Some((ca / 1000).into_series()))
20
+ s.take_materialized_series()
21
+ .timestamp(TimeUnit::Milliseconds)
22
+ .map(|ca| Some((ca / 1000).into_column()))
22
23
  },
23
24
  GetOutput::from_type(DataType::Int64),
24
25
  )
@@ -29,8 +30,12 @@ impl RbExpr {
29
30
  self.inner.clone().dt().with_time_unit(tu.0).into()
30
31
  }
31
32
 
32
- pub fn dt_convert_time_zone(&self, tz: TimeZone) -> Self {
33
- self.inner.clone().dt().convert_time_zone(tz).into()
33
+ pub fn dt_convert_time_zone(&self, time_zone: String) -> Self {
34
+ self.inner
35
+ .clone()
36
+ .dt()
37
+ .convert_time_zone(time_zone.into())
38
+ .into()
34
39
  }
35
40
 
36
41
  pub fn dt_cast_time_unit(&self, tu: Wrap<TimeUnit>) -> Self {
@@ -46,7 +51,11 @@ impl RbExpr {
46
51
  self.inner
47
52
  .clone()
48
53
  .dt()
49
- .replace_time_zone(time_zone, ambiguous.inner.clone(), non_existent.0)
54
+ .replace_time_zone(
55
+ time_zone.map(|x| x.into()),
56
+ ambiguous.inner.clone(),
57
+ non_existent.0,
58
+ )
50
59
  .into()
51
60
  }
52
61