polars-df 0.13.0 → 0.15.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +30 -0
- data/Cargo.lock +1368 -319
- data/LICENSE.txt +1 -0
- data/README.md +1 -2
- data/ext/polars/Cargo.toml +15 -6
- data/ext/polars/src/batched_csv.rs +10 -13
- data/ext/polars/src/conversion/any_value.rs +37 -21
- data/ext/polars/src/conversion/chunked_array.rs +3 -3
- data/ext/polars/src/conversion/mod.rs +159 -46
- data/ext/polars/src/dataframe/construction.rs +4 -7
- data/ext/polars/src/dataframe/export.rs +9 -2
- data/ext/polars/src/dataframe/general.rs +22 -16
- data/ext/polars/src/dataframe/io.rs +78 -174
- data/ext/polars/src/dataframe/mod.rs +1 -0
- data/ext/polars/src/dataframe/serde.rs +15 -0
- data/ext/polars/src/error.rs +31 -48
- data/ext/polars/src/exceptions.rs +24 -0
- data/ext/polars/src/expr/binary.rs +4 -42
- data/ext/polars/src/expr/datetime.rs +16 -7
- data/ext/polars/src/expr/general.rs +14 -23
- data/ext/polars/src/expr/list.rs +18 -11
- data/ext/polars/src/expr/name.rs +3 -2
- data/ext/polars/src/expr/rolling.rs +6 -7
- data/ext/polars/src/expr/string.rs +17 -37
- data/ext/polars/src/file.rs +59 -22
- data/ext/polars/src/functions/business.rs +15 -0
- data/ext/polars/src/functions/io.rs +6 -6
- data/ext/polars/src/functions/lazy.rs +17 -8
- data/ext/polars/src/functions/mod.rs +1 -0
- data/ext/polars/src/functions/range.rs +4 -2
- data/ext/polars/src/interop/arrow/mod.rs +1 -0
- data/ext/polars/src/interop/arrow/to_ruby.rs +83 -0
- data/ext/polars/src/interop/mod.rs +1 -0
- data/ext/polars/src/lazyframe/general.rs +877 -0
- data/ext/polars/src/lazyframe/mod.rs +3 -825
- data/ext/polars/src/lazyframe/serde.rs +31 -0
- data/ext/polars/src/lib.rs +44 -13
- data/ext/polars/src/map/dataframe.rs +46 -14
- data/ext/polars/src/map/lazy.rs +65 -4
- data/ext/polars/src/map/mod.rs +17 -16
- data/ext/polars/src/map/series.rs +106 -64
- data/ext/polars/src/on_startup.rs +2 -2
- data/ext/polars/src/series/aggregation.rs +1 -5
- data/ext/polars/src/series/arithmetic.rs +10 -10
- data/ext/polars/src/series/construction.rs +52 -25
- data/ext/polars/src/series/export.rs +1 -1
- data/ext/polars/src/series/general.rs +643 -0
- data/ext/polars/src/series/import.rs +55 -0
- data/ext/polars/src/series/mod.rs +11 -638
- data/ext/polars/src/series/scatter.rs +2 -2
- data/ext/polars/src/utils.rs +0 -20
- data/lib/polars/batched_csv_reader.rb +0 -2
- data/lib/polars/binary_expr.rb +133 -9
- data/lib/polars/binary_name_space.rb +101 -6
- data/lib/polars/config.rb +4 -0
- data/lib/polars/data_frame.rb +285 -62
- data/lib/polars/data_type_group.rb +28 -0
- data/lib/polars/data_types.rb +2 -0
- data/lib/polars/date_time_expr.rb +244 -0
- data/lib/polars/date_time_name_space.rb +87 -0
- data/lib/polars/expr.rb +109 -8
- data/lib/polars/functions/as_datatype.rb +51 -2
- data/lib/polars/functions/col.rb +1 -1
- data/lib/polars/functions/eager.rb +1 -3
- data/lib/polars/functions/lazy.rb +88 -10
- data/lib/polars/functions/range/time_range.rb +21 -21
- data/lib/polars/io/csv.rb +14 -16
- data/lib/polars/io/database.rb +2 -2
- data/lib/polars/io/ipc.rb +14 -12
- data/lib/polars/io/ndjson.rb +10 -0
- data/lib/polars/io/parquet.rb +168 -111
- data/lib/polars/lazy_frame.rb +649 -15
- data/lib/polars/list_name_space.rb +169 -0
- data/lib/polars/selectors.rb +1144 -0
- data/lib/polars/series.rb +470 -40
- data/lib/polars/string_cache.rb +27 -1
- data/lib/polars/string_expr.rb +0 -1
- data/lib/polars/string_name_space.rb +73 -3
- data/lib/polars/struct_name_space.rb +31 -7
- data/lib/polars/utils/various.rb +5 -1
- data/lib/polars/utils.rb +45 -10
- data/lib/polars/version.rb +1 -1
- data/lib/polars.rb +2 -1
- metadata +14 -4
- data/lib/polars/functions.rb +0 -57
@@ -1,13 +1,17 @@
|
|
1
|
-
use magnus::{prelude::*,
|
1
|
+
use magnus::{prelude::*, Value};
|
2
2
|
use polars::io::avro::AvroCompression;
|
3
3
|
use polars::io::RowIndex;
|
4
4
|
use polars::prelude::*;
|
5
|
-
use
|
5
|
+
use polars_utils::mmap::ensure_not_mapped;
|
6
|
+
use std::io::BufWriter;
|
6
7
|
use std::num::NonZeroUsize;
|
7
8
|
|
8
9
|
use super::*;
|
9
10
|
use crate::conversion::*;
|
10
|
-
use crate::file::{
|
11
|
+
use crate::file::{
|
12
|
+
get_either_file, get_file_like, get_mmap_bytes_reader, get_mmap_bytes_reader_and_path,
|
13
|
+
read_if_bytesio, EitherRustRubyFile,
|
14
|
+
};
|
11
15
|
use crate::{RbPolarsErr, RbResult};
|
12
16
|
|
13
17
|
impl RbDataFrame {
|
@@ -29,8 +33,7 @@ impl RbDataFrame {
|
|
29
33
|
let n_threads = Option::<usize>::try_convert(arguments[12])?;
|
30
34
|
let path = Option::<String>::try_convert(arguments[13])?;
|
31
35
|
let overwrite_dtype = Option::<Vec<(String, Wrap<DataType>)>>::try_convert(arguments[14])?;
|
32
|
-
|
33
|
-
let overwrite_dtype_slice = Option::<Vec<Wrap<DataType>>>::None; // Option::<Vec<Wrap<DataType>>>::try_convert(arguments[15])?;
|
36
|
+
let overwrite_dtype_slice = Option::<Vec<Wrap<DataType>>>::try_convert(arguments[15])?;
|
34
37
|
let low_memory = bool::try_convert(arguments[16])?;
|
35
38
|
let comment_prefix = Option::<String>::try_convert(arguments[17])?;
|
36
39
|
let quote_char = Option::<String>::try_convert(arguments[18])?;
|
@@ -39,18 +42,17 @@ impl RbDataFrame {
|
|
39
42
|
let try_parse_dates = bool::try_convert(arguments[21])?;
|
40
43
|
let skip_rows_after_header = usize::try_convert(arguments[22])?;
|
41
44
|
let row_index = Option::<(String, IdxSize)>::try_convert(arguments[23])?;
|
42
|
-
let
|
43
|
-
let
|
44
|
-
let
|
45
|
-
let
|
46
|
-
let
|
47
|
-
let schema = Option::<Wrap<Schema>>::try_convert(arguments[29])?;
|
45
|
+
let eol_char = String::try_convert(arguments[24])?;
|
46
|
+
let raise_if_empty = bool::try_convert(arguments[25])?;
|
47
|
+
let truncate_ragged_lines = bool::try_convert(arguments[26])?;
|
48
|
+
let decimal_comma = bool::try_convert(arguments[27])?;
|
49
|
+
let schema = Option::<Wrap<Schema>>::try_convert(arguments[28])?;
|
48
50
|
// end arguments
|
49
51
|
|
50
52
|
let null_values = null_values.map(|w| w.0);
|
51
53
|
let eol_char = eol_char.as_bytes()[0];
|
52
54
|
let row_index = row_index.map(|(name, offset)| RowIndex {
|
53
|
-
name:
|
55
|
+
name: name.into(),
|
54
56
|
offset,
|
55
57
|
});
|
56
58
|
let quote_char = if let Some(s) = quote_char {
|
@@ -68,7 +70,7 @@ impl RbDataFrame {
|
|
68
70
|
.iter()
|
69
71
|
.map(|(name, dtype)| {
|
70
72
|
let dtype = dtype.0.clone();
|
71
|
-
Field::new(name, dtype)
|
73
|
+
Field::new((&**name).into(), dtype)
|
72
74
|
})
|
73
75
|
.collect::<Schema>()
|
74
76
|
});
|
@@ -80,7 +82,8 @@ impl RbDataFrame {
|
|
80
82
|
.collect::<Vec<_>>()
|
81
83
|
});
|
82
84
|
|
83
|
-
let
|
85
|
+
let rb_f = read_if_bytesio(rb_f);
|
86
|
+
let mmap_bytes_r = get_mmap_bytes_reader(&rb_f)?;
|
84
87
|
let df = CsvReadOptions::default()
|
85
88
|
.with_path(path)
|
86
89
|
.with_infer_schema_length(infer_schema_length)
|
@@ -91,7 +94,7 @@ impl RbDataFrame {
|
|
91
94
|
.with_projection(projection.map(Arc::new))
|
92
95
|
.with_rechunk(rechunk)
|
93
96
|
.with_chunk_size(chunk_size)
|
94
|
-
.with_columns(columns.map(
|
97
|
+
.with_columns(columns.map(|x| x.into_iter().map(|x| x.into()).collect()))
|
95
98
|
.with_n_threads(n_threads)
|
96
99
|
.with_schema_overwrite(overwrite_dtype.map(Arc::new))
|
97
100
|
.with_dtype_overwrite(overwrite_dtype_slice.map(Arc::new))
|
@@ -99,7 +102,6 @@ impl RbDataFrame {
|
|
99
102
|
.with_low_memory(low_memory)
|
100
103
|
.with_skip_rows_after_header(skip_rows_after_header)
|
101
104
|
.with_row_index(row_index)
|
102
|
-
.with_sample_size(sample_size)
|
103
105
|
.with_raise_if_empty(raise_if_empty)
|
104
106
|
.with_parse_options(
|
105
107
|
CsvParseOptions::default()
|
@@ -120,59 +122,14 @@ impl RbDataFrame {
|
|
120
122
|
Ok(df.into())
|
121
123
|
}
|
122
124
|
|
123
|
-
#[allow(clippy::too_many_arguments)]
|
124
|
-
pub fn read_parquet(
|
125
|
-
rb_f: Value,
|
126
|
-
columns: Option<Vec<String>>,
|
127
|
-
projection: Option<Vec<usize>>,
|
128
|
-
n_rows: Option<usize>,
|
129
|
-
parallel: Wrap<ParallelStrategy>,
|
130
|
-
row_index: Option<(String, IdxSize)>,
|
131
|
-
low_memory: bool,
|
132
|
-
use_statistics: bool,
|
133
|
-
rechunk: bool,
|
134
|
-
) -> RbResult<Self> {
|
135
|
-
use EitherRustRubyFile::*;
|
136
|
-
|
137
|
-
let row_index = row_index.map(|(name, offset)| RowIndex {
|
138
|
-
name: Arc::from(name.as_str()),
|
139
|
-
offset,
|
140
|
-
});
|
141
|
-
let result = match get_either_file(rb_f, false)? {
|
142
|
-
Rb(f) => {
|
143
|
-
let buf = f.as_buffer();
|
144
|
-
ParquetReader::new(buf)
|
145
|
-
.with_projection(projection)
|
146
|
-
.with_columns(columns)
|
147
|
-
.read_parallel(parallel.0)
|
148
|
-
.with_slice(n_rows.map(|x| (0, x)))
|
149
|
-
.with_row_index(row_index)
|
150
|
-
.set_low_memory(low_memory)
|
151
|
-
.use_statistics(use_statistics)
|
152
|
-
.set_rechunk(rechunk)
|
153
|
-
.finish()
|
154
|
-
}
|
155
|
-
Rust(f) => ParquetReader::new(f.into_inner())
|
156
|
-
.with_projection(projection)
|
157
|
-
.with_columns(columns)
|
158
|
-
.read_parallel(parallel.0)
|
159
|
-
.with_slice(n_rows.map(|x| (0, x)))
|
160
|
-
.with_row_index(row_index)
|
161
|
-
.use_statistics(use_statistics)
|
162
|
-
.set_rechunk(rechunk)
|
163
|
-
.finish(),
|
164
|
-
};
|
165
|
-
let df = result.map_err(RbPolarsErr::from)?;
|
166
|
-
Ok(RbDataFrame::new(df))
|
167
|
-
}
|
168
|
-
|
169
125
|
pub fn read_json(
|
170
126
|
rb_f: Value,
|
171
127
|
infer_schema_length: Option<usize>,
|
172
128
|
schema: Option<Wrap<Schema>>,
|
173
129
|
schema_overrides: Option<Wrap<Schema>>,
|
174
130
|
) -> RbResult<Self> {
|
175
|
-
let
|
131
|
+
let rb_f = read_if_bytesio(rb_f);
|
132
|
+
let mmap_bytes_r = get_mmap_bytes_reader(&rb_f)?;
|
176
133
|
|
177
134
|
let mut builder = JsonReader::new(mmap_bytes_r)
|
178
135
|
.with_json_format(JsonFormat::Json)
|
@@ -196,7 +153,8 @@ impl RbDataFrame {
|
|
196
153
|
schema: Option<Wrap<Schema>>,
|
197
154
|
schema_overrides: Option<Wrap<Schema>>,
|
198
155
|
) -> RbResult<Self> {
|
199
|
-
let
|
156
|
+
let rb_f = read_if_bytesio(rb_f);
|
157
|
+
let mmap_bytes_r = get_mmap_bytes_reader(&rb_f)?;
|
200
158
|
|
201
159
|
let mut builder = JsonReader::new(mmap_bytes_r)
|
202
160
|
.with_json_format(JsonFormat::JsonLines)
|
@@ -212,7 +170,7 @@ impl RbDataFrame {
|
|
212
170
|
|
213
171
|
let out = builder
|
214
172
|
.finish()
|
215
|
-
.map_err(|e| RbPolarsErr::
|
173
|
+
.map_err(|e| RbPolarsErr::Other(format!("{e}")))?;
|
216
174
|
Ok(out.into())
|
217
175
|
}
|
218
176
|
|
@@ -222,16 +180,16 @@ impl RbDataFrame {
|
|
222
180
|
projection: Option<Vec<usize>>,
|
223
181
|
n_rows: Option<usize>,
|
224
182
|
row_index: Option<(String, IdxSize)>,
|
225
|
-
|
183
|
+
memory_map: bool,
|
226
184
|
) -> RbResult<Self> {
|
227
185
|
let row_index = row_index.map(|(name, offset)| RowIndex {
|
228
|
-
name:
|
186
|
+
name: name.into(),
|
229
187
|
offset,
|
230
188
|
});
|
231
|
-
let
|
189
|
+
let rb_f = read_if_bytesio(rb_f);
|
190
|
+
let (mmap_bytes_r, mmap_path) = get_mmap_bytes_reader_and_path(&rb_f)?;
|
232
191
|
|
233
|
-
|
234
|
-
let mmap_path = None;
|
192
|
+
let mmap_path = if memory_map { mmap_path } else { None };
|
235
193
|
let df = IpcReader::new(mmap_bytes_r)
|
236
194
|
.with_projection(projection)
|
237
195
|
.with_columns(columns)
|
@@ -252,11 +210,11 @@ impl RbDataFrame {
|
|
252
210
|
rechunk: bool,
|
253
211
|
) -> RbResult<Self> {
|
254
212
|
let row_index = row_index.map(|(name, offset)| RowIndex {
|
255
|
-
name:
|
213
|
+
name: name.into(),
|
256
214
|
offset,
|
257
215
|
});
|
258
|
-
|
259
|
-
let mmap_bytes_r = get_mmap_bytes_reader(rb_f)?;
|
216
|
+
let rb_f = read_if_bytesio(rb_f);
|
217
|
+
let mmap_bytes_r = get_mmap_bytes_reader(&rb_f)?;
|
260
218
|
let df = IpcStreamReader::new(mmap_bytes_r)
|
261
219
|
.with_projection(projection)
|
262
220
|
.with_columns(columns)
|
@@ -302,41 +260,19 @@ impl RbDataFrame {
|
|
302
260
|
) -> RbResult<()> {
|
303
261
|
let batch_size = batch_size.0;
|
304
262
|
let null = null_value.unwrap_or_default();
|
305
|
-
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
.with_null_value(null)
|
319
|
-
.finish(&mut self.df.borrow_mut())
|
320
|
-
.map_err(RbPolarsErr::from)?;
|
321
|
-
} else {
|
322
|
-
let mut buf = Cursor::new(Vec::new());
|
323
|
-
CsvWriter::new(&mut buf)
|
324
|
-
.include_header(include_header)
|
325
|
-
.with_separator(separator)
|
326
|
-
.with_quote_char(quote_char)
|
327
|
-
.with_batch_size(batch_size)
|
328
|
-
.with_datetime_format(datetime_format)
|
329
|
-
.with_date_format(date_format)
|
330
|
-
.with_time_format(time_format)
|
331
|
-
.with_float_precision(float_precision)
|
332
|
-
.with_null_value(null)
|
333
|
-
.finish(&mut self.df.borrow_mut())
|
334
|
-
.map_err(RbPolarsErr::from)?;
|
335
|
-
// TODO less copying
|
336
|
-
let rb_str = RString::from_slice(&buf.into_inner());
|
337
|
-
rb_f.funcall::<_, _, Value>("write", (rb_str,))?;
|
338
|
-
}
|
339
|
-
|
263
|
+
let mut buf = get_file_like(rb_f, true)?;
|
264
|
+
CsvWriter::new(&mut buf)
|
265
|
+
.include_header(include_header)
|
266
|
+
.with_separator(separator)
|
267
|
+
.with_quote_char(quote_char)
|
268
|
+
.with_batch_size(batch_size)
|
269
|
+
.with_datetime_format(datetime_format)
|
270
|
+
.with_date_format(date_format)
|
271
|
+
.with_time_format(time_format)
|
272
|
+
.with_float_precision(float_precision)
|
273
|
+
.with_null_value(null)
|
274
|
+
.finish(&mut self.df.borrow_mut())
|
275
|
+
.map_err(RbPolarsErr::from)?;
|
340
276
|
Ok(())
|
341
277
|
}
|
342
278
|
|
@@ -351,26 +287,14 @@ impl RbDataFrame {
|
|
351
287
|
) -> RbResult<()> {
|
352
288
|
let compression = parse_parquet_compression(&compression, compression_level)?;
|
353
289
|
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
.map_err(RbPolarsErr::from)?;
|
363
|
-
} else {
|
364
|
-
let buf = get_file_like(rb_f, true)?;
|
365
|
-
ParquetWriter::new(buf)
|
366
|
-
.with_compression(compression)
|
367
|
-
.with_statistics(statistics.0)
|
368
|
-
.with_row_group_size(row_group_size)
|
369
|
-
.with_data_page_size(data_page_size)
|
370
|
-
.finish(&mut self.df.borrow_mut())
|
371
|
-
.map_err(RbPolarsErr::from)?;
|
372
|
-
}
|
373
|
-
|
290
|
+
let buf = get_file_like(rb_f, true)?;
|
291
|
+
ParquetWriter::new(buf)
|
292
|
+
.with_compression(compression)
|
293
|
+
.with_statistics(statistics.0)
|
294
|
+
.with_row_group_size(row_group_size)
|
295
|
+
.with_data_page_size(data_page_size)
|
296
|
+
.finish(&mut self.df.borrow_mut())
|
297
|
+
.map_err(RbPolarsErr::from)?;
|
374
298
|
Ok(())
|
375
299
|
}
|
376
300
|
|
@@ -386,7 +310,7 @@ impl RbDataFrame {
|
|
386
310
|
(false, _) => serde_json::to_writer(file, &*self.df.borrow())
|
387
311
|
.map_err(|e| PolarsError::ComputeError(format!("{:?}", e).into())),
|
388
312
|
};
|
389
|
-
r.map_err(|e| RbPolarsErr::
|
313
|
+
r.map_err(|e| RbPolarsErr::Other(format!("{:?}", e)))?;
|
390
314
|
Ok(())
|
391
315
|
}
|
392
316
|
|
@@ -397,7 +321,7 @@ impl RbDataFrame {
|
|
397
321
|
.with_json_format(JsonFormat::JsonLines)
|
398
322
|
.finish(&mut self.df.borrow_mut());
|
399
323
|
|
400
|
-
r.map_err(|e| RbPolarsErr::
|
324
|
+
r.map_err(|e| RbPolarsErr::Other(format!("{:?}", e)))?;
|
401
325
|
Ok(())
|
402
326
|
}
|
403
327
|
|
@@ -405,23 +329,18 @@ impl RbDataFrame {
|
|
405
329
|
&self,
|
406
330
|
rb_f: Value,
|
407
331
|
compression: Wrap<Option<IpcCompression>>,
|
332
|
+
compat_level: RbCompatLevel,
|
408
333
|
) -> RbResult<()> {
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
.with_compression(compression.0)
|
413
|
-
.finish(&mut self.df.borrow_mut())
|
414
|
-
.map_err(RbPolarsErr::from)?;
|
415
|
-
} else {
|
416
|
-
let mut buf = Cursor::new(Vec::new());
|
417
|
-
IpcWriter::new(&mut buf)
|
418
|
-
.with_compression(compression.0)
|
419
|
-
.finish(&mut self.df.borrow_mut())
|
420
|
-
.map_err(RbPolarsErr::from)?;
|
421
|
-
// TODO less copying
|
422
|
-
let rb_str = RString::from_slice(&buf.into_inner());
|
423
|
-
rb_f.funcall::<_, _, Value>("write", (rb_str,))?;
|
334
|
+
let either = get_either_file(rb_f, true)?;
|
335
|
+
if let EitherRustRubyFile::Rust(ref f) = either {
|
336
|
+
ensure_not_mapped(f).map_err(RbPolarsErr::from)?;
|
424
337
|
}
|
338
|
+
let mut buf = either.into_dyn();
|
339
|
+
IpcWriter::new(&mut buf)
|
340
|
+
.with_compression(compression.0)
|
341
|
+
.with_compat_level(compat_level.0)
|
342
|
+
.finish(&mut self.df.borrow_mut())
|
343
|
+
.map_err(RbPolarsErr::from)?;
|
425
344
|
Ok(())
|
426
345
|
}
|
427
346
|
|
@@ -429,21 +348,14 @@ impl RbDataFrame {
|
|
429
348
|
&self,
|
430
349
|
rb_f: Value,
|
431
350
|
compression: Wrap<Option<IpcCompression>>,
|
351
|
+
compat_level: RbCompatLevel,
|
432
352
|
) -> RbResult<()> {
|
433
|
-
|
434
|
-
|
435
|
-
|
436
|
-
|
437
|
-
|
438
|
-
|
439
|
-
} else {
|
440
|
-
let mut buf = get_file_like(rb_f, true)?;
|
441
|
-
|
442
|
-
IpcStreamWriter::new(&mut buf)
|
443
|
-
.with_compression(compression.0)
|
444
|
-
.finish(&mut self.df.borrow_mut())
|
445
|
-
.map_err(RbPolarsErr::from)?;
|
446
|
-
}
|
353
|
+
let mut buf = get_file_like(rb_f, true)?;
|
354
|
+
IpcStreamWriter::new(&mut buf)
|
355
|
+
.with_compression(compression.0)
|
356
|
+
.with_compat_level(compat_level.0)
|
357
|
+
.finish(&mut self.df.borrow_mut())
|
358
|
+
.map_err(RbPolarsErr::from)?;
|
447
359
|
Ok(())
|
448
360
|
}
|
449
361
|
|
@@ -451,23 +363,15 @@ impl RbDataFrame {
|
|
451
363
|
&self,
|
452
364
|
rb_f: Value,
|
453
365
|
compression: Wrap<Option<AvroCompression>>,
|
366
|
+
name: String,
|
454
367
|
) -> RbResult<()> {
|
455
368
|
use polars::io::avro::AvroWriter;
|
456
|
-
|
457
|
-
|
458
|
-
|
459
|
-
|
460
|
-
|
461
|
-
|
462
|
-
.map_err(RbPolarsErr::from)?;
|
463
|
-
} else {
|
464
|
-
let mut buf = get_file_like(rb_f, true)?;
|
465
|
-
AvroWriter::new(&mut buf)
|
466
|
-
.with_compression(compression.0)
|
467
|
-
.finish(&mut self.df.borrow_mut())
|
468
|
-
.map_err(RbPolarsErr::from)?;
|
469
|
-
}
|
470
|
-
|
369
|
+
let mut buf = get_file_like(rb_f, true)?;
|
370
|
+
AvroWriter::new(&mut buf)
|
371
|
+
.with_compression(compression.0)
|
372
|
+
.with_name(name)
|
373
|
+
.finish(&mut self.df.borrow_mut())
|
374
|
+
.map_err(RbPolarsErr::from)?;
|
471
375
|
Ok(())
|
472
376
|
}
|
473
377
|
}
|
@@ -0,0 +1,15 @@
|
|
1
|
+
use crate::exceptions::ComputeError;
|
2
|
+
use crate::file::get_file_like;
|
3
|
+
use crate::{RbDataFrame, RbResult};
|
4
|
+
use magnus::Value;
|
5
|
+
use std::io::BufWriter;
|
6
|
+
|
7
|
+
impl RbDataFrame {
|
8
|
+
// TODO add to Ruby
|
9
|
+
pub fn serialize_json(&self, rb_f: Value) -> RbResult<()> {
|
10
|
+
let file = get_file_like(rb_f, true)?;
|
11
|
+
let writer = BufWriter::new(file);
|
12
|
+
serde_json::to_writer(writer, &self.df)
|
13
|
+
.map_err(|err| ComputeError::new_err(err.to_string()))
|
14
|
+
}
|
15
|
+
}
|
data/ext/polars/src/error.rs
CHANGED
@@ -1,67 +1,50 @@
|
|
1
|
-
use
|
1
|
+
use std::fmt::{Debug, Formatter};
|
2
|
+
|
2
3
|
use magnus::Error;
|
3
4
|
use polars::prelude::PolarsError;
|
4
5
|
|
6
|
+
use crate::exceptions::{ComputeError, InvalidOperationError};
|
5
7
|
use crate::rb_modules;
|
6
8
|
|
7
|
-
pub
|
8
|
-
|
9
|
-
|
10
|
-
// convert to Error instead of Self
|
11
|
-
pub fn from(e: PolarsError) -> Error {
|
12
|
-
match e {
|
13
|
-
PolarsError::ComputeError(err) => ComputeError::new_err(err.to_string()),
|
14
|
-
PolarsError::InvalidOperation(err) => InvalidOperationError::new_err(err.to_string()),
|
15
|
-
_ => Error::new(rb_modules::error(), e.to_string()),
|
16
|
-
}
|
17
|
-
}
|
18
|
-
|
19
|
-
pub fn io(e: std::io::Error) -> Error {
|
20
|
-
Error::new(rb_modules::error(), e.to_string())
|
21
|
-
}
|
22
|
-
|
23
|
-
pub fn other(message: String) -> Error {
|
24
|
-
Error::new(rb_modules::error(), message)
|
25
|
-
}
|
26
|
-
}
|
27
|
-
|
28
|
-
pub struct RbTypeError {}
|
29
|
-
|
30
|
-
impl RbTypeError {
|
31
|
-
pub fn new_err(message: String) -> Error {
|
32
|
-
Error::new(exception::type_error(), message)
|
33
|
-
}
|
9
|
+
pub enum RbPolarsErr {
|
10
|
+
Polars(PolarsError),
|
11
|
+
Other(String),
|
34
12
|
}
|
35
13
|
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
pub fn new_err(message: String) -> Error {
|
40
|
-
Error::new(exception::arg_error(), message)
|
14
|
+
impl From<PolarsError> for RbPolarsErr {
|
15
|
+
fn from(err: PolarsError) -> Self {
|
16
|
+
RbPolarsErr::Polars(err)
|
41
17
|
}
|
42
18
|
}
|
43
19
|
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
pub fn new_err(message: String) -> Error {
|
48
|
-
Error::new(exception::range_error(), message)
|
20
|
+
impl From<std::io::Error> for RbPolarsErr {
|
21
|
+
fn from(value: std::io::Error) -> Self {
|
22
|
+
RbPolarsErr::Other(format!("{value:?}"))
|
49
23
|
}
|
50
24
|
}
|
51
25
|
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
26
|
+
impl From<RbPolarsErr> for Error {
|
27
|
+
fn from(err: RbPolarsErr) -> Self {
|
28
|
+
match err {
|
29
|
+
RbPolarsErr::Polars(err) => match err {
|
30
|
+
PolarsError::ComputeError(err) => ComputeError::new_err(err.to_string()),
|
31
|
+
PolarsError::InvalidOperation(err) => {
|
32
|
+
InvalidOperationError::new_err(err.to_string())
|
33
|
+
}
|
34
|
+
_ => Error::new(rb_modules::error(), err.to_string()),
|
35
|
+
},
|
36
|
+
RbPolarsErr::Other(err) => Error::new(rb_modules::error(), err.to_string()),
|
37
|
+
}
|
57
38
|
}
|
58
39
|
}
|
59
40
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
41
|
+
impl Debug for RbPolarsErr {
|
42
|
+
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
43
|
+
use RbPolarsErr::*;
|
44
|
+
match self {
|
45
|
+
Polars(err) => write!(f, "{err:?}"),
|
46
|
+
Other(err) => write!(f, "BindingsError: {err:?}"),
|
47
|
+
}
|
65
48
|
}
|
66
49
|
}
|
67
50
|
|
@@ -0,0 +1,24 @@
|
|
1
|
+
use crate::rb_modules;
|
2
|
+
use magnus::{exception, Error};
|
3
|
+
use std::borrow::Cow;
|
4
|
+
|
5
|
+
macro_rules! create_exception {
|
6
|
+
($type:ident, $cls:expr) => {
|
7
|
+
pub struct $type {}
|
8
|
+
|
9
|
+
impl $type {
|
10
|
+
pub fn new_err<T>(message: T) -> Error
|
11
|
+
where
|
12
|
+
T: Into<Cow<'static, str>>,
|
13
|
+
{
|
14
|
+
Error::new($cls, message)
|
15
|
+
}
|
16
|
+
}
|
17
|
+
};
|
18
|
+
}
|
19
|
+
|
20
|
+
create_exception!(RbTypeError, exception::type_error());
|
21
|
+
create_exception!(RbValueError, exception::arg_error());
|
22
|
+
create_exception!(RbOverflowError, exception::range_error());
|
23
|
+
create_exception!(ComputeError, rb_modules::compute_error());
|
24
|
+
create_exception!(InvalidOperationError, rb_modules::invalid_operation_error());
|
@@ -1,5 +1,3 @@
|
|
1
|
-
use polars::prelude::*;
|
2
|
-
|
3
1
|
use crate::RbExpr;
|
4
2
|
|
5
3
|
impl RbExpr {
|
@@ -28,54 +26,18 @@ impl RbExpr {
|
|
28
26
|
}
|
29
27
|
|
30
28
|
pub fn bin_hex_decode(&self, strict: bool) -> Self {
|
31
|
-
self.clone()
|
32
|
-
.inner
|
33
|
-
.map(
|
34
|
-
move |s| {
|
35
|
-
s.binary()?
|
36
|
-
.hex_decode(strict)
|
37
|
-
.map(|s| Some(s.into_series()))
|
38
|
-
},
|
39
|
-
GetOutput::same_type(),
|
40
|
-
)
|
41
|
-
.with_fmt("bin.hex_decode")
|
42
|
-
.into()
|
29
|
+
self.inner.clone().binary().hex_decode(strict).into()
|
43
30
|
}
|
44
31
|
|
45
32
|
pub fn bin_base64_decode(&self, strict: bool) -> Self {
|
46
|
-
self.clone()
|
47
|
-
.inner
|
48
|
-
.map(
|
49
|
-
move |s| {
|
50
|
-
s.binary()?
|
51
|
-
.base64_decode(strict)
|
52
|
-
.map(|s| Some(s.into_series()))
|
53
|
-
},
|
54
|
-
GetOutput::same_type(),
|
55
|
-
)
|
56
|
-
.with_fmt("bin.base64_decode")
|
57
|
-
.into()
|
33
|
+
self.inner.clone().binary().base64_decode(strict).into()
|
58
34
|
}
|
59
35
|
|
60
36
|
pub fn bin_hex_encode(&self) -> Self {
|
61
|
-
self.clone()
|
62
|
-
.inner
|
63
|
-
.map(
|
64
|
-
move |s| s.binary().map(|s| Some(s.hex_encode().into_series())),
|
65
|
-
GetOutput::same_type(),
|
66
|
-
)
|
67
|
-
.with_fmt("bin.hex_encode")
|
68
|
-
.into()
|
37
|
+
self.inner.clone().binary().hex_encode().into()
|
69
38
|
}
|
70
39
|
|
71
40
|
pub fn bin_base64_encode(&self) -> Self {
|
72
|
-
self.clone()
|
73
|
-
.inner
|
74
|
-
.map(
|
75
|
-
move |s| s.binary().map(|s| Some(s.base64_encode().into_series())),
|
76
|
-
GetOutput::same_type(),
|
77
|
-
)
|
78
|
-
.with_fmt("bin.base64_encode")
|
79
|
-
.into()
|
41
|
+
self.inner.clone().binary().base64_encode().into()
|
80
42
|
}
|
81
43
|
}
|
@@ -13,12 +13,13 @@ impl RbExpr {
|
|
13
13
|
}
|
14
14
|
|
15
15
|
pub fn dt_epoch_seconds(&self) -> Self {
|
16
|
-
self.
|
17
|
-
.
|
16
|
+
self.inner
|
17
|
+
.clone()
|
18
18
|
.map(
|
19
19
|
|s| {
|
20
|
-
s.
|
21
|
-
.
|
20
|
+
s.take_materialized_series()
|
21
|
+
.timestamp(TimeUnit::Milliseconds)
|
22
|
+
.map(|ca| Some((ca / 1000).into_column()))
|
22
23
|
},
|
23
24
|
GetOutput::from_type(DataType::Int64),
|
24
25
|
)
|
@@ -29,8 +30,12 @@ impl RbExpr {
|
|
29
30
|
self.inner.clone().dt().with_time_unit(tu.0).into()
|
30
31
|
}
|
31
32
|
|
32
|
-
pub fn dt_convert_time_zone(&self,
|
33
|
-
self.inner
|
33
|
+
pub fn dt_convert_time_zone(&self, time_zone: String) -> Self {
|
34
|
+
self.inner
|
35
|
+
.clone()
|
36
|
+
.dt()
|
37
|
+
.convert_time_zone(time_zone.into())
|
38
|
+
.into()
|
34
39
|
}
|
35
40
|
|
36
41
|
pub fn dt_cast_time_unit(&self, tu: Wrap<TimeUnit>) -> Self {
|
@@ -46,7 +51,11 @@ impl RbExpr {
|
|
46
51
|
self.inner
|
47
52
|
.clone()
|
48
53
|
.dt()
|
49
|
-
.replace_time_zone(
|
54
|
+
.replace_time_zone(
|
55
|
+
time_zone.map(|x| x.into()),
|
56
|
+
ambiguous.inner.clone(),
|
57
|
+
non_existent.0,
|
58
|
+
)
|
50
59
|
.into()
|
51
60
|
}
|
52
61
|
|