parquet 0.5.5 → 0.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.lock +51 -44
- data/ext/parquet/Cargo.toml +3 -3
- data/ext/parquet/src/types/record_types.rs +26 -4
- data/ext/parquet/src/types/schema_converter.rs +14 -75
- data/lib/parquet/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: dc1d1eda7d71aa6336fbf6cc94789517439df3fab1852ec7d2e9d265e0c016c4
|
4
|
+
data.tar.gz: 6fff5321a31d3fe19a59a4f47add56222dbeb274bef7a068163b48757d65252d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ddd50f82df2b42cf844e379a7f07c0214e9aef925e7c43ec566b6b9f27be311676b6f887c163aa5d41d4523cd1d506266b15623205453bc8e08467c88e7c2b63
|
7
|
+
data.tar.gz: afb235ad09338d8c4cd59588dded3d312890c5d5d879b77040fcbf960be69653981fe5176cc591969a80ba54214d4c6a63cff96c36ceda7b9e00c75ba8e9e913
|
data/Cargo.lock
CHANGED
@@ -63,9 +63,8 @@ dependencies = [
|
|
63
63
|
|
64
64
|
[[package]]
|
65
65
|
name = "arrow-array"
|
66
|
-
version = "
|
67
|
-
source = "
|
68
|
-
checksum = "57a4a6d2896083cfbdf84a71a863b22460d0708f8206a8373c52e326cc72ea1a"
|
66
|
+
version = "55.1.0"
|
67
|
+
source = "git+https://github.com/njaremko/arrow-rs?branch=nathan%2Ffix-reading-int32-timestamp-records#f791b78a67cb5d9a0b4ec0fcab80780dcb61c346"
|
69
68
|
dependencies = [
|
70
69
|
"ahash",
|
71
70
|
"arrow-buffer",
|
@@ -79,9 +78,8 @@ dependencies = [
|
|
79
78
|
|
80
79
|
[[package]]
|
81
80
|
name = "arrow-buffer"
|
82
|
-
version = "
|
83
|
-
source = "
|
84
|
-
checksum = "cef870583ce5e4f3b123c181706f2002fb134960f9a911900f64ba4830c7a43a"
|
81
|
+
version = "55.1.0"
|
82
|
+
source = "git+https://github.com/njaremko/arrow-rs?branch=nathan%2Ffix-reading-int32-timestamp-records#f791b78a67cb5d9a0b4ec0fcab80780dcb61c346"
|
85
83
|
dependencies = [
|
86
84
|
"bytes",
|
87
85
|
"half",
|
@@ -90,9 +88,8 @@ dependencies = [
|
|
90
88
|
|
91
89
|
[[package]]
|
92
90
|
name = "arrow-cast"
|
93
|
-
version = "
|
94
|
-
source = "
|
95
|
-
checksum = "1ac7eba5a987f8b4a7d9629206ba48e19a1991762795bbe5d08497b7736017ee"
|
91
|
+
version = "55.1.0"
|
92
|
+
source = "git+https://github.com/njaremko/arrow-rs?branch=nathan%2Ffix-reading-int32-timestamp-records#f791b78a67cb5d9a0b4ec0fcab80780dcb61c346"
|
96
93
|
dependencies = [
|
97
94
|
"arrow-array",
|
98
95
|
"arrow-buffer",
|
@@ -110,9 +107,8 @@ dependencies = [
|
|
110
107
|
|
111
108
|
[[package]]
|
112
109
|
name = "arrow-data"
|
113
|
-
version = "
|
114
|
-
source = "
|
115
|
-
checksum = "b095e8a4f3c309544935d53e04c3bfe4eea4e71c3de6fe0416d1f08bb4441a83"
|
110
|
+
version = "55.1.0"
|
111
|
+
source = "git+https://github.com/njaremko/arrow-rs?branch=nathan%2Ffix-reading-int32-timestamp-records#f791b78a67cb5d9a0b4ec0fcab80780dcb61c346"
|
116
112
|
dependencies = [
|
117
113
|
"arrow-buffer",
|
118
114
|
"arrow-schema",
|
@@ -122,9 +118,8 @@ dependencies = [
|
|
122
118
|
|
123
119
|
[[package]]
|
124
120
|
name = "arrow-ipc"
|
125
|
-
version = "
|
126
|
-
source = "
|
127
|
-
checksum = "65c63da4afedde2b25ef69825cd4663ca76f78f79ffe2d057695742099130ff6"
|
121
|
+
version = "55.1.0"
|
122
|
+
source = "git+https://github.com/njaremko/arrow-rs?branch=nathan%2Ffix-reading-int32-timestamp-records#f791b78a67cb5d9a0b4ec0fcab80780dcb61c346"
|
128
123
|
dependencies = [
|
129
124
|
"arrow-array",
|
130
125
|
"arrow-buffer",
|
@@ -135,15 +130,13 @@ dependencies = [
|
|
135
130
|
|
136
131
|
[[package]]
|
137
132
|
name = "arrow-schema"
|
138
|
-
version = "
|
139
|
-
source = "
|
140
|
-
checksum = "0f40f6be8f78af1ab610db7d9b236e21d587b7168e368a36275d2e5670096735"
|
133
|
+
version = "55.1.0"
|
134
|
+
source = "git+https://github.com/njaremko/arrow-rs?branch=nathan%2Ffix-reading-int32-timestamp-records#f791b78a67cb5d9a0b4ec0fcab80780dcb61c346"
|
141
135
|
|
142
136
|
[[package]]
|
143
137
|
name = "arrow-select"
|
144
|
-
version = "
|
145
|
-
source = "
|
146
|
-
checksum = "ac265273864a820c4a179fc67182ccc41ea9151b97024e1be956f0f2369c2539"
|
138
|
+
version = "55.1.0"
|
139
|
+
source = "git+https://github.com/njaremko/arrow-rs?branch=nathan%2Ffix-reading-int32-timestamp-records#f791b78a67cb5d9a0b4ec0fcab80780dcb61c346"
|
147
140
|
dependencies = [
|
148
141
|
"ahash",
|
149
142
|
"arrow-array",
|
@@ -180,7 +173,7 @@ version = "0.69.5"
|
|
180
173
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
181
174
|
checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"
|
182
175
|
dependencies = [
|
183
|
-
"bitflags
|
176
|
+
"bitflags",
|
184
177
|
"cexpr",
|
185
178
|
"clang-sys",
|
186
179
|
"itertools 0.12.1",
|
@@ -194,12 +187,6 @@ dependencies = [
|
|
194
187
|
"syn",
|
195
188
|
]
|
196
189
|
|
197
|
-
[[package]]
|
198
|
-
name = "bitflags"
|
199
|
-
version = "1.3.2"
|
200
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
201
|
-
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
202
|
-
|
203
190
|
[[package]]
|
204
191
|
name = "bitflags"
|
205
192
|
version = "2.8.0"
|
@@ -208,9 +195,9 @@ checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36"
|
|
208
195
|
|
209
196
|
[[package]]
|
210
197
|
name = "brotli"
|
211
|
-
version = "
|
198
|
+
version = "8.0.1"
|
212
199
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
213
|
-
checksum = "
|
200
|
+
checksum = "9991eea70ea4f293524138648e41ee89b0b2b12ddef3b255effa43c8056e0e0d"
|
214
201
|
dependencies = [
|
215
202
|
"alloc-no-stdlib",
|
216
203
|
"alloc-stdlib",
|
@@ -219,9 +206,9 @@ dependencies = [
|
|
219
206
|
|
220
207
|
[[package]]
|
221
208
|
name = "brotli-decompressor"
|
222
|
-
version = "
|
209
|
+
version = "5.0.0"
|
223
210
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
224
|
-
checksum = "
|
211
|
+
checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03"
|
225
212
|
dependencies = [
|
226
213
|
"alloc-no-stdlib",
|
227
214
|
"alloc-stdlib",
|
@@ -359,11 +346,11 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
|
|
359
346
|
|
360
347
|
[[package]]
|
361
348
|
name = "flatbuffers"
|
362
|
-
version = "
|
349
|
+
version = "25.2.10"
|
363
350
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
364
|
-
checksum = "
|
351
|
+
checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1"
|
365
352
|
dependencies = [
|
366
|
-
"bitflags
|
353
|
+
"bitflags",
|
367
354
|
"rustc_version",
|
368
355
|
]
|
369
356
|
|
@@ -374,6 +361,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
374
361
|
checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc"
|
375
362
|
dependencies = [
|
376
363
|
"crc32fast",
|
364
|
+
"libz-rs-sys",
|
377
365
|
"miniz_oxide",
|
378
366
|
]
|
379
367
|
|
@@ -652,6 +640,15 @@ dependencies = [
|
|
652
640
|
"libc",
|
653
641
|
]
|
654
642
|
|
643
|
+
[[package]]
|
644
|
+
name = "libz-rs-sys"
|
645
|
+
version = "0.4.2"
|
646
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
647
|
+
checksum = "902bc563b5d65ad9bba616b490842ef0651066a1a1dc3ce1087113ffcb873c8d"
|
648
|
+
dependencies = [
|
649
|
+
"zlib-rs",
|
650
|
+
]
|
651
|
+
|
655
652
|
[[package]]
|
656
653
|
name = "linux-raw-sys"
|
657
654
|
version = "0.4.15"
|
@@ -670,7 +667,7 @@ version = "0.11.3"
|
|
670
667
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
671
668
|
checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5"
|
672
669
|
dependencies = [
|
673
|
-
"twox-hash",
|
670
|
+
"twox-hash 1.6.3",
|
674
671
|
]
|
675
672
|
|
676
673
|
[[package]]
|
@@ -840,7 +837,7 @@ dependencies = [
|
|
840
837
|
"magnus",
|
841
838
|
"mimalloc",
|
842
839
|
"num",
|
843
|
-
"parquet
|
840
|
+
"parquet 55.1.0",
|
844
841
|
"rand",
|
845
842
|
"rb-sys",
|
846
843
|
"rb-sys-env 0.2.2",
|
@@ -851,9 +848,8 @@ dependencies = [
|
|
851
848
|
|
852
849
|
[[package]]
|
853
850
|
name = "parquet"
|
854
|
-
version = "
|
855
|
-
source = "
|
856
|
-
checksum = "761c44d824fe83106e0600d2510c07bf4159a4985bf0569b513ea4288dc1b4fb"
|
851
|
+
version = "55.1.0"
|
852
|
+
source = "git+https://github.com/njaremko/arrow-rs?branch=nathan%2Ffix-reading-int32-timestamp-records#f791b78a67cb5d9a0b4ec0fcab80780dcb61c346"
|
857
853
|
dependencies = [
|
858
854
|
"ahash",
|
859
855
|
"arrow-array",
|
@@ -879,9 +875,8 @@ dependencies = [
|
|
879
875
|
"simdutf8",
|
880
876
|
"snap",
|
881
877
|
"thrift",
|
882
|
-
"twox-hash",
|
878
|
+
"twox-hash 2.1.0",
|
883
879
|
"zstd",
|
884
|
-
"zstd-sys",
|
885
880
|
]
|
886
881
|
|
887
882
|
[[package]]
|
@@ -1055,7 +1050,7 @@ version = "0.38.44"
|
|
1055
1050
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1056
1051
|
checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
|
1057
1052
|
dependencies = [
|
1058
|
-
"bitflags
|
1053
|
+
"bitflags",
|
1059
1054
|
"errno",
|
1060
1055
|
"libc",
|
1061
1056
|
"linux-raw-sys",
|
@@ -1223,6 +1218,12 @@ dependencies = [
|
|
1223
1218
|
"static_assertions",
|
1224
1219
|
]
|
1225
1220
|
|
1221
|
+
[[package]]
|
1222
|
+
name = "twox-hash"
|
1223
|
+
version = "2.1.0"
|
1224
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1225
|
+
checksum = "e7b17f197b3050ba473acf9181f7b1d3b66d1cf7356c6cc57886662276e65908"
|
1226
|
+
|
1226
1227
|
[[package]]
|
1227
1228
|
name = "unicode-ident"
|
1228
1229
|
version = "1.0.17"
|
@@ -1402,7 +1403,7 @@ version = "0.33.0"
|
|
1402
1403
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1403
1404
|
checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c"
|
1404
1405
|
dependencies = [
|
1405
|
-
"bitflags
|
1406
|
+
"bitflags",
|
1406
1407
|
]
|
1407
1408
|
|
1408
1409
|
[[package]]
|
@@ -1446,6 +1447,12 @@ dependencies = [
|
|
1446
1447
|
"syn",
|
1447
1448
|
]
|
1448
1449
|
|
1450
|
+
[[package]]
|
1451
|
+
name = "zlib-rs"
|
1452
|
+
version = "0.4.2"
|
1453
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1454
|
+
checksum = "8b20717f0917c908dc63de2e44e97f1e6b126ca58d0e391cee86d504eb8fbd05"
|
1455
|
+
|
1449
1456
|
[[package]]
|
1450
1457
|
name = "zstd"
|
1451
1458
|
version = "0.13.3"
|
data/ext/parquet/Cargo.toml
CHANGED
@@ -11,14 +11,14 @@ rb-sys-env = "^0.2"
|
|
11
11
|
|
12
12
|
[dependencies]
|
13
13
|
ahash = "0.8"
|
14
|
-
arrow-array = "
|
15
|
-
arrow-schema = "
|
14
|
+
arrow-array = { git = "https://github.com/njaremko/arrow-rs", branch = "nathan/fix-reading-int32-timestamp-records" }
|
15
|
+
arrow-schema = { git = "https://github.com/njaremko/arrow-rs", branch = "nathan/fix-reading-int32-timestamp-records" }
|
16
16
|
bytes = "^1.9"
|
17
17
|
either = "1.9"
|
18
18
|
itertools = "^0.14"
|
19
19
|
jiff = "0.2"
|
20
20
|
magnus = { version = "0.7", features = ["rb-sys"] }
|
21
|
-
parquet = {
|
21
|
+
parquet = { git = "https://github.com/njaremko/arrow-rs", branch = "nathan/fix-reading-int32-timestamp-records", features = ["json"] }
|
22
22
|
rand = "0.9"
|
23
23
|
rb-sys = "^0.9"
|
24
24
|
simdutf8 = "0.1.5"
|
@@ -238,10 +238,32 @@ impl TryIntoValue for ParquetField {
|
|
238
238
|
format_decimal_with_i32_scale(unscaled, scale)
|
239
239
|
}
|
240
240
|
Decimal::Bytes { value, scale, .. } => {
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
241
|
+
match value.len() {
|
242
|
+
4 => {
|
243
|
+
// value is a byte array containing the bytes for an i32 value in big endian order
|
244
|
+
let casted = value.as_bytes()[..4].try_into()?;
|
245
|
+
let unscaled = i32::from_be_bytes(casted);
|
246
|
+
format_decimal_with_i32_scale(unscaled, scale)
|
247
|
+
}
|
248
|
+
8 => {
|
249
|
+
// value is a byte array containing the bytes for an i64 value in big endian order
|
250
|
+
let casted = value.as_bytes()[..8].try_into()?;
|
251
|
+
let unscaled = i64::from_be_bytes(casted);
|
252
|
+
format_decimal_with_i32_scale(unscaled, scale)
|
253
|
+
}
|
254
|
+
16 => {
|
255
|
+
// value is a byte array containing the bytes for an i128 value in big endian order
|
256
|
+
let casted = value.as_bytes()[..16].try_into()?;
|
257
|
+
let unscaled = i128::from_be_bytes(casted);
|
258
|
+
format_decimal_with_i32_scale(unscaled, scale)
|
259
|
+
}
|
260
|
+
_ => {
|
261
|
+
unimplemented!(
|
262
|
+
"Unsupported decimal byte array size: {}",
|
263
|
+
value.len()
|
264
|
+
);
|
265
|
+
}
|
266
|
+
}
|
245
267
|
}
|
246
268
|
};
|
247
269
|
|
@@ -121,7 +121,8 @@ pub fn parse_legacy_schema(
|
|
121
121
|
ruby.exception_type_error(),
|
122
122
|
"Schema must be an array of field definitions or nil",
|
123
123
|
)
|
124
|
-
})
|
124
|
+
})?
|
125
|
+
.is_empty())
|
125
126
|
{
|
126
127
|
// If schema is nil or an empty array, we'll handle this in the caller
|
127
128
|
return Ok(Vec::new());
|
@@ -206,101 +207,39 @@ pub fn parse_legacy_schema(
|
|
206
207
|
|
207
208
|
// Handle decimal type with precision and scale
|
208
209
|
let mut type_result = PST::try_convert(type_str)?;
|
209
|
-
|
210
|
+
|
210
211
|
// If it's a decimal type and we have precision and scale, override the type
|
211
212
|
if let PST::Primitive(PrimitiveType::Decimal128(_, _)) = type_result {
|
212
|
-
|
213
|
-
let val: u8 = 18;
|
214
|
-
val.into_value_with(ruby)
|
215
|
-
});
|
216
|
-
let scale_value = scale.unwrap_or_else(|| {
|
217
|
-
let val: i8 = 2;
|
218
|
-
val.into_value_with(ruby)
|
219
|
-
});
|
220
|
-
|
221
|
-
let precision_u8 = u8::try_convert(precision_value).map_err(|_| {
|
222
|
-
MagnusError::new(
|
223
|
-
ruby.exception_type_error(),
|
224
|
-
"Invalid precision value for decimal type, expected a positive integer".to_string(),
|
225
|
-
)
|
226
|
-
})?;
|
227
|
-
|
228
|
-
// Validate precision is in a valid range
|
229
|
-
if precision_u8 < 1 {
|
230
|
-
return Err(MagnusError::new(
|
231
|
-
ruby.exception_arg_error(),
|
232
|
-
format!(
|
233
|
-
"Precision for decimal type must be at least 1, got {}",
|
234
|
-
precision_u8
|
235
|
-
),
|
236
|
-
));
|
237
|
-
}
|
238
|
-
|
239
|
-
if precision_u8 > 38 {
|
240
|
-
return Err(MagnusError::new(
|
241
|
-
ruby.exception_arg_error(),
|
242
|
-
format!(
|
243
|
-
"Precision for decimal type cannot exceed 38, got {}",
|
244
|
-
precision_u8
|
245
|
-
),
|
246
|
-
));
|
247
|
-
}
|
248
|
-
|
249
|
-
let scale_i8 = i8::try_convert(scale_value).map_err(|_| {
|
250
|
-
MagnusError::new(
|
251
|
-
ruby.exception_type_error(),
|
252
|
-
"Invalid scale value for decimal type, expected an integer".to_string(),
|
253
|
-
)
|
254
|
-
})?;
|
255
|
-
|
256
|
-
// Validate scale is in a valid range relative to precision
|
257
|
-
if scale_i8 < 0 {
|
258
|
-
return Err(MagnusError::new(
|
259
|
-
ruby.exception_arg_error(),
|
260
|
-
format!(
|
261
|
-
"Scale for decimal type cannot be negative, got {}",
|
262
|
-
scale_i8
|
263
|
-
),
|
264
|
-
));
|
265
|
-
}
|
266
|
-
|
267
|
-
if scale_i8 as u8 > precision_u8 {
|
268
|
-
return Err(MagnusError::new(
|
269
|
-
ruby.exception_arg_error(),
|
270
|
-
format!(
|
271
|
-
"Scale ({}) cannot be larger than precision ({}) for decimal type",
|
272
|
-
scale_i8, precision_u8
|
273
|
-
),
|
274
|
-
));
|
275
|
-
}
|
276
|
-
|
277
|
-
type_result = PST::Primitive(PrimitiveType::Decimal128(precision_u8, scale_i8));
|
213
|
+
// Do nothing
|
278
214
|
} else if let Some(type_name) = parse_string_or_symbol(ruby, type_str)? {
|
279
215
|
if type_name == "decimal" {
|
280
216
|
let precision_value = precision.unwrap_or_else(|| {
|
281
|
-
let val: u8 =
|
217
|
+
let val: u8 = 38;
|
282
218
|
val.into_value_with(ruby)
|
283
219
|
});
|
220
|
+
|
284
221
|
let scale_value = scale.unwrap_or_else(|| {
|
285
|
-
let val: i8 =
|
222
|
+
let val: i8 = 0;
|
286
223
|
val.into_value_with(ruby)
|
287
224
|
});
|
288
|
-
|
225
|
+
|
289
226
|
let precision_u8 = u8::try_convert(precision_value).map_err(|_| {
|
290
227
|
MagnusError::new(
|
291
228
|
ruby.exception_type_error(),
|
292
229
|
"Invalid precision value for decimal type, expected a positive integer".to_string(),
|
293
230
|
)
|
294
231
|
})?;
|
295
|
-
|
232
|
+
|
296
233
|
let scale_i8 = i8::try_convert(scale_value).map_err(|_| {
|
297
234
|
MagnusError::new(
|
298
235
|
ruby.exception_type_error(),
|
299
|
-
"Invalid scale value for decimal type, expected an integer"
|
236
|
+
"Invalid scale value for decimal type, expected an integer"
|
237
|
+
.to_string(),
|
300
238
|
)
|
301
239
|
})?;
|
302
|
-
|
303
|
-
type_result =
|
240
|
+
|
241
|
+
type_result =
|
242
|
+
PST::Primitive(PrimitiveType::Decimal128(precision_u8, scale_i8));
|
304
243
|
}
|
305
244
|
}
|
306
245
|
|
data/lib/parquet/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: parquet
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nathan Jaremko
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2025-
|
11
|
+
date: 2025-05-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rb_sys
|