iceberg 0.11.1 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +16 -0
- data/Cargo.lock +838 -455
- data/Cargo.toml +7 -0
- data/NOTICE.txt +2 -2
- data/README.md +32 -12
- data/ext/iceberg/Cargo.toml +17 -13
- data/ext/iceberg/src/arrow.rs +22 -7
- data/ext/iceberg/src/batch.rs +174 -0
- data/ext/iceberg/src/capsule.rs +24 -0
- data/ext/iceberg/src/catalog.rs +223 -49
- data/ext/iceberg/src/encryption.rs +32 -0
- data/ext/iceberg/src/error.rs +50 -17
- data/ext/iceberg/src/lib.rs +265 -41
- data/ext/iceberg/src/partitioning.rs +102 -0
- data/ext/iceberg/src/result.rs +201 -0
- data/ext/iceberg/src/ruby.rs +51 -0
- data/ext/iceberg/src/runtime.rs +5 -1
- data/ext/iceberg/src/scan.rs +116 -33
- data/ext/iceberg/src/schema.rs +305 -0
- data/ext/iceberg/src/snapshot.rs +85 -0
- data/ext/iceberg/src/sorting.rs +122 -0
- data/ext/iceberg/src/statistics.rs +71 -0
- data/ext/iceberg/src/table.rs +255 -263
- data/ext/iceberg/src/utils.rs +222 -164
- data/lib/iceberg/catalog.rb +54 -35
- data/lib/iceberg/glue_catalog.rb +5 -2
- data/lib/iceberg/memory_catalog.rb +5 -2
- data/lib/iceberg/rest_catalog.rb +5 -2
- data/lib/iceberg/result.rb +22 -0
- data/lib/iceberg/s3_tables_catalog.rb +5 -2
- data/lib/iceberg/sql_catalog.rb +5 -2
- data/lib/iceberg/table.rb +87 -21
- data/lib/iceberg/table_definition.rb +49 -7
- data/lib/iceberg/table_scan.rb +14 -0
- data/lib/iceberg/transforms.rb +64 -0
- data/lib/iceberg/types.rb +137 -0
- data/lib/iceberg/version.rb +1 -1
- data/lib/iceberg.rb +10 -5
- metadata +15 -3
- data/lib/iceberg/schema.rb +0 -10
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
use iceberg::spec::{NullOrder, SortDirection, SortField, SortOrder, Transform};
|
|
2
|
+
use magnus::{IntoValue, RArray, RHash, Ruby, TryConvert, Value, value::ReprValue};
|
|
3
|
+
|
|
4
|
+
use crate::RbResult;
|
|
5
|
+
use crate::error::to_rb_err;
|
|
6
|
+
use crate::utils::{Wrap, rb_transform};
|
|
7
|
+
|
|
8
|
+
#[magnus::wrap(class = "Iceberg::SortOrder")]
|
|
9
|
+
pub struct RbSortOrder {
|
|
10
|
+
pub(crate) order: SortOrder,
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
#[magnus::wrap(class = "Iceberg::SortField")]
|
|
14
|
+
pub struct RbSortField {
|
|
15
|
+
pub(crate) field: SortField,
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
impl RbSortOrder {
|
|
19
|
+
pub fn new(args: &[Value]) -> RbResult<Self> {
|
|
20
|
+
let mut fields = Vec::new();
|
|
21
|
+
for v in args {
|
|
22
|
+
fields.push(<&RbSortField>::try_convert(*v)?.field.clone());
|
|
23
|
+
}
|
|
24
|
+
let order = SortOrder::builder()
|
|
25
|
+
.with_fields(fields)
|
|
26
|
+
.build_unbound()
|
|
27
|
+
.map_err(to_rb_err)?;
|
|
28
|
+
Ok(Self { order })
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
pub fn order_id(&self) -> i64 {
|
|
32
|
+
self.order.order_id
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
pub fn fields(ruby: &Ruby, rb_self: &Self) -> RArray {
|
|
36
|
+
ruby.ary_from_iter(
|
|
37
|
+
rb_self
|
|
38
|
+
.order
|
|
39
|
+
.fields
|
|
40
|
+
.iter()
|
|
41
|
+
.map(|v| RbSortField { field: v.clone() }),
|
|
42
|
+
)
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
pub fn eq(&self, other: &Self) -> bool {
|
|
46
|
+
self.order == other.order
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
pub fn inspect(ruby: &Ruby, rb_self: &Self) -> String {
|
|
50
|
+
format!(
|
|
51
|
+
"#<Iceberg::SortOrder order_id={}, fields={}>",
|
|
52
|
+
rb_self.order_id().into_value_with(ruby).inspect(),
|
|
53
|
+
Self::fields(ruby, rb_self).inspect(),
|
|
54
|
+
)
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
impl RbSortField {
|
|
59
|
+
pub fn new(ruby: &Ruby, ob: RHash) -> RbResult<Self> {
|
|
60
|
+
let transform = ob
|
|
61
|
+
.aref::<_, Wrap<Transform>>(ruby.to_symbol("transform"))?
|
|
62
|
+
.0;
|
|
63
|
+
|
|
64
|
+
let direction = ob
|
|
65
|
+
.aref::<_, Option<Wrap<SortDirection>>>(ruby.to_symbol("direction"))?
|
|
66
|
+
.map(|v| v.0)
|
|
67
|
+
.unwrap_or(SortDirection::Ascending);
|
|
68
|
+
|
|
69
|
+
let null_order = ob
|
|
70
|
+
.aref::<_, Option<Wrap<NullOrder>>>(ruby.to_symbol("null_order"))?
|
|
71
|
+
.map(|v| v.0)
|
|
72
|
+
.unwrap_or(if direction == SortDirection::Ascending {
|
|
73
|
+
NullOrder::First
|
|
74
|
+
} else {
|
|
75
|
+
NullOrder::Last
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
let field = SortField::builder()
|
|
79
|
+
.source_id(ob.aref(ruby.to_symbol("source_id"))?)
|
|
80
|
+
.transform(transform)
|
|
81
|
+
.direction(direction)
|
|
82
|
+
.null_order(null_order)
|
|
83
|
+
.build();
|
|
84
|
+
Ok(Self { field })
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
pub fn source_id(&self) -> i32 {
|
|
88
|
+
self.field.source_id
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
pub fn transform(&self) -> RbResult<Value> {
|
|
92
|
+
rb_transform(&self.field.transform)
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
pub fn direction(&self) -> &str {
|
|
96
|
+
match self.field.direction {
|
|
97
|
+
SortDirection::Ascending => "asc",
|
|
98
|
+
SortDirection::Descending => "desc",
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
pub fn null_order(&self) -> &str {
|
|
103
|
+
match self.field.null_order {
|
|
104
|
+
NullOrder::First => "first",
|
|
105
|
+
NullOrder::Last => "last",
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
pub fn eq(&self, other: &Self) -> bool {
|
|
110
|
+
self.field == other.field
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
pub fn inspect(ruby: &Ruby, rb_self: &Self) -> RbResult<String> {
|
|
114
|
+
Ok(format!(
|
|
115
|
+
"#<Iceberg::SortField source_id={}, transform={}, direction={}, null_order={}>",
|
|
116
|
+
rb_self.source_id().into_value_with(ruby).inspect(),
|
|
117
|
+
rb_self.transform()?.into_value_with(ruby).inspect(),
|
|
118
|
+
rb_self.direction().into_value_with(ruby).inspect(),
|
|
119
|
+
rb_self.null_order().into_value_with(ruby).inspect(),
|
|
120
|
+
))
|
|
121
|
+
}
|
|
122
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
use iceberg::spec::{PartitionStatisticsFile, StatisticsFile};
|
|
2
|
+
use magnus::{IntoValue, Ruby, value::ReprValue};
|
|
3
|
+
|
|
4
|
+
#[magnus::wrap(class = "Iceberg::StatisticsFile")]
|
|
5
|
+
pub struct RbStatisticsFile {
|
|
6
|
+
pub(crate) file: StatisticsFile,
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
#[magnus::wrap(class = "Iceberg::PartitionStatisticsFile")]
|
|
10
|
+
pub struct RbPartitionStatisticsFile {
|
|
11
|
+
pub(crate) file: PartitionStatisticsFile,
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
impl RbStatisticsFile {
|
|
15
|
+
pub fn snapshot_id(&self) -> i64 {
|
|
16
|
+
self.file.snapshot_id
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
pub fn statistics_path(&self) -> &str {
|
|
20
|
+
&self.file.statistics_path
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
pub fn file_size_in_bytes(&self) -> i64 {
|
|
24
|
+
self.file.file_size_in_bytes
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
pub fn file_footer_size_in_bytes(&self) -> i64 {
|
|
28
|
+
self.file.file_footer_size_in_bytes
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
pub fn key_metadata(&self) -> Option<&str> {
|
|
32
|
+
self.file.key_metadata.as_deref()
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
pub fn inspect(ruby: &Ruby, rb_self: &Self) -> String {
|
|
36
|
+
format!(
|
|
37
|
+
"#<Iceberg::StatisticsFile snapshot_id={}, statistics_path={}, file_size_in_bytes={}, file_footer_size_in_bytes={}, key_metadata={}>",
|
|
38
|
+
rb_self.snapshot_id().into_value_with(ruby).inspect(),
|
|
39
|
+
rb_self.statistics_path().into_value_with(ruby).inspect(),
|
|
40
|
+
rb_self.file_size_in_bytes().into_value_with(ruby).inspect(),
|
|
41
|
+
rb_self
|
|
42
|
+
.file_footer_size_in_bytes()
|
|
43
|
+
.into_value_with(ruby)
|
|
44
|
+
.inspect(),
|
|
45
|
+
rb_self.key_metadata().into_value_with(ruby).inspect(),
|
|
46
|
+
)
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
impl RbPartitionStatisticsFile {
|
|
51
|
+
pub fn snapshot_id(&self) -> i64 {
|
|
52
|
+
self.file.snapshot_id
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
pub fn statistics_path(&self) -> &str {
|
|
56
|
+
&self.file.statistics_path
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
pub fn file_size_in_bytes(&self) -> i64 {
|
|
60
|
+
self.file.file_size_in_bytes
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
pub fn inspect(ruby: &Ruby, rb_self: &Self) -> String {
|
|
64
|
+
format!(
|
|
65
|
+
"#<Iceberg::PartitionStatisticsFile snapshot_id={}, statistics_path={}, file_size_in_bytes={}>",
|
|
66
|
+
rb_self.snapshot_id().into_value_with(ruby).inspect(),
|
|
67
|
+
rb_self.statistics_path().into_value_with(ruby).inspect(),
|
|
68
|
+
rb_self.file_size_in_bytes().into_value_with(ruby).inspect(),
|
|
69
|
+
)
|
|
70
|
+
}
|
|
71
|
+
}
|