iceberg 0.11.2 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +17 -0
- data/Cargo.lock +1180 -495
- data/Cargo.toml +7 -0
- data/NOTICE.txt +2 -2
- data/README.md +60 -15
- data/ext/iceberg/Cargo.toml +18 -15
- data/ext/iceberg/src/arrow.rs +22 -7
- data/ext/iceberg/src/batch.rs +174 -0
- data/ext/iceberg/src/capsule.rs +24 -0
- data/ext/iceberg/src/catalog.rs +157 -33
- data/ext/iceberg/src/encryption.rs +32 -0
- data/ext/iceberg/src/error.rs +63 -23
- data/ext/iceberg/src/lib.rs +264 -41
- data/ext/iceberg/src/partitioning.rs +102 -0
- data/ext/iceberg/src/result.rs +201 -0
- data/ext/iceberg/src/runtime.rs +5 -1
- data/ext/iceberg/src/scan.rs +106 -26
- data/ext/iceberg/src/schema.rs +306 -0
- data/ext/iceberg/src/snapshot.rs +85 -0
- data/ext/iceberg/src/sorting.rs +122 -0
- data/ext/iceberg/src/statistics.rs +71 -0
- data/ext/iceberg/src/table.rs +213 -265
- data/ext/iceberg/src/utils.rs +223 -164
- data/lib/iceberg/catalog.rb +59 -35
- data/lib/iceberg/glue_catalog.rb +5 -2
- data/lib/iceberg/memory_catalog.rb +5 -2
- data/lib/iceberg/rest_catalog.rb +5 -2
- data/lib/iceberg/result.rb +22 -0
- data/lib/iceberg/s3_tables_catalog.rb +5 -2
- data/lib/iceberg/sql_catalog.rb +5 -2
- data/lib/iceberg/table.rb +87 -21
- data/lib/iceberg/table_definition.rb +49 -7
- data/lib/iceberg/table_scan.rb +14 -0
- data/lib/iceberg/transforms.rb +64 -0
- data/lib/iceberg/types.rb +137 -0
- data/lib/iceberg/version.rb +1 -1
- data/lib/iceberg.rb +10 -5
- metadata +14 -3
- data/lib/iceberg/schema.rb +0 -10
data/ext/iceberg/src/utils.rs
CHANGED
|
@@ -1,15 +1,24 @@
|
|
|
1
|
+
use std::sync::Arc;
|
|
2
|
+
|
|
1
3
|
use iceberg::spec::{
|
|
2
|
-
EncryptedKey, Literal,
|
|
3
|
-
|
|
4
|
+
EncryptedKey, Literal, MetadataLog, NestedField, NullOrder, PartitionSpec,
|
|
5
|
+
PartitionStatisticsFile, PrimitiveLiteral, PrimitiveType, Schema, Snapshot, SnapshotLog,
|
|
6
|
+
SortDirection, SortOrder, StatisticsFile, Transform, Type,
|
|
4
7
|
};
|
|
5
8
|
use iceberg::{NamespaceIdent, TableIdent};
|
|
6
9
|
use magnus::{
|
|
7
|
-
Error as RbErr, IntoValue,
|
|
8
|
-
|
|
10
|
+
Error as RbErr, IntoValue, RClass, RModule, RString, Ruby, TryConvert, Value, prelude::*,
|
|
11
|
+
value::Lazy,
|
|
9
12
|
};
|
|
10
13
|
|
|
11
14
|
use crate::RbResult;
|
|
12
|
-
use crate::
|
|
15
|
+
use crate::encryption::RbEncryptedKey;
|
|
16
|
+
use crate::error::{to_rb_err, todo_error};
|
|
17
|
+
use crate::partitioning::RbPartitionSpec;
|
|
18
|
+
use crate::schema::{RbNestedField, RbSchema};
|
|
19
|
+
use crate::snapshot::{RbMetadataLogEntry, RbSnapshot, RbSnapshotLogEntry};
|
|
20
|
+
use crate::sorting::RbSortOrder;
|
|
21
|
+
use crate::statistics::{RbPartitionStatisticsFile, RbStatisticsFile};
|
|
13
22
|
|
|
14
23
|
pub struct Wrap<T>(pub T);
|
|
15
24
|
|
|
@@ -42,88 +51,80 @@ impl TryConvert for Wrap<TableIdent> {
|
|
|
42
51
|
}
|
|
43
52
|
}
|
|
44
53
|
|
|
45
|
-
impl TryConvert for Wrap<
|
|
54
|
+
impl TryConvert for Wrap<Transform> {
|
|
46
55
|
fn try_convert(ob: Value) -> RbResult<Self> {
|
|
47
|
-
let
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
"double" => Type::Primitive(PrimitiveType::Double),
|
|
60
|
-
// TODO PrimitiveType::Decimal
|
|
61
|
-
"date" => Type::Primitive(PrimitiveType::Date),
|
|
62
|
-
"time" => Type::Primitive(PrimitiveType::Time),
|
|
63
|
-
"timestamp" => Type::Primitive(PrimitiveType::Timestamp),
|
|
64
|
-
"timestamptz" => Type::Primitive(PrimitiveType::Timestamptz),
|
|
65
|
-
"timestamp_ns" => Type::Primitive(PrimitiveType::TimestampNs),
|
|
66
|
-
"timestamptz_ns" => Type::Primitive(PrimitiveType::TimestamptzNs),
|
|
67
|
-
"string" => Type::Primitive(PrimitiveType::String),
|
|
68
|
-
"uuid" => Type::Primitive(PrimitiveType::Uuid),
|
|
69
|
-
// TODO PrimitiveType::Fixed
|
|
70
|
-
"binary" => Type::Primitive(PrimitiveType::Binary),
|
|
71
|
-
_ => {
|
|
72
|
-
return Err(RbErr::new(
|
|
73
|
-
ruby.exception_arg_error(),
|
|
74
|
-
format!("Type not supported: {}", s),
|
|
75
|
-
));
|
|
76
|
-
}
|
|
56
|
+
let v = if let Ok(s) = RString::try_convert(ob) {
|
|
57
|
+
match unsafe { s.as_str() }? {
|
|
58
|
+
"identity" => Transform::Identity,
|
|
59
|
+
"year" => Transform::Year,
|
|
60
|
+
"month" => Transform::Month,
|
|
61
|
+
"day" => Transform::Day,
|
|
62
|
+
"hour" => Transform::Hour,
|
|
63
|
+
_ => {
|
|
64
|
+
return Err(RbErr::new(
|
|
65
|
+
Ruby::get_with(ob).exception_arg_error(),
|
|
66
|
+
"Unsupported transform",
|
|
67
|
+
));
|
|
77
68
|
}
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
}
|
|
69
|
+
}
|
|
70
|
+
} else {
|
|
71
|
+
match &*unsafe { ob.classname() } {
|
|
72
|
+
"Iceberg::IdentityTransform" => Transform::Identity,
|
|
73
|
+
"Iceberg::BucketTransform" => Transform::Bucket(ob.funcall("num_buckets", ())?),
|
|
74
|
+
"Iceberg::TruncateTransform" => Transform::Truncate(ob.funcall("width", ())?),
|
|
75
|
+
"Iceberg::YearTransform" => Transform::Year,
|
|
76
|
+
"Iceberg::MonthTransform" => Transform::Month,
|
|
77
|
+
"Iceberg::DayTransform" => Transform::Day,
|
|
78
|
+
"Iceberg::HourTransform" => Transform::Hour,
|
|
79
|
+
"Iceberg::VoidTransform" => Transform::Void,
|
|
80
|
+
"Iceberg::UnknownTransform" => Transform::Unknown,
|
|
81
|
+
_ => {
|
|
82
|
+
return Err(RbErr::new(
|
|
83
|
+
Ruby::get_with(ob).exception_arg_error(),
|
|
84
|
+
format!("Transform not supported: {}", ob),
|
|
85
|
+
));
|
|
96
86
|
}
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
87
|
+
}
|
|
88
|
+
};
|
|
89
|
+
Ok(Wrap(v))
|
|
90
|
+
}
|
|
91
|
+
}
|
|
101
92
|
|
|
102
|
-
|
|
103
|
-
|
|
93
|
+
impl TryConvert for Wrap<SortDirection> {
|
|
94
|
+
fn try_convert(ob: Value) -> RbResult<Self> {
|
|
95
|
+
let s = RString::try_convert(ob)?;
|
|
96
|
+
let v = match unsafe { s.as_str() }? {
|
|
97
|
+
"asc" => SortDirection::Ascending,
|
|
98
|
+
"desc" => SortDirection::Descending,
|
|
99
|
+
_ => {
|
|
100
|
+
return Err(RbErr::new(
|
|
101
|
+
Ruby::get_with(ob).exception_arg_error(),
|
|
102
|
+
"Unsupported sort direction",
|
|
103
|
+
));
|
|
104
|
+
}
|
|
105
|
+
};
|
|
106
|
+
Ok(Wrap(v))
|
|
107
|
+
}
|
|
108
|
+
}
|
|
104
109
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
.with_fields(fields)
|
|
120
|
-
.build()
|
|
121
|
-
.map_err(to_rb_err)?;
|
|
122
|
-
Ok(Wrap(schema))
|
|
110
|
+
impl TryConvert for Wrap<NullOrder> {
|
|
111
|
+
fn try_convert(ob: Value) -> RbResult<Self> {
|
|
112
|
+
let s = RString::try_convert(ob)?;
|
|
113
|
+
let v = match unsafe { s.as_str() }? {
|
|
114
|
+
"first" => NullOrder::First,
|
|
115
|
+
"last" => NullOrder::Last,
|
|
116
|
+
_ => {
|
|
117
|
+
return Err(RbErr::new(
|
|
118
|
+
Ruby::get_with(ob).exception_arg_error(),
|
|
119
|
+
"Unsupported null order",
|
|
120
|
+
));
|
|
121
|
+
}
|
|
122
|
+
};
|
|
123
|
+
Ok(Wrap(v))
|
|
123
124
|
}
|
|
124
125
|
}
|
|
125
126
|
|
|
126
|
-
fn default_value(ob: Value, field_type: &Type) -> RbResult<Option<Literal>> {
|
|
127
|
+
pub fn default_value(ob: Value, field_type: &Type) -> RbResult<Option<Literal>> {
|
|
127
128
|
if ob.is_nil() {
|
|
128
129
|
return Ok(None);
|
|
129
130
|
}
|
|
@@ -136,114 +137,116 @@ fn default_value(ob: Value, field_type: &Type) -> RbResult<Option<Literal>> {
|
|
|
136
137
|
PrimitiveType::Long => PrimitiveLiteral::Long(i64::try_convert(ob)?),
|
|
137
138
|
PrimitiveType::Float => PrimitiveLiteral::Float(f32::try_convert(ob)?.into()),
|
|
138
139
|
PrimitiveType::Double => PrimitiveLiteral::Double(f64::try_convert(ob)?.into()),
|
|
140
|
+
PrimitiveType::Decimal {
|
|
141
|
+
precision: _,
|
|
142
|
+
scale: _,
|
|
143
|
+
} => return Err(todo_error(field_type)),
|
|
144
|
+
PrimitiveType::Date => return Err(todo_error(field_type)),
|
|
145
|
+
PrimitiveType::Time => return Err(todo_error(field_type)),
|
|
146
|
+
PrimitiveType::Timestamp => return Err(todo_error(field_type)),
|
|
147
|
+
PrimitiveType::Timestamptz => return Err(todo_error(field_type)),
|
|
148
|
+
PrimitiveType::TimestampNs => return Err(todo_error(field_type)),
|
|
149
|
+
PrimitiveType::TimestamptzNs => return Err(todo_error(field_type)),
|
|
139
150
|
PrimitiveType::String => PrimitiveLiteral::String(String::try_convert(ob)?),
|
|
140
|
-
|
|
151
|
+
PrimitiveType::Uuid => return Err(todo_error(field_type)),
|
|
152
|
+
PrimitiveType::Fixed(_) => return Err(todo_error(field_type)),
|
|
153
|
+
PrimitiveType::Binary => {
|
|
154
|
+
let s = RString::try_convert(ob)?;
|
|
155
|
+
PrimitiveLiteral::Binary(unsafe { s.as_slice() }.to_vec())
|
|
156
|
+
}
|
|
141
157
|
};
|
|
142
158
|
Literal::Primitive(pl)
|
|
143
159
|
}
|
|
144
|
-
_ =>
|
|
160
|
+
Type::Struct(_) => return Err(todo_error(field_type)),
|
|
161
|
+
Type::List(_) => return Err(todo_error(field_type)),
|
|
162
|
+
Type::Map(_) => return Err(todo_error(field_type)),
|
|
163
|
+
Type::Variant(_) => return Err(todo_error(field_type)),
|
|
145
164
|
};
|
|
146
165
|
Ok(Some(lit))
|
|
147
166
|
}
|
|
148
167
|
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
let field_type = match &*f.field_type {
|
|
157
|
-
Type::Primitive(ty) => match ty {
|
|
158
|
-
PrimitiveType::Boolean => "boolean",
|
|
159
|
-
PrimitiveType::Int => "int",
|
|
160
|
-
PrimitiveType::Long => "long",
|
|
161
|
-
PrimitiveType::Float => "float",
|
|
162
|
-
PrimitiveType::Double => "double",
|
|
163
|
-
PrimitiveType::Decimal {
|
|
164
|
-
precision: _,
|
|
165
|
-
scale: _,
|
|
166
|
-
} => todo!(),
|
|
167
|
-
PrimitiveType::Date => "date",
|
|
168
|
-
PrimitiveType::Time => "time",
|
|
169
|
-
PrimitiveType::Timestamp => "timestamp",
|
|
170
|
-
PrimitiveType::Timestamptz => "timestamptz",
|
|
171
|
-
PrimitiveType::TimestampNs => "timestamp_ns",
|
|
172
|
-
PrimitiveType::TimestamptzNs => "timestamptz_ns",
|
|
173
|
-
PrimitiveType::String => "string",
|
|
174
|
-
PrimitiveType::Uuid => "uuid",
|
|
175
|
-
PrimitiveType::Binary => "binary",
|
|
176
|
-
PrimitiveType::Fixed(_) => todo!(),
|
|
177
|
-
},
|
|
178
|
-
_ => todo!(),
|
|
179
|
-
};
|
|
180
|
-
field.aset(ruby.to_symbol("type"), field_type)?;
|
|
181
|
-
|
|
182
|
-
field.aset(ruby.to_symbol("required"), f.required)?;
|
|
183
|
-
|
|
184
|
-
let initial_default = f.initial_default.as_ref().map(|v| rb_literal(ruby, v));
|
|
185
|
-
field.aset(ruby.to_symbol("initial_default"), initial_default)?;
|
|
186
|
-
|
|
187
|
-
let write_default = f.write_default.as_ref().map(|v| rb_literal(ruby, v));
|
|
188
|
-
field.aset(ruby.to_symbol("write_default"), write_default)?;
|
|
168
|
+
impl From<&Arc<Schema>> for RbSchema {
|
|
169
|
+
fn from(schema: &Arc<Schema>) -> Self {
|
|
170
|
+
Self {
|
|
171
|
+
schema: (**schema).clone(),
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
}
|
|
189
175
|
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
176
|
+
impl From<&Arc<NestedField>> for RbNestedField {
|
|
177
|
+
fn from(field: &Arc<NestedField>) -> Self {
|
|
178
|
+
Self {
|
|
179
|
+
field: field.clone(),
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
}
|
|
194
183
|
|
|
195
|
-
|
|
184
|
+
impl From<&Arc<Snapshot>> for RbSnapshot {
|
|
185
|
+
fn from(snapshot: &Arc<Snapshot>) -> Self {
|
|
186
|
+
Self {
|
|
187
|
+
snapshot: (**snapshot).clone(),
|
|
188
|
+
}
|
|
196
189
|
}
|
|
197
|
-
|
|
190
|
+
}
|
|
198
191
|
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
192
|
+
impl From<&SnapshotLog> for RbSnapshotLogEntry {
|
|
193
|
+
fn from(snapshot_log: &SnapshotLog) -> Self {
|
|
194
|
+
Self {
|
|
195
|
+
log: snapshot_log.clone(),
|
|
196
|
+
}
|
|
197
|
+
}
|
|
205
198
|
}
|
|
206
199
|
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
)?;
|
|
214
|
-
rb_snapshot.aset(
|
|
215
|
-
ruby.to_symbol("sequence_number"),
|
|
216
|
-
snapshot.sequence_number(),
|
|
217
|
-
)?;
|
|
218
|
-
rb_snapshot.aset(ruby.to_symbol("manifest_list"), snapshot.manifest_list())?;
|
|
219
|
-
rb_snapshot.aset(ruby.to_symbol("schema_id"), snapshot.schema_id())?;
|
|
220
|
-
Ok(rb_snapshot.as_value())
|
|
200
|
+
impl From<&MetadataLog> for RbMetadataLogEntry {
|
|
201
|
+
fn from(metadata_log: &MetadataLog) -> Self {
|
|
202
|
+
Self {
|
|
203
|
+
log: metadata_log.clone(),
|
|
204
|
+
}
|
|
205
|
+
}
|
|
221
206
|
}
|
|
222
207
|
|
|
223
|
-
|
|
224
|
-
|
|
208
|
+
impl From<&Arc<PartitionSpec>> for RbPartitionSpec {
|
|
209
|
+
fn from(partition_spec: &Arc<PartitionSpec>) -> Self {
|
|
210
|
+
Self {
|
|
211
|
+
spec: (**partition_spec).clone().into(),
|
|
212
|
+
}
|
|
213
|
+
}
|
|
225
214
|
}
|
|
226
215
|
|
|
227
|
-
|
|
228
|
-
|
|
216
|
+
impl From<&Arc<SortOrder>> for RbSortOrder {
|
|
217
|
+
fn from(sort_order: &Arc<SortOrder>) -> Self {
|
|
218
|
+
Self {
|
|
219
|
+
order: (**sort_order).clone(),
|
|
220
|
+
}
|
|
221
|
+
}
|
|
229
222
|
}
|
|
230
223
|
|
|
231
|
-
|
|
232
|
-
|
|
224
|
+
impl From<&StatisticsFile> for RbStatisticsFile {
|
|
225
|
+
fn from(statistics_file: &StatisticsFile) -> Self {
|
|
226
|
+
Self {
|
|
227
|
+
file: statistics_file.clone(),
|
|
228
|
+
}
|
|
229
|
+
}
|
|
233
230
|
}
|
|
234
231
|
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
232
|
+
impl From<&PartitionStatisticsFile> for RbPartitionStatisticsFile {
|
|
233
|
+
fn from(partition_statistics_file: &PartitionStatisticsFile) -> Self {
|
|
234
|
+
Self {
|
|
235
|
+
file: partition_statistics_file.clone(),
|
|
236
|
+
}
|
|
237
|
+
}
|
|
239
238
|
}
|
|
240
239
|
|
|
241
|
-
|
|
242
|
-
|
|
240
|
+
impl From<&EncryptedKey> for RbEncryptedKey {
|
|
241
|
+
fn from(encrypted_key: &EncryptedKey) -> Self {
|
|
242
|
+
Self {
|
|
243
|
+
key: encrypted_key.clone(),
|
|
244
|
+
}
|
|
245
|
+
}
|
|
243
246
|
}
|
|
244
247
|
|
|
245
|
-
pub fn rb_literal(ruby: &Ruby, literal: &Literal) -> Value {
|
|
246
|
-
match literal {
|
|
248
|
+
pub fn rb_literal(ruby: &Ruby, literal: &Literal) -> RbResult<Value> {
|
|
249
|
+
let v = match literal {
|
|
247
250
|
Literal::Primitive(pl) => match pl {
|
|
248
251
|
PrimitiveLiteral::Boolean(v) => v.into_value_with(ruby),
|
|
249
252
|
PrimitiveLiteral::Int(v) => v.into_value_with(ruby),
|
|
@@ -252,8 +255,64 @@ pub fn rb_literal(ruby: &Ruby, literal: &Literal) -> Value {
|
|
|
252
255
|
PrimitiveLiteral::Double(v) => v.into_value_with(ruby),
|
|
253
256
|
PrimitiveLiteral::String(v) => ruby.str_new(v).as_value(),
|
|
254
257
|
PrimitiveLiteral::Binary(v) => ruby.str_from_slice(v).as_value(),
|
|
255
|
-
|
|
258
|
+
PrimitiveLiteral::Int128(v) => v.into_value_with(ruby),
|
|
259
|
+
PrimitiveLiteral::UInt128(v) => v.into_value_with(ruby),
|
|
260
|
+
PrimitiveLiteral::AboveMax => return Err(todo_error(literal)),
|
|
261
|
+
PrimitiveLiteral::BelowMin => return Err(todo_error(literal)),
|
|
256
262
|
},
|
|
257
|
-
_ =>
|
|
258
|
-
|
|
263
|
+
Literal::Struct(_) => return Err(todo_error(literal)),
|
|
264
|
+
Literal::List(_) => return Err(todo_error(literal)),
|
|
265
|
+
Literal::Map(_) => return Err(todo_error(literal)),
|
|
266
|
+
};
|
|
267
|
+
Ok(v)
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
pub fn rb_transform(transform: &Transform) -> RbResult<Value> {
|
|
271
|
+
let ruby = Ruby::get().unwrap();
|
|
272
|
+
let iceberg = ruby.class_object().const_get::<_, RModule>("Iceberg")?;
|
|
273
|
+
let v = match transform {
|
|
274
|
+
Transform::Identity => iceberg
|
|
275
|
+
.const_get::<_, RClass>("IdentityTransform")?
|
|
276
|
+
.new_instance(())?,
|
|
277
|
+
Transform::Bucket(num_buckets) => iceberg
|
|
278
|
+
.const_get::<_, RClass>("BucketTransform")?
|
|
279
|
+
.new_instance((*num_buckets,))?,
|
|
280
|
+
Transform::Truncate(width) => iceberg
|
|
281
|
+
.const_get::<_, RClass>("TruncateTransform")?
|
|
282
|
+
.new_instance((*width,))?,
|
|
283
|
+
Transform::Year => iceberg
|
|
284
|
+
.const_get::<_, RClass>("YearTransform")?
|
|
285
|
+
.new_instance(())?,
|
|
286
|
+
Transform::Month => iceberg
|
|
287
|
+
.const_get::<_, RClass>("MonthTransform")?
|
|
288
|
+
.new_instance(())?,
|
|
289
|
+
Transform::Day => iceberg
|
|
290
|
+
.const_get::<_, RClass>("DayTransform")?
|
|
291
|
+
.new_instance(())?,
|
|
292
|
+
Transform::Hour => iceberg
|
|
293
|
+
.const_get::<_, RClass>("HourTransform")?
|
|
294
|
+
.new_instance(())?,
|
|
295
|
+
Transform::Void => iceberg
|
|
296
|
+
.const_get::<_, RClass>("VoidTransform")?
|
|
297
|
+
.new_instance(())?,
|
|
298
|
+
Transform::Unknown => iceberg
|
|
299
|
+
.const_get::<_, RClass>("UnknownTransform")?
|
|
300
|
+
.new_instance(())?,
|
|
301
|
+
};
|
|
302
|
+
Ok(v)
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
pub static EPOCH: Lazy<Value> = Lazy::new(|ruby| {
|
|
306
|
+
ruby.class_object()
|
|
307
|
+
.const_get::<_, RClass>("Date")
|
|
308
|
+
.unwrap()
|
|
309
|
+
.new_instance((1970, 1, 1))
|
|
310
|
+
.unwrap()
|
|
311
|
+
});
|
|
312
|
+
|
|
313
|
+
pub fn date_to_i32(value: Value) -> RbResult<i32> {
|
|
314
|
+
let epoch = Ruby::get_with(value).get_inner(&EPOCH);
|
|
315
|
+
value
|
|
316
|
+
.funcall::<_, _, Value>("-", (epoch,))?
|
|
317
|
+
.funcall("to_i", ())
|
|
259
318
|
}
|
data/lib/iceberg/catalog.rb
CHANGED
|
@@ -1,14 +1,18 @@
|
|
|
1
1
|
module Iceberg
|
|
2
2
|
class Catalog
|
|
3
|
+
def _initialize(catalog, default_namespace:)
|
|
4
|
+
@catalog = catalog
|
|
5
|
+
@default_namespace = default_namespace
|
|
6
|
+
end
|
|
7
|
+
|
|
3
8
|
def list_namespaces(parent = nil)
|
|
4
9
|
@catalog.list_namespaces(parent)
|
|
5
10
|
end
|
|
6
11
|
|
|
7
12
|
def create_namespace(namespace, properties: {}, if_not_exists: nil)
|
|
8
13
|
@catalog.create_namespace(namespace, properties)
|
|
9
|
-
rescue
|
|
10
|
-
|
|
11
|
-
if !if_not_exists || (e.message != "Cannot create namespace" && !e.message.include?("already exists"))
|
|
14
|
+
rescue NamespaceAlreadyExistsError => e
|
|
15
|
+
if !if_not_exists
|
|
12
16
|
raise e
|
|
13
17
|
end
|
|
14
18
|
nil
|
|
@@ -28,19 +32,18 @@ module Iceberg
|
|
|
28
32
|
|
|
29
33
|
def drop_namespace(namespace, if_exists: nil)
|
|
30
34
|
@catalog.drop_namespace(namespace)
|
|
31
|
-
rescue
|
|
32
|
-
|
|
33
|
-
if !if_exists || (e.message != "Tried to drop a namespace that does not exist" && !e.message.include?("No such namespace") && !e.message.include?("The specified namespace does not exist") && !e.message.include?("not found"))
|
|
35
|
+
rescue NoSuchNamespaceError => e
|
|
36
|
+
if !if_exists
|
|
34
37
|
raise e
|
|
35
38
|
end
|
|
36
39
|
nil
|
|
37
40
|
end
|
|
38
41
|
|
|
39
|
-
def list_tables(namespace)
|
|
40
|
-
@catalog.list_tables(namespace)
|
|
42
|
+
def list_tables(namespace = nil)
|
|
43
|
+
@catalog.list_tables(namespace || @default_namespace)
|
|
41
44
|
end
|
|
42
45
|
|
|
43
|
-
def create_table(table_name, schema: nil, location: nil)
|
|
46
|
+
def create_table(table_name, schema: nil, location: nil, partition_spec: nil, sort_order: nil, properties: {})
|
|
44
47
|
if !schema.nil? && block_given?
|
|
45
48
|
raise ArgumentError, "Must pass schema or block"
|
|
46
49
|
end
|
|
@@ -48,63 +51,84 @@ module Iceberg
|
|
|
48
51
|
if block_given?
|
|
49
52
|
table_definition = TableDefinition.new
|
|
50
53
|
yield table_definition
|
|
51
|
-
schema = Schema.new(table_definition.fields)
|
|
52
|
-
elsif schema.is_a?(
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
schema = Schema.new(fields)
|
|
54
|
+
schema = Schema.new(*table_definition.fields)
|
|
55
|
+
elsif schema.is_a?(Schema)
|
|
56
|
+
# do nothing
|
|
57
|
+
elsif schema.respond_to?(:arrow_c_schema)
|
|
58
|
+
schema = Schema.new(schema)
|
|
59
|
+
elsif schema.is_a?(Hash)
|
|
60
|
+
table_definition = TableDefinition.new
|
|
61
|
+
schema.each do |k, v|
|
|
62
|
+
table_definition.column(k, v)
|
|
63
|
+
end
|
|
64
|
+
schema = Schema.new(*table_definition.fields)
|
|
63
65
|
elsif schema.nil?
|
|
64
|
-
schema = Schema.new
|
|
66
|
+
schema = Schema.new
|
|
65
67
|
end
|
|
66
68
|
|
|
67
|
-
|
|
69
|
+
format_version = (properties["format-version"] || 2).to_i
|
|
70
|
+
if properties["format-version"] && (is_a?(MemoryCatalog) || is_a?(SqlCatalog))
|
|
71
|
+
properties = properties.except("format-version")
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
Table.new(@catalog.create_table(with_namespace(table_name), schema, location, partition_spec, sort_order, properties, format_version), @catalog)
|
|
68
75
|
end
|
|
69
76
|
|
|
70
77
|
def load_table(table_name)
|
|
71
|
-
Table.new(@catalog.load_table(table_name), @catalog)
|
|
78
|
+
Table.new(@catalog.load_table(with_namespace(table_name)), @catalog)
|
|
72
79
|
end
|
|
73
80
|
|
|
74
81
|
def drop_table(table_name, if_exists: nil)
|
|
75
|
-
@catalog.drop_table(table_name)
|
|
76
|
-
rescue
|
|
77
|
-
|
|
78
|
-
if !if_exists || (e.message != "Tried to drop a table that does not exist" && !e.message.include?("No such table") && !e.message.include?("The specified table does not exist") && !e.message.include?("not found"))
|
|
82
|
+
@catalog.drop_table(with_namespace(table_name))
|
|
83
|
+
rescue NoSuchTableError => e
|
|
84
|
+
if !if_exists
|
|
79
85
|
raise e
|
|
80
86
|
end
|
|
81
87
|
nil
|
|
82
88
|
end
|
|
83
89
|
|
|
90
|
+
def purge_table(table_name)
|
|
91
|
+
@catalog.purge_table(with_namespace(table_name))
|
|
92
|
+
end
|
|
93
|
+
|
|
84
94
|
def table_exists?(table_name)
|
|
85
|
-
@catalog.table_exists?(table_name)
|
|
86
|
-
rescue
|
|
95
|
+
@catalog.table_exists?(with_namespace(table_name))
|
|
96
|
+
rescue NoSuchNamespaceError
|
|
87
97
|
false
|
|
88
98
|
end
|
|
89
99
|
|
|
90
100
|
def rename_table(table_name, new_name)
|
|
91
|
-
@catalog.rename_table(table_name, new_name)
|
|
101
|
+
@catalog.rename_table(with_namespace(table_name), with_namespace(new_name))
|
|
92
102
|
end
|
|
93
103
|
|
|
94
104
|
def register_table(table_name, metadata_location)
|
|
95
|
-
@catalog.register_table(table_name, metadata_location)
|
|
105
|
+
@catalog.register_table(with_namespace(table_name), metadata_location)
|
|
96
106
|
end
|
|
97
107
|
|
|
98
|
-
def sql(sql)
|
|
108
|
+
def sql(sql, params = [])
|
|
99
109
|
# requires datafusion feature
|
|
100
|
-
raise Todo unless @catalog.respond_to?(:
|
|
110
|
+
raise Todo unless @catalog.respond_to?(:session_context)
|
|
101
111
|
|
|
102
|
-
|
|
112
|
+
session_context.sql(sql, params)
|
|
103
113
|
end
|
|
104
114
|
|
|
105
115
|
# hide internal state
|
|
106
116
|
def inspect
|
|
107
117
|
to_s
|
|
108
118
|
end
|
|
119
|
+
|
|
120
|
+
private
|
|
121
|
+
|
|
122
|
+
def with_namespace(table_name)
|
|
123
|
+
if @default_namespace && table_name.is_a?(String) && !table_name.include?(".")
|
|
124
|
+
[@default_namespace, table_name]
|
|
125
|
+
else
|
|
126
|
+
table_name
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def session_context
|
|
131
|
+
@session_context ||= @catalog.session_context(@default_namespace)
|
|
132
|
+
end
|
|
109
133
|
end
|
|
110
134
|
end
|
data/lib/iceberg/glue_catalog.rb
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
module Iceberg
|
|
2
2
|
class GlueCatalog < Catalog
|
|
3
3
|
# warehouse is URI of S3 storage bucket
|
|
4
|
-
def initialize(warehouse:)
|
|
5
|
-
|
|
4
|
+
def initialize(warehouse:, default_namespace: nil)
|
|
5
|
+
_initialize(
|
|
6
|
+
RbCatalog.new_glue(warehouse),
|
|
7
|
+
default_namespace:
|
|
8
|
+
)
|
|
6
9
|
end
|
|
7
10
|
end
|
|
8
11
|
end
|
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
module Iceberg
|
|
2
2
|
class MemoryCatalog < Catalog
|
|
3
3
|
# warehouse is default storage location
|
|
4
|
-
def initialize(warehouse: nil)
|
|
5
|
-
|
|
4
|
+
def initialize(warehouse: nil, default_namespace: nil)
|
|
5
|
+
_initialize(
|
|
6
|
+
RbCatalog.new_memory(warehouse),
|
|
7
|
+
default_namespace:
|
|
8
|
+
)
|
|
6
9
|
end
|
|
7
10
|
end
|
|
8
11
|
end
|
data/lib/iceberg/rest_catalog.rb
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
module Iceberg
|
|
2
2
|
class RestCatalog < Catalog
|
|
3
3
|
# warehouse is passed to REST server
|
|
4
|
-
def initialize(uri:, warehouse: nil, properties: {})
|
|
5
|
-
|
|
4
|
+
def initialize(uri:, warehouse: nil, properties: {}, default_namespace: nil)
|
|
5
|
+
_initialize(
|
|
6
|
+
RbCatalog.new_rest(uri, warehouse, properties),
|
|
7
|
+
default_namespace:
|
|
8
|
+
)
|
|
6
9
|
end
|
|
7
10
|
end
|
|
8
11
|
end
|