iceberg 0.11.2 → 0.12.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,15 +1,24 @@
1
+ use std::sync::Arc;
2
+
1
3
  use iceberg::spec::{
2
- EncryptedKey, Literal, NestedField, PartitionSpec, PartitionStatisticsFile, PrimitiveLiteral,
3
- PrimitiveType, Schema, Snapshot, SortOrder, StatisticsFile, Type,
4
+ EncryptedKey, Literal, MetadataLog, NestedField, NullOrder, PartitionSpec,
5
+ PartitionStatisticsFile, PrimitiveLiteral, PrimitiveType, Schema, Snapshot, SnapshotLog,
6
+ SortDirection, SortOrder, StatisticsFile, Transform, Type,
4
7
  };
5
8
  use iceberg::{NamespaceIdent, TableIdent};
6
9
  use magnus::{
7
- Error as RbErr, IntoValue, RArray, RClass, RHash, RModule, Ruby, TryConvert, Value, kwargs,
8
- prelude::*,
10
+ Error as RbErr, IntoValue, RClass, RModule, RString, Ruby, TryConvert, Value, prelude::*,
11
+ value::Lazy,
9
12
  };
10
13
 
11
14
  use crate::RbResult;
12
- use crate::error::to_rb_err;
15
+ use crate::encryption::RbEncryptedKey;
16
+ use crate::error::{to_rb_err, todo_error};
17
+ use crate::partitioning::RbPartitionSpec;
18
+ use crate::schema::{RbNestedField, RbSchema};
19
+ use crate::snapshot::{RbMetadataLogEntry, RbSnapshot, RbSnapshotLogEntry};
20
+ use crate::sorting::RbSortOrder;
21
+ use crate::statistics::{RbPartitionStatisticsFile, RbStatisticsFile};
13
22
 
14
23
  pub struct Wrap<T>(pub T);
15
24
 
@@ -42,88 +51,80 @@ impl TryConvert for Wrap<TableIdent> {
42
51
  }
43
52
  }
44
53
 
45
- impl TryConvert for Wrap<Schema> {
54
+ impl TryConvert for Wrap<Transform> {
46
55
  fn try_convert(ob: Value) -> RbResult<Self> {
47
- let ruby = Ruby::get_with(ob);
48
- let mut fields = Vec::new();
49
- let rb_fields: RArray = ob.funcall("fields", ())?;
50
- for rb_field in rb_fields {
51
- let rb_field = RHash::try_convert(rb_field)?;
52
- let rb_type: Value = rb_field.aref(ruby.to_symbol("type"))?;
53
- let field_type = if let Ok(s) = String::try_convert(rb_type) {
54
- match s.as_str() {
55
- "boolean" => Type::Primitive(PrimitiveType::Boolean),
56
- "int" => Type::Primitive(PrimitiveType::Int),
57
- "long" => Type::Primitive(PrimitiveType::Long),
58
- "float" => Type::Primitive(PrimitiveType::Float),
59
- "double" => Type::Primitive(PrimitiveType::Double),
60
- // TODO PrimitiveType::Decimal
61
- "date" => Type::Primitive(PrimitiveType::Date),
62
- "time" => Type::Primitive(PrimitiveType::Time),
63
- "timestamp" => Type::Primitive(PrimitiveType::Timestamp),
64
- "timestamptz" => Type::Primitive(PrimitiveType::Timestamptz),
65
- "timestamp_ns" => Type::Primitive(PrimitiveType::TimestampNs),
66
- "timestamptz_ns" => Type::Primitive(PrimitiveType::TimestamptzNs),
67
- "string" => Type::Primitive(PrimitiveType::String),
68
- "uuid" => Type::Primitive(PrimitiveType::Uuid),
69
- // TODO PrimitiveType::Fixed
70
- "binary" => Type::Primitive(PrimitiveType::Binary),
71
- _ => {
72
- return Err(RbErr::new(
73
- ruby.exception_arg_error(),
74
- format!("Type not supported: {}", s),
75
- ));
76
- }
56
+ let v = if let Ok(s) = RString::try_convert(ob) {
57
+ match unsafe { s.as_str() }? {
58
+ "identity" => Transform::Identity,
59
+ "year" => Transform::Year,
60
+ "month" => Transform::Month,
61
+ "day" => Transform::Day,
62
+ "hour" => Transform::Hour,
63
+ _ => {
64
+ return Err(RbErr::new(
65
+ Ruby::get_with(ob).exception_arg_error(),
66
+ "Unsupported transform",
67
+ ));
77
68
  }
78
- } else {
79
- let class_name = unsafe { rb_type.classname() }.to_string();
80
- match class_name.as_str() {
81
- "Polars::Boolean" => Type::Primitive(PrimitiveType::Boolean),
82
- "Polars::Int32" => Type::Primitive(PrimitiveType::Int),
83
- "Polars::Int64" => Type::Primitive(PrimitiveType::Long),
84
- "Polars::Float32" => Type::Primitive(PrimitiveType::Float),
85
- "Polars::Float64" => Type::Primitive(PrimitiveType::Double),
86
- "Polars::Date" => Type::Primitive(PrimitiveType::Date),
87
- "Polars::Time" => Type::Primitive(PrimitiveType::Time),
88
- "Polars::String" => Type::Primitive(PrimitiveType::String),
89
- "Polars::Binary" => Type::Primitive(PrimitiveType::Binary),
90
- _ => {
91
- return Err(RbErr::new(
92
- ruby.exception_arg_error(),
93
- format!("Type not supported: {}", class_name),
94
- ));
95
- }
69
+ }
70
+ } else {
71
+ match &*unsafe { ob.classname() } {
72
+ "Iceberg::IdentityTransform" => Transform::Identity,
73
+ "Iceberg::BucketTransform" => Transform::Bucket(ob.funcall("num_buckets", ())?),
74
+ "Iceberg::TruncateTransform" => Transform::Truncate(ob.funcall("width", ())?),
75
+ "Iceberg::YearTransform" => Transform::Year,
76
+ "Iceberg::MonthTransform" => Transform::Month,
77
+ "Iceberg::DayTransform" => Transform::Day,
78
+ "Iceberg::HourTransform" => Transform::Hour,
79
+ "Iceberg::VoidTransform" => Transform::Void,
80
+ "Iceberg::UnknownTransform" => Transform::Unknown,
81
+ _ => {
82
+ return Err(RbErr::new(
83
+ Ruby::get_with(ob).exception_arg_error(),
84
+ format!("Transform not supported: {}", ob),
85
+ ));
96
86
  }
97
- };
98
-
99
- let initial_default = rb_field.aref(ruby.to_symbol("initial_default"))?;
100
- let write_default = rb_field.aref(ruby.to_symbol("write_default"))?;
87
+ }
88
+ };
89
+ Ok(Wrap(v))
90
+ }
91
+ }
101
92
 
102
- let initial_default = default_value(initial_default, &field_type)?;
103
- let write_default = default_value(write_default, &field_type)?;
93
+ impl TryConvert for Wrap<SortDirection> {
94
+ fn try_convert(ob: Value) -> RbResult<Self> {
95
+ let s = RString::try_convert(ob)?;
96
+ let v = match unsafe { s.as_str() }? {
97
+ "asc" => SortDirection::Ascending,
98
+ "desc" => SortDirection::Descending,
99
+ _ => {
100
+ return Err(RbErr::new(
101
+ Ruby::get_with(ob).exception_arg_error(),
102
+ "Unsupported sort direction",
103
+ ));
104
+ }
105
+ };
106
+ Ok(Wrap(v))
107
+ }
108
+ }
104
109
 
105
- fields.push(
106
- NestedField {
107
- id: rb_field.aref(ruby.to_symbol("id"))?,
108
- name: rb_field.aref(ruby.to_symbol("name"))?,
109
- required: rb_field.aref(ruby.to_symbol("required"))?,
110
- field_type: field_type.into(),
111
- doc: rb_field.aref(ruby.to_symbol("doc"))?,
112
- initial_default,
113
- write_default,
114
- }
115
- .into(),
116
- );
117
- }
118
- let schema = Schema::builder()
119
- .with_fields(fields)
120
- .build()
121
- .map_err(to_rb_err)?;
122
- Ok(Wrap(schema))
110
+ impl TryConvert for Wrap<NullOrder> {
111
+ fn try_convert(ob: Value) -> RbResult<Self> {
112
+ let s = RString::try_convert(ob)?;
113
+ let v = match unsafe { s.as_str() }? {
114
+ "first" => NullOrder::First,
115
+ "last" => NullOrder::Last,
116
+ _ => {
117
+ return Err(RbErr::new(
118
+ Ruby::get_with(ob).exception_arg_error(),
119
+ "Unsupported null order",
120
+ ));
121
+ }
122
+ };
123
+ Ok(Wrap(v))
123
124
  }
124
125
  }
125
126
 
126
- fn default_value(ob: Value, field_type: &Type) -> RbResult<Option<Literal>> {
127
+ pub fn default_value(ob: Value, field_type: &Type) -> RbResult<Option<Literal>> {
127
128
  if ob.is_nil() {
128
129
  return Ok(None);
129
130
  }
@@ -136,114 +137,116 @@ fn default_value(ob: Value, field_type: &Type) -> RbResult<Option<Literal>> {
136
137
  PrimitiveType::Long => PrimitiveLiteral::Long(i64::try_convert(ob)?),
137
138
  PrimitiveType::Float => PrimitiveLiteral::Float(f32::try_convert(ob)?.into()),
138
139
  PrimitiveType::Double => PrimitiveLiteral::Double(f64::try_convert(ob)?.into()),
140
+ PrimitiveType::Decimal {
141
+ precision: _,
142
+ scale: _,
143
+ } => return Err(todo_error(field_type)),
144
+ PrimitiveType::Date => return Err(todo_error(field_type)),
145
+ PrimitiveType::Time => return Err(todo_error(field_type)),
146
+ PrimitiveType::Timestamp => return Err(todo_error(field_type)),
147
+ PrimitiveType::Timestamptz => return Err(todo_error(field_type)),
148
+ PrimitiveType::TimestampNs => return Err(todo_error(field_type)),
149
+ PrimitiveType::TimestamptzNs => return Err(todo_error(field_type)),
139
150
  PrimitiveType::String => PrimitiveLiteral::String(String::try_convert(ob)?),
140
- _ => todo!(),
151
+ PrimitiveType::Uuid => return Err(todo_error(field_type)),
152
+ PrimitiveType::Fixed(_) => return Err(todo_error(field_type)),
153
+ PrimitiveType::Binary => {
154
+ let s = RString::try_convert(ob)?;
155
+ PrimitiveLiteral::Binary(unsafe { s.as_slice() }.to_vec())
156
+ }
141
157
  };
142
158
  Literal::Primitive(pl)
143
159
  }
144
- _ => todo!(),
160
+ Type::Struct(_) => return Err(todo_error(field_type)),
161
+ Type::List(_) => return Err(todo_error(field_type)),
162
+ Type::Map(_) => return Err(todo_error(field_type)),
163
+ Type::Variant(_) => return Err(todo_error(field_type)),
145
164
  };
146
165
  Ok(Some(lit))
147
166
  }
148
167
 
149
- pub fn rb_schema(ruby: &Ruby, schema: &Schema) -> RbResult<Value> {
150
- let fields = ruby.ary_new();
151
- for f in schema.as_struct().fields() {
152
- let field = ruby.hash_new();
153
- field.aset(ruby.to_symbol("id"), f.id)?;
154
- field.aset(ruby.to_symbol("name"), ruby.str_new(&f.name))?;
155
-
156
- let field_type = match &*f.field_type {
157
- Type::Primitive(ty) => match ty {
158
- PrimitiveType::Boolean => "boolean",
159
- PrimitiveType::Int => "int",
160
- PrimitiveType::Long => "long",
161
- PrimitiveType::Float => "float",
162
- PrimitiveType::Double => "double",
163
- PrimitiveType::Decimal {
164
- precision: _,
165
- scale: _,
166
- } => todo!(),
167
- PrimitiveType::Date => "date",
168
- PrimitiveType::Time => "time",
169
- PrimitiveType::Timestamp => "timestamp",
170
- PrimitiveType::Timestamptz => "timestamptz",
171
- PrimitiveType::TimestampNs => "timestamp_ns",
172
- PrimitiveType::TimestamptzNs => "timestamptz_ns",
173
- PrimitiveType::String => "string",
174
- PrimitiveType::Uuid => "uuid",
175
- PrimitiveType::Binary => "binary",
176
- PrimitiveType::Fixed(_) => todo!(),
177
- },
178
- _ => todo!(),
179
- };
180
- field.aset(ruby.to_symbol("type"), field_type)?;
181
-
182
- field.aset(ruby.to_symbol("required"), f.required)?;
183
-
184
- let initial_default = f.initial_default.as_ref().map(|v| rb_literal(ruby, v));
185
- field.aset(ruby.to_symbol("initial_default"), initial_default)?;
186
-
187
- let write_default = f.write_default.as_ref().map(|v| rb_literal(ruby, v));
188
- field.aset(ruby.to_symbol("write_default"), write_default)?;
168
+ impl From<&Arc<Schema>> for RbSchema {
169
+ fn from(schema: &Arc<Schema>) -> Self {
170
+ Self {
171
+ schema: (**schema).clone(),
172
+ }
173
+ }
174
+ }
189
175
 
190
- field.aset(
191
- ruby.to_symbol("doc"),
192
- f.doc.as_ref().map(|v| ruby.str_new(v)),
193
- )?;
176
+ impl From<&Arc<NestedField>> for RbNestedField {
177
+ fn from(field: &Arc<NestedField>) -> Self {
178
+ Self {
179
+ field: field.clone(),
180
+ }
181
+ }
182
+ }
194
183
 
195
- fields.push(field)?;
184
+ impl From<&Arc<Snapshot>> for RbSnapshot {
185
+ fn from(snapshot: &Arc<Snapshot>) -> Self {
186
+ Self {
187
+ snapshot: (**snapshot).clone(),
188
+ }
196
189
  }
197
- let schema_id = schema.schema_id();
190
+ }
198
191
 
199
- ruby.class_object()
200
- .const_get::<_, RModule>("Iceberg")
201
- .unwrap()
202
- .const_get::<_, RClass>("Schema")
203
- .unwrap()
204
- .funcall("new", (fields, kwargs!("schema_id" => schema_id)))
192
+ impl From<&SnapshotLog> for RbSnapshotLogEntry {
193
+ fn from(snapshot_log: &SnapshotLog) -> Self {
194
+ Self {
195
+ log: snapshot_log.clone(),
196
+ }
197
+ }
205
198
  }
206
199
 
207
- pub fn rb_snapshot(ruby: &Ruby, snapshot: &Snapshot) -> RbResult<Value> {
208
- let rb_snapshot = ruby.hash_new();
209
- rb_snapshot.aset(ruby.to_symbol("snapshot_id"), snapshot.snapshot_id())?;
210
- rb_snapshot.aset(
211
- ruby.to_symbol("parent_snapshot_id"),
212
- snapshot.parent_snapshot_id(),
213
- )?;
214
- rb_snapshot.aset(
215
- ruby.to_symbol("sequence_number"),
216
- snapshot.sequence_number(),
217
- )?;
218
- rb_snapshot.aset(ruby.to_symbol("manifest_list"), snapshot.manifest_list())?;
219
- rb_snapshot.aset(ruby.to_symbol("schema_id"), snapshot.schema_id())?;
220
- Ok(rb_snapshot.as_value())
200
+ impl From<&MetadataLog> for RbMetadataLogEntry {
201
+ fn from(metadata_log: &MetadataLog) -> Self {
202
+ Self {
203
+ log: metadata_log.clone(),
204
+ }
205
+ }
221
206
  }
222
207
 
223
- pub fn rb_partition_spec(_partition_spec: &PartitionSpec) -> RbResult<Value> {
224
- todo!();
208
+ impl From<&Arc<PartitionSpec>> for RbPartitionSpec {
209
+ fn from(partition_spec: &Arc<PartitionSpec>) -> Self {
210
+ Self {
211
+ spec: (**partition_spec).clone().into(),
212
+ }
213
+ }
225
214
  }
226
215
 
227
- pub fn rb_sort_order(_sort_order: &SortOrder) -> RbResult<Value> {
228
- todo!();
216
+ impl From<&Arc<SortOrder>> for RbSortOrder {
217
+ fn from(sort_order: &Arc<SortOrder>) -> Self {
218
+ Self {
219
+ order: (**sort_order).clone(),
220
+ }
221
+ }
229
222
  }
230
223
 
231
- pub fn rb_statistics_file(_statistics_file: &StatisticsFile) -> RbResult<Value> {
232
- todo!();
224
+ impl From<&StatisticsFile> for RbStatisticsFile {
225
+ fn from(statistics_file: &StatisticsFile) -> Self {
226
+ Self {
227
+ file: statistics_file.clone(),
228
+ }
229
+ }
233
230
  }
234
231
 
235
- pub fn rb_partition_statistics_file(
236
- _partition_statistics_file: &PartitionStatisticsFile,
237
- ) -> RbResult<Value> {
238
- todo!();
232
+ impl From<&PartitionStatisticsFile> for RbPartitionStatisticsFile {
233
+ fn from(partition_statistics_file: &PartitionStatisticsFile) -> Self {
234
+ Self {
235
+ file: partition_statistics_file.clone(),
236
+ }
237
+ }
239
238
  }
240
239
 
241
- pub fn rb_encrypted_key(_encrypted_key: &EncryptedKey) -> RbResult<Value> {
242
- todo!();
240
+ impl From<&EncryptedKey> for RbEncryptedKey {
241
+ fn from(encrypted_key: &EncryptedKey) -> Self {
242
+ Self {
243
+ key: encrypted_key.clone(),
244
+ }
245
+ }
243
246
  }
244
247
 
245
- pub fn rb_literal(ruby: &Ruby, literal: &Literal) -> Value {
246
- match literal {
248
+ pub fn rb_literal(ruby: &Ruby, literal: &Literal) -> RbResult<Value> {
249
+ let v = match literal {
247
250
  Literal::Primitive(pl) => match pl {
248
251
  PrimitiveLiteral::Boolean(v) => v.into_value_with(ruby),
249
252
  PrimitiveLiteral::Int(v) => v.into_value_with(ruby),
@@ -252,8 +255,64 @@ pub fn rb_literal(ruby: &Ruby, literal: &Literal) -> Value {
252
255
  PrimitiveLiteral::Double(v) => v.into_value_with(ruby),
253
256
  PrimitiveLiteral::String(v) => ruby.str_new(v).as_value(),
254
257
  PrimitiveLiteral::Binary(v) => ruby.str_from_slice(v).as_value(),
255
- _ => todo!(),
258
+ PrimitiveLiteral::Int128(v) => v.into_value_with(ruby),
259
+ PrimitiveLiteral::UInt128(v) => v.into_value_with(ruby),
260
+ PrimitiveLiteral::AboveMax => return Err(todo_error(literal)),
261
+ PrimitiveLiteral::BelowMin => return Err(todo_error(literal)),
256
262
  },
257
- _ => todo!(),
258
- }
263
+ Literal::Struct(_) => return Err(todo_error(literal)),
264
+ Literal::List(_) => return Err(todo_error(literal)),
265
+ Literal::Map(_) => return Err(todo_error(literal)),
266
+ };
267
+ Ok(v)
268
+ }
269
+
270
+ pub fn rb_transform(transform: &Transform) -> RbResult<Value> {
271
+ let ruby = Ruby::get().unwrap();
272
+ let iceberg = ruby.class_object().const_get::<_, RModule>("Iceberg")?;
273
+ let v = match transform {
274
+ Transform::Identity => iceberg
275
+ .const_get::<_, RClass>("IdentityTransform")?
276
+ .new_instance(())?,
277
+ Transform::Bucket(num_buckets) => iceberg
278
+ .const_get::<_, RClass>("BucketTransform")?
279
+ .new_instance((*num_buckets,))?,
280
+ Transform::Truncate(width) => iceberg
281
+ .const_get::<_, RClass>("TruncateTransform")?
282
+ .new_instance((*width,))?,
283
+ Transform::Year => iceberg
284
+ .const_get::<_, RClass>("YearTransform")?
285
+ .new_instance(())?,
286
+ Transform::Month => iceberg
287
+ .const_get::<_, RClass>("MonthTransform")?
288
+ .new_instance(())?,
289
+ Transform::Day => iceberg
290
+ .const_get::<_, RClass>("DayTransform")?
291
+ .new_instance(())?,
292
+ Transform::Hour => iceberg
293
+ .const_get::<_, RClass>("HourTransform")?
294
+ .new_instance(())?,
295
+ Transform::Void => iceberg
296
+ .const_get::<_, RClass>("VoidTransform")?
297
+ .new_instance(())?,
298
+ Transform::Unknown => iceberg
299
+ .const_get::<_, RClass>("UnknownTransform")?
300
+ .new_instance(())?,
301
+ };
302
+ Ok(v)
303
+ }
304
+
305
+ pub static EPOCH: Lazy<Value> = Lazy::new(|ruby| {
306
+ ruby.class_object()
307
+ .const_get::<_, RClass>("Date")
308
+ .unwrap()
309
+ .new_instance((1970, 1, 1))
310
+ .unwrap()
311
+ });
312
+
313
+ pub fn date_to_i32(value: Value) -> RbResult<i32> {
314
+ let epoch = Ruby::get_with(value).get_inner(&EPOCH);
315
+ value
316
+ .funcall::<_, _, Value>("-", (epoch,))?
317
+ .funcall("to_i", ())
259
318
  }
@@ -1,14 +1,18 @@
1
1
  module Iceberg
2
2
  class Catalog
3
+ def _initialize(catalog, default_namespace:)
4
+ @catalog = catalog
5
+ @default_namespace = default_namespace
6
+ end
7
+
3
8
  def list_namespaces(parent = nil)
4
9
  @catalog.list_namespaces(parent)
5
10
  end
6
11
 
7
12
  def create_namespace(namespace, properties: {}, if_not_exists: nil)
8
13
  @catalog.create_namespace(namespace, properties)
9
- rescue Error => e
10
- # ideally all catalogs would use NamespaceAlreadyExistsError
11
- if !if_not_exists || (e.message != "Cannot create namespace" && !e.message.include?("already exists"))
14
+ rescue NamespaceAlreadyExistsError => e
15
+ if !if_not_exists
12
16
  raise e
13
17
  end
14
18
  nil
@@ -28,19 +32,18 @@ module Iceberg
28
32
 
29
33
  def drop_namespace(namespace, if_exists: nil)
30
34
  @catalog.drop_namespace(namespace)
31
- rescue Error => e
32
- # ideally all catalogs would use NamespaceNotFoundError
33
- if !if_exists || (e.message != "Tried to drop a namespace that does not exist" && !e.message.include?("No such namespace") && !e.message.include?("The specified namespace does not exist") && !e.message.include?("not found"))
35
+ rescue NoSuchNamespaceError => e
36
+ if !if_exists
34
37
  raise e
35
38
  end
36
39
  nil
37
40
  end
38
41
 
39
- def list_tables(namespace)
40
- @catalog.list_tables(namespace)
42
+ def list_tables(namespace = nil)
43
+ @catalog.list_tables(namespace || @default_namespace)
41
44
  end
42
45
 
43
- def create_table(table_name, schema: nil, location: nil)
46
+ def create_table(table_name, schema: nil, location: nil, partition_spec: nil, sort_order: nil, properties: {})
44
47
  if !schema.nil? && block_given?
45
48
  raise ArgumentError, "Must pass schema or block"
46
49
  end
@@ -48,63 +51,84 @@ module Iceberg
48
51
  if block_given?
49
52
  table_definition = TableDefinition.new
50
53
  yield table_definition
51
- schema = Schema.new(table_definition.fields)
52
- elsif schema.is_a?(Hash) || (defined?(Polars::Schema) && schema.is_a?(Polars::Schema))
53
- fields =
54
- schema.to_h.map.with_index do |(k, v), i|
55
- {
56
- id: i + 1,
57
- name: k.is_a?(Symbol) ? k.to_s : k,
58
- type: v,
59
- required: false
60
- }
61
- end
62
- schema = Schema.new(fields)
54
+ schema = Schema.new(*table_definition.fields)
55
+ elsif schema.is_a?(Schema)
56
+ # do nothing
57
+ elsif schema.respond_to?(:arrow_c_schema)
58
+ schema = Schema.new(schema)
59
+ elsif schema.is_a?(Hash)
60
+ table_definition = TableDefinition.new
61
+ schema.each do |k, v|
62
+ table_definition.column(k, v)
63
+ end
64
+ schema = Schema.new(*table_definition.fields)
63
65
  elsif schema.nil?
64
- schema = Schema.new([])
66
+ schema = Schema.new
65
67
  end
66
68
 
67
- Table.new(@catalog.create_table(table_name, schema, location), @catalog)
69
+ format_version = (properties["format-version"] || 2).to_i
70
+ if properties["format-version"] && (is_a?(MemoryCatalog) || is_a?(SqlCatalog))
71
+ properties = properties.except("format-version")
72
+ end
73
+
74
+ Table.new(@catalog.create_table(with_namespace(table_name), schema, location, partition_spec, sort_order, properties, format_version), @catalog)
68
75
  end
69
76
 
70
77
  def load_table(table_name)
71
- Table.new(@catalog.load_table(table_name), @catalog)
78
+ Table.new(@catalog.load_table(with_namespace(table_name)), @catalog)
72
79
  end
73
80
 
74
81
  def drop_table(table_name, if_exists: nil)
75
- @catalog.drop_table(table_name)
76
- rescue Error => e
77
- # ideally all catalogs would use TableNotFoundError
78
- if !if_exists || (e.message != "Tried to drop a table that does not exist" && !e.message.include?("No such table") && !e.message.include?("The specified table does not exist") && !e.message.include?("not found"))
82
+ @catalog.drop_table(with_namespace(table_name))
83
+ rescue NoSuchTableError => e
84
+ if !if_exists
79
85
  raise e
80
86
  end
81
87
  nil
82
88
  end
83
89
 
90
+ def purge_table(table_name)
91
+ @catalog.purge_table(with_namespace(table_name))
92
+ end
93
+
84
94
  def table_exists?(table_name)
85
- @catalog.table_exists?(table_name)
86
- rescue NamespaceNotFoundError
95
+ @catalog.table_exists?(with_namespace(table_name))
96
+ rescue NoSuchNamespaceError
87
97
  false
88
98
  end
89
99
 
90
100
  def rename_table(table_name, new_name)
91
- @catalog.rename_table(table_name, new_name)
101
+ @catalog.rename_table(with_namespace(table_name), with_namespace(new_name))
92
102
  end
93
103
 
94
104
  def register_table(table_name, metadata_location)
95
- @catalog.register_table(table_name, metadata_location)
105
+ @catalog.register_table(with_namespace(table_name), metadata_location)
96
106
  end
97
107
 
98
- def sql(sql)
108
+ def sql(sql, params = [])
99
109
  # requires datafusion feature
100
- raise Todo unless @catalog.respond_to?(:sql)
110
+ raise Todo unless @catalog.respond_to?(:session_context)
101
111
 
102
- @catalog.sql(sql)
112
+ session_context.sql(sql, params)
103
113
  end
104
114
 
105
115
  # hide internal state
106
116
  def inspect
107
117
  to_s
108
118
  end
119
+
120
+ private
121
+
122
+ def with_namespace(table_name)
123
+ if @default_namespace && table_name.is_a?(String) && !table_name.include?(".")
124
+ [@default_namespace, table_name]
125
+ else
126
+ table_name
127
+ end
128
+ end
129
+
130
+ def session_context
131
+ @session_context ||= @catalog.session_context(@default_namespace)
132
+ end
109
133
  end
110
134
  end
@@ -1,8 +1,11 @@
1
1
  module Iceberg
2
2
  class GlueCatalog < Catalog
3
3
  # warehouse is URI of S3 storage bucket
4
- def initialize(warehouse:)
5
- @catalog = RbCatalog.new_glue(warehouse)
4
+ def initialize(warehouse:, default_namespace: nil)
5
+ _initialize(
6
+ RbCatalog.new_glue(warehouse),
7
+ default_namespace:
8
+ )
6
9
  end
7
10
  end
8
11
  end
@@ -1,8 +1,11 @@
1
1
  module Iceberg
2
2
  class MemoryCatalog < Catalog
3
3
  # warehouse is default storage location
4
- def initialize(warehouse: nil)
5
- @catalog = RbCatalog.new_memory(warehouse)
4
+ def initialize(warehouse: nil, default_namespace: nil)
5
+ _initialize(
6
+ RbCatalog.new_memory(warehouse),
7
+ default_namespace:
8
+ )
6
9
  end
7
10
  end
8
11
  end
@@ -1,8 +1,11 @@
1
1
  module Iceberg
2
2
  class RestCatalog < Catalog
3
3
  # warehouse is passed to REST server
4
- def initialize(uri:, warehouse: nil, properties: {})
5
- @catalog = RbCatalog.new_rest(uri, warehouse, properties)
4
+ def initialize(uri:, warehouse: nil, properties: {}, default_namespace: nil)
5
+ _initialize(
6
+ RbCatalog.new_rest(uri, warehouse, properties),
7
+ default_namespace:
8
+ )
6
9
  end
7
10
  end
8
11
  end