@loaders.gl/parquet 4.0.0-beta.2 → 4.0.0-beta.4

package/dist/esm/lib/wip/convert-schema-deep.rs.disabled

@@ -1,976 +0,0 @@
-//! This module has a single entry point, [`parquet_to_arrow_schema`].
-use parquet2::schema::{
-    types::{
-        FieldInfo, GroupConvertedType, GroupLogicalType, IntegerType, ParquetType, PhysicalType,
-        PrimitiveConvertedType, PrimitiveLogicalType, PrimitiveType, TimeUnit as ParquetTimeUnit,
-    },
-    Repetition,
-};
-
-use crate::datatypes::{DataType, Field, IntervalUnit, TimeUnit};
-
-/// Converts [`ParquetType`]s to a [`Field`], ignoring parquet fields that do not contain
-/// any physical column.
-pub fn parquet_to_arrow_schema(fields: &[ParquetType]) -> Vec<Field> {
-    fields.iter().filter_map(to_field).collect::<Vec<_>>()
-}
-
-fn from_int32(
-    logical_type: Option<PrimitiveLogicalType>,
-    converted_type: Option<PrimitiveConvertedType>,
-) -> DataType {
-    use PrimitiveLogicalType::*;
-    match (logical_type, converted_type) {
-        // handle logical types first
-        (Some(Integer(t)), _) => match t {
-            IntegerType::Int8 => DataType::Int8,
-            IntegerType::Int16 => DataType::Int16,
-            IntegerType::Int32 => DataType::Int32,
-            IntegerType::UInt8 => DataType::UInt8,
-            IntegerType::UInt16 => DataType::UInt16,
-            IntegerType::UInt32 => DataType::UInt32,
-            // The above are the only possible annotations for parquet's int32. Anything else
-            // is a deviation to the parquet specification and we ignore
-            _ => DataType::Int32,
-        },
-        (Some(Decimal(precision, scale)), _) => DataType::Decimal(precision, scale),
-        (Some(Date), _) => DataType::Date32,
-        (Some(Time { unit, .. }), _) => match unit {
-            ParquetTimeUnit::Milliseconds => DataType::Time32(TimeUnit::Millisecond),
-            // MILLIS is the only possible annotation for parquet's int32. Anything else
-            // is a deviation to the parquet specification and we ignore
-            _ => DataType::Int32,
-        },
-        // handle converted types:
-        (_, Some(PrimitiveConvertedType::Uint8)) => DataType::UInt8,
-        (_, Some(PrimitiveConvertedType::Uint16)) => DataType::UInt16,
-        (_, Some(PrimitiveConvertedType::Uint32)) => DataType::UInt32,
-        (_, Some(PrimitiveConvertedType::Int8)) => DataType::Int8,
-        (_, Some(PrimitiveConvertedType::Int16)) => DataType::Int16,
-        (_, Some(PrimitiveConvertedType::Int32)) => DataType::Int32,
-        (_, Some(PrimitiveConvertedType::Date)) => DataType::Date32,
-        (_, Some(PrimitiveConvertedType::TimeMillis)) => DataType::Time32(TimeUnit::Millisecond),
-        (_, Some(PrimitiveConvertedType::Decimal(precision, scale))) => {
-            DataType::Decimal(precision, scale)
-        }
-        (_, _) => DataType::Int32,
-    }
-}
-
-fn from_int64(
-    logical_type: Option<PrimitiveLogicalType>,
-    converted_type: Option<PrimitiveConvertedType>,
-) -> DataType {
-    use PrimitiveLogicalType::*;
-    match (logical_type, converted_type) {
-        // handle logical types first
-        (Some(Integer(integer)), _) => match integer {
-            IntegerType::UInt64 => DataType::UInt64,
-            IntegerType::Int64 => DataType::Int64,
-            _ => DataType::Int64,
-        },
-        (
-            Some(Timestamp {
-                is_adjusted_to_utc,
-                unit,
-            }),
-            _,
-        ) => {
-            let timezone = if is_adjusted_to_utc {
-                // https://github.com/apache/parquet-format/blob/master/LogicalTypes.md
-                // A TIMESTAMP with isAdjustedToUTC=true is defined as [...] elapsed since the Unix epoch
-                Some("+00:00".to_string())
-            } else {
-                // PARQUET:
-                // https://github.com/apache/parquet-format/blob/master/LogicalTypes.md
-                // A TIMESTAMP with isAdjustedToUTC=false represents [...] such
-                // timestamps should always be displayed the same way, regardless of the local time zone in effect
-                // ARROW:
-                // https://github.com/apache/parquet-format/blob/master/LogicalTypes.md
-                // If the time zone is null or equal to an empty string, the data is "time
-                // zone naive" and shall be displayed *as is* to the user, not localized
-                // to the locale of the user.
-                None
-            };
-
-            match unit {
-                ParquetTimeUnit::Milliseconds => {
-                    DataType::Timestamp(TimeUnit::Millisecond, timezone)
-                }
-                ParquetTimeUnit::Microseconds => {
-                    DataType::Timestamp(TimeUnit::Microsecond, timezone)
-                }
-                ParquetTimeUnit::Nanoseconds => DataType::Timestamp(TimeUnit::Nanosecond, timezone),
-            }
-        }
-        (Some(Time { unit, .. }), _) => match unit {
-            ParquetTimeUnit::Microseconds => DataType::Time64(TimeUnit::Microsecond),
-            ParquetTimeUnit::Nanoseconds => DataType::Time64(TimeUnit::Nanosecond),
-            // MILLIS is only possible for int32. Appearing in int64 is a deviation
-            // to parquet's spec, which we ignore
-            _ => DataType::Int64,
-        },
-        (Some(Decimal(precision, scale)), _) => DataType::Decimal(precision, scale),
-        // handle converted types:
-        (_, Some(PrimitiveConvertedType::TimeMicros)) => DataType::Time64(TimeUnit::Microsecond),
-        (_, Some(PrimitiveConvertedType::TimestampMillis)) => {
-            DataType::Timestamp(TimeUnit::Millisecond, None)
-        }
-        (_, Some(PrimitiveConvertedType::TimestampMicros)) => {
-            DataType::Timestamp(TimeUnit::Microsecond, None)
-        }
-        (_, Some(PrimitiveConvertedType::Int64)) => DataType::Int64,
-        (_, Some(PrimitiveConvertedType::Uint64)) => DataType::UInt64,
-        (_, Some(PrimitiveConvertedType::Decimal(precision, scale))) => {
-            DataType::Decimal(precision, scale)
-        }
-
-        (_, _) => DataType::Int64,
-    }
-}
-
-fn from_byte_array(
-    logical_type: &Option<PrimitiveLogicalType>,
-    converted_type: &Option<PrimitiveConvertedType>,
-) -> DataType {
-    match (logical_type, converted_type) {
-        (Some(PrimitiveLogicalType::String), _) => DataType::Utf8,
-        (Some(PrimitiveLogicalType::Json), _) => DataType::Binary,
-        (Some(PrimitiveLogicalType::Bson), _) => DataType::Binary,
-        (Some(PrimitiveLogicalType::Enum), _) => DataType::Binary,
-        (_, Some(PrimitiveConvertedType::Json)) => DataType::Binary,
-        (_, Some(PrimitiveConvertedType::Bson)) => DataType::Binary,
-        (_, Some(PrimitiveConvertedType::Enum)) => DataType::Binary,
-        (_, Some(PrimitiveConvertedType::Utf8)) => DataType::Utf8,
-        (_, _) => DataType::Binary,
-    }
-}
-
-fn from_fixed_len_byte_array(
-    length: usize,
-    logical_type: Option<PrimitiveLogicalType>,
-    converted_type: Option<PrimitiveConvertedType>,
-) -> DataType {
-    match (logical_type, converted_type) {
-        (Some(PrimitiveLogicalType::Decimal(precision, scale)), _) => {
-            DataType::Decimal(precision, scale)
-        }
-        (None, Some(PrimitiveConvertedType::Decimal(precision, scale))) => {
-            DataType::Decimal(precision, scale)
-        }
-        (None, Some(PrimitiveConvertedType::Interval)) => {
-            // There is currently no reliable way of determining which IntervalUnit
-            // to return. Thus without the original Arrow schema, the results
-            // would be incorrect if all 12 bytes of the interval are populated
-            DataType::Interval(IntervalUnit::DayTime)
-        }
-        _ => DataType::FixedSizeBinary(length),
-    }
-}
-
-/// Maps a [`PhysicalType`] with optional metadata to a [`DataType`]
-fn to_primitive_type_inner(primitive_type: &PrimitiveType) -> DataType {
-    match primitive_type.physical_type {
-        PhysicalType::Boolean => DataType::Boolean,
-        PhysicalType::Int32 => {
-            from_int32(primitive_type.logical_type, primitive_type.converted_type)
-        }
-        PhysicalType::Int64 => {
-            from_int64(primitive_type.logical_type, primitive_type.converted_type)
-        }
-        PhysicalType::Int96 => DataType::Timestamp(TimeUnit::Nanosecond, None),
-        PhysicalType::Float => DataType::Float32,
-        PhysicalType::Double => DataType::Float64,
-        PhysicalType::ByteArray => {
-            from_byte_array(&primitive_type.logical_type, &primitive_type.converted_type)
-        }
-        PhysicalType::FixedLenByteArray(length) => from_fixed_len_byte_array(
-            length,
-            primitive_type.logical_type,
-            primitive_type.converted_type,
-        ),
-    }
-}
-
-/// Entry point for converting parquet primitive type to arrow type.
-///
-/// This function takes care of repetition.
-fn to_primitive_type(primitive_type: &PrimitiveType) -> DataType {
-    let base_type = to_primitive_type_inner(primitive_type);
-
-    if primitive_type.field_info.repetition == Repetition::Repeated {
-        DataType::List(Box::new(Field::new(
-            &primitive_type.field_info.name,
-            base_type,
-            is_nullable(&primitive_type.field_info),
-        )))
-    } else {
-        base_type
-    }
-}
-
-fn non_repeated_group(
-    logical_type: &Option<GroupLogicalType>,
-    converted_type: &Option<GroupConvertedType>,
-    fields: &[ParquetType],
-    parent_name: &str,
-) -> Option<DataType> {
-    debug_assert!(!fields.is_empty());
-    match (logical_type, converted_type) {
-        (Some(GroupLogicalType::List), _) => to_list(fields, parent_name),
-        (None, Some(GroupConvertedType::List)) => to_list(fields, parent_name),
-        (Some(GroupLogicalType::Map), _) => to_list(fields, parent_name),
-        (None, Some(GroupConvertedType::Map) | Some(GroupConvertedType::MapKeyValue)) => {
-            to_map(fields)
-        }
-        _ => to_struct(fields),
-    }
-}
-
-/// Converts a parquet group type to an arrow [`DataType::Struct`].
-/// Returns [`None`] if all its fields are empty
-fn to_struct(fields: &[ParquetType]) -> Option<DataType> {
-    let fields = fields.iter().filter_map(to_field).collect::<Vec<Field>>();
-    if fields.is_empty() {
-        None
-    } else {
-        Some(DataType::Struct(fields))
-    }
-}
-
-/// Converts a parquet group type to an arrow [`DataType::Struct`].
-/// Returns [`None`] if all its fields are empty
-fn to_map(fields: &[ParquetType]) -> Option<DataType> {
-    let inner = to_field(&fields[0])?;
-    Some(DataType::Map(Box::new(inner), false))
-}
-
-/// Entry point for converting parquet group type.
-///
-/// This function takes care of logical type and repetition.
-fn to_group_type(
-    field_info: &FieldInfo,
-    logical_type: &Option<GroupLogicalType>,
-    converted_type: &Option<GroupConvertedType>,
-    fields: &[ParquetType],
-    parent_name: &str,
-) -> Option<DataType> {
-    debug_assert!(!fields.is_empty());
-    if field_info.repetition == Repetition::Repeated {
-        Some(DataType::List(Box::new(Field::new(
-            &field_info.name,
-            to_struct(fields)?,
-            is_nullable(field_info),
-        ))))
-    } else {
-        non_repeated_group(logical_type, converted_type, fields, parent_name)
-    }
-}
-
-/// Checks whether this schema is nullable.
-pub(crate) fn is_nullable(field_info: &FieldInfo) -> bool {
-    match field_info.repetition {
-        Repetition::Optional => true,
-        Repetition::Repeated => true,
-        Repetition::Required => false,
-    }
-}
-
-/// Converts parquet schema to arrow field.
-/// Returns `None` iff the parquet type has no associated primitive types,
-/// i.e. if it is a column-less group type.
-fn to_field(type_: &ParquetType) -> Option<Field> {
-    Some(Field::new(
-        &type_.get_field_info().name,
-        to_data_type(type_)?,
-        is_nullable(type_.get_field_info()),
-    ))
-}
-
-/// Converts a parquet list to arrow list.
-///
-/// To fully understand this algorithm, please refer to
-/// [parquet doc](https://github.com/apache/parquet-format/blob/master/LogicalTypes.md).
-fn to_list(fields: &[ParquetType], parent_name: &str) -> Option<DataType> {
-    let item = fields.first().unwrap();
-
-    let item_type = match item {
-        ParquetType::PrimitiveType(primitive) => Some(to_primitive_type_inner(primitive)),
-        ParquetType::GroupType { fields, .. } => {
-            if fields.len() == 1
-                && item.name() != "array"
-                && item.name() != format!("{parent_name}_tuple")
-            {
-                // extract the repetition field
-                let nested_item = fields.first().unwrap();
-                to_data_type(nested_item)
-            } else {
-                to_struct(fields)
-            }
-        }
-    }?;
-
-    // Check that the name of the list child is "list", in which case we
-    // get the child nullability and name (normally "element") from the nested
-    // group type.
-    // Without this step, the child incorrectly inherits the parent's optionality
-    let (list_item_name, item_is_optional) = match item {
-        ParquetType::GroupType {
-            field_info, fields, ..
-        } if field_info.name == "list" && fields.len() == 1 => {
-            let field = fields.first().unwrap();
-            (
-                &field.get_field_info().name,
-                field.get_field_info().repetition != Repetition::Required,
-            )
-        }
-        _ => (
-            &item.get_field_info().name,
-            item.get_field_info().repetition != Repetition::Required,
-        ),
-    };
-
-    Some(DataType::List(Box::new(Field::new(
-        list_item_name,
-        item_type,
-        item_is_optional,
-    ))))
-}
-
-/// Converts parquet schema to arrow data type.
-///
-/// This function discards schema name.
-///
-/// If this schema is a primitive type and not included in the leaves, the result is
-/// Ok(None).
-///
-/// If this schema is a group type and none of its children is reserved in the
-/// conversion, the result is Ok(None).
-pub(crate) fn to_data_type(type_: &ParquetType) -> Option<DataType> {
-    match type_ {
-        ParquetType::PrimitiveType(primitive) => Some(to_primitive_type(primitive)),
-        ParquetType::GroupType {
-            field_info,
-            logical_type,
-            converted_type,
-            fields,
-        } => {
-            if fields.is_empty() {
-                None
-            } else {
-                to_group_type(
-                    field_info,
-                    logical_type,
-                    converted_type,
-                    fields,
-                    &field_info.name,
-                )
-            }
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use parquet2::metadata::SchemaDescriptor;
-
-    use super::*;
-
-    use crate::datatypes::{DataType, Field, TimeUnit};
-    use crate::error::Result;
-
-    #[test]
-    fn test_flat_primitives() -> Result<()> {
-        let message = "
-        message test_schema {
-            REQUIRED BOOLEAN boolean;
-            REQUIRED INT32   int8  (INT_8);
-            REQUIRED INT32   int16 (INT_16);
-            REQUIRED INT32   uint8 (INTEGER(8,false));
-            REQUIRED INT32   uint16 (INTEGER(16,false));
-            REQUIRED INT32   int32;
-            REQUIRED INT64   int64 ;
-            OPTIONAL DOUBLE  double;
-            OPTIONAL FLOAT   float;
-            OPTIONAL BINARY  string (UTF8);
-            OPTIONAL BINARY  string_2 (STRING);
-        }
-        ";
-        let expected = &[
-            Field::new("boolean", DataType::Boolean, false),
-            Field::new("int8", DataType::Int8, false),
-            Field::new("int16", DataType::Int16, false),
-            Field::new("uint8", DataType::UInt8, false),
-            Field::new("uint16", DataType::UInt16, false),
-            Field::new("int32", DataType::Int32, false),
-            Field::new("int64", DataType::Int64, false),
-            Field::new("double", DataType::Float64, true),
-            Field::new("float", DataType::Float32, true),
-            Field::new("string", DataType::Utf8, true),
-            Field::new("string_2", DataType::Utf8, true),
-        ];
-
-        let parquet_schema = SchemaDescriptor::try_from_message(message)?;
-        let fields = parquet_to_arrow_schema(parquet_schema.fields());
-
-        assert_eq!(fields, expected);
-        Ok(())
-    }
-
-    #[test]
-    fn test_byte_array_fields() -> Result<()> {
-        let message = "
-        message test_schema {
-            REQUIRED BYTE_ARRAY binary;
-            REQUIRED FIXED_LEN_BYTE_ARRAY (20) fixed_binary;
-        }
-        ";
-        let expected = vec![
-            Field::new("binary", DataType::Binary, false),
-            Field::new("fixed_binary", DataType::FixedSizeBinary(20), false),
-        ];
-
-        let parquet_schema = SchemaDescriptor::try_from_message(message)?;
-        let fields = parquet_to_arrow_schema(parquet_schema.fields());
-
-        assert_eq!(fields, expected);
-        Ok(())
-    }
-
-    #[test]
-    fn test_duplicate_fields() -> Result<()> {
-        let message = "
-        message test_schema {
-            REQUIRED BOOLEAN boolean;
-            REQUIRED INT32 int8 (INT_8);
-        }
-        ";
-        let expected = &[
-            Field::new("boolean", DataType::Boolean, false),
-            Field::new("int8", DataType::Int8, false),
-        ];
-
-        let parquet_schema = SchemaDescriptor::try_from_message(message)?;
-        let fields = parquet_to_arrow_schema(parquet_schema.fields());
-
-        assert_eq!(fields, expected);
-        Ok(())
-    }
-
-    #[test]
-    fn test_parquet_lists() -> Result<()> {
-        let mut arrow_fields = Vec::new();
-
-        // LIST encoding example taken from parquet-format/LogicalTypes.md
-        let message_type = "
-        message test_schema {
-          REQUIRED GROUP my_list (LIST) {
-            REPEATED GROUP list {
-              OPTIONAL BINARY element (UTF8);
-            }
-          }
-          OPTIONAL GROUP my_list (LIST) {
-            REPEATED GROUP list {
-              REQUIRED BINARY element (UTF8);
-            }
-          }
-          OPTIONAL GROUP array_of_arrays (LIST) {
-            REPEATED GROUP list {
-              REQUIRED GROUP element (LIST) {
-                REPEATED GROUP list {
-                  REQUIRED INT32 element;
-                }
-              }
-            }
-          }
-          OPTIONAL GROUP my_list (LIST) {
-            REPEATED GROUP element {
-              REQUIRED BINARY str (UTF8);
-            }
-          }
-          OPTIONAL GROUP my_list (LIST) {
-            REPEATED INT32 element;
-          }
-          OPTIONAL GROUP my_list (LIST) {
-            REPEATED GROUP element {
-              REQUIRED BINARY str (UTF8);
-              REQUIRED INT32 num;
-            }
-          }
-          OPTIONAL GROUP my_list (LIST) {
-            REPEATED GROUP array {
-              REQUIRED BINARY str (UTF8);
-            }
-
-          }
-          OPTIONAL GROUP my_list (LIST) {
-            REPEATED GROUP my_list_tuple {
-              REQUIRED BINARY str (UTF8);
-            }
-          }
-          REPEATED INT32 name;
-        }
-        ";
-
-        // // List<String> (list non-null, elements nullable)
-        // required group my_list (LIST) {
-        //   repeated group list {
-        //     optional binary element (UTF8);
-        //   }
-        // }
-        {
-            arrow_fields.push(Field::new(
-                "my_list",
-                DataType::List(Box::new(Field::new("element", DataType::Utf8, true))),
-                false,
-            ));
-        }
-
-        // // List<String> (list nullable, elements non-null)
-        // optional group my_list (LIST) {
-        //   repeated group list {
-        //     required binary element (UTF8);
-        //   }
-        // }
-        {
-            arrow_fields.push(Field::new(
-                "my_list",
-                DataType::List(Box::new(Field::new("element", DataType::Utf8, false))),
-                true,
-            ));
-        }
-
-        // Element types can be nested structures. For example, a list of lists:
-        //
-        // // List<List<Integer>>
-        // optional group array_of_arrays (LIST) {
-        //   repeated group list {
-        //     required group element (LIST) {
-        //       repeated group list {
-        //         required int32 element;
-        //       }
-        //     }
-        //   }
-        // }
-        {
-            let arrow_inner_list =
-                DataType::List(Box::new(Field::new("element", DataType::Int32, false)));
-            arrow_fields.push(Field::new(
-                "array_of_arrays",
-                DataType::List(Box::new(Field::new("element", arrow_inner_list, false))),
-                true,
-            ));
-        }
-
-        // // List<String> (list nullable, elements non-null)
-        // optional group my_list (LIST) {
-        //   repeated group element {
-        //     required binary str (UTF8);
-        //   };
-        // }
-        {
-            arrow_fields.push(Field::new(
-                "my_list",
-                DataType::List(Box::new(Field::new("element", DataType::Utf8, true))),
-                true,
-            ));
-        }
-
-        // // List<Integer> (nullable list, non-null elements)
-        // optional group my_list (LIST) {
-        //   repeated int32 element;
-        // }
-        {
-            arrow_fields.push(Field::new(
-                "my_list",
-                DataType::List(Box::new(Field::new("element", DataType::Int32, true))),
-                true,
-            ));
-        }
-
-        // // List<Tuple<String, Integer>> (nullable list, non-null elements)
-        // optional group my_list (LIST) {
-        //   repeated group element {
-        //     required binary str (UTF8);
-        //     required int32 num;
-        //   };
-        // }
-        {
-            let arrow_struct = DataType::Struct(vec![
-                Field::new("str", DataType::Utf8, false),
-                Field::new("num", DataType::Int32, false),
-            ]);
-            arrow_fields.push(Field::new(
-                "my_list",
-                DataType::List(Box::new(Field::new("element", arrow_struct, true))),
-                true,
-            ));
-        }
-
-        // // List<OneTuple<String>> (nullable list, non-null elements)
-        // optional group my_list (LIST) {
-        //   repeated group array {
-        //     required binary str (UTF8);
-        //   };
-        // }
-        // Special case: group is named array
-        {
-            let arrow_struct = DataType::Struct(vec![Field::new("str", DataType::Utf8, false)]);
-            arrow_fields.push(Field::new(
-                "my_list",
-                DataType::List(Box::new(Field::new("array", arrow_struct, true))),
-                true,
-            ));
-        }
-
-        // // List<OneTuple<String>> (nullable list, non-null elements)
-        // optional group my_list (LIST) {
-        //   repeated group my_list_tuple {
-        //     required binary str (UTF8);
-        //   };
-        // }
-        // Special case: group named ends in _tuple
-        {
-            let arrow_struct = DataType::Struct(vec![Field::new("str", DataType::Utf8, false)]);
-            arrow_fields.push(Field::new(
-                "my_list",
-                DataType::List(Box::new(Field::new("my_list_tuple", arrow_struct, true))),
-                true,
-            ));
-        }
-
-        // One-level encoding: Only allows required lists with required cells
-        //   repeated value_type name
-        {
-            arrow_fields.push(Field::new(
-                "name",
-                DataType::List(Box::new(Field::new("name", DataType::Int32, true))),
-                true,
-            ));
-        }
-
-        let parquet_schema = SchemaDescriptor::try_from_message(message_type)?;
-        let fields = parquet_to_arrow_schema(parquet_schema.fields());
-
-        assert_eq!(arrow_fields, fields);
-        Ok(())
-    }
-
-    #[test]
-    fn test_parquet_list_nullable() -> Result<()> {
-        let mut arrow_fields = Vec::new();
-
-        let message_type = "
-        message test_schema {
-          REQUIRED GROUP my_list1 (LIST) {
-            REPEATED GROUP list {
-              OPTIONAL BINARY element (UTF8);
-            }
-          }
-          OPTIONAL GROUP my_list2 (LIST) {
-            REPEATED GROUP list {
-              REQUIRED BINARY element (UTF8);
-            }
-          }
-          REQUIRED GROUP my_list3 (LIST) {
-            REPEATED GROUP list {
-              REQUIRED BINARY element (UTF8);
-            }
-          }
-        }
-        ";
-
-        // // List<String> (list non-null, elements nullable)
-        // required group my_list1 (LIST) {
-        //   repeated group list {
-        //     optional binary element (UTF8);
-        //   }
-        // }
-        {
-            arrow_fields.push(Field::new(
-                "my_list1",
-                DataType::List(Box::new(Field::new("element", DataType::Utf8, true))),
-                false,
-            ));
-        }
-
-        // // List<String> (list nullable, elements non-null)
-        // optional group my_list2 (LIST) {
-        //   repeated group list {
-        //     required binary element (UTF8);
-        //   }
-        // }
-        {
-            arrow_fields.push(Field::new(
-                "my_list2",
-                DataType::List(Box::new(Field::new("element", DataType::Utf8, false))),
-                true,
-            ));
-        }
-
-        // // List<String> (list non-null, elements non-null)
-        // repeated group my_list3 (LIST) {
-        //   repeated group list {
-        //     required binary element (UTF8);
-        //   }
-        // }
-        {
-            arrow_fields.push(Field::new(
-                "my_list3",
-                DataType::List(Box::new(Field::new("element", DataType::Utf8, false))),
-                false,
-            ));
-        }
-
-        let parquet_schema = SchemaDescriptor::try_from_message(message_type)?;
-        let fields = parquet_to_arrow_schema(parquet_schema.fields());
-
-        assert_eq!(arrow_fields, fields);
-        Ok(())
-    }
-
-    #[test]
-    fn test_nested_schema() -> Result<()> {
-        let mut arrow_fields = Vec::new();
-        {
-            let group1_fields = vec![
-                Field::new("leaf1", DataType::Boolean, false),
-                Field::new("leaf2", DataType::Int32, false),
-            ];
-            let group1_struct = Field::new("group1", DataType::Struct(group1_fields), false);
-            arrow_fields.push(group1_struct);
-
-            let leaf3_field = Field::new("leaf3", DataType::Int64, false);
-            arrow_fields.push(leaf3_field);
-        }
-
-        let message_type = "
-        message test_schema {
-          REQUIRED GROUP group1 {
-            REQUIRED BOOLEAN leaf1;
-            REQUIRED INT32 leaf2;
-          }
-          REQUIRED INT64 leaf3;
-        }
-        ";
-
-        let parquet_schema = SchemaDescriptor::try_from_message(message_type)?;
-        let fields = parquet_to_arrow_schema(parquet_schema.fields());
-
-        assert_eq!(arrow_fields, fields);
-        Ok(())
-    }
-
-    #[test]
-    fn test_repeated_nested_schema() -> Result<()> {
-        let mut arrow_fields = Vec::new();
-        {
-            arrow_fields.push(Field::new("leaf1", DataType::Int32, true));
-
-            let inner_group_list = Field::new(
-                "innerGroup",
-                DataType::List(Box::new(Field::new(
-                    "innerGroup",
-                    DataType::Struct(vec![Field::new("leaf3", DataType::Int32, true)]),
-                    true,
-                ))),
-                true,
-            );
-
-            let outer_group_list = Field::new(
-                "outerGroup",
-                DataType::List(Box::new(Field::new(
-                    "outerGroup",
-                    DataType::Struct(vec![
-                        Field::new("leaf2", DataType::Int32, true),
-                        inner_group_list,
-                    ]),
-                    true,
-                ))),
-                true,
-            );
-            arrow_fields.push(outer_group_list);
-        }
-
-        let message_type = "
-        message test_schema {
-          OPTIONAL INT32 leaf1;
-          REPEATED GROUP outerGroup {
-            OPTIONAL INT32 leaf2;
-            REPEATED GROUP innerGroup {
-              OPTIONAL INT32 leaf3;
-            }
-          }
-        }
-        ";
-
-        let parquet_schema = SchemaDescriptor::try_from_message(message_type)?;
-        let fields = parquet_to_arrow_schema(parquet_schema.fields());
-
-        assert_eq!(arrow_fields, fields);
-        Ok(())
-    }
-
-    #[test]
-    fn test_column_desc_to_field() -> Result<()> {
-        let message_type = "
-        message test_schema {
-            REQUIRED BOOLEAN boolean;
-            REQUIRED INT32   int8  (INT_8);
-            REQUIRED INT32   uint8 (INTEGER(8,false));
-            REQUIRED INT32   int16 (INT_16);
-            REQUIRED INT32   uint16 (INTEGER(16,false));
-            REQUIRED INT32   int32;
-            REQUIRED INT64   int64;
-            OPTIONAL DOUBLE  double;
-            OPTIONAL FLOAT   float;
-            OPTIONAL BINARY  string (UTF8);
-            REPEATED BOOLEAN bools;
-            OPTIONAL INT32   date       (DATE);
-            OPTIONAL INT32   time_milli (TIME_MILLIS);
-            OPTIONAL INT64   time_micro (TIME_MICROS);
-            OPTIONAL INT64   time_nano (TIME(NANOS,false));
-            OPTIONAL INT64   ts_milli (TIMESTAMP_MILLIS);
-            REQUIRED INT64   ts_micro (TIMESTAMP_MICROS);
-            REQUIRED INT64   ts_nano (TIMESTAMP(NANOS,true));
-        }
-        ";
-        let arrow_fields = vec![
-            Field::new("boolean", DataType::Boolean, false),
-            Field::new("int8", DataType::Int8, false),
-            Field::new("uint8", DataType::UInt8, false),
-            Field::new("int16", DataType::Int16, false),
-            Field::new("uint16", DataType::UInt16, false),
-            Field::new("int32", DataType::Int32, false),
-            Field::new("int64", DataType::Int64, false),
-            Field::new("double", DataType::Float64, true),
-            Field::new("float", DataType::Float32, true),
-            Field::new("string", DataType::Utf8, true),
-            Field::new(
-                "bools",
-                DataType::List(Box::new(Field::new("bools", DataType::Boolean, true))),
-                true,
-            ),
-            Field::new("date", DataType::Date32, true),
-            Field::new("time_milli", DataType::Time32(TimeUnit::Millisecond), true),
-            Field::new("time_micro", DataType::Time64(TimeUnit::Microsecond), true),
-            Field::new("time_nano", DataType::Time64(TimeUnit::Nanosecond), true),
-            Field::new(
-                "ts_milli",
-                DataType::Timestamp(TimeUnit::Millisecond, None),
-                true,
-            ),
-            Field::new(
-                "ts_micro",
-                DataType::Timestamp(TimeUnit::Microsecond, None),
-                false,
-            ),
-            Field::new(
-                "ts_nano",
-                DataType::Timestamp(TimeUnit::Nanosecond, Some("+00:00".to_string())),
-                false,
-            ),
-        ];
-
-        let parquet_schema = SchemaDescriptor::try_from_message(message_type)?;
-        let fields = parquet_to_arrow_schema(parquet_schema.fields());
-
-        assert_eq!(arrow_fields, fields);
-        Ok(())
-    }
-
-    #[test]
-    fn test_field_to_column_desc() -> Result<()> {
-        let message_type = "
-        message arrow_schema {
-            REQUIRED BOOLEAN boolean;
-            REQUIRED INT32   int8  (INT_8);
-            REQUIRED INT32   int16 (INTEGER(16,true));
-            REQUIRED INT32   int32;
-            REQUIRED INT64   int64;
-            OPTIONAL DOUBLE  double;
-            OPTIONAL FLOAT   float;
-            OPTIONAL BINARY  string (STRING);
-            OPTIONAL GROUP   bools (LIST) {
-                REPEATED GROUP list {
-                    OPTIONAL BOOLEAN element;
-                }
-            }
-            REQUIRED GROUP   bools_non_null (LIST) {
-                REPEATED GROUP list {
-                    REQUIRED BOOLEAN element;
-                }
-            }
-            OPTIONAL INT32   date       (DATE);
-            OPTIONAL INT32   time_milli (TIME(MILLIS,false));
-            OPTIONAL INT64   time_micro (TIME_MICROS);
-            OPTIONAL INT64   ts_milli (TIMESTAMP_MILLIS);
-            REQUIRED INT64   ts_micro (TIMESTAMP(MICROS,false));
-            REQUIRED GROUP struct {
-                REQUIRED BOOLEAN bools;
-                REQUIRED INT32 uint32 (INTEGER(32,false));
-                REQUIRED GROUP   int32 (LIST) {
-                    REPEATED GROUP list {
-                        OPTIONAL INT32 element;
-                    }
-                }
-            }
-            REQUIRED BINARY  dictionary_strings (STRING);
-        }
-        ";
-
-        let arrow_fields = vec![
-            Field::new("boolean", DataType::Boolean, false),
-            Field::new("int8", DataType::Int8, false),
-            Field::new("int16", DataType::Int16, false),
-            Field::new("int32", DataType::Int32, false),
-            Field::new("int64", DataType::Int64, false),
-            Field::new("double", DataType::Float64, true),
-            Field::new("float", DataType::Float32, true),
-            Field::new("string", DataType::Utf8, true),
-            Field::new(
-                "bools",
-                DataType::List(Box::new(Field::new("element", DataType::Boolean, true))),
-                true,
-            ),
-            Field::new(
-                "bools_non_null",
-                DataType::List(Box::new(Field::new("element", DataType::Boolean, false))),
-                false,
-            ),
-            Field::new("date", DataType::Date32, true),
-            Field::new("time_milli", DataType::Time32(TimeUnit::Millisecond), true),
-            Field::new("time_micro", DataType::Time64(TimeUnit::Microsecond), true),
-            Field::new(
-                "ts_milli",
-                DataType::Timestamp(TimeUnit::Millisecond, None),
-                true,
-            ),
-            Field::new(
-                "ts_micro",
-                DataType::Timestamp(TimeUnit::Microsecond, None),
-                false,
-            ),
-            Field::new(
-                "struct",
-                DataType::Struct(vec![
-                    Field::new("bools", DataType::Boolean, false),
-                    Field::new("uint32", DataType::UInt32, false),
-                    Field::new(
-                        "int32",
-                        DataType::List(Box::new(Field::new("element", DataType::Int32, true))),
-                        false,
-                    ),
-                ]),
-                false,
-            ),
-            Field::new("dictionary_strings", DataType::Utf8, false),
-        ];
-
-        let parquet_schema = SchemaDescriptor::try_from_message(message_type)?;
-        let fields = parquet_to_arrow_schema(parquet_schema.fields());
-
-        assert_eq!(arrow_fields, fields);
-        Ok(())
-    }
-}