duckdb 0.7.2-dev402.0 → 0.7.2-dev586.0

package/binding.gyp

@@ -223,18 +223,18 @@
                 "src/duckdb/third_party/zstd/compress/zstd_lazy.cpp", 
                 "src/duckdb/third_party/zstd/compress/zstd_ldm.cpp", 
                 "src/duckdb/third_party/zstd/compress/zstd_opt.cpp", 
-                "src/duckdb/extension/icu/./icu-dateadd.cpp", 
-                "src/duckdb/extension/icu/./icu-table-range.cpp", 
-                "src/duckdb/extension/icu/./icu-datetrunc.cpp", 
-                "src/duckdb/extension/icu/./icu-strptime.cpp", 
-                "src/duckdb/extension/icu/./icu-datefunc.cpp", 
-                "src/duckdb/extension/icu/./icu-extension.cpp", 
-                "src/duckdb/extension/icu/./icu-makedate.cpp", 
                 "src/duckdb/extension/icu/./icu-list-range.cpp", 
+                "src/duckdb/extension/icu/./icu-datepart.cpp", 
+                "src/duckdb/extension/icu/./icu-timebucket.cpp", 
                 "src/duckdb/extension/icu/./icu-timezone.cpp", 
+                "src/duckdb/extension/icu/./icu-datefunc.cpp", 
+                "src/duckdb/extension/icu/./icu-makedate.cpp", 
+                "src/duckdb/extension/icu/./icu-datetrunc.cpp", 
+                "src/duckdb/extension/icu/./icu-table-range.cpp", 
+                "src/duckdb/extension/icu/./icu-extension.cpp", 
+                "src/duckdb/extension/icu/./icu-dateadd.cpp", 
                 "src/duckdb/extension/icu/./icu-datesub.cpp", 
-                "src/duckdb/extension/icu/./icu-timebucket.cpp", 
-                "src/duckdb/extension/icu/./icu-datepart.cpp", 
+                "src/duckdb/extension/icu/./icu-strptime.cpp", 
                 "src/duckdb/ub_extension_icu_third_party_icu_common.cpp", 
                 "src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp", 
                 "src/duckdb/extension/icu/third_party/icu/stubdata/stubdata.cpp", 

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.7.2-dev402.0",
+  "version": "0.7.2-dev586.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/extension/parquet/include/parquet_timestamp.hpp

@@ -16,7 +16,6 @@ struct Int96 {
 	uint32_t value[3];
 };
 
-int64_t ImpalaTimestampToNanoseconds(const Int96 &impala_timestamp);
 timestamp_t ImpalaTimestampToTimestamp(const Int96 &raw_ts);
 Int96 TimestampToImpalaTimestamp(timestamp_t &ts);
 timestamp_t ParquetTimestampMicrosToTimestamp(const int64_t &raw_ts);

package/src/duckdb/extension/parquet/parquet-extension.cpp

@@ -354,6 +354,7 @@ public:
 		return_types.assign(union_col_types.begin(), union_col_types.end());
 		result->SetInitialReader(result->union_readers[0]);
 		D_ASSERT(names.size() == return_types.size());
+		result->types = union_col_types;
 
 		return std::move(result);
 	}

package/src/duckdb/extension/parquet/parquet_timestamp.cpp

@@ -12,24 +12,26 @@ namespace duckdb {
 // surely they are joking
 static constexpr int64_t JULIAN_TO_UNIX_EPOCH_DAYS = 2440588LL;
 static constexpr int64_t MILLISECONDS_PER_DAY = 86400000LL;
-static constexpr int64_t NANOSECONDS_PER_DAY = MILLISECONDS_PER_DAY * 1000LL * 1000LL;
+static constexpr int64_t MICROSECONDS_PER_DAY = MILLISECONDS_PER_DAY * 1000LL;
+static constexpr int64_t NANOSECONDS_PER_MICRO = 1000LL;
 
-int64_t ImpalaTimestampToNanoseconds(const Int96 &impala_timestamp) {
+static int64_t ImpalaTimestampToMicroseconds(const Int96 &impala_timestamp) {
 	int64_t days_since_epoch = impala_timestamp.value[2] - JULIAN_TO_UNIX_EPOCH_DAYS;
 	auto nanoseconds = Load<int64_t>((data_ptr_t)impala_timestamp.value);
-	return days_since_epoch * NANOSECONDS_PER_DAY + nanoseconds;
+	auto microseconds = nanoseconds / NANOSECONDS_PER_MICRO;
+	return days_since_epoch * MICROSECONDS_PER_DAY + microseconds;
 }
 
 timestamp_t ImpalaTimestampToTimestamp(const Int96 &raw_ts) {
-	auto impala_ns = ImpalaTimestampToNanoseconds(raw_ts);
-	return Timestamp::FromEpochNanoSeconds(impala_ns);
+	auto impala_us = ImpalaTimestampToMicroseconds(raw_ts);
+	return Timestamp::FromEpochMicroSeconds(impala_us);
 }
 
 Int96 TimestampToImpalaTimestamp(timestamp_t &ts) {
 	int32_t hour, min, sec, msec;
 	Time::Convert(Timestamp::GetTime(ts), hour, min, sec, msec);
 	uint64_t ms_since_midnight = hour * 60 * 60 * 1000 + min * 60 * 1000 + sec * 1000 + msec;
-	auto days_since_epoch = Date::Epoch(Timestamp::GetDate(ts)) / (24 * 60 * 60);
+	auto days_since_epoch = Date::Epoch(Timestamp::GetDate(ts)) / int64_t(24 * 60 * 60);
 	// first two uint32 in Int96 are nanoseconds since midnights
 	// last uint32 is number of days since year 4713 BC ("Julian date")
 	Int96 impala_ts;

package/src/duckdb/src/catalog/catalog.cpp

@@ -625,6 +625,19 @@ vector<SchemaCatalogEntry *> Catalog::GetSchemas(ClientContext &context) {
 	return schemas;
 }
 
+bool Catalog::TypeExists(ClientContext &context, const string &catalog_name, const string &schema, const string &name) {
+	CatalogEntry *entry;
+	entry = GetEntry(context, CatalogType::TYPE_ENTRY, catalog_name, schema, name, true);
+	if (!entry) {
+		// look in the system catalog
+		entry = GetEntry(context, CatalogType::TYPE_ENTRY, SYSTEM_CATALOG, schema, name, true);
+		if (!entry) {
+			return false;
+		}
+	}
+	return true;
+}
+
 vector<SchemaCatalogEntry *> Catalog::GetSchemas(ClientContext &context, const string &catalog_name) {
 	vector<Catalog *> catalogs;
 	if (IsInvalidCatalog(catalog_name)) {

package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp

@@ -682,27 +682,8 @@ void DuckTableEntry::SetAsRoot() {
 	storage->info->table = name;
 }
 
-void DuckTableEntry::CommitAlter(AlterInfo &info) {
-	D_ASSERT(info.type == AlterType::ALTER_TABLE);
-	auto &alter_table = (AlterTableInfo &)info;
-	string column_name;
-	switch (alter_table.alter_table_type) {
-	case AlterTableType::REMOVE_COLUMN: {
-		auto &remove_info = (RemoveColumnInfo &)alter_table;
-		column_name = remove_info.removed_column;
-		break;
-	}
-	case AlterTableType::ALTER_COLUMN_TYPE: {
-		auto &change_info = (ChangeColumnTypeInfo &)alter_table;
-		column_name = change_info.column_name;
-		break;
-	}
-	default:
-		break;
-	}
-	if (column_name.empty()) {
-		return;
-	}
+void DuckTableEntry::CommitAlter(string &column_name) {
+	D_ASSERT(!column_name.empty());
 	idx_t removed_index = DConstants::INVALID_INDEX;
 	for (auto &col : columns.Logical()) {
 		if (col.Name() == column_name) {

package/src/duckdb/src/catalog/catalog_entry/type_catalog_entry.cpp

@@ -23,7 +23,13 @@ void TypeCatalogEntry::Serialize(Serializer &serializer) {
 	FieldWriter writer(serializer);
 	writer.WriteString(schema->name);
 	writer.WriteString(name);
-	writer.WriteSerializable(user_type);
+	if (user_type.id() == LogicalTypeId::ENUM) {
+		// We have to serialize Enum Values
+		writer.AddField();
+		user_type.SerializeEnumType(writer.GetSerializer());
+	} else {
+		writer.WriteSerializable(user_type);
+	}
 	writer.Finalize();
 }
 
@@ -43,7 +49,7 @@ string TypeCatalogEntry::ToSQL() {
 	std::stringstream ss;
 	switch (user_type.id()) {
 	case (LogicalTypeId::ENUM): {
-		Vector values_insert_order(EnumType::GetValuesInsertOrder(user_type));
+		auto &values_insert_order = EnumType::GetValuesInsertOrder(user_type);
 		idx_t size = EnumType::GetSize(user_type);
 		ss << "CREATE TYPE ";
 		ss << KeywordHelper::WriteOptionallyQuoted(name);

package/src/duckdb/src/catalog/catalog_set.cpp

@@ -258,6 +258,7 @@ bool CatalogSet::AlterEntry(CatalogTransaction transaction, const string &name,
 
 	// serialize the AlterInfo into a temporary buffer
 	BufferedSerializer serializer;
+	serializer.WriteString(alter_info->GetColumnName());
 	alter_info->Serialize(serializer);
 	BinaryData serialized_alter = serializer.GetData();
 

package/src/duckdb/src/common/arrow/arrow_appender.cpp

@@ -4,6 +4,7 @@
 #include "duckdb/common/array.hpp"
 #include "duckdb/common/types/interval.hpp"
 #include "duckdb/common/types/uuid.hpp"
+#include "duckdb/function/table/arrow.hpp"
 
 namespace duckdb {
 
@@ -127,7 +128,11 @@ struct ArrowScalarConverter {
 struct ArrowIntervalConverter {
 	template <class TGT, class SRC>
 	static TGT Operation(SRC input) {
-		return Interval::GetMilli(input);
+		ArrowInterval result;
+		result.months = input.months;
+		result.days = input.days;
+		result.nanoseconds = input.micros * Interval::NANOS_PER_MICRO;
+		return result;
 	}
 
 	static bool SkipNulls() {
@@ -136,7 +141,6 @@ struct ArrowIntervalConverter {
 
 	template <class TGT>
 	static void SetNull(TGT &value) {
-		value = 0;
 	}
 };
 
@@ -185,11 +189,51 @@ struct ArrowScalarData : public ArrowScalarBaseData<TGT, SRC, OP> {
 //===--------------------------------------------------------------------===//
 template <class TGT>
 struct ArrowEnumData : public ArrowScalarBaseData<TGT> {
+	static idx_t GetLength(string_t input) {
+		return input.GetSize();
+	}
+	static void WriteData(data_ptr_t target, string_t input) {
+		memcpy(target, input.GetDataUnsafe(), input.GetSize());
+	}
+	static void EnumAppendVector(ArrowAppendData &append_data, const Vector &input, idx_t size) {
+		D_ASSERT(input.GetVectorType() == VectorType::FLAT_VECTOR);
+
+		// resize the validity mask and set up the validity buffer for iteration
+		ResizeValidity(append_data.validity, append_data.row_count + size);
+
+		// resize the offset buffer - the offset buffer holds the offsets into the child array
+		append_data.main_buffer.resize(append_data.main_buffer.size() + sizeof(uint32_t) * (size + 1));
+		auto data = (string_t *)FlatVector::GetData<string_t>(input);
+		auto offset_data = (uint32_t *)append_data.main_buffer.data();
+		if (append_data.row_count == 0) {
+			// first entry
+			offset_data[0] = 0;
+		}
+		// now append the string data to the auxiliary buffer
+		// the auxiliary buffer's length depends on the string lengths, so we resize as required
+		auto last_offset = offset_data[append_data.row_count];
+		for (idx_t i = 0; i < size; i++) {
+			auto offset_idx = append_data.row_count + i + 1;
+
+			auto string_length = GetLength(data[i]);
+
+			// append the offset data
+			auto current_offset = last_offset + string_length;
+			offset_data[offset_idx] = current_offset;
+
+			// resize the string buffer if required, and write the string data
+			append_data.aux_buffer.resize(current_offset);
+			WriteData(append_data.aux_buffer.data() + last_offset, data[i]);
+
+			last_offset = current_offset;
+		}
+		append_data.row_count += size;
+	}
 	static void Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity) {
 		result.main_buffer.reserve(capacity * sizeof(TGT));
 		// construct the enum child data
 		auto enum_data = InitializeArrowChild(LogicalType::VARCHAR, EnumType::GetSize(type));
-		enum_data->append_vector(*enum_data, EnumType::GetValuesInsertOrder(type), EnumType::GetSize(type));
+		EnumAppendVector(*enum_data, EnumType::GetValuesInsertOrder(type), EnumType::GetSize(type));
 		result.child_data.push_back(std::move(enum_data));
 	}
 
@@ -629,7 +673,7 @@ static void InitializeFunctionPointers(ArrowAppendData &append_data, const Logic
 		}
 		break;
 	case LogicalTypeId::INTERVAL:
-		InitializeFunctionPointers<ArrowScalarData<int64_t, interval_t, ArrowIntervalConverter>>(append_data);
+		InitializeFunctionPointers<ArrowScalarData<ArrowInterval, interval_t, ArrowIntervalConverter>>(append_data);
 		break;
 	case LogicalTypeId::STRUCT:
 		InitializeFunctionPointers<ArrowStructData>(append_data);

package/src/duckdb/src/common/arrow/arrow_converter.cpp

@@ -150,7 +150,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 		child.format = "tsm:";
 		break;
 	case LogicalTypeId::INTERVAL:
-		child.format = "tDm";
+		child.format = "tin";
 		break;
 	case LogicalTypeId::DECIMAL: {
 		uint8_t width, scale;

package/src/duckdb/src/common/field_writer.cpp

@@ -8,6 +8,7 @@ namespace duckdb {
 FieldWriter::FieldWriter(Serializer &serializer_p)
     : serializer(serializer_p), buffer(make_unique<BufferedSerializer>()), field_count(0), finalized(false) {
 	buffer->SetVersion(serializer.GetVersion());
+	buffer->is_query_plan = serializer.is_query_plan;
 }
 
 FieldWriter::~FieldWriter() {

package/src/duckdb/src/common/serializer/buffered_deserializer.cpp

@@ -20,4 +20,8 @@ void BufferedDeserializer::ReadData(data_ptr_t buffer, idx_t read_size) {
 	ptr += read_size;
 }
 
+ClientContext &BufferedContextDeserializer::GetContext() {
+	return context;
+}
+
 } // namespace duckdb

package/src/duckdb/src/common/serializer/buffered_file_reader.cpp

@@ -7,8 +7,10 @@
 
 namespace duckdb {
 
-BufferedFileReader::BufferedFileReader(FileSystem &fs, const char *path, FileLockType lock_type, FileOpener *opener)
-    : fs(fs), data(unique_ptr<data_t[]>(new data_t[FILE_BUFFER_SIZE])), offset(0), read_data(0), total_read(0) {
+BufferedFileReader::BufferedFileReader(FileSystem &fs, const char *path, ClientContext *context, FileLockType lock_type,
+                                       FileOpener *opener)
+    : fs(fs), data(unique_ptr<data_t[]>(new data_t[FILE_BUFFER_SIZE])), offset(0), read_data(0), context(context),
+      total_read(0) {
 	handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ, lock_type, FileSystem::DEFAULT_COMPRESSION, opener);
 	file_size = fs.GetFileSize(*handle);
 }
@@ -54,4 +56,15 @@ uint64_t BufferedFileReader::CurrentOffset() {
 	return total_read + offset;
 }
 
+ClientContext &BufferedFileReader::GetContext() {
+	if (!context) {
+		throw InternalException("Trying to acquire a client context that does not exist");
+	}
+	return *context;
+}
+
+Catalog *BufferedFileReader::GetCatalog() {
+	return catalog;
+}
+
 } // namespace duckdb