duckdb 0.6.2-dev2115.0 → 0.6.2-dev2226.0

package/src/duckdb/extension/json/json_scan.cpp

@@ -92,6 +92,7 @@ void JSONScanData::Serialize(FieldWriter &writer) {
 	writer.WriteField<idx_t>(sample_size);
 	writer.WriteList<string>(names);
 	writer.WriteField<idx_t>(max_depth);
+	writer.WriteField<bool>(objects);
 }
 
 void JSONScanData::Deserialize(FieldReader &reader) {
@@ -105,6 +106,7 @@ void JSONScanData::Deserialize(FieldReader &reader) {
 	sample_size = reader.ReadRequired<idx_t>();
 	names = reader.ReadRequiredList<string>();
 	max_depth = reader.ReadRequired<idx_t>();
+	objects = reader.ReadRequired<bool>();
 }
 
 JSONScanGlobalState::JSONScanGlobalState(ClientContext &context, JSONScanData &bind_data_p)
@@ -144,16 +146,20 @@ unique_ptr<GlobalTableFunctionState> JSONGlobalTableFunctionState::Init(ClientCo
 	auto &bind_data = (JSONScanData &)*input.bind_data;
 	auto result = make_unique<JSONGlobalTableFunctionState>(context, input);
 
-	// Check if we need to do projection pushdown
-	if (bind_data.type == JSONScanType::READ_JSON && input.column_ids.size() != bind_data.names.size()) {
-		D_ASSERT(input.column_ids.size() < bind_data.names.size()); // Can't project to have more columns
+	// Perform projection pushdown
+	if (bind_data.type == JSONScanType::READ_JSON) {
+		D_ASSERT(input.column_ids.size() <= bind_data.names.size()); // Can't project to have more columns
+		if (bind_data.auto_detect && input.column_ids.size() < bind_data.names.size()) {
+			// If we are auto-detecting, but don't need all columns present in the file,
+			// then we don't need to throw an error if we encounter an unseen column
+			bind_data.transform_options.error_unknown_key = false;
+		}
 		vector<string> names;
 		names.reserve(input.column_ids.size());
 		for (const auto &id : input.column_ids) {
 			names.push_back(std::move(bind_data.names[id]));
 		}
 		bind_data.names = std::move(names);
-		bind_data.transform_options.error_unknown_key = false;
 	}
 	return result;
 }
@@ -170,7 +176,14 @@ unique_ptr<LocalTableFunctionState> JSONLocalTableFunctionState::Init(ExecutionC
                                                                       TableFunctionInitInput &input,
                                                                       GlobalTableFunctionState *global_state) {
 	auto &gstate = (JSONGlobalTableFunctionState &)*global_state;
-	return make_unique<JSONLocalTableFunctionState>(context.client, gstate.state);
+	auto result = make_unique<JSONLocalTableFunctionState>(context.client, gstate.state);
+
+	// Copy the transform options / date format map because we need to do thread-local stuff
+	result->state.date_format_map = gstate.state.bind_data.date_format_map;
+	result->state.transform_options = gstate.state.bind_data.transform_options;
+	result->state.transform_options.date_format_map = &result->state.date_format_map;
+
+	return result;
 }
 
 idx_t JSONLocalTableFunctionState::GetBatchIndex() const {
@@ -604,4 +617,12 @@ yyjson_alc *JSONScanLocalState::GetAllocator() {
 	return json_allocator.GetYYJSONAllocator();
 }
 
+void JSONScanLocalState::ThrowTransformError(idx_t count, idx_t object_index, const string &error_message) {
+	D_ASSERT(current_reader);
+	D_ASSERT(current_buffer_handle);
+	D_ASSERT(object_index != DConstants::INVALID_INDEX);
+	auto line_or_object_in_buffer = lines_or_objects_in_buffer - count + object_index;
+	current_reader->ThrowTransformError(current_buffer_handle->buffer_index, line_or_object_in_buffer, error_message);
+}
+
 } // namespace duckdb

package/src/duckdb/extension/parquet/parquet-extension.cpp

@@ -174,6 +174,7 @@ public:
 		table_function.named_parameters["file_row_number"] = LogicalType::BOOLEAN;
 		table_function.named_parameters["hive_partitioning"] = LogicalType::BOOLEAN;
 		table_function.named_parameters["union_by_name"] = LogicalType::BOOLEAN;
+		table_function.named_parameters["compression"] = LogicalType::VARCHAR;
 		table_function.get_batch_index = ParquetScanGetBatchIndex;
 		table_function.serialize = ParquetScanSerialize;
 		table_function.deserialize = ParquetScanDeserialize;

package/src/duckdb/src/catalog/catalog.cpp

@@ -552,20 +552,19 @@ SchemaCatalogEntry *Catalog::GetSchema(ClientContext &context, const string &cat
 	return result;
 }
 
+LogicalType Catalog::GetType(ClientContext &context, const string &schema, const string &name, bool if_exists) {
+	auto type_entry = GetEntry<TypeCatalogEntry>(context, schema, name, if_exists);
+	if (!type_entry) {
+		return LogicalType::INVALID;
+	}
+	auto result_type = type_entry->user_type;
+	LogicalType::SetCatalog(result_type, type_entry);
+	return result_type;
+}
+
 LogicalType Catalog::GetType(ClientContext &context, const string &catalog_name, const string &schema,
                              const string &name) {
-	CatalogEntry *entry;
-	entry = GetEntry(context, CatalogType::TYPE_ENTRY, catalog_name, schema, name, true);
-	if (!entry) {
-		// look in the system catalog
-		entry = GetEntry(context, CatalogType::TYPE_ENTRY, SYSTEM_CATALOG, schema, name, true);
-		if (!entry) {
-			// repeat the search to get the error
-			GetEntry(context, CatalogType::TYPE_ENTRY, catalog_name, schema, name);
-			throw InternalException("Catalog::GetType - second type lookup somehow succeeded!?");
-		}
-	}
-	auto type_entry = (TypeCatalogEntry *)entry;
+	auto type_entry = Catalog::GetEntry<TypeCatalogEntry>(context, catalog_name, schema, name);
 	auto result_type = type_entry->user_type;
 	LogicalType::SetCatalog(result_type, type_entry);
 	return result_type;

package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp

@@ -286,7 +286,7 @@ unique_ptr<CatalogEntry> DuckTableEntry::AddColumn(ClientContext &context, AddCo
 	for (auto &constraint : constraints) {
 		create_info->constraints.push_back(constraint->Copy());
 	}
-	Binder::BindLogicalType(context, info.new_column.TypeMutable(), catalog->GetName(), schema->name);
+	Binder::BindLogicalType(context, info.new_column.TypeMutable(), catalog, schema->name);
 	info.new_column.SetOid(columns.LogicalColumnCount());
 	info.new_column.SetStorageOid(columns.PhysicalColumnCount());
 	auto col = info.new_column.Copy();

package/src/duckdb/src/common/box_renderer.cpp

@@ -590,10 +590,18 @@ void BoxRenderer::Render(ClientContext &context, const vector<string> &names, co
 		bottom_rows = rows_to_render - top_rows;
 	}
 	auto row_count_str = to_string(row_count) + " rows";
+	bool has_limited_rows = config.limit > 0 && row_count == config.limit;
+	if (has_limited_rows) {
+		row_count_str = "? rows";
+	}
 	string shown_str;
 	bool has_hidden_rows = top_rows < row_count;
 	if (has_hidden_rows) {
-		shown_str = "(" + to_string(top_rows + bottom_rows) + " shown)";
+		shown_str = "(";
+		if (has_limited_rows) {
+			shown_str += ">" + to_string(config.limit - 1) + " rows, ";
+		}
+		shown_str += to_string(top_rows + bottom_rows) + " shown)";
 	}
 	auto minimum_row_length = MaxValue<idx_t>(row_count_str.size(), shown_str.size()) + 4;
 

package/src/duckdb/src/common/compressed_file_system.cpp

@@ -10,7 +10,7 @@ CompressedFile::CompressedFile(CompressedFileSystem &fs, unique_ptr<FileHandle>
 }
 
 CompressedFile::~CompressedFile() {
-	Close();
+	CompressedFile::Close();
 }
 
 void CompressedFile::Initialize(bool write) {

package/src/duckdb/src/common/enums/relation_type.cpp

@@ -43,6 +43,8 @@ string RelationTypeToString(RelationType type) {
 		return "UPDATE_RELATION";
 	case RelationType::WRITE_CSV_RELATION:
 		return "WRITE_CSV_RELATION";
+	case RelationType::WRITE_PARQUET_RELATION:
+		return "WRITE_PARQUET_RELATION";
 	case RelationType::READ_CSV_RELATION:
 		return "READ_CSV_RELATION";
 	case RelationType::SUBQUERY_RELATION:

package/src/duckdb/src/common/gzip_file_system.cpp

@@ -90,7 +90,7 @@ MiniZStreamWrapper::~MiniZStreamWrapper() {
 		return;
 	}
 	try {
-		Close();
+		MiniZStreamWrapper::Close();
 	} catch (...) {
 	}
 }

package/src/duckdb/src/common/local_file_system.cpp

@@ -123,7 +123,7 @@ public:
 	UnixFileHandle(FileSystem &file_system, string path, int fd) : FileHandle(file_system, std::move(path)), fd(fd) {
 	}
 	~UnixFileHandle() override {
-		Close();
+		UnixFileHandle::Close();
 	}
 
 	int fd;

package/src/duckdb/src/common/row_operations/row_aggregate.cpp

@@ -22,7 +22,7 @@ void RowOperations::InitializeStates(RowLayout &layout, Vector &addresses, const
 	auto &offsets = layout.GetOffsets();
 	auto aggr_idx = layout.ColumnCount();
 
-	for (auto &aggr : layout.GetAggregates()) {
+	for (const auto &aggr : layout.GetAggregates()) {
 		for (idx_t i = 0; i < count; ++i) {
 			auto row_idx = sel.get_index(i);
 			auto row = pointers[row_idx];
@@ -38,7 +38,7 @@ void RowOperations::DestroyStates(RowLayout &layout, Vector &addresses, idx_t co
 	}
 	//	Move to the first aggregate state
 	VectorOperations::AddInPlace(addresses, layout.GetAggrOffset(), count);
-	for (auto &aggr : layout.GetAggregates()) {
+	for (const auto &aggr : layout.GetAggregates()) {
 		if (aggr.function.destructor) {
 			aggr.function.destructor(addresses, count);
 		}

package/src/duckdb/src/common/types/column_data_allocator.cpp

@@ -182,8 +182,8 @@ void ColumnDataAllocator::UnswizzlePointers(ChunkManagementState &state, Vector
 	auto strings = FlatVector::GetData<string_t>(result);
 
 	// find first non-inlined string
-	auto i = v_offset;
-	const auto end = v_offset + count;
+	uint32_t i = v_offset;
+	const uint32_t end = v_offset + count;
 	for (; i < end; i++) {
 		if (!validity.RowIsValid(i)) {
 			continue;

package/src/duckdb/src/common/types/date.cpp

@@ -425,13 +425,18 @@ int64_t Date::Epoch(date_t date) {
 }
 
 int64_t Date::EpochNanoseconds(date_t date) {
-	return ((int64_t)date.days) * (Interval::MICROS_PER_DAY * 1000);
+	int64_t result;
+	if (!TryMultiplyOperator::Operation<int64_t, int64_t, int64_t>(date.days, Interval::MICROS_PER_DAY * 1000,
+	                                                               result)) {
+		throw ConversionException("Could not convert DATE (%s) to nanoseconds", Date::ToString(date));
+	}
+	return result;
 }
 
 int64_t Date::EpochMicroseconds(date_t date) {
 	int64_t result;
 	if (!TryMultiplyOperator::Operation<int64_t, int64_t, int64_t>(date.days, Interval::MICROS_PER_DAY, result)) {
-		throw ConversionException("Could not convert DATE to microseconds");
+		throw ConversionException("Could not convert DATE (%s) to microseconds", Date::ToString(date));
 	}
 	return result;
 }

package/src/duckdb/src/common/types/vector.cpp

@@ -652,7 +652,7 @@ string Vector::ToString(idx_t count) const {
 	return retval;
 }
 
-void Vector::Print(idx_t count) {
+void Vector::Print(idx_t count) const {
 	Printer::Print(ToString(count));
 }
 
@@ -676,7 +676,7 @@ string Vector::ToString() const {
 	return retval;
 }
 
-void Vector::Print() {
+void Vector::Print() const {
 	Printer::Print(ToString());
 }
 // LCOV_EXCL_STOP
@@ -1322,6 +1322,7 @@ void ConstantVector::Reference(Vector &vector, Vector &source, idx_t position, i
 			ConstantVector::Reference(*target_entries[i], *source_entries[i], position, count);
 		}
 		vector.SetVectorType(VectorType::CONSTANT_VECTOR);
+		vector.validity.Set(0, true);
 		break;
 	}
 	default:

package/src/duckdb/src/common/virtual_file_system.cpp

@@ -7,7 +7,7 @@
 namespace duckdb {
 
 VirtualFileSystem::VirtualFileSystem() : default_fs(FileSystem::CreateLocal()) {
-	RegisterSubSystem(FileCompressionType::GZIP, make_unique<GZipFileSystem>());
+	VirtualFileSystem::RegisterSubSystem(FileCompressionType::GZIP, make_unique<GZipFileSystem>());
 }
 
 unique_ptr<FileHandle> VirtualFileSystem::OpenFile(const string &path, uint8_t flags, FileLockType lock,

package/src/duckdb/src/execution/index/art/art.cpp

@@ -27,7 +27,7 @@ ART::ART(const vector<column_t> &column_ids, TableIOManager &table_io_manager,
 	tree = nullptr;
 	if (block_id != DConstants::INVALID_INDEX) {
 		tree = Node::Deserialize(*this, block_id, block_offset);
-		Verify();
+		ART::Verify();
 	}
 	serialized_data_pointer = BlockPointer(block_id, block_offset);
 
@@ -58,7 +58,7 @@ ART::~ART() {
 	if (!tree) {
 		return;
 	}
-	Verify();
+	ART::Verify();
 	if (track_memory) {
 		buffer_manager.DecreaseUsedMemory(memory_size);
 	}
@@ -70,7 +70,7 @@ ART::~ART() {
 // Initialize Predicate Scans
 //===--------------------------------------------------------------------===//
 
-unique_ptr<IndexScanState> ART::InitializeScanSinglePredicate(Transaction &transaction, Value value,
+unique_ptr<IndexScanState> ART::InitializeScanSinglePredicate(const Transaction &transaction, const Value &value,
                                                               ExpressionType expression_type) {
 	auto result = make_unique<ARTIndexScanState>();
 	result->values[0] = value;
@@ -78,8 +78,8 @@ unique_ptr<IndexScanState> ART::InitializeScanSinglePredicate(Transaction &trans
 	return std::move(result);
 }
 
-unique_ptr<IndexScanState> ART::InitializeScanTwoPredicates(Transaction &transaction, Value low_value,
-                                                            ExpressionType low_expression_type, Value high_value,
+unique_ptr<IndexScanState> ART::InitializeScanTwoPredicates(Transaction &transaction, const Value &low_value,
+                                                            ExpressionType low_expression_type, const Value &high_value,
                                                             ExpressionType high_expression_type) {
 	auto result = make_unique<ARTIndexScanState>();
 	result->values[0] = low_value;

package/src/duckdb/src/execution/join_hashtable.cpp

@@ -444,7 +444,6 @@ void ScanStructure::Next(DataChunk &keys, DataChunk &left, DataChunk &result) {
 	if (finished) {
 		return;
 	}
-
 	switch (ht.join_type) {
 	case JoinType::INNER:
 	case JoinType::RIGHT:
@@ -823,10 +822,10 @@ void ScanStructure::NextSingleJoin(DataChunk &keys, DataChunk &input, DataChunk
 	for (idx_t i = 0; i < ht.build_types.size(); i++) {
 		auto &vector = result.data[input.ColumnCount() + i];
 		// set NULL entries for every entry that was not found
-		auto &mask = FlatVector::Validity(vector);
-		mask.SetAllInvalid(input.size());
-		for (idx_t j = 0; j < result_count; j++) {
-			mask.SetValid(result_sel.get_index(j));
+		for (idx_t j = 0; j < input.size(); j++) {
+			if (!found_match[j]) {
+				FlatVector::SetNull(vector, j, true);
+			}
 		}
 		// for the remaining values we fetch the values
 		GatherResult(vector, result_sel, result_sel, result_count, i + ht.condition_types.size());

package/src/duckdb/src/execution/operator/persistent/physical_update.cpp

@@ -70,7 +70,9 @@ SinkResultType PhysicalUpdate::Sink(ExecutionContext &context, GlobalSinkState &
 	// update data in the base table
 	// the row ids are given to us as the last column of the child chunk
 	auto &row_ids = chunk.data[chunk.ColumnCount() - 1];
+	update_chunk.Reset();
 	update_chunk.SetCardinality(chunk);
+
 	for (idx_t i = 0; i < expressions.size(); i++) {
 		if (expressions[i]->type == ExpressionType::VALUE_DEFAULT) {
 			// default expression, set to the default value of the column