duckdb 0.5.1-dev13.0 → 0.5.1-dev134.0

package/src/duckdb.cpp

@@ -620,7 +620,88 @@ public:
 
 } // namespace duckdb
 
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// extension_functions.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
 
+
+
+namespace duckdb {
+
+struct ExtensionFunction {
+	char function[48];
+	char extension[48];
+};
+
+static constexpr ExtensionFunction EXTENSION_FUNCTIONS[] = {
+    {"->>", "json"},
+    {"array_to_json", "json"},
+    {"create_fts_index", "fts"},
+    {"dbgen", "tpch"},
+    {"drop_fts_index", "fts"},
+    {"dsdgen", "tpcds"},
+    {"excel_text", "excel"},
+    {"from_json", "json"},
+    {"from_json_strict", "json"},
+    {"from_substrait", "substrait"},
+    {"get_substrait", "substrait"},
+    {"get_substrait_json", "substrait"},
+    {"icu_calendar_names", "icu"},
+    {"icu_sort_key", "icu"},
+    {"json", "json"},
+    {"json_array", "json"},
+    {"json_array_length", "json"},
+    {"json_extract", "json"},
+    {"json_extract_path", "json"},
+    {"json_extract_path_text", "json"},
+    {"json_extract_string", "json"},
+    {"json_group_array", "json"},
+    {"json_group_object", "json"},
+    {"json_group_structure", "json"},
+    {"json_merge_patch", "json"},
+    {"json_object", "json"},
+    {"json_quote", "json"},
+    {"json_structure", "json"},
+    {"json_transform", "json"},
+    {"json_transform_strict", "json"},
+    {"json_type", "json"},
+    {"json_valid", "json"},
+    {"make_timestamptz", "icu"},
+    {"parquet_metadata", "parquet"},
+    {"parquet_scan", "parquet"},
+    {"parquet_schema", "parquet"},
+    {"pg_timezone_names", "icu"},
+    {"postgres_attach", "postgres_scanner"},
+    {"postgres_scan", "postgres_scanner"},
+    {"postgres_scan_pushdown", "postgres_scanner"},
+    {"read_json_objects", "json"},
+    {"read_ndjson_objects", "json"},
+    {"read_parquet", "parquet"},
+    {"row_to_json", "json"},
+    {"sqlite_attach", "sqlite_scanner"},
+    {"sqlite_scan", "sqlite_scanner"},
+    {"stem", "fts"},
+    {"text", "excel"},
+    {"to_json", "json"},
+    {"tpcds", "tpcds"},
+    {"tpcds_answers", "tpcds"},
+    {"tpcds_queries", "tpcds"},
+    {"tpch", "tpch"},
+    {"tpch_answers", "tpch"},
+    {"tpch_queries", "tpch"},
+    {"visualize_diff_profiling_output", "visualizer"},
+    {"visualize_json_profiling_output", "visualizer"},
+    {"visualize_last_profiling_output", "visualizer"},
+};
+} // namespace duckdb
+
+#include <algorithm>
 namespace duckdb {
 
 string SimilarCatalogEntry::GetQualifiedName() const {
@@ -823,6 +904,16 @@ SimilarCatalogEntry Catalog::SimilarEntryInSchemas(ClientContext &context, const
 	return {most_similar.first, most_similar.second, schema_of_most_similar};
 }
 
+string FindExtension(const string &function_name) {
+	auto size = sizeof(EXTENSION_FUNCTIONS) / sizeof(ExtensionFunction);
+	auto it = std::lower_bound(
+	    EXTENSION_FUNCTIONS, EXTENSION_FUNCTIONS + size, function_name,
+	    [](const ExtensionFunction &element, const string &value) { return element.function < value; });
+	if (it != EXTENSION_FUNCTIONS + size && it->function == function_name) {
+		return it->extension;
+	}
+	return "";
+}
 CatalogException Catalog::CreateMissingEntryException(ClientContext &context, const string &entry_name,
                                                       CatalogType type, const vector<SchemaCatalogEntry *> &schemas,
                                                       QueryErrorContext error_context) {
@@ -836,7 +927,12 @@ CatalogException Catalog::CreateMissingEntryException(ClientContext &context, co
 		}
 	});
 	auto unseen_entry = SimilarEntryInSchemas(context, entry_name, type, unseen_schemas);
-
+	auto extension_name = FindExtension(entry_name);
+	if (!extension_name.empty()) {
+		return CatalogException("Function with name %s is not on the catalog, but it exists in the %s extension. To "
+		                        "Install and Load the extension, run: INSTALL %s; LOAD %s;",
+		                        entry_name, extension_name, extension_name, extension_name);
+	}
 	string did_you_mean;
 	if (unseen_entry.Found() && unseen_entry.distance < entry.distance) {
 		did_you_mean = "\nDid you mean \"" + unseen_entry.GetQualifiedName() + "\"?";
@@ -4934,12 +5030,16 @@ bool CatalogSet::AlterEntry(ClientContext &context, const string &name, AlterInf
 				throw CatalogException(rename_err_msg, original_name, value->name);
 			}
 		}
-		PutMapping(context, value->name, entry_index);
-		DeleteMapping(context, original_name);
 	}
 	//! Check the dependency manager to verify that there are no conflicting dependencies with this alter
 	catalog.dependency_manager->AlterObject(context, entry, value.get());
 
+	if (value->name != original_name) {
+		// Do PutMapping and DeleteMapping after dependency check
+		PutMapping(context, value->name, entry_index);
+		DeleteMapping(context, original_name);
+	}
+
 	value->timestamp = transaction.transaction_id;
 	value->child = move(entries[entry_index]);
 	value->child->parent = value.get();
@@ -6506,7 +6606,7 @@ static void GetBitPosition(idx_t row_idx, idx_t &current_byte, uint8_t &current_
 }
 
 static void UnsetBit(uint8_t *data, idx_t current_byte, uint8_t current_bit) {
-	data[current_byte] &= ~(1 << current_bit);
+	data[current_byte] &= ~((uint64_t)1 << current_bit);
 }
 
 static void NextBit(idx_t &current_byte, uint8_t &current_bit) {
@@ -28558,7 +28658,7 @@ template <idx_t radix_bits>
 struct RadixPartitioningConstants {
 public:
 	static constexpr const idx_t NUM_RADIX_BITS = radix_bits;
-	static constexpr const idx_t NUM_PARTITIONS = 1 << NUM_RADIX_BITS;
+	static constexpr const idx_t NUM_PARTITIONS = (idx_t)1 << NUM_RADIX_BITS;
 	static constexpr const idx_t TMP_BUF_SIZE = 8;
 
 public:
@@ -28576,7 +28676,7 @@ private:
 struct RadixPartitioning {
 public:
 	static idx_t NumberOfPartitions(idx_t radix_bits) {
-		return 1 << radix_bits;
+		return (idx_t)1 << radix_bits;
 	}
 
 	//! Partition the data in block_collection/string_heap to multiple partitions
@@ -33336,6 +33436,9 @@ void RowOperations::UnswizzleHeapPointer(const RowLayout &layout, const data_ptr
 
 static inline void VerifyUnswizzledString(const RowLayout &layout, const idx_t &col_idx, const data_ptr_t &row_ptr) {
 #ifdef DEBUG
+	if (layout.GetTypes()[col_idx] == LogicalTypeId::BLOB) {
+		return;
+	}
 	idx_t entry_idx;
 	idx_t idx_in_entry;
 	ValidityBytes::GetEntryIndex(col_idx, entry_idx, idx_in_entry);
@@ -35673,7 +35776,10 @@ struct SortConstants {
 
 struct SortLayout {
 public:
+	SortLayout() {
+	}
 	explicit SortLayout(const vector<BoundOrderByNode> &orders);
+	SortLayout GetPrefixComparisonLayout(idx_t num_prefix_cols) const;
 
 public:
 	idx_t column_count;
@@ -37324,6 +37430,32 @@ SortLayout::SortLayout(const vector<BoundOrderByNode> &orders)
 	blob_layout.Initialize(blob_layout_types);
 }
 
+SortLayout SortLayout::GetPrefixComparisonLayout(idx_t num_prefix_cols) const {
+	SortLayout result;
+	result.column_count = num_prefix_cols;
+	result.all_constant = true;
+	result.comparison_size = 0;
+	for (idx_t col_idx = 0; col_idx < num_prefix_cols; col_idx++) {
+		result.order_types.push_back(order_types[col_idx]);
+		result.order_by_null_types.push_back(order_by_null_types[col_idx]);
+		result.logical_types.push_back(logical_types[col_idx]);
+
+		result.all_constant = result.all_constant && constant_size[col_idx];
+		result.constant_size.push_back(constant_size[col_idx]);
+
+		result.comparison_size += column_sizes[col_idx];
+		result.column_sizes.push_back(column_sizes[col_idx]);
+
+		result.prefix_lengths.push_back(prefix_lengths[col_idx]);
+		result.stats.push_back(stats[col_idx]);
+		result.has_null.push_back(has_null[col_idx]);
+	}
+	result.entry_size = entry_size;
+	result.blob_layout = blob_layout;
+	result.sorting_to_blob_col = sorting_to_blob_col;
+	return result;
+}
+
 LocalSortState::LocalSortState() : initialized(false) {
 }
 
@@ -47573,11 +47705,36 @@ Value Value::CreateValue(dtime_t value) {
 	return Value::TIME(value);
 }
 
+template <>
+Value Value::CreateValue(dtime_tz_t value) {
+	return Value::TIMETZ(value);
+}
+
 template <>
 Value Value::CreateValue(timestamp_t value) {
 	return Value::TIMESTAMP(value);
 }
 
+template <>
+Value Value::CreateValue(timestamp_sec_t value) {
+	return Value::TIMESTAMPSEC(value);
+}
+
+template <>
+Value Value::CreateValue(timestamp_ms_t value) {
+	return Value::TIMESTAMPMS(value);
+}
+
+template <>
+Value Value::CreateValue(timestamp_ns_t value) {
+	return Value::TIMESTAMPNS(value);
+}
+
+template <>
+Value Value::CreateValue(timestamp_tz_t value) {
+	return Value::TIMESTAMPTZ(value);
+}
+
 template <>
 Value Value::CreateValue(const char *value) {
 	return Value(string(value));
@@ -49150,19 +49307,6 @@ void Vector::Resize(idx_t cur_size, idx_t new_size) {
 	}
 }
 
-// FIXME Just like DECIMAL, it's important that type_info gets considered when determining whether or not to cast
-// just comparing internal type is not always enough
-static bool ValueShouldBeCast(const LogicalType &incoming, const LogicalType &target) {
-	if (incoming.InternalType() != target.InternalType()) {
-		return true;
-	}
-	if (incoming.id() == LogicalTypeId::DECIMAL && incoming.id() == target.id()) {
-		//! Compare the type_info
-		return incoming != target;
-	}
-	return false;
-}
-
 void Vector::SetValue(idx_t index, const Value &val) {
 	if (GetVectorType() == VectorType::DICTIONARY_VECTOR) {
 		// dictionary: apply dictionary and forward to child
@@ -49170,10 +49314,11 @@ void Vector::SetValue(idx_t index, const Value &val) {
 		auto &child = DictionaryVector::Child(*this);
 		return child.SetValue(sel_vector.get_index(index), val);
 	}
-	if (ValueShouldBeCast(val.type(), GetType())) {
+	if (val.type() != GetType()) {
 		SetValue(index, val.CastAs(GetType()));
 		return;
 	}
+	D_ASSERT(val.type().InternalType() == GetType().InternalType());
 
 	validity.EnsureWritable();
 	validity.Set(index, !val.IsNull());
@@ -49424,7 +49569,10 @@ Value Vector::GetValue(const Vector &v_p, idx_t index_p) {
 	auto value = GetValueInternal(v_p, index_p);
 	// set the alias of the type to the correct value, if there is a type alias
 	if (v_p.GetType().HasAlias()) {
-		value.type().SetAlias(v_p.GetType().GetAlias());
+		value.type().CopyAuxInfo(v_p.GetType());
+	}
+	if (v_p.GetType().id() != LogicalTypeId::AGGREGATE_STATE && value.type().id() != LogicalTypeId::AGGREGATE_STATE) {
+		D_ASSERT(v_p.GetType() == value.type());
 	}
 	return value;
 }
@@ -51491,6 +51639,7 @@ public:
 				if (!alias.empty()) {
 					return false;
 				}
+				//! We only need to compare aliases when both types have them in this case
 				return true;
 			}
 			if (alias != other_p->alias) {
@@ -51504,8 +51653,7 @@ public:
 		if (type != other_p->type) {
 			return false;
 		}
-		auto &other = (ExtraTypeInfo &)*other_p;
-		return alias == other.alias && EqualsInternal(other_p);
+		return alias == other_p->alias && EqualsInternal(other_p);
 	}
 	//! Serializes a ExtraTypeInfo to a stand-alone binary blob
 	virtual void Serialize(FieldWriter &writer) const {};
@@ -52184,10 +52332,7 @@ LogicalType LogicalType::Deserialize(Deserializer &source) {
 	return LogicalType(id, move(info));
 }
 
-bool LogicalType::operator==(const LogicalType &rhs) const {
-	if (id_ != rhs.id_) {
-		return false;
-	}
+bool LogicalType::EqualTypeInfo(const LogicalType &rhs) const {
 	if (type_info_.get() == rhs.type_info_.get()) {
 		return true;
 	}
@@ -52199,6 +52344,13 @@ bool LogicalType::operator==(const LogicalType &rhs) const {
 	}
 }
 
+bool LogicalType::operator==(const LogicalType &rhs) const {
+	if (id_ != rhs.id_) {
+		return false;
+	}
+	return EqualTypeInfo(rhs);
+}
+
 } // namespace duckdb
 
 
@@ -64927,12 +65079,14 @@ public:
 
 	WindowGlobalHashGroup(BufferManager &buffer_manager, const Orders &partitions, const Orders &orders,
 	                      const Types &payload_types, idx_t max_mem, bool external)
-	    : memory_per_thread(max_mem), count(0), partition_layout(partitions) {
+	    : memory_per_thread(max_mem), count(0) {
 
 		RowLayout payload_layout;
 		payload_layout.Initialize(payload_types);
 		global_sort = make_unique<GlobalSortState>(buffer_manager, orders, payload_layout);
 		global_sort->external = external;
+
+		partition_layout = global_sort->sort_layout.GetPrefixComparisonLayout(partitions.size());
 	}
 
 	void Combine(LocalSortState &local_sort) {
@@ -80935,7 +81089,7 @@ PerfectAggregateHashTable::PerfectAggregateHashTable(Allocator &allocator, Buffe
 		total_required_bits += group_bits;
 	}
 	// the total amount of groups we allocate space for is 2^required_bits
-	total_groups = 1 << total_required_bits;
+	total_groups = (uint64_t)1 << total_required_bits;
 	// we don't need to store the groups in a perfect hash table, since the group keys can be deduced by their location
 	grouping_columns = group_types_p.size();
 	layout.Initialize(move(aggregate_objects_p));
@@ -81119,7 +81273,7 @@ static void ReconstructGroupVectorTemplated(uint32_t group_values[], Value &min,
 static void ReconstructGroupVector(uint32_t group_values[], Value &min, idx_t required_bits, idx_t shift,
                                    idx_t entry_count, Vector &result) {
 	// construct the mask for this entry
-	idx_t mask = (1 << required_bits) - 1;
+	idx_t mask = ((uint64_t)1 << required_bits) - 1;
 	switch (result.GetType().InternalType()) {
 	case PhysicalType::INT8:
 		ReconstructGroupVectorTemplated<int8_t>(group_values, min, mask, shift, entry_count, result);
@@ -85366,7 +85520,7 @@ void RadixPartitionedHashTable::SetGroupingValues() {
 		for (idx_t i = 0; i < grouping.size(); i++) {
 			if (grouping_set.find(grouping[i]) == grouping_set.end()) {
 				// we don't group on this value!
-				grouping_value += 1 << (grouping.size() - (i + 1));
+				grouping_value += (int64_t)1 << (grouping.size() - (i + 1));
 			}
 		}
 		grouping_values.push_back(Value::BIGINT(grouping_value));
@@ -93281,21 +93435,21 @@ AggregateFunction GetHistogramFunction(const LogicalType &type) {
 	case LogicalType::VARCHAR:
 		return GetMapType<HistogramStringFunctor, string, IS_ORDERED>(type);
 	case LogicalType::TIMESTAMP:
-		return GetMapType<HistogramFunctor, int64_t, IS_ORDERED>(type);
+		return GetMapType<HistogramFunctor, timestamp_t, IS_ORDERED>(type);
 	case LogicalType::TIMESTAMP_TZ:
-		return GetMapType<HistogramFunctor, int64_t, IS_ORDERED>(type);
+		return GetMapType<HistogramFunctor, timestamp_tz_t, IS_ORDERED>(type);
 	case LogicalType::TIMESTAMP_S:
-		return GetMapType<HistogramFunctor, int64_t, IS_ORDERED>(type);
+		return GetMapType<HistogramFunctor, timestamp_sec_t, IS_ORDERED>(type);
 	case LogicalType::TIMESTAMP_MS:
-		return GetMapType<HistogramFunctor, int64_t, IS_ORDERED>(type);
+		return GetMapType<HistogramFunctor, timestamp_ms_t, IS_ORDERED>(type);
 	case LogicalType::TIMESTAMP_NS:
-		return GetMapType<HistogramFunctor, int64_t, IS_ORDERED>(type);
+		return GetMapType<HistogramFunctor, timestamp_ns_t, IS_ORDERED>(type);
 	case LogicalType::TIME:
-		return GetMapType<HistogramFunctor, int64_t, IS_ORDERED>(type);
+		return GetMapType<HistogramFunctor, dtime_t, IS_ORDERED>(type);
 	case LogicalType::TIME_TZ:
-		return GetMapType<HistogramFunctor, int64_t, IS_ORDERED>(type);
+		return GetMapType<HistogramFunctor, dtime_tz_t, IS_ORDERED>(type);
 	case LogicalType::DATE:
-		return GetMapType<HistogramFunctor, int32_t, IS_ORDERED>(type);
+		return GetMapType<HistogramFunctor, date_t, IS_ORDERED>(type);
 	default:
 		throw InternalException("Unimplemented histogram aggregate");
 	}
@@ -96859,7 +97013,8 @@ struct DateDiff {
 	struct WeekOperator {
 		template <class TA, class TB, class TR>
 		static inline TR Operation(TA startdate, TB enddate) {
-			return Date::Epoch(enddate) / Interval::SECS_PER_WEEK - Date::Epoch(startdate) / Interval::SECS_PER_WEEK;
+			return Date::Epoch(Date::GetMondayOfCurrentWeek(enddate)) / Interval::SECS_PER_WEEK -
+			       Date::Epoch(Date::GetMondayOfCurrentWeek(startdate)) / Interval::SECS_PER_WEEK;
 		}
 	};
 
@@ -103243,12 +103398,49 @@ static void ListAggregatesFunction(DataChunk &args, ExpressionState &state, Vect
 			    result, state_vector.state_vector, count);
 			break;
 		case PhysicalType::INT32:
-			FUNCTION_FUNCTOR::template ListExecuteFunction<FinalizeValueFunctor, int32_t>(
-			    result, state_vector.state_vector, count);
+			if (key_type.id() == LogicalTypeId::DATE) {
+				FUNCTION_FUNCTOR::template ListExecuteFunction<FinalizeValueFunctor, date_t>(
+				    result, state_vector.state_vector, count);
+			} else {
+				FUNCTION_FUNCTOR::template ListExecuteFunction<FinalizeValueFunctor, int32_t>(
+				    result, state_vector.state_vector, count);
+			}
 			break;
 		case PhysicalType::INT64:
-			FUNCTION_FUNCTOR::template ListExecuteFunction<FinalizeValueFunctor, int64_t>(
-			    result, state_vector.state_vector, count);
+			switch (key_type.id()) {
+			case LogicalTypeId::TIME:
+				FUNCTION_FUNCTOR::template ListExecuteFunction<FinalizeValueFunctor, dtime_t>(
+				    result, state_vector.state_vector, count);
+				break;
+			case LogicalTypeId::TIME_TZ:
+				FUNCTION_FUNCTOR::template ListExecuteFunction<FinalizeValueFunctor, dtime_tz_t>(
+				    result, state_vector.state_vector, count);
+				break;
+			case LogicalTypeId::TIMESTAMP:
+				FUNCTION_FUNCTOR::template ListExecuteFunction<FinalizeValueFunctor, timestamp_t>(
+				    result, state_vector.state_vector, count);
+				break;
+			case LogicalTypeId::TIMESTAMP_MS:
+				FUNCTION_FUNCTOR::template ListExecuteFunction<FinalizeValueFunctor, timestamp_ms_t>(
+				    result, state_vector.state_vector, count);
+				break;
+			case LogicalTypeId::TIMESTAMP_NS:
+				FUNCTION_FUNCTOR::template ListExecuteFunction<FinalizeValueFunctor, timestamp_ns_t>(
+				    result, state_vector.state_vector, count);
+				break;
+			case LogicalTypeId::TIMESTAMP_SEC:
+				FUNCTION_FUNCTOR::template ListExecuteFunction<FinalizeValueFunctor, timestamp_sec_t>(
+				    result, state_vector.state_vector, count);
+				break;
+			case LogicalTypeId::TIMESTAMP_TZ:
+				FUNCTION_FUNCTOR::template ListExecuteFunction<FinalizeValueFunctor, timestamp_tz_t>(
+				    result, state_vector.state_vector, count);
+				break;
+			default:
+				FUNCTION_FUNCTOR::template ListExecuteFunction<FinalizeValueFunctor, int64_t>(
+				    result, state_vector.state_vector, count);
+				break;
+			}
 			break;
 		case PhysicalType::FLOAT:
 			FUNCTION_FUNCTOR::template ListExecuteFunction<FinalizeValueFunctor, float>(
@@ -104318,18 +104510,12 @@ void SinkDataChunk(Vector *child_vector, SelectionVector &sel, idx_t offset_list
 static void ListSortFunction(DataChunk &args, ExpressionState &state, Vector &result) {
 	D_ASSERT(args.ColumnCount() >= 1 && args.ColumnCount() <= 3);
 	auto count = args.size();
-	Vector &lists = args.data[0];
+	Vector &input_lists = args.data[0];
 
 	result.SetVectorType(VectorType::FLAT_VECTOR);
 	auto &result_validity = FlatVector::Validity(result);
 
-	for (auto &v : args.data) {
-		if (v.GetVectorType() != VectorType::FLAT_VECTOR && v.GetVectorType() != VectorType::CONSTANT_VECTOR) {
-			v.Flatten(count);
-		}
-	}
-
-	if (lists.GetType().id() == LogicalTypeId::SQLNULL) {
+	if (input_lists.GetType().id() == LogicalTypeId::SQLNULL) {
 		result_validity.SetInvalid(0);
 		return;
 	}
@@ -104344,15 +104530,18 @@ static void ListSortFunction(DataChunk &args, ExpressionState &state, Vector &re
 	LocalSortState local_sort_state;
 	local_sort_state.Initialize(global_sort_state, buffer_manager);
 
+	// this ensures that we do not change the order of the entries in the input chunk
+	VectorOperations::Copy(input_lists, result, count, 0, 0);
+
 	// get the child vector
-	auto lists_size = ListVector::GetListSize(lists);
-	auto &child_vector = ListVector::GetEntry(lists);
+	auto lists_size = ListVector::GetListSize(result);
+	auto &child_vector = ListVector::GetEntry(result);
 	UnifiedVectorFormat child_data;
 	child_vector.ToUnifiedFormat(lists_size, child_data);
 
 	// get the lists data
 	UnifiedVectorFormat lists_data;
-	lists.ToUnifiedFormat(count, lists_data);
+	result.ToUnifiedFormat(count, lists_data);
 	auto list_entries = (list_entry_t *)lists_data.data;
 
 	// create the lists_indices vector, this contains an element for each list's entry,
@@ -104449,8 +104638,6 @@ static void ListSortFunction(DataChunk &args, ExpressionState &state, Vector &re
 		child_vector.Flatten(sel_sorted_idx);
 	}
 
-	result.Reference(lists);
-
 	if (args.AllConstant()) {
 		result.SetVectorType(VectorType::CONSTANT_VECTOR);
 	}
@@ -117625,6 +117812,7 @@ static void ReadCSVAddNamedParameters(TableFunction &table_function) {
 	table_function.named_parameters["skip"] = LogicalType::BIGINT;
 	table_function.named_parameters["max_line_size"] = LogicalType::VARCHAR;
 	table_function.named_parameters["maximum_line_size"] = LogicalType::VARCHAR;
+	table_function.named_parameters["ignore_errors"] = LogicalType::BOOLEAN;
 }
 
 double CSVReaderProgress(ClientContext &context, const FunctionData *bind_data_p,
@@ -123028,7 +123216,7 @@ bool duckdb_validity_row_is_valid(uint64_t *validity, idx_t row) {
 	}
 	idx_t entry_idx = row / 64;
 	idx_t idx_in_entry = row % 64;
-	return validity[entry_idx] & (1 << idx_in_entry);
+	return validity[entry_idx] & ((idx_t)1 << idx_in_entry);
 }
 
 void duckdb_validity_set_row_validity(uint64_t *validity, idx_t row, bool valid) {
@@ -123045,7 +123233,7 @@ void duckdb_validity_set_row_invalid(uint64_t *validity, idx_t row) {
 	}
 	idx_t entry_idx = row / 64;
 	idx_t idx_in_entry = row % 64;
-	validity[entry_idx] &= ~(1 << idx_in_entry);
+	validity[entry_idx] &= ~((uint64_t)1 << idx_in_entry);
 }
 
 void duckdb_validity_set_row_valid(uint64_t *validity, idx_t row) {
@@ -123054,7 +123242,7 @@ void duckdb_validity_set_row_valid(uint64_t *validity, idx_t row) {
 	}
 	idx_t entry_idx = row / 64;
 	idx_t idx_in_entry = row % 64;
-	validity[entry_idx] |= 1 << idx_in_entry;
+	validity[entry_idx] |= (uint64_t)1 << idx_in_entry;
 }
 
 
@@ -141402,7 +141590,7 @@ Value ForceCompressionSetting::GetSetting(ClientContext &context) {
 //===--------------------------------------------------------------------===//
 void HomeDirectorySetting::SetLocal(ClientContext &context, const Value &input) {
 	auto &config = ClientConfig::GetConfig(context);
-	config.home_directory = input.IsNull() ? input.ToString() : string();
+	config.home_directory = input.IsNull() ? string() : input.ToString();
 }
 
 Value HomeDirectorySetting::GetSetting(ClientContext &context) {
@@ -151925,6 +152113,7 @@ unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalGet
 
 
 
+
 namespace duckdb {
 
 void StatisticsPropagator::PropagateStatistics(LogicalComparisonJoin &join, unique_ptr<LogicalOperator> *node_ptr) {
@@ -151954,10 +152143,15 @@ void StatisticsPropagator::PropagateStatistics(LogicalComparisonJoin &join, uniq
 					// semi or inner join on false; entire node can be pruned
 					ReplaceWithEmptyResult(*node_ptr);
 					return;
-				case JoinType::ANTI:
-					// anti join: replace entire join with LHS
-					*node_ptr = move(join.children[0]);
+				case JoinType::ANTI: {
+					// when the right child has data, return the left child
+					// when the right child has no data, return an empty set
+					auto limit = make_unique<LogicalLimit>(1, 0, nullptr, nullptr);
+					limit->AddChild(move(join.children[1]));
+					auto cross_product = LogicalCrossProduct::Create(move(join.children[0]), move(limit));
+					*node_ptr = move(cross_product);
 					return;
+				}
 				case JoinType::LEFT:
 					// anti/left outer join: replace right side with empty node
 					ReplaceWithEmptyResult(join.children[1]);
@@ -151985,10 +152179,15 @@ void StatisticsPropagator::PropagateStatistics(LogicalComparisonJoin &join, uniq
 				} else {
 					// this is the only condition and it is always true: all conditions are true
 					switch (join.join_type) {
-					case JoinType::SEMI:
-						// semi join on true: replace entire join with LHS
-						*node_ptr = move(join.children[0]);
+					case JoinType::SEMI: {
+						// when the right child has data, return the left child
+						// when the right child has no data, return an empty set
+						auto limit = make_unique<LogicalLimit>(1, 0, nullptr, nullptr);
+						limit->AddChild(move(join.children[1]));
+						auto cross_product = LogicalCrossProduct::Create(move(join.children[0]), move(limit));
+						*node_ptr = move(cross_product);
 						return;
+					}
 					case JoinType::INNER:
 					case JoinType::LEFT:
 					case JoinType::RIGHT:
@@ -152105,6 +152304,7 @@ unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalJoin
 	// then propagate into the join conditions
 	switch (join.type) {
 	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN:
+	case LogicalOperatorType::LOGICAL_DELIM_JOIN:
 		PropagateStatistics((LogicalComparisonJoin &)join, node_ptr);
 		break;
 	case LogicalOperatorType::LOGICAL_ANY_JOIN:
@@ -171348,7 +171548,7 @@ void Transformer::TransformCTE(duckdb_libpgquery::PGWithClause *de_with_clause,
 		}
 		// we need a query
 		if (!cte->ctequery || cte->ctequery->type != duckdb_libpgquery::T_PGSelectStmt) {
-			throw InternalException("A CTE needs a SELECT");
+			throw NotImplementedException("A CTE needs a SELECT");
 		}
 
 		// CTE transformation can either result in inlining for non recursive CTEs, or in recursive CTE bindings
@@ -171765,7 +171965,7 @@ LogicalType Transformer::TransformTypeName(duckdb_libpgquery::PGTypeName *type_n
 
 		result_type = LogicalType::MAP(move(children));
 	} else {
-		int8_t width, scale;
+		int64_t width, scale;
 		if (base_type == LogicalTypeId::DECIMAL) {
 			// default decimal width/scale
 			width = 18;
@@ -179786,11 +179986,13 @@ unique_ptr<BoundCreateTableInfo> Binder::BindCreateTableInfo(unique_ptr<CreateIn
 		BindDefaultValues(base.columns, result->bound_defaults);
 	}
 
+	idx_t regular_column_count = 0;
 	// bind collations to detect any unsupported collation errors
 	for (auto &column : base.columns) {
 		if (column.Generated()) {
 			continue;
 		}
+		regular_column_count++;
 		if (column.Type().id() == LogicalTypeId::VARCHAR) {
 			ExpressionBinder::TestCollation(context, StringType::GetCollation(column.Type()));
 		}
@@ -179802,6 +180004,9 @@ unique_ptr<BoundCreateTableInfo> Binder::BindCreateTableInfo(unique_ptr<CreateIn
 			result->dependencies.insert(type_dependency);
 		}
 	}
+	if (regular_column_count == 0) {
+		throw BinderException("Creating a table without physical (non-generated) columns is not supported");
+	}
 	properties.allow_stream_result = false;
 	return result;
 }
@@ -181189,7 +181394,20 @@ BoundStatement Binder::Bind(VacuumStatement &stmt) {
 			auto &get = (LogicalGet &)*ref->get;
 			columns.insert(columns.end(), get.names.begin(), get.names.end());
 		}
+
+		case_insensitive_set_t column_name_set;
+		vector<string> non_generated_column_names;
 		for (auto &col_name : columns) {
+			if (column_name_set.count(col_name) > 0) {
+				throw BinderException("Vacuum the same column twice(same name in column name list)");
+			}
+			column_name_set.insert(col_name);
+			auto &col = ref->table->GetColumn(col_name);
+			// ignore generated column
+			if (col.Generated()) {
+				continue;
+			}
+			non_generated_column_names.push_back(col_name);
 			ColumnRefExpression colref(col_name, ref->table->name);
 			auto result = bind_context.BindColumn(colref, 0);
 			if (result.HasError()) {
@@ -181197,17 +181415,29 @@ BoundStatement Binder::Bind(VacuumStatement &stmt) {
 			}
 			select_list.push_back(move(result.expression));
 		}
-		auto table_scan = CreatePlan(*ref);
-		D_ASSERT(table_scan->type == LogicalOperatorType::LOGICAL_GET);
-		auto &get = (LogicalGet &)*table_scan;
-		for (idx_t i = 0; i < get.column_ids.size(); i++) {
-			stmt.info->column_id_map[i] = get.column_ids[i];
-		}
+		stmt.info->columns = move(non_generated_column_names);
+		if (!select_list.empty()) {
+			auto table_scan = CreatePlan(*ref);
+			D_ASSERT(table_scan->type == LogicalOperatorType::LOGICAL_GET);
 
-		auto projection = make_unique<LogicalProjection>(GenerateTableIndex(), move(select_list));
-		projection->children.push_back(move(table_scan));
+			auto &get = (LogicalGet &)*table_scan;
+
+			D_ASSERT(select_list.size() == get.column_ids.size());
+			D_ASSERT(stmt.info->columns.size() == get.column_ids.size());
+			for (idx_t i = 0; i < get.column_ids.size(); i++) {
+				stmt.info->column_id_map[i] = ref->table->columns[get.column_ids[i]].StorageOid();
+			}
+
+			auto projection = make_unique<LogicalProjection>(GenerateTableIndex(), move(select_list));
+			projection->children.push_back(move(table_scan));
 
-		root = move(projection);
+			root = move(projection);
+		} else {
+			// eg. CREATE TABLE test (x AS (1));
+			//     ANALYZE test;
+			// Make it not a SINK so it doesn't have to do anything
+			stmt.info->has_table = false;
+		}
 	}
 	auto vacuum = make_unique<LogicalSimple>(LogicalOperatorType::LOGICAL_VACUUM, move(stmt.info));
 	if (root) {
@@ -322193,6 +322423,8 @@ exit:
 // See the end of this file for a list
 
 
+// otherwise we have different definitions for mbedtls_pk_context / mbedtls_sha256_context
+#define MBEDTLS_ALLOW_PRIVATE_ACCESS