duckdb 0.3.2-dev1304.0 → 0.3.2-dev1316.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.3.2-dev1304.0",
+  "version": "0.3.2-dev1316.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -3101,7 +3101,15 @@ string TableCatalogEntry::ToSQL() {
 			ss << ", ";
 		}
 		auto &column = columns[i];
-		ss << KeywordHelper::WriteOptionallyQuoted(column.name) << " " << column.type.ToString();
+		ss << KeywordHelper::WriteOptionallyQuoted(column.name) << " ";
+		switch (column.type.id()) {
+		case LogicalTypeId::ENUM:
+			ss << KeywordHelper::WriteOptionallyQuoted(column.type.ToString());
+			break;
+		default:
+			ss << column.type.ToString();
+			break;
+		}
 		bool not_null = not_null_columns.find(column.oid) != not_null_columns.end();
 		bool is_single_key_pk = pk_columns.find(column.oid) != pk_columns.end();
 		bool is_multi_key_pk = multi_key_pks.find(column.name) != multi_key_pks.end();
@@ -3209,6 +3217,7 @@ TableFunctionCatalogEntry::TableFunctionCatalogEntry(Catalog *catalog, SchemaCat
 
 
 
+
 #include <algorithm>
 #include <sstream>
 
@@ -3245,7 +3254,7 @@ string TypeCatalogEntry::ToSQL() {
 		Vector values_insert_order(EnumType::GetValuesInsertOrder(user_type));
 		idx_t size = EnumType::GetSize(user_type);
 		ss << "CREATE TYPE ";
-		ss << name;
+		ss << KeywordHelper::WriteOptionallyQuoted(name);
 		ss << " AS ENUM ( ";
 
 		for (idx_t i = 0; i < size; i++) {
@@ -57313,11 +57322,11 @@ void BufferedCSVReader::InitParseChunk(idx_t num_cols) {
 void BufferedCSVReader::JumpToBeginning(idx_t skip_rows = 0, bool skip_header = false) {
 	ResetBuffer();
 	ResetStream();
-	SkipRowsAndReadHeader(skip_rows, skip_header);
 	sample_chunk_idx = 0;
 	bytes_in_chunk = 0;
 	end_of_file_reached = false;
 	bom_checked = false;
+	SkipRowsAndReadHeader(skip_rows, skip_header);
 }
 
 void BufferedCSVReader::SkipRowsAndReadHeader(idx_t skip_rows, bool skip_header) {
@@ -80656,17 +80665,11 @@ static void ArraySliceFunction(DataChunk &args, ExpressionState &state, Vector &
 	D_ASSERT(args.data.size() == 3);
 	auto count = args.size();
 
-	result.SetVectorType(VectorType::CONSTANT_VECTOR);
-	for (idx_t i = 0; i < args.ColumnCount(); i++) {
-		if (args.data[i].GetVectorType() != VectorType::CONSTANT_VECTOR) {
-			result.SetVectorType(VectorType::FLAT_VECTOR);
-		}
-	}
-
 	Vector &s = args.data[0];
 	Vector &b = args.data[1];
 	Vector &e = args.data[2];
 
+	s.Normalify(count);
 	switch (result.GetType().id()) {
 	case LogicalTypeId::LIST:
 		// Share the value dictionary as we are just going to slice it
@@ -80679,6 +80682,14 @@ static void ArraySliceFunction(DataChunk &args, ExpressionState &state, Vector &
 	default:
 		throw NotImplementedException("Specifier type not implemented");
 	}
+
+	result.SetVectorType(VectorType::CONSTANT_VECTOR);
+	for (idx_t i = 0; i < args.ColumnCount(); i++) {
+		if (args.data[i].GetVectorType() != VectorType::CONSTANT_VECTOR) {
+			result.SetVectorType(VectorType::FLAT_VECTOR);
+			break;
+		}
+	}
 }
 
 static unique_ptr<FunctionData> ArraySliceBind(ClientContext &context, ScalarFunction &bound_function,
@@ -121268,6 +121279,10 @@ unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalCros
 
 namespace duckdb {
 
+static bool IsCompareDistinct(ExpressionType type) {
+	return type == ExpressionType::COMPARE_DISTINCT_FROM || type == ExpressionType::COMPARE_NOT_DISTINCT_FROM;
+}
+
 bool StatisticsPropagator::ExpressionIsConstant(Expression &expr, const Value &val) {
 	if (expr.GetExpressionClass() != ExpressionClass::BOUND_CONSTANT) {
 		return false;
@@ -121295,8 +121310,10 @@ void StatisticsPropagator::SetStatisticsNotNull(ColumnBinding binding) {
 
 void StatisticsPropagator::UpdateFilterStatistics(BaseStatistics &stats, ExpressionType comparison_type,
                                                   const Value &constant) {
-	// any comparison filter removes all null values
-	stats.validity_stats = make_unique<ValidityStatistics>(false);
+	// regular comparisons removes all null values
+	if (!IsCompareDistinct(comparison_type)) {
+		stats.validity_stats = make_unique<ValidityStatistics>(false);
+	}
 	if (!stats.type.IsNumeric()) {
 		// don't handle non-numeric columns here (yet)
 		return;
@@ -121332,9 +121349,11 @@ void StatisticsPropagator::UpdateFilterStatistics(BaseStatistics &stats, Express
 
 void StatisticsPropagator::UpdateFilterStatistics(BaseStatistics &lstats, BaseStatistics &rstats,
                                                   ExpressionType comparison_type) {
-	// any comparison filter removes all null values
-	lstats.validity_stats = make_unique<ValidityStatistics>(false);
-	rstats.validity_stats = make_unique<ValidityStatistics>(false);
+	// regular comparisons removes all null values
+	if (!IsCompareDistinct(comparison_type)) {
+		lstats.validity_stats = make_unique<ValidityStatistics>(false);
+		rstats.validity_stats = make_unique<ValidityStatistics>(false);
+	}
 	D_ASSERT(lstats.type == rstats.type);
 	if (!lstats.type.IsNumeric()) {
 		// don't handle non-numeric columns here (yet)
@@ -121380,6 +121399,7 @@ void StatisticsPropagator::UpdateFilterStatistics(BaseStatistics &lstats, BaseSt
 		}
 		break;
 	case ExpressionType::COMPARE_EQUAL:
+	case ExpressionType::COMPARE_NOT_DISTINCT_FROM:
 		// LEFT = RIGHT
 		// only the tightest bounds pass
 		// so if we have e.g. left = [-50, 250] and right = [-100, 100]
@@ -121405,10 +121425,11 @@ void StatisticsPropagator::UpdateFilterStatistics(BaseStatistics &lstats, BaseSt
 void StatisticsPropagator::UpdateFilterStatistics(Expression &left, Expression &right, ExpressionType comparison_type) {
 	// first check if either side is a bound column ref
 	// any column ref involved in a comparison will not be null after the comparison
-	if (left.type == ExpressionType::BOUND_COLUMN_REF) {
+	bool compare_distinct = IsCompareDistinct(comparison_type);
+	if (!compare_distinct && left.type == ExpressionType::BOUND_COLUMN_REF) {
 		SetStatisticsNotNull(((BoundColumnRefExpression &)left).binding);
 	}
-	if (right.type == ExpressionType::BOUND_COLUMN_REF) {
+	if (!compare_distinct && right.type == ExpressionType::BOUND_COLUMN_REF) {
 		SetStatisticsNotNull(((BoundColumnRefExpression &)right).binding);
 	}
 	// check if this is a comparison between a constant and a column ref
@@ -153545,8 +153566,9 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 		join->children.push_back(move(plan->children[1]));
 		return move(join);
 	}
+	case LogicalOperatorType::LOGICAL_ANY_JOIN:
 	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN: {
-		auto &join = (LogicalComparisonJoin &)*plan;
+		auto &join = (LogicalJoin &)*plan;
 		D_ASSERT(plan->children.size() == 2);
 		// check the correlated expressions in the children of the join
 		bool left_has_correlation = has_correlated_expressions.find(plan->children[0].get())->second;
@@ -153571,6 +153593,13 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 				plan->children[0] = PushDownDependentJoinInternal(move(plan->children[0]));
 				return plan;
 			}
+		} else if (join.join_type == JoinType::RIGHT) {
+			// left outer join
+			if (!left_has_correlation) {
+				// only right has correlation: push into right
+				plan->children[1] = PushDownDependentJoinInternal(move(plan->children[1]));
+				return plan;
+			}
 		} else if (join.join_type == JoinType::MARK) {
 			if (right_has_correlation) {
 				throw Exception("MARK join with correlation in RHS not supported");
@@ -153586,27 +153615,42 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 		}
 		// both sides have correlation
 		// push into both sides
-		// NOTE: for OUTER JOINS it matters what the BASE BINDING is after the join
-		// for the LEFT OUTER JOIN, we want the LEFT side to be the base binding after we push
-		// because the RIGHT binding might contain NULL values
 		plan->children[0] = PushDownDependentJoinInternal(move(plan->children[0]));
 		auto left_binding = this->base_binding;
 		plan->children[1] = PushDownDependentJoinInternal(move(plan->children[1]));
 		auto right_binding = this->base_binding;
+		// NOTE: for OUTER JOINS it matters what the BASE BINDING is after the join
+		// for the LEFT OUTER JOIN, we want the LEFT side to be the base binding after we push
+		// because the RIGHT binding might contain NULL values
 		if (join.join_type == JoinType::LEFT) {
 			this->base_binding = left_binding;
+		} else if (join.join_type == JoinType::RIGHT) {
+			this->base_binding = right_binding;
 		}
 		// add the correlated columns to the join conditions
 		for (idx_t i = 0; i < correlated_columns.size(); i++) {
-			JoinCondition cond;
-
-			cond.left = make_unique<BoundColumnRefExpression>(
+			auto left = make_unique<BoundColumnRefExpression>(
 			    correlated_columns[i].type, ColumnBinding(left_binding.table_index, left_binding.column_index + i));
-			cond.right = make_unique<BoundColumnRefExpression>(
+			auto right = make_unique<BoundColumnRefExpression>(
 			    correlated_columns[i].type, ColumnBinding(right_binding.table_index, right_binding.column_index + i));
-			cond.comparison = ExpressionType::COMPARE_EQUAL;
-			cond.null_values_are_equal = true;
-			join.conditions.push_back(move(cond));
+
+			if (join.type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) {
+				JoinCondition cond;
+				cond.left = move(left);
+				cond.right = move(right);
+				cond.comparison = ExpressionType::COMPARE_EQUAL;
+				cond.null_values_are_equal = true;
+
+				auto &comparison_join = (LogicalComparisonJoin &)join;
+				comparison_join.conditions.push_back(move(cond));
+			} else {
+				auto &any_join = (LogicalAnyJoin &)join;
+				auto comparison = make_unique<BoundComparisonExpression>(ExpressionType::COMPARE_NOT_DISTINCT_FROM,
+				                                                         move(left), move(right));
+				auto conjunction = make_unique<BoundConjunctionExpression>(ExpressionType::CONJUNCTION_AND,
+				                                                           move(comparison), move(any_join.condition));
+				any_join.condition = move(conjunction);
+			}
 		}
 		// then we replace any correlated expressions with the corresponding entry in the correlated_map
 		RewriteCorrelatedExpressions rewriter(right_binding, correlated_map);

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "180367f93"
-#define DUCKDB_VERSION "v0.3.2-dev1304"
+#define DUCKDB_SOURCE_ID "9a485b0fd"
+#define DUCKDB_VERSION "v0.3.2-dev1316"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //