duckdb 0.4.1-dev2371.0 → 0.4.1-dev2385.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.4.1-dev2371.0",
+  "version": "0.4.1-dev2385.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -9279,6 +9279,12 @@ string StatementReturnTypeToString(StatementReturnType type) {
 
 
 
+#ifdef DUCKDB_CRASH_ON_ASSERT
+
+#include <stdio.h>
+#include <stdlib.h>
+#endif
+
 namespace duckdb {
 
 Exception::Exception(const string &msg) : std::exception(), type(ExceptionType::INVALID), raw_message_(msg) {
@@ -9561,6 +9567,10 @@ FatalException::FatalException(ExceptionType type, const string &msg) : Exceptio
 }
 
 InternalException::InternalException(const string &msg) : FatalException(ExceptionType::INTERNAL, msg) {
+#ifdef DUCKDB_CRASH_ON_ASSERT
+	Printer::Print("ABORT THROWN BY INTERNAL EXCEPTION: " + msg);
+	abort();
+#endif
 }
 
 InvalidInputException::InvalidInputException(const string &msg) : Exception(ExceptionType::INVALID_INPUT, msg) {
@@ -27686,9 +27696,8 @@ void PreservedError::Throw(const string &prepended_message) const {
 	if (!prepended_message.empty()) {
 		string new_message = prepended_message + raw_message;
 		Exception::ThrowAsTypeWithMessage(type, new_message);
-	} else {
-		Exception::ThrowAsTypeWithMessage(type, raw_message);
 	}
+	Exception::ThrowAsTypeWithMessage(type, raw_message);
 }
 
 const ExceptionType &PreservedError::Type() const {
@@ -41401,6 +41410,14 @@ struct ColumnDataMetaData {
 	}
 };
 
+//! Explicitly initialized without types
+ColumnDataCollection::ColumnDataCollection(Allocator &allocator_p) {
+	types.clear();
+	count = 0;
+	this->finished_append = false;
+	allocator = make_shared<ColumnDataAllocator>(allocator_p);
+}
+
 ColumnDataCollection::ColumnDataCollection(Allocator &allocator_p, vector<LogicalType> types_p) {
 	Initialize(move(types_p));
 	allocator = make_shared<ColumnDataAllocator>(allocator_p);
@@ -42506,6 +42523,15 @@ void DataChunk::SetValue(idx_t col_idx, idx_t index, const Value &val) {
 	data[col_idx].SetValue(index, val);
 }
 
+bool DataChunk::AllConstant() const {
+	for (auto &v : data) {
+		if (v.GetVectorType() != VectorType::CONSTANT_VECTOR) {
+			return false;
+		}
+	}
+	return true;
+}
+
 void DataChunk::Reference(DataChunk &chunk) {
 	D_ASSERT(chunk.ColumnCount() <= ColumnCount());
 	SetCardinality(chunk);
@@ -55053,7 +55079,7 @@ static bool ListCastSwitch(Vector &source, Vector &result, idx_t count, string *
 	case LogicalTypeId::LIST: {
 		// only handle constant and flat vectors here for now
 		if (source.GetVectorType() == VectorType::CONSTANT_VECTOR) {
-			result.SetVectorType(source.GetVectorType());
+			result.SetVectorType(VectorType::CONSTANT_VECTOR);
 			ConstantVector::SetNull(result, ConstantVector::IsNull(source));
 
 			auto ldata = ConstantVector::GetData<list_entry_t>(source);
@@ -58731,6 +58757,8 @@ bool ExpressionExecutor::TryEvaluateScalar(const Expression &expr, Value &result
 	try {
 		result = EvaluateScalar(expr);
 		return true;
+	} catch (InternalException &ex) {
+		throw ex;
 	} catch (...) {
 		return false;
 	}
@@ -84141,6 +84169,13 @@ protected:
 	void ResolveTypes() override {
 		types.emplace_back(LogicalType::BOOLEAN);
 	}
+
+	bool RequireOptimizer() const override {
+		if (!prepared->properties.bound_all_parameters) {
+			return false;
+		}
+		return children[0]->RequireOptimizer();
+	}
 };
 } // namespace duckdb
 
@@ -93174,8 +93209,8 @@ unique_ptr<FunctionData> HistogramBindFunction(ClientContext &context, Aggregate
 
 	D_ASSERT(arguments.size() == 1);
 	child_list_t<LogicalType> struct_children;
-	struct_children.push_back({"bucket", LogicalType::LIST(arguments[0]->return_type)});
-	struct_children.push_back({"count", LogicalType::LIST(LogicalType::UBIGINT)});
+	struct_children.push_back({"key", LogicalType::LIST(arguments[0]->return_type)});
+	struct_children.push_back({"value", LogicalType::LIST(LogicalType::UBIGINT)});
 	auto struct_type = LogicalType::MAP(move(struct_children));
 
 	function.return_type = struct_type;
@@ -102462,10 +102497,13 @@ static void ArraySliceFunction(DataChunk &args, ExpressionState &state, Vector &
 	Vector &b = args.data[1];
 	Vector &e = args.data[2];
 
-	s.Flatten(count);
+	result.SetVectorType(args.AllConstant() ? VectorType::CONSTANT_VECTOR : VectorType::FLAT_VECTOR);
 	switch (result.GetType().id()) {
 	case LogicalTypeId::LIST:
 		// Share the value dictionary as we are just going to slice it
+		if (s.GetVectorType() != VectorType::FLAT_VECTOR && s.GetVectorType() != VectorType::CONSTANT_VECTOR) {
+			s.Flatten(count);
+		}
 		ListVector::ReferenceEntry(result, s);
 		ExecuteSlice<list_entry_t, int64_t>(result, s, b, e, count);
 		break;
@@ -102475,14 +102513,6 @@ static void ArraySliceFunction(DataChunk &args, ExpressionState &state, Vector &
 	default:
 		throw NotImplementedException("Specifier type not implemented");
 	}
-
-	result.SetVectorType(VectorType::CONSTANT_VECTOR);
-	for (idx_t i = 0; i < args.ColumnCount(); i++) {
-		if (args.data[i].GetVectorType() != VectorType::CONSTANT_VECTOR) {
-			result.SetVectorType(VectorType::FLAT_VECTOR);
-			break;
-		}
-	}
 }
 
 static unique_ptr<FunctionData> ArraySliceBind(ClientContext &context, ScalarFunction &bound_function,
@@ -102617,6 +102647,10 @@ static void TemplatedContainsOrPosition(DataChunk &args, ExpressionState &state,
 			}
 		}
 	}
+
+	if (args.AllConstant()) {
+		result.SetVectorType(VectorType::CONSTANT_VECTOR);
+	}
 }
 
 template <class T, class OP>
@@ -102816,6 +102850,9 @@ void ListFlattenFunction(DataChunk &args, ExpressionState &state, Vector &result
 			result_entries[i].offset = 0;
 			result_entries[i].length = 0;
 		}
+		if (args.AllConstant()) {
+			result.SetVectorType(VectorType::CONSTANT_VECTOR);
+		}
 		return;
 	}
 
@@ -102861,7 +102898,7 @@ void ListFlattenFunction(DataChunk &args, ExpressionState &state, Vector &result
 		offset += length;
 	}
 
-	if (input.GetVectorType() == VectorType::CONSTANT_VECTOR) {
+	if (args.AllConstant()) {
 		result.SetVectorType(VectorType::CONSTANT_VECTOR);
 	}
 }
@@ -103215,6 +103252,10 @@ static void ListAggregatesFunction(DataChunk &args, ExpressionState &state, Vect
 			throw InternalException("Unimplemented histogram aggregate");
 		}
 	}
+
+	if (args.AllConstant()) {
+		result.SetVectorType(VectorType::CONSTANT_VECTOR);
+	}
 }
 
 static void ListAggregateFunction(DataChunk &args, ExpressionState &state, Vector &result) {
@@ -104061,6 +104102,10 @@ static void ListLambdaFunction(DataChunk &args, ExpressionState &state, Vector &
 		AppendFilteredToResult(lambda_vector, result_entries, elem_cnt, result, curr_list_len, curr_list_offset,
 		                       appended_lists_cnt, lists_len, curr_original_list_len, input_chunk);
 	}
+
+	if (args.AllConstant()) {
+		result.SetVectorType(VectorType::CONSTANT_VECTOR);
+	}
 }
 
 static void ListTransformFunction(DataChunk &args, ExpressionState &state, Vector &result) {
@@ -104264,7 +104309,12 @@ static void ListSortFunction(DataChunk &args, ExpressionState &state, Vector &re
 	result.SetVectorType(VectorType::FLAT_VECTOR);
 	auto &result_validity = FlatVector::Validity(result);
 
-	args.Flatten();
+	for (auto &v : args.data) {
+		if (v.GetVectorType() != VectorType::FLAT_VECTOR && v.GetVectorType() != VectorType::CONSTANT_VECTOR) {
+			v.Flatten(count);
+		}
+	}
+
 	if (lists.GetType().id() == LogicalTypeId::SQLNULL) {
 		result_validity.SetInvalid(0);
 		return;
@@ -104386,6 +104436,10 @@ static void ListSortFunction(DataChunk &args, ExpressionState &state, Vector &re
 	}
 
 	result.Reference(lists);
+
+	if (args.AllConstant()) {
+		result.SetVectorType(VectorType::CONSTANT_VECTOR);
+	}
 }
 
 static unique_ptr<FunctionData> ListSortBind(ClientContext &context, ScalarFunction &bound_function,
@@ -105318,6 +105372,10 @@ static void MapFromEntriesFunction(DataChunk &args, ExpressionState &state, Vect
 	}
 	MapConversionVerify(result, count);
 	result.Verify(count);
+
+	if (args.AllConstant()) {
+		result.SetVectorType(VectorType::CONSTANT_VECTOR);
+	}
 }
 
 static unique_ptr<FunctionData> MapFromEntriesBind(ClientContext &context, ScalarFunction &bound_function,
@@ -113280,8 +113338,7 @@ static void StringSplitExecutor(DataChunk &args, ExpressionState &state, Vector
 	}
 
 	D_ASSERT(ListVector::GetListSize(result) == total_len);
-	if (args.data[0].GetVectorType() == VectorType::CONSTANT_VECTOR &&
-	    args.data[1].GetVectorType() == VectorType::CONSTANT_VECTOR) {
+	if (args.AllConstant()) {
 		result.SetVectorType(VectorType::CONSTANT_VECTOR);
 	}
 }
@@ -113978,7 +114035,6 @@ static void StructInsertFunction(DataChunk &args, ExpressionState &state, Vector
 	starting_vec.Verify(args.size());
 
 	auto &starting_child_entries = StructVector::GetEntries(starting_vec);
-
 	auto &result_child_entries = StructVector::GetEntries(result);
 
 	// Assign the starting vector entries to the result vector
@@ -113993,6 +114049,10 @@ static void StructInsertFunction(DataChunk &args, ExpressionState &state, Vector
 	}
 
 	result.Verify(args.size());
+
+	if (args.AllConstant()) {
+		result.SetVectorType(VectorType::CONSTANT_VECTOR);
+	}
 }
 
 static unique_ptr<FunctionData> StructInsertBind(ClientContext &context, ScalarFunction &bound_function,
@@ -114809,10 +114869,8 @@ typedef unique_ptr<ArrowArrayStreamWrapper> (*stream_factory_produce_t)(uintptr_
 typedef void (*stream_factory_get_schema_t)(uintptr_t stream_factory_ptr, ArrowSchemaWrapper &schema);
 
 struct ArrowScanFunctionData : public PyTableFunctionData {
-	ArrowScanFunctionData(idx_t rows_per_thread_p, stream_factory_produce_t scanner_producer_p,
-	                      uintptr_t stream_factory_ptr_p)
-	    : lines_read(0), rows_per_thread(rows_per_thread_p), stream_factory_ptr(stream_factory_ptr_p),
-	      scanner_producer(scanner_producer_p), number_of_rows(0) {
+	ArrowScanFunctionData(stream_factory_produce_t scanner_producer_p, uintptr_t stream_factory_ptr_p)
+	    : lines_read(0), stream_factory_ptr(stream_factory_ptr_p), scanner_producer(scanner_producer_p) {
 	}
 	//! This holds the original list type (col_idx, [ArrowListType,size])
 	unordered_map<idx_t, unique_ptr<ArrowConvertData>> arrow_convert_data;
@@ -114823,8 +114881,6 @@ struct ArrowScanFunctionData : public PyTableFunctionData {
 	uintptr_t stream_factory_ptr;
 	//! Pointer to the scanner factory produce
 	stream_factory_produce_t scanner_producer;
-	//! Number of rows (Used in cardinality and progress bar)
-	int64_t number_of_rows;
 };
 
 struct ArrowScanLocalState : public LocalTableFunctionState {
@@ -115080,9 +115136,8 @@ unique_ptr<FunctionData> ArrowTableFunction::ArrowScanBind(ClientContext &contex
 	auto stream_factory_ptr = input.inputs[0].GetPointer();
 	auto stream_factory_produce = (stream_factory_produce_t)input.inputs[1].GetPointer();
 	auto stream_factory_get_schema = (stream_factory_get_schema_t)input.inputs[2].GetPointer();
-	auto rows_per_thread = input.inputs[3].GetValue<uint64_t>();
 
-	auto res = make_unique<ArrowScanFunctionData>(rows_per_thread, stream_factory_produce, stream_factory_ptr);
+	auto res = make_unique<ArrowScanFunctionData>(stream_factory_produce, stream_factory_ptr);
 
 	auto &data = *res;
 	stream_factory_get_schema(stream_factory_ptr, data.schema_root);
@@ -115127,11 +115182,7 @@ unique_ptr<ArrowArrayStreamWrapper> ProduceArrowScan(const ArrowScanFunctionData
 }
 
 idx_t ArrowTableFunction::ArrowScanMaxThreads(ClientContext &context, const FunctionData *bind_data_p) {
-	auto &bind_data = (const ArrowScanFunctionData &)*bind_data_p;
-	if (bind_data.number_of_rows <= 0 || ClientConfig::GetConfig(context).verify_parallelism) {
-		return context.db->NumberOfThreads();
-	}
-	return ((bind_data.number_of_rows + bind_data.rows_per_thread - 1) / bind_data.rows_per_thread) + 1;
+	return context.db->NumberOfThreads();
 }
 
 bool ArrowScanParallelStateNext(ClientContext &context, const FunctionData *bind_data_p, ArrowScanLocalState &state,
@@ -115197,28 +115248,15 @@ void ArrowTableFunction::ArrowScanFunction(ClientContext &context, TableFunction
 }
 
 unique_ptr<NodeStatistics> ArrowTableFunction::ArrowScanCardinality(ClientContext &context, const FunctionData *data) {
-	auto &bind_data = (ArrowScanFunctionData &)*data;
-	return make_unique<NodeStatistics>(bind_data.number_of_rows, bind_data.number_of_rows);
-}
-
-double ArrowTableFunction::ArrowProgress(ClientContext &context, const FunctionData *bind_data_p,
-                                         const GlobalTableFunctionState *global_state) {
-	auto &bind_data = (const ArrowScanFunctionData &)*bind_data_p;
-	if (bind_data.number_of_rows == 0) {
-		return 100;
-	}
-	auto percentage = bind_data.lines_read * 100.0 / bind_data.number_of_rows;
-	return percentage;
+	return make_unique<NodeStatistics>();
 }
 
 void ArrowTableFunction::RegisterFunction(BuiltinFunctions &set) {
-	TableFunction arrow("arrow_scan",
-	                    {LogicalType::POINTER, LogicalType::POINTER, LogicalType::POINTER, LogicalType::UBIGINT},
+	TableFunction arrow("arrow_scan", {LogicalType::POINTER, LogicalType::POINTER, LogicalType::POINTER},
 	                    ArrowScanFunction, ArrowScanBind, ArrowScanInitGlobal, ArrowScanInitLocal);
 	arrow.cardinality = ArrowScanCardinality;
 	arrow.projection_pushdown = true;
 	arrow.filter_pushdown = true;
-	arrow.table_scan_progress = ArrowProgress;
 	set.AddFunction(arrow);
 }
 
@@ -126087,7 +126125,7 @@ shared_ptr<PreparedStatementData> ClientContext::CreatePreparedStatement(ClientC
 #ifdef DEBUG
 	plan->Verify(*this);
 #endif
-	if (config.enable_optimizer) {
+	if (config.enable_optimizer && plan->RequireOptimizer()) {
 		profiler.StartPhase("optimizer");
 		Optimizer optimizer(*planner.binder, *this);
 		plan = optimizer.Optimize(move(plan));
@@ -126510,9 +126548,8 @@ unique_ptr<QueryResult> ClientContext::Query(const string &query, bool allow_str
 	if (statements.empty()) {
 		// no statements, return empty successful result
 		StatementProperties properties;
-		vector<LogicalType> types;
 		vector<string> names;
-		auto collection = make_unique<ColumnDataCollection>(Allocator::DefaultAllocator(), move(types));
+		auto collection = make_unique<ColumnDataCollection>(Allocator::DefaultAllocator());
 		return make_unique<MaterializedQueryResult>(StatementType::INVALID_STATEMENT, properties, move(names),
 		                                            move(collection), GetClientProperties());
 	}
@@ -137483,8 +137520,11 @@ PendingQueryResult::~PendingQueryResult() {
 
 unique_ptr<ClientContextLock> PendingQueryResult::LockContext() {
 	if (!context) {
-		throw InvalidInputException("Attempting to execute an unsuccessful or closed pending query result\nError: %s",
-		                            GetError());
+		if (HasError()) {
+			throw InvalidInputException(
+			    "Attempting to execute an unsuccessful or closed pending query result\nError: %s", GetError());
+		}
+		throw InvalidInputException("Attempting to execute an unsuccessful or closed pending query result");
 	}
 	return context->LockContext();
 }
@@ -137495,8 +137535,11 @@ void PendingQueryResult::CheckExecutableInternal(ClientContextLock &lock) {
 		invalidated = !context->IsActiveResult(lock, this);
 	}
 	if (invalidated) {
-		throw InvalidInputException("Attempting to execute an unsuccessful or closed pending query result\nError: %s",
-		                            GetError());
+		if (HasError()) {
+			throw InvalidInputException(
+			    "Attempting to execute an unsuccessful or closed pending query result\nError: %s", GetError());
+		}
+		throw InvalidInputException("Attempting to execute an unsuccessful or closed pending query result");
 	}
 }
 
@@ -166733,12 +166776,12 @@ vector<OrderByNode> Parser::ParseOrderList(const string &select_list, ParserOpti
 	}
 	auto &select = (SelectStatement &)*parser.statements[0];
 	if (select.node->type != QueryNodeType::SELECT_NODE) {
-		throw InternalException("Expected a single SELECT node");
+		throw ParserException("Expected a single SELECT node");
 	}
 	auto &select_node = (SelectNode &)*select.node;
 	if (select_node.modifiers.empty() || select_node.modifiers[0]->type != ResultModifierType::ORDER_MODIFIER ||
 	    select_node.modifiers.size() != 1) {
-		throw InternalException("Expected a single ORDER clause");
+		throw ParserException("Expected a single ORDER clause");
 	}
 	auto &order = (OrderModifier &)*select_node.modifiers[0];
 	return move(order.orders);
@@ -166776,7 +166819,7 @@ vector<vector<unique_ptr<ParsedExpression>>> Parser::ParseValuesList(const strin
 	}
 	auto &select_node = (SelectNode &)*select.node;
 	if (!select_node.from_table || select_node.from_table->type != TableReferenceType::EXPRESSION_LIST) {
-		throw InternalException("Expected a single VALUES statement");
+		throw ParserException("Expected a single VALUES statement");
 	}
 	auto &values_list = (ExpressionListRef &)*select_node.from_table;
 	return move(values_list.values);
@@ -208021,7 +208064,7 @@ bool PreparedStatementVerifier::Run(
 		}
 		auto execute_result = run(string(), move(execute_statement));
 		if (execute_result->HasError()) {
-			prepare_result->ThrowError("Failed execute during verify: ");
+			execute_result->ThrowError("Failed execute during verify: ");
 		}
 		materialized_result = unique_ptr_cast<QueryResult, MaterializedQueryResult>(move(execute_result));
 	} catch (const Exception &ex) {

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "3825e0ee7"
-#define DUCKDB_VERSION "v0.4.1-dev2371"
+#define DUCKDB_SOURCE_ID "94920472d"
+#define DUCKDB_VERSION "v0.4.1-dev2385"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -4608,6 +4608,9 @@ public:
 	DUCKDB_API Value GetValue(idx_t col_idx, idx_t index) const;
 	DUCKDB_API void SetValue(idx_t col_idx, idx_t index, const Value &val);
 
+	//! Returns true if all vectors in the DataChunk are constant
+	DUCKDB_API bool AllConstant() const;
+
 	//! Set the DataChunk to reference another data chunk
 	DUCKDB_API void Reference(DataChunk &chunk);
 	//! Set the DataChunk to own the data of data chunk, destroying the other chunk in the process
@@ -9971,6 +9974,10 @@ public:
 
 	static unique_ptr<LogicalOperator> Deserialize(Deserializer &deserializer, PlanDeserializationState &state);
 
+	virtual bool RequireOptimizer() const {
+		return true;
+	}
+
 protected:
 	//! Resolve types for this specific operator
 	virtual void ResolveTypes() = 0;
@@ -11271,6 +11278,8 @@ class ColumnDataCollection {
 public:
 	//! Constructs an in-memory column data collection from an allocator
 	DUCKDB_API ColumnDataCollection(Allocator &allocator, vector<LogicalType> types);
+	//! Constructs an empty (but valid) in-memory column data collection from an allocator
+	DUCKDB_API ColumnDataCollection(Allocator &allocator);
 	//! Constructs a buffer-managed column data collection
 	DUCKDB_API ColumnDataCollection(BufferManager &buffer_manager, vector<LogicalType> types);
 	//! Constructs either an in-memory or a buffer-managed column data collection