duckdb 0.5.2-dev2245.0 → 0.5.2-dev2256.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.5.2-dev2245.0",
+  "version": "0.5.2-dev2256.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -9120,8 +9120,8 @@ void BoxRenderer::RenderRowCount(string row_count_str, string shown_str, const s
 	}
 	auto minimum_length = row_count_str.size() + column_count_str.size() + 6;
 	bool render_rows_and_columns = total_length >= minimum_length &&
-	                               ((has_hidden_columns && row_count > 0) || (row_count > 1 && column_count > 1));
-	bool render_rows = total_length >= minimum_row_length && row_count != 1;
+	                               ((has_hidden_columns && row_count > 0) || (row_count >= 10 && column_count > 1));
+	bool render_rows = total_length >= minimum_row_length && (row_count == 0 || row_count >= 10);
 	bool render_anything = true;
 	if (!render_rows && !render_rows_and_columns) {
 		render_anything = false;
@@ -81718,6 +81718,16 @@ bool ParallelCSVReader::SetPosition(DataChunk &insert_chunk) {
 	if (buffer->buffer->IsCSVFileFirstBuffer() && start_buffer == position_buffer &&
 	    start_buffer == buffer->buffer->GetStart()) {
 		// First buffer doesn't need any setting
+		// Unless we have a header
+		if (options.header && options.auto_detect) {
+			for (; position_buffer < end_buffer; position_buffer++) {
+				if (StringUtil::CharacterIsNewline((*buffer)[position_buffer])) {
+					position_buffer++;
+					return true;
+				}
+			}
+			return false;
+		}
 		return true;
 	}
 
@@ -81841,8 +81851,9 @@ normal : {
 			goto add_value;
 		} else if (StringUtil::CharacterIsNewline(c)) {
 			// newline: add row
-			D_ASSERT(try_add_line || column == insert_chunk.ColumnCount() - 1);
-			goto add_row;
+			if (column > 0 || try_add_line) {
+				goto add_row;
+			}
 		}
 	}
 	if (!BufferRemainder()) {
@@ -82016,9 +82027,8 @@ final_state : {
 	}
 	// If this is the last buffer, we have to read the last value
 	if (buffer->buffer->IsCSVFileLastBuffer() || (buffer->next_buffer->IsCSVFileLastBuffer())) {
-		if (column > 0 || position_buffer > start_buffer) {
+		if (column > 0 || try_add_line) {
 			// remaining values to be added to the chunk
-			D_ASSERT(column == insert_chunk.ColumnCount() - 1);
 			AddValue(buffer->GetValue(start_buffer, position_buffer, offset), column, escape_positions, has_quotes);
 			if (try_add_line) {
 				bool success = column == sql_types.size();
@@ -86768,6 +86778,7 @@ public:
 	vector<ColumnBinding> GetColumnBindings() override;
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
+	idx_t EstimateCardinality(ClientContext &context) override;
 
 protected:
 	void ResolveTypes() override;
@@ -87387,6 +87398,7 @@ public:
 public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
+	idx_t EstimateCardinality(ClientContext &context) override;
 
 protected:
 	void ResolveTypes() override {
@@ -87453,6 +87465,7 @@ public:
 public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
+	idx_t EstimateCardinality(ClientContext &context) override;
 
 protected:
 	void ResolveTypes() override {
@@ -87590,6 +87603,7 @@ public:
 public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
+	idx_t EstimateCardinality(ClientContext &context) override;
 
 protected:
 	void ResolveTypes() override {
@@ -87729,6 +87743,7 @@ public:
 public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
+	idx_t EstimateCardinality(ClientContext &context) override;
 
 protected:
 	vector<ColumnBinding> GetColumnBindings() override {
@@ -88099,6 +88114,9 @@ public:
 	}
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
+	idx_t EstimateCardinality(ClientContext &context) override {
+		return 0;
+	}
 
 protected:
 	void ResolveTypes() override {
@@ -88245,6 +88263,9 @@ public:
 public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
+	idx_t EstimateCardinality(ClientContext &context) override {
+		return 3;
+	}
 
 protected:
 	void ResolveTypes() override {
@@ -88419,6 +88440,9 @@ public:
 	}
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
+	idx_t EstimateCardinality(ClientContext &context) override {
+		return expressions.size();
+	}
 
 protected:
 	void ResolveTypes() override {
@@ -89126,6 +89150,8 @@ protected:
 			types.emplace_back(LogicalType::BIGINT);
 		}
 	}
+
+	idx_t EstimateCardinality(ClientContext &context) override;
 };
 } // namespace duckdb
 
@@ -89419,6 +89445,7 @@ public:
 public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
+	idx_t EstimateCardinality(ClientContext &context) override;
 
 protected:
 	void ResolveTypes() override {
@@ -89472,6 +89499,7 @@ public:
 public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
+	idx_t EstimateCardinality(ClientContext &context) override;
 
 protected:
 	void ResolveTypes() override {
@@ -89818,6 +89846,7 @@ public:
 public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
+	idx_t EstimateCardinality(ClientContext &context) override;
 
 protected:
 	void ResolveTypes() override {
@@ -90054,6 +90083,7 @@ public:
 public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
+	idx_t EstimateCardinality(ClientContext &context) override;
 
 protected:
 	void ResolveTypes() override {
@@ -90128,6 +90158,7 @@ public:
 	}
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
+	idx_t EstimateCardinality(ClientContext &context) override;
 
 protected:
 	void ResolveTypes() override {
@@ -90236,6 +90267,7 @@ public:
 public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
+	idx_t EstimateCardinality(ClientContext &context) override;
 
 protected:
 	vector<ColumnBinding> GetColumnBindings() override {
@@ -126385,13 +126417,10 @@ static unique_ptr<GlobalTableFunctionState> ParallelCSVInitGlobal(ClientContext
 		return make_unique<ParallelCSVGlobalState>();
 	}
 	unique_ptr<CSVFileHandle> file_handle;
-	if (bind_data.initial_reader) {
-		file_handle = move(bind_data.initial_reader->file_handle);
-		bind_data.initial_reader.reset();
-	} else {
-		bind_data.options.file_path = bind_data.files[0];
-		file_handle = ReadCSV::OpenCSV(bind_data.options, context);
-	}
+
+	bind_data.options.file_path = bind_data.files[0];
+	file_handle = ReadCSV::OpenCSV(bind_data.options, context);
+
 	idx_t rows_to_skip = bind_data.options.skip_rows + (bind_data.options.has_header ? 1 : 0);
 	return make_unique<ParallelCSVGlobalState>(context, move(file_handle), bind_data.files,
 	                                           context.db->NumberOfThreads(), bind_data.options.buffer_size,
@@ -126663,6 +126692,19 @@ void CSVComplexFilterPushdown(ClientContext &context, LogicalGet &get, FunctionD
 	}
 }
 
+unique_ptr<NodeStatistics> CSVReaderCardinality(ClientContext &context, const FunctionData *bind_data_p) {
+	auto &bind_data = (ReadCSVData &)*bind_data_p;
+	idx_t per_file_cardinality = 0;
+	if (bind_data.initial_reader && bind_data.initial_reader->file_handle) {
+		auto estimated_row_width = (bind_data.sql_types.size() * 5);
+		per_file_cardinality = bind_data.initial_reader->file_handle->FileSize() / estimated_row_width;
+	} else {
+		// determined through the scientific method as the average amount of rows in a CSV file
+		per_file_cardinality = 42;
+	}
+	return make_unique<NodeStatistics>(bind_data.files.size() * per_file_cardinality);
+}
+
 void BufferedCSVReaderOptions::Serialize(FieldWriter &writer) const {
 	// common options
 	writer.WriteField<bool>(has_delimiter);
@@ -126757,6 +126799,7 @@ TableFunction ReadCSVTableFunction::GetFunction(bool list_parameter) {
 	read_csv.serialize = CSVReaderSerialize;
 	read_csv.deserialize = CSVReaderDeserialize;
 	read_csv.get_batch_index = CSVReaderGetBatchIndex;
+	read_csv.cardinality = CSVReaderCardinality;
 	ReadCSVAddNamedParameters(read_csv);
 	return read_csv;
 }
@@ -126770,6 +126813,7 @@ TableFunction ReadCSVTableFunction::GetAutoFunction(bool list_parameter) {
 	read_csv_auto.serialize = CSVReaderSerialize;
 	read_csv_auto.deserialize = CSVReaderDeserialize;
 	read_csv_auto.get_batch_index = CSVReaderGetBatchIndex;
+	read_csv_auto.cardinality = CSVReaderCardinality;
 	ReadCSVAddNamedParameters(read_csv_auto);
 	return read_csv_auto;
 }
@@ -198594,6 +198638,14 @@ unique_ptr<LogicalOperator> LogicalAggregate::Deserialize(LogicalDeserialization
 	return move(result);
 }
 
+idx_t LogicalAggregate::EstimateCardinality(ClientContext &context) {
+	if (groups.empty()) {
+		// ungrouped aggregate
+		return 1;
+	}
+	return LogicalOperator::EstimateCardinality(context);
+}
+
 } // namespace duckdb
 
 
@@ -198762,6 +198814,10 @@ unique_ptr<LogicalOperator> LogicalCopyToFile::Deserialize(LogicalDeserializatio
 	return move(result);
 }
 
+idx_t LogicalCopyToFile::EstimateCardinality(ClientContext &context) {
+	return 1;
+}
+
 } // namespace duckdb
 
 
@@ -198782,6 +198838,10 @@ unique_ptr<LogicalOperator> LogicalCreate::Deserialize(LogicalDeserializationSta
 	return make_unique<LogicalCreate>(state.type, move(info), schema_catalog_entry);
 }
 
+idx_t LogicalCreate::EstimateCardinality(ClientContext &context) {
+	return 1;
+}
+
 } // namespace duckdb
 
 
@@ -198846,6 +198906,10 @@ unique_ptr<LogicalOperator> LogicalCreateTable::Deserialize(LogicalDeserializati
 	return make_unique<LogicalCreateTable>(schema, move(info));
 }
 
+idx_t LogicalCreateTable::EstimateCardinality(ClientContext &context) {
+	return 1;
+}
+
 } // namespace duckdb
 
 
@@ -198938,6 +199002,10 @@ unique_ptr<LogicalOperator> LogicalDelete::Deserialize(LogicalDeserializationSta
 	return move(result);
 }
 
+idx_t LogicalDelete::EstimateCardinality(ClientContext &context) {
+	return return_chunk ? LogicalOperator::EstimateCardinality(context) : 1;
+}
+
 } // namespace duckdb
 
 
@@ -199424,6 +199492,10 @@ unique_ptr<LogicalOperator> LogicalInsert::Deserialize(LogicalDeserializationSta
 	return move(result);
 }
 
+idx_t LogicalInsert::EstimateCardinality(ClientContext &context) {
+	return return_chunk ? LogicalOperator::EstimateCardinality(context) : 1;
+}
+
 } // namespace duckdb
 
 
@@ -199544,6 +199616,7 @@ unique_ptr<LogicalOperator> LogicalLimit::Deserialize(LogicalDeserializationStat
 	auto offset = reader.ReadOptional<Expression>(nullptr, state.gstate);
 	return make_unique<LogicalLimit>(limit_val, offset_val, move(limit), move(offset));
 }
+
 } // namespace duckdb
 
 
@@ -199564,6 +199637,15 @@ unique_ptr<LogicalOperator> LogicalLimitPercent::Deserialize(LogicalDeserializat
 	auto offset = reader.ReadOptional<Expression>(nullptr, state.gstate);
 	return make_unique<LogicalLimitPercent>(limit_percent, offset_val, move(limit), move(offset));
 }
+
+idx_t LogicalLimitPercent::EstimateCardinality(ClientContext &context) {
+	auto child_cardinality = LogicalOperator::EstimateCardinality(context);
+	if (limit_percent < 0 || limit_percent > 100) {
+		return child_cardinality;
+	}
+	return idx_t(child_cardinality * (limit_percent / 100.0));
+}
+
 } // namespace duckdb
 
 
@@ -199597,6 +199679,10 @@ unique_ptr<LogicalOperator> LogicalPragma::Deserialize(LogicalDeserializationSta
 	throw NotImplementedException(LogicalOperatorToString(state.type));
 }
 
+idx_t LogicalPragma::EstimateCardinality(ClientContext &context) {
+	return 1;
+}
+
 } // namespace duckdb
 
 
@@ -199610,6 +199696,10 @@ unique_ptr<LogicalOperator> LogicalPrepare::Deserialize(LogicalDeserializationSt
 	throw NotImplementedException(LogicalOperatorToString(state.type));
 }
 
+idx_t LogicalPrepare::EstimateCardinality(ClientContext &context) {
+	return 1;
+}
+
 } // namespace duckdb
 
 
@@ -199722,6 +199812,10 @@ unique_ptr<LogicalOperator> LogicalSet::Deserialize(LogicalDeserializationState
 	return make_unique<LogicalSet>(name, value, scope);
 }
 
+idx_t LogicalSet::EstimateCardinality(ClientContext &context) {
+	return 1;
+}
+
 } // namespace duckdb
 
 
@@ -199773,6 +199867,10 @@ unique_ptr<LogicalOperator> LogicalSimple::Deserialize(LogicalDeserializationSta
 	throw NotImplementedException(LogicalOperatorToString(state.type));
 }
 
+idx_t LogicalSimple::EstimateCardinality(ClientContext &context) {
+	return 1;
+}
+
 } // namespace duckdb
 
 
@@ -199791,6 +199889,15 @@ unique_ptr<LogicalOperator> LogicalTopN::Deserialize(LogicalDeserializationState
 	auto limit = reader.ReadRequired<idx_t>();
 	return make_unique<LogicalTopN>(move(orders), limit, offset);
 }
+
+idx_t LogicalTopN::EstimateCardinality(ClientContext &context) {
+	auto child_cardinality = LogicalOperator::EstimateCardinality(context);
+	if (limit >= 0 && child_cardinality < idx_t(limit)) {
+		return limit;
+	}
+	return child_cardinality;
+}
+
 } // namespace duckdb
 
 
@@ -199861,6 +199968,10 @@ unique_ptr<LogicalOperator> LogicalUpdate::Deserialize(LogicalDeserializationSta
 	return move(result);
 }
 
+idx_t LogicalUpdate::EstimateCardinality(ClientContext &context) {
+	return return_chunk ? LogicalOperator::EstimateCardinality(context) : 1;
+}
+
 } // namespace duckdb
 
 

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "5fde799851"
-#define DUCKDB_VERSION "v0.5.2-dev2245"
+#define DUCKDB_SOURCE_ID "47bb125a91"
+#define DUCKDB_VERSION "v0.5.2-dev2256"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -16880,6 +16880,7 @@ public:
 
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
+	idx_t EstimateCardinality(ClientContext &context) override;
 
 protected:
 	void ResolveTypes() override {