duckdb 0.6.2-dev2015.0 → 0.6.2-dev2042.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.6.2-dev2015.0",
+  "version": "0.6.2-dev2042.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp

@@ -60,6 +60,25 @@ static int64_t ParseInteger(const Value &value, const string &loption) {
 	return value.GetValue<int64_t>();
 }
 
+void BufferedCSVReaderOptions::SetHeader(bool input) {
+	this->header = input;
+	this->has_header = true;
+}
+
+void BufferedCSVReaderOptions::SetCompression(const string &compression) {
+	this->compression = FileCompressionTypeFromString(compression);
+}
+
+void BufferedCSVReaderOptions::SetEscape(const string &input) {
+	this->escape = input;
+	this->has_escape = true;
+}
+
+void BufferedCSVReaderOptions::SetParallel(bool use_parallel) {
+	this->has_parallel = true;
+	this->use_parallel = use_parallel;
+}
+
 void BufferedCSVReaderOptions::SetDelimiter(const string &input) {
 	this->delimiter = StringUtil::Replace(input, "\\t", "\t");
 	this->has_delimiter = true;
@@ -68,6 +87,11 @@ void BufferedCSVReaderOptions::SetDelimiter(const string &input) {
 	}
 }
 
+void BufferedCSVReaderOptions::SetQuote(const string &quote) {
+	this->quote = quote;
+	this->has_quote = true;
+}
+
 void BufferedCSVReaderOptions::SetNewline(const string &input) {
 	if (input == "\\n" || input == "\\r") {
 		new_line = NewLineIdentifier::SINGLE;
@@ -102,6 +126,8 @@ void BufferedCSVReaderOptions::SetReadOption(const string &loption, const Value
 	}
 	if (loption == "auto_detect") {
 		auto_detect = ParseBoolean(value, loption);
+	} else if (loption == "parallel") {
+		SetParallel(ParseBoolean(value, loption));
 	} else if (loption == "sample_size") {
 		int64_t sample_size = ParseInteger(value, loption);
 		if (sample_size < 1 && sample_size != -1) {
@@ -143,9 +169,6 @@ void BufferedCSVReaderOptions::SetReadOption(const string &loption, const Value
 	} else if (loption == "timestamp_format" || loption == "timestampformat") {
 		string format = ParseString(value, loption);
 		SetDateFormat(LogicalTypeId::TIMESTAMP, format, true);
-	} else if (loption == "escape") {
-		escape = ParseString(value, loption);
-		has_escape = true;
 	} else if (loption == "ignore_errors") {
 		ignore_errors = ParseBoolean(value, loption);
 	} else if (loption == "union_by_name") {
@@ -193,16 +216,13 @@ bool BufferedCSVReaderOptions::SetBaseOption(const string &loption, const Value
 	if (StringUtil::StartsWith(loption, "delim") || StringUtil::StartsWith(loption, "sep")) {
 		SetDelimiter(ParseString(value, loption));
 	} else if (loption == "quote") {
-		quote = ParseString(value, loption);
-		has_quote = true;
+		SetQuote(ParseString(value, loption));
 	} else if (loption == "new_line") {
 		SetNewline(ParseString(value, loption));
 	} else if (loption == "escape") {
-		escape = ParseString(value, loption);
-		has_escape = true;
+		SetEscape(ParseString(value, loption));
 	} else if (loption == "header") {
-		header = ParseBoolean(value, loption);
-		has_header = true;
+		SetHeader(ParseBoolean(value, loption));
 	} else if (loption == "null" || loption == "nullstr") {
 		null_str = ParseString(value, loption);
 	} else if (loption == "encoding") {
@@ -211,7 +231,7 @@ bool BufferedCSVReaderOptions::SetBaseOption(const string &loption, const Value
 			throw BinderException("Copy is only supported for UTF-8 encoded files, ENCODING 'UTF-8'");
 		}
 	} else if (loption == "compression") {
-		compression = FileCompressionTypeFromString(ParseString(value, loption));
+		SetCompression(ParseString(value, loption));
 	} else {
 		// unrecognized option in base CSV
 		return false;
@@ -227,7 +247,7 @@ std::string BufferedCSVReaderOptions::ToString() const {
 	       "\n  header=" + std::to_string(header) +
 	       (has_header ? "" : (auto_detect ? " (auto detected)" : "' (default)")) +
 	       "\n  sample_size=" + std::to_string(sample_chunk_size * sample_chunks) +
-	       "\n  ignore_erros=" + std::to_string(ignore_errors) + "\n  all_varchar=" + std::to_string(all_varchar);
+	       "\n  ignore_errors=" + std::to_string(ignore_errors) + "\n  all_varchar=" + std::to_string(all_varchar);
 }
 
 } // namespace duckdb

package/src/duckdb/src/function/table/read_csv.cpp

@@ -38,6 +38,10 @@ void ReadCSVData::FinalizeRead(ClientContext &context) {
 	BaseCSVData::Finalize();
 	auto &config = DBConfig::GetConfig(context);
 	single_threaded = !config.options.experimental_parallel_csv_reader;
+	if (options.has_parallel) {
+		// Override the option set in the config
+		single_threaded = !options.use_parallel;
+	}
 	bool null_or_empty = options.delimiter.empty() || options.escape.empty() || options.quote.empty() ||
 	                     options.delimiter[0] == '\0' || options.escape[0] == '\0' || options.quote[0] == '\0';
 	bool complex_options = options.delimiter.size() > 1 || options.escape.size() > 1 || options.quote.size() > 1;
@@ -789,6 +793,7 @@ static void ReadCSVAddNamedParameters(TableFunction &table_function) {
 	table_function.named_parameters["union_by_name"] = LogicalType::BOOLEAN;
 	table_function.named_parameters["buffer_size"] = LogicalType::UBIGINT;
 	table_function.named_parameters["decimal_separator"] = LogicalType::VARCHAR;
+	table_function.named_parameters["parallel"] = LogicalType::BOOLEAN;
 }
 
 double CSVReaderProgress(ClientContext &context, const FunctionData *bind_data_p,

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.6.2-dev2015"
+#define DUCKDB_VERSION "0.6.2-dev2042"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "cbf43430b3"
+#define DUCKDB_SOURCE_ID "67e219929b"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp

@@ -38,6 +38,11 @@ struct BufferedCSVReaderOptions {
 	//! New Line separator
 	NewLineIdentifier new_line = NewLineIdentifier::NOT_SET;
 
+	//! Whether or not an option was provided for parallel
+	bool has_parallel = false;
+	//! Whether or not the read will use the ParallelCSVReader
+	bool use_parallel = false;
+	//! Whether or not a quote was defined by the user
 	bool has_quote = false;
 	//! Quote used for columns that contain reserved characters, e.g., delimiter
 	string quote = "\"";
@@ -122,7 +127,12 @@ struct BufferedCSVReaderOptions {
 	void Serialize(FieldWriter &writer) const;
 	void Deserialize(FieldReader &reader);
 
+	void SetCompression(const string &compression);
+	void SetHeader(bool has_header);
+	void SetEscape(const string &escape);
+	void SetQuote(const string &quote);
 	void SetDelimiter(const string &delimiter);
+	void SetParallel(bool use_parallel);
 
 	void SetNewline(const string &input);
 	//! Set an option that is supported by both reading and writing functions, called by

package/src/duckdb/src/include/duckdb/function/function_set.hpp

@@ -34,9 +34,11 @@ public:
 		return functions.size();
 	}
 	T GetFunctionByOffset(idx_t offset) {
+		D_ASSERT(offset < functions.size());
 		return functions[offset];
 	}
 	T &GetFunctionReferenceByOffset(idx_t offset) {
+		D_ASSERT(offset < functions.size());
 		return functions[offset];
 	}
 	bool MergeFunctionSet(FunctionSet<T> new_functions) {

package/src/duckdb/src/include/duckdb/main/connection.hpp

@@ -30,6 +30,7 @@ class DatabaseInstance;
 class DuckDB;
 class LogicalOperator;
 class SelectStatement;
+struct BufferedCSVReaderOptions;
 
 typedef void (*warning_callback)(std::string);
 
@@ -127,8 +128,10 @@ public:
 	DUCKDB_API shared_ptr<Relation> Values(const string &values);
 	DUCKDB_API shared_ptr<Relation> Values(const string &values, const vector<string> &column_names,
 	                                       const string &alias = "values");
+
 	//! Reads CSV file
 	DUCKDB_API shared_ptr<Relation> ReadCSV(const string &csv_file);
+	DUCKDB_API shared_ptr<Relation> ReadCSV(const string &csv_file, BufferedCSVReaderOptions &options);
 	DUCKDB_API shared_ptr<Relation> ReadCSV(const string &csv_file, const vector<string> &columns);
 
 	//! Reads Parquet file

package/src/duckdb/src/include/duckdb/main/relation/read_csv_relation.hpp

@@ -8,26 +8,26 @@
 
 #pragma once
 
-#include "duckdb/main/relation.hpp"
+#include "duckdb/execution/operator/persistent/csv_reader_options.hpp"
+#include "duckdb/main/relation/table_function_relation.hpp"
 
 namespace duckdb {
 
-class ReadCSVRelation : public Relation {
+struct BufferedCSVReaderOptions;
+
+class ReadCSVRelation : public TableFunctionRelation {
 public:
 	ReadCSVRelation(const std::shared_ptr<ClientContext> &context, string csv_file, vector<ColumnDefinition> columns,
-	                bool auto_detect = false, string alias = string());
+	                string alias = string());
+	ReadCSVRelation(const std::shared_ptr<ClientContext> &context, string csv_file, BufferedCSVReaderOptions options,
+	                string alias = string());
 
-	string csv_file;
-	bool auto_detect;
 	string alias;
-	vector<ColumnDefinition> columns;
+	bool auto_detect;
+	string csv_file;
 
 public:
-	unique_ptr<QueryNode> GetQueryNode() override;
-	const vector<ColumnDefinition> &Columns() override;
-	string ToString(idx_t depth) override;
 	string GetAlias() override;
-	unique_ptr<TableRef> GetTableRef() override;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/relation/table_function_relation.hpp

@@ -15,10 +15,11 @@ namespace duckdb {
 class TableFunctionRelation : public Relation {
 public:
 	TableFunctionRelation(const std::shared_ptr<ClientContext> &context, string name, vector<Value> parameters,
-	                      named_parameter_map_t named_parameters, shared_ptr<Relation> input_relation_p = nullptr);
+	                      named_parameter_map_t named_parameters, shared_ptr<Relation> input_relation_p = nullptr,
+	                      bool auto_init = true);
 
 	TableFunctionRelation(const std::shared_ptr<ClientContext> &context, string name, vector<Value> parameters,
-	                      shared_ptr<Relation> input_relation_p = nullptr);
+	                      shared_ptr<Relation> input_relation_p = nullptr, bool auto_init = true);
 
 	string name;
 	vector<Value> parameters;
@@ -33,6 +34,14 @@ public:
 	const vector<ColumnDefinition> &Columns() override;
 	string ToString(idx_t depth) override;
 	string GetAlias() override;
+	void AddNamedParameter(const string &name, Value argument);
+
+private:
+	void InitializeColumns();
+
+private:
+	//! Whether or not to auto initialize the columns on construction
+	bool auto_initialize;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/parser/tableref/table_function_ref.hpp

@@ -26,7 +26,7 @@ public:
 	// if the function takes a subquery as argument its in here
 	unique_ptr<SelectStatement> subquery;
 
-	// External dependencies of this table funcion
+	// External dependencies of this table function
 	unique_ptr<ExternalDependency> external_dependency;
 
 public:

package/src/duckdb/src/main/connection.cpp

@@ -220,14 +220,13 @@ shared_ptr<Relation> Connection::Values(const string &values, const vector<strin
 
 shared_ptr<Relation> Connection::ReadCSV(const string &csv_file) {
 	BufferedCSVReaderOptions options;
+	return ReadCSV(csv_file, options);
+}
+
+shared_ptr<Relation> Connection::ReadCSV(const string &csv_file, BufferedCSVReaderOptions &options) {
 	options.file_path = csv_file;
 	options.auto_detect = true;
-	BufferedCSVReader reader(*context, options);
-	vector<ColumnDefinition> column_list;
-	for (idx_t i = 0; i < reader.return_types.size(); i++) {
-		column_list.emplace_back(reader.names[i], reader.return_types[i]);
-	}
-	return make_shared<ReadCSVRelation>(context, csv_file, std::move(column_list), true);
+	return make_shared<ReadCSVRelation>(context, csv_file, options);
 }
 
 shared_ptr<Relation> Connection::ReadCSV(const string &csv_file, const vector<string> &columns) {

package/src/duckdb/src/main/relation/read_csv_relation.cpp

@@ -8,59 +8,51 @@
 #include "duckdb/parser/expression/constant_expression.hpp"
 #include "duckdb/parser/expression/function_expression.hpp"
 #include "duckdb/common/string_util.hpp"
+#include "duckdb/execution/operator/persistent/buffered_csv_reader.hpp"
 
 namespace duckdb {
 
 ReadCSVRelation::ReadCSVRelation(const std::shared_ptr<ClientContext> &context, string csv_file_p,
-                                 vector<ColumnDefinition> columns_p, bool auto_detect, string alias_p)
-    : Relation(context, RelationType::READ_CSV_RELATION), csv_file(std::move(csv_file_p)), auto_detect(auto_detect),
-      alias(std::move(alias_p)), columns(std::move(columns_p)) {
+                                 vector<ColumnDefinition> columns_p, string alias_p)
+    : TableFunctionRelation(context, "read_csv", {Value(csv_file_p)}, nullptr, false), alias(std::move(alias_p)),
+      auto_detect(false), csv_file(std::move(csv_file_p)) {
+
 	if (alias.empty()) {
 		alias = StringUtil::Split(csv_file, ".")[0];
 	}
-}
 
-unique_ptr<QueryNode> ReadCSVRelation::GetQueryNode() {
-	auto result = make_unique<SelectNode>();
-	result->select_list.push_back(make_unique<StarExpression>());
-	result->from_table = GetTableRef();
-	return std::move(result);
-}
+	columns = move(columns_p);
 
-unique_ptr<TableRef> ReadCSVRelation::GetTableRef() {
-	auto table_ref = make_unique<TableFunctionRef>();
-	table_ref->alias = alias;
-	vector<unique_ptr<ParsedExpression>> children;
-	// CSV file
-	children.push_back(make_unique<ConstantExpression>(Value(csv_file)));
-	if (!auto_detect) {
-		// parameters
-		child_list_t<Value> column_names;
-		for (idx_t i = 0; i < columns.size(); i++) {
-			column_names.push_back(make_pair(columns[i].Name(), Value(columns[i].Type().ToString())));
-		}
-		auto colnames = make_unique<ConstantExpression>(Value::STRUCT(std::move(column_names)));
-		children.push_back(make_unique<ComparisonExpression>(
-		    ExpressionType::COMPARE_EQUAL, make_unique<ColumnRefExpression>("columns"), std::move(colnames)));
-	} else {
-		children.push_back(make_unique<ComparisonExpression>(ExpressionType::COMPARE_EQUAL,
-		                                                     make_unique<ColumnRefExpression>("auto_detect"),
-		                                                     make_unique<ConstantExpression>(Value::BOOLEAN(true))));
+	child_list_t<Value> column_names;
+	for (idx_t i = 0; i < columns.size(); i++) {
+		column_names.push_back(make_pair(columns[i].Name(), Value(columns[i].Type().ToString())));
 	}
-	table_ref->function = make_unique<FunctionExpression>("read_csv", std::move(children));
-	return std::move(table_ref);
-}
 
-string ReadCSVRelation::GetAlias() {
-	return alias;
+	AddNamedParameter("columns", Value::STRUCT(std::move(column_names)));
 }
 
-const vector<ColumnDefinition> &ReadCSVRelation::Columns() {
-	return columns;
+ReadCSVRelation::ReadCSVRelation(const std::shared_ptr<ClientContext> &context, string csv_file_p,
+                                 BufferedCSVReaderOptions options, string alias_p)
+    : TableFunctionRelation(context, "read_csv_auto", {Value(csv_file_p)}, nullptr, false), alias(std::move(alias_p)),
+      auto_detect(true), csv_file(std::move(csv_file_p)) {
+
+	if (alias.empty()) {
+		alias = StringUtil::Split(csv_file, ".")[0];
+	}
+
+	// Force auto_detect for this constructor
+	options.auto_detect = true;
+	BufferedCSVReader reader(*context, move(options));
+
+	for (idx_t i = 0; i < reader.return_types.size(); i++) {
+		columns.emplace_back(reader.names[i], reader.return_types[i]);
+	}
+
+	AddNamedParameter("auto_detect", Value::BOOLEAN(true));
 }
 
-string ReadCSVRelation::ToString(idx_t depth) {
-	return RenderWhitespace(depth) + "Read CSV [" + csv_file + "]";
+string ReadCSVRelation::GetAlias() {
+	return alias;
 }
 
 } // namespace duckdb

package/src/duckdb/src/main/relation/table_function_relation.cpp

@@ -12,21 +12,32 @@
 
 namespace duckdb {
 
+void TableFunctionRelation::AddNamedParameter(const string &name, Value argument) {
+	named_parameters[name] = move(argument);
+}
+
 TableFunctionRelation::TableFunctionRelation(const std::shared_ptr<ClientContext> &context, string name_p,
                                              vector<Value> parameters_p, named_parameter_map_t named_parameters,
-                                             shared_ptr<Relation> input_relation_p)
+                                             shared_ptr<Relation> input_relation_p, bool auto_init)
     : Relation(context, RelationType::TABLE_FUNCTION_RELATION), name(std::move(name_p)),
       parameters(std::move(parameters_p)), named_parameters(std::move(named_parameters)),
-      input_relation(std::move(input_relation_p)) {
+      input_relation(std::move(input_relation_p)), auto_initialize(auto_init) {
 	context->TryBindRelation(*this, this->columns);
 }
-TableFunctionRelation::TableFunctionRelation(const std::shared_ptr<ClientContext> &context, string name_p,
-                                             vector<Value> parameters_p,
 
-                                             shared_ptr<Relation> input_relation_p)
+TableFunctionRelation::TableFunctionRelation(const std::shared_ptr<ClientContext> &context, string name_p,
+                                             vector<Value> parameters_p, shared_ptr<Relation> input_relation_p,
+                                             bool auto_init)
     : Relation(context, RelationType::TABLE_FUNCTION_RELATION), name(std::move(name_p)),
-      parameters(std::move(parameters_p)), input_relation(std::move(input_relation_p)) {
-	context->TryBindRelation(*this, this->columns);
+      parameters(std::move(parameters_p)), input_relation(std::move(input_relation_p)), auto_initialize(auto_init) {
+	InitializeColumns();
+}
+
+void TableFunctionRelation::InitializeColumns() {
+	if (!auto_initialize) {
+		return;
+	}
+	context.GetContext()->TryBindRelation(*this, this->columns);
 }
 
 unique_ptr<QueryNode> TableFunctionRelation::GetQueryNode() {