duckdb 0.6.2-dev761.0 → 0.6.2-dev768.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.6.2-dev761.0",
+  "version": "0.6.2-dev768.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/extension/parquet/column_writer.cpp

@@ -185,7 +185,7 @@ ColumnWriterState::~ColumnWriterState() {
 
 void ColumnWriter::CompressPage(BufferedSerializer &temp_writer, size_t &compressed_size, data_ptr_t &compressed_data,
                                 unique_ptr<data_t[]> &compressed_buf) {
-	switch (writer.codec) {
+	switch (writer.GetCodec()) {
 	case CompressionCodec::UNCOMPRESSED:
 		compressed_size = temp_writer.blob.size;
 		compressed_data = temp_writer.blob.data.get();
@@ -396,10 +396,10 @@ unique_ptr<ColumnWriterState> BasicColumnWriter::InitializeWriteState(duckdb_par
 void BasicColumnWriter::RegisterToRowGroup(duckdb_parquet::format::RowGroup &row_group) {
 	format::ColumnChunk column_chunk;
 	column_chunk.__isset.meta_data = true;
-	column_chunk.meta_data.codec = writer.codec;
+	column_chunk.meta_data.codec = writer.GetCodec();
 	column_chunk.meta_data.path_in_schema = schema_path;
 	column_chunk.meta_data.num_values = 0;
-	column_chunk.meta_data.type = writer.file_meta_data.schema[schema_idx].type;
+	column_chunk.meta_data.type = writer.GetType(schema_idx);
 	row_group.columns.push_back(move(column_chunk));
 }
 
@@ -645,7 +645,8 @@ void BasicColumnWriter::FinalizeWrite(ColumnWriterState &state_p) {
 	// flush the last page (if any remains)
 	FlushPage(state);
 
-	auto start_offset = writer.writer->GetTotalWritten();
+	auto &column_writer = writer.GetWriter();
+	auto start_offset = column_writer.GetTotalWritten();
 	auto page_offset = start_offset;
 	// flush the dictionary
 	if (HasDictionary(state)) {
@@ -665,14 +666,14 @@ void BasicColumnWriter::FinalizeWrite(ColumnWriterState &state_p) {
 	idx_t total_uncompressed_size = 0;
 	for (auto &write_info : state.write_info) {
 		D_ASSERT(write_info.page_header.uncompressed_page_size > 0);
-		auto header_start_offset = writer.writer->GetTotalWritten();
-		write_info.page_header.write(writer.protocol.get());
+		auto header_start_offset = column_writer.GetTotalWritten();
+		write_info.page_header.write(writer.GetProtocol());
 		// total uncompressed size in the column chunk includes the header size (!)
-		total_uncompressed_size += writer.writer->GetTotalWritten() - header_start_offset;
+		total_uncompressed_size += column_writer.GetTotalWritten() - header_start_offset;
 		total_uncompressed_size += write_info.page_header.uncompressed_page_size;
-		writer.writer->WriteData(write_info.compressed_data, write_info.compressed_size);
+		column_writer.WriteData(write_info.compressed_data, write_info.compressed_size);
 	}
-	column_chunk.meta_data.total_compressed_size = writer.writer->GetTotalWritten() - start_offset;
+	column_chunk.meta_data.total_compressed_size = column_writer.GetTotalWritten() - start_offset;
 	column_chunk.meta_data.total_uncompressed_size = total_uncompressed_size;
 }
 

package/src/duckdb/extension/parquet/include/parquet_writer.hpp

@@ -26,11 +26,6 @@ class FileSystem;
 class FileOpener;
 
 class ParquetWriter {
-	friend class ColumnWriter;
-	friend class BasicColumnWriter;
-	friend class ListColumnWriter;
-	friend class StructColumnWriter;
-
 public:
 	ParquetWriter(FileSystem &fs, string file_name, FileOpener *file_opener, vector<LogicalType> types,
 	              vector<string> names, duckdb_parquet::format::CompressionCodec::type codec);
@@ -42,6 +37,19 @@ public:
 	static duckdb_parquet::format::Type::type DuckDBTypeToParquetType(const LogicalType &duckdb_type);
 	static void SetSchemaProperties(const LogicalType &duckdb_type, duckdb_parquet::format::SchemaElement &schema_ele);
 
+	duckdb_apache::thrift::protocol::TProtocol *GetProtocol() {
+		return protocol.get();
+	}
+	duckdb_parquet::format::CompressionCodec::type GetCodec() {
+		return codec;
+	}
+	duckdb_parquet::format::Type::type GetType(idx_t schema_idx) {
+		return file_meta_data.schema[schema_idx].type;
+	}
+	BufferedFileWriter &GetWriter() {
+		return *writer;
+	}
+
 private:
 	string file_name;
 	vector<LogicalType> sql_types;

package/src/duckdb/extension/parquet/parquet-extension.cpp

@@ -733,6 +733,17 @@ unique_ptr<LocalFunctionData> ParquetWriteInitializeLocal(ExecutionContext &cont
 	return make_unique<ParquetWriteLocalState>(context.client, bind_data.sql_types);
 }
 
+//===--------------------------------------------------------------------===//
+// Parallel
+//===--------------------------------------------------------------------===//
+bool ParquetWriteIsParallel(ClientContext &context, FunctionData &bind_data) {
+	auto &config = DBConfig::GetConfig(context);
+	if (config.options.preserve_insertion_order) {
+		return false;
+	}
+	return true;
+}
+
 unique_ptr<TableFunctionRef> ParquetScanReplacement(ClientContext &context, const string &table_name,
                                                     ReplacementScanData *data) {
 	auto lower_name = StringUtil::Lower(table_name);
@@ -769,6 +780,7 @@ void ParquetExtension::Load(DuckDB &db) {
 	function.copy_to_sink = ParquetWriteSink;
 	function.copy_to_combine = ParquetWriteCombine;
 	function.copy_to_finalize = ParquetWriteFinalize;
+	function.parallel = ParquetWriteIsParallel;
 	function.copy_from_bind = ParquetScanFunction::ParquetReadBind;
 	function.copy_from_function = scan_fun.functions[0];
 

package/src/duckdb/extension/parquet/parquet_writer.cpp

@@ -249,18 +249,17 @@ void ParquetWriter::Flush(ColumnDataCollection &buffer) {
 	if (buffer.Count() == 0) {
 		return;
 	}
-	lock_guard<mutex> glock(lock);
 
 	// set up a new row group for this chunk collection
 	ParquetRowGroup row_group;
 	row_group.num_rows = buffer.Count();
-	row_group.file_offset = writer->GetTotalWritten();
 	row_group.__isset.file_offset = true;
 
+	vector<unique_ptr<ColumnWriterState>> states;
 	// iterate over each of the columns of the chunk collection and write them
 	D_ASSERT(buffer.ColumnCount() == column_writers.size());
 	for (idx_t col_idx = 0; col_idx < buffer.ColumnCount(); col_idx++) {
-		const unique_ptr<ColumnWriter> &col_writer = column_writers[col_idx];
+		const auto &col_writer = column_writers[col_idx];
 		auto write_state = col_writer->InitializeWriteState(row_group, buffer.GetAllocator());
 		if (col_writer->HasAnalyze()) {
 			for (auto &chunk : buffer.Chunks()) {
@@ -275,6 +274,14 @@ void ParquetWriter::Flush(ColumnDataCollection &buffer) {
 		for (auto &chunk : buffer.Chunks()) {
 			col_writer->Write(*write_state, chunk.data[col_idx], chunk.size());
 		}
+		states.push_back(move(write_state));
+	}
+
+	lock_guard<mutex> glock(lock);
+	row_group.file_offset = writer->GetTotalWritten();
+	for (idx_t col_idx = 0; col_idx < buffer.ColumnCount(); col_idx++) {
+		const auto &col_writer = column_writers[col_idx];
+		auto write_state = move(states[col_idx]);
 		col_writer->FinalizeWrite(*write_state);
 	}
 

package/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp

@@ -40,7 +40,7 @@ void MoveTmpFile(ClientContext &context, const string &tmp_file_path) {
 PhysicalCopyToFile::PhysicalCopyToFile(vector<LogicalType> types, CopyFunction function_p,
                                        unique_ptr<FunctionData> bind_data, idx_t estimated_cardinality)
     : PhysicalOperator(PhysicalOperatorType::COPY_TO_FILE, move(types), estimated_cardinality),
-      function(move(function_p)), bind_data(move(bind_data)) {
+      function(move(function_p)), bind_data(move(bind_data)), parallel(false) {
 }
 
 SinkResultType PhysicalCopyToFile::Sink(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate,

package/src/duckdb/src/execution/physical_plan/plan_copy_to_file.cpp

@@ -18,6 +18,9 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalCopyToFile
 	copy->file_path = op.file_path;
 	copy->use_tmp_file = use_tmp_file;
 	copy->per_thread_output = op.per_thread_output;
+	if (op.function.parallel) {
+		copy->parallel = op.function.parallel(context, *copy->bind_data);
+	}
 
 	copy->children.push_back(move(plan));
 	return move(copy);

package/src/duckdb/src/function/table/copy_csv.cpp

@@ -8,6 +8,7 @@
 #include "duckdb/common/types/string_type.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/function/scalar/string_functions.hpp"
+#include "duckdb/main/config.hpp"
 #include <limits>
 
 namespace duckdb {
@@ -390,6 +391,17 @@ void WriteCSVFinalize(ClientContext &context, FunctionData &bind_data, GlobalFun
 	global_state.handle.reset();
 }
 
+//===--------------------------------------------------------------------===//
+// Parallel
+//===--------------------------------------------------------------------===//
+bool WriteCSVIsParallel(ClientContext &context, FunctionData &bind_data) {
+	auto &config = DBConfig::GetConfig(context);
+	if (config.options.preserve_insertion_order) {
+		return false;
+	}
+	return true;
+}
+
 void CSVCopyFunction::RegisterFunction(BuiltinFunctions &set) {
 	CopyFunction info("csv");
 	info.copy_to_bind = WriteCSVBind;
@@ -398,6 +410,7 @@ void CSVCopyFunction::RegisterFunction(BuiltinFunctions &set) {
 	info.copy_to_sink = WriteCSVSink;
 	info.copy_to_combine = WriteCSVCombine;
 	info.copy_to_finalize = WriteCSVFinalize;
+	info.parallel = WriteCSVIsParallel;
 
 	info.copy_from_bind = ReadCSVBind;
 	info.copy_from_function = ReadCSVTableFunction::GetFunction();

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.6.2-dev761"
+#define DUCKDB_VERSION "0.6.2-dev768"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "a133a60550"
+#define DUCKDB_SOURCE_ID "bade062078"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_copy_to_file.hpp

@@ -24,6 +24,7 @@ public:
 	unique_ptr<FunctionData> bind_data;
 	string file_path;
 	bool use_tmp_file;
+	bool parallel;
 	bool per_thread_output;
 
 public:
@@ -51,7 +52,7 @@ public:
 	}
 
 	bool ParallelSink() const override {
-		return per_thread_output;
+		return per_thread_output || parallel;
 	}
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/function/copy_function.hpp

@@ -44,13 +44,14 @@ typedef unique_ptr<FunctionData> (*copy_to_deserialize_t)(ClientContext &context
 typedef unique_ptr<FunctionData> (*copy_from_bind_t)(ClientContext &context, CopyInfo &info,
                                                      vector<string> &expected_names,
                                                      vector<LogicalType> &expected_types);
+typedef bool (*copy_to_is_parallel_t)(ClientContext &context, FunctionData &bind_data);
 
 class CopyFunction : public Function {
 public:
 	explicit CopyFunction(string name)
 	    : Function(name), copy_to_bind(nullptr), copy_to_initialize_local(nullptr), copy_to_initialize_global(nullptr),
-	      copy_to_sink(nullptr), copy_to_combine(nullptr), copy_to_finalize(nullptr), serialize(nullptr),
-	      deserialize(nullptr), copy_from_bind(nullptr) {
+	      copy_to_sink(nullptr), copy_to_combine(nullptr), copy_to_finalize(nullptr), parallel(nullptr),
+	      serialize(nullptr), deserialize(nullptr), copy_from_bind(nullptr) {
 	}
 
 	copy_to_bind_t copy_to_bind;
@@ -59,6 +60,7 @@ public:
 	copy_to_sink_t copy_to_sink;
 	copy_to_combine_t copy_to_combine;
 	copy_to_finalize_t copy_to_finalize;
+	copy_to_is_parallel_t parallel;
 
 	copy_to_serialize_t serialize;
 	copy_to_deserialize_t deserialize;

package/src/duckdb/src/main/extension/extension_load.cpp

@@ -49,7 +49,7 @@ ExtensionInitResult ExtensionHelper::InitialLoad(DBConfig &config, FileOpener *o
 		}
 		throw IOException("Extension \"%s\" not found.\n%s", filename, message);
 	}
-	{
+	if (!config.options.allow_unsigned_extensions) {
 		auto handle = fs.OpenFile(filename, FileFlags::FILE_FLAGS_READ);
 
 		// signature is the last 265 bytes of the file
@@ -75,7 +75,7 @@ ExtensionInitResult ExtensionHelper::InitialLoad(DBConfig &config, FileOpener *o
 				break;
 			}
 		}
-		if (!any_valid && !config.options.allow_unsigned_extensions) {
+		if (!any_valid) {
 			throw IOException(config.error_manager->FormatException(ErrorType::UNSIGNED_EXTENSION, filename));
 		}
 	}