duckdb 0.6.2-dev758.0 → 0.6.2-dev766.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.6.2-dev758.0",
+  "version": "0.6.2-dev766.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/extension/parquet/column_writer.cpp

@@ -185,7 +185,7 @@ ColumnWriterState::~ColumnWriterState() {
 
 void ColumnWriter::CompressPage(BufferedSerializer &temp_writer, size_t &compressed_size, data_ptr_t &compressed_data,
                                 unique_ptr<data_t[]> &compressed_buf) {
-	switch (writer.codec) {
+	switch (writer.GetCodec()) {
 	case CompressionCodec::UNCOMPRESSED:
 		compressed_size = temp_writer.blob.size;
 		compressed_data = temp_writer.blob.data.get();
@@ -396,10 +396,10 @@ unique_ptr<ColumnWriterState> BasicColumnWriter::InitializeWriteState(duckdb_par
 void BasicColumnWriter::RegisterToRowGroup(duckdb_parquet::format::RowGroup &row_group) {
 	format::ColumnChunk column_chunk;
 	column_chunk.__isset.meta_data = true;
-	column_chunk.meta_data.codec = writer.codec;
+	column_chunk.meta_data.codec = writer.GetCodec();
 	column_chunk.meta_data.path_in_schema = schema_path;
 	column_chunk.meta_data.num_values = 0;
-	column_chunk.meta_data.type = writer.file_meta_data.schema[schema_idx].type;
+	column_chunk.meta_data.type = writer.GetType(schema_idx);
 	row_group.columns.push_back(move(column_chunk));
 }
 
@@ -645,7 +645,8 @@ void BasicColumnWriter::FinalizeWrite(ColumnWriterState &state_p) {
 	// flush the last page (if any remains)
 	FlushPage(state);
 
-	auto start_offset = writer.writer->GetTotalWritten();
+	auto &column_writer = writer.GetWriter();
+	auto start_offset = column_writer.GetTotalWritten();
 	auto page_offset = start_offset;
 	// flush the dictionary
 	if (HasDictionary(state)) {
@@ -665,14 +666,14 @@ void BasicColumnWriter::FinalizeWrite(ColumnWriterState &state_p) {
 	idx_t total_uncompressed_size = 0;
 	for (auto &write_info : state.write_info) {
 		D_ASSERT(write_info.page_header.uncompressed_page_size > 0);
-		auto header_start_offset = writer.writer->GetTotalWritten();
-		write_info.page_header.write(writer.protocol.get());
+		auto header_start_offset = column_writer.GetTotalWritten();
+		write_info.page_header.write(writer.GetProtocol());
 		// total uncompressed size in the column chunk includes the header size (!)
-		total_uncompressed_size += writer.writer->GetTotalWritten() - header_start_offset;
+		total_uncompressed_size += column_writer.GetTotalWritten() - header_start_offset;
 		total_uncompressed_size += write_info.page_header.uncompressed_page_size;
-		writer.writer->WriteData(write_info.compressed_data, write_info.compressed_size);
+		column_writer.WriteData(write_info.compressed_data, write_info.compressed_size);
 	}
-	column_chunk.meta_data.total_compressed_size = writer.writer->GetTotalWritten() - start_offset;
+	column_chunk.meta_data.total_compressed_size = column_writer.GetTotalWritten() - start_offset;
 	column_chunk.meta_data.total_uncompressed_size = total_uncompressed_size;
 }
 

package/src/duckdb/extension/parquet/include/parquet_writer.hpp

@@ -26,11 +26,6 @@ class FileSystem;
 class FileOpener;
 
 class ParquetWriter {
-	friend class ColumnWriter;
-	friend class BasicColumnWriter;
-	friend class ListColumnWriter;
-	friend class StructColumnWriter;
-
 public:
 	ParquetWriter(FileSystem &fs, string file_name, FileOpener *file_opener, vector<LogicalType> types,
 	              vector<string> names, duckdb_parquet::format::CompressionCodec::type codec);
@@ -42,6 +37,19 @@ public:
 	static duckdb_parquet::format::Type::type DuckDBTypeToParquetType(const LogicalType &duckdb_type);
 	static void SetSchemaProperties(const LogicalType &duckdb_type, duckdb_parquet::format::SchemaElement &schema_ele);
 
+	duckdb_apache::thrift::protocol::TProtocol *GetProtocol() {
+		return protocol.get();
+	}
+	duckdb_parquet::format::CompressionCodec::type GetCodec() {
+		return codec;
+	}
+	duckdb_parquet::format::Type::type GetType(idx_t schema_idx) {
+		return file_meta_data.schema[schema_idx].type;
+	}
+	BufferedFileWriter &GetWriter() {
+		return *writer;
+	}
+
 private:
 	string file_name;
 	vector<LogicalType> sql_types;

package/src/duckdb/extension/parquet/parquet-extension.cpp

@@ -733,6 +733,17 @@ unique_ptr<LocalFunctionData> ParquetWriteInitializeLocal(ExecutionContext &cont
 	return make_unique<ParquetWriteLocalState>(context.client, bind_data.sql_types);
 }
 
+//===--------------------------------------------------------------------===//
+// Parallel
+//===--------------------------------------------------------------------===//
+bool ParquetWriteIsParallel(ClientContext &context, FunctionData &bind_data) {
+	auto &config = DBConfig::GetConfig(context);
+	if (config.options.preserve_insertion_order) {
+		return false;
+	}
+	return true;
+}
+
 unique_ptr<TableFunctionRef> ParquetScanReplacement(ClientContext &context, const string &table_name,
                                                     ReplacementScanData *data) {
 	auto lower_name = StringUtil::Lower(table_name);
@@ -769,6 +780,7 @@ void ParquetExtension::Load(DuckDB &db) {
 	function.copy_to_sink = ParquetWriteSink;
 	function.copy_to_combine = ParquetWriteCombine;
 	function.copy_to_finalize = ParquetWriteFinalize;
+	function.parallel = ParquetWriteIsParallel;
 	function.copy_from_bind = ParquetScanFunction::ParquetReadBind;
 	function.copy_from_function = scan_fun.functions[0];
 

package/src/duckdb/extension/parquet/parquet_writer.cpp

@@ -249,18 +249,17 @@ void ParquetWriter::Flush(ColumnDataCollection &buffer) {
 	if (buffer.Count() == 0) {
 		return;
 	}
-	lock_guard<mutex> glock(lock);
 
 	// set up a new row group for this chunk collection
 	ParquetRowGroup row_group;
 	row_group.num_rows = buffer.Count();
-	row_group.file_offset = writer->GetTotalWritten();
 	row_group.__isset.file_offset = true;
 
+	vector<unique_ptr<ColumnWriterState>> states;
 	// iterate over each of the columns of the chunk collection and write them
 	D_ASSERT(buffer.ColumnCount() == column_writers.size());
 	for (idx_t col_idx = 0; col_idx < buffer.ColumnCount(); col_idx++) {
-		const unique_ptr<ColumnWriter> &col_writer = column_writers[col_idx];
+		const auto &col_writer = column_writers[col_idx];
 		auto write_state = col_writer->InitializeWriteState(row_group, buffer.GetAllocator());
 		if (col_writer->HasAnalyze()) {
 			for (auto &chunk : buffer.Chunks()) {
@@ -275,6 +274,14 @@ void ParquetWriter::Flush(ColumnDataCollection &buffer) {
 		for (auto &chunk : buffer.Chunks()) {
 			col_writer->Write(*write_state, chunk.data[col_idx], chunk.size());
 		}
+		states.push_back(move(write_state));
+	}
+
+	lock_guard<mutex> glock(lock);
+	row_group.file_offset = writer->GetTotalWritten();
+	for (idx_t col_idx = 0; col_idx < buffer.ColumnCount(); col_idx++) {
+		const auto &col_writer = column_writers[col_idx];
+		auto write_state = move(states[col_idx]);
 		col_writer->FinalizeWrite(*write_state);
 	}
 

package/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp

@@ -40,7 +40,7 @@ void MoveTmpFile(ClientContext &context, const string &tmp_file_path) {
 PhysicalCopyToFile::PhysicalCopyToFile(vector<LogicalType> types, CopyFunction function_p,
                                        unique_ptr<FunctionData> bind_data, idx_t estimated_cardinality)
     : PhysicalOperator(PhysicalOperatorType::COPY_TO_FILE, move(types), estimated_cardinality),
-      function(move(function_p)), bind_data(move(bind_data)) {
+      function(move(function_p)), bind_data(move(bind_data)), parallel(false) {
 }
 
 SinkResultType PhysicalCopyToFile::Sink(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate,

package/src/duckdb/src/execution/physical_plan/plan_copy_to_file.cpp

@@ -18,6 +18,9 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalCopyToFile
 	copy->file_path = op.file_path;
 	copy->use_tmp_file = use_tmp_file;
 	copy->per_thread_output = op.per_thread_output;
+	if (op.function.parallel) {
+		copy->parallel = op.function.parallel(context, *copy->bind_data);
+	}
 
 	copy->children.push_back(move(plan));
 	return move(copy);

package/src/duckdb/src/function/table/copy_csv.cpp

@@ -8,6 +8,7 @@
 #include "duckdb/common/types/string_type.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/function/scalar/string_functions.hpp"
+#include "duckdb/main/config.hpp"
 #include <limits>
 
 namespace duckdb {
@@ -390,6 +391,17 @@ void WriteCSVFinalize(ClientContext &context, FunctionData &bind_data, GlobalFun
 	global_state.handle.reset();
 }
 
+//===--------------------------------------------------------------------===//
+// Parallel
+//===--------------------------------------------------------------------===//
+bool WriteCSVIsParallel(ClientContext &context, FunctionData &bind_data) {
+	auto &config = DBConfig::GetConfig(context);
+	if (config.options.preserve_insertion_order) {
+		return false;
+	}
+	return true;
+}
+
 void CSVCopyFunction::RegisterFunction(BuiltinFunctions &set) {
 	CopyFunction info("csv");
 	info.copy_to_bind = WriteCSVBind;
@@ -398,6 +410,7 @@ void CSVCopyFunction::RegisterFunction(BuiltinFunctions &set) {
 	info.copy_to_sink = WriteCSVSink;
 	info.copy_to_combine = WriteCSVCombine;
 	info.copy_to_finalize = WriteCSVFinalize;
+	info.parallel = WriteCSVIsParallel;
 
 	info.copy_from_bind = ReadCSVBind;
 	info.copy_from_function = ReadCSVTableFunction::GetFunction();

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.6.2-dev758"
+#define DUCKDB_VERSION "0.6.2-dev766"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "cd29769dcd"
+#define DUCKDB_SOURCE_ID "81c7b8cfb6"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_copy_to_file.hpp

@@ -24,6 +24,7 @@ public:
 	unique_ptr<FunctionData> bind_data;
 	string file_path;
 	bool use_tmp_file;
+	bool parallel;
 	bool per_thread_output;
 
 public:
@@ -51,7 +52,7 @@ public:
 	}
 
 	bool ParallelSink() const override {
-		return per_thread_output;
+		return per_thread_output || parallel;
 	}
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/function/copy_function.hpp

@@ -44,13 +44,14 @@ typedef unique_ptr<FunctionData> (*copy_to_deserialize_t)(ClientContext &context
 typedef unique_ptr<FunctionData> (*copy_from_bind_t)(ClientContext &context, CopyInfo &info,
                                                      vector<string> &expected_names,
                                                      vector<LogicalType> &expected_types);
+typedef bool (*copy_to_is_parallel_t)(ClientContext &context, FunctionData &bind_data);
 
 class CopyFunction : public Function {
 public:
 	explicit CopyFunction(string name)
 	    : Function(name), copy_to_bind(nullptr), copy_to_initialize_local(nullptr), copy_to_initialize_global(nullptr),
-	      copy_to_sink(nullptr), copy_to_combine(nullptr), copy_to_finalize(nullptr), serialize(nullptr),
-	      deserialize(nullptr), copy_from_bind(nullptr) {
+	      copy_to_sink(nullptr), copy_to_combine(nullptr), copy_to_finalize(nullptr), parallel(nullptr),
+	      serialize(nullptr), deserialize(nullptr), copy_from_bind(nullptr) {
 	}
 
 	copy_to_bind_t copy_to_bind;
@@ -59,6 +60,7 @@ public:
 	copy_to_sink_t copy_to_sink;
 	copy_to_combine_t copy_to_combine;
 	copy_to_finalize_t copy_to_finalize;
+	copy_to_is_parallel_t parallel;
 
 	copy_to_serialize_t serialize;
 	copy_to_deserialize_t deserialize;

package/src/duckdb/src/include/duckdb/storage/storage_info.hpp

@@ -28,6 +28,8 @@ struct FileHandle;
 //! The version number of the database storage format
 extern const uint64_t VERSION_NUMBER;
 
+const char *GetDuckDBVersion(idx_t version_number);
+
 using block_id_t = int64_t;
 
 #define INVALID_BLOCK (-1)

package/src/duckdb/src/storage/single_file_block_manager.cpp

@@ -46,6 +46,29 @@ MainHeader MainHeader::Deserialize(Deserializer &source) {
 		throw IOException("The file is not a valid DuckDB database file!");
 	}
 	header.version_number = source.Read<uint64_t>();
+	// check the version number
+	if (header.version_number != VERSION_NUMBER) {
+		auto version = GetDuckDBVersion(header.version_number);
+		string version_text;
+		if (version) {
+			// known version
+			version_text = "DuckDB version " + string(version);
+		} else {
+			version_text = string("an ") + (VERSION_NUMBER > header.version_number ? "older development" : "newer") +
+			               string(" version of DuckDB");
+		}
+		throw IOException(
+		    "Trying to read a database file with version number %lld, but we can only read version %lld.\n"
+		    "The database file was created with %s.\n\n"
+		    "The storage of DuckDB is not yet stable; newer versions of DuckDB cannot read old database files and "
+		    "vice versa.\n"
+		    "The storage will be stabilized when version 1.0 releases.\n\n"
+		    "For now, we recommend that you load the database file in a supported version of DuckDB, and use the "
+		    "EXPORT DATABASE command "
+		    "followed by IMPORT DATABASE on the current version of DuckDB.\n\n"
+		    "See the storage page for more information: https://duckdb.org/internals/storage",
+		    header.version_number, VERSION_NUMBER, version_text);
+	}
 	// read the flags
 	FieldReader reader(source);
 	for (idx_t i = 0; i < FLAG_COUNT; i++) {
@@ -150,20 +173,7 @@ SingleFileBlockManager::SingleFileBlockManager(DatabaseInstance &db, string path
 		MainHeader::CheckMagicBytes(*handle);
 		// otherwise, we check the metadata of the file
 		header_buffer.ReadAndChecksum(*handle, 0);
-		MainHeader header = DeserializeHeaderStructure<MainHeader>(header_buffer.buffer);
-		// check the version number
-		if (header.version_number != VERSION_NUMBER) {
-			throw IOException(
-			    "Trying to read a database file with version number %lld, but we can only read version %lld.\n"
-			    "The database file was created with an %s version of DuckDB.\n\n"
-			    "The storage of DuckDB is not yet stable; newer versions of DuckDB cannot read old database files and "
-			    "vice versa.\n"
-			    "The storage will be stabilized when version 1.0 releases.\n\n"
-			    "For now, we recommend that you load the database file in a supported version of DuckDB, and use the "
-			    "EXPORT DATABASE command "
-			    "followed by IMPORT DATABASE on the current version of DuckDB.",
-			    header.version_number, VERSION_NUMBER, VERSION_NUMBER > header.version_number ? "older" : "newer");
-		}
+		DeserializeHeaderStructure<MainHeader>(header_buffer.buffer);
 
 		// read the database headers from disk
 		DatabaseHeader h1, h2;

package/src/duckdb/src/storage/storage_info.cpp

@@ -4,4 +4,35 @@ namespace duckdb {
 
 const uint64_t VERSION_NUMBER = 40;
 
+struct StorageVersionInfo {
+	const char *version_name;
+	idx_t storage_version;
+};
+
+static StorageVersionInfo storage_version_info[] = {{"v0.6.0 or v0.6.1", 39},
+                                                    {"v0.5.0 or v0.5.1", 38},
+                                                    {"v0.3.3, v0.3.4 or v0.4.0", 33},
+                                                    {"v0.3.2", 31},
+                                                    {"v0.3.1", 27},
+                                                    {"v0.3.0", 25},
+                                                    {"v0.2.9", 21},
+                                                    {"v0.2.8", 18},
+                                                    {"v0.2.7", 17},
+                                                    {"v0.2.6", 15},
+                                                    {"v0.2.5", 13},
+                                                    {"v0.2.4", 11},
+                                                    {"v0.2.3", 6},
+                                                    {"v0.2.2", 4},
+                                                    {"v0.2.1 and prior", 1},
+                                                    {nullptr, 0}};
+
+const char *GetDuckDBVersion(idx_t version_number) {
+	for (idx_t i = 0; storage_version_info[i].version_name; i++) {
+		if (version_number == storage_version_info[i].storage_version) {
+			return storage_version_info[i].version_name;
+		}
+	}
+	return nullptr;
+}
+
 } // namespace duckdb