duckdb 0.3.2-dev816.0 → 0.3.2-dev843.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.3.2-dev816.0",
+  "version": "0.3.2-dev843.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -21139,12 +21139,14 @@ public:
 
 	int64_t GetFileSize(FileHandle &handle) override;
 
+	void Reset(FileHandle &handle) override;
 	bool OnDiskFile(FileHandle &handle) override {
 		return false;
 	};
 	bool CanSeek() override {
 		return false;
 	}
+	void FileSync(FileHandle &handle) override;
 
 	std::string GetName() const override {
 		return "PipeFileSystem";
@@ -21181,6 +21183,10 @@ int64_t PipeFile::WriteChunk(void *buffer, int64_t nr_bytes) {
 	return child_handle->Write(buffer, nr_bytes);
 }
 
+void PipeFileSystem::Reset(FileHandle &handle) {
+	throw InternalException("Cannot reset pipe file system");
+}
+
 int64_t PipeFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes) {
 	auto &pipe = (PipeFile &)handle;
 	return pipe.ReadChunk(buffer, nr_bytes);
@@ -21195,6 +21201,9 @@ int64_t PipeFileSystem::GetFileSize(FileHandle &handle) {
 	return 0;
 }
 
+void PipeFileSystem::FileSync(FileHandle &handle) {
+}
+
 unique_ptr<FileHandle> PipeFileSystem::OpenPipe(unique_ptr<FileHandle> handle) {
 	auto path = handle->path;
 	return make_unique<PipeFile>(move(handle), path);
@@ -56617,6 +56626,125 @@ string PhysicalTopN::ParamsToString() const {
 
 namespace duckdb {
 
+struct CSVFileHandle {
+public:
+	explicit CSVFileHandle(unique_ptr<FileHandle> file_handle_p) : file_handle(move(file_handle_p)) {
+		can_seek = file_handle->CanSeek();
+		plain_file_source = file_handle->OnDiskFile() && can_seek;
+		file_size = file_handle->GetFileSize();
+	}
+
+	bool CanSeek() {
+		return can_seek;
+	}
+	void Seek(idx_t position) {
+		if (!can_seek) {
+			throw InternalException("Cannot seek in this file");
+		}
+		file_handle->Seek(position);
+	}
+	idx_t SeekPosition() {
+		if (!can_seek) {
+			throw InternalException("Cannot seek in this file");
+		}
+		return file_handle->SeekPosition();
+	}
+	void Reset() {
+		if (plain_file_source) {
+			file_handle->Reset();
+		} else {
+			if (!reset_enabled) {
+				throw InternalException("Reset called but reset is not enabled for this CSV Handle");
+			}
+			read_position = 0;
+		}
+	}
+	bool PlainFileSource() {
+		return plain_file_source;
+	}
+
+	idx_t FileSize() {
+		return file_size;
+	}
+
+	idx_t Read(void *buffer, idx_t nr_bytes) {
+		if (!plain_file_source) {
+			// not a plain file source: we need to do some bookkeeping around the reset functionality
+			idx_t result_offset = 0;
+			if (read_position < buffer_size) {
+				// we need to read from our cached buffer
+				auto buffer_read_count = MinValue<idx_t>(nr_bytes, buffer_size - read_position);
+				memcpy(buffer, cached_buffer.get() + read_position, buffer_read_count);
+				result_offset += buffer_read_count;
+				read_position += buffer_read_count;
+				if (result_offset == nr_bytes) {
+					return nr_bytes;
+				}
+			} else if (!reset_enabled && cached_buffer) {
+				// reset is disabled but we still have cached data
+				// we can remove any cached data
+				cached_buffer.reset();
+				buffer_size = 0;
+				buffer_capacity = 0;
+				read_position = 0;
+			}
+			// we have data left to read from the file
+			// read directly into the buffer
+			auto bytes_read = file_handle->Read((char *)buffer + result_offset, nr_bytes - result_offset);
+			read_position += bytes_read;
+			if (reset_enabled) {
+				// if reset caching is enabled, we need to cache the bytes that we have read
+				if (buffer_size + bytes_read >= buffer_capacity) {
+					// no space; first enlarge the buffer
+					buffer_capacity = MaxValue<idx_t>(NextPowerOfTwo(buffer_size + bytes_read), buffer_capacity * 2);
+
+					auto new_buffer = unique_ptr<data_t[]>(new data_t[buffer_capacity]);
+					if (buffer_size > 0) {
+						memcpy(new_buffer.get(), cached_buffer.get(), buffer_size);
+					}
+					cached_buffer = move(new_buffer);
+				}
+				memcpy(cached_buffer.get() + buffer_size, (char *)buffer + result_offset, bytes_read);
+				buffer_size += bytes_read;
+			}
+
+			return result_offset + bytes_read;
+		} else {
+			return file_handle->Read(buffer, nr_bytes);
+		}
+	}
+
+	string ReadLine() {
+		string result;
+		char buffer[1];
+		while (true) {
+			idx_t tuples_read = Read(buffer, 1);
+			if (tuples_read == 0 || buffer[0] == '\n') {
+				return result;
+			}
+			if (buffer[0] != '\r') {
+				result += buffer[0];
+			}
+		}
+	}
+
+	void DisableReset() {
+		this->reset_enabled = false;
+	}
+
+private:
+	unique_ptr<FileHandle> file_handle;
+	bool reset_enabled = true;
+	bool can_seek = false;
+	bool plain_file_source = false;
+	idx_t file_size = 0;
+	// reset support
+	unique_ptr<data_t[]> cached_buffer;
+	idx_t read_position = 0;
+	idx_t buffer_size = 0;
+	idx_t buffer_capacity = 0;
+};
+
 void BufferedCSVReaderOptions::SetDelimiter(const string &input) {
 	this->delimiter = StringUtil::Replace(input, "\\t", "\t");
 	this->has_delimiter = true;
@@ -56749,6 +56877,13 @@ BufferedCSVReader::BufferedCSVReader(ClientContext &context, BufferedCSVReaderOp
                         requested_types) {
 }
 
+BufferedCSVReader::~BufferedCSVReader() {
+}
+
+idx_t BufferedCSVReader::GetFileSize() {
+	return file_handle ? file_handle->FileSize() : 0;
+}
+
 void BufferedCSVReader::Initialize(const vector<LogicalType> &requested_types) {
 	PrepareComplexParser();
 	if (options.auto_detect) {
@@ -56765,6 +56900,9 @@ void BufferedCSVReader::Initialize(const vector<LogicalType> &requested_types) {
 		SkipRowsAndReadHeader(options.skip_rows, options.header);
 	}
 	InitParseChunk(sql_types.size());
+	// we only need reset support during the automatic CSV type detection
+	// since reset support might require caching (in the case of streams), we disable it for the remainder
+	file_handle->DisableReset();
 }
 
 void BufferedCSVReader::PrepareComplexParser() {
@@ -56773,12 +56911,10 @@ void BufferedCSVReader::PrepareComplexParser() {
 	quote_search = TextSearchShiftArray(options.quote);
 }
 
-unique_ptr<FileHandle> BufferedCSVReader::OpenCSV(const BufferedCSVReaderOptions &options) {
-	auto result = fs.OpenFile(options.file_path.c_str(), FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK,
-	                          options.compression, this->opener);
-	plain_file_source = result->OnDiskFile() && result->CanSeek();
-	file_size = result->GetFileSize();
-	return result;
+unique_ptr<CSVFileHandle> BufferedCSVReader::OpenCSV(const BufferedCSVReaderOptions &options) {
+	auto file_handle = fs.OpenFile(options.file_path.c_str(), FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK,
+	                               options.compression, this->opener);
+	return make_unique<CSVFileHandle>(move(file_handle));
 }
 
 // Helper function to generate column names
@@ -56944,7 +57080,7 @@ bool BufferedCSVReader::JumpToNextSample() {
 	// assess if it makes sense to jump, based on size of the first chunk relative to size of the entire file
 	if (sample_chunk_idx == 0) {
 		idx_t bytes_first_chunk = bytes_in_chunk;
-		double chunks_fit = (file_size / (double)bytes_first_chunk);
+		double chunks_fit = (file_handle->FileSize() / (double)bytes_first_chunk);
 		jumping_samples = chunks_fit >= options.sample_chunks;
 
 		// jump back to the beginning
@@ -56959,7 +57095,7 @@ bool BufferedCSVReader::JumpToNextSample() {
 
 	// if we deal with any other sources than plaintext files, jumping_samples can be tricky. In that case
 	// we just read x continuous chunks from the stream TODO: make jumps possible for zipfiles.
-	if (!plain_file_source || !jumping_samples) {
+	if (!file_handle->PlainFileSource() || !jumping_samples) {
 		sample_chunk_idx++;
 		return true;
 	}
@@ -56969,13 +57105,13 @@ bool BufferedCSVReader::JumpToNextSample() {
 	bytes_per_line_avg = ((bytes_per_line_avg * (sample_chunk_idx)) + bytes_per_line) / (sample_chunk_idx + 1);
 
 	// if none of the previous conditions were met, we can jump
-	idx_t partition_size = (idx_t)round(file_size / (double)options.sample_chunks);
+	idx_t partition_size = (idx_t)round(file_handle->FileSize() / (double)options.sample_chunks);
 
 	// calculate offset to end of the current partition
 	int64_t offset = partition_size - bytes_in_chunk - remaining_bytes_in_buffer;
 	auto current_pos = file_handle->SeekPosition();
 
-	if (current_pos + offset < file_size) {
+	if (current_pos + offset < file_handle->FileSize()) {
 		// set position in stream and clear failure bits
 		file_handle->Seek(current_pos + offset);
 
@@ -56986,10 +57122,10 @@ bool BufferedCSVReader::JumpToNextSample() {
 		// seek backwards from the end in last chunk and hope to catch the end of the file
 		// TODO: actually it would be good to make sure that the end of file is being reached, because
 		// messy end-lines are quite common. For this case, however, we first need a skip_end detection anyways.
-		file_handle->Seek(file_size - bytes_in_chunk);
+		file_handle->Seek(file_handle->FileSize() - bytes_in_chunk);
 
 		// estimate linenr
-		linenr = (idx_t)round((file_size - bytes_in_chunk) / bytes_per_line_avg);
+		linenr = (idx_t)round((file_handle->FileSize() - bytes_in_chunk) / bytes_per_line_avg);
 		linenr_estimated = true;
 	}
 
@@ -57143,6 +57279,7 @@ void BufferedCSVReader::DetectDialect(const vector<LogicalType> &requested_types
 
 					JumpToBeginning(original_options.skip_rows);
 					sniffed_column_counts.clear();
+
 					if (!TryParseCSV(ParserMode::SNIFFING_DIALECT)) {
 						continue;
 					}
@@ -88587,9 +88724,9 @@ static void GenerateUUIDFunction(DataChunk &args, ExpressionState &state, Vector
 }
 
 void UUIDFun::RegisterFunction(BuiltinFunctions &set) {
+	ScalarFunction uuid_function({}, LogicalType::UUID, GenerateUUIDFunction, true, UUIDRandomBind);
 	// generate a random uuid
-	set.AddFunction(
-	    ScalarFunction("gen_random_uuid", {}, LogicalType::UUID, GenerateUUIDFunction, true, UUIDRandomBind));
+	set.AddFunction({"uuid", "gen_random_uuid"}, uuid_function);
 }
 
 } // namespace duckdb
@@ -91542,7 +91679,7 @@ static unique_ptr<FunctionOperatorData> ReadCSVInit(ClientContext &context, cons
 		result->csv_reader = make_unique<BufferedCSVReader>(context, bind_data.options, bind_data.sql_types);
 	}
 	bind_data.bytes_read = 0;
-	bind_data.file_size = result->csv_reader->file_size;
+	bind_data.file_size = result->csv_reader->GetFileSize();
 	result->file_index = 1;
 	return move(result);
 }
@@ -98287,8 +98424,8 @@ string ClientContext::VerifyQuery(ClientContextLock &lock, const string &query,
 	} catch (std::exception &ex) {
 		original_result->error = ex.what();
 		original_result->success = false;
-		interrupted = false;
 	}
+	interrupted = false;
 
 	// check explain, only if q does not already contain EXPLAIN
 	if (original_result->success) {
@@ -98308,8 +98445,8 @@ string ClientContext::VerifyQuery(ClientContextLock &lock, const string &query,
 	} catch (std::exception &ex) {
 		copied_result->error = ex.what();
 		copied_result->success = false;
-		interrupted = false;
 	}
+	interrupted = false;
 	// now execute the deserialized statement
 	try {
 		auto result = RunStatementInternal(lock, query, move(deserialized_stmt), false, false);
@@ -98317,8 +98454,8 @@ string ClientContext::VerifyQuery(ClientContextLock &lock, const string &query,
 	} catch (std::exception &ex) {
 		deserialized_result->error = ex.what();
 		deserialized_result->success = false;
-		interrupted = false;
 	}
+	interrupted = false;
 	// now execute the unoptimized statement
 	config.enable_optimizer = false;
 	try {
@@ -98327,8 +98464,8 @@ string ClientContext::VerifyQuery(ClientContextLock &lock, const string &query,
 	} catch (std::exception &ex) {
 		unoptimized_result->error = ex.what();
 		unoptimized_result->success = false;
-		interrupted = false;
 	}
+	interrupted = false;
 	config.enable_optimizer = true;
 
 	if (profiling_is_enabled) {
@@ -119251,6 +119388,10 @@ FilterPropagateResult StatisticsPropagator::PropagateComparison(BaseStatistics &
 	// only handle numerics for now
 	switch (left.type.InternalType()) {
 	case PhysicalType::BOOL:
+	case PhysicalType::UINT8:
+	case PhysicalType::UINT16:
+	case PhysicalType::UINT32:
+	case PhysicalType::UINT64:
 	case PhysicalType::INT8:
 	case PhysicalType::INT16:
 	case PhysicalType::INT32:
@@ -119378,6 +119519,10 @@ namespace duckdb {
 unique_ptr<BaseStatistics> StatisticsPropagator::StatisticsFromValue(const Value &input) {
 	switch (input.type().InternalType()) {
 	case PhysicalType::BOOL:
+	case PhysicalType::UINT8:
+	case PhysicalType::UINT16:
+	case PhysicalType::UINT32:
+	case PhysicalType::UINT64:
 	case PhysicalType::INT8:
 	case PhysicalType::INT16:
 	case PhysicalType::INT32:
@@ -160562,7 +160707,6 @@ unique_ptr<BaseStatistics> StringStatistics::Copy() {
 	memcpy(stats->max, max, MAX_STRING_MINMAX_SIZE);
 	stats->has_unicode = has_unicode;
 	stats->max_string_length = max_string_length;
-	stats->max_string_length = max_string_length;
 	if (validity_stats) {
 		stats->validity_stats = validity_stats->Copy();
 	}

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "684aaf4e7"
-#define DUCKDB_VERSION "0.3.2-dev816"
+#define DUCKDB_SOURCE_ID "dbe8a9905"
+#define DUCKDB_VERSION "0.3.2-dev843"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -20341,6 +20341,7 @@ protected:
 
 namespace duckdb {
 struct CopyInfo;
+struct CSVFileHandle;
 struct FileHandle;
 struct StrpTimeFormat;
 
@@ -20441,15 +20442,14 @@ public:
 
 	BufferedCSVReader(FileSystem &fs, FileOpener *opener, BufferedCSVReaderOptions options,
 	                  const vector<LogicalType> &requested_types = vector<LogicalType>());
+	~BufferedCSVReader();
 
 	FileSystem &fs;
 	FileOpener *opener;
 	BufferedCSVReaderOptions options;
 	vector<LogicalType> sql_types;
 	vector<string> col_names;
-	unique_ptr<FileHandle> file_handle;
-	bool plain_file_source = false;
-	idx_t file_size = 0;
+	unique_ptr<CSVFileHandle> file_handle;
 
 	unique_ptr<char[]> buffer;
 	idx_t buffer_size;
@@ -20481,6 +20481,8 @@ public:
 	//! Extract a single DataChunk from the CSV file and stores it in insert_chunk
 	void ParseCSV(DataChunk &insert_chunk);
 
+	idx_t GetFileSize();
+
 private:
 	//! Initialize Parser
 	void Initialize(const vector<LogicalType> &requested_types);
@@ -20527,7 +20529,7 @@ private:
 	//! Reads a new buffer from the CSV file if the current one has been exhausted
 	bool ReadBuffer(idx_t &start);
 
-	unique_ptr<FileHandle> OpenCSV(const BufferedCSVReaderOptions &options);
+	unique_ptr<CSVFileHandle> OpenCSV(const BufferedCSVReaderOptions &options);
 
 	//! First phase of auto detection: detect CSV dialect (i.e. delimiter, quote rules, etc)
 	void DetectDialect(const vector<LogicalType> &requested_types, BufferedCSVReaderOptions &original_options,