duckdb 0.3.2-dev24.0 → 0.3.2-dev31.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.3.2-dev24.0",
+  "version": "0.3.2-dev31.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -18315,7 +18315,7 @@ struct RegexpFun {
 
 struct SubstringFun {
 	static void RegisterFunction(BuiltinFunctions &set);
-	static string_t SubstringScalarFunction(Vector &result, string_t input, int32_t offset, int32_t length);
+	static string_t SubstringScalarFunction(Vector &result, string_t input, int64_t offset, int64_t length);
 };
 
 struct PrintfFun {
@@ -61228,15 +61228,9 @@ struct BufferedCSVReaderOptions {
 	//! Whether or not a type format is specified
 	std::map<LogicalTypeId, bool> has_format = {{LogicalTypeId::DATE, false}, {LogicalTypeId::TIMESTAMP, false}};
 
-	std::string toString() const {
-		return "DELIMITER='" + delimiter + (has_delimiter ? "'" : (auto_detect ? "' (auto detected)" : "' (default)")) +
-		       ", QUOTE='" + quote + (has_quote ? "'" : (auto_detect ? "' (auto detected)" : "' (default)")) +
-		       ", ESCAPE='" + escape + (has_escape ? "'" : (auto_detect ? "' (auto detected)" : "' (default)")) +
-		       ", HEADER=" + std::to_string(header) +
-		       (has_header ? "" : (auto_detect ? " (auto detected)" : "' (default)")) +
-		       ", SAMPLE_SIZE=" + std::to_string(sample_chunk_size * sample_chunks) +
-		       ", ALL_VARCHAR=" + std::to_string(all_varchar);
-	}
+	void SetDelimiter(const string &delimiter);
+
+	std::string ToString() const;
 };
 
 enum class ParserMode : uint8_t { PARSING = 0, SNIFFING_DIALECT = 1, SNIFFING_DATATYPES = 2, PARSING_HEADER = 3 };
@@ -61389,6 +61383,24 @@ private:
 
 namespace duckdb {
 
+void BufferedCSVReaderOptions::SetDelimiter(const string &input) {
+	this->delimiter = StringUtil::Replace(input, "\\t", "\t");
+	this->has_delimiter = true;
+	if (input.empty()) {
+		throw BinderException("DELIM or SEP must not be empty");
+	}
+}
+
+std::string BufferedCSVReaderOptions::ToString() const {
+	return "DELIMITER='" + delimiter + (has_delimiter ? "'" : (auto_detect ? "' (auto detected)" : "' (default)")) +
+	       ", QUOTE='" + quote + (has_quote ? "'" : (auto_detect ? "' (auto detected)" : "' (default)")) +
+	       ", ESCAPE='" + escape + (has_escape ? "'" : (auto_detect ? "' (auto detected)" : "' (default)")) +
+	       ", HEADER=" + std::to_string(header) +
+	       (has_header ? "" : (auto_detect ? " (auto detected)" : "' (default)")) +
+	       ", SAMPLE_SIZE=" + std::to_string(sample_chunk_size * sample_chunks) +
+	       ", ALL_VARCHAR=" + std::to_string(all_varchar);
+}
+
 static string GetLineNumberStr(idx_t linenr, bool linenr_estimated) {
 	string estimated = (linenr_estimated ? string(" (estimated)") : string(""));
 	return to_string(linenr + 1) + estimated;
@@ -62202,7 +62214,7 @@ vector<LogicalType> BufferedCSVReader::RefineTypeDetection(const vector<LogicalT
 		if (requested_types.size() != options.num_cols) {
 			throw InvalidInputException(
 			    "Error while determining column types: found %lld columns but expected %d. (%s)", options.num_cols,
-			    requested_types.size(), options.toString());
+			    requested_types.size(), options.ToString());
 		} else {
 			detected_types = requested_types;
 		}
@@ -62460,7 +62472,7 @@ in_quotes:
 	} while (ReadBuffer(start));
 	// still in quoted state at the end of the file, error:
 	error_message = StringUtil::Format("Error in file \"%s\" on line %s: unterminated quotes. (%s)", options.file_path,
-	                                   GetLineNumberStr(linenr, linenr_estimated).c_str(), options.toString());
+	                                   GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
 	return false;
 unquote:
 	/* state: unquote */
@@ -62490,7 +62502,7 @@ unquote:
 				error_message = StringUtil::Format(
 				    "Error in file \"%s\" on line %s: quote should be followed by end of value, end "
 				    "of row or another quote. (%s)",
-				    options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), options.toString());
+				    options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
 				return false;
 			}
 			if (delimiter_pos == options.delimiter.size()) {
@@ -62507,7 +62519,7 @@ unquote:
 	} while (ReadBuffer(start));
 	error_message = StringUtil::Format(
 	    "Error in file \"%s\" on line %s: quote should be followed by end of value, end of row or another quote. (%s)",
-	    options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), options.toString());
+	    options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
 	return false;
 handle_escape:
 	escape_pos = 0;
@@ -62522,7 +62534,7 @@ handle_escape:
 			if (count > escape_pos && count > quote_pos) {
 				error_message = StringUtil::Format(
 				    "Error in file \"%s\" on line %s: neither QUOTE nor ESCAPE is proceeded by ESCAPE. (%s)",
-				    options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), options.toString());
+				    options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
 				return false;
 			}
 			if (quote_pos == options.quote.size() || escape_pos == options.escape.size()) {
@@ -62533,7 +62545,7 @@ handle_escape:
 	} while (ReadBuffer(start));
 	error_message =
 	    StringUtil::Format("Error in file \"%s\" on line %s: neither QUOTE nor ESCAPE is proceeded by ESCAPE. (%s)",
-	                       options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), options.toString());
+	                       options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
 	return false;
 carriage_return:
 	/* state: carriage_return */
@@ -62665,7 +62677,7 @@ in_quotes:
 	} while (ReadBuffer(start));
 	// still in quoted state at the end of the file, error:
 	throw InvalidInputException("Error in file \"%s\" on line %s: unterminated quotes. (%s)", options.file_path,
-	                            GetLineNumberStr(linenr, linenr_estimated).c_str(), options.toString());
+	                            GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
 unquote:
 	/* state: unquote */
 	// this state handles the state directly after we unquote
@@ -62692,7 +62704,7 @@ unquote:
 		error_message = StringUtil::Format(
 		    "Error in file \"%s\" on line %s: quote should be followed by end of value, end of "
 		    "row or another quote. (%s)",
-		    options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), options.toString());
+		    options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
 		return false;
 	}
 handle_escape:
@@ -62702,13 +62714,13 @@ handle_escape:
 	if (position >= buffer_size && !ReadBuffer(start)) {
 		error_message = StringUtil::Format(
 		    "Error in file \"%s\" on line %s: neither QUOTE nor ESCAPE is proceeded by ESCAPE. (%s)", options.file_path,
-		    GetLineNumberStr(linenr, linenr_estimated).c_str(), options.toString());
+		    GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
 		return false;
 	}
 	if (buffer[position] != options.quote[0] && buffer[position] != options.escape[0]) {
 		error_message = StringUtil::Format(
 		    "Error in file \"%s\" on line %s: neither QUOTE nor ESCAPE is proceeded by ESCAPE. (%s)", options.file_path,
-		    GetLineNumberStr(linenr, linenr_estimated).c_str(), options.toString());
+		    GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
 		return false;
 	}
 	// escape was followed by quote or escape, go back to quoted state
@@ -62848,7 +62860,7 @@ void BufferedCSVReader::AddValue(char *str_val, idx_t length, idx_t &column, vec
 	if (column >= sql_types.size()) {
 		throw InvalidInputException("Error on line %s: expected %lld values per row, but got more. (%s)",
 		                            GetLineNumberStr(linenr, linenr_estimated).c_str(), sql_types.size(),
-		                            options.toString());
+		                            options.ToString());
 	}
 
 	// insert the line number into the chunk
@@ -62903,7 +62915,7 @@ bool BufferedCSVReader::AddRow(DataChunk &insert_chunk, idx_t &column) {
 	if (column < sql_types.size() && mode != ParserMode::SNIFFING_DIALECT) {
 		throw InvalidInputException("Error on line %s: expected %lld values per row, but got %d. (%s)",
 		                            GetLineNumberStr(linenr, linenr_estimated).c_str(), sql_types.size(), column,
-		                            options.toString());
+		                            options.ToString());
 	}
 
 	if (mode == ParserMode::SNIFFING_DIALECT) {
@@ -62956,7 +62968,7 @@ void BufferedCSVReader::Flush(DataChunk &insert_chunk) {
 						throw InvalidInputException("Error in file \"%s\" between line %llu and %llu in column \"%s\": "
 						                            "file is not valid UTF8. Parser options: %s",
 						                            options.file_path, linenr - parse_chunk.size(), linenr, col_name,
-						                            options.toString());
+						                            options.ToString());
 					}
 				}
 			}
@@ -62990,11 +63002,11 @@ void BufferedCSVReader::Flush(DataChunk &insert_chunk) {
 					                            "(SAMPLE_SIZE=X [X rows] or SAMPLE_SIZE=-1 [all rows]), "
 					                            "or skipping column conversion (ALL_VARCHAR=1)",
 					                            error_message, col_name, linenr - parse_chunk.size() + 1, linenr,
-					                            options.toString());
+					                            options.ToString());
 				} else {
 					throw InvalidInputException("%s between line %llu and %llu in column %s. Parser options: %s ",
 					                            error_message, linenr - parse_chunk.size(), linenr, col_name,
-					                            options.toString());
+					                            options.ToString());
 				}
 			}
 		}
@@ -92198,7 +92210,7 @@ string_t SubstringASCII(Vector &result, string_t input, int64_t offset, int64_t
 	return SubstringSlice(result, input_data, start, end - start);
 }
 
-string_t SubstringFun::SubstringScalarFunction(Vector &result, string_t input, int32_t offset, int32_t length) {
+string_t SubstringFun::SubstringScalarFunction(Vector &result, string_t input, int64_t offset, int64_t length) {
 	auto input_data = input.GetDataUnsafe();
 	auto input_size = input.GetSize();
 
@@ -92265,16 +92277,16 @@ static void SubstringFunction(DataChunk &args, ExpressionState &state, Vector &r
 	if (args.ColumnCount() == 3) {
 		auto &length_vector = args.data[2];
 
-		TernaryExecutor::Execute<string_t, int32_t, int32_t, string_t>(
+		TernaryExecutor::Execute<string_t, int64_t, int64_t, string_t>(
 		    input_vector, offset_vector, length_vector, result, args.size(),
-		    [&](string_t input_string, int32_t offset, int32_t length) {
+		    [&](string_t input_string, int64_t offset, int64_t length) {
 			    return SubstringFun::SubstringScalarFunction(result, input_string, offset, length);
 		    });
 	} else {
-		BinaryExecutor::Execute<string_t, int32_t, string_t>(
-		    input_vector, offset_vector, result, args.size(), [&](string_t input_string, int32_t offset) {
+		BinaryExecutor::Execute<string_t, int64_t, string_t>(
+		    input_vector, offset_vector, result, args.size(), [&](string_t input_string, int64_t offset) {
 			    return SubstringFun::SubstringScalarFunction(result, input_string, offset,
-			                                                 NumericLimits<int32_t>::Maximum());
+			                                                 NumericLimits<int64_t>::Maximum() - offset);
 		    });
 	}
 }
@@ -92285,15 +92297,15 @@ static void SubstringFunctionASCII(DataChunk &args, ExpressionState &state, Vect
 	if (args.ColumnCount() == 3) {
 		auto &length_vector = args.data[2];
 
-		TernaryExecutor::Execute<string_t, int32_t, int32_t, string_t>(
+		TernaryExecutor::Execute<string_t, int64_t, int64_t, string_t>(
 		    input_vector, offset_vector, length_vector, result, args.size(),
-		    [&](string_t input_string, int32_t offset, int32_t length) {
+		    [&](string_t input_string, int64_t offset, int64_t length) {
 			    return SubstringASCII(result, input_string, offset, length);
 		    });
 	} else {
-		BinaryExecutor::Execute<string_t, int32_t, string_t>(
-		    input_vector, offset_vector, result, args.size(), [&](string_t input_string, int32_t offset) {
-			    return SubstringASCII(result, input_string, offset, NumericLimits<int32_t>::Maximum());
+		BinaryExecutor::Execute<string_t, int64_t, string_t>(
+		    input_vector, offset_vector, result, args.size(), [&](string_t input_string, int64_t offset) {
+			    return SubstringASCII(result, input_string, offset, NumericLimits<int64_t>::Maximum() - offset);
 		    });
 	}
 }
@@ -92315,10 +92327,10 @@ static unique_ptr<BaseStatistics> SubstringPropagateStats(ClientContext &context
 
 void SubstringFun::RegisterFunction(BuiltinFunctions &set) {
 	ScalarFunctionSet substr("substring");
-	substr.AddFunction(ScalarFunction({LogicalType::VARCHAR, LogicalType::INTEGER, LogicalType::INTEGER},
+	substr.AddFunction(ScalarFunction({LogicalType::VARCHAR, LogicalType::BIGINT, LogicalType::BIGINT},
 	                                  LogicalType::VARCHAR, SubstringFunction, false, nullptr, nullptr,
 	                                  SubstringPropagateStats));
-	substr.AddFunction(ScalarFunction({LogicalType::VARCHAR, LogicalType::INTEGER}, LogicalType::VARCHAR,
+	substr.AddFunction(ScalarFunction({LogicalType::VARCHAR, LogicalType::BIGINT}, LogicalType::VARCHAR,
 	                                  SubstringFunction, false, nullptr, nullptr, SubstringPropagateStats));
 	set.AddFunction(substr);
 	substr.name = "substr";
@@ -94500,11 +94512,7 @@ static int64_t ParseInteger(vector<Value> &set) {
 //===--------------------------------------------------------------------===//
 static bool ParseBaseOption(BufferedCSVReaderOptions &options, string &loption, vector<Value> &set) {
 	if (StringUtil::StartsWith(loption, "delim") || StringUtil::StartsWith(loption, "sep")) {
-		options.delimiter = ParseString(set);
-		options.has_delimiter = true;
-		if (options.delimiter.length() == 0) {
-			throw BinderException("DELIM or SEP must not be empty");
-		}
+		options.SetDelimiter(ParseString(set));
 	} else if (loption == "quote") {
 		options.quote = ParseString(set);
 		options.has_quote = true;
@@ -95789,8 +95797,7 @@ static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, vector<Value
 		if (kv.first == "auto_detect") {
 			options.auto_detect = kv.second.value_.boolean;
 		} else if (kv.first == "sep" || kv.first == "delim") {
-			options.delimiter = kv.second.str_value;
-			options.has_delimiter = true;
+			options.SetDelimiter(kv.second.str_value);
 		} else if (kv.first == "header") {
 			options.header = kv.second.value_.boolean;
 			options.has_header = true;
@@ -212139,7 +212146,6 @@ unsigned char *unicode_to_utf8(pg_wchar c, unsigned char *utf8string);
 
 // max parse tree size approx 100 MB, should be enough
 #define PG_MALLOC_SIZE 10240
-#define PG_MALLOC_LIMIT 1000
 
 namespace duckdb_libpgquery {
 
@@ -212151,7 +212157,8 @@ struct pg_parser_state_str {
 
 	size_t malloc_pos;
 	size_t malloc_ptr_idx;
-	char *malloc_ptrs[PG_MALLOC_LIMIT];
+	char **malloc_ptrs;
+	size_t malloc_ptr_size;
 };
 
 static __thread parser_state pg_parser_state;
@@ -212161,8 +212168,13 @@ __thread PGNode *duckdb_newNodeMacroHolder;
 #endif
 
 static void allocate_new(parser_state *state, size_t n) {
-	if (state->malloc_ptr_idx + 1 >= PG_MALLOC_LIMIT) {
-		throw std::runtime_error("Memory allocation failure");
+	if (state->malloc_ptr_idx >= state->malloc_ptr_size) {
+		size_t new_size = state->malloc_ptr_size * 2;
+		auto new_malloc_ptrs = (char **) malloc(sizeof(char *) * new_size);
+		memcpy(new_malloc_ptrs, state->malloc_ptrs, state->malloc_ptr_size * sizeof(char*));
+		free(state->malloc_ptrs);
+		state->malloc_ptr_size = new_size;
+		state->malloc_ptrs = new_malloc_ptrs;
 	}
 	if (n < PG_MALLOC_SIZE) {
 		n = PG_MALLOC_SIZE;
@@ -212193,6 +212205,8 @@ void pg_parser_init() {
 	pg_parser_state.pg_err_code = PGUNDEFINED;
 	pg_parser_state.pg_err_msg[0] = '\0';
 
+	pg_parser_state.malloc_ptr_size = 4;
+	pg_parser_state.malloc_ptrs = (char **) malloc(sizeof(char *) * pg_parser_state.malloc_ptr_size);
 	pg_parser_state.malloc_ptr_idx = 0;
 	allocate_new(&pg_parser_state, 1);
 }
@@ -212202,8 +212216,16 @@ void pg_parser_parse(const char *query, parse_result *res) {
 	try {
 		res->parse_tree = duckdb_libpgquery::raw_parser(query);
 		res->success = pg_parser_state.pg_err_code == PGUNDEFINED;
-	} catch (...) {
+	} catch (std::exception &ex) {
 		res->success = false;
+		// copy the error message of the exception
+		auto error_message = ex.what();
+		uint32_t pos = 0;
+		while(pos < 1023 && error_message[pos]) {
+			pg_parser_state.pg_err_msg[pos] = error_message[pos];
+			pos++;
+		}
+		pg_parser_state.pg_err_msg[pos] = '\0';
 	}
 	res->error_message = pg_parser_state.pg_err_msg;
 	res->error_location = pg_parser_state.pg_err_pos;
@@ -212217,6 +212239,7 @@ void pg_parser_cleanup() {
 			pg_parser_state.malloc_ptrs[ptr_idx] = nullptr;
 		}
 	}
+	free(pg_parser_state.malloc_ptrs);
 }
 
 int ereport(int code, ...) {

package/src/duckdb.hpp

@@ -10,8 +10,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 #pragma once
 #define DUCKDB_AMALGAMATION 1
-#define DUCKDB_SOURCE_ID "6926d0118"
-#define DUCKDB_VERSION "0.3.2-dev24"
+#define DUCKDB_SOURCE_ID "275e368e7"
+#define DUCKDB_VERSION "0.3.2-dev31"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //