duckdb 0.4.1-dev182.0 → 0.4.1-dev188.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.4.1-dev182.0",
+  "version": "0.4.1-dev188.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -7663,6 +7663,8 @@ string StatementTypeToString(StatementType type) {
 		return "SET";
 	case StatementType::LOAD_STATEMENT:
 		return "LOAD";
+	case StatementType::EXTENSION_STATEMENT:
+		return "EXTENSION";
 	case StatementType::INVALID_STATEMENT:
 		break;
 	}
@@ -112254,7 +112256,11 @@ TableFunction::TableFunction(const vector<LogicalType> &arguments, table_functio
                              table_function_init_local_t init_local)
     : TableFunction(string(), arguments, function, bind, init_global, init_local) {
 }
-TableFunction::TableFunction() : SimpleNamedParameterFunction("", {}) {
+TableFunction::TableFunction()
+    : SimpleNamedParameterFunction("", {}), bind(nullptr), init_global(nullptr), init_local(nullptr), function(nullptr),
+      in_out_function(nullptr), statistics(nullptr), dependency(nullptr), cardinality(nullptr),
+      pushdown_complex_filter(nullptr), to_string(nullptr), table_scan_progress(nullptr), get_batch_index(nullptr),
+      projection_pushdown(false), filter_pushdown(false) {
 }
 
 } // namespace duckdb
@@ -116965,6 +116971,7 @@ ParserOptions ClientContext::GetParserOptions() {
 	ParserOptions options;
 	options.preserve_identifier_case = ClientConfig::GetConfig(*this).preserve_identifier_case;
 	options.max_expression_depth = ClientConfig::GetConfig(*this).max_expression_depth;
+	options.extensions = &DBConfig::GetConfig(*this).parser_extensions;
 	return options;
 }
 
@@ -118240,6 +118247,7 @@ void DatabaseInstance::Configure(DBConfig &new_config) {
 	config.replacement_scans = move(new_config.replacement_scans);
 	config.initialize_default_database = new_config.initialize_default_database;
 	config.disabled_optimizers = move(new_config.disabled_optimizers);
+	config.parser_extensions = move(new_config.parser_extensions);
 }
 
 DBConfig &DBConfig::GetConfig(ClientContext &context) {
@@ -155261,6 +155269,36 @@ vector<string> ReadPgListToString(duckdb_libpgquery::PGList *column_list);
 
 
 
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/parser/statement/extension_statement.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+
+namespace duckdb {
+
+class ExtensionStatement : public SQLStatement {
+public:
+	ExtensionStatement(ParserExtension extension, unique_ptr<ParserExtensionParseData> parse_data);
+
+	//! The ParserExtension this statement was generated from
+	ParserExtension extension;
+	//! The parse data for this specific statement
+	unique_ptr<ParserExtensionParseData> parse_data;
+
+public:
+	unique_ptr<SQLStatement> Copy() const override;
+};
+
+} // namespace duckdb
+
 
 
 
@@ -155358,6 +155396,7 @@ public:
 
 
 
+
 // LICENSE_CHANGE_BEGIN
 // The following code up to LICENSE_CHANGE_END is subject to THIRD PARTY LICENSE #11
 // See the end of this file for a list
@@ -155420,6 +155459,22 @@ void Parser::ParseQuery(const string &query) {
 		parser.Parse(query);
 
 		if (!parser.success) {
+			if (options.extensions) {
+				for (auto &ext : *options.extensions) {
+					D_ASSERT(ext.parse_function);
+					auto result = ext.parse_function(ext.parser_info.get(), query);
+					if (result.type == ParserExtensionResultType::PARSE_SUCCESSFUL) {
+						auto statement = make_unique<ExtensionStatement>(ext, move(result.parse_data));
+						statement->stmt_length = query.size();
+						statement->stmt_location = 0;
+						statements.push_back(move(statement));
+						return;
+					}
+					if (result.type == ParserExtensionResultType::DISPLAY_EXTENSION_ERROR) {
+						throw ParserException(result.error);
+					}
+				}
+			}
 			throw ParserException(QueryErrorContext::Format(query, parser.error_message, parser.error_location - 1));
 		}
 
@@ -156812,6 +156867,19 @@ unique_ptr<SQLStatement> ExportStatement::Copy() const {
 } // namespace duckdb
 
 
+namespace duckdb {
+
+ExtensionStatement::ExtensionStatement(ParserExtension extension_p, unique_ptr<ParserExtensionParseData> parse_data_p)
+    : SQLStatement(StatementType::EXTENSION_STATEMENT), extension(move(extension_p)), parse_data(move(parse_data_p)) {
+}
+
+unique_ptr<SQLStatement> ExtensionStatement::Copy() const {
+	return make_unique<ExtensionStatement>(extension, parse_data->Copy());
+}
+
+} // namespace duckdb
+
+
 
 
 namespace duckdb {
@@ -168128,6 +168196,37 @@ BoundStatement Binder::Bind(ExportStatement &stmt) {
 
 
 
+namespace duckdb {
+
+BoundStatement Binder::Bind(ExtensionStatement &stmt) {
+	BoundStatement result;
+
+	// perform the planning of the function
+	D_ASSERT(stmt.extension.plan_function);
+	auto parse_result = stmt.extension.plan_function(stmt.extension.parser_info.get(), context, move(stmt.parse_data));
+
+	properties.read_only = parse_result.read_only;
+	properties.requires_valid_transaction = parse_result.requires_valid_transaction;
+	properties.return_type = parse_result.return_type;
+
+	// create the plan as a scan of the given table function
+	result.plan = BindTableFunction(parse_result.function, move(parse_result.parameters));
+	D_ASSERT(result.plan->type == LogicalOperatorType::LOGICAL_GET);
+	auto &get = (LogicalGet &)*result.plan;
+	result.names = get.names;
+	result.types = get.returned_types;
+	get.column_ids.clear();
+	for (idx_t i = 0; i < get.returned_types.size(); i++) {
+		get.column_ids.push_back(i);
+	}
+	return result;
+}
+
+} // namespace duckdb
+
+
+
+
 
 
 //===----------------------------------------------------------------------===//
@@ -169748,9 +169847,60 @@ bool Binder::BindTableFunctionParameters(TableFunctionCatalogEntry &table_functi
 	return true;
 }
 
+unique_ptr<LogicalOperator>
+Binder::BindTableFunctionInternal(TableFunction &table_function, const string &function_name, vector<Value> parameters,
+                                  named_parameter_map_t named_parameters, vector<LogicalType> input_table_types,
+                                  vector<string> input_table_names, const vector<string> &column_name_alias,
+                                  unique_ptr<ExternalDependency> external_dependency) {
+	auto bind_index = GenerateTableIndex();
+	// perform the binding
+	unique_ptr<FunctionData> bind_data;
+	vector<LogicalType> return_types;
+	vector<string> return_names;
+	if (table_function.bind) {
+		TableFunctionBindInput bind_input(parameters, named_parameters, input_table_types, input_table_names,
+		                                  table_function.function_info.get());
+		bind_data = table_function.bind(context, bind_input, return_types, return_names);
+		if (table_function.name == "pandas_scan" || table_function.name == "arrow_scan") {
+			auto arrow_bind = (PyTableFunctionData *)bind_data.get();
+			arrow_bind->external_dependency = move(external_dependency);
+		}
+	}
+	if (return_types.size() != return_names.size()) {
+		throw InternalException(
+		    "Failed to bind \"%s\": Table function return_types and return_names must be of the same size",
+		    table_function.name);
+	}
+	if (return_types.empty()) {
+		throw InternalException("Failed to bind \"%s\": Table function must return at least one column",
+		                        table_function.name);
+	}
+	// overwrite the names with any supplied aliases
+	for (idx_t i = 0; i < column_name_alias.size() && i < return_names.size(); i++) {
+		return_names[i] = column_name_alias[i];
+	}
+	for (idx_t i = 0; i < return_names.size(); i++) {
+		if (return_names[i].empty()) {
+			return_names[i] = "C" + to_string(i);
+		}
+	}
+	auto get = make_unique<LogicalGet>(bind_index, table_function, move(bind_data), return_types, return_names);
+	// now add the table function to the bind context so its columns can be bound
+	bind_context.AddTableFunction(bind_index, function_name, return_names, return_types, *get);
+	return move(get);
+}
+
+unique_ptr<LogicalOperator> Binder::BindTableFunction(TableFunction &function, vector<Value> parameters) {
+	named_parameter_map_t named_parameters;
+	vector<LogicalType> input_table_types;
+	vector<string> input_table_names;
+	vector<string> column_name_aliases;
+	return BindTableFunctionInternal(function, function.name, move(parameters), move(named_parameters),
+	                                 move(input_table_types), move(input_table_names), column_name_aliases, nullptr);
+}
+
 unique_ptr<BoundTableRef> Binder::Bind(TableFunctionRef &ref) {
 	QueryErrorContext error_context(root_statement, ref.query_location);
-	auto bind_index = GenerateTableIndex();
 
 	D_ASSERT(ref.function->type == ExpressionType::FUNCTION);
 	auto fexpr = (FunctionExpression *)ref.function.get();
@@ -169824,42 +169974,9 @@ unique_ptr<BoundTableRef> Binder::Bind(TableFunctionRef &ref) {
 		input_table_types = subquery->subquery->types;
 		input_table_names = subquery->subquery->names;
 	}
-
-	// perform the binding
-	unique_ptr<FunctionData> bind_data;
-	vector<LogicalType> return_types;
-	vector<string> return_names;
-	if (table_function.bind) {
-		TableFunctionBindInput bind_input(parameters, named_parameters, input_table_types, input_table_names,
-		                                  table_function.function_info.get());
-		bind_data = table_function.bind(context, bind_input, return_types, return_names);
-		if (table_function.name == "pandas_scan" || table_function.name == "arrow_scan") {
-			auto arrow_bind = (PyTableFunctionData *)bind_data.get();
-			arrow_bind->external_dependency = move(ref.external_dependency);
-		}
-	}
-	if (return_types.size() != return_names.size()) {
-		throw InternalException(
-		    "Failed to bind \"%s\": Table function return_types and return_names must be of the same size",
-		    table_function.name);
-	}
-	if (return_types.empty()) {
-		throw InternalException("Failed to bind \"%s\": Table function must return at least one column",
-		                        table_function.name);
-	}
-	// overwrite the names with any supplied aliases
-	for (idx_t i = 0; i < ref.column_name_alias.size() && i < return_names.size(); i++) {
-		return_names[i] = ref.column_name_alias[i];
-	}
-	for (idx_t i = 0; i < return_names.size(); i++) {
-		if (return_names[i].empty()) {
-			return_names[i] = "C" + to_string(i);
-		}
-	}
-	auto get = make_unique<LogicalGet>(bind_index, table_function, move(bind_data), return_types, return_names);
-	// now add the table function to the bind context so its columns can be bound
-	bind_context.AddTableFunction(bind_index, ref.alias.empty() ? fexpr->function_name : ref.alias, return_names,
-	                              return_types, *get);
+	auto get = BindTableFunctionInternal(table_function, ref.alias.empty() ? fexpr->function_name : ref.alias,
+	                                     move(parameters), move(named_parameters), move(input_table_types),
+	                                     move(input_table_names), ref.column_name_alias, move(ref.external_dependency));
 	if (subquery) {
 		get->children.push_back(Binder::CreatePlan(*subquery));
 	}
@@ -170288,6 +170405,8 @@ BoundStatement Binder::Bind(SQLStatement &statement) {
 		return Bind((SetStatement &)statement);
 	case StatementType::LOAD_STATEMENT:
 		return Bind((LoadStatement &)statement);
+	case StatementType::EXTENSION_STATEMENT:
+		return Bind((ExtensionStatement &)statement);
 	default: // LCOV_EXCL_START
 		throw NotImplementedException("Unimplemented statement type \"%s\" for Bind",
 		                              StatementTypeToString(statement.type));
@@ -174225,6 +174344,7 @@ void Planner::CreatePlan(unique_ptr<SQLStatement> statement) {
 	case StatementType::SHOW_STATEMENT:
 	case StatementType::SET_STATEMENT:
 	case StatementType::LOAD_STATEMENT:
+	case StatementType::EXTENSION_STATEMENT:
 		CreatePlan(*statement);
 		break;
 	case StatementType::EXECUTE_STATEMENT:

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "f0bbcabe3"
-#define DUCKDB_VERSION "v0.4.1-dev182"
+#define DUCKDB_SOURCE_ID "b5a0b2595"
+#define DUCKDB_VERSION "v0.4.1-dev188"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -9935,7 +9935,8 @@ enum class StatementType : uint8_t {
 	CALL_STATEMENT,         // CALL statement type
 	SET_STATEMENT,          // SET statement type
 	LOAD_STATEMENT,         // LOAD statement type
-	RELATION_STATEMENT
+	RELATION_STATEMENT,
+	EXTENSION_STATEMENT
 };
 
 string StatementTypeToString(StatementType type);
@@ -13127,6 +13128,7 @@ class CopyStatement;
 class CreateStatement;
 class DeleteStatement;
 class DropStatement;
+class ExtensionStatement;
 class InsertStatement;
 class SelectStatement;
 class TransactionStatement;
@@ -14036,6 +14038,7 @@ private:
 	BoundStatement Bind(ShowStatement &stmt);
 	BoundStatement Bind(CallStatement &stmt);
 	BoundStatement Bind(ExportStatement &stmt);
+	BoundStatement Bind(ExtensionStatement &stmt);
 	BoundStatement Bind(SetStatement &stmt);
 	BoundStatement Bind(LoadStatement &stmt);
 	BoundStatement BindReturning(vector<unique_ptr<ParsedExpression>> returning_list, TableCatalogEntry *table,
@@ -14069,6 +14072,12 @@ private:
 	                                 unique_ptr<BoundSubqueryRef> &subquery, string &error);
 	bool BindTableInTableOutFunction(vector<unique_ptr<ParsedExpression>> &expressions,
 	                                 unique_ptr<BoundSubqueryRef> &subquery, string &error);
+	unique_ptr<LogicalOperator> BindTableFunction(TableFunction &function, vector<Value> parameters);
+	unique_ptr<LogicalOperator>
+	BindTableFunctionInternal(TableFunction &table_function, const string &function_name, vector<Value> parameters,
+	                          named_parameter_map_t named_parameters, vector<LogicalType> input_table_types,
+	                          vector<string> input_table_names, const vector<string> &column_name_alias,
+	                          unique_ptr<ExternalDependency> external_dependency);
 
 	unique_ptr<LogicalOperator> CreatePlan(BoundBaseTableRef &ref);
 	unique_ptr<LogicalOperator> CreatePlan(BoundCrossProductRef &ref);
@@ -18568,6 +18577,100 @@ enum class SetScope : uint8_t {
 
 } // namespace duckdb
 
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/parser/parser_extension.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+
+
+namespace duckdb {
+
+//! The ParserExtensionInfo holds static information relevant to the parser extension
+//! It is made available in the parse_function, and will be kept alive as long as the database system is kept alive
+struct ParserExtensionInfo {
+	DUCKDB_API virtual ~ParserExtensionInfo() {
+	}
+};
+
+//===--------------------------------------------------------------------===//
+// Parse
+//===--------------------------------------------------------------------===//
+enum class ParserExtensionResultType : uint8_t { PARSE_SUCCESSFUL, DISPLAY_ORIGINAL_ERROR, DISPLAY_EXTENSION_ERROR };
+
+//! The ParserExtensionParseData holds the result of a successful parse step
+//! It will be passed along to the subsequent plan function
+struct ParserExtensionParseData {
+	DUCKDB_API virtual ~ParserExtensionParseData() {
+	}
+
+	virtual unique_ptr<ParserExtensionParseData> Copy() const = 0;
+};
+
+struct ParserExtensionParseResult {
+	ParserExtensionParseResult() : type(ParserExtensionResultType::DISPLAY_ORIGINAL_ERROR) {
+	}
+	ParserExtensionParseResult(string error_p)
+	    : type(ParserExtensionResultType::DISPLAY_EXTENSION_ERROR), error(move(error_p)) {
+	}
+	ParserExtensionParseResult(unique_ptr<ParserExtensionParseData> parse_data_p)
+	    : type(ParserExtensionResultType::PARSE_SUCCESSFUL), parse_data(move(parse_data_p)) {
+	}
+
+	//! Whether or not parsing was successful
+	ParserExtensionResultType type;
+	//! The parse data (if successful)
+	unique_ptr<ParserExtensionParseData> parse_data;
+	//! The error message (if unsuccessful)
+	string error;
+};
+
+typedef ParserExtensionParseResult (*parse_function_t)(ParserExtensionInfo *info, const string &query);
+//===--------------------------------------------------------------------===//
+// Plan
+//===--------------------------------------------------------------------===//
+struct ParserExtensionPlanResult {
+	//! The table function to execute
+	TableFunction function;
+	//! Parameters to the function
+	vector<Value> parameters;
+	//! Whether or not the statement is read_only (i.e. can be executed in a read_only database)
+	bool read_only = false;
+	//! Whether or not the statement requires a valid transaction to be executed
+	bool requires_valid_transaction = true;
+	//! What type of result set the statement returns
+	StatementReturnType return_type = StatementReturnType::NOTHING;
+};
+
+typedef ParserExtensionPlanResult (*plan_function_t)(ParserExtensionInfo *info, ClientContext &context,
+                                                     unique_ptr<ParserExtensionParseData> parse_data);
+
+//===--------------------------------------------------------------------===//
+// ParserExtension
+//===--------------------------------------------------------------------===//
+class ParserExtension {
+public:
+	//! The parse function of the parser extension.
+	//! Takes a query string as input and returns ParserExtensionParseData (on success) or an error
+	parse_function_t parse_function;
+
+	//! The plan function of the parser extension
+	//! Takes as input the result of the parse_function, and outputs various properties of the resulting plan
+	plan_function_t plan_function;
+
+	//! Additional parser info passed to the parse function
+	shared_ptr<ParserExtensionInfo> parser_info;
+};
+
+} // namespace duckdb
+
 
 namespace duckdb {
 class ClientContext;
@@ -18673,6 +18776,8 @@ public:
 	WindowAggregationMode window_mode = WindowAggregationMode::WINDOW;
 	//! Whether or not preserving insertion order should be preserved
 	bool preserve_insertion_order = true;
+	//! Extensions made to the parser
+	vector<ParserExtension> parser_extensions;
 
 	//! Extra parameters that can be SET for loaded extensions
 	case_insensitive_map_t<ExtensionOption> extension_parameters;
@@ -21900,10 +22005,12 @@ struct PGList;
 } // namespace duckdb_libpgquery
 
 namespace duckdb {
+class ParserExtension;
 
 struct ParserOptions {
 	bool preserve_identifier_case = true;
 	idx_t max_expression_depth = 1000;
+	vector<ParserExtension> *extensions = nullptr;
 };
 
 //! The parser is responsible for parsing the query and converting it into a set