duckdb 0.3.2-dev50.0 → 0.3.2-dev70.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.3.2-dev50.0",
+  "version": "0.3.2-dev70.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -9481,6 +9481,7 @@ static DefaultMacro internal_macros[] = {
 	{DEFAULT_SCHEMA, "array_append", {"arr", "el", nullptr}, "list_append(arr, el)"},
 	{DEFAULT_SCHEMA, "list_prepend", {"e", "l", nullptr}, "list_concat(list_value(e), l)"},
 	{DEFAULT_SCHEMA, "array_prepend", {"el", "arr", nullptr}, "list_prepend(el, arr)"},
+	{DEFAULT_SCHEMA, "generate_subscripts", {"arr", "dim", nullptr}, "unnest(generate_series(1, array_length(arr, dim)))"},
 	{nullptr, nullptr, {nullptr}, nullptr}};
 
 static unique_ptr<CreateFunctionInfo> GetDefaultFunction(const string &schema, const string &name) {
@@ -11177,6 +11178,8 @@ string PhysicalOperatorToString(PhysicalOperatorType type) {
 		return "CREATE_INDEX";
 	case PhysicalOperatorType::EXPLAIN:
 		return "EXPLAIN";
+	case PhysicalOperatorType::EXPLAIN_ANALYZE:
+		return "EXPLAIN_ANALYZE";
 	case PhysicalOperatorType::EXECUTE:
 		return "EXECUTE";
 	case PhysicalOperatorType::VACUUM:
@@ -31970,6 +31973,7 @@ struct ExpressionInfo {
 	vector<unique_ptr<ExpressionInfo>> children;
 	// Extract ExpressionInformation from a given expression state
 	void ExtractExpressionsRecursive(unique_ptr<ExpressionState> &state);
+
 	//! Whether or not expression has function
 	bool hasfunction;
 	//! The function Name
@@ -31985,6 +31989,7 @@ struct ExpressionInfo {
 //! The ExpressionRootInfo keeps information related to the root of an expression tree
 struct ExpressionRootInfo {
 	ExpressionRootInfo(ExpressionExecutorState &executor, string name);
+
 	//! Count the number of time the executor called
 	uint64_t total_count = 0;
 	//! Count the number of time the executor called since last sampling
@@ -32008,6 +32013,7 @@ struct ExpressionRootInfo {
 struct ExpressionExecutorInfo {
 	explicit ExpressionExecutorInfo() {};
 	explicit ExpressionExecutorInfo(ExpressionExecutor &executor, const string &name, int id);
+
 	//! A vector which contain the pointer to all ExpressionRootInfo
 	vector<unique_ptr<ExpressionRootInfo>> roots;
 	//! Id, it will be used as index for executors_info vector
@@ -32015,11 +32021,12 @@ struct ExpressionExecutorInfo {
 };
 
 struct OperatorInformation {
+	explicit OperatorInformation(double time_ = 0, idx_t elements_ = 0) : time(time_), elements(elements_) {
+	}
+
 	double time = 0;
 	idx_t elements = 0;
 	string name;
-	explicit OperatorInformation(double time_ = 0, idx_t elements_ = 0) : time(time_), elements(elements_) {
-	}
 	//! A vector of Expression Executor Info
 	vector<unique_ptr<ExpressionExecutorInfo>> executors_info;
 };
@@ -32057,9 +32064,16 @@ class QueryProfiler {
 public:
 	DUCKDB_API QueryProfiler()
 	    : automatic_print_format(ProfilerPrintFormat::NONE), enabled(false), detailed_enabled(false), running(false),
-	      query_requires_profiling(false) {
+	      query_requires_profiling(false), is_explain_analyze(false), stored_enabled(false),
+	      stored_automatic_print_format(ProfilerPrintFormat::NONE) {
 	}
 
+	//! The format to automatically print query profiling information in (default: disabled)
+	ProfilerPrintFormat automatic_print_format;
+	//! The file to save query profiling information to, instead of printing it to the console (empty = print to
+	//! console)
+	string save_location;
+
 public:
 	struct TreeNode {
 		PhysicalOperatorType type;
@@ -32103,9 +32117,11 @@ public:
 		return detailed_enabled;
 	}
 
-	DUCKDB_API void StartQuery(string query);
+	DUCKDB_API void StartQuery(string query, bool is_explain_analyze = false);
 	DUCKDB_API void EndQuery();
 
+	DUCKDB_API void StartExplainAnalyze();
+
 	//! Adds the timings gathered by an OperatorProfiler to this query profiler
 	DUCKDB_API void Flush(OperatorProfiler &profiler);
 
@@ -32121,12 +32137,6 @@ public:
 	DUCKDB_API string ToJSON() const;
 	DUCKDB_API void WriteToFile(const char *path, string &info) const;
 
-	//! The format to automatically print query profiling information in (default: disabled)
-	ProfilerPrintFormat automatic_print_format;
-	//! The file to save query profiling information to, instead of printing it to the console (empty = print to
-	//! console)
-	string save_location;
-
 	idx_t OperatorSize() {
 		return tree_map.size();
 	}
@@ -32152,6 +32162,11 @@ private:
 	//! A map of a Physical Operator pointer to a tree node
 	TreeMap tree_map;
 
+	bool is_explain_analyze;
+	bool stored_enabled;
+	ProfilerPrintFormat stored_automatic_print_format;
+	string stored_save_location;
+
 public:
 	const TreeMap &GetTreeMap() const {
 		return tree_map;
@@ -54034,7 +54049,6 @@ public:
 	}
 
 	bool ParallelSink() const override {
-		// we can only parallelize if all aggregates are combinable
 		return true;
 	}
 };
@@ -56018,6 +56032,112 @@ PhysicalExecute::PhysicalExecute(PhysicalOperator *plan)
     : PhysicalOperator(PhysicalOperatorType::EXECUTE, plan->types, -1), plan(plan) {
 }
 
+} // namespace duckdb
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/helper/physical_explain_analyze.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+
+namespace duckdb {
+
+class PhysicalExplainAnalyze : public PhysicalOperator {
+public:
+	PhysicalExplainAnalyze(vector<LogicalType> types)
+	    : PhysicalOperator(PhysicalOperatorType::EXPLAIN_ANALYZE, move(types), 1) {
+	}
+
+public:
+	// Source interface
+	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
+	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
+	             LocalSourceState &lstate) const override;
+
+public:
+	// Sink Interface
+	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,
+	                    DataChunk &input) const override;
+	SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
+	                          GlobalSinkState &gstate) const override;
+
+	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
+
+	bool IsSink() const override {
+		return true;
+	}
+
+	bool ParallelSink() const override {
+		return true;
+	}
+};
+
+} // namespace duckdb
+
+
+
+
+namespace duckdb {
+
+//===--------------------------------------------------------------------===//
+// Sink
+//===--------------------------------------------------------------------===//
+class ExplainAnalyzeStateGlobalState : public GlobalSinkState {
+public:
+	string analyzed_plan;
+};
+
+SinkResultType PhysicalExplainAnalyze::Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,
+                                            DataChunk &input) const {
+	return SinkResultType::NEED_MORE_INPUT;
+}
+
+SinkFinalizeType PhysicalExplainAnalyze::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
+                                                  GlobalSinkState &gstate_p) const {
+	auto &gstate = (ExplainAnalyzeStateGlobalState &)gstate_p;
+	gstate.analyzed_plan = context.profiler->ToString();
+	return SinkFinalizeType::READY;
+}
+
+unique_ptr<GlobalSinkState> PhysicalExplainAnalyze::GetGlobalSinkState(ClientContext &context) const {
+	return make_unique<ExplainAnalyzeStateGlobalState>();
+}
+
+//===--------------------------------------------------------------------===//
+// Source
+//===--------------------------------------------------------------------===//
+class ExplainAnalyzeState : public GlobalSourceState {
+public:
+	ExplainAnalyzeState() : finished(false) {
+	}
+
+	bool finished;
+};
+
+unique_ptr<GlobalSourceState> PhysicalExplainAnalyze::GetGlobalSourceState(ClientContext &context) const {
+	return make_unique<ExplainAnalyzeState>();
+}
+
+void PhysicalExplainAnalyze::GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &source_state,
+                                     LocalSourceState &lstate) const {
+	auto &state = (ExplainAnalyzeState &)source_state;
+	auto &gstate = (ExplainAnalyzeStateGlobalState &)*sink_state;
+	if (state.finished) {
+		return;
+	}
+	chunk.SetValue(0, 0, Value("analyzed_plan"));
+	chunk.SetValue(1, 0, Value(gstate.analyzed_plan));
+	chunk.SetCardinality(1);
+
+	state.finished = true;
+}
+
 } // namespace duckdb
 //===----------------------------------------------------------------------===//
 //                         DuckDB
@@ -67794,15 +67914,47 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalExecute &o
 
 
 
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/parser/statement/explain_statement.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+
+namespace duckdb {
+
+enum class ExplainType : uint8_t { EXPLAIN_STANDARD, EXPLAIN_ANALYZE };
+
+class ExplainStatement : public SQLStatement {
+public:
+	ExplainStatement(unique_ptr<SQLStatement> stmt, ExplainType explain_type = ExplainType::EXPLAIN_STANDARD);
+
+	unique_ptr<SQLStatement> stmt;
+	ExplainType explain_type;
+
+public:
+	unique_ptr<SQLStatement> Copy() const override;
+};
+
+} // namespace duckdb
+
 
 namespace duckdb {
 
 class LogicalExplain : public LogicalOperator {
 public:
-	explicit LogicalExplain(unique_ptr<LogicalOperator> plan) : LogicalOperator(LogicalOperatorType::LOGICAL_EXPLAIN) {
+	LogicalExplain(unique_ptr<LogicalOperator> plan, ExplainType explain_type)
+	    : LogicalOperator(LogicalOperatorType::LOGICAL_EXPLAIN), explain_type(explain_type) {
 		children.push_back(move(plan));
 	}
 
+	ExplainType explain_type;
 	string physical_plan;
 	string logical_plan_unopt;
 	string logical_plan_opt;
@@ -67821,15 +67973,20 @@ protected:
 
 
 
+
 namespace duckdb {
 
 unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalExplain &op) {
 	D_ASSERT(op.children.size() == 1);
 	auto logical_plan_opt = op.children[0]->ToString();
 	auto plan = CreatePlan(*op.children[0]);
+	if (op.explain_type == ExplainType::EXPLAIN_ANALYZE) {
+		auto result = make_unique<PhysicalExplainAnalyze>(op.types);
+		result->children.push_back(move(plan));
+		return move(result);
+	}
 
 	op.physical_plan = plan->ToString();
-
 	// the output of the explain
 	vector<string> keys, values;
 	switch (context.explain_output_type) {
@@ -79278,6 +79435,14 @@ static void PragmaDisableForceExternal(ClientContext &context, const FunctionPar
 	context.force_external = false;
 }
 
+static void PragmaEnableParquetBinaryAsString(ClientContext &context, const FunctionParameters &parameters) {
+	context.parquet_binary_as_strings = true;
+}
+
+static void PragmaDisableParquetBinaryAsString(ClientContext &context, const FunctionParameters &parameters) {
+	context.parquet_binary_as_strings = false;
+}
+
 static void PragmaEnableObjectCache(ClientContext &context, const FunctionParameters &parameters) {
 	DBConfig::GetConfig(context).object_cache_enable = true;
 }
@@ -79453,6 +79618,11 @@ void PragmaFunctions::RegisterFunction(BuiltinFunctions &set) {
 	set.AddFunction(PragmaFunction::PragmaStatement("enable_print_progress_bar", PragmaEnablePrintProgressBar));
 	set.AddFunction(PragmaFunction::PragmaStatement("disable_print_progress_bar", PragmaDisablePrintProgressBar));
 
+	set.AddFunction(
+	    PragmaFunction::PragmaStatement("enable_parquet_binary_as_string", PragmaEnableParquetBinaryAsString));
+	set.AddFunction(
+	    PragmaFunction::PragmaStatement("disable_parquet_binary_as_string", PragmaDisableParquetBinaryAsString));
+
 	set.AddFunction(
 	    PragmaFunction::PragmaAssignment("set_progress_bar_time", PragmaSetProgressBarWaitTime, LogicalType::INTEGER));
 
@@ -100949,32 +101119,6 @@ public:
 };
 } // namespace duckdb
 
-//===----------------------------------------------------------------------===//
-//                         DuckDB
-//
-// duckdb/parser/statement/explain_statement.hpp
-//
-//
-//===----------------------------------------------------------------------===//
-
-
-
-
-
-
-namespace duckdb {
-
-class ExplainStatement : public SQLStatement {
-public:
-	explicit ExplainStatement(unique_ptr<SQLStatement> stmt);
-
-	unique_ptr<SQLStatement> stmt;
-
-public:
-	unique_ptr<SQLStatement> Copy() const override;
-};
-
-} // namespace duckdb
 
 //===----------------------------------------------------------------------===//
 //                         DuckDB
@@ -101529,6 +101673,17 @@ unique_ptr<QueryResult> ClientContext::RunStatementInternal(ClientContextLock &l
 	return ExecutePreparedStatement(lock, query, move(prepared), move(bound_values), allow_stream_result);
 }
 
+static bool IsExplainAnalyze(SQLStatement *statement) {
+	if (!statement) {
+		return false;
+	}
+	if (statement->type != StatementType::EXPLAIN_STATEMENT) {
+		return false;
+	}
+	auto &explain = (ExplainStatement &)*statement;
+	return explain.explain_type == ExplainType::EXPLAIN_ANALYZE;
+}
+
 unique_ptr<QueryResult> ClientContext::RunStatementOrPreparedStatement(ClientContextLock &lock, const string &query,
                                                                        unique_ptr<SQLStatement> statement,
                                                                        shared_ptr<PreparedStatementData> &prepared,
@@ -101560,7 +101715,7 @@ unique_ptr<QueryResult> ClientContext::RunStatementOrPreparedStatement(ClientCon
 		statement = move(copied_statement);
 	}
 	// start the profiler
-	profiler->StartQuery(query);
+	profiler->StartQuery(query, IsExplainAnalyze(statement ? statement.get() : prepared->unbound_statement.get()));
 	try {
 		if (statement) {
 			result = RunStatementInternal(lock, query, move(statement), allow_stream_result);
@@ -103348,7 +103503,10 @@ using std::ofstream;
 
 namespace duckdb {
 
-void QueryProfiler::StartQuery(string query) {
+void QueryProfiler::StartQuery(string query, bool is_explain_analyze) {
+	if (is_explain_analyze) {
+		StartExplainAnalyze();
+	}
 	if (!enabled) {
 		return;
 	}
@@ -103404,6 +103562,17 @@ void QueryProfiler::Finalize(TreeNode &node) {
 	}
 }
 
+void QueryProfiler::StartExplainAnalyze() {
+	this->is_explain_analyze = true;
+	this->stored_enabled = enabled;
+	this->stored_automatic_print_format = automatic_print_format;
+	this->stored_save_location = save_location;
+
+	this->enabled = true;
+	this->save_location = string();
+	this->automatic_print_format = ProfilerPrintFormat::NONE;
+}
+
 void QueryProfiler::EndQuery() {
 	if (!enabled || !running) {
 		return;
@@ -103433,6 +103602,12 @@ void QueryProfiler::EndQuery() {
 			WriteToFile(save_location.c_str(), query_info);
 		}
 	}
+	if (is_explain_analyze) {
+		this->is_explain_analyze = false;
+		this->enabled = stored_enabled;
+		this->automatic_print_format = stored_automatic_print_format;
+		this->save_location = stored_save_location;
+	}
 }
 
 void QueryProfiler::StartPhase(string new_phase) {
@@ -116078,6 +116253,7 @@ void Executor::BuildPipelines(PhysicalOperator *op, Pipeline *current) {
 		case PhysicalOperatorType::COPY_TO_FILE:
 		case PhysicalOperatorType::LIMIT:
 		case PhysicalOperatorType::EXPRESSION_SCAN:
+		case PhysicalOperatorType::EXPLAIN_ANALYZE:
 			D_ASSERT(op->children.size() == 1);
 			// single operator:
 			// the operator becomes the data source of the current pipeline
@@ -130320,12 +130496,12 @@ unique_ptr<SQLStatement> ExecuteStatement::Copy() const {
 
 namespace duckdb {
 
-ExplainStatement::ExplainStatement(unique_ptr<SQLStatement> stmt)
-    : SQLStatement(StatementType::EXPLAIN_STATEMENT), stmt(move(stmt)) {
+ExplainStatement::ExplainStatement(unique_ptr<SQLStatement> stmt, ExplainType explain_type)
+    : SQLStatement(StatementType::EXPLAIN_STATEMENT), stmt(move(stmt)), explain_type(explain_type) {
 }
 
 unique_ptr<SQLStatement> ExplainStatement::Copy() const {
-	auto result = make_unique<ExplainStatement>(stmt->Copy());
+	auto result = make_unique<ExplainStatement>(stmt->Copy(), explain_type);
 	return move(result);
 }
 
@@ -134584,7 +134760,19 @@ namespace duckdb {
 unique_ptr<ExplainStatement> Transformer::TransformExplain(duckdb_libpgquery::PGNode *node) {
 	auto stmt = reinterpret_cast<duckdb_libpgquery::PGExplainStmt *>(node);
 	D_ASSERT(stmt);
-	return make_unique<ExplainStatement>(TransformStatement(stmt->query));
+	auto explain_type = ExplainType::EXPLAIN_STANDARD;
+	if (stmt->options) {
+		for (auto n = stmt->options->head; n; n = n->next) {
+			auto def_elem = ((duckdb_libpgquery::PGDefElem *)n->data.ptr_value)->defname;
+			string elem(def_elem);
+			if (elem == "analyze") {
+				explain_type = ExplainType::EXPLAIN_ANALYZE;
+			} else {
+				throw NotImplementedException("Unimplemented explain type: %s", elem);
+			}
+		}
+	}
+	return make_unique<ExplainStatement>(TransformStatement(stmt->query), explain_type);
 }
 
 } // namespace duckdb
@@ -140177,7 +140365,7 @@ BoundStatement Binder::Bind(ExplainStatement &stmt) {
 	auto plan = Bind(*stmt.stmt);
 	// get the unoptimized logical plan, and create the explain statement
 	auto logical_plan_unopt = plan.plan->ToString();
-	auto explain = make_unique<LogicalExplain>(move(plan.plan));
+	auto explain = make_unique<LogicalExplain>(move(plan.plan), stmt.explain_type);
 	explain->logical_plan_unopt = logical_plan_unopt;
 
 	result.plan = move(explain);

package/src/duckdb.hpp

@@ -10,8 +10,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 #pragma once
 #define DUCKDB_AMALGAMATION 1
-#define DUCKDB_SOURCE_ID "1f398ce3b"
-#define DUCKDB_VERSION "0.3.2-dev50"
+#define DUCKDB_SOURCE_ID "7a1864144"
+#define DUCKDB_VERSION "0.3.2-dev70"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -12697,6 +12697,7 @@ enum class PhysicalOperatorType : uint8_t {
 	// Helpers
 	// -----------------------------
 	EXPLAIN,
+	EXPLAIN_ANALYZE,
 	EMPTY_RESULT,
 	EXECUTE,
 	PREPARE,
@@ -13613,6 +13614,8 @@ public:
 	bool force_index_join = false;
 	//! Force out-of-core computation for operators that support it, used for testing
 	bool force_external = false;
+	//! If binaries should be converted to strings by default on the parquet reader
+	bool parquet_binary_as_strings = false;
 	//! Maximum bits allowed for using a perfect hash table (i.e. the perfect HT can hold up to 2^perfect_ht_threshold
 	//! elements)
 	idx_t perfect_ht_threshold = 12;