duckdb 0.3.5-dev396.0 → 0.3.5-dev411.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.3.5-dev396.0",
+  "version": "0.3.5-dev411.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -5311,6 +5311,7 @@ public:
 	idx_t batch_size;
 	vector<LogicalType> column_types;
 	vector<string> column_names;
+	string timezone_config;
 
 private:
 	static int MyStreamGetSchema(struct ArrowArrayStream *stream, struct ArrowSchema *out);
@@ -5390,7 +5391,7 @@ int ResultArrowArrayStreamWrapper::MyStreamGetSchema(struct ArrowArrayStream *st
 	}
 	auto my_stream = (ResultArrowArrayStreamWrapper *)stream->private_data;
 	if (!my_stream->column_types.empty()) {
-		QueryResult::ToArrowSchema(out, my_stream->column_types, my_stream->column_names);
+		QueryResult::ToArrowSchema(out, my_stream->column_types, my_stream->column_names, my_stream->timezone_config);
 		return 0;
 	}
 
@@ -5410,7 +5411,7 @@ int ResultArrowArrayStreamWrapper::MyStreamGetSchema(struct ArrowArrayStream *st
 		my_stream->column_types = result.types;
 		my_stream->column_names = result.names;
 	}
-	QueryResult::ToArrowSchema(out, my_stream->column_types, my_stream->column_names);
+	QueryResult::ToArrowSchema(out, my_stream->column_types, my_stream->column_names, my_stream->timezone_config);
 	return 0;
 }
 
@@ -102593,6 +102594,21 @@ LogicalType GetArrowLogicalType(ArrowSchema &schema,
 		idx_t fixed_size = std::stoi(parameters);
 		arrow_convert_data[col_idx]->variable_sz_type.emplace_back(ArrowVariableSizeType::FIXED_SIZE, fixed_size);
 		return LogicalType::BLOB;
+	} else if (format[0] == 't' && format[1] == 's') {
+		// Timestamp with Timezone
+		if (format[2] == 'n') {
+			arrow_convert_data[col_idx]->date_time_precision.emplace_back(ArrowDateTimeType::NANOSECONDS);
+		} else if (format[2] == 'u') {
+			arrow_convert_data[col_idx]->date_time_precision.emplace_back(ArrowDateTimeType::MICROSECONDS);
+		} else if (format[2] == 'm') {
+			arrow_convert_data[col_idx]->date_time_precision.emplace_back(ArrowDateTimeType::MILLISECONDS);
+		} else if (format[2] == 's') {
+			arrow_convert_data[col_idx]->date_time_precision.emplace_back(ArrowDateTimeType::SECONDS);
+		} else {
+			throw NotImplementedException(" Timestamptz precision of not accepted");
+		}
+		// TODO right now we just get the UTC value. We probably want to support this properly in the future
+		return LogicalType::TIMESTAMP_TZ;
 	} else {
 		throw NotImplementedException("Unsupported Internal Arrow Type %s", format);
 	}
@@ -102950,7 +102966,25 @@ void TimeConversion(Vector &vector, ArrowArray &array, ArrowScanState &scan_stat
 			continue;
 		}
 		if (!TryMultiplyOperator::Operation((int64_t)src_ptr[row], conversion, tgt_ptr[row].micros)) {
-			throw ConversionException("Could not convert Interval to Microsecond");
+			throw ConversionException("Could not convert Time to Microsecond");
+		}
+	}
+}
+
+void TimestampTZConversion(Vector &vector, ArrowArray &array, ArrowScanState &scan_state, int64_t nested_offset,
+                           idx_t size, int64_t conversion) {
+	auto tgt_ptr = (timestamp_t *)FlatVector::GetData(vector);
+	auto &validity_mask = FlatVector::Validity(vector);
+	auto src_ptr = (int64_t *)array.buffers[1] + scan_state.chunk_offset + array.offset;
+	if (nested_offset != -1) {
+		src_ptr = (int64_t *)array.buffers[1] + nested_offset + array.offset;
+	}
+	for (idx_t row = 0; row < size; row++) {
+		if (!validity_mask.RowIsValid(row)) {
+			continue;
+		}
+		if (!TryMultiplyOperator::Operation(src_ptr[row], conversion, tgt_ptr[row].value)) {
+			throw ConversionException("Could not convert TimestampTZ to Microsecond");
 		}
 	}
 }
@@ -103115,6 +103149,37 @@ void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowScanState &scan
 		}
 		break;
 	}
+	case LogicalTypeId::TIMESTAMP_TZ: {
+		auto precision = arrow_convert_data[col_idx]->date_time_precision[arrow_convert_idx.second++];
+		switch (precision) {
+		case ArrowDateTimeType::SECONDS: {
+			TimestampTZConversion(vector, array, scan_state, nested_offset, size, 1000000);
+			break;
+		}
+		case ArrowDateTimeType::MILLISECONDS: {
+			TimestampTZConversion(vector, array, scan_state, nested_offset, size, 1000);
+			break;
+		}
+		case ArrowDateTimeType::MICROSECONDS: {
+			DirectConversion(vector, array, scan_state, nested_offset);
+			break;
+		}
+		case ArrowDateTimeType::NANOSECONDS: {
+			auto tgt_ptr = (timestamp_t *)FlatVector::GetData(vector);
+			auto src_ptr = (int64_t *)array.buffers[1] + scan_state.chunk_offset + array.offset;
+			if (nested_offset != -1) {
+				src_ptr = (int64_t *)array.buffers[1] + nested_offset + array.offset;
+			}
+			for (idx_t row = 0; row < size; row++) {
+				tgt_ptr[row].value = src_ptr[row] / 1000;
+			}
+			break;
+		}
+		default:
+			throw std::runtime_error("Unsupported precision for TimestampTZ Type ");
+		}
+		break;
+	}
 	case LogicalTypeId::INTERVAL: {
 		auto precision = arrow_convert_data[col_idx]->date_time_precision[arrow_convert_idx.second++];
 		switch (precision) {
@@ -109206,6 +109271,7 @@ struct PreparedStatementWrapper {
 struct ArrowResultWrapper {
 	unique_ptr<MaterializedQueryResult> result;
 	unique_ptr<DataChunk> current_chunk;
+	string timezone_config;
 };
 
 struct AppenderWrapper {
@@ -109462,7 +109528,8 @@ duckdb_state duckdb_query_arrow_schema(duckdb_arrow result, duckdb_arrow_schema
 		return DuckDBSuccess;
 	}
 	auto wrapper = (ArrowResultWrapper *)result;
-	QueryResult::ToArrowSchema((ArrowSchema *)*out_schema, wrapper->result->types, wrapper->result->names);
+	QueryResult::ToArrowSchema((ArrowSchema *)*out_schema, wrapper->result->types, wrapper->result->names,
+	                           wrapper->timezone_config);
 	return DuckDBSuccess;
 }
 
@@ -109524,6 +109591,14 @@ duckdb_state duckdb_execute_prepared_arrow(duckdb_prepared_statement prepared_st
 		return DuckDBError;
 	}
 	auto arrow_wrapper = new ArrowResultWrapper();
+	if (wrapper->statement->context->config.set_variables.find("TimeZone") ==
+	    wrapper->statement->context->config.set_variables.end()) {
+		arrow_wrapper->timezone_config = "UTC";
+	} else {
+		arrow_wrapper->timezone_config =
+		    wrapper->statement->context->config.set_variables["TimeZone"].GetValue<std::string>();
+	}
+
 	auto result = wrapper->statement->Execute(wrapper->values, false);
 	D_ASSERT(result->type == QueryResultType::MATERIALIZED_RESULT);
 	arrow_wrapper->result = duckdb::unique_ptr_cast<QueryResult, MaterializedQueryResult>(move(result));
@@ -112463,7 +112538,7 @@ unique_ptr<QueryResult> ClientContext::FetchResultInternal(ClientContextLock &lo
 		active_query->progress_bar.reset();
 		query_progress = -1;
 
-		// successfully compiled SELECT clause and it is the last statement
+		// successfully compiled SELECT clause, and it is the last statement
 		// return a StreamQueryResult so the client can call Fetch() on it and stream the result
 		auto stream_result = make_unique<StreamQueryResult>(pending.statement_type, pending.properties,
 		                                                    shared_from_this(), pending.types, pending.names);
@@ -112471,8 +112546,8 @@ unique_ptr<QueryResult> ClientContext::FetchResultInternal(ClientContextLock &lo
 		return move(stream_result);
 	}
 	// create a materialized result by continuously fetching
-	auto result =
-	    make_unique<MaterializedQueryResult>(pending.statement_type, pending.properties, pending.types, pending.names);
+	auto result = make_unique<MaterializedQueryResult>(pending.statement_type, pending.properties, pending.types,
+	                                                   pending.names, shared_from_this());
 	result->properties = pending.properties;
 	while (true) {
 		auto chunk = FetchInternal(lock, GetExecutor(), *result);
@@ -112899,7 +112974,7 @@ unique_ptr<QueryResult> ClientContext::Query(const string &query, bool allow_str
 		vector<LogicalType> types;
 		vector<string> names;
 		return make_unique<MaterializedQueryResult>(StatementType::INVALID_STATEMENT, properties, move(types),
-		                                            move(names));
+		                                            move(names), shared_from_this());
 	}
 
 	unique_ptr<QueryResult> result;
@@ -114649,6 +114724,14 @@ void DuckDB::SetExtensionLoaded(const std::string &name) {
 	instance->loaded_extensions.insert(name);
 }
 
+string ClientConfig::ExtractTimezoneFromConfig(ClientConfig &config) {
+	if (config.set_variables.find("TimeZone") == config.set_variables.end()) {
+		return "UTC";
+	} else {
+		return config.set_variables["TimeZone"].GetValue<std::string>();
+	}
+}
+
 } // namespace duckdb
 
 
@@ -123266,8 +123349,10 @@ Extension::~Extension() {
 namespace duckdb {
 
 MaterializedQueryResult::MaterializedQueryResult(StatementType statement_type, StatementProperties properties,
-                                                 vector<LogicalType> types, vector<string> names)
-    : QueryResult(QueryResultType::MATERIALIZED_RESULT, statement_type, properties, move(types), move(names)) {
+                                                 vector<LogicalType> types, vector<string> names,
+                                                 const shared_ptr<ClientContext> &context)
+    : QueryResult(QueryResultType::MATERIALIZED_RESULT, statement_type, properties, move(types), move(names)),
+      context(context) {
 }
 
 MaterializedQueryResult::MaterializedQueryResult(string error)
@@ -124193,6 +124278,7 @@ ExpressionRootInfo::ExpressionRootInfo(ExpressionExecutorState &state, string na
 
 
 
+
 namespace duckdb {
 
 BaseQueryResult::BaseQueryResult(QueryResultType type, StatementType statement_type, StatementProperties properties,
@@ -124338,9 +124424,11 @@ void InitializeChild(ArrowSchema &child, const string &name = "") {
 	child.metadata = nullptr;
 	child.dictionary = nullptr;
 }
-void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, const LogicalType &type);
+void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, const LogicalType &type,
+                    string &config_timezone);
 
-void SetArrowMapFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, const LogicalType &type) {
+void SetArrowMapFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, const LogicalType &type,
+                       string &config_timezone) {
 	child.format = "+m";
 	//! Map has one child which is a struct
 	child.n_children = 1;
@@ -124355,10 +124443,11 @@ void SetArrowMapFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child,
 	struct_child_types.push_back(std::make_pair("key", ListType::GetChildType(StructType::GetChildType(type, 0))));
 	struct_child_types.push_back(std::make_pair("value", ListType::GetChildType(StructType::GetChildType(type, 1))));
 	auto struct_type = LogicalType::STRUCT(move(struct_child_types));
-	SetArrowFormat(root_holder, *child.children[0], struct_type);
+	SetArrowFormat(root_holder, *child.children[0], struct_type, config_timezone);
 }
 
-void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, const LogicalType &type) {
+void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, const LogicalType &type,
+                    string &config_timezone) {
 	switch (type.id()) {
 	case LogicalTypeId::BOOLEAN:
 		child.format = "b";
@@ -124409,9 +124498,19 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 		child.format = "ttu";
 		break;
 	case LogicalTypeId::TIMESTAMP:
-	case LogicalTypeId::TIMESTAMP_TZ:
 		child.format = "tsu:";
 		break;
+	case LogicalTypeId::TIMESTAMP_TZ: {
+		string format = "tsu:" + config_timezone;
+		unique_ptr<char[]> format_ptr = unique_ptr<char[]>(new char[format.size() + 1]);
+		for (size_t i = 0; i < format.size(); i++) {
+			format_ptr[i] = format[i];
+		}
+		format_ptr[format.size()] = '\0';
+		root_holder.owned_type_names.push_back(move(format_ptr));
+		child.format = root_holder.owned_type_names.back().get();
+		break;
+	}
 	case LogicalTypeId::TIMESTAMP_SEC:
 		child.format = "tss:";
 		break;
@@ -124455,7 +124554,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 		InitializeChild(root_holder.nested_children.back()[0]);
 		child.children = &root_holder.nested_children_ptr.back()[0];
 		child.children[0]->name = "l";
-		SetArrowFormat(root_holder, **child.children, ListType::GetChildType(type));
+		SetArrowFormat(root_holder, **child.children, ListType::GetChildType(type), config_timezone);
 		break;
 	}
 	case LogicalTypeId::STRUCT: {
@@ -124483,12 +124582,12 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 			root_holder.owned_type_names.push_back(move(name_ptr));
 
 			child.children[type_idx]->name = root_holder.owned_type_names.back().get();
-			SetArrowFormat(root_holder, *child.children[type_idx], child_types[type_idx].second);
+			SetArrowFormat(root_holder, *child.children[type_idx], child_types[type_idx].second, config_timezone);
 		}
 		break;
 	}
 	case LogicalTypeId::MAP: {
-		SetArrowMapFormat(root_holder, child, type);
+		SetArrowMapFormat(root_holder, child, type, config_timezone);
 		break;
 	}
 	case LogicalTypeId::ENUM: {
@@ -124519,7 +124618,8 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 	}
 }
 
-void QueryResult::ToArrowSchema(ArrowSchema *out_schema, vector<LogicalType> &types, vector<string> &names) {
+void QueryResult::ToArrowSchema(ArrowSchema *out_schema, vector<LogicalType> &types, vector<string> &names,
+                                string &config_timezone) {
 	D_ASSERT(out_schema);
 	D_ASSERT(types.size() == names.size());
 	idx_t column_count = types.size();
@@ -124547,7 +124647,7 @@ void QueryResult::ToArrowSchema(ArrowSchema *out_schema, vector<LogicalType> &ty
 
 		auto &child = root_holder->children[col_idx];
 		InitializeChild(child, names[col_idx]);
-		SetArrowFormat(*root_holder, child, types[col_idx]);
+		SetArrowFormat(*root_holder, child, types[col_idx], config_timezone);
 	}
 
 	// Release ownership to caller
@@ -124555,6 +124655,23 @@ void QueryResult::ToArrowSchema(ArrowSchema *out_schema, vector<LogicalType> &ty
 	out_schema->release = ReleaseDuckDBArrowSchema;
 }
 
+string QueryResult::GetConfigTimezone(QueryResult &query_result) {
+	switch (query_result.type) {
+	case QueryResultType::MATERIALIZED_RESULT: {
+		auto actual_context = ((MaterializedQueryResult &)query_result).context.lock();
+		if (!actual_context) {
+			throw std::runtime_error("This connection is closed");
+		}
+		return ClientConfig::ExtractTimezoneFromConfig(actual_context->config);
+	}
+	case QueryResultType::STREAM_RESULT: {
+		return ClientConfig::ExtractTimezoneFromConfig(((StreamQueryResult &)query_result).context->config);
+	}
+	default:
+		throw std::runtime_error("Can't extract timezone configuration from query type ");
+	}
+}
+
 } // namespace duckdb
 //===----------------------------------------------------------------------===//
 //                         DuckDB
@@ -127554,7 +127671,7 @@ unique_ptr<MaterializedQueryResult> StreamQueryResult::Materialize() {
 	if (!success) {
 		return make_unique<MaterializedQueryResult>(error);
 	}
-	auto result = make_unique<MaterializedQueryResult>(statement_type, properties, types, names);
+	auto result = make_unique<MaterializedQueryResult>(statement_type, properties, types, names, context);
 	while (true) {
 		auto chunk = Fetch();
 		if (!chunk || chunk->size() == 0) {