duckdb 0.8.2-dev33.0 → 0.8.2-dev3334.0

package/src/duckdb/third_party/libpg_query/src_backend_parser_scansup.cpp

@@ -30,6 +30,10 @@
 #include "parser/scansup.hpp"
 #include "mb/pg_wchar.hpp"
 
+#ifdef __MVS__
+#include <zos-tls.h>
+#endif
+
 namespace duckdb_libpgquery {
 
 /* ----------------
@@ -60,7 +64,12 @@ char *downcase_truncate_identifier(const char *ident, int len, bool warn) {
 	return downcase_identifier(ident, len, warn, true);
 }
 
+#ifdef __MVS__
+static __tlssim<bool> pg_preserve_identifier_case_impl(false);
+#define pg_preserve_identifier_case (*pg_preserve_identifier_case_impl.access())
+#else
 static __thread bool pg_preserve_identifier_case = false;
+#endif
 
 void set_preserve_identifier_case(bool preserve) {
 	pg_preserve_identifier_case = preserve;

package/src/duckdb/third_party/mbedtls/include/mbedtls_wrapper.hpp

@@ -19,5 +19,15 @@ public:
 	static void Hmac256(const char* key, size_t key_len, const char* message, size_t message_len, char* out);
 
 	static constexpr size_t SHA256_HASH_BYTES = 32;
+
+	class SHA256State {
+	public:
+		SHA256State();
+		~SHA256State();
+		void AddString(const std::string & str);
+		std::string Finalize();
+	private:
+		void *sha_context;
+	};
 };
 }

package/src/duckdb/third_party/mbedtls/mbedtls_wrapper.cpp

@@ -81,4 +81,34 @@ void MbedTlsWrapper::Hmac256(const char* key, size_t key_len, const char* messag
 		throw runtime_error("HMAC256 Error");
 	}
 	mbedtls_md_free(&hmac_ctx);
-}
+}
+
+MbedTlsWrapper::SHA256State::SHA256State() : sha_context(new mbedtls_sha256_context()) {
+	mbedtls_sha256_init((mbedtls_sha256_context*)sha_context);
+
+	if (mbedtls_sha256_starts((mbedtls_sha256_context*)sha_context, false)) {
+		throw std::runtime_error("SHA256 Error");
+	}
+}
+
+MbedTlsWrapper::SHA256State::~SHA256State() {
+	mbedtls_sha256_free((mbedtls_sha256_context*)sha_context);
+	delete (mbedtls_sha256_context*)sha_context;
+}
+
+void MbedTlsWrapper::SHA256State::AddString(const std::string & str) {
+	if (mbedtls_sha256_update((mbedtls_sha256_context*)sha_context, (unsigned char*)str.data(), str.size())) {
+		throw std::runtime_error("SHA256 Error");
+	}
+}
+
+std::string MbedTlsWrapper::SHA256State::Finalize() {
+	string hash;
+	hash.resize(MbedTlsWrapper::SHA256_HASH_BYTES);
+
+	if (mbedtls_sha256_finish((mbedtls_sha256_context*)sha_context, (unsigned char*)hash.data())) {
+		throw std::runtime_error("SHA256 Error");
+	}
+
+	return hash;
+}

package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp

@@ -350,10 +350,6 @@
 
 #include "extension/icu/third_party/icu/i18n/double-conversion-fast-dtoa.cpp"
 
-#include "extension/icu/third_party/icu/i18n/double-conversion-double-to-string.cpp"
-
-#include "extension/icu/third_party/icu/i18n/double-conversion-strtod.cpp"
-
 #include "extension/icu/third_party/icu/i18n/double-conversion-cached-powers.cpp"
 
 #include "extension/icu/third_party/icu/i18n/double-conversion-string-to-double.cpp"
@@ -362,3 +358,7 @@
 
 #include "extension/icu/third_party/icu/i18n/double-conversion-bignum.cpp"
 
+#include "extension/icu/third_party/icu/i18n/double-conversion-strtod.cpp"
+
+#include "extension/icu/third_party/icu/i18n/double-conversion-double-to-string.cpp"
+

package/src/duckdb/ub_src_common.cpp

@@ -18,6 +18,8 @@
 
 #include "src/common/exception_format_value.cpp"
 
+#include "src/common/extra_type_info.cpp"
+
 #include "src/common/field_writer.cpp"
 
 #include "src/common/file_buffer.cpp"
@@ -32,6 +34,8 @@
 
 #include "src/common/hive_partitioning.cpp"
 
+#include "src/common/http_state.cpp"
+
 #include "src/common/pipe_file_system.cpp"
 
 #include "src/common/local_file_system.cpp"

package/src/duckdb/ub_src_common_adbc_nanoarrow.cpp

@@ -0,0 +1,8 @@
+#include "src/common/adbc/nanoarrow/metadata.cpp"
+
+#include "src/common/adbc/nanoarrow/schema.cpp"
+
+#include "src/common/adbc/nanoarrow/allocator.cpp"
+
+#include "src/common/adbc/nanoarrow/single_batch_array_stream.cpp"
+

package/src/duckdb/ub_src_common_arrow_appender.cpp

@@ -0,0 +1,10 @@
+#include "src/common/arrow/appender/bool_data.cpp"
+
+#include "src/common/arrow/appender/list_data.cpp"
+
+#include "src/common/arrow/appender/map_data.cpp"
+
+#include "src/common/arrow/appender/struct_data.cpp"
+
+#include "src/common/arrow/appender/union_data.cpp"
+

package/src/duckdb/ub_src_common_serializer.cpp

@@ -10,3 +10,5 @@
 
 #include "src/common/serializer/buffered_serializer.cpp"
 
+#include "src/common/serializer/format_serializer.cpp"
+

package/src/duckdb/ub_src_core_functions_scalar_debug.cpp

@@ -0,0 +1,2 @@
+#include "src/core_functions/scalar/debug/vector_type.cpp"
+

package/src/duckdb/ub_src_core_functions_scalar_string.cpp

@@ -38,6 +38,8 @@
 
 #include "src/core_functions/scalar/string/string_split.cpp"
 
+#include "src/core_functions/scalar/string/to_base.cpp"
+
 #include "src/core_functions/scalar/string/translate.cpp"
 
 #include "src/core_functions/scalar/string/trim.cpp"

package/src/duckdb/ub_src_execution.cpp

@@ -24,5 +24,7 @@
 
 #include "src/execution/reservoir_sample.cpp"
 
+#include "src/execution/window_executor.cpp"
+
 #include "src/execution/window_segment_tree.cpp"
 

package/src/duckdb/ub_src_execution_index_art.cpp

@@ -8,8 +8,6 @@
 
 #include "src/execution/index/art/leaf.cpp"
 
-#include "src/execution/index/art/leaf_segment.cpp"
-
 #include "src/execution/index/art/node4.cpp"
 
 #include "src/execution/index/art/node16.cpp"
@@ -18,11 +16,7 @@
 
 #include "src/execution/index/art/node256.cpp"
 
-#include "src/execution/index/art/swizzleable_pointer.cpp"
-
 #include "src/execution/index/art/prefix.cpp"
 
-#include "src/execution/index/art/prefix_segment.cpp"
-
 #include "src/execution/index/art/art.cpp"
 

package/src/duckdb/ub_src_execution_operator_persistent.cpp

@@ -10,6 +10,8 @@
 
 #include "src/execution/operator/persistent/csv_reader_options.cpp"
 
+#include "src/execution/operator/persistent/csv_rejects_table.cpp"
+
 #include "src/execution/operator/persistent/physical_fixed_batch_copy.cpp"
 
 #include "src/execution/operator/persistent/physical_batch_copy_to_file.cpp"

package/src/duckdb/ub_src_execution_operator_schema.cpp

@@ -2,7 +2,7 @@
 
 #include "src/execution/operator/schema/physical_attach.cpp"
 
-#include "src/execution/operator/schema/physical_create_index.cpp"
+#include "src/execution/operator/schema/physical_create_art_index.cpp"
 
 #include "src/execution/operator/schema/physical_create_schema.cpp"
 

package/src/duckdb/ub_src_execution_operator_set.cpp

@@ -2,3 +2,5 @@
 
 #include "src/execution/operator/set/physical_recursive_cte.cpp"
 
+#include "src/execution/operator/set/physical_cte.cpp"
+

package/src/duckdb/ub_src_execution_physical_plan.cpp

@@ -82,3 +82,5 @@
 
 #include "src/execution/physical_plan/plan_recursive_cte.cpp"
 
+#include "src/execution/physical_plan/plan_cte.cpp"
+

package/src/duckdb/ub_src_function_scalar.cpp

@@ -1,3 +1,5 @@
+#include "src/function/scalar/compressed_materialization_functions.cpp"
+
 #include "src/function/scalar/generic_functions.cpp"
 
 #include "src/function/scalar/string_functions.cpp"

package/src/duckdb/ub_src_function_scalar_compressed_materialization.cpp

@@ -0,0 +1,4 @@
+#include "src/function/scalar/compressed_materialization/compress_integral.cpp"
+
+#include "src/function/scalar/compressed_materialization/compress_string.cpp"
+

package/src/duckdb/ub_src_function_scalar_list.cpp

@@ -4,3 +4,5 @@
 
 #include "src/function/scalar/list/list_extract.cpp"
 
+#include "src/function/scalar/list/list_resize.cpp"
+

package/src/duckdb/ub_src_function_table_arrow.cpp

@@ -0,0 +1,2 @@
+#include "src/function/table/arrow/arrow_duck_schema.cpp"
+

package/src/duckdb/ub_src_main.cpp

@@ -10,6 +10,8 @@
 
 #include "src/main/client_verify.cpp"
 
+#include "src/main/chunk_scan_state.cpp"
+
 #include "src/main/config.cpp"
 
 #include "src/main/connection.cpp"

package/src/duckdb/ub_src_main_chunk_scan_state.cpp

@@ -0,0 +1,2 @@
+#include "src/main/chunk_scan_state/query_result.cpp"
+

package/src/duckdb/ub_src_optimizer.cpp

@@ -1,5 +1,9 @@
+#include "src/optimizer/column_binding_replacer.cpp"
+
 #include "src/optimizer/common_aggregate_optimizer.cpp"
 
+#include "src/optimizer/compressed_materialization.cpp"
+
 #include "src/optimizer/cse_optimizer.cpp"
 
 #include "src/optimizer/deliminator.cpp"
@@ -24,6 +28,8 @@
 
 #include "src/optimizer/regex_range_filter.cpp"
 
+#include "src/optimizer/remove_duplicate_groups.cpp"
+
 #include "src/optimizer/remove_unused_columns.cpp"
 
 #include "src/optimizer/statistics_propagator.cpp"

package/src/duckdb/ub_src_optimizer_compressed_materialization.cpp

@@ -0,0 +1,6 @@
+#include "src/optimizer/compressed_materialization/compress_aggregate.cpp"
+
+#include "src/optimizer/compressed_materialization/compress_distinct.cpp"
+
+#include "src/optimizer/compressed_materialization/compress_order.cpp"
+

package/src/duckdb/ub_src_optimizer_join_order.cpp

@@ -10,3 +10,13 @@
 
 #include "src/optimizer/join_order/cardinality_estimator.cpp"
 
+#include "src/optimizer/join_order/cost_model.cpp"
+
+#include "src/optimizer/join_order/plan_enumerator.cpp"
+
+#include "src/optimizer/join_order/relation_manager.cpp"
+
+#include "src/optimizer/join_order/query_graph_manager.cpp"
+
+#include "src/optimizer/join_order/relation_statistics_helper.cpp"
+

package/src/duckdb/ub_src_optimizer_statistics_expression.cpp

@@ -1,7 +1,5 @@
 #include "src/optimizer/statistics/expression/propagate_aggregate.cpp"
 
-#include "src/optimizer/statistics/expression/propagate_and_compress.cpp"
-
 #include "src/optimizer/statistics/expression/propagate_between.cpp"
 
 #include "src/optimizer/statistics/expression/propagate_case.cpp"

package/src/duckdb/ub_src_parser.cpp

@@ -6,8 +6,6 @@
 
 #include "src/parser/constraint.cpp"
 
-#include "src/parser/common_table_expression_info.cpp"
-
 #include "src/parser/expression_util.cpp"
 
 #include "src/parser/keyword_helper.cpp"

package/src/duckdb/ub_src_parser_query_node.cpp

@@ -1,5 +1,7 @@
 #include "src/parser/query_node/recursive_cte_node.cpp"
 
+#include "src/parser/query_node/cte_node.cpp"
+
 #include "src/parser/query_node/select_node.cpp"
 
 #include "src/parser/query_node/set_operation_node.cpp"

package/src/duckdb/ub_src_parser_transform_expression.cpp

@@ -34,3 +34,5 @@
 
 #include "src/parser/transform/expression/transform_subquery.cpp"
 
+#include "src/parser/transform/expression/transform_multi_assign_reference.cpp"
+

package/src/duckdb/ub_src_planner_binder_query_node.cpp

@@ -4,12 +4,16 @@
 
 #include "src/planner/binder/query_node/bind_recursive_cte_node.cpp"
 
+#include "src/planner/binder/query_node/bind_cte_node.cpp"
+
 #include "src/planner/binder/query_node/bind_table_macro_node.cpp"
 
 #include "src/planner/binder/query_node/plan_query_node.cpp"
 
 #include "src/planner/binder/query_node/plan_recursive_cte_node.cpp"
 
+#include "src/planner/binder/query_node/plan_cte_node.cpp"
+
 #include "src/planner/binder/query_node/plan_select_node.cpp"
 
 #include "src/planner/binder/query_node/plan_setop.cpp"

package/src/duckdb/ub_src_planner_operator.cpp

@@ -2,8 +2,6 @@
 
 #include "src/planner/operator/logical_any_join.cpp"
 
-#include "src/planner/operator/logical_asof_join.cpp"
-
 #include "src/planner/operator/logical_column_data_get.cpp"
 
 #include "src/planner/operator/logical_comparison_join.cpp"
@@ -24,7 +22,7 @@
 
 #include "src/planner/operator/logical_delim_get.cpp"
 
-#include "src/planner/operator/logical_delim_join.cpp"
+#include "src/planner/operator/logical_dependent_join.cpp"
 
 #include "src/planner/operator/logical_distinct.cpp"
 
@@ -68,6 +66,8 @@
 
 #include "src/planner/operator/logical_recursive_cte.cpp"
 
+#include "src/planner/operator/logical_materialized_cte.cpp"
+
 #include "src/planner/operator/logical_sample.cpp"
 
 #include "src/planner/operator/logical_set.cpp"

package/src/duckdb/ub_src_storage.cpp

@@ -14,10 +14,6 @@
 
 #include "src/storage/magic_bytes.cpp"
 
-#include "src/storage/meta_block_reader.cpp"
-
-#include "src/storage/meta_block_writer.cpp"
-
 #include "src/storage/storage_manager.cpp"
 
 #include "src/storage/standard_buffer_manager.cpp"

package/src/duckdb/ub_src_storage_compression.cpp

@@ -14,6 +14,8 @@
 
 #include "src/storage/compression/bitpacking.cpp"
 
+#include "src/storage/compression/bitpacking_hugeint.cpp"
+
 #include "src/storage/compression/patas.cpp"
 
 #include "src/storage/compression/fsst.cpp"

package/src/duckdb/ub_src_storage_metadata.cpp

@@ -0,0 +1,6 @@
+#include "src/storage/metadata/metadata_manager.cpp"
+
+#include "src/storage/metadata/metadata_reader.cpp"
+
+#include "src/storage/metadata/metadata_writer.cpp"
+

package/src/duckdb/ub_src_storage_serialization.cpp

@@ -0,0 +1,28 @@
+#include "src/storage/serialization/serialize_constraint.cpp"
+
+#include "src/storage/serialization/serialize_create_info.cpp"
+
+#include "src/storage/serialization/serialize_expression.cpp"
+
+#include "src/storage/serialization/serialize_logical_operator.cpp"
+
+#include "src/storage/serialization/serialize_macro_function.cpp"
+
+#include "src/storage/serialization/serialize_nodes.cpp"
+
+#include "src/storage/serialization/serialize_parsed_expression.cpp"
+
+#include "src/storage/serialization/serialize_parse_info.cpp"
+
+#include "src/storage/serialization/serialize_query_node.cpp"
+
+#include "src/storage/serialization/serialize_result_modifier.cpp"
+
+#include "src/storage/serialization/serialize_statement.cpp"
+
+#include "src/storage/serialization/serialize_table_filter.cpp"
+
+#include "src/storage/serialization/serialize_tableref.cpp"
+
+#include "src/storage/serialization/serialize_types.cpp"
+

package/src/duckdb_node.hpp

@@ -7,6 +7,7 @@
 #include <unordered_map>
 
 #include "duckdb/common/vector.hpp"
+#include "duckdb/common/arrow/arrow.hpp"
 
 using duckdb::vector;
 

package/src/statement.cpp

@@ -147,7 +147,7 @@ static Napi::Value convert_col_val(Napi::Env &env, duckdb::Value dval, duckdb::L
 		value = Napi::Number::New(env, duckdb::IntegerValue::Get(dval));
 	} break;
 	case duckdb::LogicalTypeId::BIGINT: {
-		value = Napi::Number::New(env, duckdb::BigIntValue::Get(dval));
+		value = Napi::BigInt::New(env, duckdb::BigIntValue::Get(dval));
 	} break;
 	case duckdb::LogicalTypeId::UTINYINT: {
 		value = Napi::Number::New(env, duckdb::UTinyIntValue::Get(dval));
@@ -159,7 +159,7 @@ static Napi::Value convert_col_val(Napi::Env &env, duckdb::Value dval, duckdb::L
 		value = Napi::Number::New(env, duckdb::UIntegerValue::Get(dval));
 	} break;
 	case duckdb::LogicalTypeId::UBIGINT: {
-		value = Napi::Number::New(env, duckdb::UBigIntValue::Get(dval));
+		value = Napi::BigInt::New(env, duckdb::UBigIntValue::Get(dval));
 	} break;
 	case duckdb::LogicalTypeId::FLOAT: {
 		value = Napi::Number::New(env, duckdb::FloatValue::Get(dval));
@@ -168,7 +168,14 @@ static Napi::Value convert_col_val(Napi::Env &env, duckdb::Value dval, duckdb::L
 		value = Napi::Number::New(env, duckdb::DoubleValue::Get(dval));
 	} break;
 	case duckdb::LogicalTypeId::HUGEINT: {
-		value = Napi::Number::New(env, dval.GetValue<double>());
+		auto val = duckdb::HugeIntValue::Get(dval);
+		auto negative = val.upper < 0;
+		if (negative) {
+			duckdb::Hugeint::NegateInPlace(val); // remove signing bit
+		}
+		D_ASSERT(val.upper >= 0);
+		const uint64_t words[] = {val.lower, static_cast<uint64_t>(val.upper)};
+		value = Napi::BigInt::New(env, negative, 2, words);
 	} break;
 	case duckdb::LogicalTypeId::DECIMAL: {
 		value = Napi::Number::New(env, dval.GetValue<double>());
@@ -541,7 +548,6 @@ static Napi::Value TypeToObject(Napi::Env &env, const duckdb::LogicalType &type)
 		obj.Set("value", TypeToObject(env, value_type));
 	} break;
 	case duckdb::LogicalTypeId::ENUM: {
-		auto name = duckdb::EnumType::GetTypeName(type);
 		auto &values_vec = duckdb::EnumType::GetValuesInsertOrder(type);
 		auto enum_size = duckdb::EnumType::GetSize(type);
 		auto arr = Napi::Array::New(env, enum_size);
@@ -549,7 +555,6 @@ static Napi::Value TypeToObject(Napi::Env &env, const duckdb::LogicalType &type)
 			auto child_name = values_vec.GetValue(i).GetValue<duckdb::string>();
 			arr.Set(i, child_name);
 		}
-		obj.Set("name", name);
 		obj.Set("values", arr);
 	} break;
 	case duckdb::LogicalTypeId::UNION: {

package/test/columns.test.ts

@@ -12,7 +12,7 @@ describe('Column Types', function() {
 
           let cols = stmt.columns();
           
-          assert.equal(cols.length, 41);
+          assert.equal(cols.length, 42);
 
           var expected = [
             { name: 'bool', type: { id: 'BOOLEAN',  sql_type: 'BOOLEAN' } },
@@ -48,8 +48,7 @@ describe('Column Types', function() {
               name: 'small_enum',
               type: {
                 id: 'ENUM',
-                sql_type: 'small_enum',
-                name: 'small_enum',
+                sql_type: "ENUM('DUCK_DUCK_ENUM', 'GOOSE')",
                 values: [
                   "DUCK_DUCK_ENUM",
                   "GOOSE"
@@ -232,6 +231,29 @@ describe('Column Types', function() {
                   sql_type: 'VARCHAR'
                 }
               }
+            },
+            {
+              name: "union",
+              type: {
+                id: "UNION",
+                sql_type: "UNION(name VARCHAR, age SMALLINT)",
+                children: [
+                  {
+                    name: "name",
+                    type: {
+                      id: "VARCHAR",
+                      sql_type: "VARCHAR"
+                    }
+                  },
+                  {
+                    name: "age",
+                    type: {
+                      id: "SMALLINT",
+                      sql_type: "SMALLINT",
+                    }
+                  }
+                ],
+              }
             }
         ]
 

package/test/extension.test.ts

@@ -37,7 +37,7 @@ const test_httpfs = async function (db: duckdb.Database) {
         }));
     await chai.assert.isRejected(promise, 'IO Error: Connection error for HTTP HEAD');
 
-    await new Promise<void>((resolve, reject) => db.all("SELECT id, first_name, last_name FROM PARQUET_SCAN('https://raw.githubusercontent.com/cwida/duckdb/master/data/parquet-testing/userdata1.parquet') LIMIT 3;", function (err: null | Error, rows: TableData) {
+    await new Promise<void>((resolve, reject) => db.all("SELECT id, first_name, last_name FROM PARQUET_SCAN('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/userdata1.parquet') LIMIT 3;", function (err: null | Error, rows: TableData) {
         if (err) {
             if (err.message.startsWith("Unable to connect to URL")) {
                 console.warn("Warning: HTTP request failed in extension.test.js");