duckdb 0.8.2-dev15.0 → 0.8.2-dev1549.0

package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp

@@ -348,17 +348,17 @@
 
 #include "extension/icu/third_party/icu/i18n/wintzimpl.cpp"
 
-#include "extension/icu/third_party/icu/i18n/double-conversion-fast-dtoa.cpp"
-
-#include "extension/icu/third_party/icu/i18n/double-conversion-double-to-string.cpp"
-
 #include "extension/icu/third_party/icu/i18n/double-conversion-strtod.cpp"
 
-#include "extension/icu/third_party/icu/i18n/double-conversion-cached-powers.cpp"
+#include "extension/icu/third_party/icu/i18n/double-conversion-fast-dtoa.cpp"
 
 #include "extension/icu/third_party/icu/i18n/double-conversion-string-to-double.cpp"
 
+#include "extension/icu/third_party/icu/i18n/double-conversion-bignum.cpp"
+
+#include "extension/icu/third_party/icu/i18n/double-conversion-double-to-string.cpp"
+
 #include "extension/icu/third_party/icu/i18n/double-conversion-bignum-dtoa.cpp"
 
-#include "extension/icu/third_party/icu/i18n/double-conversion-bignum.cpp"
+#include "extension/icu/third_party/icu/i18n/double-conversion-cached-powers.cpp"
 

package/src/duckdb/ub_src_common.cpp

@@ -18,6 +18,8 @@
 
 #include "src/common/exception_format_value.cpp"
 
+#include "src/common/extra_type_info.cpp"
+
 #include "src/common/field_writer.cpp"
 
 #include "src/common/file_buffer.cpp"

package/src/duckdb/ub_src_execution_index_art.cpp

@@ -22,7 +22,5 @@
 
 #include "src/execution/index/art/prefix.cpp"
 
-#include "src/execution/index/art/prefix_segment.cpp"
-
 #include "src/execution/index/art/art.cpp"
 

package/src/duckdb/ub_src_execution_operator_persistent.cpp

@@ -10,6 +10,8 @@
 
 #include "src/execution/operator/persistent/csv_reader_options.cpp"
 
+#include "src/execution/operator/persistent/csv_rejects_table.cpp"
+
 #include "src/execution/operator/persistent/physical_fixed_batch_copy.cpp"
 
 #include "src/execution/operator/persistent/physical_batch_copy_to_file.cpp"

package/src/duckdb/ub_src_execution_operator_set.cpp

@@ -2,3 +2,5 @@
 
 #include "src/execution/operator/set/physical_recursive_cte.cpp"
 
+#include "src/execution/operator/set/physical_cte.cpp"
+

package/src/duckdb/ub_src_execution_physical_plan.cpp

@@ -82,3 +82,5 @@
 
 #include "src/execution/physical_plan/plan_recursive_cte.cpp"
 
+#include "src/execution/physical_plan/plan_cte.cpp"
+

package/src/duckdb/ub_src_function_scalar.cpp

@@ -1,3 +1,5 @@
+#include "src/function/scalar/compressed_materialization_functions.cpp"
+
 #include "src/function/scalar/generic_functions.cpp"
 
 #include "src/function/scalar/string_functions.cpp"

package/src/duckdb/ub_src_function_scalar_compressed_materialization.cpp

@@ -0,0 +1,4 @@
+#include "src/function/scalar/compressed_materialization/compress_integral.cpp"
+
+#include "src/function/scalar/compressed_materialization/compress_string.cpp"
+

package/src/duckdb/ub_src_function_scalar_list.cpp

@@ -4,3 +4,5 @@
 
 #include "src/function/scalar/list/list_extract.cpp"
 
+#include "src/function/scalar/list/list_resize.cpp"
+

package/src/duckdb/ub_src_optimizer.cpp

@@ -1,5 +1,9 @@
+#include "src/optimizer/column_binding_replacer.cpp"
+
 #include "src/optimizer/common_aggregate_optimizer.cpp"
 
+#include "src/optimizer/compressed_materialization.cpp"
+
 #include "src/optimizer/cse_optimizer.cpp"
 
 #include "src/optimizer/deliminator.cpp"
@@ -24,6 +28,8 @@
 
 #include "src/optimizer/regex_range_filter.cpp"
 
+#include "src/optimizer/remove_duplicate_groups.cpp"
+
 #include "src/optimizer/remove_unused_columns.cpp"
 
 #include "src/optimizer/statistics_propagator.cpp"

package/src/duckdb/ub_src_optimizer_compressed_materialization.cpp

@@ -0,0 +1,6 @@
+#include "src/optimizer/compressed_materialization/compress_aggregate.cpp"
+
+#include "src/optimizer/compressed_materialization/compress_distinct.cpp"
+
+#include "src/optimizer/compressed_materialization/compress_order.cpp"
+

package/src/duckdb/ub_src_optimizer_statistics_expression.cpp

@@ -1,7 +1,5 @@
 #include "src/optimizer/statistics/expression/propagate_aggregate.cpp"
 
-#include "src/optimizer/statistics/expression/propagate_and_compress.cpp"
-
 #include "src/optimizer/statistics/expression/propagate_between.cpp"
 
 #include "src/optimizer/statistics/expression/propagate_case.cpp"

package/src/duckdb/ub_src_parser_query_node.cpp

@@ -1,5 +1,7 @@
 #include "src/parser/query_node/recursive_cte_node.cpp"
 
+#include "src/parser/query_node/cte_node.cpp"
+
 #include "src/parser/query_node/select_node.cpp"
 
 #include "src/parser/query_node/set_operation_node.cpp"

package/src/duckdb/ub_src_parser_transform_expression.cpp

@@ -34,3 +34,5 @@
 
 #include "src/parser/transform/expression/transform_subquery.cpp"
 
+#include "src/parser/transform/expression/transform_multi_assign_reference.cpp"
+

package/src/duckdb/ub_src_planner_binder_query_node.cpp

@@ -4,12 +4,16 @@
 
 #include "src/planner/binder/query_node/bind_recursive_cte_node.cpp"
 
+#include "src/planner/binder/query_node/bind_cte_node.cpp"
+
 #include "src/planner/binder/query_node/bind_table_macro_node.cpp"
 
 #include "src/planner/binder/query_node/plan_query_node.cpp"
 
 #include "src/planner/binder/query_node/plan_recursive_cte_node.cpp"
 
+#include "src/planner/binder/query_node/plan_cte_node.cpp"
+
 #include "src/planner/binder/query_node/plan_select_node.cpp"
 
 #include "src/planner/binder/query_node/plan_setop.cpp"

package/src/duckdb/ub_src_planner_operator.cpp

@@ -26,6 +26,8 @@
 
 #include "src/planner/operator/logical_delim_join.cpp"
 
+#include "src/planner/operator/logical_dependent_join.cpp"
+
 #include "src/planner/operator/logical_distinct.cpp"
 
 #include "src/planner/operator/logical_dummy_scan.cpp"
@@ -68,6 +70,8 @@
 
 #include "src/planner/operator/logical_recursive_cte.cpp"
 
+#include "src/planner/operator/logical_materialized_cte.cpp"
+
 #include "src/planner/operator/logical_sample.cpp"
 
 #include "src/planner/operator/logical_set.cpp"

package/src/duckdb/ub_src_storage_serialization.cpp

@@ -0,0 +1,10 @@
+#include "src/storage/serialization/serialize_parsed_expression.cpp"
+
+#include "src/storage/serialization/serialize_query_node.cpp"
+
+#include "src/storage/serialization/serialize_result_modifier.cpp"
+
+#include "src/storage/serialization/serialize_tableref.cpp"
+
+#include "src/storage/serialization/serialize_types.cpp"
+

package/src/duckdb_node.hpp

@@ -165,6 +165,7 @@ public:
 	Napi::Value Run(const Napi::CallbackInfo &info);
 	Napi::Value Finish(const Napi::CallbackInfo &info);
 	Napi::Value Stream(const Napi::CallbackInfo &info);
+	Napi::Value Columns(const Napi::CallbackInfo &info);
 
 public:
 	static Napi::FunctionReference constructor;

package/src/statement.cpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/common/helper.hpp"
 #include "duckdb/common/vector.hpp"
+#include "duckdb/common/types.hpp"
 
 using duckdb::unique_ptr;
 using duckdb::vector;
@@ -25,7 +26,8 @@ Napi::Object Statement::Init(Napi::Env env, Napi::Object exports) {
 	    DefineClass(env, "Statement",
 	                {InstanceMethod("run", &Statement::Run), InstanceMethod("all", &Statement::All),
 	                 InstanceMethod("arrowIPCAll", &Statement::ArrowIPCAll), InstanceMethod("each", &Statement::Each),
-	                 InstanceMethod("finalize", &Statement::Finish), InstanceMethod("stream", &Statement::Stream)});
+	                 InstanceMethod("finalize", &Statement::Finish), InstanceMethod("stream", &Statement::Stream),
+	                 InstanceMethod("columns", &Statement::Columns)});
 
 	constructor = Napi::Persistent(t);
 	constructor.SuppressDestruct();
@@ -145,7 +147,7 @@ static Napi::Value convert_col_val(Napi::Env &env, duckdb::Value dval, duckdb::L
 		value = Napi::Number::New(env, duckdb::IntegerValue::Get(dval));
 	} break;
 	case duckdb::LogicalTypeId::BIGINT: {
-		value = Napi::Number::New(env, duckdb::BigIntValue::Get(dval));
+		value = Napi::BigInt::New(env, duckdb::BigIntValue::Get(dval));
 	} break;
 	case duckdb::LogicalTypeId::UTINYINT: {
 		value = Napi::Number::New(env, duckdb::UTinyIntValue::Get(dval));
@@ -157,7 +159,7 @@ static Napi::Value convert_col_val(Napi::Env &env, duckdb::Value dval, duckdb::L
 		value = Napi::Number::New(env, duckdb::UIntegerValue::Get(dval));
 	} break;
 	case duckdb::LogicalTypeId::UBIGINT: {
-		value = Napi::Number::New(env, duckdb::UBigIntValue::Get(dval));
+		value = Napi::BigInt::New(env, duckdb::UBigIntValue::Get(dval));
 	} break;
 	case duckdb::LogicalTypeId::FLOAT: {
 		value = Napi::Number::New(env, duckdb::FloatValue::Get(dval));
@@ -166,7 +168,14 @@ static Napi::Value convert_col_val(Napi::Env &env, duckdb::Value dval, duckdb::L
 		value = Napi::Number::New(env, duckdb::DoubleValue::Get(dval));
 	} break;
 	case duckdb::LogicalTypeId::HUGEINT: {
-		value = Napi::Number::New(env, dval.GetValue<double>());
+		auto val = duckdb::HugeIntValue::Get(dval);
+		auto negative = val.upper < 0;
+		if (negative) {
+			duckdb::Hugeint::NegateInPlace(val); // remove signing bit
+		}
+		D_ASSERT(val.upper >= 0);
+		const uint64_t words[] = {val.lower, (uint64_t)val.upper};
+		value = Napi::BigInt::New(env, negative, 2, words);
 	} break;
 	case duckdb::LogicalTypeId::DECIMAL: {
 		value = Napi::Number::New(env, dval.GetValue<double>());
@@ -503,6 +512,97 @@ Napi::Value Statement::Stream(const Napi::CallbackInfo &info) {
 	return deferred.Promise();
 }
 
+static Napi::Value TypeToObject(Napi::Env &env, const duckdb::LogicalType &type) {
+	auto obj = Napi::Object::New(env);
+
+	auto id = duckdb::LogicalTypeIdToString(type.id());
+	obj.Set("id", id);
+	obj.Set("sql_type", type.ToString());
+
+	if (type.HasAlias()) {
+		obj.Set("alias", type.GetAlias());
+	}
+
+	switch (type.id()) {
+	case duckdb::LogicalTypeId::STRUCT: {
+		auto &child_types = duckdb::StructType::GetChildTypes(type);
+		auto arr = Napi::Array::New(env, child_types.size());
+		for (size_t i = 0; i < child_types.size(); i++) {
+			auto child_name = child_types[i].first;
+			auto child_type = child_types[i].second;
+			auto child_obj = Napi::Object::New(env);
+			child_obj.Set("name", child_name);
+			child_obj.Set("type", TypeToObject(env, child_type));
+			arr.Set(i, child_obj);
+		}
+		obj.Set("children", arr);
+	} break;
+	case duckdb::LogicalTypeId::LIST: {
+		auto &child_type = duckdb::ListType::GetChildType(type);
+		obj.Set("child", TypeToObject(env, child_type));
+	} break;
+	case duckdb::LogicalTypeId::MAP: {
+		auto &key_type = duckdb::MapType::KeyType(type);
+		auto &value_type = duckdb::MapType::ValueType(type);
+		obj.Set("key", TypeToObject(env, key_type));
+		obj.Set("value", TypeToObject(env, value_type));
+	} break;
+	case duckdb::LogicalTypeId::ENUM: {
+		auto name = duckdb::EnumType::GetTypeName(type);
+		auto &values_vec = duckdb::EnumType::GetValuesInsertOrder(type);
+		auto enum_size = duckdb::EnumType::GetSize(type);
+		auto arr = Napi::Array::New(env, enum_size);
+		for (size_t i = 0; i < enum_size; i++) {
+			auto child_name = values_vec.GetValue(i).GetValue<duckdb::string>();
+			arr.Set(i, child_name);
+		}
+		obj.Set("name", name);
+		obj.Set("values", arr);
+	} break;
+	case duckdb::LogicalTypeId::UNION: {
+		auto child_count = duckdb::UnionType::GetMemberCount(type);
+		auto arr = Napi::Array::New(env, child_count);
+		for (size_t i = 0; i < child_count; i++) {
+			auto &child_name = duckdb::UnionType::GetMemberName(type, i);
+			auto &child_type = duckdb::UnionType::GetMemberType(type, i);
+			auto child_obj = Napi::Object::New(env);
+			child_obj.Set("name", child_name);
+			child_obj.Set("type", TypeToObject(env, child_type));
+			arr.Set(i, child_obj);
+		}
+		obj.Set("children", arr);
+	} break;
+	case duckdb::LogicalTypeId::DECIMAL: {
+		auto width = duckdb::DecimalType::GetWidth(type);
+		auto scale = duckdb::DecimalType::GetScale(type);
+		obj.Set("width", width);
+		obj.Set("scale", scale);
+	} break;
+	default:
+		break;
+	}
+	return obj;
+}
+
+Napi::Value Statement::Columns(const Napi::CallbackInfo &info) {
+	Napi::Env env = info.Env();
+
+	if (!statement) {
+		return env.Null();
+	}
+
+	auto &names = statement->GetNames();
+	auto &types = statement->GetTypes();
+	auto arr = Napi::Array::New(env, names.size());
+	for (size_t i = 0; i < names.size(); i++) {
+		auto obj = Napi::Object::New(env);
+		obj.Set("name", Napi::String::New(env, names[i]));
+		obj.Set("type", TypeToObject(env, types[i]));
+		arr.Set(i, obj);
+	}
+	return arr;
+}
+
 struct FinishTask : public Task {
 	FinishTask(Statement &statement, Napi::Function callback) : Task(statement, callback) {
 	}

package/test/columns.test.ts

@@ -0,0 +1,243 @@
+import * as duckdb from '..';
+import * as assert from 'assert';
+
+describe('Column Types', function() {
+    let db: duckdb.Database;
+    before(function(done) { db = new duckdb.Database(':memory:', done); });
+
+    it('should prepare a statement and return the columns and their types', function(done) {
+      // we dont include the large_enum and small_enum since they are huge and test the same code path as the small_enum
+      var stmt = db.prepare("SELECT * EXCLUDE(medium_enum, large_enum) FROM test_all_types()", function(err: null | Error) {
+          if (err) throw err;
+
+          let cols = stmt.columns();
+          
+          assert.equal(cols.length, 41);
+
+          var expected = [
+            { name: 'bool', type: { id: 'BOOLEAN',  sql_type: 'BOOLEAN' } },
+            { name: 'tinyint', type: { id: 'TINYINT', sql_type: 'TINYINT' } },
+            { name: 'smallint', type: { id: 'SMALLINT', sql_type: 'SMALLINT' } },
+            { name: 'int', type: { id: 'INTEGER', sql_type: 'INTEGER' } },
+            { name: 'bigint', type: { id: 'BIGINT', sql_type: 'BIGINT' } },
+            { name: 'hugeint', type: { id: 'HUGEINT', sql_type: 'HUGEINT' } },
+            { name: 'utinyint', type: { id: 'UTINYINT', sql_type: 'UTINYINT' } },
+            { name: 'usmallint', type: { id: 'USMALLINT', sql_type: 'USMALLINT' } },
+            { name: 'uint', type: { id: 'UINTEGER', sql_type: 'UINTEGER' } },
+            { name: 'ubigint', type: { id: 'UBIGINT', sql_type: 'UBIGINT' } },
+            { name: 'date', type: { id: 'DATE', sql_type: 'DATE' } },
+            { name: 'time', type: { id: 'TIME', sql_type: 'TIME' } },
+            { name: 'timestamp', type: { id: 'TIMESTAMP', sql_type: 'TIMESTAMP' } },
+            { name: 'timestamp_s', type: { id: 'TIMESTAMP_S', sql_type: 'TIMESTAMP_S' } },
+            { name: 'timestamp_ms', type: { id: 'TIMESTAMP_MS', sql_type: 'TIMESTAMP_MS' } },
+            { name: 'timestamp_ns', type: { id: 'TIMESTAMP_NS', sql_type: 'TIMESTAMP_NS' } },
+            { name: 'time_tz', type: { id: 'TIME WITH TIME ZONE', sql_type: 'TIME WITH TIME ZONE' } },
+            { name: 'timestamp_tz', type: { id: 'TIMESTAMP WITH TIME ZONE', sql_type: 'TIMESTAMP WITH TIME ZONE' } },
+            { name: 'float', type: { id: 'FLOAT', sql_type: 'FLOAT' } },
+            { name: 'double', type: { id: 'DOUBLE', sql_type: 'DOUBLE' } },
+            { name: 'dec_4_1', type: { id: 'DECIMAL', sql_type: 'DECIMAL(4,1)', width: 4, scale: 1 } },
+            { name: 'dec_9_4', type: { id: 'DECIMAL', sql_type: 'DECIMAL(9,4)', width: 9, scale: 4 } },
+            { name: 'dec_18_6', type: { id: 'DECIMAL', sql_type: 'DECIMAL(18,6)', width: 18, scale: 6 } },
+            { name: 'dec38_10', type: { id: 'DECIMAL', sql_type: 'DECIMAL(38,10)', width: 38, scale: 10 } },
+            { name: 'uuid', type: { id: 'UUID', sql_type: 'UUID' } },
+            { name: 'interval', type: { id: 'INTERVAL', sql_type: 'INTERVAL' } },
+            { name: 'varchar', type: { id: 'VARCHAR', sql_type: 'VARCHAR' } },
+            { name: 'blob', type: { id: 'BLOB', sql_type: 'BLOB' } },
+            { name: 'bit', type: { id: 'BIT', sql_type: 'BIT' } },
+            {
+              name: 'small_enum',
+              type: {
+                id: 'ENUM',
+                sql_type: 'small_enum',
+                name: 'small_enum',
+                values: [
+                  "DUCK_DUCK_ENUM",
+                  "GOOSE"
+                ]
+              }
+            },
+            {
+              name: 'int_array',
+              type: { 
+                id: 'LIST', 
+                sql_type: 'INTEGER[]', 
+                child: {
+                  id: 'INTEGER',
+                  sql_type: 'INTEGER'
+                } 
+              }
+            },
+            {
+              name: 'double_array',
+              type: { 
+                id: 'LIST', 
+                sql_type: 'DOUBLE[]', 
+                child: {
+                  id: 'DOUBLE',
+                  sql_type: 'DOUBLE'
+                } 
+              }
+            },
+            {
+              name: 'date_array',
+              type: { 
+                id: 'LIST', 
+                sql_type: 'DATE[]', 
+                child: {
+                  id: 'DATE',
+                  sql_type: 'DATE'
+                } 
+              }
+            },
+            {
+              name: 'timestamp_array',
+              type: { 
+                id: 'LIST', 
+                sql_type: 'TIMESTAMP[]', 
+                child: {
+                  id: 'TIMESTAMP',
+                  sql_type: 'TIMESTAMP'
+                } 
+              }
+            },
+            {
+              name: 'timestamptz_array',
+              type: {
+                id: 'LIST',
+                sql_type: 'TIMESTAMP WITH TIME ZONE[]',
+                child: {
+                  id: 'TIMESTAMP WITH TIME ZONE',
+                  sql_type: 'TIMESTAMP WITH TIME ZONE'
+                }
+              }
+            },
+            {
+              name: 'varchar_array',
+              type: { 
+                id: 'LIST', 
+                sql_type: 'VARCHAR[]', 
+                child: {
+                  id: 'VARCHAR',
+                  sql_type: 'VARCHAR'
+                }
+              }  
+            },
+            {
+              name: 'nested_int_array',
+              type: { 
+                id: 'LIST', 
+                sql_type: 'INTEGER[][]', 
+                child: {
+                  id: 'LIST',
+                  sql_type: 'INTEGER[]',
+                  child: {
+                    id: 'INTEGER',
+                    sql_type: 'INTEGER'
+                  }
+                } 
+              }
+            },
+            {
+              name: 'struct',
+              type: {
+                id: 'STRUCT',
+                sql_type: 'STRUCT(a INTEGER, b VARCHAR)',
+                children: [
+                  { 
+                    name: 'a',
+                    type: {
+                      id: 'INTEGER',
+                      sql_type: 'INTEGER'
+                    }
+                  },
+                  {
+                    name: 'b',
+                    type: {
+                      id: 'VARCHAR',
+                      sql_type: 'VARCHAR'
+                    }
+                  }
+                ]
+              }
+            },
+            {
+              name: 'struct_of_arrays',
+              type: {
+                id: 'STRUCT',
+                sql_type: 'STRUCT(a INTEGER[], b VARCHAR[])',
+                children: [
+                  {
+                    name: 'a',
+                    type: {
+                      id: 'LIST',
+                      sql_type: 'INTEGER[]',
+                      child: {
+                        id: 'INTEGER',
+                        sql_type: 'INTEGER'
+                      }
+                    }
+                  },
+                  {
+                    name: 'b',
+                    type: {
+                      id: 'LIST',
+                      sql_type: 'VARCHAR[]',
+                      child: {
+                        id: 'VARCHAR',
+                        sql_type: 'VARCHAR'
+                      }
+                    }
+                  }
+                ]
+              }
+            },
+            {
+              name: 'array_of_structs',
+              type: {
+                id: 'LIST',
+                sql_type: 'STRUCT(a INTEGER, b VARCHAR)[]',
+                child: {
+                  id: 'STRUCT',
+                  sql_type: 'STRUCT(a INTEGER, b VARCHAR)',
+                  children: [
+                    {
+                      name: 'a',
+                      type: {
+                        id: 'INTEGER',
+                        sql_type: 'INTEGER'
+                      }
+                    },
+                    {
+                      name: 'b',
+                      type: {
+                        id: 'VARCHAR',
+                        sql_type: 'VARCHAR'
+                      }
+                    }
+                  ]
+                }
+              }
+            },
+            {
+              name: 'map',
+              type: {
+                id: 'MAP',
+                sql_type: 'MAP(VARCHAR, VARCHAR)',
+                key: {
+                  id: 'VARCHAR',
+                  sql_type: 'VARCHAR'
+                },
+                value: {
+                  id: 'VARCHAR',
+                  sql_type: 'VARCHAR'
+                }
+              }
+            }
+        ]
+
+        assert.deepEqual(cols, expected);
+
+      });
+      stmt.finalize(done);
+  });
+});