duckdb 0.8.2-dev161.0 → 0.8.2-dev1764.0

package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp

@@ -348,17 +348,17 @@
 
 #include "extension/icu/third_party/icu/i18n/wintzimpl.cpp"
 
-#include "extension/icu/third_party/icu/i18n/double-conversion-double-to-string.cpp"
-
-#include "extension/icu/third_party/icu/i18n/double-conversion-string-to-double.cpp"
-
-#include "extension/icu/third_party/icu/i18n/double-conversion-cached-powers.cpp"
+#include "extension/icu/third_party/icu/i18n/double-conversion-strtod.cpp"
 
 #include "extension/icu/third_party/icu/i18n/double-conversion-fast-dtoa.cpp"
 
+#include "extension/icu/third_party/icu/i18n/double-conversion-string-to-double.cpp"
+
 #include "extension/icu/third_party/icu/i18n/double-conversion-bignum.cpp"
 
+#include "extension/icu/third_party/icu/i18n/double-conversion-double-to-string.cpp"
+
 #include "extension/icu/third_party/icu/i18n/double-conversion-bignum-dtoa.cpp"
 
-#include "extension/icu/third_party/icu/i18n/double-conversion-strtod.cpp"
+#include "extension/icu/third_party/icu/i18n/double-conversion-cached-powers.cpp"
 

package/src/duckdb/ub_src_common.cpp

@@ -18,6 +18,8 @@
 
 #include "src/common/exception_format_value.cpp"
 
+#include "src/common/extra_type_info.cpp"
+
 #include "src/common/field_writer.cpp"
 
 #include "src/common/file_buffer.cpp"

package/src/duckdb/ub_src_execution_index_art.cpp

@@ -22,7 +22,5 @@
 
 #include "src/execution/index/art/prefix.cpp"
 
-#include "src/execution/index/art/prefix_segment.cpp"
-
 #include "src/execution/index/art/art.cpp"
 

package/src/duckdb/ub_src_execution_operator_persistent.cpp

@@ -10,6 +10,8 @@
 
 #include "src/execution/operator/persistent/csv_reader_options.cpp"
 
+#include "src/execution/operator/persistent/csv_rejects_table.cpp"
+
 #include "src/execution/operator/persistent/physical_fixed_batch_copy.cpp"
 
 #include "src/execution/operator/persistent/physical_batch_copy_to_file.cpp"

package/src/duckdb/ub_src_execution_operator_set.cpp

@@ -2,3 +2,5 @@
 
 #include "src/execution/operator/set/physical_recursive_cte.cpp"
 
+#include "src/execution/operator/set/physical_cte.cpp"
+

package/src/duckdb/ub_src_execution_physical_plan.cpp

@@ -82,3 +82,5 @@
 
 #include "src/execution/physical_plan/plan_recursive_cte.cpp"
 
+#include "src/execution/physical_plan/plan_cte.cpp"
+

package/src/duckdb/ub_src_function_scalar.cpp

@@ -1,3 +1,5 @@
+#include "src/function/scalar/compressed_materialization_functions.cpp"
+
 #include "src/function/scalar/generic_functions.cpp"
 
 #include "src/function/scalar/string_functions.cpp"

package/src/duckdb/ub_src_function_scalar_compressed_materialization.cpp

@@ -0,0 +1,4 @@
+#include "src/function/scalar/compressed_materialization/compress_integral.cpp"
+
+#include "src/function/scalar/compressed_materialization/compress_string.cpp"
+

package/src/duckdb/ub_src_function_scalar_list.cpp

@@ -4,3 +4,5 @@
 
 #include "src/function/scalar/list/list_extract.cpp"
 
+#include "src/function/scalar/list/list_resize.cpp"
+

package/src/duckdb/ub_src_optimizer.cpp

@@ -1,5 +1,9 @@
+#include "src/optimizer/column_binding_replacer.cpp"
+
 #include "src/optimizer/common_aggregate_optimizer.cpp"
 
+#include "src/optimizer/compressed_materialization.cpp"
+
 #include "src/optimizer/cse_optimizer.cpp"
 
 #include "src/optimizer/deliminator.cpp"
@@ -24,6 +28,8 @@
 
 #include "src/optimizer/regex_range_filter.cpp"
 
+#include "src/optimizer/remove_duplicate_groups.cpp"
+
 #include "src/optimizer/remove_unused_columns.cpp"
 
 #include "src/optimizer/statistics_propagator.cpp"

package/src/duckdb/ub_src_optimizer_compressed_materialization.cpp

@@ -0,0 +1,6 @@
+#include "src/optimizer/compressed_materialization/compress_aggregate.cpp"
+
+#include "src/optimizer/compressed_materialization/compress_distinct.cpp"
+
+#include "src/optimizer/compressed_materialization/compress_order.cpp"
+

package/src/duckdb/ub_src_optimizer_statistics_expression.cpp

@@ -1,7 +1,5 @@
 #include "src/optimizer/statistics/expression/propagate_aggregate.cpp"
 
-#include "src/optimizer/statistics/expression/propagate_and_compress.cpp"
-
 #include "src/optimizer/statistics/expression/propagate_between.cpp"
 
 #include "src/optimizer/statistics/expression/propagate_case.cpp"

package/src/duckdb/ub_src_parser_query_node.cpp

@@ -1,5 +1,7 @@
 #include "src/parser/query_node/recursive_cte_node.cpp"
 
+#include "src/parser/query_node/cte_node.cpp"
+
 #include "src/parser/query_node/select_node.cpp"
 
 #include "src/parser/query_node/set_operation_node.cpp"

package/src/duckdb/ub_src_parser_transform_expression.cpp

@@ -34,3 +34,5 @@
 
 #include "src/parser/transform/expression/transform_subquery.cpp"
 
+#include "src/parser/transform/expression/transform_multi_assign_reference.cpp"
+

package/src/duckdb/ub_src_planner_binder_query_node.cpp

@@ -4,12 +4,16 @@
 
 #include "src/planner/binder/query_node/bind_recursive_cte_node.cpp"
 
+#include "src/planner/binder/query_node/bind_cte_node.cpp"
+
 #include "src/planner/binder/query_node/bind_table_macro_node.cpp"
 
 #include "src/planner/binder/query_node/plan_query_node.cpp"
 
 #include "src/planner/binder/query_node/plan_recursive_cte_node.cpp"
 
+#include "src/planner/binder/query_node/plan_cte_node.cpp"
+
 #include "src/planner/binder/query_node/plan_select_node.cpp"
 
 #include "src/planner/binder/query_node/plan_setop.cpp"

package/src/duckdb/ub_src_planner_operator.cpp

@@ -26,6 +26,8 @@
 
 #include "src/planner/operator/logical_delim_join.cpp"
 
+#include "src/planner/operator/logical_dependent_join.cpp"
+
 #include "src/planner/operator/logical_distinct.cpp"
 
 #include "src/planner/operator/logical_dummy_scan.cpp"
@@ -68,6 +70,8 @@
 
 #include "src/planner/operator/logical_recursive_cte.cpp"
 
+#include "src/planner/operator/logical_materialized_cte.cpp"
+
 #include "src/planner/operator/logical_sample.cpp"
 
 #include "src/planner/operator/logical_set.cpp"

package/src/duckdb/ub_src_storage_serialization.cpp

@@ -0,0 +1,10 @@
+#include "src/storage/serialization/serialize_parsed_expression.cpp"
+
+#include "src/storage/serialization/serialize_query_node.cpp"
+
+#include "src/storage/serialization/serialize_result_modifier.cpp"
+
+#include "src/storage/serialization/serialize_tableref.cpp"
+
+#include "src/storage/serialization/serialize_types.cpp"
+

package/src/statement.cpp

@@ -147,7 +147,7 @@ static Napi::Value convert_col_val(Napi::Env &env, duckdb::Value dval, duckdb::L
 		value = Napi::Number::New(env, duckdb::IntegerValue::Get(dval));
 	} break;
 	case duckdb::LogicalTypeId::BIGINT: {
-		value = Napi::Number::New(env, duckdb::BigIntValue::Get(dval));
+		value = Napi::BigInt::New(env, duckdb::BigIntValue::Get(dval));
 	} break;
 	case duckdb::LogicalTypeId::UTINYINT: {
 		value = Napi::Number::New(env, duckdb::UTinyIntValue::Get(dval));
@@ -159,7 +159,7 @@ static Napi::Value convert_col_val(Napi::Env &env, duckdb::Value dval, duckdb::L
 		value = Napi::Number::New(env, duckdb::UIntegerValue::Get(dval));
 	} break;
 	case duckdb::LogicalTypeId::UBIGINT: {
-		value = Napi::Number::New(env, duckdb::UBigIntValue::Get(dval));
+		value = Napi::BigInt::New(env, duckdb::UBigIntValue::Get(dval));
 	} break;
 	case duckdb::LogicalTypeId::FLOAT: {
 		value = Napi::Number::New(env, duckdb::FloatValue::Get(dval));
@@ -168,7 +168,14 @@ static Napi::Value convert_col_val(Napi::Env &env, duckdb::Value dval, duckdb::L
 		value = Napi::Number::New(env, duckdb::DoubleValue::Get(dval));
 	} break;
 	case duckdb::LogicalTypeId::HUGEINT: {
-		value = Napi::Number::New(env, dval.GetValue<double>());
+		auto val = duckdb::HugeIntValue::Get(dval);
+		auto negative = val.upper < 0;
+		if (negative) {
+			duckdb::Hugeint::NegateInPlace(val); // remove signing bit
+		}
+		D_ASSERT(val.upper >= 0);
+		const uint64_t words[] = {val.lower, (uint64_t)val.upper};
+		value = Napi::BigInt::New(env, negative, 2, words);
 	} break;
 	case duckdb::LogicalTypeId::DECIMAL: {
 		value = Napi::Number::New(env, dval.GetValue<double>());

package/test/columns.test.ts

@@ -12,7 +12,7 @@ describe('Column Types', function() {
 
           let cols = stmt.columns();
           
-          assert.equal(cols.length, 41);
+          assert.equal(cols.length, 42);
 
           var expected = [
             { name: 'bool', type: { id: 'BOOLEAN',  sql_type: 'BOOLEAN' } },
@@ -232,6 +232,29 @@ describe('Column Types', function() {
                   sql_type: 'VARCHAR'
                 }
               }
+            },
+            {
+              name: "union",
+              type: {
+                id: "UNION",
+                sql_type: "UNION(name VARCHAR, age SMALLINT)",
+                children: [
+                  {
+                    name: "name",
+                    type: {
+                      id: "VARCHAR",
+                      sql_type: "VARCHAR"
+                    }
+                  },
+                  {
+                    name: "age",
+                    type: {
+                      id: "SMALLINT",
+                      sql_type: "SMALLINT",
+                    }
+                  }
+                ],
+              }
             }
         ]
 

package/test/test_all_types.test.ts

@@ -0,0 +1,234 @@
+import { expect } from "chai";
+import duckdb, { DuckDbError, TableData } from "..";
+
+function get_all_types(): Promise<string[]> {
+  return new Promise((resolve, reject) => {
+    const conn = new duckdb.Database(":memory:");
+    conn.all(
+      "describe select * from test_all_types()",
+      (error: DuckDbError | null, data: TableData) => {
+        if (error) reject(error);
+        resolve(data.map((row) => row.column_name));
+      }
+    );
+  });
+}
+
+function timedelta(obj: { days: number; micros: number; months: number }) {
+  return obj;
+}
+
+// We replace these values since the extreme ranges are not supported in native-node.
+const replacement_values: Record<string, string> = {
+  timestamp:
+    "'1990-01-01 00:00:00'::TIMESTAMP, '9999-12-31 23:59:59'::TIMESTAMP, NULL::TIMESTAMP",
+  // TODO: fix these, they are currently being returned as strings
+  //   timestamp_s: "'1990-01-01 00:00:00'::TIMESTAMP_S",
+  //   timestamp_ns: "'1990-01-01 00:00:00'::TIMESTAMP_NS",
+  //   timestamp_ms: "'1990-01-01 00:00:00'::TIMESTAMP_MS",
+  timestamp_tz:
+    "'1990-01-01 00:00:00Z'::TIMESTAMPTZ, '9999-12-31 23:59:59.999999Z'::TIMESTAMPTZ, NULL::TIMESTAMPTZ",
+  date: "'1990-01-01'::DATE, '9999-12-31'::DATE, NULL::DATE",
+  date_array:
+    "[], ['1970-01-01'::DATE, NULL, '0001-01-01'::DATE, '9999-12-31'::DATE,], [NULL::DATE,]",
+  timestamp_array:
+    "[], ['1970-01-01'::TIMESTAMP, NULL, '0001-01-01'::TIMESTAMP, '9999-12-31 23:59:59.999999'::TIMESTAMP,], [NULL::TIMESTAMP,]",
+  timestamptz_array:
+    "[], ['1970-01-01 00:00:00Z'::TIMESTAMPTZ, NULL, '0001-01-01 00:00:00Z'::TIMESTAMPTZ, '9999-12-31 23:59:59.999999Z'::TIMESTAMPTZ,], [NULL::TIMESTAMPTZ,]",
+};
+
+const correct_answer_map: Record<string, any[]> = {
+  bool: [false, true, null],
+
+  tinyint: [-128, 127, null],
+  smallint: [-32768, 32767, null],
+
+  int: [-2147483648, 2147483647, null],
+  bigint: [BigInt("-9223372036854775808"), BigInt("9223372036854775807"), null],
+
+  hugeint: [
+    BigInt("-170141183460469231731687303715884105727"),
+    BigInt("170141183460469231731687303715884105727"),
+    null,
+  ],
+
+  utinyint: [0, 255, null],
+  usmallint: [0, 65535, null],
+
+  uint: [0, 4294967295, null],
+  ubigint: [BigInt(0), BigInt("18446744073709551615"), null],
+
+  time: ["00:00:00", "23:59:59.999999", null],
+
+  float: [-3.4028234663852886e38, 3.4028234663852886e38, null],
+  double: [-1.7976931348623157e308, 1.7976931348623157e308, null],
+
+  dec_4_1: [-999.9, 999.9, null],
+  dec_9_4: [-99999.9999, 99999.9999, null],
+  dec_18_6: ["-999999999999.999999", "999999999999.999999", null],
+  dec38_10: [
+    "-9999999999999999999999999999.9999999999",
+    "9999999999999999999999999999.9999999999",
+    null,
+  ],
+  uuid: [
+    "00000000-0000-0000-0000-000000000001",
+    "ffffffff-ffff-ffff-ffff-ffffffffffff",
+    null,
+  ],
+  varchar: ["🦆🦆🦆🦆🦆🦆", "goo\0se", null],
+  json: ["🦆🦆🦆🦆🦆🦆", "goose", null],
+  blob: [
+    Buffer.from("thisisalongblob\x00withnullbytes"),
+    Buffer.from("\x00\x00\x00a"),
+    null,
+  ],
+  bit: ["0010001001011100010101011010111", "10101", null],
+  small_enum: ["DUCK_DUCK_ENUM", "GOOSE", null],
+  medium_enum: ["enum_0", "enum_299", null],
+  large_enum: ["enum_0", "enum_69999", null],
+  date_array: [
+    [],
+    [
+      new Date(1970, 0, 1),
+      null,
+      new Date("0001-01-01T00:00:00.000Z"),
+      new Date("9999-12-31T00:00:00.000Z"),
+    ],
+    [null],
+  ],
+  timestamp_array: [
+    [],
+    [
+      new Date(1970, 0, 1),
+      null,
+      new Date("0001-01-01T00:00:00.000Z"),
+      new Date("9999-12-31T23:59:59.999Z"),
+    ],
+    [null],
+  ],
+
+  timestamptz_array: [
+    [],
+    [
+      new Date(1970, 0, 1),
+      null,
+      new Date("0001-01-01T00:00:00.000Z"),
+      new Date("9999-12-31T23:59:59.999Z"),
+    ],
+    [null],
+  ],
+
+  int_array: [[], [42, 999, null, null, -42], null],
+  varchar_array: [[], ["🦆🦆🦆🦆🦆🦆", "goose", null, ""], null],
+
+  double_array: [
+    [],
+    [
+      42.0,
+      Number.NaN,
+      Number.POSITIVE_INFINITY,
+      Number.NEGATIVE_INFINITY,
+      null,
+      -42.0,
+    ],
+    null,
+  ],
+
+  nested_int_array: [
+    [],
+    [[], [42, 999, null, null, -42], null, [], [42, 999, null, null, -42]],
+    null,
+  ],
+  struct: [{ a: null, b: null }, { a: 42, b: "🦆🦆🦆🦆🦆🦆" }, null],
+
+  struct_of_arrays: [
+    { a: null, b: null },
+    {
+      a: [42, 999, null, null, -42],
+      b: ["🦆🦆🦆🦆🦆🦆", "goose", null, ""],
+    },
+    null,
+  ],
+
+  array_of_structs: [
+    [],
+    [{ a: null, b: null }, { a: 42, b: "🦆🦆🦆🦆🦆🦆" }, null],
+    null,
+  ],
+  map: ["{}", "{key1=🦆🦆🦆🦆🦆🦆, key2=goose}", null],
+  union: ['Frank', '5', null],
+
+  time_tz: ["00:00:00+00", "23:59:59.999999+00", null],
+  interval: [
+    timedelta({
+      days: 0,
+      months: 0,
+      micros: 0,
+    }),
+    timedelta({ days: 999, months: 999, micros: 999999999 }),
+    null,
+  ],
+
+  timestamp: [
+    new Date("1990-01-01T00:00"),
+    new Date("9999-12-31T23:59:59.000Z"),
+    null,
+  ],
+  date: [new Date("1990-01-01"), new Date("9999-12-31"), null],
+  timestamp_s: ["290309-12-22 (BC) 00:00:00", "294247-01-10 04:00:54", null],
+
+  timestamp_ns: [
+    "1677-09-21 00:12:43.145225",
+    "2262-04-11 23:47:16.854775",
+    null,
+  ],
+  timestamp_ms: [
+    "290309-12-22 (BC) 00:00:00",
+    "294247-01-10 04:00:54.775",
+    null,
+  ],
+  timestamp_tz: [
+    new Date("1990-01-01T00:00:00.000Z"),
+    new Date("9999-12-31T23:59:59.999Z"),
+    null,
+  ],
+};
+
+const suite = describe("test_all_types", () => {
+  before(async function () {
+    const all_types = await get_all_types();
+
+    for (const cur_type of all_types) {
+      // FIXME: these currently have too high a precision to be tested
+      if (["dec_18_6", "dec38_10"].includes(cur_type)) continue;
+
+      suite.addTest(
+        it(cur_type, async () => {
+          const conn = new duckdb.Database(":memory:");
+
+          let query: string;
+          if (cur_type in replacement_values) {
+            query = `select UNNEST([${replacement_values[cur_type]}]) AS ${cur_type}`;
+          } else {
+            query = `select "${cur_type}" from test_all_types()`;
+          }
+
+          let result = await new Promise<any[]>((resolve, reject) =>
+            conn.all(query, (err: DuckDbError | null, data: TableData) =>
+              err ? reject(err) : resolve(data)
+            )
+          );
+
+          result = result.map((row) => row[cur_type]); // pluck values
+
+          const correct_result = correct_answer_map[cur_type];
+
+          expect(result).deep.eq(correct_result);
+        })
+      );
+    }
+  });
+
+  it("dummy", () => {});
+});

package/tsconfig.json

@@ -2,6 +2,7 @@
   "compilerOptions": {
     "allowJs": false,
     "esModuleInterop": true,
+    "lib": ["ES2015"],
     "noImplicitAny": true,
     "strict": true,
     "outDir": "./dist",

package/src/duckdb/src/execution/index/art/prefix_segment.cpp

@@ -1,42 +0,0 @@
-#include "duckdb/execution/index/art/prefix_segment.hpp"
-
-#include "duckdb/execution/index/art/art.hpp"
-#include "duckdb/execution/index/art/node.hpp"
-
-namespace duckdb {
-
-PrefixSegment &PrefixSegment::New(ART &art, Node &node) {
-
-	node.SetPtr(Node::GetAllocator(art, NType::PREFIX_SEGMENT).New());
-	node.type = (uint8_t)NType::PREFIX_SEGMENT;
-
-	auto &segment = PrefixSegment::Get(art, node);
-	segment.next.Reset();
-	return segment;
-}
-
-PrefixSegment &PrefixSegment::Append(ART &art, uint32_t &count, const uint8_t byte) {
-
-	reference<PrefixSegment> segment(*this);
-	auto position = count % Node::PREFIX_SEGMENT_SIZE;
-
-	// we need a new segment
-	if (position == 0 && count != 0) {
-		segment = PrefixSegment::New(art, next);
-	}
-
-	segment.get().bytes[position] = byte;
-	count++;
-	return segment.get();
-}
-
-PrefixSegment &PrefixSegment::GetTail(const ART &art) {
-
-	reference<PrefixSegment> segment(*this);
-	while (segment.get().next.IsSet()) {
-		segment = PrefixSegment::Get(art, segment.get().next);
-	}
-	return segment.get();
-}
-
-} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/index/art/prefix_segment.hpp

@@ -1,40 +0,0 @@
-//===----------------------------------------------------------------------===//
-//                         DuckDB
-//
-// duckdb/execution/index/art/prefix_segment.hpp
-//
-//
-//===----------------------------------------------------------------------===//
-#pragma once
-
-#include "duckdb/execution/index/art/art.hpp"
-#include "duckdb/execution/index/art/node.hpp"
-
-namespace duckdb {
-
-class PrefixSegment {
-public:
-	//! Constructor of an empty prefix segment containing bytes.
-	//! NOTE: only use this constructor for temporary prefix segments
-	PrefixSegment() {};
-
-	//! The prefix bytes stored in this segment
-	uint8_t bytes[Node::PREFIX_SEGMENT_SIZE];
-	//! The position of the next segment, if the prefix exceeds this segment
-	Node next;
-
-public:
-	//! Get a new prefix segment node, might cause a new buffer allocation, and initialize it
-	static PrefixSegment &New(ART &art, Node &node);
-	//! Get a reference to the prefix segment
-	static inline PrefixSegment &Get(const ART &art, const Node ptr) {
-		return *Node::GetAllocator(art, NType::PREFIX_SEGMENT).Get<PrefixSegment>(ptr);
-	}
-
-	//! Append a byte to the current segment, or create a new segment containing that byte
-	PrefixSegment &Append(ART &art, uint32_t &count, const uint8_t byte);
-	//! Get the tail of a list of segments
-	PrefixSegment &GetTail(const ART &art);
-};
-
-} // namespace duckdb