duckdb 0.7.2-dev654.0 → 0.7.2-dev832.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +2 -0
- package/lib/duckdb.d.ts +12 -1
- package/lib/duckdb.js +19 -0
- package/package.json +1 -1
- package/src/duckdb/extension/json/include/json_common.hpp +1 -0
- package/src/duckdb/extension/json/include/json_functions.hpp +2 -0
- package/src/duckdb/extension/json/include/json_serializer.hpp +77 -0
- package/src/duckdb/extension/json/json_functions/json_serialize_sql.cpp +147 -0
- package/src/duckdb/extension/json/json_functions.cpp +12 -4
- package/src/duckdb/extension/json/json_scan.cpp +2 -2
- package/src/duckdb/extension/json/json_serializer.cpp +217 -0
- package/src/duckdb/src/common/enums/expression_type.cpp +8 -222
- package/src/duckdb/src/common/enums/join_type.cpp +3 -22
- package/src/duckdb/src/common/exception.cpp +2 -2
- package/src/duckdb/src/common/serializer/enum_serializer.cpp +1172 -0
- package/src/duckdb/src/common/types/column_data_collection_segment.cpp +11 -6
- package/src/duckdb/src/common/types/value.cpp +117 -0
- package/src/duckdb/src/common/types/vector.cpp +140 -1
- package/src/duckdb/src/common/types.cpp +166 -89
- package/src/duckdb/src/common/vector_operations/vector_cast.cpp +2 -1
- package/src/duckdb/src/execution/aggregate_hashtable.cpp +10 -5
- package/src/duckdb/src/execution/expression_executor/execute_cast.cpp +2 -1
- package/src/duckdb/src/execution/index/art/art.cpp +5 -5
- package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +3 -0
- package/src/duckdb/src/execution/partitionable_hashtable.cpp +14 -2
- package/src/duckdb/src/function/cast/cast_function_set.cpp +1 -1
- package/src/duckdb/src/function/cast/enum_casts.cpp +25 -3
- package/src/duckdb/src/function/cast/list_casts.cpp +17 -4
- package/src/duckdb/src/function/cast/map_cast.cpp +5 -2
- package/src/duckdb/src/function/cast/string_cast.cpp +36 -10
- package/src/duckdb/src/function/cast/struct_cast.cpp +23 -3
- package/src/duckdb/src/function/cast/union_casts.cpp +33 -7
- package/src/duckdb/src/function/scalar/string/regexp/regexp_extract_all.cpp +243 -0
- package/src/duckdb/src/function/scalar/string/regexp/regexp_util.cpp +79 -0
- package/src/duckdb/src/function/scalar/string/regexp.cpp +21 -80
- package/src/duckdb/src/function/table/arrow_conversion.cpp +7 -1
- package/src/duckdb/src/function/table/table_scan.cpp +1 -1
- package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
- package/src/duckdb/src/include/duckdb/common/enums/aggregate_handling.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/enums/expression_type.hpp +2 -3
- package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/enums/order_type.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/enums/set_operation_type.hpp +2 -1
- package/src/duckdb/src/include/duckdb/common/exception.hpp +40 -9
- package/src/duckdb/src/include/duckdb/common/optional_ptr.hpp +45 -0
- package/src/duckdb/src/include/duckdb/common/preserved_error.hpp +3 -0
- package/src/duckdb/src/include/duckdb/common/serializer/enum_serializer.hpp +113 -0
- package/src/duckdb/src/include/duckdb/common/serializer/format_deserializer.hpp +336 -0
- package/src/duckdb/src/include/duckdb/common/serializer/format_serializer.hpp +268 -0
- package/src/duckdb/src/include/duckdb/common/serializer/serialization_traits.hpp +126 -0
- package/src/duckdb/src/include/duckdb/common/string_util.hpp +12 -0
- package/src/duckdb/src/include/duckdb/common/types/value.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/types/vector.hpp +3 -0
- package/src/duckdb/src/include/duckdb/common/types.hpp +8 -2
- package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +1 -0
- package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp +3 -0
- package/src/duckdb/src/include/duckdb/function/cast/bound_cast_data.hpp +84 -0
- package/src/duckdb/src/include/duckdb/function/cast/cast_function_set.hpp +2 -2
- package/src/duckdb/src/include/duckdb/function/cast/default_casts.hpp +28 -64
- package/src/duckdb/src/include/duckdb/function/scalar/regexp.hpp +81 -1
- package/src/duckdb/src/include/duckdb/main/extension_entries.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/bound_expression.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parser/expression/case_expression.hpp +5 -0
- package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parser/expression/conjunction_expression.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/default_expression.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parser/expression/operator_expression.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parser/expression/parameter_expression.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parser/expression/star_expression.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parser/expression/subquery_expression.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +5 -0
- package/src/duckdb/src/include/duckdb/parser/parsed_data/sample_options.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parser/parsed_expression.hpp +5 -0
- package/src/duckdb/src/include/duckdb/parser/query_node/recursive_cte_node.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/query_node/select_node.hpp +5 -0
- package/src/duckdb/src/include/duckdb/parser/query_node/set_operation_node.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/query_node.hpp +11 -1
- package/src/duckdb/src/include/duckdb/parser/result_modifier.hpp +24 -1
- package/src/duckdb/src/include/duckdb/parser/sql_statement.hpp +2 -1
- package/src/duckdb/src/include/duckdb/parser/statement/select_statement.hpp +6 -1
- package/src/duckdb/src/include/duckdb/parser/tableref/basetableref.hpp +4 -0
- package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parser/tableref/expressionlistref.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/tableref/pivotref.hpp +9 -0
- package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/tableref/table_function_ref.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/tableref.hpp +3 -1
- package/src/duckdb/src/include/duckdb/storage/data_table.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/index.hpp +4 -3
- package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -2
- package/src/duckdb/src/main/extension/extension_install.cpp +7 -2
- package/src/duckdb/src/optimizer/deliminator.cpp +1 -1
- package/src/duckdb/src/optimizer/filter_combiner.cpp +1 -1
- package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +3 -3
- package/src/duckdb/src/optimizer/rule/move_constants.cpp +2 -2
- package/src/duckdb/src/optimizer/statistics/operator/propagate_filter.cpp +1 -1
- package/src/duckdb/src/parser/common_table_expression_info.cpp +19 -0
- package/src/duckdb/src/parser/expression/between_expression.cpp +17 -0
- package/src/duckdb/src/parser/expression/case_expression.cpp +28 -0
- package/src/duckdb/src/parser/expression/cast_expression.cpp +17 -0
- package/src/duckdb/src/parser/expression/collate_expression.cpp +16 -0
- package/src/duckdb/src/parser/expression/columnref_expression.cpp +15 -0
- package/src/duckdb/src/parser/expression/comparison_expression.cpp +16 -0
- package/src/duckdb/src/parser/expression/conjunction_expression.cpp +15 -0
- package/src/duckdb/src/parser/expression/constant_expression.cpp +14 -0
- package/src/duckdb/src/parser/expression/default_expression.cpp +7 -0
- package/src/duckdb/src/parser/expression/function_expression.cpp +35 -0
- package/src/duckdb/src/parser/expression/lambda_expression.cpp +16 -0
- package/src/duckdb/src/parser/expression/operator_expression.cpp +15 -0
- package/src/duckdb/src/parser/expression/parameter_expression.cpp +15 -0
- package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +14 -0
- package/src/duckdb/src/parser/expression/star_expression.cpp +20 -0
- package/src/duckdb/src/parser/expression/subquery_expression.cpp +20 -0
- package/src/duckdb/src/parser/expression/window_expression.cpp +43 -0
- package/src/duckdb/src/parser/parsed_data/sample_options.cpp +22 -10
- package/src/duckdb/src/parser/parsed_expression.cpp +72 -0
- package/src/duckdb/src/parser/query_node/recursive_cte_node.cpp +21 -0
- package/src/duckdb/src/parser/query_node/select_node.cpp +31 -0
- package/src/duckdb/src/parser/query_node/set_operation_node.cpp +17 -0
- package/src/duckdb/src/parser/query_node.cpp +50 -0
- package/src/duckdb/src/parser/result_modifier.cpp +78 -0
- package/src/duckdb/src/parser/statement/select_statement.cpp +12 -0
- package/src/duckdb/src/parser/tableref/basetableref.cpp +21 -0
- package/src/duckdb/src/parser/tableref/emptytableref.cpp +4 -0
- package/src/duckdb/src/parser/tableref/expressionlistref.cpp +17 -0
- package/src/duckdb/src/parser/tableref/joinref.cpp +25 -0
- package/src/duckdb/src/parser/tableref/pivotref.cpp +53 -0
- package/src/duckdb/src/parser/tableref/subqueryref.cpp +15 -0
- package/src/duckdb/src/parser/tableref/table_function.cpp +17 -0
- package/src/duckdb/src/parser/tableref.cpp +46 -0
- package/src/duckdb/src/parser/transform/expression/transform_bool_expr.cpp +1 -1
- package/src/duckdb/src/parser/transform/expression/transform_function.cpp +1 -1
- package/src/duckdb/src/parser/transform/expression/transform_operator.cpp +1 -1
- package/src/duckdb/src/parser/transform/expression/transform_subquery.cpp +1 -1
- package/src/duckdb/src/planner/binder/expression/bind_subquery_expression.cpp +4 -0
- package/src/duckdb/src/planner/binder/statement/bind_copy.cpp +3 -1
- package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +1 -1
- package/src/duckdb/src/planner/expression/bound_expression.cpp +4 -0
- package/src/duckdb/src/storage/data_table.cpp +15 -13
- package/src/duckdb/src/storage/index.cpp +12 -1
- package/src/duckdb/src/storage/local_storage.cpp +20 -23
- package/src/duckdb/src/verification/deserialized_statement_verifier.cpp +0 -1
- package/src/duckdb/third_party/re2/re2/re2.cc +9 -0
- package/src/duckdb/third_party/re2/re2/re2.h +2 -0
- package/src/duckdb/ub_extension_json_json_functions.cpp +2 -0
- package/src/duckdb/ub_src_common_serializer.cpp +2 -0
- package/src/duckdb/ub_src_function_scalar_string_regexp.cpp +4 -0
- package/src/duckdb/ub_src_parser.cpp +2 -0
- package/src/utils.cpp +12 -0
- package/test/extension.test.ts +44 -26
@@ -1,6 +1,8 @@
|
|
1
1
|
#include "duckdb/parser/tableref/table_function_ref.hpp"
|
2
2
|
#include "duckdb/common/vector.hpp"
|
3
3
|
#include "duckdb/common/field_writer.hpp"
|
4
|
+
#include "duckdb/common/serializer/format_serializer.hpp"
|
5
|
+
#include "duckdb/common/serializer/format_deserializer.hpp"
|
4
6
|
|
5
7
|
namespace duckdb {
|
6
8
|
|
@@ -25,6 +27,21 @@ void TableFunctionRef::Serialize(FieldWriter &writer) const {
|
|
25
27
|
writer.WriteList<string>(column_name_alias);
|
26
28
|
}
|
27
29
|
|
30
|
+
void TableFunctionRef::FormatSerialize(FormatSerializer &serializer) const {
|
31
|
+
TableRef::FormatSerialize(serializer);
|
32
|
+
serializer.WriteProperty("function", function);
|
33
|
+
serializer.WriteProperty("alias", alias);
|
34
|
+
serializer.WriteProperty("column_name_alias", column_name_alias);
|
35
|
+
}
|
36
|
+
|
37
|
+
unique_ptr<TableRef> TableFunctionRef::FormatDeserialize(FormatDeserializer &deserializer) {
|
38
|
+
auto result = make_unique<TableFunctionRef>();
|
39
|
+
deserializer.ReadProperty("function", result->function);
|
40
|
+
deserializer.ReadProperty("alias", result->alias);
|
41
|
+
deserializer.ReadProperty("column_name_alias", result->column_name_alias);
|
42
|
+
return std::move(result);
|
43
|
+
}
|
44
|
+
|
28
45
|
unique_ptr<TableRef> TableFunctionRef::Deserialize(FieldReader &reader) {
|
29
46
|
auto result = make_unique<TableFunctionRef>();
|
30
47
|
result->function = reader.ReadRequiredSerializable<ParsedExpression>();
|
@@ -3,6 +3,8 @@
|
|
3
3
|
#include "duckdb/common/printer.hpp"
|
4
4
|
#include "duckdb/common/field_writer.hpp"
|
5
5
|
#include "duckdb/parser/tableref/list.hpp"
|
6
|
+
#include "duckdb/common/serializer/format_serializer.hpp"
|
7
|
+
#include "duckdb/common/serializer/format_deserializer.hpp"
|
6
8
|
#include "duckdb/common/to_string.hpp"
|
7
9
|
|
8
10
|
namespace duckdb {
|
@@ -52,6 +54,50 @@ void TableRef::Serialize(Serializer &serializer) const {
|
|
52
54
|
writer.Finalize();
|
53
55
|
}
|
54
56
|
|
57
|
+
void TableRef::FormatSerialize(FormatSerializer &serializer) const {
|
58
|
+
serializer.WriteProperty("type", type);
|
59
|
+
serializer.WriteProperty("alias", alias);
|
60
|
+
serializer.WriteOptionalProperty("sample", sample);
|
61
|
+
}
|
62
|
+
|
63
|
+
unique_ptr<TableRef> TableRef::FormatDeserialize(FormatDeserializer &deserializer) {
|
64
|
+
auto type = deserializer.ReadProperty<TableReferenceType>("type");
|
65
|
+
auto alias = deserializer.ReadProperty<string>("alias");
|
66
|
+
auto sample = deserializer.ReadOptionalProperty<unique_ptr<SampleOptions>>("sample");
|
67
|
+
|
68
|
+
unique_ptr<TableRef> result;
|
69
|
+
|
70
|
+
switch (type) {
|
71
|
+
case TableReferenceType::BASE_TABLE:
|
72
|
+
result = BaseTableRef::FormatDeserialize(deserializer);
|
73
|
+
break;
|
74
|
+
case TableReferenceType::JOIN:
|
75
|
+
result = JoinRef::FormatDeserialize(deserializer);
|
76
|
+
break;
|
77
|
+
case TableReferenceType::SUBQUERY:
|
78
|
+
result = SubqueryRef::FormatDeserialize(deserializer);
|
79
|
+
break;
|
80
|
+
case TableReferenceType::TABLE_FUNCTION:
|
81
|
+
result = TableFunctionRef::FormatDeserialize(deserializer);
|
82
|
+
break;
|
83
|
+
case TableReferenceType::EMPTY:
|
84
|
+
result = EmptyTableRef::FormatDeserialize(deserializer);
|
85
|
+
break;
|
86
|
+
case TableReferenceType::EXPRESSION_LIST:
|
87
|
+
result = ExpressionListRef::FormatDeserialize(deserializer);
|
88
|
+
break;
|
89
|
+
case TableReferenceType::PIVOT:
|
90
|
+
result = PivotRef::FormatDeserialize(deserializer);
|
91
|
+
break;
|
92
|
+
case TableReferenceType::CTE:
|
93
|
+
case TableReferenceType::INVALID:
|
94
|
+
throw InternalException("Unsupported type for TableRef::FormatDeserialize");
|
95
|
+
}
|
96
|
+
result->alias = alias;
|
97
|
+
result->sample = std::move(sample);
|
98
|
+
return result;
|
99
|
+
}
|
100
|
+
|
55
101
|
unique_ptr<TableRef> TableRef::Deserialize(Deserializer &source) {
|
56
102
|
FieldReader reader(source);
|
57
103
|
|
@@ -37,7 +37,7 @@ unique_ptr<ParsedExpression> Transformer::TransformBoolExpr(duckdb_libpgquery::P
|
|
37
37
|
next->type <= ExpressionType::COMPARE_GREATERTHANOREQUALTO) {
|
38
38
|
// NOT on a comparison: we can negate the comparison
|
39
39
|
// e.g. NOT(x > y) is equivalent to x <= y
|
40
|
-
next->type =
|
40
|
+
next->type = NegateComparisonExpression(next->type);
|
41
41
|
result = std::move(next);
|
42
42
|
} else {
|
43
43
|
result = make_unique<OperatorExpression>(ExpressionType::OPERATOR_NOT, std::move(next));
|
@@ -134,7 +134,7 @@ unique_ptr<ParsedExpression> Transformer::TransformFuncCall(duckdb_libpgquery::P
|
|
134
134
|
schema = INVALID_SCHEMA;
|
135
135
|
function_name = reinterpret_cast<duckdb_libpgquery::PGValue *>(name->head->data.ptr_value)->val.str;
|
136
136
|
} else {
|
137
|
-
throw
|
137
|
+
throw ParserException("TransformFuncCall - Expected 1, 2 or 3 qualifications");
|
138
138
|
}
|
139
139
|
|
140
140
|
// transform children
|
@@ -84,7 +84,7 @@ unique_ptr<ParsedExpression> Transformer::TransformAExprInternal(duckdb_libpgque
|
|
84
84
|
// ALL sublink is equivalent to NOT(ANY) with inverted comparison
|
85
85
|
// e.g. [= ALL()] is equivalent to [NOT(<> ANY())]
|
86
86
|
// first invert the comparison type
|
87
|
-
subquery_expr->comparison_type =
|
87
|
+
subquery_expr->comparison_type = NegateComparisonExpression(subquery_expr->comparison_type);
|
88
88
|
return make_unique<OperatorExpression>(ExpressionType::OPERATOR_NOT, std::move(subquery_expr));
|
89
89
|
}
|
90
90
|
return std::move(subquery_expr);
|
@@ -44,7 +44,7 @@ unique_ptr<ParsedExpression> Transformer::TransformSubquery(duckdb_libpgquery::P
|
|
44
44
|
// ALL sublink is equivalent to NOT(ANY) with inverted comparison
|
45
45
|
// e.g. [= ALL()] is equivalent to [NOT(<> ANY())]
|
46
46
|
// first invert the comparison type
|
47
|
-
subquery_expr->comparison_type =
|
47
|
+
subquery_expr->comparison_type = NegateComparisonExpression(subquery_expr->comparison_type);
|
48
48
|
return make_unique<OperatorExpression>(ExpressionType::OPERATOR_NOT, std::move(subquery_expr));
|
49
49
|
}
|
50
50
|
break;
|
@@ -32,6 +32,10 @@ public:
|
|
32
32
|
void Serialize(FieldWriter &writer) const override {
|
33
33
|
throw InternalException("Cannot serialize bound subquery node");
|
34
34
|
}
|
35
|
+
|
36
|
+
void FormatSerialize(FormatSerializer &serializer) const override {
|
37
|
+
throw InternalException("Cannot serialize bound subquery node");
|
38
|
+
}
|
35
39
|
};
|
36
40
|
|
37
41
|
BindResult ExpressionBinder::BindExpression(SubqueryExpression &expr, idx_t depth) {
|
@@ -158,7 +158,9 @@ BoundStatement Binder::BindCopyFrom(CopyStatement &stmt) {
|
|
158
158
|
result.types = {LogicalType::BIGINT};
|
159
159
|
result.names = {"Count"};
|
160
160
|
|
161
|
-
|
161
|
+
if (stmt.info->table.empty()) {
|
162
|
+
throw ParserException("COPY FROM requires a table name to be specified");
|
163
|
+
}
|
162
164
|
// COPY FROM a file
|
163
165
|
// generate an insert statement for the the to-be-inserted table
|
164
166
|
InsertStatement insert;
|
@@ -33,7 +33,7 @@ static bool CreateJoinCondition(Expression &expr, const unordered_set<idx_t> &le
|
|
33
33
|
if (left_side == JoinSide::RIGHT) {
|
34
34
|
// left = right, right = left, flip the comparison symbol and reverse sides
|
35
35
|
swap(left, right);
|
36
|
-
condition.comparison =
|
36
|
+
condition.comparison = FlipComparisonExpression(expr.type);
|
37
37
|
}
|
38
38
|
condition.left = std::move(left);
|
39
39
|
condition.right = std::move(right);
|
@@ -28,4 +28,8 @@ void BoundExpression::Serialize(FieldWriter &writer) const {
|
|
28
28
|
throw SerializationException("Cannot copy or serialize bound expression");
|
29
29
|
}
|
30
30
|
|
31
|
+
void BoundExpression::FormatSerialize(FormatSerializer &serializer) const {
|
32
|
+
throw SerializationException("Cannot copy or serialize bound expression");
|
33
|
+
}
|
34
|
+
|
31
35
|
} // namespace duckdb
|
@@ -819,9 +819,10 @@ void DataTable::RevertAppend(idx_t start_row, idx_t count) {
|
|
819
819
|
//===--------------------------------------------------------------------===//
|
820
820
|
// Indexes
|
821
821
|
//===--------------------------------------------------------------------===//
|
822
|
-
|
822
|
+
PreservedError DataTable::AppendToIndexes(TableIndexList &indexes, DataChunk &chunk, row_t row_start) {
|
823
|
+
PreservedError error;
|
823
824
|
if (indexes.Empty()) {
|
824
|
-
return
|
825
|
+
return error;
|
825
826
|
}
|
826
827
|
// first generate the vector of row identifiers
|
827
828
|
Vector row_identifiers(LogicalType::ROW_TYPE);
|
@@ -832,11 +833,13 @@ bool DataTable::AppendToIndexes(TableIndexList &indexes, DataChunk &chunk, row_t
|
|
832
833
|
// now append the entries to the indices
|
833
834
|
indexes.Scan([&](Index &index) {
|
834
835
|
try {
|
835
|
-
|
836
|
-
|
837
|
-
|
838
|
-
|
839
|
-
|
836
|
+
error = index.Append(chunk, row_identifiers);
|
837
|
+
} catch (Exception &ex) {
|
838
|
+
error = PreservedError(ex);
|
839
|
+
} catch (std::exception &ex) {
|
840
|
+
error = PreservedError(ex);
|
841
|
+
}
|
842
|
+
if (error) {
|
840
843
|
append_failed = true;
|
841
844
|
return true;
|
842
845
|
}
|
@@ -850,12 +853,11 @@ bool DataTable::AppendToIndexes(TableIndexList &indexes, DataChunk &chunk, row_t
|
|
850
853
|
for (auto *index : already_appended) {
|
851
854
|
index->Delete(chunk, row_identifiers);
|
852
855
|
}
|
853
|
-
return false;
|
854
856
|
}
|
855
|
-
return
|
857
|
+
return error;
|
856
858
|
}
|
857
859
|
|
858
|
-
|
860
|
+
PreservedError DataTable::AppendToIndexes(DataChunk &chunk, row_t row_start) {
|
859
861
|
D_ASSERT(is_root);
|
860
862
|
return AppendToIndexes(info->indexes, chunk, row_start);
|
861
863
|
}
|
@@ -1204,9 +1206,9 @@ void DataTable::WALAddIndex(ClientContext &context, unique_ptr<Index> index,
|
|
1204
1206
|
index->ExecuteExpressions(intermediate, result);
|
1205
1207
|
|
1206
1208
|
// insert into the index
|
1207
|
-
|
1208
|
-
|
1209
|
-
|
1209
|
+
auto error = index->Insert(lock, result, intermediate.data[intermediate.ColumnCount() - 1]);
|
1210
|
+
if (error) {
|
1211
|
+
throw InternalException("Error during WAL replay: %s", error.Message());
|
1210
1212
|
}
|
1211
1213
|
}
|
1212
1214
|
}
|
@@ -36,7 +36,7 @@ void Index::InitializeLock(IndexLock &state) {
|
|
36
36
|
state.index_lock = unique_lock<mutex>(lock);
|
37
37
|
}
|
38
38
|
|
39
|
-
|
39
|
+
PreservedError Index::Append(DataChunk &entries, Vector &row_identifiers) {
|
40
40
|
IndexLock state;
|
41
41
|
InitializeLock(state);
|
42
42
|
return Append(state, entries, row_identifiers);
|
@@ -90,4 +90,15 @@ BlockPointer Index::Serialize(MetaBlockWriter &writer) {
|
|
90
90
|
throw NotImplementedException("The implementation of this index serialization does not exist.");
|
91
91
|
}
|
92
92
|
|
93
|
+
string Index::AppendRowError(DataChunk &input, idx_t index) {
|
94
|
+
string error;
|
95
|
+
for (idx_t c = 0; c < input.ColumnCount(); c++) {
|
96
|
+
if (c > 0) {
|
97
|
+
error += ", ";
|
98
|
+
}
|
99
|
+
error += input.GetValue(c, index).ToString();
|
100
|
+
}
|
101
|
+
return error;
|
102
|
+
}
|
103
|
+
|
93
104
|
} // namespace duckdb
|
@@ -197,16 +197,16 @@ void LocalTableStorage::FlushToDisk() {
|
|
197
197
|
optimistic_writer.FinalFlush();
|
198
198
|
}
|
199
199
|
|
200
|
-
|
201
|
-
|
202
|
-
|
200
|
+
PreservedError LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, RowGroupCollection &source,
|
201
|
+
TableIndexList &index_list, const vector<LogicalType> &table_types,
|
202
|
+
row_t &start_row) {
|
203
203
|
// only need to scan for index append
|
204
204
|
// figure out which columns we need to scan for the set of indexes
|
205
205
|
auto columns = index_list.GetRequiredColumns();
|
206
206
|
// create an empty mock chunk that contains all the correct types for the table
|
207
207
|
DataChunk mock_chunk;
|
208
208
|
mock_chunk.InitializeEmpty(table_types);
|
209
|
-
|
209
|
+
PreservedError error;
|
210
210
|
source.Scan(transaction, columns, [&](DataChunk &chunk) -> bool {
|
211
211
|
// construct the mock chunk by referencing the required columns
|
212
212
|
for (idx_t i = 0; i < columns.size(); i++) {
|
@@ -214,28 +214,28 @@ bool LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, RowGroupCo
|
|
214
214
|
}
|
215
215
|
mock_chunk.SetCardinality(chunk);
|
216
216
|
// append this chunk to the indexes of the table
|
217
|
-
|
218
|
-
|
217
|
+
error = DataTable::AppendToIndexes(index_list, mock_chunk, start_row);
|
218
|
+
if (error) {
|
219
219
|
return false;
|
220
220
|
}
|
221
221
|
start_row += chunk.size();
|
222
222
|
return true;
|
223
223
|
});
|
224
|
-
return
|
224
|
+
return error;
|
225
225
|
}
|
226
226
|
|
227
227
|
void LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, TableAppendState &append_state,
|
228
228
|
idx_t append_count, bool append_to_table) {
|
229
|
-
bool constraint_violated = false;
|
230
229
|
if (append_to_table) {
|
231
230
|
table->InitializeAppend(transaction, append_state, append_count);
|
232
231
|
}
|
232
|
+
PreservedError error;
|
233
233
|
if (append_to_table) {
|
234
234
|
// appending: need to scan entire
|
235
235
|
row_groups->Scan(transaction, [&](DataChunk &chunk) -> bool {
|
236
236
|
// append this chunk to the indexes of the table
|
237
|
-
|
238
|
-
|
237
|
+
error = table->AppendToIndexes(chunk, append_state.current_row);
|
238
|
+
if (error) {
|
239
239
|
return false;
|
240
240
|
}
|
241
241
|
// append to base table
|
@@ -243,11 +243,10 @@ void LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, TableAppen
|
|
243
243
|
return true;
|
244
244
|
});
|
245
245
|
} else {
|
246
|
-
|
247
|
-
|
246
|
+
error = AppendToIndexes(transaction, *row_groups, table->info->indexes, table->GetTypes(),
|
247
|
+
append_state.current_row);
|
248
248
|
}
|
249
|
-
if (
|
250
|
-
PreservedError error;
|
249
|
+
if (error) {
|
251
250
|
// need to revert the append
|
252
251
|
row_t current_row = append_state.row_start;
|
253
252
|
// remove the data from the indexes, if there are any indexes
|
@@ -273,10 +272,7 @@ void LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, TableAppen
|
|
273
272
|
if (append_to_table) {
|
274
273
|
table->RevertAppendInternal(append_state.row_start, append_count);
|
275
274
|
}
|
276
|
-
|
277
|
-
error.Throw();
|
278
|
-
}
|
279
|
-
throw ConstraintException("PRIMARY KEY or UNIQUE constraint violated: duplicated key");
|
275
|
+
error.Throw();
|
280
276
|
}
|
281
277
|
}
|
282
278
|
|
@@ -412,8 +408,9 @@ void LocalStorage::Append(LocalAppendState &state, DataChunk &chunk) {
|
|
412
408
|
// append to unique indices (if any)
|
413
409
|
auto storage = state.storage;
|
414
410
|
idx_t base_id = MAX_ROW_ID + storage->row_groups->GetTotalRows() + state.append_state.total_append_count;
|
415
|
-
|
416
|
-
|
411
|
+
auto error = DataTable::AppendToIndexes(storage->indexes, chunk, base_id);
|
412
|
+
if (error) {
|
413
|
+
error.Throw();
|
417
414
|
}
|
418
415
|
|
419
416
|
//! Append the chunk to the local storage
|
@@ -434,9 +431,9 @@ void LocalStorage::LocalMerge(DataTable *table, RowGroupCollection &collection)
|
|
434
431
|
if (!storage->indexes.Empty()) {
|
435
432
|
// append data to indexes if required
|
436
433
|
row_t base_id = MAX_ROW_ID + storage->row_groups->GetTotalRows();
|
437
|
-
|
438
|
-
if (
|
439
|
-
|
434
|
+
auto error = storage->AppendToIndexes(transaction, collection, storage->indexes, table->GetTypes(), base_id);
|
435
|
+
if (error) {
|
436
|
+
error.Throw();
|
440
437
|
}
|
441
438
|
}
|
442
439
|
storage->row_groups->MergeStorage(collection);
|
@@ -566,6 +566,15 @@ static int ascii_strcasecmp(const char* a, const char* b, size_t len) {
|
|
566
566
|
return 0;
|
567
567
|
}
|
568
568
|
|
569
|
+
RE2::Anchor RE2::Anchored() const {
|
570
|
+
if (prog_->anchor_start()) {
|
571
|
+
if (prog_->anchor_end()) {
|
572
|
+
return Anchor::ANCHOR_BOTH;
|
573
|
+
}
|
574
|
+
return Anchor::ANCHOR_START;
|
575
|
+
}
|
576
|
+
return Anchor::UNANCHORED;
|
577
|
+
}
|
569
578
|
|
570
579
|
/***** Actual matching and rewriting code *****/
|
571
580
|
|
@@ -475,6 +475,8 @@ class RE2 {
|
|
475
475
|
ANCHOR_BOTH // Anchor at start and end
|
476
476
|
};
|
477
477
|
|
478
|
+
Anchor Anchored() const;
|
479
|
+
|
478
480
|
// Return the number of capturing subpatterns, or -1 if the
|
479
481
|
// regexp wasn't valid on construction. The overall match ($0)
|
480
482
|
// does not count: if the regexp is "(a)(b)", returns 2.
|
@@ -20,6 +20,8 @@
|
|
20
20
|
|
21
21
|
#include "extension/json/json_functions/json_valid.cpp"
|
22
22
|
|
23
|
+
#include "extension/json/json_functions/json_serialize_sql.cpp"
|
24
|
+
|
23
25
|
#include "extension/json/json_functions/read_json.cpp"
|
24
26
|
|
25
27
|
#include "extension/json/json_functions/read_json_objects.cpp"
|
package/src/utils.cpp
CHANGED
@@ -18,6 +18,18 @@ static void SetString(Napi::Object &obj, const std::string &key, const std::stri
|
|
18
18
|
|
19
19
|
Napi::Object Utils::CreateError(Napi::Env env, duckdb::PreservedError &error) {
|
20
20
|
auto obj = Utils::CreateError(env, error.Message());
|
21
|
+
if (error.Type() == duckdb::ExceptionType::HTTP) {
|
22
|
+
const auto &e = error.GetError()->AsHTTPException();
|
23
|
+
obj.Set(Napi::String::New(env, "statusCode"), Napi::Number::New(env, e.GetStatusCode()));
|
24
|
+
SetString(obj, "response", e.GetResponseBody());
|
25
|
+
SetString(obj, "reason", e.GetReason());
|
26
|
+
|
27
|
+
auto headers = Napi::Object::New(env);
|
28
|
+
for (const auto &item : e.GetHeaders()) {
|
29
|
+
SetString(headers, item.first, item.second);
|
30
|
+
}
|
31
|
+
obj.Set(Napi::String::New(env, "headers"), headers);
|
32
|
+
}
|
21
33
|
|
22
34
|
SetString(obj, "errorType", duckdb::Exception::ExceptionTypeToString(error.Type()));
|
23
35
|
|
package/test/extension.test.ts
CHANGED
@@ -1,9 +1,8 @@
|
|
1
1
|
import * as duckdb from '..';
|
2
|
-
import {Database, TableData} from '..';
|
2
|
+
import {Database, DuckDbError, HttpError, TableData} from '..';
|
3
3
|
import * as fs from 'fs';
|
4
4
|
import * as assert from 'assert';
|
5
5
|
import * as path from 'path';
|
6
|
-
import {Done} from "mocha";
|
7
6
|
|
8
7
|
const extension_base_path = "../../../build/release/extension";
|
9
8
|
|
@@ -21,46 +20,63 @@ if (fs.existsSync(extension_full_path)) {
|
|
21
20
|
}).filter(a => a) as string[];
|
22
21
|
}
|
23
22
|
|
23
|
+
function isHTTPException(err: DuckDbError): err is HttpError {
|
24
|
+
return err.errorType === 'HTTP';
|
25
|
+
}
|
26
|
+
|
24
27
|
// Note: test will pass on http request failing due to connection issues.
|
25
|
-
const test_httpfs = async function (db: duckdb.Database
|
26
|
-
db.all("SELECT id, first_name, last_name FROM PARQUET_SCAN('https://raw.githubusercontent.com/cwida/duckdb/master/data/parquet-testing/userdata1.parquet') LIMIT 3;", function(err: null | Error, rows: TableData) {
|
28
|
+
const test_httpfs = async function (db: duckdb.Database) {
|
29
|
+
await new Promise<void>((resolve, reject) => db.all("SELECT id, first_name, last_name FROM PARQUET_SCAN('https://raw.githubusercontent.com/cwida/duckdb/master/data/parquet-testing/userdata1.parquet') LIMIT 3;", function (err: null | Error, rows: TableData) {
|
27
30
|
if (err) {
|
28
31
|
if (err.message.startsWith("Unable to connect to URL")) {
|
29
32
|
console.warn("Warning: HTTP request failed in extension.test.js");
|
30
|
-
|
33
|
+
resolve();
|
31
34
|
} else {
|
32
|
-
|
35
|
+
reject(err);
|
33
36
|
}
|
34
37
|
} else {
|
35
38
|
assert.deepEqual(rows, [
|
36
|
-
{
|
37
|
-
{
|
38
|
-
{
|
39
|
+
{id: 1, first_name: 'Amanda', last_name: 'Jordan'},
|
40
|
+
{id: 2, first_name: 'Albert', last_name: 'Freeman'},
|
41
|
+
{id: 3, first_name: 'Evelyn', last_name: 'Morgan'},
|
39
42
|
]);
|
40
|
-
|
43
|
+
resolve();
|
41
44
|
}
|
42
|
-
});
|
45
|
+
}));
|
46
|
+
|
47
|
+
await new Promise<void>((resolve) => {
|
48
|
+
db.exec("select * from read_csv_auto('https://example.com/hello.csv')", (err: DuckDbError | null) => {
|
49
|
+
assert.ok(err);
|
50
|
+
assert.ok(isHTTPException(err));
|
51
|
+
if (isHTTPException(err)) {
|
52
|
+
assert.equal(err.statusCode, 404);
|
53
|
+
assert.equal(err.reason, 'Not Found');
|
54
|
+
assert.equal(err.response, '');
|
55
|
+
assert.ok('Content-Length' in err.headers, JSON.stringify(err.headers));
|
56
|
+
}
|
57
|
+
resolve();
|
58
|
+
});
|
59
|
+
})
|
43
60
|
};
|
44
61
|
|
45
|
-
const test_tpch = async function (db: Database
|
46
|
-
db.all("CALL DBGEN(sf=0.01);", function(err: null | Error) {
|
62
|
+
const test_tpch = async function (db: Database) {
|
63
|
+
await new Promise<void>((resolve, reject) => db.all("CALL DBGEN(sf=0.01);", function (err: null | Error) {
|
47
64
|
if (err) {
|
48
|
-
|
65
|
+
reject(err);
|
49
66
|
}
|
50
|
-
|
51
|
-
});
|
67
|
+
resolve();
|
68
|
+
}));
|
52
69
|
};
|
53
70
|
|
54
|
-
const test_extension = function(extension_name: string, db: duckdb.Database
|
55
|
-
switch(extension_name) {
|
71
|
+
const test_extension = async function (extension_name: string, db: duckdb.Database) {
|
72
|
+
switch (extension_name) {
|
56
73
|
case 'httpfs.duckdb_extension':
|
57
|
-
test_httpfs(db
|
74
|
+
await test_httpfs(db);
|
58
75
|
break;
|
59
76
|
case 'tpch.duckdb_extension':
|
60
|
-
test_tpch(db
|
77
|
+
await test_tpch(db);
|
61
78
|
break;
|
62
79
|
default:
|
63
|
-
done();
|
64
80
|
break;
|
65
81
|
}
|
66
82
|
};
|
@@ -79,13 +95,15 @@ describe('Extension loading', function() {
|
|
79
95
|
continue;
|
80
96
|
}
|
81
97
|
|
82
|
-
it(extension_name, function(
|
83
|
-
db.run(`LOAD '${extension_path}';`, function(err: null | Error) {
|
98
|
+
it(extension_name, async function () {
|
99
|
+
await new Promise<void>((resolve, reject) => db.run(`LOAD '${extension_path}';`, function (err: null | Error) {
|
84
100
|
if (err) {
|
85
|
-
|
101
|
+
reject(err);
|
86
102
|
}
|
87
|
-
|
88
|
-
});
|
103
|
+
resolve()
|
104
|
+
}));
|
105
|
+
|
106
|
+
await test_extension(extension_name, db);
|
89
107
|
});
|
90
108
|
}
|
91
109
|
});
|