duckdb 0.8.2-dev2809.0 → 0.8.2-dev2850.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/duckdb/extension/json/include/json_deserializer.hpp +1 -1
- package/src/duckdb/extension/json/include/json_serializer.hpp +1 -1
- package/src/duckdb/extension/json/json_deserializer.cpp +16 -14
- package/src/duckdb/extension/json/json_scan.cpp +2 -2
- package/src/duckdb/extension/json/json_serializer.cpp +11 -11
- package/src/duckdb/extension/json/serialize_json.cpp +44 -44
- package/src/duckdb/extension/parquet/parquet_extension.cpp +11 -10
- package/src/duckdb/extension/parquet/serialize_parquet.cpp +6 -6
- package/src/duckdb/src/common/extra_type_info.cpp +2 -2
- package/src/duckdb/src/common/serializer/binary_deserializer.cpp +5 -3
- package/src/duckdb/src/common/serializer/binary_serializer.cpp +10 -5
- package/src/duckdb/src/common/types/column/column_data_collection.cpp +4 -4
- package/src/duckdb/src/common/types/value.cpp +33 -33
- package/src/duckdb/src/common/types/vector.cpp +20 -20
- package/src/duckdb/src/core_functions/aggregate/holistic/approximate_quantile.cpp +2 -2
- package/src/duckdb/src/core_functions/aggregate/holistic/quantile.cpp +6 -6
- package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +4 -4
- package/src/duckdb/src/core_functions/scalar/list/list_lambdas.cpp +4 -4
- package/src/duckdb/src/function/table/read_csv.cpp +4 -4
- package/src/duckdb/src/function/table/table_scan.cpp +14 -14
- package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
- package/src/duckdb/src/include/duckdb/common/index_vector.hpp +2 -2
- package/src/duckdb/src/include/duckdb/common/serializer/binary_deserializer.hpp +7 -3
- package/src/duckdb/src/include/duckdb/common/serializer/binary_serializer.hpp +2 -1
- package/src/duckdb/src/include/duckdb/common/serializer/format_deserializer.hpp +18 -17
- package/src/duckdb/src/include/duckdb/common/serializer/format_serializer.hpp +10 -9
- package/src/duckdb/src/include/duckdb/common/serializer/serialization_traits.hpp +4 -0
- package/src/duckdb/src/include/duckdb/function/function_serialization.hpp +10 -10
- package/src/duckdb/src/include/duckdb/main/relation/aggregate_relation.hpp +4 -1
- package/src/duckdb/src/include/duckdb/parser/group_by_node.hpp +11 -0
- package/src/duckdb/src/include/duckdb/parser/parser.hpp +4 -0
- package/src/duckdb/src/main/relation/aggregate_relation.cpp +20 -10
- package/src/duckdb/src/main/relation.cpp +4 -4
- package/src/duckdb/src/parser/parser.cpp +18 -3
- package/src/duckdb/src/parser/tableref/pivotref.cpp +6 -6
- package/src/duckdb/src/parser/transform/constraint/transform_constraint.cpp +55 -38
- package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +10 -10
- package/src/duckdb/src/planner/expression/bound_function_expression.cpp +6 -6
- package/src/duckdb/src/planner/expression/bound_window_expression.cpp +24 -24
- package/src/duckdb/src/planner/operator/logical_extension_operator.cpp +2 -2
- package/src/duckdb/src/planner/operator/logical_get.cpp +22 -22
- package/src/duckdb/src/storage/serialization/serialize_constraint.cpp +26 -26
- package/src/duckdb/src/storage/serialization/serialize_create_info.cpp +66 -66
- package/src/duckdb/src/storage/serialization/serialize_expression.cpp +78 -78
- package/src/duckdb/src/storage/serialization/serialize_logical_operator.cpp +250 -250
- package/src/duckdb/src/storage/serialization/serialize_macro_function.cpp +10 -10
- package/src/duckdb/src/storage/serialization/serialize_nodes.cpp +206 -206
- package/src/duckdb/src/storage/serialization/serialize_parse_info.cpp +116 -116
- package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +110 -110
- package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +48 -48
- package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +16 -16
- package/src/duckdb/src/storage/serialization/serialize_statement.cpp +2 -2
- package/src/duckdb/src/storage/serialization/serialize_table_filter.cpp +10 -10
- package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +54 -54
- package/src/duckdb/src/storage/serialization/serialize_types.cpp +22 -22
- package/src/duckdb/src/storage/table/update_segment.cpp +1 -1
package/package.json
CHANGED
@@ -48,7 +48,7 @@ private:
|
|
48
48
|
void ThrowTypeError(yyjson_val *val, const char *expected);
|
49
49
|
|
50
50
|
// Set the 'tag' of the property to read
|
51
|
-
void SetTag(const char *tag) final;
|
51
|
+
void SetTag(const field_id_t, const char *tag) final;
|
52
52
|
|
53
53
|
//===--------------------------------------------------------------------===//
|
54
54
|
// Nested Types Hooks
|
@@ -3,7 +3,7 @@
|
|
3
3
|
|
4
4
|
namespace duckdb {
|
5
5
|
|
6
|
-
void JsonDeserializer::SetTag(const char *tag) {
|
6
|
+
void JsonDeserializer::SetTag(const field_id_t, const char *tag) {
|
7
7
|
current_tag = tag;
|
8
8
|
}
|
9
9
|
|
@@ -43,8 +43,10 @@ void JsonDeserializer::ThrowTypeError(yyjson_val *val, const char *expected) {
|
|
43
43
|
if (yyjson_is_obj(parent.val)) {
|
44
44
|
auto msg =
|
45
45
|
StringUtil::Format("property '%s' expected type '%s', but got type: '%s'", current_tag, expected, actual);
|
46
|
+
throw ParserException(msg);
|
46
47
|
} else if (yyjson_is_arr(parent.val)) {
|
47
48
|
auto msg = StringUtil::Format("Sequence expect child of type '%s', but got type: %s", expected, actual);
|
49
|
+
throw ParserException(msg);
|
48
50
|
} else {
|
49
51
|
// unreachable?
|
50
52
|
throw InternalException("cannot get nested value from non object or array-type");
|
@@ -122,11 +124,11 @@ void JsonDeserializer::OnMapEntryBegin() {
|
|
122
124
|
}
|
123
125
|
|
124
126
|
void JsonDeserializer::OnMapKeyBegin() {
|
125
|
-
SetTag("key");
|
127
|
+
SetTag(100, "key");
|
126
128
|
}
|
127
129
|
|
128
130
|
void JsonDeserializer::OnMapValueBegin() {
|
129
|
-
SetTag("value");
|
131
|
+
SetTag(101, "value");
|
130
132
|
}
|
131
133
|
|
132
134
|
void JsonDeserializer::OnMapEntryEnd() {
|
@@ -146,11 +148,11 @@ void JsonDeserializer::OnPairBegin() {
|
|
146
148
|
}
|
147
149
|
|
148
150
|
void JsonDeserializer::OnPairKeyBegin() {
|
149
|
-
SetTag("key");
|
151
|
+
SetTag(100, "key");
|
150
152
|
}
|
151
153
|
|
152
154
|
void JsonDeserializer::OnPairValueBegin() {
|
153
|
-
SetTag("value");
|
155
|
+
SetTag(101, "value");
|
154
156
|
}
|
155
157
|
|
156
158
|
void JsonDeserializer::OnPairEnd() {
|
@@ -178,7 +180,7 @@ bool JsonDeserializer::ReadBool() {
|
|
178
180
|
|
179
181
|
int8_t JsonDeserializer::ReadSignedInt8() {
|
180
182
|
auto val = GetNextValue();
|
181
|
-
if (!
|
183
|
+
if (!yyjson_is_int(val)) {
|
182
184
|
ThrowTypeError(val, "int8_t");
|
183
185
|
}
|
184
186
|
return yyjson_get_sint(val);
|
@@ -194,7 +196,7 @@ uint8_t JsonDeserializer::ReadUnsignedInt8() {
|
|
194
196
|
|
195
197
|
int16_t JsonDeserializer::ReadSignedInt16() {
|
196
198
|
auto val = GetNextValue();
|
197
|
-
if (!
|
199
|
+
if (!yyjson_is_int(val)) {
|
198
200
|
ThrowTypeError(val, "int16_t");
|
199
201
|
}
|
200
202
|
return yyjson_get_sint(val);
|
@@ -210,7 +212,7 @@ uint16_t JsonDeserializer::ReadUnsignedInt16() {
|
|
210
212
|
|
211
213
|
int32_t JsonDeserializer::ReadSignedInt32() {
|
212
214
|
auto val = GetNextValue();
|
213
|
-
if (!
|
215
|
+
if (!yyjson_is_int(val)) {
|
214
216
|
ThrowTypeError(val, "int32_t");
|
215
217
|
}
|
216
218
|
return yyjson_get_sint(val);
|
@@ -226,7 +228,7 @@ uint32_t JsonDeserializer::ReadUnsignedInt32() {
|
|
226
228
|
|
227
229
|
int64_t JsonDeserializer::ReadSignedInt64() {
|
228
230
|
auto val = GetNextValue();
|
229
|
-
if (!
|
231
|
+
if (!yyjson_is_int(val)) {
|
230
232
|
ThrowTypeError(val, "int64_t");
|
231
233
|
}
|
232
234
|
return yyjson_get_sint(val);
|
@@ -271,9 +273,9 @@ interval_t JsonDeserializer::ReadInterval() {
|
|
271
273
|
}
|
272
274
|
Push(val);
|
273
275
|
interval_t result;
|
274
|
-
ReadProperty("months", result.months);
|
275
|
-
ReadProperty("days", result.days);
|
276
|
-
ReadProperty("micros", result.micros);
|
276
|
+
ReadProperty(100, "months", result.months);
|
277
|
+
ReadProperty(101, "days", result.days);
|
278
|
+
ReadProperty(102, "micros", result.micros);
|
277
279
|
Pop();
|
278
280
|
return result;
|
279
281
|
}
|
@@ -285,8 +287,8 @@ hugeint_t JsonDeserializer::ReadHugeInt() {
|
|
285
287
|
}
|
286
288
|
Push(val);
|
287
289
|
hugeint_t result;
|
288
|
-
ReadProperty("upper", result.upper);
|
289
|
-
ReadProperty("lower", result.lower);
|
290
|
+
ReadProperty(100, "upper", result.upper);
|
291
|
+
ReadProperty(101, "lower", result.lower);
|
290
292
|
Pop();
|
291
293
|
return result;
|
292
294
|
}
|
@@ -999,12 +999,12 @@ unique_ptr<FunctionData> JSONScan::Deserialize(PlanDeserializationState &state,
|
|
999
999
|
void JSONScan::FormatSerialize(FormatSerializer &serializer, const optional_ptr<FunctionData> bind_data_p,
|
1000
1000
|
const TableFunction &function) {
|
1001
1001
|
auto &bind_data = bind_data_p->Cast<JSONScanData>();
|
1002
|
-
serializer.WriteProperty("scan_data", bind_data);
|
1002
|
+
serializer.WriteProperty(100, "scan_data", bind_data);
|
1003
1003
|
}
|
1004
1004
|
|
1005
1005
|
unique_ptr<FunctionData> JSONScan::FormatDeserialize(FormatDeserializer &deserializer, TableFunction &function) {
|
1006
1006
|
unique_ptr<JSONScanData> result;
|
1007
|
-
deserializer.ReadProperty("scan_data", result);
|
1007
|
+
deserializer.ReadProperty(100, "scan_data", result);
|
1008
1008
|
return std::move(result);
|
1009
1009
|
}
|
1010
1010
|
|
@@ -19,7 +19,7 @@ void JsonSerializer::PushValue(yyjson_mut_val *val) {
|
|
19
19
|
}
|
20
20
|
}
|
21
21
|
|
22
|
-
void JsonSerializer::SetTag(const char *tag) {
|
22
|
+
void JsonSerializer::SetTag(const field_id_t, const char *tag) {
|
23
23
|
current_tag = yyjson_mut_strcpy(doc, tag);
|
24
24
|
}
|
25
25
|
|
@@ -63,11 +63,11 @@ void JsonSerializer::OnMapEntryBegin() {
|
|
63
63
|
}
|
64
64
|
|
65
65
|
void JsonSerializer::OnMapKeyBegin() {
|
66
|
-
SetTag("key");
|
66
|
+
SetTag(100, "key");
|
67
67
|
}
|
68
68
|
|
69
69
|
void JsonSerializer::OnMapValueBegin() {
|
70
|
-
SetTag("value");
|
70
|
+
SetTag(101, "value");
|
71
71
|
}
|
72
72
|
|
73
73
|
void JsonSerializer::OnMapEntryEnd() {
|
@@ -127,11 +127,11 @@ void JsonSerializer::OnPairBegin() {
|
|
127
127
|
}
|
128
128
|
|
129
129
|
void JsonSerializer::OnPairKeyBegin() {
|
130
|
-
SetTag("key");
|
130
|
+
SetTag(100, "key");
|
131
131
|
}
|
132
132
|
|
133
133
|
void JsonSerializer::OnPairValueBegin() {
|
134
|
-
SetTag("value");
|
134
|
+
SetTag(101, "value");
|
135
135
|
}
|
136
136
|
|
137
137
|
void JsonSerializer::OnPairEnd() {
|
@@ -172,7 +172,7 @@ void JsonSerializer::WriteValue(uint32_t value) {
|
|
172
172
|
}
|
173
173
|
|
174
174
|
void JsonSerializer::WriteValue(int32_t value) {
|
175
|
-
auto val =
|
175
|
+
auto val = yyjson_mut_sint(doc, value);
|
176
176
|
PushValue(val);
|
177
177
|
}
|
178
178
|
|
@@ -190,8 +190,8 @@ void JsonSerializer::WriteValue(hugeint_t value) {
|
|
190
190
|
auto val = yyjson_mut_obj(doc);
|
191
191
|
PushValue(val);
|
192
192
|
stack.push_back(val);
|
193
|
-
WriteProperty("upper", value.upper);
|
194
|
-
WriteProperty("lower", value.lower);
|
193
|
+
WriteProperty(100, "upper", value.upper);
|
194
|
+
WriteProperty(101, "lower", value.lower);
|
195
195
|
stack.pop_back();
|
196
196
|
}
|
197
197
|
|
@@ -209,9 +209,9 @@ void JsonSerializer::WriteValue(interval_t value) {
|
|
209
209
|
auto val = yyjson_mut_obj(doc);
|
210
210
|
PushValue(val);
|
211
211
|
stack.push_back(val);
|
212
|
-
WriteProperty("months", value.months);
|
213
|
-
WriteProperty("days", value.days);
|
214
|
-
WriteProperty("micros", value.micros);
|
212
|
+
WriteProperty(100, "months", value.months);
|
213
|
+
WriteProperty(101, "days", value.days);
|
214
|
+
WriteProperty(102, "micros", value.micros);
|
215
215
|
stack.pop_back();
|
216
216
|
}
|
217
217
|
|
@@ -12,51 +12,51 @@
|
|
12
12
|
namespace duckdb {
|
13
13
|
|
14
14
|
void BufferedJSONReaderOptions::FormatSerialize(FormatSerializer &serializer) const {
|
15
|
-
serializer.WriteProperty("format", format);
|
16
|
-
serializer.WriteProperty("record_type", record_type);
|
17
|
-
serializer.WriteProperty("compression", compression);
|
18
|
-
serializer.WriteProperty("file_options", file_options);
|
15
|
+
serializer.WriteProperty(100, "format", format);
|
16
|
+
serializer.WriteProperty(101, "record_type", record_type);
|
17
|
+
serializer.WriteProperty(102, "compression", compression);
|
18
|
+
serializer.WriteProperty(103, "file_options", file_options);
|
19
19
|
}
|
20
20
|
|
21
21
|
BufferedJSONReaderOptions BufferedJSONReaderOptions::FormatDeserialize(FormatDeserializer &deserializer) {
|
22
22
|
BufferedJSONReaderOptions result;
|
23
|
-
deserializer.ReadProperty("format", result.format);
|
24
|
-
deserializer.ReadProperty("record_type", result.record_type);
|
25
|
-
deserializer.ReadProperty("compression", result.compression);
|
26
|
-
deserializer.ReadProperty("file_options", result.file_options);
|
23
|
+
deserializer.ReadProperty(100, "format", result.format);
|
24
|
+
deserializer.ReadProperty(101, "record_type", result.record_type);
|
25
|
+
deserializer.ReadProperty(102, "compression", result.compression);
|
26
|
+
deserializer.ReadProperty(103, "file_options", result.file_options);
|
27
27
|
return result;
|
28
28
|
}
|
29
29
|
|
30
30
|
void JSONScanData::FormatSerialize(FormatSerializer &serializer) const {
|
31
|
-
serializer.WriteProperty("json_type", type);
|
32
|
-
serializer.WriteProperty("options", options);
|
33
|
-
serializer.WriteProperty("reader_bind", reader_bind);
|
34
|
-
serializer.WriteProperty("files", files);
|
35
|
-
serializer.WriteProperty("ignore_errors", ignore_errors);
|
36
|
-
serializer.WriteProperty("maximum_object_size", maximum_object_size);
|
37
|
-
serializer.WriteProperty("auto_detect", auto_detect);
|
38
|
-
serializer.WriteProperty("sample_size", sample_size);
|
39
|
-
serializer.WriteProperty("max_depth", max_depth);
|
40
|
-
serializer.WriteProperty("transform_options", transform_options);
|
41
|
-
serializer.WriteProperty("names", names);
|
42
|
-
serializer.WriteProperty("date_format", GetDateFormat());
|
43
|
-
serializer.WriteProperty("timestamp_format", GetTimestampFormat());
|
31
|
+
serializer.WriteProperty(100, "json_type", type);
|
32
|
+
serializer.WriteProperty(101, "options", options);
|
33
|
+
serializer.WriteProperty(102, "reader_bind", reader_bind);
|
34
|
+
serializer.WriteProperty(103, "files", files);
|
35
|
+
serializer.WriteProperty(104, "ignore_errors", ignore_errors);
|
36
|
+
serializer.WriteProperty(105, "maximum_object_size", maximum_object_size);
|
37
|
+
serializer.WriteProperty(106, "auto_detect", auto_detect);
|
38
|
+
serializer.WriteProperty(107, "sample_size", sample_size);
|
39
|
+
serializer.WriteProperty(108, "max_depth", max_depth);
|
40
|
+
serializer.WriteProperty(109, "transform_options", transform_options);
|
41
|
+
serializer.WriteProperty(110, "names", names);
|
42
|
+
serializer.WriteProperty(111, "date_format", GetDateFormat());
|
43
|
+
serializer.WriteProperty(112, "timestamp_format", GetTimestampFormat());
|
44
44
|
}
|
45
45
|
|
46
46
|
unique_ptr<JSONScanData> JSONScanData::FormatDeserialize(FormatDeserializer &deserializer) {
|
47
|
-
auto type = deserializer.ReadProperty<JSONScanType>("json_type");
|
48
|
-
auto options = deserializer.ReadProperty<BufferedJSONReaderOptions>("options");
|
49
|
-
auto reader_bind = deserializer.ReadProperty<MultiFileReaderBindData>("reader_bind");
|
50
|
-
auto files = deserializer.ReadProperty<vector<string>>("files");
|
51
|
-
auto ignore_errors = deserializer.ReadProperty<bool>("ignore_errors");
|
52
|
-
auto maximum_object_size = deserializer.ReadProperty<idx_t>("maximum_object_size");
|
53
|
-
auto auto_detect = deserializer.ReadProperty<bool>("auto_detect");
|
54
|
-
auto sample_size = deserializer.ReadProperty<idx_t>("sample_size");
|
55
|
-
auto max_depth = deserializer.ReadProperty<idx_t>("max_depth");
|
56
|
-
auto transform_options = deserializer.ReadProperty<JSONTransformOptions>("transform_options");
|
57
|
-
auto names = deserializer.ReadProperty<vector<string>>("names");
|
58
|
-
auto date_format = deserializer.ReadProperty<string>("date_format");
|
59
|
-
auto timestamp_format = deserializer.ReadProperty<string>("timestamp_format");
|
47
|
+
auto type = deserializer.ReadProperty<JSONScanType>(100, "json_type");
|
48
|
+
auto options = deserializer.ReadProperty<BufferedJSONReaderOptions>(101, "options");
|
49
|
+
auto reader_bind = deserializer.ReadProperty<MultiFileReaderBindData>(102, "reader_bind");
|
50
|
+
auto files = deserializer.ReadProperty<vector<string>>(103, "files");
|
51
|
+
auto ignore_errors = deserializer.ReadProperty<bool>(104, "ignore_errors");
|
52
|
+
auto maximum_object_size = deserializer.ReadProperty<idx_t>(105, "maximum_object_size");
|
53
|
+
auto auto_detect = deserializer.ReadProperty<bool>(106, "auto_detect");
|
54
|
+
auto sample_size = deserializer.ReadProperty<idx_t>(107, "sample_size");
|
55
|
+
auto max_depth = deserializer.ReadProperty<idx_t>(108, "max_depth");
|
56
|
+
auto transform_options = deserializer.ReadProperty<JSONTransformOptions>(109, "transform_options");
|
57
|
+
auto names = deserializer.ReadProperty<vector<string>>(110, "names");
|
58
|
+
auto date_format = deserializer.ReadProperty<string>(111, "date_format");
|
59
|
+
auto timestamp_format = deserializer.ReadProperty<string>(112, "timestamp_format");
|
60
60
|
auto result = duckdb::unique_ptr<JSONScanData>(new JSONScanData(deserializer.Get<ClientContext &>(), std::move(files), std::move(date_format), std::move(timestamp_format)));
|
61
61
|
result->type = type;
|
62
62
|
result->options = options;
|
@@ -72,20 +72,20 @@ unique_ptr<JSONScanData> JSONScanData::FormatDeserialize(FormatDeserializer &des
|
|
72
72
|
}
|
73
73
|
|
74
74
|
void JSONTransformOptions::FormatSerialize(FormatSerializer &serializer) const {
|
75
|
-
serializer.WriteProperty("strict_cast", strict_cast);
|
76
|
-
serializer.WriteProperty("error_duplicate_key", error_duplicate_key);
|
77
|
-
serializer.WriteProperty("error_missing_key", error_missing_key);
|
78
|
-
serializer.WriteProperty("error_unknown_key", error_unknown_key);
|
79
|
-
serializer.WriteProperty("delay_error", delay_error);
|
75
|
+
serializer.WriteProperty(100, "strict_cast", strict_cast);
|
76
|
+
serializer.WriteProperty(101, "error_duplicate_key", error_duplicate_key);
|
77
|
+
serializer.WriteProperty(102, "error_missing_key", error_missing_key);
|
78
|
+
serializer.WriteProperty(103, "error_unknown_key", error_unknown_key);
|
79
|
+
serializer.WriteProperty(104, "delay_error", delay_error);
|
80
80
|
}
|
81
81
|
|
82
82
|
JSONTransformOptions JSONTransformOptions::FormatDeserialize(FormatDeserializer &deserializer) {
|
83
83
|
JSONTransformOptions result;
|
84
|
-
deserializer.ReadProperty("strict_cast", result.strict_cast);
|
85
|
-
deserializer.ReadProperty("error_duplicate_key", result.error_duplicate_key);
|
86
|
-
deserializer.ReadProperty("error_missing_key", result.error_missing_key);
|
87
|
-
deserializer.ReadProperty("error_unknown_key", result.error_unknown_key);
|
88
|
-
deserializer.ReadProperty("delay_error", result.delay_error);
|
84
|
+
deserializer.ReadProperty(100, "strict_cast", result.strict_cast);
|
85
|
+
deserializer.ReadProperty(101, "error_duplicate_key", result.error_duplicate_key);
|
86
|
+
deserializer.ReadProperty(102, "error_missing_key", result.error_missing_key);
|
87
|
+
deserializer.ReadProperty(103, "error_unknown_key", result.error_unknown_key);
|
88
|
+
deserializer.ReadProperty(104, "delay_error", result.delay_error);
|
89
89
|
return result;
|
90
90
|
}
|
91
91
|
|
@@ -201,8 +201,9 @@ public:
|
|
201
201
|
|
202
202
|
for (auto &option : info.options) {
|
203
203
|
auto loption = StringUtil::Lower(option.first);
|
204
|
-
if (loption == "compression" || loption == "codec") {
|
205
|
-
// CODEC
|
204
|
+
if (loption == "compression" || loption == "codec" || loption == "row_group_size") {
|
205
|
+
// CODEC/COMPRESSION and ROW_GROUP_SIZE options have no effect on parquet read.
|
206
|
+
// These options are determined from the file.
|
206
207
|
continue;
|
207
208
|
} else if (loption == "binary_as_string") {
|
208
209
|
parquet_options.binary_as_string = true;
|
@@ -437,19 +438,19 @@ public:
|
|
437
438
|
static void ParquetScanFormatSerialize(FormatSerializer &serializer, const optional_ptr<FunctionData> bind_data_p,
|
438
439
|
const TableFunction &function) {
|
439
440
|
auto &bind_data = bind_data_p->Cast<ParquetReadBindData>();
|
440
|
-
serializer.WriteProperty("files", bind_data.files);
|
441
|
-
serializer.WriteProperty("types", bind_data.types);
|
442
|
-
serializer.WriteProperty("names", bind_data.names);
|
443
|
-
serializer.WriteProperty("parquet_options", bind_data.parquet_options);
|
441
|
+
serializer.WriteProperty(100, "files", bind_data.files);
|
442
|
+
serializer.WriteProperty(101, "types", bind_data.types);
|
443
|
+
serializer.WriteProperty(102, "names", bind_data.names);
|
444
|
+
serializer.WriteProperty(103, "parquet_options", bind_data.parquet_options);
|
444
445
|
}
|
445
446
|
|
446
447
|
static unique_ptr<FunctionData> ParquetScanFormatDeserialize(FormatDeserializer &deserializer,
|
447
448
|
TableFunction &function) {
|
448
449
|
auto &context = deserializer.Get<ClientContext &>();
|
449
|
-
auto files = deserializer.ReadProperty<vector<string>>("files");
|
450
|
-
auto types = deserializer.ReadProperty<vector<LogicalType>>("types");
|
451
|
-
auto names = deserializer.ReadProperty<vector<string>>("names");
|
452
|
-
auto parquet_options = deserializer.ReadProperty<ParquetOptions>("parquet_options");
|
450
|
+
auto files = deserializer.ReadProperty<vector<string>>(100, "files");
|
451
|
+
auto types = deserializer.ReadProperty<vector<LogicalType>>(101, "types");
|
452
|
+
auto names = deserializer.ReadProperty<vector<string>>(102, "names");
|
453
|
+
auto parquet_options = deserializer.ReadProperty<ParquetOptions>(103, "parquet_options");
|
453
454
|
return ParquetScanBindInternal(context, files, types, names, parquet_options);
|
454
455
|
}
|
455
456
|
|
@@ -10,16 +10,16 @@
|
|
10
10
|
namespace duckdb {
|
11
11
|
|
12
12
|
void ParquetOptions::FormatSerialize(FormatSerializer &serializer) const {
|
13
|
-
serializer.WriteProperty("binary_as_string", binary_as_string);
|
14
|
-
serializer.WriteProperty("file_row_number", file_row_number);
|
15
|
-
serializer.WriteProperty("file_options", file_options);
|
13
|
+
serializer.WriteProperty(100, "binary_as_string", binary_as_string);
|
14
|
+
serializer.WriteProperty(101, "file_row_number", file_row_number);
|
15
|
+
serializer.WriteProperty(102, "file_options", file_options);
|
16
16
|
}
|
17
17
|
|
18
18
|
ParquetOptions ParquetOptions::FormatDeserialize(FormatDeserializer &deserializer) {
|
19
19
|
ParquetOptions result;
|
20
|
-
deserializer.ReadProperty("binary_as_string", result.binary_as_string);
|
21
|
-
deserializer.ReadProperty("file_row_number", result.file_row_number);
|
22
|
-
deserializer.ReadProperty("file_options", result.file_options);
|
20
|
+
deserializer.ReadProperty(100, "binary_as_string", result.binary_as_string);
|
21
|
+
deserializer.ReadProperty(101, "file_row_number", result.file_row_number);
|
22
|
+
deserializer.ReadProperty(102, "file_options", result.file_options);
|
23
23
|
return result;
|
24
24
|
}
|
25
25
|
|
@@ -426,7 +426,7 @@ shared_ptr<ExtraTypeInfo> EnumTypeInfo::Deserialize(FieldReader &reader) {
|
|
426
426
|
}
|
427
427
|
|
428
428
|
shared_ptr<ExtraTypeInfo> EnumTypeInfo::FormatDeserialize(FormatDeserializer &deserializer) {
|
429
|
-
auto enum_size = deserializer.ReadProperty<idx_t>("enum_size");
|
429
|
+
auto enum_size = deserializer.ReadProperty<idx_t>(200, "enum_size");
|
430
430
|
auto enum_internal_type = EnumTypeInfo::DictType(enum_size);
|
431
431
|
switch (enum_internal_type) {
|
432
432
|
case PhysicalType::UINT8:
|
@@ -476,7 +476,7 @@ void EnumTypeInfo::Serialize(FieldWriter &writer) const {
|
|
476
476
|
|
477
477
|
void EnumTypeInfo::FormatSerialize(FormatSerializer &serializer) const {
|
478
478
|
ExtraTypeInfo::FormatSerialize(serializer);
|
479
|
-
serializer.WriteProperty("dict_size", dict_size);
|
479
|
+
serializer.WriteProperty(200, "dict_size", dict_size);
|
480
480
|
((Vector &)values_insert_order).FormatSerialize(serializer, dict_size); // NOLINT - FIXME
|
481
481
|
}
|
482
482
|
|
@@ -2,7 +2,8 @@
|
|
2
2
|
|
3
3
|
namespace duckdb {
|
4
4
|
|
5
|
-
void BinaryDeserializer::SetTag(const char *tag) {
|
5
|
+
void BinaryDeserializer::SetTag(const field_id_t field_id, const char *tag) {
|
6
|
+
current_field_id = field_id;
|
6
7
|
current_tag = tag;
|
7
8
|
stack.back().read_field_count++;
|
8
9
|
if (stack.back().read_field_count > stack.back().expected_field_count) {
|
@@ -14,12 +15,13 @@ void BinaryDeserializer::SetTag(const char *tag) {
|
|
14
15
|
// Nested Types Hooks
|
15
16
|
//===--------------------------------------------------------------------===//
|
16
17
|
void BinaryDeserializer::OnObjectBegin() {
|
18
|
+
auto expected_field_id = ReadPrimitive<field_id_t>();
|
17
19
|
auto expected_field_count = ReadPrimitive<uint32_t>();
|
18
20
|
auto expected_size = ReadPrimitive<uint64_t>();
|
19
21
|
D_ASSERT(expected_field_count > 0);
|
20
22
|
D_ASSERT(expected_size > 0);
|
21
|
-
|
22
|
-
stack.emplace_back(expected_field_count, expected_size);
|
23
|
+
D_ASSERT(expected_field_id == current_field_id);
|
24
|
+
stack.emplace_back(expected_field_count, expected_size, expected_field_id);
|
23
25
|
}
|
24
26
|
|
25
27
|
void BinaryDeserializer::OnObjectEnd() {
|
@@ -2,9 +2,9 @@
|
|
2
2
|
|
3
3
|
namespace duckdb {
|
4
4
|
|
5
|
-
void BinarySerializer::SetTag(const char *tag) {
|
5
|
+
void BinarySerializer::SetTag(const field_id_t field_id, const char *tag) {
|
6
|
+
current_field_id = field_id;
|
6
7
|
current_tag = tag;
|
7
|
-
|
8
8
|
// Increment the number of fields
|
9
9
|
stack.back().field_count++;
|
10
10
|
}
|
@@ -45,6 +45,9 @@ void BinarySerializer::OnMapEnd(idx_t count) {
|
|
45
45
|
|
46
46
|
void BinarySerializer::OnObjectBegin() {
|
47
47
|
stack.push_back(State({0, 0, data.size()}));
|
48
|
+
// Store the field id
|
49
|
+
Write<field_id_t>(current_field_id);
|
50
|
+
// Store the offset so we can patch the field count and size later
|
48
51
|
Write<uint32_t>(0); // Placeholder for the field count
|
49
52
|
Write<uint64_t>(0); // Placeholder for the size
|
50
53
|
}
|
@@ -52,9 +55,11 @@ void BinarySerializer::OnObjectBegin() {
|
|
52
55
|
void BinarySerializer::OnObjectEnd() {
|
53
56
|
auto &frame = stack.back();
|
54
57
|
// Patch the field count and size
|
55
|
-
auto
|
56
|
-
|
57
|
-
Store<
|
58
|
+
auto ptr = &data[frame.offset];
|
59
|
+
ptr += sizeof(field_id_t); // Skip the field id
|
60
|
+
Store<uint32_t>(frame.field_count, ptr);
|
61
|
+
ptr += sizeof(uint32_t); // Skip the field count
|
62
|
+
Store<uint64_t>(frame.size, ptr);
|
58
63
|
stack.pop_back();
|
59
64
|
}
|
60
65
|
|
@@ -1069,13 +1069,13 @@ void ColumnDataCollection::FormatSerialize(FormatSerializer &serializer) const {
|
|
1069
1069
|
}
|
1070
1070
|
}
|
1071
1071
|
}
|
1072
|
-
serializer.WriteProperty("types", types);
|
1073
|
-
serializer.WriteProperty("values", values);
|
1072
|
+
serializer.WriteProperty(100, "types", types);
|
1073
|
+
serializer.WriteProperty(101, "values", values);
|
1074
1074
|
}
|
1075
1075
|
|
1076
1076
|
unique_ptr<ColumnDataCollection> ColumnDataCollection::FormatDeserialize(FormatDeserializer &deserializer) {
|
1077
|
-
auto types = deserializer.ReadProperty<vector<LogicalType>>("types");
|
1078
|
-
auto values = deserializer.ReadProperty<vector<vector<Value>>>("values");
|
1077
|
+
auto types = deserializer.ReadProperty<vector<LogicalType>>(100, "types");
|
1078
|
+
auto values = deserializer.ReadProperty<vector<vector<Value>>>(101, "values");
|
1079
1079
|
|
1080
1080
|
auto collection = make_uniq<ColumnDataCollection>(Allocator::DefaultAllocator(), types);
|
1081
1081
|
if (values.empty()) {
|
@@ -1763,55 +1763,55 @@ Value Value::Deserialize(Deserializer &main_source) {
|
|
1763
1763
|
}
|
1764
1764
|
|
1765
1765
|
void Value::FormatSerialize(FormatSerializer &serializer) const {
|
1766
|
-
serializer.WriteProperty("type", type_);
|
1767
|
-
serializer.WriteProperty("is_null", is_null);
|
1766
|
+
serializer.WriteProperty(100, "type", type_);
|
1767
|
+
serializer.WriteProperty(101, "is_null", is_null);
|
1768
1768
|
if (!IsNull()) {
|
1769
1769
|
switch (type_.InternalType()) {
|
1770
1770
|
case PhysicalType::BOOL:
|
1771
|
-
serializer.WriteProperty("value", value_.boolean);
|
1771
|
+
serializer.WriteProperty(100, "value", value_.boolean);
|
1772
1772
|
break;
|
1773
1773
|
case PhysicalType::INT8:
|
1774
|
-
serializer.WriteProperty("value", value_.tinyint);
|
1774
|
+
serializer.WriteProperty(100, "value", value_.tinyint);
|
1775
1775
|
break;
|
1776
1776
|
case PhysicalType::INT16:
|
1777
|
-
serializer.WriteProperty("value", value_.smallint);
|
1777
|
+
serializer.WriteProperty(100, "value", value_.smallint);
|
1778
1778
|
break;
|
1779
1779
|
case PhysicalType::INT32:
|
1780
|
-
serializer.WriteProperty("value", value_.integer);
|
1780
|
+
serializer.WriteProperty(100, "value", value_.integer);
|
1781
1781
|
break;
|
1782
1782
|
case PhysicalType::INT64:
|
1783
|
-
serializer.WriteProperty("value", value_.bigint);
|
1783
|
+
serializer.WriteProperty(100, "value", value_.bigint);
|
1784
1784
|
break;
|
1785
1785
|
case PhysicalType::UINT8:
|
1786
|
-
serializer.WriteProperty("value", value_.utinyint);
|
1786
|
+
serializer.WriteProperty(100, "value", value_.utinyint);
|
1787
1787
|
break;
|
1788
1788
|
case PhysicalType::UINT16:
|
1789
|
-
serializer.WriteProperty("value", value_.usmallint);
|
1789
|
+
serializer.WriteProperty(100, "value", value_.usmallint);
|
1790
1790
|
break;
|
1791
1791
|
case PhysicalType::UINT32:
|
1792
|
-
serializer.WriteProperty("value", value_.uinteger);
|
1792
|
+
serializer.WriteProperty(100, "value", value_.uinteger);
|
1793
1793
|
break;
|
1794
1794
|
case PhysicalType::UINT64:
|
1795
|
-
serializer.WriteProperty("value", value_.ubigint);
|
1795
|
+
serializer.WriteProperty(100, "value", value_.ubigint);
|
1796
1796
|
break;
|
1797
1797
|
case PhysicalType::INT128:
|
1798
|
-
serializer.WriteProperty("value", value_.hugeint);
|
1798
|
+
serializer.WriteProperty(100, "value", value_.hugeint);
|
1799
1799
|
break;
|
1800
1800
|
case PhysicalType::FLOAT:
|
1801
|
-
serializer.WriteProperty("value", value_.float_);
|
1801
|
+
serializer.WriteProperty(100, "value", value_.float_);
|
1802
1802
|
break;
|
1803
1803
|
case PhysicalType::DOUBLE:
|
1804
|
-
serializer.WriteProperty("value", value_.double_);
|
1804
|
+
serializer.WriteProperty(100, "value", value_.double_);
|
1805
1805
|
break;
|
1806
1806
|
case PhysicalType::INTERVAL:
|
1807
|
-
serializer.WriteProperty("value", value_.interval);
|
1807
|
+
serializer.WriteProperty(100, "value", value_.interval);
|
1808
1808
|
break;
|
1809
1809
|
case PhysicalType::VARCHAR:
|
1810
1810
|
if (type_.id() == LogicalTypeId::BLOB) {
|
1811
1811
|
auto blob_str = Blob::ToString(StringValue::Get(*this));
|
1812
|
-
serializer.WriteProperty("value", blob_str);
|
1812
|
+
serializer.WriteProperty(100, "value", blob_str);
|
1813
1813
|
} else {
|
1814
|
-
serializer.WriteProperty("value", StringValue::Get(*this));
|
1814
|
+
serializer.WriteProperty(100, "value", StringValue::Get(*this));
|
1815
1815
|
}
|
1816
1816
|
break;
|
1817
1817
|
default: {
|
@@ -1824,8 +1824,8 @@ void Value::FormatSerialize(FormatSerializer &serializer) const {
|
|
1824
1824
|
}
|
1825
1825
|
|
1826
1826
|
Value Value::FormatDeserialize(FormatDeserializer &deserializer) {
|
1827
|
-
auto type = deserializer.ReadProperty<LogicalType>("type");
|
1828
|
-
auto is_null = deserializer.ReadProperty<bool>("is_null");
|
1827
|
+
auto type = deserializer.ReadProperty<LogicalType>(100, "type");
|
1828
|
+
auto is_null = deserializer.ReadProperty<bool>(101, "is_null");
|
1829
1829
|
Value new_value = Value(type);
|
1830
1830
|
if (is_null) {
|
1831
1831
|
return new_value;
|
@@ -1833,46 +1833,46 @@ Value Value::FormatDeserialize(FormatDeserializer &deserializer) {
|
|
1833
1833
|
new_value.is_null = false;
|
1834
1834
|
switch (type.InternalType()) {
|
1835
1835
|
case PhysicalType::BOOL:
|
1836
|
-
new_value.value_.boolean = deserializer.ReadProperty<bool>("value");
|
1836
|
+
new_value.value_.boolean = deserializer.ReadProperty<bool>(100, "value");
|
1837
1837
|
break;
|
1838
1838
|
case PhysicalType::UINT8:
|
1839
|
-
new_value.value_.utinyint = deserializer.ReadProperty<uint8_t>("value");
|
1839
|
+
new_value.value_.utinyint = deserializer.ReadProperty<uint8_t>(100, "value");
|
1840
1840
|
break;
|
1841
1841
|
case PhysicalType::INT8:
|
1842
|
-
new_value.value_.tinyint = deserializer.ReadProperty<int8_t>("value");
|
1842
|
+
new_value.value_.tinyint = deserializer.ReadProperty<int8_t>(100, "value");
|
1843
1843
|
break;
|
1844
1844
|
case PhysicalType::UINT16:
|
1845
|
-
new_value.value_.usmallint = deserializer.ReadProperty<uint16_t>("value");
|
1845
|
+
new_value.value_.usmallint = deserializer.ReadProperty<uint16_t>(100, "value");
|
1846
1846
|
break;
|
1847
1847
|
case PhysicalType::INT16:
|
1848
|
-
new_value.value_.smallint = deserializer.ReadProperty<int16_t>("value");
|
1848
|
+
new_value.value_.smallint = deserializer.ReadProperty<int16_t>(100, "value");
|
1849
1849
|
break;
|
1850
1850
|
case PhysicalType::UINT32:
|
1851
|
-
new_value.value_.uinteger = deserializer.ReadProperty<uint32_t>("value");
|
1851
|
+
new_value.value_.uinteger = deserializer.ReadProperty<uint32_t>(100, "value");
|
1852
1852
|
break;
|
1853
1853
|
case PhysicalType::INT32:
|
1854
|
-
new_value.value_.integer = deserializer.ReadProperty<int32_t>("value");
|
1854
|
+
new_value.value_.integer = deserializer.ReadProperty<int32_t>(100, "value");
|
1855
1855
|
break;
|
1856
1856
|
case PhysicalType::UINT64:
|
1857
|
-
new_value.value_.ubigint = deserializer.ReadProperty<uint64_t>("value");
|
1857
|
+
new_value.value_.ubigint = deserializer.ReadProperty<uint64_t>(100, "value");
|
1858
1858
|
break;
|
1859
1859
|
case PhysicalType::INT64:
|
1860
|
-
new_value.value_.bigint = deserializer.ReadProperty<int64_t>("value");
|
1860
|
+
new_value.value_.bigint = deserializer.ReadProperty<int64_t>(100, "value");
|
1861
1861
|
break;
|
1862
1862
|
case PhysicalType::INT128:
|
1863
|
-
new_value.value_.hugeint = deserializer.ReadProperty<hugeint_t>("value");
|
1863
|
+
new_value.value_.hugeint = deserializer.ReadProperty<hugeint_t>(100, "value");
|
1864
1864
|
break;
|
1865
1865
|
case PhysicalType::FLOAT:
|
1866
|
-
new_value.value_.float_ = deserializer.ReadProperty<float>("value");
|
1866
|
+
new_value.value_.float_ = deserializer.ReadProperty<float>(100, "value");
|
1867
1867
|
break;
|
1868
1868
|
case PhysicalType::DOUBLE:
|
1869
|
-
new_value.value_.double_ = deserializer.ReadProperty<double>("value");
|
1869
|
+
new_value.value_.double_ = deserializer.ReadProperty<double>(100, "value");
|
1870
1870
|
break;
|
1871
1871
|
case PhysicalType::INTERVAL:
|
1872
|
-
new_value.value_.interval = deserializer.ReadProperty<interval_t>("value");
|
1872
|
+
new_value.value_.interval = deserializer.ReadProperty<interval_t>(100, "value");
|
1873
1873
|
break;
|
1874
1874
|
case PhysicalType::VARCHAR: {
|
1875
|
-
auto str = deserializer.ReadProperty<string>("value");
|
1875
|
+
auto str = deserializer.ReadProperty<string>(100, "value");
|
1876
1876
|
if (type.id() == LogicalTypeId::BLOB) {
|
1877
1877
|
new_value.value_info_ = make_shared<StringValueInfo>(Blob::ToBlob(str));
|
1878
1878
|
} else {
|