duckdb 0.7.2-dev3666.0 → 0.7.2-dev3763.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/database.cpp +0 -1
- package/src/duckdb/extension/json/json_functions/json_transform.cpp +1 -1
- package/src/duckdb/extension/json/json_functions/read_json.cpp +4 -4
- package/src/duckdb/extension/json/json_functions/read_json_objects.cpp +1 -1
- package/src/duckdb/extension/json/json_scan.cpp +16 -12
- package/src/duckdb/src/common/arrow/arrow_converter.cpp +4 -4
- package/src/duckdb/src/common/compressed_file_system.cpp +2 -2
- package/src/duckdb/src/common/exception.cpp +17 -0
- package/src/duckdb/src/common/exception_format_value.cpp +14 -0
- package/src/duckdb/src/common/file_system.cpp +53 -31
- package/src/duckdb/src/common/local_file_system.cpp +5 -3
- package/src/duckdb/src/common/row_operations/row_gather.cpp +2 -2
- package/src/duckdb/src/common/serializer/binary_deserializer.cpp +1 -1
- package/src/duckdb/src/common/serializer/buffered_file_reader.cpp +1 -1
- package/src/duckdb/src/common/serializer/buffered_file_writer.cpp +1 -1
- package/src/duckdb/src/common/serializer/buffered_serializer.cpp +3 -3
- package/src/duckdb/src/common/serializer.cpp +1 -1
- package/src/duckdb/src/common/sort/radix_sort.cpp +5 -5
- package/src/duckdb/src/common/string_util.cpp +6 -2
- package/src/duckdb/src/common/types/bit.cpp +2 -2
- package/src/duckdb/src/common/types/blob.cpp +2 -2
- package/src/duckdb/src/common/types/data_chunk.cpp +2 -2
- package/src/duckdb/src/common/types/date.cpp +1 -1
- package/src/duckdb/src/common/types/decimal.cpp +2 -2
- package/src/duckdb/src/common/types/selection_vector.cpp +1 -1
- package/src/duckdb/src/common/types/time.cpp +1 -1
- package/src/duckdb/src/common/types/vector.cpp +7 -7
- package/src/duckdb/src/common/types.cpp +1 -1
- package/src/duckdb/src/common/windows_util.cpp +2 -2
- package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +1 -1
- package/src/duckdb/src/core_functions/scalar/list/list_lambdas.cpp +1 -1
- package/src/duckdb/src/core_functions/scalar/string/printf.cpp +1 -1
- package/src/duckdb/src/execution/aggregate_hashtable.cpp +1 -1
- package/src/duckdb/src/execution/join_hashtable.cpp +3 -3
- package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +2 -2
- package/src/duckdb/src/execution/operator/join/outer_join_marker.cpp +1 -1
- package/src/duckdb/src/execution/operator/join/perfect_hash_join_executor.cpp +1 -1
- package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +1 -1
- package/src/duckdb/src/execution/operator/persistent/buffered_csv_reader.cpp +1 -1
- package/src/duckdb/src/execution/operator/persistent/physical_export.cpp +2 -2
- package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +91 -30
- package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp +1 -1
- package/src/duckdb/src/execution/operator/schema/physical_create_index.cpp +1 -1
- package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +2 -2
- package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +3 -3
- package/src/duckdb/src/execution/window_segment_tree.cpp +1 -1
- package/src/duckdb/src/function/pragma/pragma_queries.cpp +2 -2
- package/src/duckdb/src/function/scalar/strftime_format.cpp +1 -1
- package/src/duckdb/src/function/scalar/string/concat.cpp +1 -1
- package/src/duckdb/src/function/scalar/string/like.cpp +2 -2
- package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +5 -5
- package/src/duckdb/src/function/table/copy_csv.cpp +1 -1
- package/src/duckdb/src/function/table/read_csv.cpp +3 -0
- package/src/duckdb/src/function/table/table_scan.cpp +7 -3
- package/src/duckdb/src/function/table/version/pragma_version.cpp +4 -6
- package/src/duckdb/src/include/duckdb/common/compressed_file_system.hpp +2 -2
- package/src/duckdb/src/include/duckdb/common/exception_format_value.hpp +26 -0
- package/src/duckdb/src/include/duckdb/common/file_system.hpp +6 -0
- package/src/duckdb/src/include/duckdb/common/helper.hpp +9 -9
- package/src/duckdb/src/include/duckdb/common/http_state.hpp +2 -2
- package/src/duckdb/src/include/duckdb/common/serializer/buffered_file_reader.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/serializer/buffered_file_writer.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/serializer/buffered_serializer.hpp +2 -2
- package/src/duckdb/src/include/duckdb/common/sort/duckdb_pdqsort.hpp +10 -10
- package/src/duckdb/src/include/duckdb/common/string_util.hpp +20 -0
- package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/selection_vector.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/validity_mask.hpp +3 -3
- package/src/duckdb/src/include/duckdb/common/types/vector_buffer.hpp +4 -4
- package/src/duckdb/src/include/duckdb/common/unique_ptr.hpp +8 -8
- package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/join_hashtable.hpp +3 -3
- package/src/duckdb/src/include/duckdb/execution/operator/join/outer_join_marker.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/join/perfect_hash_join_executor.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_range_join.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/buffered_csv_reader.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/parallel_csv_reader.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_insert.hpp +2 -3
- package/src/duckdb/src/include/duckdb/execution/perfect_aggregate_hashtable.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp +1 -1
- package/src/duckdb/src/include/duckdb/function/table/read_csv.hpp +1 -1
- package/src/duckdb/src/include/duckdb/main/client_data.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp +3 -3
- package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/expression/operator_expression.hpp +2 -2
- package/src/duckdb/src/include/duckdb/parser/keyword_helper.hpp +5 -0
- package/src/duckdb/src/include/duckdb/storage/data_table.hpp +4 -0
- package/src/duckdb/src/include/duckdb/storage/statistics/base_statistics.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/table/append_state.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/table/scan_state.hpp +5 -5
- package/src/duckdb/src/include/duckdb/storage/table/update_segment.hpp +2 -2
- package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +1 -1
- package/src/duckdb/src/main/client_context.cpp +4 -4
- package/src/duckdb/src/main/db_instance_cache.cpp +5 -3
- package/src/duckdb/src/main/extension/extension_install.cpp +22 -18
- package/src/duckdb/src/optimizer/join_order/join_relation_set.cpp +5 -5
- package/src/duckdb/src/parser/expression/collate_expression.cpp +1 -1
- package/src/duckdb/src/parser/keyword_helper.cpp +11 -1
- package/src/duckdb/src/parser/query_node/select_node.cpp +1 -1
- package/src/duckdb/src/parser/statement/copy_statement.cpp +2 -2
- package/src/duckdb/src/parser/tableref.cpp +1 -1
- package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +7 -1
- package/src/duckdb/src/planner/expression_binder/index_binder.cpp +1 -1
- package/src/duckdb/src/storage/checkpoint/write_overflow_strings_to_disk.cpp +1 -1
- package/src/duckdb/src/storage/compression/string_uncompressed.cpp +2 -2
- package/src/duckdb/src/storage/data_table.cpp +75 -44
- package/src/duckdb/src/storage/local_storage.cpp +1 -1
- package/src/duckdb/src/storage/statistics/list_stats.cpp +1 -1
- package/src/duckdb/src/storage/statistics/struct_stats.cpp +1 -1
- package/src/duckdb/src/storage/storage_manager.cpp +3 -0
- package/src/duckdb/src/storage/table/row_group.cpp +11 -11
- package/src/duckdb/src/storage/table/scan_state.cpp +1 -1
- package/src/duckdb/src/storage/table/update_segment.cpp +6 -6
package/package.json
CHANGED
package/src/database.cpp
CHANGED
@@ -328,7 +328,6 @@ ScanReplacement(duckdb::ClientContext &context, const std::string &table_name, d
|
|
328
328
|
children.push_back(duckdb::make_uniq<duckdb::ConstantExpression>(std::move(param)));
|
329
329
|
}
|
330
330
|
table_function->function = duckdb::make_uniq<duckdb::FunctionExpression>(jsargs.function, std::move(children));
|
331
|
-
table_function->alias = table_name;
|
332
331
|
return std::move(table_function);
|
333
332
|
}
|
334
333
|
return nullptr;
|
@@ -659,7 +659,7 @@ static bool TransformObjectToMap(yyjson_val *objects[], yyjson_alc *alc, Vector
|
|
659
659
|
// Transform keys
|
660
660
|
if (!JSONTransform::Transform(keys, alc, MapVector::GetKeys(result), list_size, options)) {
|
661
661
|
throw ConversionException(
|
662
|
-
StringUtil::Format(options.error_message
|
662
|
+
StringUtil::Format(options.error_message + ". Cannot default to NULL, because map keys cannot be NULL"));
|
663
663
|
}
|
664
664
|
|
665
665
|
// Transform values
|
@@ -161,7 +161,7 @@ unique_ptr<FunctionData> ReadJSONBind(ClientContext &context, TableFunctionBindI
|
|
161
161
|
StrpTimeFormat format;
|
162
162
|
auto error = StrTimeFormat::ParseFormatSpecifier(format_string, format);
|
163
163
|
if (!error.empty()) {
|
164
|
-
throw
|
164
|
+
throw BinderException("read_json could not parse \"dateformat\": '%s'.", error.c_str());
|
165
165
|
}
|
166
166
|
} else if (loption == "timestampformat" || loption == "timestamp_format") {
|
167
167
|
auto format_string = StringValue::Get(kv.second);
|
@@ -173,7 +173,7 @@ unique_ptr<FunctionData> ReadJSONBind(ClientContext &context, TableFunctionBindI
|
|
173
173
|
StrpTimeFormat format;
|
174
174
|
auto error = StrTimeFormat::ParseFormatSpecifier(format_string, format);
|
175
175
|
if (!error.empty()) {
|
176
|
-
throw
|
176
|
+
throw BinderException("read_json could not parse \"timestampformat\": '%s'.", error.c_str());
|
177
177
|
}
|
178
178
|
} else if (loption == "records") {
|
179
179
|
auto arg = StringValue::Get(kv.second);
|
@@ -184,7 +184,7 @@ unique_ptr<FunctionData> ReadJSONBind(ClientContext &context, TableFunctionBindI
|
|
184
184
|
} else if (arg == "false") {
|
185
185
|
bind_data->options.record_type = JSONRecordType::VALUES;
|
186
186
|
} else {
|
187
|
-
throw
|
187
|
+
throw BinderException("read_json requires \"records\" to be one of ['auto', 'true', 'false'].");
|
188
188
|
}
|
189
189
|
}
|
190
190
|
}
|
@@ -225,7 +225,7 @@ unique_ptr<FunctionData> ReadJSONBind(ClientContext &context, TableFunctionBindI
|
|
225
225
|
transform_options.error_unknown_key = bind_data->auto_detect && !bind_data->ignore_errors;
|
226
226
|
transform_options.delay_error = true;
|
227
227
|
|
228
|
-
return bind_data;
|
228
|
+
return std::move(bind_data);
|
229
229
|
}
|
230
230
|
|
231
231
|
static void ReadJSONFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) {
|
@@ -16,7 +16,7 @@ unique_ptr<FunctionData> ReadJSONObjectsBind(ClientContext &context, TableFuncti
|
|
16
16
|
bind_data->reader_bind =
|
17
17
|
MultiFileReader::BindOptions(bind_data->options.file_options, bind_data->files, return_types, names);
|
18
18
|
|
19
|
-
return bind_data;
|
19
|
+
return std::move(bind_data);
|
20
20
|
}
|
21
21
|
|
22
22
|
static void ReadJSONObjectsFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) {
|
@@ -28,19 +28,23 @@ void JSONScanData::Bind(ClientContext &context, TableFunctionBindInput &input) {
|
|
28
28
|
} else if (loption == "maximum_object_size") {
|
29
29
|
maximum_object_size = MaxValue<idx_t>(UIntegerValue::Get(kv.second), maximum_object_size);
|
30
30
|
} else if (loption == "format") {
|
31
|
-
auto arg = StringValue::Get(kv.second);
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
31
|
+
auto arg = StringUtil::Lower(StringValue::Get(kv.second));
|
32
|
+
static const auto format_options =
|
33
|
+
case_insensitive_map_t<JSONFormat> {{"auto", JSONFormat::AUTO_DETECT},
|
34
|
+
{"unstructured", JSONFormat::UNSTRUCTURED},
|
35
|
+
{"newline_delimited", JSONFormat::NEWLINE_DELIMITED},
|
36
|
+
{"nd", JSONFormat::NEWLINE_DELIMITED},
|
37
|
+
{"array", JSONFormat::ARRAY}};
|
38
|
+
auto lookup = format_options.find(arg);
|
39
|
+
if (lookup == format_options.end()) {
|
40
|
+
vector<string> valid_options;
|
41
|
+
for (auto &pair : format_options) {
|
42
|
+
valid_options.push_back(StringUtil::Format("'%s'", pair.first));
|
43
|
+
}
|
44
|
+
throw BinderException("format must be one of [%s], not '%s'", StringUtil::Join(valid_options, ", "),
|
45
|
+
arg);
|
43
46
|
}
|
47
|
+
options.format = lookup->second;
|
44
48
|
} else if (loption == "compression") {
|
45
49
|
SetCompression(StringUtil::Lower(StringValue::Get(kv.second)));
|
46
50
|
}
|
@@ -33,7 +33,7 @@ struct DuckDBArrowSchemaHolder {
|
|
33
33
|
std::list<vector<ArrowSchema>> nested_children;
|
34
34
|
std::list<vector<ArrowSchema *>> nested_children_ptr;
|
35
35
|
//! This holds strings created to represent decimal types
|
36
|
-
vector<
|
36
|
+
vector<unsafe_unique_array<char>> owned_type_names;
|
37
37
|
};
|
38
38
|
|
39
39
|
static void ReleaseDuckDBArrowSchema(ArrowSchema *schema) {
|
@@ -131,7 +131,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
|
|
131
131
|
break;
|
132
132
|
case LogicalTypeId::TIMESTAMP_TZ: {
|
133
133
|
string format = "tsu:" + config_timezone;
|
134
|
-
auto format_ptr =
|
134
|
+
auto format_ptr = make_unsafe_uniq_array<char>(format.size() + 1);
|
135
135
|
for (size_t i = 0; i < format.size(); i++) {
|
136
136
|
format_ptr[i] = format[i];
|
137
137
|
}
|
@@ -156,7 +156,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
|
|
156
156
|
uint8_t width, scale;
|
157
157
|
type.GetDecimalProperties(width, scale);
|
158
158
|
string format = "d:" + to_string(width) + "," + to_string(scale);
|
159
|
-
auto format_ptr =
|
159
|
+
auto format_ptr = make_unsafe_uniq_array<char>(format.size() + 1);
|
160
160
|
for (size_t i = 0; i < format.size(); i++) {
|
161
161
|
format_ptr[i] = format[i];
|
162
162
|
}
|
@@ -204,7 +204,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
|
|
204
204
|
InitializeChild(*child.children[type_idx]);
|
205
205
|
|
206
206
|
auto &struct_col_name = child_types[type_idx].first;
|
207
|
-
auto name_ptr =
|
207
|
+
auto name_ptr = make_unsafe_uniq_array<char>(struct_col_name.size() + 1);
|
208
208
|
for (size_t i = 0; i < struct_col_name.size(); i++) {
|
209
209
|
name_ptr[i] = struct_col_name[i];
|
210
210
|
}
|
@@ -19,10 +19,10 @@ void CompressedFile::Initialize(bool write) {
|
|
19
19
|
this->write = write;
|
20
20
|
stream_data.in_buf_size = compressed_fs.InBufferSize();
|
21
21
|
stream_data.out_buf_size = compressed_fs.OutBufferSize();
|
22
|
-
stream_data.in_buff =
|
22
|
+
stream_data.in_buff = make_unsafe_uniq_array<data_t>(stream_data.in_buf_size);
|
23
23
|
stream_data.in_buff_start = stream_data.in_buff.get();
|
24
24
|
stream_data.in_buff_end = stream_data.in_buff.get();
|
25
|
-
stream_data.out_buff =
|
25
|
+
stream_data.out_buff = make_unsafe_uniq_array<data_t>(stream_data.out_buf_size);
|
26
26
|
stream_data.out_buff_start = stream_data.out_buff.get();
|
27
27
|
stream_data.out_buff_end = stream_data.out_buff.get();
|
28
28
|
|
@@ -59,6 +59,23 @@ string Exception::GetStackTrace(int max_depth) {
|
|
59
59
|
}
|
60
60
|
|
61
61
|
string Exception::ConstructMessageRecursive(const string &msg, std::vector<ExceptionFormatValue> &values) {
|
62
|
+
#ifdef DEBUG
|
63
|
+
// Verify that we have the required amount of values for the message
|
64
|
+
idx_t parameter_count = 0;
|
65
|
+
for (idx_t i = 0; i < msg.size(); i++) {
|
66
|
+
if (msg[i] != '%') {
|
67
|
+
continue;
|
68
|
+
}
|
69
|
+
if (i < msg.size() && msg[i + 1] == '%') {
|
70
|
+
i++;
|
71
|
+
continue;
|
72
|
+
}
|
73
|
+
parameter_count++;
|
74
|
+
}
|
75
|
+
if (parameter_count != values.size()) {
|
76
|
+
throw InternalException("Expected %d parameters, received %d", parameter_count, values.size());
|
77
|
+
}
|
78
|
+
#endif
|
62
79
|
return ExceptionFormatValue::Format(msg, values);
|
63
80
|
}
|
64
81
|
|
@@ -3,6 +3,7 @@
|
|
3
3
|
#include "fmt/format.h"
|
4
4
|
#include "fmt/printf.h"
|
5
5
|
#include "duckdb/common/types/hugeint.hpp"
|
6
|
+
#include "duckdb/parser/keyword_helper.hpp"
|
6
7
|
|
7
8
|
namespace duckdb {
|
8
9
|
|
@@ -40,6 +41,19 @@ template <>
|
|
40
41
|
ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(string value) {
|
41
42
|
return ExceptionFormatValue(std::move(value));
|
42
43
|
}
|
44
|
+
|
45
|
+
template <>
|
46
|
+
ExceptionFormatValue
|
47
|
+
ExceptionFormatValue::CreateFormatValue(SQLString value) { // NOLINT: templating requires us to copy value here
|
48
|
+
return KeywordHelper::WriteQuoted(value.raw_string, '\'');
|
49
|
+
}
|
50
|
+
|
51
|
+
template <>
|
52
|
+
ExceptionFormatValue
|
53
|
+
ExceptionFormatValue::CreateFormatValue(SQLIdentifier value) { // NOLINT: templating requires us to copy value here
|
54
|
+
return KeywordHelper::WriteOptionallyQuoted(value.raw_string, '"');
|
55
|
+
}
|
56
|
+
|
43
57
|
template <>
|
44
58
|
ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const char *value) {
|
45
59
|
return ExceptionFormatValue(string(value));
|
@@ -11,6 +11,7 @@
|
|
11
11
|
#include "duckdb/main/client_data.hpp"
|
12
12
|
#include "duckdb/main/database.hpp"
|
13
13
|
#include "duckdb/main/extension_helper.hpp"
|
14
|
+
#include "duckdb/common/windows_util.hpp"
|
14
15
|
|
15
16
|
#include <cstdint>
|
16
17
|
#include <cstdio>
|
@@ -53,6 +54,14 @@ bool PathMatched(const string &path, const string &sub_path) {
|
|
53
54
|
|
54
55
|
#ifndef _WIN32
|
55
56
|
|
57
|
+
string FileSystem::GetEnvVariable(const string &name) {
|
58
|
+
const char *env = getenv(name.c_str());
|
59
|
+
if (!env) {
|
60
|
+
return string();
|
61
|
+
}
|
62
|
+
return env;
|
63
|
+
}
|
64
|
+
|
56
65
|
bool FileSystem::IsPathAbsolute(const string &path) {
|
57
66
|
auto path_separator = FileSystem::PathSeparator();
|
58
67
|
return PathMatched(path, path_separator);
|
@@ -78,43 +87,66 @@ idx_t FileSystem::GetAvailableMemory() {
|
|
78
87
|
}
|
79
88
|
|
80
89
|
string FileSystem::GetWorkingDirectory() {
|
81
|
-
auto buffer =
|
90
|
+
auto buffer = make_unsafe_uniq_array<char>(PATH_MAX);
|
82
91
|
char *ret = getcwd(buffer.get(), PATH_MAX);
|
83
92
|
if (!ret) {
|
84
93
|
throw IOException("Could not get working directory!");
|
85
94
|
}
|
86
95
|
return string(buffer.get());
|
87
96
|
}
|
97
|
+
|
98
|
+
string FileSystem::NormalizeAbsolutePath(const string &path) {
|
99
|
+
D_ASSERT(IsPathAbsolute(path));
|
100
|
+
return path;
|
101
|
+
}
|
102
|
+
|
88
103
|
#else
|
89
104
|
|
90
|
-
|
91
|
-
//
|
92
|
-
auto
|
93
|
-
|
94
|
-
|
105
|
+
string FileSystem::GetEnvVariable(const string &env) {
|
106
|
+
// first convert the environment variable name to the correct encoding
|
107
|
+
auto env_w = WindowsUtil::UTF8ToUnicode(env.c_str());
|
108
|
+
// use _wgetenv to get the value
|
109
|
+
auto res_w = _wgetenv(env_w.c_str());
|
110
|
+
if (!res_w) {
|
111
|
+
// no environment variable of this name found
|
112
|
+
return string();
|
95
113
|
}
|
96
|
-
|
97
|
-
|
98
|
-
|
114
|
+
return WindowsUtil::UnicodeToUTF8(res_w);
|
115
|
+
}
|
116
|
+
|
117
|
+
bool FileSystem::IsPathAbsolute(const string &path) {
|
118
|
+
// 1) A single backslash or forward-slash
|
119
|
+
if (PathMatched(path, "\\") || PathMatched(path, "/")) {
|
99
120
|
return true;
|
100
121
|
}
|
101
|
-
//
|
122
|
+
// 2) A disk designator with a backslash (e.g., C:\ or C:/)
|
102
123
|
auto path_aux = path;
|
103
124
|
path_aux.erase(0, 1);
|
104
|
-
|
105
|
-
if (PathMatched(path_aux, sub_path)) {
|
125
|
+
if (PathMatched(path_aux, ":\\") || PathMatched(path_aux, ":/")) {
|
106
126
|
return true;
|
107
127
|
}
|
108
128
|
return false;
|
109
129
|
}
|
110
130
|
|
131
|
+
string FileSystem::NormalizeAbsolutePath(const string &path) {
|
132
|
+
D_ASSERT(IsPathAbsolute(path));
|
133
|
+
auto result = StringUtil::Lower(FileSystem::ConvertSeparators(path));
|
134
|
+
if (PathMatched(result, "\\")) {
|
135
|
+
// Path starts with a single backslash or forward slash
|
136
|
+
// prepend drive letter
|
137
|
+
return GetWorkingDirectory().substr(0, 2) + result;
|
138
|
+
}
|
139
|
+
return result;
|
140
|
+
}
|
141
|
+
|
111
142
|
string FileSystem::PathSeparator() {
|
112
143
|
return "\\";
|
113
144
|
}
|
114
145
|
|
115
146
|
void FileSystem::SetWorkingDirectory(const string &path) {
|
116
|
-
|
117
|
-
|
147
|
+
auto unicode_path = WindowsUtil::UTF8ToUnicode(path.c_str());
|
148
|
+
if (!SetCurrentDirectoryW(unicode_path.c_str())) {
|
149
|
+
throw IOException("Could not change working directory to \"%s\"", path);
|
118
150
|
}
|
119
151
|
}
|
120
152
|
|
@@ -134,16 +166,16 @@ idx_t FileSystem::GetAvailableMemory() {
|
|
134
166
|
}
|
135
167
|
|
136
168
|
string FileSystem::GetWorkingDirectory() {
|
137
|
-
idx_t count =
|
169
|
+
idx_t count = GetCurrentDirectoryW(0, nullptr);
|
138
170
|
if (count == 0) {
|
139
171
|
throw IOException("Could not get working directory!");
|
140
172
|
}
|
141
|
-
auto buffer =
|
142
|
-
idx_t ret =
|
173
|
+
auto buffer = make_unsafe_uniq_array<wchar_t>(count);
|
174
|
+
idx_t ret = GetCurrentDirectoryW(count, buffer.get());
|
143
175
|
if (count != ret + 1) {
|
144
176
|
throw IOException("Could not get working directory!");
|
145
177
|
}
|
146
|
-
return
|
178
|
+
return WindowsUtil::UnicodeToUTF8(buffer.get());
|
147
179
|
}
|
148
180
|
|
149
181
|
#endif
|
@@ -161,13 +193,7 @@ string FileSystem::ConvertSeparators(const string &path) {
|
|
161
193
|
return path;
|
162
194
|
}
|
163
195
|
// on windows-based systems we accept both
|
164
|
-
|
165
|
-
for (idx_t i = 0; i < result.size(); i++) {
|
166
|
-
if (result[i] == '/') {
|
167
|
-
result[i] = separator;
|
168
|
-
}
|
169
|
-
}
|
170
|
-
return result;
|
196
|
+
return StringUtil::Replace(path, "/", separator_str);
|
171
197
|
}
|
172
198
|
|
173
199
|
string FileSystem::ExtractName(const string &path) {
|
@@ -202,14 +228,10 @@ string FileSystem::GetHomeDirectory(optional_ptr<FileOpener> opener) {
|
|
202
228
|
}
|
203
229
|
// fallback to the default home directories for the specified system
|
204
230
|
#ifdef DUCKDB_WINDOWS
|
205
|
-
|
231
|
+
return FileSystem::GetEnvVariable("USERPROFILE");
|
206
232
|
#else
|
207
|
-
|
233
|
+
return FileSystem::GetEnvVariable("HOME");
|
208
234
|
#endif
|
209
|
-
if (homedir) {
|
210
|
-
return homedir;
|
211
|
-
}
|
212
|
-
return string();
|
213
235
|
}
|
214
236
|
|
215
237
|
string FileSystem::GetHomeDirectory() {
|
@@ -162,8 +162,9 @@ static FileType GetFileTypeInternal(int fd) { // LCOV_EXCL_START
|
|
162
162
|
}
|
163
163
|
} // LCOV_EXCL_STOP
|
164
164
|
|
165
|
-
unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &
|
165
|
+
unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path_p, uint8_t flags, FileLockType lock_type,
|
166
166
|
FileCompressionType compression, FileOpener *opener) {
|
167
|
+
auto path = FileSystem::ExpandPath(path_p, opener);
|
167
168
|
if (compression != FileCompressionType::UNCOMPRESSED) {
|
168
169
|
throw NotImplementedException("Unsupported compression type for default file system");
|
169
170
|
}
|
@@ -506,8 +507,9 @@ public:
|
|
506
507
|
};
|
507
508
|
};
|
508
509
|
|
509
|
-
unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &
|
510
|
+
unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path_p, uint8_t flags, FileLockType lock_type,
|
510
511
|
FileCompressionType compression, FileOpener *opener) {
|
512
|
+
auto path = FileSystem::ExpandPath(path_p, opener);
|
511
513
|
if (compression != FileCompressionType::UNCOMPRESSED) {
|
512
514
|
throw NotImplementedException("Unsupported compression type for default file system");
|
513
515
|
}
|
@@ -775,7 +777,7 @@ void LocalFileSystem::MoveFile(const string &source, const string &target) {
|
|
775
777
|
auto source_unicode = WindowsUtil::UTF8ToUnicode(source.c_str());
|
776
778
|
auto target_unicode = WindowsUtil::UTF8ToUnicode(target.c_str());
|
777
779
|
if (!MoveFileW(source_unicode.c_str(), target_unicode.c_str())) {
|
778
|
-
throw IOException("Could not move file");
|
780
|
+
throw IOException("Could not move file: %s", GetLastErrorAsString());
|
779
781
|
}
|
780
782
|
}
|
781
783
|
|
@@ -95,8 +95,8 @@ static void GatherNestedVector(Vector &rows, const SelectionVector &row_sel, Vec
|
|
95
95
|
auto ptrs = FlatVector::GetData<data_ptr_t>(rows);
|
96
96
|
|
97
97
|
// Build the gather locations
|
98
|
-
auto data_locations =
|
99
|
-
auto mask_locations =
|
98
|
+
auto data_locations = make_unsafe_uniq_array<data_ptr_t>(count);
|
99
|
+
auto mask_locations = make_unsafe_uniq_array<data_ptr_t>(count);
|
100
100
|
for (idx_t i = 0; i < count; i++) {
|
101
101
|
auto row_idx = row_sel.get_index(i);
|
102
102
|
auto row = ptrs[row_idx];
|
@@ -123,7 +123,7 @@ string BinaryDeserializer::ReadString() {
|
|
123
123
|
if (size == 0) {
|
124
124
|
return string();
|
125
125
|
}
|
126
|
-
auto buffer =
|
126
|
+
auto buffer = make_unsafe_uniq_array<data_t>(size);
|
127
127
|
ReadData(buffer.get(), size);
|
128
128
|
return string((char *)buffer.get(), size);
|
129
129
|
}
|
@@ -9,7 +9,7 @@ namespace duckdb {
|
|
9
9
|
|
10
10
|
BufferedFileReader::BufferedFileReader(FileSystem &fs, const char *path, optional_ptr<ClientContext> context,
|
11
11
|
FileLockType lock_type, optional_ptr<FileOpener> opener)
|
12
|
-
: fs(fs), data(
|
12
|
+
: fs(fs), data(make_unsafe_uniq_array<data_t>(FILE_BUFFER_SIZE)), offset(0), read_data(0), total_read(0),
|
13
13
|
context(context) {
|
14
14
|
handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ, lock_type, FileSystem::DEFAULT_COMPRESSION, opener.get());
|
15
15
|
file_size = fs.GetFileSize(*handle);
|
@@ -9,7 +9,7 @@ namespace duckdb {
|
|
9
9
|
constexpr uint8_t BufferedFileWriter::DEFAULT_OPEN_FLAGS;
|
10
10
|
|
11
11
|
BufferedFileWriter::BufferedFileWriter(FileSystem &fs, const string &path_p, uint8_t open_flags)
|
12
|
-
: fs(fs), path(path_p), data(
|
12
|
+
: fs(fs), path(path_p), data(make_unsafe_uniq_array<data_t>(FILE_BUFFER_SIZE)), offset(0), total_written(0) {
|
13
13
|
handle = fs.OpenFile(path, open_flags, FileLockType::WRITE_LOCK);
|
14
14
|
}
|
15
15
|
|
@@ -5,10 +5,10 @@
|
|
5
5
|
namespace duckdb {
|
6
6
|
|
7
7
|
BufferedSerializer::BufferedSerializer(idx_t maximum_size)
|
8
|
-
: BufferedSerializer(
|
8
|
+
: BufferedSerializer(make_unsafe_uniq_array<data_t>(maximum_size), maximum_size) {
|
9
9
|
}
|
10
10
|
|
11
|
-
BufferedSerializer::BufferedSerializer(
|
11
|
+
BufferedSerializer::BufferedSerializer(unsafe_unique_array<data_t> data, idx_t size)
|
12
12
|
: maximum_size(size), data(data.get()) {
|
13
13
|
blob.size = 0;
|
14
14
|
blob.data = std::move(data);
|
@@ -26,7 +26,7 @@ void BufferedSerializer::WriteData(const_data_ptr_t buffer, idx_t write_size) {
|
|
26
26
|
auto new_data = new data_t[maximum_size];
|
27
27
|
memcpy(new_data, data, blob.size);
|
28
28
|
data = new_data;
|
29
|
-
blob.data =
|
29
|
+
blob.data = unsafe_unique_array<data_t>(new_data);
|
30
30
|
}
|
31
31
|
|
32
32
|
memcpy(data + blob.size, buffer, write_size);
|
@@ -17,7 +17,7 @@ static void SortTiedBlobs(BufferManager &buffer_manager, const data_ptr_t datapt
|
|
17
17
|
return;
|
18
18
|
}
|
19
19
|
// Fill pointer array for sorting
|
20
|
-
auto ptr_block =
|
20
|
+
auto ptr_block = make_unsafe_uniq_array<data_ptr_t>(end - start);
|
21
21
|
auto entry_ptrs = (data_ptr_t *)ptr_block.get();
|
22
22
|
for (idx_t i = start; i < end; i++) {
|
23
23
|
entry_ptrs[i - start] = row_ptr;
|
@@ -158,7 +158,7 @@ inline void InsertionSort(const data_ptr_t orig_ptr, const data_ptr_t temp_ptr,
|
|
158
158
|
const data_ptr_t target_ptr = swap ? orig_ptr : temp_ptr;
|
159
159
|
if (count > 1) {
|
160
160
|
const idx_t total_offset = col_offset + offset;
|
161
|
-
auto temp_val =
|
161
|
+
auto temp_val = make_unsafe_uniq_array<data_t>(row_width);
|
162
162
|
const data_ptr_t val = temp_val.get();
|
163
163
|
const auto comp_width = total_comp_width - offset;
|
164
164
|
for (idx_t i = 1; i < count; i++) {
|
@@ -249,7 +249,7 @@ void RadixSort(BufferManager &buffer_manager, const data_ptr_t &dataptr, const i
|
|
249
249
|
RadixSortLSD(buffer_manager, dataptr, count, col_offset, sort_layout.entry_size, sorting_size);
|
250
250
|
} else {
|
251
251
|
auto temp_block = buffer_manager.Allocate(MaxValue(count * sort_layout.entry_size, (idx_t)Storage::BLOCK_SIZE));
|
252
|
-
auto preallocated_array =
|
252
|
+
auto preallocated_array = make_unsafe_uniq_array<idx_t>(sorting_size * SortConstants::MSD_RADIX_LOCATIONS);
|
253
253
|
RadixSortMSD(dataptr, temp_block.Ptr(), count, col_offset, sort_layout.entry_size, sorting_size, 0,
|
254
254
|
preallocated_array.get(), false);
|
255
255
|
}
|
@@ -291,7 +291,7 @@ void LocalSortState::SortInMemory() {
|
|
291
291
|
// Radix sort and break ties until no more ties, or until all columns are sorted
|
292
292
|
idx_t sorting_size = 0;
|
293
293
|
idx_t col_offset = 0;
|
294
|
-
|
294
|
+
unsafe_unique_array<bool> ties_ptr;
|
295
295
|
bool *ties = nullptr;
|
296
296
|
bool contains_string = false;
|
297
297
|
for (idx_t i = 0; i < sort_layout->column_count; i++) {
|
@@ -305,7 +305,7 @@ void LocalSortState::SortInMemory() {
|
|
305
305
|
if (!ties) {
|
306
306
|
// This is the first sort
|
307
307
|
RadixSort(*buffer_manager, dataptr, count, col_offset, sorting_size, *sort_layout, contains_string);
|
308
|
-
ties_ptr =
|
308
|
+
ties_ptr = make_unsafe_uniq_array<bool>(count);
|
309
309
|
ties = ties_ptr.get();
|
310
310
|
std::fill_n(ties, count - 1, true);
|
311
311
|
ties[count - 1] = false;
|
@@ -193,6 +193,10 @@ string StringUtil::Lower(const string &str) {
|
|
193
193
|
return (copy);
|
194
194
|
}
|
195
195
|
|
196
|
+
bool StringUtil::IsLower(const string &str) {
|
197
|
+
return str == Lower(str);
|
198
|
+
}
|
199
|
+
|
196
200
|
// Jenkins hash function: https://en.wikipedia.org/wiki/Jenkins_hash_function
|
197
201
|
uint64_t StringUtil::CIHash(const string &str) {
|
198
202
|
uint32_t hash = 0;
|
@@ -277,7 +281,7 @@ vector<string> StringUtil::TopNStrings(vector<pair<string, idx_t>> scores, idx_t
|
|
277
281
|
|
278
282
|
struct LevenshteinArray {
|
279
283
|
LevenshteinArray(idx_t len1, idx_t len2) : len1(len1) {
|
280
|
-
dist =
|
284
|
+
dist = make_unsafe_uniq_array<idx_t>(len1 * len2);
|
281
285
|
}
|
282
286
|
|
283
287
|
idx_t &Score(idx_t i, idx_t j) {
|
@@ -286,7 +290,7 @@ struct LevenshteinArray {
|
|
286
290
|
|
287
291
|
private:
|
288
292
|
idx_t len1;
|
289
|
-
|
293
|
+
unsafe_unique_array<idx_t> dist;
|
290
294
|
|
291
295
|
idx_t GetIndex(idx_t i, idx_t j) {
|
292
296
|
return j * len1 + i;
|
@@ -78,7 +78,7 @@ void Bit::ToString(string_t bits, char *output) {
|
|
78
78
|
|
79
79
|
string Bit::ToString(string_t str) {
|
80
80
|
auto len = BitLength(str);
|
81
|
-
auto buffer =
|
81
|
+
auto buffer = make_unsafe_uniq_array<char>(len);
|
82
82
|
ToString(str, buffer.get());
|
83
83
|
return string(buffer.get(), len);
|
84
84
|
}
|
@@ -140,7 +140,7 @@ void Bit::ToBit(string_t str, string_t &output_str) {
|
|
140
140
|
|
141
141
|
string Bit::ToBit(string_t str) {
|
142
142
|
auto bit_len = GetBitSize(str);
|
143
|
-
auto buffer =
|
143
|
+
auto buffer = make_unsafe_uniq_array<char>(bit_len);
|
144
144
|
string_t output_str(buffer.get(), bit_len);
|
145
145
|
Bit::ToBit(str, output_str);
|
146
146
|
return output_str.GetString();
|
@@ -64,7 +64,7 @@ void Blob::ToString(string_t blob, char *output) {
|
|
64
64
|
|
65
65
|
string Blob::ToString(string_t blob) {
|
66
66
|
auto str_len = GetStringSize(blob);
|
67
|
-
auto buffer =
|
67
|
+
auto buffer = make_unsafe_uniq_array<char>(str_len);
|
68
68
|
Blob::ToString(blob, buffer.get());
|
69
69
|
return string(buffer.get(), str_len);
|
70
70
|
}
|
@@ -136,7 +136,7 @@ void Blob::ToBlob(string_t str, data_ptr_t output) {
|
|
136
136
|
|
137
137
|
string Blob::ToBlob(string_t str) {
|
138
138
|
auto blob_len = GetBlobSize(str);
|
139
|
-
auto buffer =
|
139
|
+
auto buffer = make_unsafe_uniq_array<char>(blob_len);
|
140
140
|
Blob::ToBlob(str, (data_ptr_t)buffer.get());
|
141
141
|
return string(buffer.get(), blob_len);
|
142
142
|
}
|
@@ -291,8 +291,8 @@ void DataChunk::Slice(DataChunk &other, const SelectionVector &sel, idx_t count_
|
|
291
291
|
}
|
292
292
|
}
|
293
293
|
|
294
|
-
|
295
|
-
auto orrified_data =
|
294
|
+
unsafe_unique_array<UnifiedVectorFormat> DataChunk::ToUnifiedFormat() {
|
295
|
+
auto orrified_data = make_unsafe_uniq_array<UnifiedVectorFormat>(ColumnCount());
|
296
296
|
for (idx_t col_idx = 0; col_idx < ColumnCount(); col_idx++) {
|
297
297
|
data[col_idx].ToUnifiedFormat(size(), orrified_data[col_idx]);
|
298
298
|
}
|
@@ -362,7 +362,7 @@ string Date::ToString(date_t date) {
|
|
362
362
|
Date::Convert(date, date_units[0], date_units[1], date_units[2]);
|
363
363
|
|
364
364
|
auto length = DateToStringCast::Length(date_units, year_length, add_bc);
|
365
|
-
auto buffer =
|
365
|
+
auto buffer = make_unsafe_uniq_array<char>(length);
|
366
366
|
DateToStringCast::Format(buffer.get(), date_units, year_length, add_bc);
|
367
367
|
return string(buffer.get(), length);
|
368
368
|
}
|
@@ -6,7 +6,7 @@ namespace duckdb {
|
|
6
6
|
template <class SIGNED, class UNSIGNED>
|
7
7
|
string TemplatedDecimalToString(SIGNED value, uint8_t width, uint8_t scale) {
|
8
8
|
auto len = DecimalToString::DecimalLength<SIGNED, UNSIGNED>(value, width, scale);
|
9
|
-
auto data =
|
9
|
+
auto data = make_unsafe_uniq_array<char>(len + 1);
|
10
10
|
DecimalToString::FormatDecimal<SIGNED, UNSIGNED>(value, width, scale, data.get(), len);
|
11
11
|
return string(data.get(), len);
|
12
12
|
}
|
@@ -25,7 +25,7 @@ string Decimal::ToString(int64_t value, uint8_t width, uint8_t scale) {
|
|
25
25
|
|
26
26
|
string Decimal::ToString(hugeint_t value, uint8_t width, uint8_t scale) {
|
27
27
|
auto len = HugeintToStringCast::DecimalLength(value, width, scale);
|
28
|
-
auto data =
|
28
|
+
auto data = make_unsafe_uniq_array<char>(len + 1);
|
29
29
|
HugeintToStringCast::FormatDecimal(value, width, scale, data.get(), len);
|
30
30
|
return string(data.get(), len);
|
31
31
|
}
|
@@ -5,7 +5,7 @@
|
|
5
5
|
namespace duckdb {
|
6
6
|
|
7
7
|
SelectionData::SelectionData(idx_t count) {
|
8
|
-
owned_data =
|
8
|
+
owned_data = make_unsafe_uniq_array<sel_t>(count);
|
9
9
|
#ifdef DEBUG
|
10
10
|
for (idx_t i = 0; i < count; i++) {
|
11
11
|
owned_data[i] = std::numeric_limits<sel_t>::max();
|
@@ -158,7 +158,7 @@ string Time::ToString(dtime_t time) {
|
|
158
158
|
|
159
159
|
char micro_buffer[6];
|
160
160
|
auto length = TimeToStringCast::Length(time_units, micro_buffer);
|
161
|
-
auto buffer =
|
161
|
+
auto buffer = make_unsafe_uniq_array<char>(length);
|
162
162
|
TimeToStringCast::Format(buffer.get(), length, time_units, micro_buffer);
|
163
163
|
return string(buffer.get(), length);
|
164
164
|
}
|