duckdb 0.8.1-dev96.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +8 -8
- package/package.json +3 -1
- package/src/duckdb/extension/icu/icu-datepart.cpp +1 -1
- package/src/duckdb/extension/icu/icu-extension.cpp +1 -1
- package/src/duckdb/extension/icu/icu-makedate.cpp +5 -4
- package/src/duckdb/extension/icu/icu-strptime.cpp +1 -1
- package/src/duckdb/extension/icu/third_party/icu/i18n/nfsubs.cpp +0 -2
- package/src/duckdb/extension/json/buffered_json_reader.cpp +23 -14
- package/src/duckdb/extension/json/include/buffered_json_reader.hpp +6 -6
- package/src/duckdb/extension/json/include/json_common.hpp +12 -2
- package/src/duckdb/extension/json/include/json_scan.hpp +5 -2
- package/src/duckdb/extension/json/json_functions/json_contains.cpp +5 -0
- package/src/duckdb/extension/json/json_functions/json_create.cpp +10 -10
- package/src/duckdb/extension/json/json_functions/json_merge_patch.cpp +2 -2
- package/src/duckdb/extension/json/json_functions/json_serialize_sql.cpp +2 -2
- package/src/duckdb/extension/json/json_functions/json_structure.cpp +5 -3
- package/src/duckdb/extension/json/json_functions/json_transform.cpp +11 -11
- package/src/duckdb/extension/json/json_functions/read_json.cpp +2 -1
- package/src/duckdb/extension/json/json_functions.cpp +6 -3
- package/src/duckdb/extension/json/json_scan.cpp +43 -27
- package/src/duckdb/extension/parquet/column_reader.cpp +5 -1
- package/src/duckdb/extension/parquet/include/decode_utils.hpp +6 -0
- package/src/duckdb/extension/parquet/parquet-extension.cpp +26 -1
- package/src/duckdb/src/catalog/catalog.cpp +5 -17
- package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +7 -1
- package/src/duckdb/src/catalog/catalog_entry/table_catalog_entry.cpp +121 -0
- package/src/duckdb/src/catalog/catalog_search_path.cpp +49 -12
- package/src/duckdb/src/catalog/default/default_types.cpp +9 -84
- package/src/duckdb/src/common/adbc/adbc.cpp +118 -12
- package/src/duckdb/src/common/adbc/driver_manager.cpp +0 -20
- package/src/duckdb/src/common/arrow/arrow_converter.cpp +11 -12
- package/src/duckdb/src/common/arrow/arrow_wrapper.cpp +4 -3
- package/src/duckdb/src/common/exception.cpp +4 -1
- package/src/duckdb/src/common/exception_format_value.cpp +24 -15
- package/src/duckdb/src/common/multi_file_reader.cpp +3 -0
- package/src/duckdb/src/common/random_engine.cpp +1 -1
- package/src/duckdb/src/common/types/row/row_data_collection_scanner.cpp +5 -4
- package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp +44 -7
- package/src/duckdb/src/common/types/time.cpp +2 -8
- package/src/duckdb/src/common/types/timestamp.cpp +37 -1
- package/src/duckdb/src/common/types/value.cpp +1 -0
- package/src/duckdb/src/common/types.cpp +4 -0
- package/src/duckdb/src/core_functions/aggregate/algebraic/avg.cpp +30 -33
- package/src/duckdb/src/core_functions/aggregate/algebraic/covar.cpp +0 -4
- package/src/duckdb/src/core_functions/aggregate/distributive/approx_count.cpp +30 -33
- package/src/duckdb/src/core_functions/aggregate/distributive/arg_min_max.cpp +52 -65
- package/src/duckdb/src/core_functions/aggregate/distributive/bitagg.cpp +48 -48
- package/src/duckdb/src/core_functions/aggregate/distributive/bitstring_agg.cpp +39 -40
- package/src/duckdb/src/core_functions/aggregate/distributive/bool.cpp +32 -32
- package/src/duckdb/src/core_functions/aggregate/distributive/entropy.cpp +34 -34
- package/src/duckdb/src/core_functions/aggregate/distributive/kurtosis.cpp +30 -31
- package/src/duckdb/src/core_functions/aggregate/distributive/minmax.cpp +88 -100
- package/src/duckdb/src/core_functions/aggregate/distributive/product.cpp +17 -17
- package/src/duckdb/src/core_functions/aggregate/distributive/skew.cpp +25 -27
- package/src/duckdb/src/core_functions/aggregate/distributive/string_agg.cpp +37 -38
- package/src/duckdb/src/core_functions/aggregate/distributive/sum.cpp +22 -22
- package/src/duckdb/src/core_functions/aggregate/holistic/approximate_quantile.cpp +44 -80
- package/src/duckdb/src/core_functions/aggregate/holistic/mode.cpp +49 -51
- package/src/duckdb/src/core_functions/aggregate/holistic/quantile.cpp +104 -122
- package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +57 -93
- package/src/duckdb/src/core_functions/aggregate/nested/histogram.cpp +22 -23
- package/src/duckdb/src/core_functions/aggregate/nested/list.cpp +18 -19
- package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp +16 -18
- package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp +22 -25
- package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp +19 -24
- package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp +18 -23
- package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp +14 -18
- package/src/duckdb/src/core_functions/function_list.cpp +1 -0
- package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +1 -1
- package/src/duckdb/src/core_functions/scalar/date/make_date.cpp +3 -0
- package/src/duckdb/src/core_functions/scalar/generic/system_functions.cpp +14 -0
- package/src/duckdb/src/core_functions/scalar/list/array_slice.cpp +1 -1
- package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +23 -6
- package/src/duckdb/src/core_functions/scalar/list/list_lambdas.cpp +1 -2
- package/src/duckdb/src/core_functions/scalar/map/map_concat.cpp +3 -0
- package/src/duckdb/src/core_functions/scalar/math/numeric.cpp +3 -3
- package/src/duckdb/src/execution/index/art/art.cpp +80 -7
- package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +20 -1
- package/src/duckdb/src/execution/index/art/leaf.cpp +11 -11
- package/src/duckdb/src/execution/index/art/leaf_segment.cpp +10 -0
- package/src/duckdb/src/execution/index/art/node.cpp +48 -35
- package/src/duckdb/src/execution/index/art/node16.cpp +3 -0
- package/src/duckdb/src/execution/index/art/node256.cpp +1 -0
- package/src/duckdb/src/execution/index/art/node4.cpp +3 -0
- package/src/duckdb/src/execution/index/art/node48.cpp +2 -0
- package/src/duckdb/src/execution/index/art/prefix.cpp +2 -0
- package/src/duckdb/src/execution/join_hashtable.cpp +2 -0
- package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +26 -9
- package/src/duckdb/src/execution/operator/helper/physical_reset.cpp +5 -2
- package/src/duckdb/src/execution/operator/helper/physical_set.cpp +5 -1
- package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp +3 -6
- package/src/duckdb/src/execution/operator/projection/physical_tableinout_function.cpp +1 -0
- package/src/duckdb/src/execution/operator/projection/physical_unnest.cpp +8 -3
- package/src/duckdb/src/execution/operator/schema/physical_attach.cpp +0 -1
- package/src/duckdb/src/execution/operator/schema/physical_create_index.cpp +29 -3
- package/src/duckdb/src/execution/reservoir_sample.cpp +18 -4
- package/src/duckdb/src/function/aggregate/distributive/count.cpp +159 -21
- package/src/duckdb/src/function/aggregate/distributive/first.cpp +67 -74
- package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +7 -7
- package/src/duckdb/src/function/cast/list_casts.cpp +2 -4
- package/src/duckdb/src/function/pragma/pragma_queries.cpp +33 -23
- package/src/duckdb/src/function/scalar/list/list_extract.cpp +1 -1
- package/src/duckdb/src/function/scalar/operators/arithmetic.cpp +1 -1
- package/src/duckdb/src/function/scalar/string/regexp/regexp_util.cpp +6 -2
- package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +2 -2
- package/src/duckdb/src/function/table/arrow.cpp +2 -2
- package/src/duckdb/src/function/table/checkpoint.cpp +3 -0
- package/src/duckdb/src/function/table/read_csv.cpp +15 -17
- package/src/duckdb/src/function/table/repeat.cpp +3 -0
- package/src/duckdb/src/function/table/repeat_row.cpp +8 -1
- package/src/duckdb/src/function/table/system/pragma_storage_info.cpp +4 -4
- package/src/duckdb/src/function/table/system/test_vector_types.cpp +81 -25
- package/src/duckdb/src/function/table/table_scan.cpp +2 -2
- package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
- package/src/duckdb/src/include/duckdb/catalog/catalog.hpp +0 -3
- package/src/duckdb/src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp +2 -0
- package/src/duckdb/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp +11 -1
- package/src/duckdb/src/include/duckdb/catalog/catalog_search_path.hpp +8 -2
- package/src/duckdb/src/include/duckdb/catalog/default/builtin_types/types.hpp +97 -0
- package/src/duckdb/src/include/duckdb/common/arrow/arrow_appender.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/arrow/arrow_converter.hpp +2 -3
- package/src/duckdb/src/include/duckdb/common/arrow/arrow_options.hpp +8 -1
- package/src/duckdb/src/include/duckdb/common/arrow/result_arrow_wrapper.hpp +0 -1
- package/src/duckdb/src/include/duckdb/common/bit_utils.hpp +16 -22
- package/src/duckdb/src/include/duckdb/common/exception.hpp +3 -0
- package/src/duckdb/src/include/duckdb/common/types/time.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/types/timestamp.hpp +4 -14
- package/src/duckdb/src/include/duckdb/common/vector_operations/aggregate_executor.hpp +92 -57
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/corr.hpp +20 -24
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/covar.hpp +36 -39
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/stddev.hpp +57 -53
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_count.hpp +8 -9
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_slope.hpp +16 -18
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/sum_helpers.hpp +7 -8
- package/src/duckdb/src/include/duckdb/core_functions/scalar/generic_functions.hpp +9 -0
- package/src/duckdb/src/include/duckdb/core_functions/scalar/map_functions.hpp +2 -6
- package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +16 -36
- package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +10 -4
- package/src/duckdb/src/include/duckdb/execution/index/art/fixed_size_allocator.hpp +3 -0
- package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/index/art/leaf_segment.hpp +2 -0
- package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +13 -3
- package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +1 -0
- package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +9 -30
- package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp +95 -0
- package/src/duckdb/src/include/duckdb/function/function_serialization.hpp +4 -2
- package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp +1 -1
- package/src/duckdb/src/include/duckdb/function/scalar_function.hpp +2 -1
- package/src/duckdb/src/include/duckdb/function/table_function.hpp +3 -2
- package/src/duckdb/src/include/duckdb/main/attached_database.hpp +4 -1
- package/src/duckdb/src/include/duckdb/main/capi/capi_internal.hpp +3 -1
- package/src/duckdb/src/include/duckdb/main/config.hpp +5 -0
- package/src/duckdb/src/include/duckdb/main/database_manager.hpp +1 -0
- package/src/duckdb/src/include/duckdb/main/extension_entries.hpp +142 -136
- package/src/duckdb/src/include/duckdb/main/query_result.hpp +6 -0
- package/src/duckdb/src/include/duckdb/main/settings.hpp +19 -0
- package/src/duckdb/src/include/duckdb/optimizer/unnest_rewriter.hpp +4 -0
- package/src/duckdb/src/include/duckdb/parallel/meta_pipeline.hpp +12 -3
- package/src/duckdb/src/include/duckdb/parser/parser.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/tableref/pivotref.hpp +2 -2
- package/src/duckdb/src/include/duckdb/parser/transformer.hpp +0 -2
- package/src/duckdb/src/include/duckdb/planner/bind_context.hpp +1 -1
- package/src/duckdb/src/include/duckdb/planner/bound_result_modifier.hpp +3 -0
- package/src/duckdb/src/include/duckdb/planner/expression_binder.hpp +1 -1
- package/src/duckdb/src/include/duckdb/planner/logical_operator.hpp +5 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_copy_to_file.hpp +6 -2
- package/src/duckdb/src/include/duckdb/planner/operator/logical_execute.hpp +4 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_explain.hpp +5 -1
- package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +5 -1
- package/src/duckdb/src/include/duckdb/planner/operator/logical_pragma.hpp +6 -2
- package/src/duckdb/src/include/duckdb/planner/operator/logical_prepare.hpp +4 -0
- package/src/duckdb/src/include/duckdb/storage/compression/chimp/algorithm/byte_reader.hpp +4 -0
- package/src/duckdb/src/include/duckdb/storage/data_table.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/in_memory_block_manager.hpp +13 -13
- package/src/duckdb/src/include/duckdb/storage/index.hpp +4 -2
- package/src/duckdb/src/include/duckdb/storage/storage_extension.hpp +0 -6
- package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/table/list_column_data.hpp +2 -1
- package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +2 -1
- package/src/duckdb/src/include/duckdb/storage/table/standard_column_data.hpp +2 -1
- package/src/duckdb/src/include/duckdb/storage/table/struct_column_data.hpp +2 -1
- package/src/duckdb/src/include/duckdb/storage/table_storage_info.hpp +0 -2
- package/src/duckdb/src/main/attached_database.cpp +8 -0
- package/src/duckdb/src/main/capi/arrow-c.cpp +4 -4
- package/src/duckdb/src/main/capi/config-c.cpp +2 -5
- package/src/duckdb/src/main/client_context.cpp +4 -3
- package/src/duckdb/src/main/config.cpp +2 -0
- package/src/duckdb/src/main/database.cpp +1 -0
- package/src/duckdb/src/main/database_manager.cpp +21 -0
- package/src/duckdb/src/main/query_result.cpp +6 -2
- package/src/duckdb/src/main/settings/settings.cpp +41 -6
- package/src/duckdb/src/optimizer/unnest_rewriter.cpp +27 -16
- package/src/duckdb/src/parallel/executor.cpp +38 -14
- package/src/duckdb/src/parallel/meta_pipeline.cpp +17 -3
- package/src/duckdb/src/parser/column_definition.cpp +5 -8
- package/src/duckdb/src/parser/parsed_data/create_info.cpp +0 -3
- package/src/duckdb/src/parser/parser.cpp +95 -35
- package/src/duckdb/src/parser/transform/expression/transform_function.cpp +1 -0
- package/src/duckdb/src/parser/transform/expression/transform_operator.cpp +3 -0
- package/src/duckdb/src/parser/transform/helpers/nodetype_to_string.cpp +0 -2
- package/src/duckdb/src/parser/transform/statement/transform_drop.cpp +0 -3
- package/src/duckdb/src/parser/transform/tableref/transform_pivot.cpp +3 -0
- package/src/duckdb/src/parser/transformer.cpp +0 -2
- package/src/duckdb/src/planner/bind_context.cpp +3 -4
- package/src/duckdb/src/planner/binder/expression/bind_function_expression.cpp +0 -2
- package/src/duckdb/src/planner/binder/expression/bind_positional_reference_expression.cpp +8 -3
- package/src/duckdb/src/planner/binder/query_node/bind_select_node.cpp +31 -15
- package/src/duckdb/src/planner/binder/statement/bind_create.cpp +0 -27
- package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +18 -1
- package/src/duckdb/src/planner/binder/statement/bind_drop.cpp +0 -25
- package/src/duckdb/src/planner/binder/statement/bind_update.cpp +2 -114
- package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +6 -1
- package/src/duckdb/src/planner/binder/tableref/bind_table_function.cpp +5 -0
- package/src/duckdb/src/planner/bound_result_modifier.cpp +14 -0
- package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +6 -5
- package/src/duckdb/src/planner/expression/bound_default_expression.cpp +7 -1
- package/src/duckdb/src/planner/expression.cpp +3 -0
- package/src/duckdb/src/planner/expression_binder.cpp +3 -2
- package/src/duckdb/src/planner/operator/logical_distinct.cpp +5 -4
- package/src/duckdb/src/planner/operator/logical_pivot.cpp +14 -2
- package/src/duckdb/src/planner/planner.cpp +5 -15
- package/src/duckdb/src/storage/data_table.cpp +10 -8
- package/src/duckdb/src/storage/index.cpp +13 -0
- package/src/duckdb/src/storage/storage_manager.cpp +6 -0
- package/src/duckdb/src/storage/table/column_data.cpp +7 -3
- package/src/duckdb/src/storage/table/list_column_data.cpp +5 -4
- package/src/duckdb/src/storage/table/row_group.cpp +4 -3
- package/src/duckdb/src/storage/table/row_group_collection.cpp +6 -3
- package/src/duckdb/src/storage/table/standard_column_data.cpp +4 -3
- package/src/duckdb/src/storage/table/struct_column_data.cpp +4 -3
- package/src/duckdb/src/storage/wal_replay.cpp +4 -5
- package/src/duckdb/third_party/libpg_query/include/nodes/nodes.hpp +0 -1
- package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +0 -14
- package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +12828 -12956
- package/src/duckdb/third_party/zstd/compress/zstd_compress.cpp +3 -0
- package/src/duckdb/third_party/zstd/include/zstd/compress/zstd_cwksp.h +4 -0
- package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +5 -5
- package/src/duckdb/ub_src_parser_transform_statement.cpp +0 -2
- package/test/extension.test.ts +11 -0
- package/src/duckdb/src/include/duckdb/parser/parsed_data/create_database_info.hpp +0 -46
- package/src/duckdb/src/parser/transform/statement/transform_create_database.cpp +0 -27
package/binding.gyp
CHANGED
@@ -237,18 +237,18 @@
|
|
237
237
|
"src/duckdb/third_party/zstd/compress/zstd_lazy.cpp",
|
238
238
|
"src/duckdb/third_party/zstd/compress/zstd_ldm.cpp",
|
239
239
|
"src/duckdb/third_party/zstd/compress/zstd_opt.cpp",
|
240
|
-
"src/duckdb/extension/icu/./icu-dateadd.cpp",
|
241
|
-
"src/duckdb/extension/icu/./icu-datetrunc.cpp",
|
242
|
-
"src/duckdb/extension/icu/./icu-datesub.cpp",
|
243
|
-
"src/duckdb/extension/icu/./icu-table-range.cpp",
|
244
|
-
"src/duckdb/extension/icu/./icu-timebucket.cpp",
|
245
240
|
"src/duckdb/extension/icu/./icu-list-range.cpp",
|
241
|
+
"src/duckdb/extension/icu/./icu-datefunc.cpp",
|
246
242
|
"src/duckdb/extension/icu/./icu-datepart.cpp",
|
247
|
-
"src/duckdb/extension/icu/./icu-
|
243
|
+
"src/duckdb/extension/icu/./icu-datetrunc.cpp",
|
244
|
+
"src/duckdb/extension/icu/./icu-table-range.cpp",
|
245
|
+
"src/duckdb/extension/icu/./icu-dateadd.cpp",
|
248
246
|
"src/duckdb/extension/icu/./icu-extension.cpp",
|
249
|
-
"src/duckdb/extension/icu/./icu-makedate.cpp",
|
250
|
-
"src/duckdb/extension/icu/./icu-datefunc.cpp",
|
251
247
|
"src/duckdb/extension/icu/./icu-strptime.cpp",
|
248
|
+
"src/duckdb/extension/icu/./icu-datesub.cpp",
|
249
|
+
"src/duckdb/extension/icu/./icu-makedate.cpp",
|
250
|
+
"src/duckdb/extension/icu/./icu-timezone.cpp",
|
251
|
+
"src/duckdb/extension/icu/./icu-timebucket.cpp",
|
252
252
|
"src/duckdb/ub_extension_icu_third_party_icu_common.cpp",
|
253
253
|
"src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp",
|
254
254
|
"src/duckdb/extension/icu/third_party/icu/stubdata/stubdata.cpp",
|
package/package.json
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
"name": "duckdb",
|
3
3
|
"main": "./lib/duckdb.js",
|
4
4
|
"types": "./lib/duckdb.d.ts",
|
5
|
-
"version": "0.8.1
|
5
|
+
"version": "0.8.1",
|
6
6
|
"description": "DuckDB node.js API",
|
7
7
|
"gypfile": true,
|
8
8
|
"dependencies": {
|
@@ -28,11 +28,13 @@
|
|
28
28
|
},
|
29
29
|
"devDependencies": {
|
30
30
|
"@types/chai": "^4.3.4",
|
31
|
+
"@types/chai-as-promised": "^7.1.5",
|
31
32
|
"@types/mocha": "^10.0.0",
|
32
33
|
"@types/node": "^18.11.0",
|
33
34
|
"apache-arrow": "^9.0.0",
|
34
35
|
"aws-sdk": "^2.790.0",
|
35
36
|
"chai": "^4.3.6",
|
37
|
+
"chai-as-promised": "^7.1.1",
|
36
38
|
"jsdoc3-parser": "^2.0.0",
|
37
39
|
"mocha": "^8.3.0",
|
38
40
|
"ts-node": "^10.9.1",
|
@@ -432,7 +432,7 @@ struct ICUDatePart : public ICUDateFunc {
|
|
432
432
|
throw NotImplementedException("FIXME: serialize icu-datepart");
|
433
433
|
}
|
434
434
|
|
435
|
-
static duckdb::unique_ptr<FunctionData> DeserializeFunction(
|
435
|
+
static duckdb::unique_ptr<FunctionData> DeserializeFunction(PlanDeserializationState &state, FieldReader &reader,
|
436
436
|
ScalarFunction &bound_function) {
|
437
437
|
throw NotImplementedException("FIXME: serialize icu-datepart");
|
438
438
|
}
|
@@ -145,7 +145,7 @@ static void ICUCollateSerialize(FieldWriter &writer, const FunctionData *bind_da
|
|
145
145
|
throw NotImplementedException("FIXME: serialize icu-collate");
|
146
146
|
}
|
147
147
|
|
148
|
-
static duckdb::unique_ptr<FunctionData> ICUCollateDeserialize(
|
148
|
+
static duckdb::unique_ptr<FunctionData> ICUCollateDeserialize(PlanDeserializationState &state, FieldReader &reader,
|
149
149
|
ScalarFunction &bound_function) {
|
150
150
|
throw NotImplementedException("FIXME: serialize icu-collate");
|
151
151
|
}
|
@@ -1,3 +1,4 @@
|
|
1
|
+
#include "duckdb/common/operator/cast_operators.hpp"
|
1
2
|
#include "duckdb/common/types/date.hpp"
|
2
3
|
#include "duckdb/common/types/time.hpp"
|
3
4
|
#include "duckdb/common/types/timestamp.hpp"
|
@@ -36,8 +37,8 @@ struct ICUMakeDate : public ICUDateFunc {
|
|
36
37
|
|
37
38
|
static bool CastToDate(Vector &source, Vector &result, idx_t count, CastParameters ¶meters) {
|
38
39
|
auto &cast_data = parameters.cast_data->Cast<CastData>();
|
39
|
-
auto info =
|
40
|
-
CalendarPtr calendar(info
|
40
|
+
auto &info = cast_data.info->Cast<BindData>();
|
41
|
+
CalendarPtr calendar(info.calendar->clone());
|
41
42
|
|
42
43
|
UnaryExecutor::Execute<timestamp_t, date_t>(
|
43
44
|
source, result, count, [&](timestamp_t input) { return Operation(calendar.get(), input); });
|
@@ -67,7 +68,7 @@ struct ICUMakeTimestampTZFunc : public ICUDateFunc {
|
|
67
68
|
static inline timestamp_t Operation(icu::Calendar *calendar, T yyyy, T mm, T dd, T hr, T mn, double ss) {
|
68
69
|
const auto year = yyyy + (yyyy < 0);
|
69
70
|
|
70
|
-
const auto secs = int32_t(ss);
|
71
|
+
const auto secs = Cast::Operation<double, int32_t>(ss);
|
71
72
|
ss -= secs;
|
72
73
|
ss *= Interval::MSECS_PER_SEC;
|
73
74
|
const auto millis = int32_t(ss);
|
@@ -87,7 +88,7 @@ struct ICUMakeTimestampTZFunc : public ICUDateFunc {
|
|
87
88
|
template <typename T>
|
88
89
|
static void Execute(DataChunk &input, ExpressionState &state, Vector &result) {
|
89
90
|
auto &func_expr = state.expr.Cast<BoundFunctionExpression>();
|
90
|
-
auto &info =
|
91
|
+
auto &info = func_expr.bind_info->Cast<BindData>();
|
91
92
|
CalendarPtr calendar_ptr(info.calendar->clone());
|
92
93
|
auto calendar = calendar_ptr.get();
|
93
94
|
|
@@ -53,7 +53,7 @@ struct ICUStrptime : public ICUDateFunc {
|
|
53
53
|
throw NotImplementedException("FIXME: serialize icu-strptime");
|
54
54
|
}
|
55
55
|
|
56
|
-
static duckdb::unique_ptr<FunctionData> Deserialize(
|
56
|
+
static duckdb::unique_ptr<FunctionData> Deserialize(PlanDeserializationState &state, FieldReader &reader,
|
57
57
|
ScalarFunction &bound_function) {
|
58
58
|
throw NotImplementedException("FIXME: serialize icu-strptime");
|
59
59
|
}
|
@@ -1310,10 +1310,8 @@ NumeratorSubstitution::doParse(const UnicodeString& text,
|
|
1310
1310
|
// compute the 'effective' base and prescale the value down
|
1311
1311
|
int64_t n = result.getLong(status); // force conversion!
|
1312
1312
|
int64_t d = 1;
|
1313
|
-
int32_t pow = 0;
|
1314
1313
|
while (d <= n) {
|
1315
1314
|
d *= 10;
|
1316
|
-
++pow;
|
1317
1315
|
}
|
1318
1316
|
// now add the zeros
|
1319
1317
|
while (zeroCount > 0) {
|
@@ -30,6 +30,10 @@ JSONFileHandle::JSONFileHandle(unique_ptr<FileHandle> file_handle_p, Allocator &
|
|
30
30
|
requested_reads(0), actual_reads(0), cached_size(0) {
|
31
31
|
}
|
32
32
|
|
33
|
+
bool JSONFileHandle::IsOpen() const {
|
34
|
+
return file_handle != nullptr;
|
35
|
+
}
|
36
|
+
|
33
37
|
void JSONFileHandle::Close() {
|
34
38
|
if (file_handle) {
|
35
39
|
file_handle->Close();
|
@@ -62,19 +66,20 @@ idx_t JSONFileHandle::GetPositionAndSize(idx_t &position, idx_t requested_size)
|
|
62
66
|
if (actual_size != 0) {
|
63
67
|
requested_reads++;
|
64
68
|
}
|
69
|
+
|
65
70
|
return actual_size;
|
66
71
|
}
|
67
72
|
|
68
|
-
void JSONFileHandle::ReadAtPosition(
|
73
|
+
void JSONFileHandle::ReadAtPosition(char *pointer, idx_t size, idx_t position, bool sample_run) {
|
69
74
|
D_ASSERT(size != 0);
|
70
75
|
if (plain_file_source) {
|
71
|
-
file_handle->Read(
|
76
|
+
file_handle->Read(pointer, size, position);
|
72
77
|
actual_reads++;
|
73
78
|
return;
|
74
79
|
}
|
75
80
|
|
76
81
|
if (sample_run) { // Cache the buffer
|
77
|
-
file_handle->Read(
|
82
|
+
file_handle->Read(pointer, size, position);
|
78
83
|
actual_reads++;
|
79
84
|
cached_buffers.emplace_back(allocator.Allocate(size));
|
80
85
|
memcpy(cached_buffers.back().get(), pointer, size);
|
@@ -87,12 +92,12 @@ void JSONFileHandle::ReadAtPosition(const char *pointer, idx_t size, idx_t posit
|
|
87
92
|
actual_reads++;
|
88
93
|
}
|
89
94
|
if (size != 0) {
|
90
|
-
file_handle->Read(
|
95
|
+
file_handle->Read(pointer, size, position);
|
91
96
|
actual_reads++;
|
92
97
|
}
|
93
98
|
}
|
94
99
|
|
95
|
-
idx_t JSONFileHandle::Read(
|
100
|
+
idx_t JSONFileHandle::Read(char *pointer, idx_t requested_size, bool sample_run) {
|
96
101
|
D_ASSERT(requested_size != 0);
|
97
102
|
if (plain_file_source) {
|
98
103
|
auto actual_size = ReadInternal(pointer, requested_size);
|
@@ -121,7 +126,7 @@ idx_t JSONFileHandle::Read(const char *pointer, idx_t requested_size, bool sampl
|
|
121
126
|
return actual_size;
|
122
127
|
}
|
123
128
|
|
124
|
-
idx_t JSONFileHandle::ReadFromCache(
|
129
|
+
idx_t JSONFileHandle::ReadFromCache(char *&pointer, idx_t &size, idx_t &position) {
|
125
130
|
idx_t read_size = 0;
|
126
131
|
idx_t total_offset = 0;
|
127
132
|
|
@@ -134,7 +139,7 @@ idx_t JSONFileHandle::ReadFromCache(const char *&pointer, idx_t &size, idx_t &po
|
|
134
139
|
if (position < total_offset + cached_buffer.GetSize()) {
|
135
140
|
idx_t within_buffer_offset = position - total_offset;
|
136
141
|
idx_t copy_size = MinValue<idx_t>(size, cached_buffer.GetSize() - within_buffer_offset);
|
137
|
-
memcpy(
|
142
|
+
memcpy(pointer, cached_buffer.get() + within_buffer_offset, copy_size);
|
138
143
|
|
139
144
|
read_size += copy_size;
|
140
145
|
pointer += copy_size;
|
@@ -147,11 +152,11 @@ idx_t JSONFileHandle::ReadFromCache(const char *&pointer, idx_t &size, idx_t &po
|
|
147
152
|
return read_size;
|
148
153
|
}
|
149
154
|
|
150
|
-
idx_t JSONFileHandle::ReadInternal(
|
155
|
+
idx_t JSONFileHandle::ReadInternal(char *pointer, const idx_t requested_size) {
|
151
156
|
// Deal with reading from pipes
|
152
157
|
idx_t total_read_size = 0;
|
153
158
|
while (total_read_size < requested_size) {
|
154
|
-
auto read_size = file_handle->Read(
|
159
|
+
auto read_size = file_handle->Read(pointer + total_read_size, requested_size - total_read_size);
|
155
160
|
if (read_size == 0) {
|
156
161
|
break;
|
157
162
|
}
|
@@ -165,6 +170,7 @@ BufferedJSONReader::BufferedJSONReader(ClientContext &context, BufferedJSONReade
|
|
165
170
|
}
|
166
171
|
|
167
172
|
void BufferedJSONReader::OpenJSONFile() {
|
173
|
+
D_ASSERT(!IsDone());
|
168
174
|
lock_guard<mutex> guard(lock);
|
169
175
|
auto &file_system = FileSystem::GetFileSystem(context);
|
170
176
|
auto regular_file_handle =
|
@@ -186,6 +192,13 @@ bool BufferedJSONReader::IsOpen() const {
|
|
186
192
|
return file_handle != nullptr;
|
187
193
|
}
|
188
194
|
|
195
|
+
bool BufferedJSONReader::IsDone() const {
|
196
|
+
if (file_handle) {
|
197
|
+
return !file_handle->IsOpen();
|
198
|
+
}
|
199
|
+
return false;
|
200
|
+
}
|
201
|
+
|
189
202
|
BufferedJSONReaderOptions &BufferedJSONReader::GetOptions() {
|
190
203
|
return options;
|
191
204
|
}
|
@@ -212,10 +225,6 @@ void BufferedJSONReader::SetRecordType(duckdb::JSONRecordType type) {
|
|
212
225
|
options.record_type = type;
|
213
226
|
}
|
214
227
|
|
215
|
-
bool BufferedJSONReader::IsParallel() const {
|
216
|
-
return options.format == JSONFormat::NEWLINE_DELIMITED && file_handle->CanSeek();
|
217
|
-
}
|
218
|
-
|
219
228
|
const string &BufferedJSONReader::GetFileName() const {
|
220
229
|
return file_name;
|
221
230
|
}
|
@@ -288,7 +297,7 @@ void BufferedJSONReader::ThrowTransformError(idx_t buf_index, idx_t line_or_obje
|
|
288
297
|
const string &error_message) {
|
289
298
|
string unit = options.format == JSONFormat::NEWLINE_DELIMITED ? "line" : "record/value";
|
290
299
|
auto line = GetLineNumber(buf_index, line_or_object_in_buf);
|
291
|
-
throw InvalidInputException("JSON transform error in file \"%s\", in %s %llu: %s
|
300
|
+
throw InvalidInputException("JSON transform error in file \"%s\", in %s %llu: %s", file_name, unit, line,
|
292
301
|
error_message);
|
293
302
|
}
|
294
303
|
|
@@ -71,6 +71,7 @@ public:
|
|
71
71
|
struct JSONFileHandle {
|
72
72
|
public:
|
73
73
|
JSONFileHandle(unique_ptr<FileHandle> file_handle, Allocator &allocator);
|
74
|
+
bool IsOpen() const;
|
74
75
|
void Close();
|
75
76
|
|
76
77
|
idx_t FileSize() const;
|
@@ -80,15 +81,15 @@ public:
|
|
80
81
|
void Seek(idx_t position);
|
81
82
|
|
82
83
|
idx_t GetPositionAndSize(idx_t &position, idx_t requested_size);
|
83
|
-
void ReadAtPosition(
|
84
|
-
idx_t Read(
|
84
|
+
void ReadAtPosition(char *pointer, idx_t size, idx_t position, bool sample_run);
|
85
|
+
idx_t Read(char *pointer, idx_t requested_size, bool sample_run);
|
85
86
|
|
86
87
|
void Reset();
|
87
88
|
bool RequestedReadsComplete();
|
88
89
|
|
89
90
|
private:
|
90
|
-
idx_t ReadFromCache(
|
91
|
-
idx_t ReadInternal(
|
91
|
+
idx_t ReadFromCache(char *&pointer, idx_t &size, idx_t &position);
|
92
|
+
idx_t ReadInternal(char *pointer, const idx_t requested_size);
|
92
93
|
|
93
94
|
private:
|
94
95
|
//! The JSON file handle
|
@@ -139,6 +140,7 @@ public:
|
|
139
140
|
void OpenJSONFile();
|
140
141
|
void CloseJSONFile();
|
141
142
|
bool IsOpen() const;
|
143
|
+
bool IsDone() const;
|
142
144
|
|
143
145
|
BufferedJSONReaderOptions &GetOptions();
|
144
146
|
const BufferedJSONReaderOptions &GetOptions() const;
|
@@ -148,8 +150,6 @@ public:
|
|
148
150
|
JSONRecordType GetRecordType() const;
|
149
151
|
void SetRecordType(JSONRecordType type);
|
150
152
|
|
151
|
-
bool IsParallel() const;
|
152
|
-
|
153
153
|
const string &GetFileName() const;
|
154
154
|
JSONFileHandle &GetFileHandle() const;
|
155
155
|
|
@@ -170,6 +170,16 @@ public:
|
|
170
170
|
}
|
171
171
|
|
172
172
|
public:
|
173
|
+
template <class T>
|
174
|
+
static T *AllocateArray(yyjson_alc *alc, idx_t count) {
|
175
|
+
return reinterpret_cast<T *>(alc->malloc(alc->ctx, sizeof(T) * count));
|
176
|
+
}
|
177
|
+
|
178
|
+
template <class T>
|
179
|
+
static T *AllocateArray(yyjson_mut_doc *doc, idx_t count) {
|
180
|
+
return AllocateArray<T>(&doc->alc, count);
|
181
|
+
}
|
182
|
+
|
173
183
|
static inline yyjson_mut_doc *CreateDocument(yyjson_alc *alc) {
|
174
184
|
D_ASSERT(alc);
|
175
185
|
return yyjson_mut_doc_new(alc);
|
@@ -419,11 +429,11 @@ private:
|
|
419
429
|
|
420
430
|
template <>
|
421
431
|
inline char *JSONCommon::WriteVal(yyjson_val *val, yyjson_alc *alc, idx_t &len) {
|
422
|
-
return yyjson_val_write_opts(val, JSONCommon::WRITE_FLAG, alc,
|
432
|
+
return yyjson_val_write_opts(val, JSONCommon::WRITE_FLAG, alc, reinterpret_cast<size_t *>(&len), nullptr);
|
423
433
|
}
|
424
434
|
template <>
|
425
435
|
inline char *JSONCommon::WriteVal(yyjson_mut_val *val, yyjson_alc *alc, idx_t &len) {
|
426
|
-
return yyjson_mut_val_write_opts(val, JSONCommon::WRITE_FLAG, alc,
|
436
|
+
return yyjson_mut_val_write_opts(val, JSONCommon::WRITE_FLAG, alc, reinterpret_cast<size_t *>(&len), nullptr);
|
427
437
|
}
|
428
438
|
|
429
439
|
template <>
|
@@ -232,6 +232,8 @@ private:
|
|
232
232
|
void ThrowObjectSizeError(const idx_t object_size);
|
233
233
|
void ThrowInvalidAtEndError();
|
234
234
|
|
235
|
+
bool IsParallel(JSONScanGlobalState &gstate) const;
|
236
|
+
|
235
237
|
private:
|
236
238
|
//! Bind data
|
237
239
|
const JSONScanData &bind_data;
|
@@ -245,7 +247,7 @@ private:
|
|
245
247
|
bool is_last;
|
246
248
|
|
247
249
|
//! Current buffer read info
|
248
|
-
|
250
|
+
char *buffer_ptr;
|
249
251
|
idx_t buffer_size;
|
250
252
|
idx_t buffer_offset;
|
251
253
|
idx_t prev_buffer_remainder;
|
@@ -290,7 +292,8 @@ public:
|
|
290
292
|
vector<unique_ptr<Expression>> &filters);
|
291
293
|
|
292
294
|
static void Serialize(FieldWriter &writer, const FunctionData *bind_data_p, const TableFunction &function);
|
293
|
-
static unique_ptr<FunctionData> Deserialize(
|
295
|
+
static unique_ptr<FunctionData> Deserialize(PlanDeserializationState &state, FieldReader &reader,
|
296
|
+
TableFunction &function);
|
294
297
|
|
295
298
|
static void TableFunctionDefaults(TableFunction &table_function);
|
296
299
|
};
|
@@ -114,6 +114,11 @@ static void JSONContainsFunction(DataChunk &args, ExpressionState &state, Vector
|
|
114
114
|
auto &needles = args.data[1];
|
115
115
|
|
116
116
|
if (needles.GetVectorType() == VectorType::CONSTANT_VECTOR) {
|
117
|
+
if (ConstantVector::IsNull(needles)) {
|
118
|
+
result.SetVectorType(VectorType::CONSTANT_VECTOR);
|
119
|
+
ConstantVector::SetNull(result, true);
|
120
|
+
return;
|
121
|
+
}
|
117
122
|
auto &needle_str = *ConstantVector::GetData<string_t>(needles);
|
118
123
|
auto needle_doc = JSONCommon::ReadDocument(needle_str, JSONCommon::READ_FLAG, lstate.json_allocator.GetYYAlc());
|
119
124
|
UnaryExecutor::Execute<string_t, bool>(haystacks, result, args.size(), [&](string_t haystack_str) {
|
@@ -276,7 +276,7 @@ static void CreateValuesStruct(const StructNames &names, yyjson_mut_doc *doc, yy
|
|
276
276
|
vals[i] = yyjson_mut_obj(doc);
|
277
277
|
}
|
278
278
|
// Initialize re-usable array for the nested values
|
279
|
-
auto nested_vals =
|
279
|
+
auto nested_vals = JSONCommon::AllocateArray<yyjson_mut_val *>(doc, count);
|
280
280
|
|
281
281
|
// Add the key/value pairs to the values
|
282
282
|
auto &entries = StructVector::GetEntries(value_v);
|
@@ -301,12 +301,12 @@ static void CreateValuesMap(const StructNames &names, yyjson_mut_doc *doc, yyjso
|
|
301
301
|
// Create nested keys
|
302
302
|
auto &map_key_v = MapVector::GetKeys(value_v);
|
303
303
|
auto map_key_count = ListVector::GetListSize(value_v);
|
304
|
-
auto nested_keys =
|
304
|
+
auto nested_keys = JSONCommon::AllocateArray<yyjson_mut_val *>(doc, map_key_count);
|
305
305
|
TemplatedCreateValues<string_t, string_t>(doc, nested_keys, map_key_v, map_key_count);
|
306
306
|
// Create nested values
|
307
307
|
auto &map_val_v = MapVector::GetValues(value_v);
|
308
308
|
auto map_val_count = ListVector::GetListSize(value_v);
|
309
|
-
auto nested_vals =
|
309
|
+
auto nested_vals = JSONCommon::AllocateArray<yyjson_mut_val *>(doc, map_val_count);
|
310
310
|
CreateValues(names, doc, nested_vals, map_val_v, map_val_count);
|
311
311
|
// Add the key/value pairs to the values
|
312
312
|
UnifiedVectorFormat map_data;
|
@@ -338,7 +338,7 @@ static void CreateValuesUnion(const StructNames &names, yyjson_mut_doc *doc, yyj
|
|
338
338
|
}
|
339
339
|
|
340
340
|
// Initialize re-usable array for the nested values
|
341
|
-
auto nested_vals =
|
341
|
+
auto nested_vals = JSONCommon::AllocateArray<yyjson_mut_val *>(doc, count);
|
342
342
|
|
343
343
|
auto &tag_v = UnionVector::GetTags(value_v);
|
344
344
|
UnifiedVectorFormat tag_data;
|
@@ -384,7 +384,7 @@ static void CreateValuesList(const StructNames &names, yyjson_mut_doc *doc, yyjs
|
|
384
384
|
// Initialize array for the nested values
|
385
385
|
auto &child_v = ListVector::GetEntry(value_v);
|
386
386
|
auto child_count = ListVector::GetListSize(value_v);
|
387
|
-
auto nested_vals =
|
387
|
+
auto nested_vals = JSONCommon::AllocateArray<yyjson_mut_val *>(doc, child_count);
|
388
388
|
// Fill nested_vals with list values
|
389
389
|
CreateValues(names, doc, nested_vals, child_v, child_count);
|
390
390
|
// Now we add the values to the appropriate JSON arrays
|
@@ -501,12 +501,12 @@ static void ObjectFunction(DataChunk &args, ExpressionState &state, Vector &resu
|
|
501
501
|
// Initialize values
|
502
502
|
const idx_t count = args.size();
|
503
503
|
auto doc = JSONCommon::CreateDocument(alc);
|
504
|
-
auto objs =
|
504
|
+
auto objs = JSONCommon::AllocateArray<yyjson_mut_val *>(doc, count);
|
505
505
|
for (idx_t i = 0; i < count; i++) {
|
506
506
|
objs[i] = yyjson_mut_obj(doc);
|
507
507
|
}
|
508
508
|
// Initialize a re-usable value array
|
509
|
-
auto vals =
|
509
|
+
auto vals = JSONCommon::AllocateArray<yyjson_mut_val *>(doc, count);
|
510
510
|
// Loop through key/value pairs
|
511
511
|
for (idx_t pair_idx = 0; pair_idx < args.data.size() / 2; pair_idx++) {
|
512
512
|
Vector &key_v = args.data[pair_idx * 2];
|
@@ -533,12 +533,12 @@ static void ArrayFunction(DataChunk &args, ExpressionState &state, Vector &resul
|
|
533
533
|
// Initialize arrays
|
534
534
|
const idx_t count = args.size();
|
535
535
|
auto doc = JSONCommon::CreateDocument(alc);
|
536
|
-
auto arrs =
|
536
|
+
auto arrs = JSONCommon::AllocateArray<yyjson_mut_val *>(doc, count);
|
537
537
|
for (idx_t i = 0; i < count; i++) {
|
538
538
|
arrs[i] = yyjson_mut_arr(doc);
|
539
539
|
}
|
540
540
|
// Initialize a re-usable value array
|
541
|
-
auto vals =
|
541
|
+
auto vals = JSONCommon::AllocateArray<yyjson_mut_val *>(doc, count);
|
542
542
|
// Loop through args
|
543
543
|
for (auto &v : args.data) {
|
544
544
|
CreateValues(info.const_struct_names, doc, vals, v, count);
|
@@ -561,7 +561,7 @@ static void ToJSONFunctionInternal(const StructNames &names, Vector &input, cons
|
|
561
561
|
yyjson_alc *alc) {
|
562
562
|
// Initialize array for values
|
563
563
|
auto doc = JSONCommon::CreateDocument(alc);
|
564
|
-
auto vals =
|
564
|
+
auto vals = JSONCommon::AllocateArray<yyjson_mut_val *>(doc, count);
|
565
565
|
CreateValues(names, doc, vals, input, count);
|
566
566
|
|
567
567
|
// Write JSON values to string
|
@@ -59,11 +59,11 @@ static void MergePatchFunction(DataChunk &args, ExpressionState &state, Vector &
|
|
59
59
|
const auto count = args.size();
|
60
60
|
|
61
61
|
// Read the first json arg
|
62
|
-
auto origs =
|
62
|
+
auto origs = JSONCommon::AllocateArray<yyjson_mut_val *>(alc, count);
|
63
63
|
ReadObjects(doc, args.data[0], origs, count);
|
64
64
|
|
65
65
|
// Read the next json args one by one and merge them into the first json arg
|
66
|
-
auto patches =
|
66
|
+
auto patches = JSONCommon::AllocateArray<yyjson_mut_val *>(alc, count);
|
67
67
|
for (idx_t arg_idx = 1; arg_idx < args.data.size(); arg_idx++) {
|
68
68
|
ReadObjects(doc, args.data[arg_idx], patches, count);
|
69
69
|
for (idx_t i = 0; i < count; i++) {
|
@@ -108,7 +108,7 @@ static void JsonSerializeFunction(DataChunk &args, ExpressionState &state, Vecto
|
|
108
108
|
idx_t len;
|
109
109
|
auto data = yyjson_mut_val_write_opts(result_obj,
|
110
110
|
info.format ? JSONCommon::WRITE_PRETTY_FLAG : JSONCommon::WRITE_FLAG,
|
111
|
-
alc,
|
111
|
+
alc, reinterpret_cast<size_t *>(&len), nullptr);
|
112
112
|
if (data == nullptr) {
|
113
113
|
throw SerializationException(
|
114
114
|
"Failed to serialize json, perhaps the query contains invalid utf8 characters?");
|
@@ -124,7 +124,7 @@ static void JsonSerializeFunction(DataChunk &args, ExpressionState &state, Vecto
|
|
124
124
|
idx_t len;
|
125
125
|
auto data = yyjson_mut_val_write_opts(result_obj,
|
126
126
|
info.format ? JSONCommon::WRITE_PRETTY_FLAG : JSONCommon::WRITE_FLAG,
|
127
|
-
alc,
|
127
|
+
alc, reinterpret_cast<size_t *>(&len), nullptr);
|
128
128
|
return StringVector::AddString(result, data, len);
|
129
129
|
}
|
130
130
|
});
|
@@ -150,7 +150,8 @@ void JSONStructureNode::RefineCandidateTypesArray(yyjson_val *vals[], idx_t coun
|
|
150
150
|
}
|
151
151
|
|
152
152
|
idx_t offset = 0;
|
153
|
-
auto child_vals =
|
153
|
+
auto child_vals =
|
154
|
+
reinterpret_cast<yyjson_val **>(allocator.AllocateAligned(total_list_size * sizeof(yyjson_val *)));
|
154
155
|
|
155
156
|
size_t idx, max;
|
156
157
|
yyjson_val *child_val;
|
@@ -173,11 +174,12 @@ void JSONStructureNode::RefineCandidateTypesObject(yyjson_val *vals[], idx_t cou
|
|
173
174
|
vector<yyjson_val **> child_vals;
|
174
175
|
child_vals.reserve(child_count);
|
175
176
|
for (idx_t child_idx = 0; child_idx < child_count; child_idx++) {
|
176
|
-
child_vals.emplace_back(
|
177
|
+
child_vals.emplace_back(
|
178
|
+
reinterpret_cast<yyjson_val **>(allocator.AllocateAligned(count * sizeof(yyjson_val *))));
|
177
179
|
}
|
178
180
|
|
179
181
|
idx_t found_key_count;
|
180
|
-
auto found_keys =
|
182
|
+
auto found_keys = reinterpret_cast<bool *>(allocator.AllocateAligned(sizeof(bool) * child_count));
|
181
183
|
|
182
184
|
const auto &key_map = desc.key_map;
|
183
185
|
size_t idx, max;
|
@@ -216,7 +216,7 @@ static inline bool GetValueString(yyjson_val *val, yyjson_alc *alc, string_t &re
|
|
216
216
|
|
217
217
|
template <class T>
|
218
218
|
static bool TransformNumerical(yyjson_val *vals[], Vector &result, const idx_t count, JSONTransformOptions &options) {
|
219
|
-
auto data =
|
219
|
+
auto data = FlatVector::GetData<T>(result);
|
220
220
|
auto &validity = FlatVector::Validity(result);
|
221
221
|
|
222
222
|
bool success = true;
|
@@ -238,7 +238,7 @@ static bool TransformNumerical(yyjson_val *vals[], Vector &result, const idx_t c
|
|
238
238
|
template <class T>
|
239
239
|
static bool TransformDecimal(yyjson_val *vals[], Vector &result, const idx_t count, uint8_t width, uint8_t scale,
|
240
240
|
JSONTransformOptions &options) {
|
241
|
-
auto data =
|
241
|
+
auto data = FlatVector::GetData<T>(result);
|
242
242
|
auto &validity = FlatVector::Validity(result);
|
243
243
|
|
244
244
|
bool success = true;
|
@@ -373,7 +373,7 @@ static bool TransformFromStringWithFormat(yyjson_val *vals[], Vector &result, co
|
|
373
373
|
}
|
374
374
|
|
375
375
|
static bool TransformToString(yyjson_val *vals[], yyjson_alc *alc, Vector &result, const idx_t count) {
|
376
|
-
auto data =
|
376
|
+
auto data = FlatVector::GetData<string_t>(result);
|
377
377
|
auto &validity = FlatVector::Validity(result);
|
378
378
|
for (idx_t i = 0; i < count; i++) {
|
379
379
|
const auto &val = vals[i];
|
@@ -400,11 +400,11 @@ bool JSONTransform::TransformObject(yyjson_val *objects[], yyjson_alc *alc, cons
|
|
400
400
|
nested_vals.reserve(column_count);
|
401
401
|
for (idx_t col_idx = 0; col_idx < column_count; col_idx++) {
|
402
402
|
key_map.insert({{names[col_idx].c_str(), names[col_idx].length()}, col_idx});
|
403
|
-
nested_vals.push_back(
|
403
|
+
nested_vals.push_back(JSONCommon::AllocateArray<yyjson_val *>(alc, count));
|
404
404
|
}
|
405
405
|
|
406
406
|
idx_t found_key_count;
|
407
|
-
auto found_keys =
|
407
|
+
auto found_keys = JSONCommon::AllocateArray<bool>(alc, column_count);
|
408
408
|
|
409
409
|
bool success = true;
|
410
410
|
|
@@ -558,7 +558,7 @@ static bool TransformArray(yyjson_val *arrays[], yyjson_alc *alc, Vector &result
|
|
558
558
|
ListVector::Reserve(result, offset);
|
559
559
|
|
560
560
|
// Initialize array for the nested values
|
561
|
-
auto nested_vals =
|
561
|
+
auto nested_vals = JSONCommon::AllocateArray<yyjson_val *>(alc, offset);
|
562
562
|
|
563
563
|
// Get array values
|
564
564
|
size_t idx, max;
|
@@ -617,8 +617,8 @@ static bool TransformObjectToMap(yyjson_val *objects[], yyjson_alc *alc, Vector
|
|
617
617
|
auto list_entries = FlatVector::GetData<list_entry_t>(result);
|
618
618
|
auto &list_validity = FlatVector::Validity(result);
|
619
619
|
|
620
|
-
auto keys =
|
621
|
-
auto vals =
|
620
|
+
auto keys = JSONCommon::AllocateArray<yyjson_val *>(alc, list_size);
|
621
|
+
auto vals = JSONCommon::AllocateArray<yyjson_val *>(alc, list_size);
|
622
622
|
|
623
623
|
bool success = true;
|
624
624
|
idx_t list_offset = 0;
|
@@ -675,7 +675,7 @@ static bool TransformObjectToMap(yyjson_val *objects[], yyjson_alc *alc, Vector
|
|
675
675
|
}
|
676
676
|
|
677
677
|
bool TransformToJSON(yyjson_val *vals[], yyjson_alc *alc, Vector &result, const idx_t count) {
|
678
|
-
auto data =
|
678
|
+
auto data = FlatVector::GetData<string_t>(result);
|
679
679
|
auto &validity = FlatVector::Validity(result);
|
680
680
|
for (idx_t i = 0; i < count; i++) {
|
681
681
|
const auto &val = vals[i];
|
@@ -779,8 +779,8 @@ static bool TransformFunctionInternal(Vector &input, const idx_t count, Vector &
|
|
779
779
|
auto inputs = UnifiedVectorFormat::GetData<string_t>(input_data);
|
780
780
|
|
781
781
|
// Read documents
|
782
|
-
auto docs =
|
783
|
-
auto vals =
|
782
|
+
auto docs = JSONCommon::AllocateArray<yyjson_doc *>(alc, count);
|
783
|
+
auto vals = JSONCommon::AllocateArray<yyjson_val *>(alc, count);
|
784
784
|
auto &result_validity = FlatVector::Validity(result);
|
785
785
|
for (idx_t i = 0; i < count; i++) {
|
786
786
|
auto idx = input_data.sel->get_index(i);
|
@@ -258,7 +258,8 @@ static void ReadJSONFunction(ClientContext &context, TableFunctionInput &data_p,
|
|
258
258
|
string hint =
|
259
259
|
gstate.bind_data.auto_detect
|
260
260
|
? "\nTry increasing 'sample_size', reducing 'maximum_depth', specifying 'columns', 'format' or "
|
261
|
-
"'records' manually, or setting '
|
261
|
+
"'records' manually, setting 'ignore_errors' to true, or setting 'union_by_name' to true when "
|
262
|
+
"reading multiple files with a different structure."
|
262
263
|
: "\nTry setting 'auto_detect' to true, specifying 'format' or 'records' manually, or setting "
|
263
264
|
"'ignore_errors' to true.";
|
264
265
|
lstate.ThrowTransformError(lstate.transform_options.object_index,
|
@@ -15,6 +15,9 @@ namespace duckdb {
|
|
15
15
|
static void CheckPath(const Value &path_val, string &path, size_t &len) {
|
16
16
|
string error;
|
17
17
|
Value path_str_val;
|
18
|
+
if (path_val.IsNull()) {
|
19
|
+
throw InvalidInputException("JSON path cannot be NULL");
|
20
|
+
}
|
18
21
|
if (!path_val.DefaultTryCastAs(LogicalType::VARCHAR, path_str_val, &error)) {
|
19
22
|
throw InvalidInputException(error);
|
20
23
|
}
|
@@ -220,8 +223,8 @@ static bool CastVarcharToJSON(Vector &source, Vector &result, idx_t count, CastP
|
|
220
223
|
bool success = true;
|
221
224
|
UnaryExecutor::ExecuteWithNulls<string_t, string_t>(
|
222
225
|
source, result, count, [&](string_t input, ValidityMask &mask, idx_t idx) {
|
223
|
-
auto data =
|
224
|
-
auto length = input.GetSize();
|
226
|
+
auto data = input.GetDataWriteable();
|
227
|
+
const auto length = input.GetSize();
|
225
228
|
|
226
229
|
yyjson_read_err error;
|
227
230
|
auto doc = JSONCommon::ReadDocumentUnsafe(data, length, JSONCommon::READ_FLAG, alc, &error);
|
@@ -236,7 +239,7 @@ static bool CastVarcharToJSON(Vector &source, Vector &result, idx_t count, CastP
|
|
236
239
|
}
|
237
240
|
return input;
|
238
241
|
});
|
239
|
-
result
|
242
|
+
StringVector::AddHeapReference(result, source);
|
240
243
|
return success;
|
241
244
|
}
|
242
245
|
|